From ab1965d16d4f10d94b37a082517f6c7e15009710 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 11 May 2017 12:35:51 +0300 Subject: [PATCH 0001/2107] Version 1.1.11 --- pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_probackup.c b/pg_probackup.c index 24a0d5473..13d329c3b 100644 --- a/pg_probackup.c +++ b/pg_probackup.c @@ -16,7 +16,7 @@ #include #include -const char *PROGRAM_VERSION = "1.1.10"; +const char *PROGRAM_VERSION = "1.1.11"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 41877ab8b..adc3ad0d7 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.5 +pg_probackup 1.1.11 From ec8d9b8ce4c8528b594e1b57ebfcefbe943f776d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 May 2017 16:43:16 +0300 Subject: [PATCH 0002/2107] ptrack page header fix --- tests/backup_test.py | 14 +-- tests/expected/option_help.out | 10 +- tests/expected/option_version.out | 2 +- tests/ptrack_clean.py | 10 +- tests/ptrack_cluster.py | 25 ++--- tests/ptrack_helpers.py | 149 ++++++++++++------------- tests/ptrack_move_to_tablespace.py | 3 +- tests/ptrack_recovery.py | 3 +- tests/ptrack_vacuum.py | 8 +- tests/ptrack_vacuum_bits_frozen.py | 3 +- tests/ptrack_vacuum_bits_visibility.py | 3 +- tests/ptrack_vacuum_full.py | 3 +- tests/ptrack_vacuum_truncate.py | 3 +- tests/restore_test.py | 13 --- tests/retention_test.py | 2 - tests/validate_test.py | 49 +++----- 16 files changed, 125 insertions(+), 175 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 6c654d49a..62c5ceffa 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -17,7 +17,6 @@ def __init__(self, *args, **kwargs): def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -74,7 +73,6 @@ def test_backup_modes_archive(self): def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -94,7 +92,6 @@ def test_smooth_checkpoint(self): def test_page_backup_without_full(self): """page-level backup without validated full backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -114,10 +111,12 @@ def test_page_backup_without_full(self): # @unittest.skip("123") def test_ptrack_threads(self): """ptrack multi thread backup mode""" - node = self.make_bnode( - base_dir="tmp_dirs/backup/ptrack_threads_4", - options={"ptrack_enable": "on", 'max_wal_senders': '2'} - ) + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} + ) node.start() self.assertEqual(self.init_pb(node), six.b("")) @@ -137,7 +136,6 @@ def test_ptrack_threads(self): def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums'], diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index d2a4957cd..53b46eb4a 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -1,7 +1,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. - pg_probackup help + pg_probackup help [COMMAND] pg_probackup version @@ -21,20 +21,20 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-d dbname] [-h host] [-p port] [-U username] pg_probackup restore -B backup-dir - [-D pgdata-dir] [-i backup-id] + [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] [--time=time|--xid=xid [--inclusive=boolean]] [--timeline=timeline] [-T OLDDIR=NEWDIR] pg_probackup validate -B backup-dir - [-D pgdata-dir] [-i backup-id] + [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--timeline=timeline] pg_probackup show -B backup-dir [-i backup-id] pg_probackup delete -B backup-dir - [--wal] [-i backup-id | --expired] [--force] + [--wal] [-i backup-id | --expired] Read the website for details. Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 5c184bf63..adc3ad0d7 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.11 \ No newline at end of file +pg_probackup 1.1.11 diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 2a458c751..597ea7e73 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -9,10 +9,10 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) def teardown(self): - # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_clean(self): fname = self.id().split('.')[3] node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), @@ -45,7 +45,7 @@ def test_ptrack_clean(self): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) # Update everything, vacuum it and make PTRACK BACKUP @@ -62,7 +62,7 @@ def test_ptrack_clean(self): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) @@ -81,7 +81,7 @@ def test_ptrack_clean(self): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 8a902fc3e..e4525bfdf 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -12,10 +12,9 @@ def teardown(self): # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_btree(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -63,7 +62,7 @@ def test_ptrack_cluster_btree(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -71,10 +70,9 @@ def test_ptrack_cluster_btree(self): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -122,7 +120,7 @@ def test_ptrack_cluster_spgist(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -130,10 +128,9 @@ def test_ptrack_cluster_spgist(self): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -181,7 +178,7 @@ def test_ptrack_cluster_brin(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -189,10 +186,9 @@ def test_ptrack_cluster_brin(self): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -240,7 +236,7 @@ def test_ptrack_cluster_gist(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) @@ -248,10 +244,9 @@ def test_ptrack_cluster_gist(self): self.clean_pb(node) node.stop() - @unittest.skip("123") + # @unittest.skip("123") def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -299,7 +294,7 @@ def test_ptrack_cluster_gin(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_helpers.py b/tests/ptrack_helpers.py index b233e4d5a..40c77c1ad 100644 --- a/tests/ptrack_helpers.py +++ b/tests/ptrack_helpers.py @@ -152,29 +152,6 @@ def arcwal_dir(self, node): def backup_dir(self, node): return os.path.abspath("%s/backup" % node.base_dir) - def make_bnode(self, base_dir=None, allows_streaming=False, options={}): - real_base_dir = os.path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - node = get_new_node('test', base_dir=real_base_dir) - node.init(allows_streaming=allows_streaming) - - if not allows_streaming: - node.append_conf("postgresql.auto.conf", "wal_level = hot_standby") - node.append_conf("postgresql.auto.conf", "archive_mode = on") - node.append_conf( - "postgresql.auto.conf", - """archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node)) - ) - - for key, value in six.iteritems(options): - node.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return node - -# def print_started(self, fname): -# print - def make_simple_node(self, base_dir=None, set_replication=False, set_archiving=False, initdb_params=[], pg_options={}): real_base_dir = os.path.join(self.dir_path, base_dir) @@ -184,6 +161,7 @@ def make_simple_node(self, base_dir=None, set_replication=False, node.init(initdb_params=initdb_params) # Sane default parameters, not a shit with fsync = off from testgres + node.append_conf("postgresql.auto.conf", "{0} = {1}".format('shared_buffers', '10MB')) node.append_conf("postgresql.auto.conf", "{0} = {1}".format('fsync', 'on')) node.append_conf("postgresql.auto.conf", "{0} = {1}".format('wal_level', 'minimal')) @@ -199,7 +177,6 @@ def make_simple_node(self, base_dir=None, set_replication=False, self.set_archiving_conf(node, self.arcwal_dir(node)) return node - def create_tblspace_in_node(self, node, tblspc_name, cfs=False): res = node.execute( "postgres", "select exists (select 1 from pg_tablespace where spcname = '{0}')".format( @@ -236,12 +213,16 @@ def get_md5_per_page_for_fork(self, file, size): os.close(file) return md5_per_page - def get_ptrack_bits_per_page_for_fork(self, file, size): + def get_ptrack_bits_per_page_for_fork(self, node, file, size): + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 ptrack_bits_for_fork = [] byte_size = os.path.getsize(file + '_ptrack') - byte_size_minus_header = byte_size - 24 + byte_size_minus_header = byte_size - header_size file = os.open(file + '_ptrack', os.O_RDONLY) - os.lseek(file, 24, 0) + os.lseek(file, header_size, 0) lot_of_bytes = os.read(file, byte_size_minus_header) for byte in lot_of_bytes: byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] @@ -316,20 +297,32 @@ def check_ptrack_clean(self, idx_dict, size): success = False self.assertEqual(success, True) - def run_pb(self, command): + def run_pb(self, command, async=False): try: -# print [self.probackup_path] + command - output = subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) + #print ' '.join(map(str,[self.probackup_path] + command)) + if async is True: + return subprocess.Popen( + [self.probackup_path] + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ) if command[0] == 'backup': if '-q' in command or '--quiet' in command: return None + elif '-v' in command or '--verbose' in command: + return output else: # return backup ID - return output.split()[2] + for line in output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] else: return output except subprocess.CalledProcessError as e: @@ -346,50 +339,38 @@ def init_pb(self, node): def clean_pb(self, node): shutil.rmtree(self.backup_dir(node), ignore_errors=True) - def backup_pb(self, node, backup_type="full", options=[]): - cmd_list = [ - "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), - "-p", "%i" % node.port, - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] - - return self.run_pb(cmd_list + options) + def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[], async=False): + if data_dir is None: + data_dir = node.data_dir + if backup_dir is None: + backup_dir = self.backup_dir(node) - def backup_pb_proc(self, node, backup_type="full", - stdout=None, stderr=None, options=[]): cmd_list = [ - self.probackup_path, "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), - "-p", "%i" % (node.port), + "-B", backup_dir, + "-D", data_dir, + "-p", "%i" % node.port, "-d", "postgres" ] if backup_type: cmd_list += ["-b", backup_type] - proc = subprocess.Popen( - cmd_list + options, - stdout=stdout, - stderr=stderr - ) + return self.run_pb(cmd_list + options, async) - return proc + def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]): + if data_dir is None: + data_dir = node.data_dir + if backup_dir is None: + backup_dir = self.backup_dir(node) - def restore_pb(self, node, id=None, options=[]): cmd_list = [ "restore", - "-D", node.data_dir, - "-B", self.backup_dir(node) + "-B", backup_dir, + "-D", data_dir ] if id: cmd_list += ["-i", id] - # print(cmd_list) return self.run_pb(cmd_list + options) def show_pb(self, node, id=None, options=[], as_text=False): @@ -417,13 +398,17 @@ def show_pb(self, node, id=None, options=[], as_text=False): body = body[::-1] # split string in list with string for every header element header_split = re.split(" +", header) - # CRUNCH, remove last item, because it empty, like that '' - header_split.pop() + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) for backup_record in body: # split string in list with string for every backup record element backup_record_split = re.split(" +", backup_record) - # CRUNCH, remove last item, because it empty, like that '' - backup_record_split.pop() + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) if len(header_split) != len(backup_record_split): print warning.format( header=header, body=body, @@ -500,25 +485,34 @@ def get_recovery_conf(self, node): out_dict[key.strip()] = value.strip(" '").replace("'\n", "") return out_dict - def set_archiving_conf(self, node, archive_dir): + def set_archiving_conf(self, node, archive_dir=False, replica=False): + if not archive_dir: + archive_dir = self.arcwal_dir(node) + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + node.append_conf( "postgresql.auto.conf", "wal_level = archive" ) node.append_conf( "postgresql.auto.conf", - "archive_mode = on" + "archive_mode = {0}".format(archive_mode) ) if os.name == 'posix': node.append_conf( "postgresql.auto.conf", "archive_command = 'test ! -f {0}/%f && cp %p {0}/%f'".format(archive_dir) ) - elif os.name == 'nt': - node.append_conf( - "postgresql.auto.conf", - "archive_command = 'copy %p {0}\\%f'".format(archive_dir) - ) + #elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) def wrong_wal_clean(self, node, wal_size): wals_dir = os.path.join(self.backup_dir(node), "wal") @@ -536,4 +530,9 @@ def guc_wal_block_size(self, node): var = node.execute("postgres", "select setting from pg_settings where name = 'wal_block_size'") return int(var[0][0]) -# def ptrack_node(self, ptrack_enable=False, wal_level='minimal', max_wal_senders='2', allow_replication=True) + def get_pgpro_edition(self, node): + if node.execute("postgres", "select exists(select 1 from pg_proc where proname = 'pgpro_edition')")[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 5acd5bbd9..d43282f1e 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -17,7 +17,6 @@ def teardown(self): def test_ptrack_recovery(self): fname = self.id().split(".")[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -49,7 +48,7 @@ def test_ptrack_recovery(self): idx_ptrack[i]['path'] = self.get_fork_path(node, i) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index d2a78bd98..73e9e085a 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -17,7 +17,6 @@ def teardown(self): def test_ptrack_recovery(self): fname = self.id().split(".")[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -51,7 +50,7 @@ def test_ptrack_recovery(self): for i in idx_ptrack: # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['size']) # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 6cb66b9a7..484c5c508 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -12,10 +12,10 @@ def teardown(self): # clean_all() stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -51,7 +51,7 @@ def test_ptrack_vacuum(self): self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) # Delete some rows, vacuum it and make checkpoint @@ -69,7 +69,7 @@ def test_ptrack_vacuum(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index ba7e0a39d..75d25909f 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -14,7 +14,6 @@ def teardown(self): stop_all() def test_ptrack_vacuum_bits_frozen(self): - print 'test_ptrack_vacuum_bits_frozen started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_frozen", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -60,7 +59,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index f24ae240c..4fc12419c 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -14,7 +14,6 @@ def teardown(self): stop_all() def test_ptrack_vacuum_bits_visibility(self): - print 'test_ptrack_vacuum_bits_visibility started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_visibility", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -60,7 +59,7 @@ def test_ptrack_vacuum_bits_visibility(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index ade1fd309..98af70be2 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -27,7 +27,6 @@ def teardown(self): def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -75,7 +74,7 @@ def test_ptrack_vacuum_full(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 035c7e467..eba15da4e 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -14,7 +14,6 @@ def teardown(self): stop_all() def test_ptrack_vacuum_truncate(self): - print 'test_ptrack_vacuum_truncate started' node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_truncate", set_replication=True, initdb_params=['--data-checksums', '-A trust'], @@ -62,7 +61,7 @@ def test_ptrack_vacuum_truncate(self): idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # get ptrack for every idx idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + node, idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) diff --git a/tests/restore_test.py b/tests/restore_test.py index 8ef455272..ea35a0b17 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -22,7 +22,6 @@ def tearDownClass(cls): def test_restore_full_to_latest(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -61,7 +60,6 @@ def test_restore_full_to_latest(self): def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -101,7 +99,6 @@ def test_restore_full_page_to_latest(self): def test_restore_to_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -156,7 +153,6 @@ def test_restore_to_timeline(self): def test_restore_to_time(self): """recovery to target timeline""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -195,7 +191,6 @@ def test_restore_to_time(self): def test_restore_to_xid(self): """recovery to target xid""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -249,7 +244,6 @@ def test_restore_to_xid(self): def test_restore_full_ptrack(self): """recovery to latest from full + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -297,7 +291,6 @@ def test_restore_full_ptrack(self): def test_restore_full_ptrack_ptrack(self): """recovery to latest from full + ptrack + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -352,7 +345,6 @@ def test_restore_full_ptrack_ptrack(self): def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_replication=True, initdb_params=['--data-checksums'], @@ -397,7 +389,6 @@ def test_restore_full_ptrack_stream(self): def test_restore_full_ptrack_under_load(self): """recovery to latest from full + ptrack backups with loads when ptrack backup do""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, set_replication=True, @@ -456,7 +447,6 @@ def test_restore_full_ptrack_under_load(self): def test_restore_full_under_load_ptrack(self): """recovery to latest from full + page backups with loads when full backup do""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, set_replication=True, @@ -516,7 +506,6 @@ def test_restore_full_under_load_ptrack(self): def test_restore_to_xid_inclusive(self): """recovery with target inclusive false""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -575,7 +564,6 @@ def test_restore_to_xid_inclusive(self): def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -663,7 +651,6 @@ def test_restore_with_tablespace_mapping_1(self): def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], diff --git a/tests/retention_test.py b/tests/retention_test.py index c30bdf934..265ed8dab 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -19,7 +19,6 @@ def tearDownClass(cls): def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -65,7 +64,6 @@ def test_retention_redundancy_1(self): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], diff --git a/tests/validate_test.py b/tests/validate_test.py index 80c793abc..f3fdf9664 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -12,18 +12,14 @@ class ValidateTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ValidateTest, self).__init__(*args, **kwargs) -# @classmethod -# def tearDownClass(cls): -# try: -# stop_all() -# except: -# pass + @classmethod + def tearDownClass(cls): + stop_all() # @unittest.skip("123") def test_validate_wal_1(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - print '\n {0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -62,19 +58,15 @@ def test_validate_wal_1(self): self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( after_backup_time - timedelta(days=2))]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: Full backup satisfying target options is not found.\n' - ) + self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n') # Validate to unreal time #2 try: self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( after_backup_time + timedelta(days=2))]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") except ProbackupException, e: self.assertEqual( True, @@ -95,8 +87,7 @@ def test_validate_wal_1(self): # Validate to unreal xid try: self.validate_pb(node, options=["--xid=%d" % (int(target_xid) + 1000)]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal xid") except ProbackupException, e: self.assertEqual( True, @@ -120,22 +111,16 @@ def test_validate_wal_1(self): try: self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") except ProbackupException, e: - self.assertEqual( - True, - 'Possible WAL CORRUPTION' in e.message - ) + self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) try: self.validate_pb(node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") except ProbackupException, e: - self.assertEqual( - True, - 'Possible WAL CORRUPTION' in e.message - ) + self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) node.stop() @@ -143,7 +128,6 @@ def test_validate_wal_1(self): def test_validate_wal_lost_segment_1(self): """Loose segment which belong to some backup""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -167,19 +151,15 @@ def test_validate_wal_lost_segment_1(self): os.remove(os.path.join(self.backup_dir(node), "wal", wals[1])) try: self.validate_pb(node) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance") except ProbackupException, e: - self.assertEqual( - True, - 'is absent' in e.message - ) + self.assertTrue('is absent' in e.message) node.stop() + @unittest.expectedFailure def test_validate_wal_lost_segment_2(self): """Loose segment located between backups """ fname = self.id().split('.')[3] - print '{0} started'.format(fname) node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), set_archiving=True, initdb_params=['--data-checksums'], @@ -209,7 +189,6 @@ def test_validate_wal_lost_segment_2(self): wals = map(int, wals) # delete last wal segment - print os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals))) os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) # Need more accurate error message about loosing wal segment between backups From 7685097d2ed1d4c3db37ab55d7e26ae31d25936e Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 22 Jun 2017 15:10:52 +0300 Subject: [PATCH 0003/2107] Version 1.1.17 --- .gitignore | 17 +- COPYRIGHT | 3 +- Makefile | 99 ++-- configure.c | 126 ----- help.c | 238 ---------- pg_probackup.c | 285 ------------ src/archive.c | 106 +++++ backup.c => src/backup.c | 610 ++++++++++++++++++------- catalog.c => src/catalog.c | 59 ++- src/configure.c | 240 ++++++++++ data.c => src/data.c | 310 +++++++++---- delete.c => src/delete.c | 75 ++- dir.c => src/dir.c | 37 +- fetch.c => src/fetch.c | 0 src/help.c | 339 ++++++++++++++ init.c => src/init.c | 62 ++- parsexlog.c => src/parsexlog.c | 278 ++++++++--- src/pg_probackup.c | 475 +++++++++++++++++++ pg_probackup.h => src/pg_probackup.h | 131 ++++-- restore.c => src/restore.c | 106 +++-- show.c => src/show.c | 75 ++- status.c => src/status.c | 0 util.c => src/util.c | 10 +- src/utils/logger.c | 532 +++++++++++++++++++++ src/utils/logger.h | 44 ++ parray.c => src/utils/parray.c | 2 +- parray.h => src/utils/parray.h | 0 {pgut => src/utils}/pgut.c | 340 +++++++++----- {pgut => src/utils}/pgut.h | 46 +- validate.c => src/validate.c | 145 +++++- tests/__init__.py | 11 +- tests/backup_test.py | 210 +++++---- tests/class_check.py | 24 - tests/class_check1.py | 15 - tests/class_check2.py | 23 - tests/delete_test.py | 112 +++-- tests/expected/option_help.out | 62 ++- tests/expected/option_version.out | 2 +- tests/false_positive.py | 155 +++++++ tests/helpers/__init__.py | 2 + tests/{ => helpers}/ptrack_helpers.py | 163 ++++--- tests/init_test.py | 70 +-- tests/option_test.py | 196 ++++---- tests/pgpro560.py | 78 ++++ tests/pgpro589.py | 97 ++++ tests/pgpro688.py | 201 ++++++++ tests/ptrack_clean.py | 25 +- tests/ptrack_cluster.py | 76 +-- tests/ptrack_move_to_tablespace.py | 17 +- tests/ptrack_recovery.py | 17 +- tests/ptrack_vacuum.py | 17 +- tests/ptrack_vacuum_bits_frozen.py | 22 +- tests/ptrack_vacuum_bits_visibility.py | 22 +- tests/ptrack_vacuum_full.py | 32 +- tests/ptrack_vacuum_truncate.py | 22 +- tests/replica.py | 129 ++++++ tests/restore_test.py | 549 ++++++++++++---------- tests/retention_test.py | 62 +-- tests/show_test.py | 44 +- tests/validate_test.py | 278 +++++++---- 60 files changed, 5361 insertions(+), 2162 deletions(-) delete mode 100644 configure.c delete mode 100644 help.c delete mode 100644 pg_probackup.c create mode 100644 src/archive.c rename backup.c => src/backup.c (76%) rename catalog.c => src/catalog.c (89%) create mode 100644 src/configure.c rename data.c => src/data.c (73%) rename delete.c => src/delete.c (84%) rename dir.c => src/dir.c (95%) rename fetch.c => src/fetch.c (100%) create mode 100644 src/help.c rename init.c => src/init.c (53%) rename parsexlog.c => src/parsexlog.c (69%) create mode 100644 src/pg_probackup.c rename pg_probackup.h => src/pg_probackup.h (83%) rename restore.c => src/restore.c (92%) rename show.c => src/show.c (71%) rename status.c => src/status.c (100%) rename util.c => src/util.c (95%) create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h rename parray.c => src/utils/parray.c (99%) rename parray.h => src/utils/parray.h (100%) rename {pgut => src/utils}/pgut.c (81%) rename {pgut => src/utils}/pgut.h (82%) rename validate.c => src/validate.c (54%) delete mode 100644 tests/class_check.py delete mode 100644 tests/class_check1.py delete mode 100644 tests/class_check2.py create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py rename tests/{ => helpers}/ptrack_helpers.py (81%) create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/pgpro688.py create mode 100644 tests/replica.py diff --git a/.gitignore b/.gitignore index 1c2655643..417410c3b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,13 +29,14 @@ /tests/__pycache__/ /tests/tmp_dirs/ /tests/*pyc +/tests/helpers/*pyc # Extra files -/datapagemap.c -/datapagemap.h -/logging.h -/receivelog.c -/receivelog.h -/streamutil.c -/streamutil.h -/xlogreader.c +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c diff --git a/COPYRIGHT b/COPYRIGHT index dc7e2935f..49d704724 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,4 +1,5 @@ -Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/Makefile b/Makefile index 8cf6c6690..e76a3ea9c 100644 --- a/Makefile +++ b/Makefile @@ -1,54 +1,67 @@ PROGRAM = pg_probackup -OBJS = backup.o \ - catalog.o \ - configure.o \ - data.o \ - delete.o \ - dir.o \ - fetch.o \ - help.o \ - init.o \ - parray.o \ - pg_probackup.o \ - restore.o \ - show.o \ - status.o \ - util.o \ - validate.o \ - datapagemap.o \ - parsexlog.o \ - xlogreader.o \ - streamutil.o \ - receivelog.o \ - pgut/pgut.o +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o -EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.c receivelog.h streamutil.c streamutil.h logging.h +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h -all: checksrcdir datapagemap.h logging.h receivelog.h streamutil.h pg_probackup +all: checksrcdir src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h pg_probackup -MAKE_GLOBAL="../../src/Makefile.global" -TEST_GLOBAL:=$(shell test -e ../../src/Makefile.global) -ifeq ($(.SHELLSTATUS),1) +ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) - -.PHONY: checksrcdir -checksrcdir: ifndef top_srcdir @echo "You must have PostgreSQL source tree available to compile." @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" @echo "variable: \"make top_srcdir=\"" @exit 1 endif +# Those files are symlinked from the PostgreSQL sources. +src/xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $< ./src/xlogreader.c +src/datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $< ./src/datapagemap.c +src/datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $< src/datapagemap.h +src/logging.h: % : $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $< ./src +src/receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $< ./src +src/receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $< ./src +src/streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $< ./src +src/streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $< ./src else -#TODO: fix me -REGRESS = subdir=contrib/pg_probackup top_builddir=../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk +# Those files are symlinked from the PostgreSQL sources. +src/xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) ../$< ./src/xlogreader.c +src/datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) ../$< ./src/datapagemap.c +src/datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) ../$< src/datapagemap.h +src/logging.h: % : $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) ../$< ./src +src/receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) ../$< ./src +src/receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) ../$< ./src +src/streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) ../$< ./src +src/streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) ../$< ./src endif + PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} @@ -58,7 +71,7 @@ ifeq ($(PORTNAME), aix) endif envtest: - : top_srcdir=$(top_srcdir) + : top_srcdir=$( ) : libpq_srcdir = $(libpq_srcdir) # This rule's only purpose is to give the user instructions on how to pass @@ -71,23 +84,3 @@ ifndef top_srcdir @echo "variable: \"make top_srcdir=\"" @exit 1 endif - -# Those files are symlinked from the PostgreSQL sources. -xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% - rm -f $@ && $(LN_S) $< . -datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f $@ && $(LN_S) $< . -datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f && $(LN_S) $< . -#logging.c: % : $(top_srcdir)/src/bin/pg_rewind/% -# rm -f && $(LN_S) $< . -logging.h: % : $(top_srcdir)/src/bin/pg_rewind/% - rm -f && $(LN_S) $< . -receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . -streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/% - rm -f && $(LN_S) $< . diff --git a/configure.c b/configure.c deleted file mode 100644 index fbd1b6b0f..000000000 --- a/configure.c +++ /dev/null @@ -1,126 +0,0 @@ -/*------------------------------------------------------------------------- - * - * configure.c: - manage backup catalog. - * - * Portions Copyright (c) 2017-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -/* Set configure options */ -int -do_configure(bool show_only) -{ - pgBackupConfig *config = readBackupCatalogConfigFile(); - if (pgdata) - config->pgdata = pgdata; - if (pgut_dbname) - config->pgdatabase = pgut_dbname; - if (host) - config->pghost = host; - if (port) - config->pgport = port; - if (username) - config->pguser = username; - - if (retention_redundancy) - config->retention_redundancy = retention_redundancy; - if (retention_window) - config->retention_window = retention_window; - - if (show_only) - writeBackupCatalogConfig(stderr, config); - else - writeBackupCatalogConfigFile(config); - - return 0; -} - -void -pgBackupConfigInit(pgBackupConfig *config) -{ - config->system_identifier = 0; - config->pgdata = NULL; - config->pgdatabase = NULL; - config->pghost = NULL; - config->pgport = NULL; - config->pguser = NULL; - - config->retention_redundancy = 0; - config->retention_window = 0; -} - -void -writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) -{ - fprintf(out, "#Backup instance info\n"); - fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = %li\n", config->system_identifier); - - fprintf(out, "#Connection parameters:\n"); - if (config->pgdatabase) - fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); - if (config->pghost) - fprintf(out, "PGHOST = %s\n", config->pghost); - if (config->pgport) - fprintf(out, "PGPORT = %s\n", config->pgport); - if (config->pguser) - fprintf(out, "PGUSER = %s\n", config->pguser); - - fprintf(out, "#Retention parameters:\n"); - if (config->retention_redundancy) - fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); - if (config->retention_window) - fprintf(out, "retention-window = %u\n", config->retention_window); - -} - -void -writeBackupCatalogConfigFile(pgBackupConfig *config) -{ - char path[MAXPGPATH]; - FILE *fp; - - join_path_components(path, backup_path, BACKUPS_DIR); - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - fp = fopen(path, "wt"); - if (fp == NULL) - elog(ERROR, "cannot create %s: %s", - BACKUP_CATALOG_CONF_FILE, strerror(errno)); - - writeBackupCatalogConfig(fp, config); - fclose(fp); -} - - -pgBackupConfig* -readBackupCatalogConfigFile(void) -{ - pgBackupConfig *config = pgut_new(pgBackupConfig); - char path[MAXPGPATH]; - - pgut_option options[] = - { - /* configure options */ - { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, - { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, - { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, - { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, - { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, - { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, - { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, - { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, - {0} - }; - - join_path_components(path, backup_path, BACKUPS_DIR); - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - - pgBackupConfigInit(config); - pgut_readopt(path, options, ERROR); - - return config; - -} diff --git a/help.c b/help.c deleted file mode 100644 index 9936ab6e0..000000000 --- a/help.c +++ /dev/null @@ -1,238 +0,0 @@ -/*------------------------------------------------------------------------- - * - * help.c - * - * Portions Copyright (c) 2017-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ -#include "pg_probackup.h" - -static void help_init(void); -static void help_backup(void); -static void help_restore(void); -static void help_validate(void); -static void help_show(void); -static void help_delete(void); -static void help_set_config(void); -static void help_show_config(void); - -void -help_command(char *command) -{ - if (strcmp(command, "init") == 0) - help_init(); - else if (strcmp(command, "backup") == 0) - help_backup(); - else if (strcmp(command, "restore") == 0) - help_restore(); - else if (strcmp(command, "validate") == 0) - help_validate(); - else if (strcmp(command, "show") == 0) - help_show(); - else if (strcmp(command, "delete") == 0) - help_delete(); - else if (strcmp(command, "set-config") == 0) - help_set_config(); - else if (strcmp(command, "show-config") == 0) - help_show_config(); - else if (strcmp(command, "--help") == 0 - || strcmp(command, "help") == 0 - || strcmp(command, "-?") == 0 - || strcmp(command, "--version") == 0 - || strcmp(command, "version") == 0 - || strcmp(command, "-V") == 0) - printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); - else - printf(_("Unknown command. Try pg_probackup help\n")); - exit(0); -} - -void -help_pg_probackup(void) -{ - printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); - - printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); - - printf(_("\n %s version\n"), PROGRAM_NAME); - - printf(_("\n %s init -B backup-path -D pgdata-dir\n"), PROGRAM_NAME); - - printf(_("\n %s set-config -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]]\n")); - printf(_(" [--retention-window=retention-window]\n")); - - printf(_("\n %s show-config -B backup-dir\n"), PROGRAM_NAME); - - printf(_("\n %s backup -B backup-path -b backup-mode\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress] [-q] [-v] [--delete-expired]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - - printf(_("\n %s restore -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); - - printf(_("\n %s validate -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline]\n")); - - printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-i backup-id]\n")); - - printf(_("\n %s delete -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired]\n")); - - if ((PROGRAM_URL || PROGRAM_EMAIL)) - { - printf("\n"); - if (PROGRAM_URL) - printf("Read the website for details. <%s>\n", PROGRAM_URL); - if (PROGRAM_EMAIL) - printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); - } - exit(0); -} - -static void -help_init(void) -{ - printf(_("%s init -B backup-path -D pgdata-dir\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); -} - -static void -help_backup(void) -{ - printf(_("%s backup -B backup-path -b backup-mode\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress] [-q] [-v] [--delete-expired]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|PTRACK\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); - printf(_(" --stream stream the transaction log and include it in the backup\n")); - printf(_(" --archive-timeout wait timeout for WAL segment archiving\n")); - printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --backup-pg-log backup of pg_log directory\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --delete-expired delete backups expired according to current\n")); - printf(_(" retention policy after successful backup completion\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -d, --dbname=DBNAME database to connect\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); -} - -static void -help_restore(void) -{ - printf(_("%s restore -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -i, --backup-id=backup-id backup to restore\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); -} - -static void -help_validate(void) -{ - printf(_("%s validate -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v]\n")); - printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); - printf(_(" -i, --backup-id=backup-id backup to validate\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -q, --quiet don't write any messages\n")); - printf(_(" -v, --verbose verbose mode\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); -} - -static void -help_show(void) -{ - printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-i backup-id]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); -} - -static void -help_delete(void) -{ - printf(_("%s delete -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --wal remove unnecessary wal files\n")); - printf(_(" -i, --backup-id=backup-id backup to delete\n")); - printf(_(" --expired delete backups expired according to current\n")); - printf(_(" retention policy\n")); -} - -static void -help_set_config(void) -{ - printf(_("%s set-config -B backup-dir\n"), PROGRAM_NAME); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]]\n")); - printf(_(" [--retention-window=retention-window]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -d, --dbname=DBNAME database to connect\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); - - printf(_("\n Retention options:\n")); - printf(_(" --retention-redundancy=retention-redundancy\n")); - printf(_(" number of full backups to keep\n")); - printf(_(" --retention-window=retention-window\n")); - printf(_(" number of days of recoverability\n")); - -} - -static void -help_show_config(void) -{ - printf(_("%s show-config -B backup-dir\n\n"), PROGRAM_NAME); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); -} diff --git a/pg_probackup.c b/pg_probackup.c deleted file mode 100644 index deb2934b1..000000000 --- a/pg_probackup.c +++ /dev/null @@ -1,285 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_probackup.c: Backup/Recovery manager for PostgreSQL. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "streamutil.h" - -#include -#include -#include -#include - -const char *PROGRAM_VERSION = "1.1.11"; -const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; -const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; - -/* path configuration */ -char *backup_path; -char *pgdata; -char arclog_path[MAXPGPATH]; - -/* directory configuration */ -pgBackup current; -ProbackupSubcmd backup_subcmd; - -bool help = false; - -char *backup_id_string_param = NULL; -bool backup_logs = false; - -bool smooth_checkpoint; -int num_threads = 1; -bool stream_wal = false; -bool from_replica = false; -bool progress = false; -bool delete_wal = false; -bool delete_expired = false; -bool apply_to_all = false; -bool force_delete = false; -uint32 archive_timeout = 300; /* Wait timeout for WAL segment archiving */ - -uint64 system_identifier = 0; - -uint32 retention_redundancy = 0; -uint32 retention_window = 0; - -/* restore configuration */ -static char *target_time; -static char *target_xid; -static char *target_inclusive; -static TimeLineID target_tli; - -static void opt_backup_mode(pgut_option *opt, const char *arg); - -static pgut_option options[] = -{ - /* directory options */ - { 'b', 1, "help", &help, SOURCE_CMDLINE }, - { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, - { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, - /* common options */ - { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, - { 'b', 8, "stream", &stream_wal, SOURCE_CMDLINE }, - { 'b', 11, "progress", &progress, SOURCE_CMDLINE }, - { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, - /* backup options */ - { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, - { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, - { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, - { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, - { 'u', 2, "archive-timeout", &archive_timeout, SOURCE_CMDLINE }, - { 'b', 19, "delete-expired", &delete_expired, SOURCE_CMDLINE }, - /* restore options */ - { 's', 3, "time", &target_time, SOURCE_CMDLINE }, - { 's', 4, "xid", &target_xid, SOURCE_CMDLINE }, - { 's', 5, "inclusive", &target_inclusive, SOURCE_CMDLINE }, - { 'u', 6, "timeline", &target_tli, SOURCE_CMDLINE }, - { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, - /* delete options */ - { 'b', 12, "wal", &delete_wal, SOURCE_CMDLINE }, - { 'b', 16, "expired", &delete_expired, SOURCE_CMDLINE }, - { 'b', 17, "all", &apply_to_all, SOURCE_CMDLINE }, - /* TODO not implemented yet */ - { 'b', 18, "force", &force_delete, SOURCE_CMDLINE }, - /* configure options */ - { 'u', 13, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, - { 'u', 14, "retention-window", &retention_window, SOURCE_CMDLINE }, - /* other */ - { 'U', 15, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, - - { 's', 'd', "pgdatabase" , &pgut_dbname, SOURCE_CMDLINE }, - { 's', 'h', "pghost" , &host, SOURCE_CMDLINE }, - { 's', 'p', "pgport" , &port, SOURCE_CMDLINE }, - { 's', 'U', "pguser" , &username, SOURCE_CMDLINE }, - { 'b', 'q', "quiet" , &quiet, SOURCE_CMDLINE }, - { 'b', 'v', "verbose" , &verbose, SOURCE_CMDLINE }, - { 'B', 'w', "no-password" , &prompt_password, SOURCE_CMDLINE }, - { 0 } -}; - -/* - * Entry point of pg_probackup command. - */ -int -main(int argc, char *argv[]) -{ - char path[MAXPGPATH]; - /* Check if backup_path is directory. */ - struct stat stat_buf; - int rc; - - /* initialize configuration */ - pgBackup_init(¤t); - - PROGRAM_NAME = get_progname(argv[0]); - set_pglocale_pgservice(argv[0], "pgscripts"); - - /* Parse subcommands and non-subcommand options */ - if (argc > 1) - { - if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG; - else if (strcmp(argv[1], "--help") == 0 - || strcmp(argv[1], "help") == 0 - || strcmp(argv[1], "-?") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) - { - if (argc == 2) - { - fprintf(stderr, "%s %s\n", PROGRAM_NAME, PROGRAM_VERSION); - exit(0); - } - else if (strcmp(argv[2], "--help") == 0) - help_command(argv[1]); - else - elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); - } - else - elog(ERROR, "Unknown subcommand"); - } - - /* Parse command line arguments */ - pgut_getopt(argc, argv, options); - - if (help) - help_command(argv[2]); - - if (backup_path == NULL) - { - /* Try to read BACKUP_PATH from environment variable */ - backup_path = getenv("BACKUP_PATH"); - if (backup_path == NULL) - elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); - } - - rc = stat(backup_path, &stat_buf); - /* If rc == -1, there is no file or directory. So it's OK. */ - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); - - /* Do not read options from file or env if we're going to set them */ - if (backup_subcmd != SET_CONFIG) - { - /* Read options from configuration file */ - join_path_components(path, backup_path, BACKUP_CATALOG_CONF_FILE); - pgut_readopt(path, options, ERROR); - - /* Read environment variables */ - pgut_getopt_env(options); - } - - if (backup_id_string_param != NULL) - { - current.backup_id = base36dec(backup_id_string_param); - if (current.backup_id == 0) - elog(ERROR, "Invalid backup-id"); - } - - /* setup stream options */ - if (pgut_dbname != NULL) - dbname = pstrdup(pgut_dbname); - if (host != NULL) - dbhost = pstrdup(host); - if (port != NULL) - dbport = pstrdup(port); - if (username != NULL) - dbuser = pstrdup(username); - - /* path must be absolute */ - if (!is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); - if (pgdata != NULL && !is_absolute_path(pgdata)) - elog(ERROR, "-D, --pgdata must be an absolute path"); - - join_path_components(arclog_path, backup_path, "wal"); - - /* setup exclusion list for file search */ - if (!backup_logs) - { - int i; - - for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ - - /* Set 'pg_log' in first empty slot */ - pgdata_exclude_dir[i] = "pg_log"; - } - - if (target_time != NULL && target_xid != NULL) - elog(ERROR, "You can't specify recovery-target-time and recovery-target-xid at the same time"); - - if (num_threads < 1) - num_threads = 1; - - /* do actual operation */ - switch (backup_subcmd) - { - case INIT: - return do_init(); - case BACKUP: - return do_backup(); - case RESTORE: - return do_restore_or_validate(current.backup_id, - target_time, target_xid, - target_inclusive, target_tli, - true); - case VALIDATE: - return do_restore_or_validate(current.backup_id, - target_time, target_xid, - target_inclusive, target_tli, - false); - case SHOW: - return do_show(current.backup_id); - case DELETE: - if (delete_expired && backup_id_string_param) - elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); - if (delete_expired) - return do_retention_purge(); - else - return do_delete(current.backup_id); - case SHOW_CONFIG: - if (argc > 4) - elog(ERROR, "show-config command doesn't accept any options"); - return do_configure(true); - case SET_CONFIG: - if (argc == 4) - elog(ERROR, "set-config command requires at least one option"); - return do_configure(false); - } - - return 0; -} - -static void -opt_backup_mode(pgut_option *opt, const char *arg) -{ - current.backup_mode = parse_backup_mode(arg); -} diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 000000000..1bb17643a --- /dev/null +++ b/src/archive.c @@ -0,0 +1,106 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: compress, list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal_file_name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal_file_path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", + wal_file_name, instance_name, config->system_identifier, system_id); + + if (strcmp(current_dir, config->pgdata) != 0) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have PGDATA = %s instead of %s", + wal_file_name, instance_name, config->pgdata, current_dir); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + copy_wal_file(absolute_wal_file_path, backup_wal_file_path); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal_file_name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal_file_path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", backup_wal_file_path, absolute_wal_file_path); + copy_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} \ No newline at end of file diff --git a/backup.c b/src/backup.c similarity index 76% rename from backup.c rename to src/backup.c index f502bfd6c..e8cb56e1c 100644 --- a/backup.c +++ b/src/backup.c @@ -28,17 +28,38 @@ static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; -static pthread_mutex_t check_stream_mut = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t start_stream_mut = PTHREAD_MUTEX_INITIALIZER; +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +static pthread_t stream_thread; static int is_ptrack_enable = false; -/* Backup connection */ +/* Backup connections */ static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; + +static bool exclusive_backup = false; +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; typedef struct { @@ -61,10 +82,11 @@ static void do_backup_database(parray *backup_list); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_switch_wal(void); static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); static void add_pgdata_files(parray *files, const char *root); static void write_backup_file_list(parray *files, const char *root); -static void wait_archive_lsn(XLogRecPtr lsn, bool prev_segno); +static void wait_wal_lsn(XLogRecPtr lsn); static void make_pagemap_from_ptrack(parray *files); static void StreamLog(void *arg); @@ -73,6 +95,7 @@ static void pg_ptrack_clear(void); static bool pg_ptrack_support(void); static bool pg_ptrack_enable(void); static bool pg_is_in_recovery(void); +static bool pg_archive_enabled(void); static char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid, @@ -104,7 +127,6 @@ do_backup_database(parray *backup_list) XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; pthread_t backup_threads[num_threads]; - pthread_t stream_thread; backup_files_args *backup_threads_args[num_threads]; pgBackup *prev_backup = NULL; @@ -141,29 +163,11 @@ do_backup_database(parray *backup_list) strncat(label, " with pg_probackup", lengthof(label)); pg_start_backup(label, smooth_checkpoint, ¤t); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - /* start stream replication */ - if (stream_wal) - { - join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); - dir_create_dir(dst_backup_path, DIR_PERMISSION); - - pthread_mutex_lock(&check_stream_mut); - pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); - pthread_mutex_lock(&check_stream_mut); - if (conn == NULL) - elog(ERROR, "Cannot continue backup because stream connect has failed."); - - pthread_mutex_unlock(&check_stream_mut); - } - /* * If backup_label does not exist in $PGDATA, stop taking backup. * NOTE. We can check it only on master, though. */ - if(!from_replica) + if (exclusive_backup) { char label_path[MAXPGPATH]; join_path_components(label_path, pgdata, PG_BACKUP_LABEL_FILE); @@ -177,6 +181,24 @@ do_backup_database(parray *backup_list) } } + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + pthread_mutex_lock(&start_stream_mut); + pthread_create(&stream_thread, NULL, (void *(*)(void *)) StreamLog, dst_backup_path); + pthread_mutex_lock(&start_stream_mut); + if (conn == NULL) + elog(ERROR, "Cannot continue backup because stream connect has failed."); + + pthread_mutex_unlock(&start_stream_mut); + } + /* * To take incremental backup get the filelist of the last completed database */ @@ -215,16 +237,6 @@ do_backup_database(parray *backup_list) */ if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) { - /* - * Switch to a new WAL segment. It is necessary to get archived WAL - * segment, which includes start LSN of current backup. - * - * Do not switch for standby node and if backup is stream. - */ - if (!from_replica && !stream_wal) - pg_switch_wal(); - if (!stream_wal) - wait_archive_lsn(current.start_lsn, false); /* * Build the page map. Obtain information about changed pages * reading WAL segments present in archives up to the point @@ -276,8 +288,7 @@ do_backup_database(parray *backup_list) char dirpath[MAXPGPATH]; char *dir_name = GetRelativePath(file->path, pgdata); - if (verbose) - elog(LOG, "Create directory \"%s\"", dir_name); + elog(LOG, "Create directory \"%s\"", dir_name); join_path_components(dirpath, database_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); @@ -305,12 +316,11 @@ do_backup_database(parray *backup_list) /* Run threads */ for (i = 0; i < num_threads; i++) { - if (verbose) - elog(WARNING, "Start thread num:%li", parray_num(backup_threads_args[i]->backup_files_list)); + elog(LOG, "Start thread num:%li", parray_num(backup_threads_args[i]->backup_files_list)); pthread_create(&backup_threads[i], NULL, (void *(*)(void *)) backup_files, backup_threads_args[i]); } - /* Wait theads */ + /* Wait threads */ for (i = 0; i < num_threads; i++) { pthread_join(backup_threads[i], NULL); @@ -333,9 +343,6 @@ do_backup_database(parray *backup_list) parray *xlog_files_list; char pg_xlog_path[MAXPGPATH]; - /* Wait for the completion of stream */ - pthread_join(stream_thread, NULL); - /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); @@ -367,11 +374,12 @@ do_backup_database(parray *backup_list) { pgFile *file = (pgFile *) parray_get(backup_files_list, i); - if (!S_ISREG(file->mode)) - continue; + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; /* Count the amount of the data actually copied */ - current.data_bytes += file->write_size; + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; } if (backup_files_list) @@ -400,13 +408,15 @@ do_backup(void) backup_conn = pgut_connect(pgut_dbname); pgut_atexit_push(backup_disconnect, NULL); - /* Confirm that this server version is supported */ - check_server_version(); /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); confirm_block_size("wal_block_size", XLOG_BLCKSZ); from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + current.checksum_version = get_data_checksum_version(true); current.stream = stream_wal; @@ -422,6 +432,21 @@ do_backup(void) elog(ERROR, "Ptrack is disabled"); } + /* archiving check */ + if (!current.stream && !pg_archive_enabled()) + elog(ERROR, "Archiving must be enabled for archive backup"); + + if (from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, + master_db, master_user, password); + } + /* Get exclusive lock of backup catalog */ catalog_lock(); @@ -458,6 +483,13 @@ do_backup(void) do_backup_database(backup_list); pgut_atexit_pop(backup_cleanup, NULL); + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = XLOG_SEG_SIZE * + (current.stop_lsn/XLogSegSize - current.start_lsn/XLogSegSize + 1); + } + /* Backup is done. Update backup status */ current.end_time = time(NULL); current.status = BACKUP_STATUS_DONE; @@ -486,7 +518,6 @@ do_backup(void) static void check_server_version(void) { - static int server_version = 0; /* confirm server version */ server_version = PQserverVersion(backup_conn); @@ -503,6 +534,9 @@ check_server_version(void) server_version / 10000, (server_version / 100) % 100, server_version % 100, "9.6"); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600; } /* @@ -519,7 +553,7 @@ check_system_identifiers(void) uint64 system_id_pgdata; char *val; - system_id_pgdata = get_system_identifier(); + system_id_pgdata = get_system_identifier(pgdata); res = pgut_execute(backup_conn, "SELECT system_identifier FROM pg_control_system()", @@ -578,7 +612,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - if (from_replica) + if (!exclusive_backup) res = pgut_execute(backup_conn, "SELECT pg_start_backup($1, $2, false)", 2, @@ -589,12 +623,30 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) 2, params); + backup_in_progress = true; + /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; PQclear(res); + + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + * + * Do not switch for standby node and if backup is stream. + */ + if (!from_replica && !stream_wal) + pg_switch_wal(); + if (!stream_wal) + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + wait_wal_lsn(backup->start_lsn); } /* @@ -610,19 +662,19 @@ pg_switch_wal(void) NULL); PQclear(res); -#if PG_VERSION_NUM >= 100000 - res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_wal()", 0, - NULL); -#else - res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_xlog()", 0, - NULL); -#endif + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_wal()", 0, + NULL); + else + res = pgut_execute(backup_conn, "SELECT * FROM pg_switch_xlog()", 0, + NULL); + PQclear(res); } /* * Check if the instance supports ptrack - * TODO Implement check of ptrack_version() instead of existing one + * TODO Maybe we should rather check ptrack_version()? */ static bool pg_ptrack_support(void) @@ -677,6 +729,25 @@ pg_is_in_recovery(void) return false; } +/* + * Check if archiving is enabled + */ +static bool +pg_archive_enabled(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show archive_mode", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "off") == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + /* Clear ptrack files in all databases of the instance we connected to */ static void pg_ptrack_clear(void) @@ -757,33 +828,61 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid, } /* - * Wait for target 'lsn' to be archived in archive 'wal' directory with - * WAL segment file. + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_xlog' directory. */ static void -wait_archive_lsn(XLogRecPtr lsn, bool prev_segno) +wait_wal_lsn(XLogRecPtr lsn) { TimeLineID tli; XLogSegNo targetSegNo; - char wal_path[MAXPGPATH]; - char wal_file[MAXFNAMELEN]; - uint32 try_count = 0; - - Assert(!stream_wal); + char wal_dir[MAXPGPATH], + wal_segment_full_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + uint32 try_count = 0, + timeout; tli = get_current_timeline(false); /* Compute the name of the WAL file containig requested LSN */ XLByteToSeg(lsn, targetSegNo); - if (prev_segno) - targetSegNo--; - XLogFileName(wal_file, tli, targetSegNo); + XLogFileName(wal_segment, tli, targetSegNo); - join_path_components(wal_path, arclog_path, wal_file); + if (stream_wal) + { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_full_path, wal_dir, wal_segment); - /* Wait until switched WAL is archived */ - while (!fileExists(wal_path)) + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_full_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + /* Wait until target LSN is archived or streamed */ + while (true) { + bool file_exists = fileExists(wal_segment_full_path); + + if (file_exists) + { + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli))) + /* Target LSN was found */ + return; + } + sleep(1); if (interrupted) elog(ERROR, "interrupted during waiting for WAL archiving"); @@ -792,21 +891,20 @@ wait_archive_lsn(XLogRecPtr lsn, bool prev_segno) /* Inform user if WAL segment is absent in first attempt */ if (try_count == 1) elog(INFO, "wait for LSN %X/%X in archived WAL segment %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_path); + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_full_path); - if (archive_timeout > 0 && try_count > archive_timeout) - elog(ERROR, - "switched WAL segment %s could not be archived in %d seconds", - wal_file, archive_timeout); + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + else + elog(ERROR, + "switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } } - - /* - * WAL segment was archived. Check LSN on it if we waited current WAL - * segment, not previous. - */ - if (!prev_segno && !wal_contains_lsn(arclog_path, lsn, tli)) - elog(ERROR, "WAL segment %s doesn't contain target LSN %X/%X", - wal_file, (uint32) (lsn >> 32), (uint32) lsn); } /* @@ -818,6 +916,9 @@ pg_stop_backup(pgBackup *backup) PGresult *res; uint32 xlogid; uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + bool sent = false; + int pg_stop_backup_timeout = 0; /* * We will use this values if there are no transactions between start_lsn @@ -826,36 +927,173 @@ pg_stop_backup(pgBackup *backup) time_t recovery_time; TransactionId recovery_xid; + if (!backup_in_progress) + elog(FATAL, "backup is not in progress"); + /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); - if (from_replica) + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + char *backup_id; + + backup_id = base36enc(backup->start_time); + + if (!from_replica) + { + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + backup_id); + params[0] = name; + + res = pgut_execute(backup_conn, "SELECT pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + else + { + uint32 try_count = 0; + + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + backup_id); + params[0] = name; + + res = pgut_execute(master_conn, "SELECT pg_create_restore_point($1)", + 1, params); + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + restore_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* Wait for restore_lsn from master */ + while (true) + { + XLogRecPtr min_recovery_lsn; + + res = pgut_execute(backup_conn, "SELECT min_recovery_end_location from pg_control_recovery()", + 0, NULL); + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + min_recovery_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* restore_lsn was streamed and applied to the replica */ + if (min_recovery_lsn >= restore_lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for restore point LSN"); + try_count++; + + /* Inform user if restore_lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for restore point LSN %X/%X to be streamed " + "to replica", + (uint32) (restore_lsn >> 32), (uint32) restore_lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Restore point LSN %X/%X could not be " + "streamed to replica in %d seconds", + (uint32) (restore_lsn >> 32), (uint32) restore_lsn, + replica_timeout); + } + } + + pfree(backup_id); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + if (!exclusive_backup) /* * Stop the non-exclusive backup. Besides stop_lsn it returns from * pg_stop_backup(false) copy of the backup label and tablespace map * so they can be written to disk by the caller. */ - res = pgut_execute(backup_conn, + sent = pgut_send(backup_conn, "SELECT *, txid_snapshot_xmax(txid_current_snapshot())," " current_timestamp(0)::timestamp" " FROM pg_stop_backup(false)", - 0, NULL); + 0, NULL, WARNING); else - res = pgut_execute(backup_conn, + sent = pgut_send(backup_conn, "SELECT *, txid_snapshot_xmax(txid_current_snapshot())," " current_timestamp(0)::timestamp" " FROM pg_stop_backup()", - 0, NULL); + 0, NULL, WARNING); + + if (!sent) + elog(WARNING, "Failed to send pg_stop_backup query"); + + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + elog(INFO, "wait for pg_stop_backup()"); + + while (1) + { + if (!PQconsumeInput(backup_conn) || PQisBusy(backup_conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(backup_conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(backup_conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(backup_conn); + break; + } + } + + if (!res) + elog(ERROR, "pg_stop backup() failed"); + + backup_in_progress = false; /* Extract timeline and LSN from results of pg_stop_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); /* Calculate LSN */ stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + /* Write backup_label and tablespace_map for backup from replica */ - if (from_replica) + if (!exclusive_backup) { char path[MAXPGPATH]; char backup_label[MAXPGPATH]; @@ -876,12 +1114,18 @@ pg_stop_backup(pgBackup *backup) fwrite(PQgetvalue(res, 0, 1), 1, strlen(PQgetvalue(res, 0, 1)), fp); fclose(fp); - /* TODO What for do we save the file into backup_list? */ - file = pgFileNew(backup_label, true); - calc_file_checksum(file); - free(file->path); - file->path = strdup(PG_BACKUP_LABEL_FILE); - parray_append(backup_files_list, file); + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } /* Write tablespace_map */ if (strlen(PQgetvalue(res, 0, 2)) > 0) @@ -927,8 +1171,9 @@ pg_stop_backup(pgBackup *backup) PQclear(res); - if (!stream_wal) - wait_archive_lsn(stop_backup_lsn, false); + if (stream_wal) + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); /* Fill in fields if that is the correct end of backup. */ if (backup != NULL) @@ -936,9 +1181,17 @@ pg_stop_backup(pgBackup *backup) char *xlog_path, stream_xlog_path[MAXPGPATH]; + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn); + if (stream_wal) { - join_path_components(stream_xlog_path, pgdata, PG_XLOG_DIR); + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); xlog_path = stream_xlog_path; } else @@ -957,6 +1210,33 @@ pg_stop_backup(pgBackup *backup) } } +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + PQclear(res); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + return val_int; +} + /* * Return true if the path is a existing regular file. */ @@ -981,16 +1261,6 @@ fileExists(const char *path) static void backup_cleanup(bool fatal, void *userdata) { - char path[MAXPGPATH]; - - /* If backup_label exists in $PGDATA, notify stop of backup to PostgreSQL */ - join_path_components(path, pgdata, PG_BACKUP_LABEL_FILE); - if (fileExists(path)) - { - elog(LOG, "%s exists, stop backup", PG_BACKUP_LABEL_FILE); - pg_stop_backup(NULL); /* don't care stop_lsn on error case */ - } - /* * Update status of backup in BACKUP_CONTROL_FILE to ERROR. * end_time != 0 means backup finished @@ -1002,6 +1272,15 @@ backup_cleanup(bool fatal, void *userdata) current.status = BACKUP_STATUS_ERROR; pgBackupWriteBackupControlFile(¤t); } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(LOG, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } } /* @@ -1011,6 +1290,8 @@ static void backup_disconnect(bool fatal, void *userdata) { pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); } /* Count bytes in file */ @@ -1089,7 +1370,6 @@ backup_compressed_file_partially(pgFile *file, void *arg, size_t *skip_size) * verify checksum and copy. * In incremental backup mode, copy only files or datafiles' pages changed after * previous backup. - * TODO review */ static void backup_files(void *arg) @@ -1144,32 +1424,11 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { - /* skip files which have not been modified since last backup */ - /* TODO Implement: compare oldfile and newfile checksum. Now it's just a stub */ - if (arguments->prev_backup_filelist) - { - pgFile *prev_file = NULL; - pgFile **p = (pgFile **) parray_bsearch(arguments->prev_backup_filelist, - file, pgFileComparePath); - if (p) - prev_file = *p; - - if (prev_file && false) - { - file->write_size = BYTES_INVALID; - if (verbose) - elog(LOG, "File \"%s\" has not changed since previous backup", - file->path); - continue; - } - } - /* copy the file into backup */ if (file->is_datafile) { - if (is_compressed_data_file(file)) + if (file->is_cfs) { - /* TODO review */ size_t skip_size = 0; if (backup_compressed_file_partially(file, arguments, &skip_size)) { @@ -1178,9 +1437,7 @@ backup_files(void *arg) arguments->to_root, file, skip_size)) { - /* record as skipped file in file_xxx.txt */ file->write_size = BYTES_INVALID; - elog(LOG, "skip"); continue; } } @@ -1188,9 +1445,7 @@ backup_files(void *arg) arguments->to_root, file)) { - /* record as skipped file in file_xxx.txt */ file->write_size = BYTES_INVALID; - elog(LOG, "skip"); continue; } } @@ -1222,7 +1477,6 @@ backup_files(void *arg) /* * Append files to the backup list array. - * TODO review */ static void add_pgdata_files(parray *files, const char *root) @@ -1247,7 +1501,7 @@ add_pgdata_files(parray *files, const char *root) /* data files are under "base", "global", or "pg_tblspc" */ relative = GetRelativePath(file->path, root); if (!path_is_prefix_of_path("base", relative) && - /*!path_is_prefix_of_path("global", relative) &&*/ //TODO What's wrong with this line? + !path_is_prefix_of_path("global", relative) && !path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) continue; @@ -1477,7 +1731,10 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pg_free(rel_path); } -/* TODO review it */ +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + */ static void make_pagemap_from_ptrack(parray *files) { @@ -1499,6 +1756,8 @@ make_pagemap_from_ptrack(parray *files) size_t ptrack_nonparsed_size = 0; size_t start_addr; + /* Compute db_oid and rel_oid of the relation from the path */ + tablespace = strstr(p->ptrack_path, PG_TBLSPC_DIR); if (tablespace) @@ -1524,19 +1783,25 @@ make_pagemap_from_ptrack(parray *files) p->path); sscanf(p->path + sep_iter + 1, "%u/%u", &db_oid, &rel_oid); - + + /* get ptrack map for all segments of the relation in a raw format */ ptrack_nonparsed = pg_ptrack_get_and_clear(tablespace_oid, db_oid, rel_oid, &ptrack_nonparsed_size); - /* TODO What is 8? */ - start_addr = (RELSEG_SIZE/8)*p->segno; - if (start_addr + RELSEG_SIZE/8 > ptrack_nonparsed_size) + /* + * FIXME When do we cut VARHDR from ptrack_nonparsed? + * Compute the beginning of the ptrack map related to this segment + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*p->segno; + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) p->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; else - p->pagemap.bitmapsize = RELSEG_SIZE/8; + p->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; p->pagemap.bitmap = pg_malloc(p->pagemap.bitmapsize); memcpy(p->pagemap.bitmap, ptrack_nonparsed+start_addr, p->pagemap.bitmapsize); + pg_free(ptrack_nonparsed); } } @@ -1554,10 +1819,9 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) static XLogRecPtr prevpos = InvalidXLogRecPtr; /* we assume that we get called once at the end of each segment */ - if (verbose && segment_finished) - fprintf(stderr, _("%s: finished segment at %X/%X (timeline %u)\n"), - PROGRAM_NAME, (uint32) (xlogpos >> 32), (uint32) xlogpos, - timeline); + if (segment_finished) + elog(LOG, _("finished segment at %X/%X (timeline %u)\n"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); /* * Note that we report the previous, not current, position here. After a @@ -1568,12 +1832,31 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) * timeline, but it's close enough for reporting purposes. */ if (prevtimeline != 0 && prevtimeline != timeline) - fprintf(stderr, _("%s: switched to timeline %u at %X/%X\n"), - PROGRAM_NAME, timeline, - (uint32) (prevpos >> 32), (uint32) prevpos); + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); - if (stop_backup_lsn != InvalidXLogRecPtr && xlogpos > stop_backup_lsn) - return true; + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + return true; + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } prevtimeline = timeline; prevpos = xlogpos; @@ -1598,7 +1881,7 @@ StreamLog(void *arg) conn = GetConnection(); if (!conn) { - pthread_mutex_unlock(&check_stream_mut); + pthread_mutex_unlock(&start_stream_mut); /* Error message already written in GetConnection() */ return; } @@ -1622,7 +1905,7 @@ StreamLog(void *arg) disconnect_and_exit(1); /* Ok we have normal stream connect and main process can work again */ - pthread_mutex_unlock(&check_stream_mut); + pthread_mutex_unlock(&start_stream_mut); /* * We must use startpos as start_lsn from start_backup @@ -1634,14 +1917,15 @@ StreamLog(void *arg) */ startpos -= startpos % XLOG_SEG_SIZE; + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + /* * Start the replication */ - if (verbose) - fprintf(stderr, - _("%s: starting log streaming at %X/%X (timeline %u)\n"), - PROGRAM_NAME, (uint32) (startpos >> 32), (uint32) startpos, - starttli); + elog(LOG, _("starting log streaming at %X/%X (timeline %u)\n"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); #if PG_VERSION_NUM >= 90600 { @@ -1656,13 +1940,13 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; if(ReceiveXlogStream(conn, &ctl) == false) - elog(ERROR, "Problem in recivexlog"); + elog(ERROR, "Problem in receivexlog"); } #else if(ReceiveXlogStream(conn, startpos, starttli, NULL, basedir, stop_streaming, standby_message_timeout, NULL, false, false) == false) - elog(ERROR, "Problem in recivexlog"); + elog(ERROR, "Problem in receivexlog"); #endif PQfinish(conn); @@ -1673,7 +1957,7 @@ StreamLog(void *arg) * cfs_mmap() and cfs_munmap() function definitions mirror ones * from cfs.h, but doesn't use atomic variables, since they are * not allowed in frontend code. - * TODO Is it so? + * * Since we cannot take atomic lock on files compressed by CFS, * it should care about not changing files while backup is running. */ diff --git a/catalog.c b/src/catalog.c similarity index 89% rename from catalog.c rename to src/catalog.c index 5583dfdda..1b5cc383f 100644 --- a/catalog.c +++ b/src/catalog.c @@ -50,7 +50,7 @@ catalog_lock(void) pid_t my_pid, my_p_pid; - join_path_components(lock_file, backup_path, BACKUP_CATALOG_PID); + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); /* * If the PID in the lockfile is our own PID or our parent's or @@ -84,7 +84,7 @@ catalog_lock(void) /* * We need a loop here because of race conditions. But don't loop forever - * (for example, a non-writable $backup_path directory might cause a failure + * (for example, a non-writable $backup_instance_path directory might cause a failure * that won't go away). 100 tries seems like plenty. */ for (ntries = 0;; ntries++) @@ -253,16 +253,14 @@ catalog_get_backup_list(time_t requested_backup_id) { DIR *date_dir = NULL; struct dirent *date_ent = NULL; - char backups_path[MAXPGPATH]; parray *backups = NULL; pgBackup *backup = NULL; - /* open backup root directory */ - join_path_components(backups_path, backup_path, BACKUPS_DIR); - date_dir = opendir(backups_path); + /* open backup instance backups directory */ + date_dir = opendir(backup_instance_path); if (date_dir == NULL) { - elog(WARNING, "cannot open directory \"%s\": %s", backups_path, + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, strerror(errno)); goto err_proc; } @@ -275,12 +273,12 @@ catalog_get_backup_list(time_t requested_backup_id) char date_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backups_path, date_ent->d_name) + if (!IsDir(backup_instance_path, date_ent->d_name) || date_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(date_path, backups_path, date_ent->d_name); + join_path_components(date_path, backup_instance_path, date_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ snprintf(backup_conf_path, MAXPGPATH, "%s/%s", date_path, BACKUP_CONTROL_FILE); @@ -309,7 +307,7 @@ catalog_get_backup_list(time_t requested_backup_id) if (errno) { elog(WARNING, "cannot read backup root directory \"%s\": %s", - backups_path, strerror(errno)); + backup_instance_path, strerror(errno)); goto err_proc; } @@ -388,6 +386,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "#Configuration\n"); fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); fprintf(out, "stream = %s\n", backup->stream?"true":"false"); + fprintf(out, "compress-alg = %s\n", deparse_compress_alg(compress_alg)); + fprintf(out, "compress-level = %d\n", compress_level); + fprintf(out, "from-replica = %s\n", from_replica?"true":"false"); fprintf(out, "\n#Compatibility\n"); fprintf(out, "block-size = %u\n", backup->block_size); @@ -397,11 +398,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fprintf(out, "\n#Result backup info\n"); fprintf(out, "timelineid = %d\n", backup->tli); /* LSN returned by pg_start_backup */ - fprintf(out, "start-lsn = %x/%08x\n", + fprintf(out, "start-lsn = %X/%X\n", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); /* LSN returned by pg_stop_backup */ - fprintf(out, "stop-lsn = %x/%08x\n", + fprintf(out, "stop-lsn = %X/%X\n", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); @@ -426,6 +427,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->data_bytes != BYTES_INVALID) fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + fprintf(out, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ @@ -470,6 +474,9 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; + char *compress_alg = NULL; + int *compress_level; + bool *from_replica; pgut_option options[] = { @@ -482,12 +489,16 @@ readBackupControlFile(const char *path) {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum_version", &backup->checksum_version, SOURCE_FILE_STRICT}, {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &from_replica, SOURCE_FILE_STRICT}, {0} }; @@ -615,14 +626,32 @@ pgBackupCompareIdDesc(const void *l, const void *r) */ void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) { char *datetime; datetime = base36enc(backup->start_time); - if (subdir) - snprintf(path, len, "%s/%s/%s/%s", backup_path, BACKUPS_DIR, datetime, subdir); + + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, datetime); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, datetime, subdir1); + /* "subdir1" and "subdir2" is not NULL */ else - snprintf(path, len, "%s/%s/%s", backup_path, BACKUPS_DIR, datetime); + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + datetime, subdir1, subdir2); + free(datetime); make_native_path(path); diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 000000000..8d131c728 --- /dev/null +++ b/src/configure.c @@ -0,0 +1,240 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +static void opt_log_level(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static pgBackupConfig *cur_config = NULL; + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + if (replica_timeout != 300) /* 300 is default value */ + config->replica_timeout = replica_timeout; + + if (log_level_defined) + config->log_level = log_level; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg != NOT_DEFINED_COMPRESS) + config->compress_alg = compress_alg; + if (compress_level != DEFAULT_COMPRESS_LEVEL) + config->compress_level = compress_level; + + if (show_only) + writeBackupCatalogConfig(stderr, config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = INT_MIN; /* INT_MIN means "undefined" */ + + config->log_level = INT_MIN; /* INT_MIN means "undefined" */ + config->log_filename = NULL; + config->error_log_filename = NULL; + config->log_directory = NULL; + config->log_rotation_size = 0; + config->log_rotation_age = 0; + + config->retention_redundancy = 0; + config->retention_window = 0; + + config->compress_alg = NOT_DEFINED_COMPRESS; + config->compress_level = DEFAULT_COMPRESS_LEVEL; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = %li\n", config->system_identifier); + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + if (config->replica_timeout != INT_MIN) + fprintf(out, "replica_timeout = %d\n", config->replica_timeout); + + fprintf(out, "#Logging parameters:\n"); + if (config->log_level != INT_MIN) + fprintf(out, "log-level = %s\n", deparse_log_level(config->log_level)); + if (config->log_filename) + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + if (config->log_directory) + fprintf(out, "log-directory = %s\n", config->log_directory); + if (config->log_rotation_size) + fprintf(out, "log-rotation-size = %d\n", config->log_rotation_size); + if (config->log_rotation_age) + fprintf(out, "log-rotation-age = %d\n", config->log_rotation_age); + + fprintf(out, "#Retention parameters:\n"); + if (config->retention_redundancy) + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + if (config->retention_window) + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + + if (compress_level != config->compress_level) + fprintf(out, "compress-level = %d\n", compress_level); + else + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 37, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, + { 's', 41, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 42, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 43, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 44, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE }, + { 'u', 45, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR); + + return config; + +} + +static void +opt_log_level(pgut_option *opt, const char *arg) +{ + cur_config->log_level = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} diff --git a/data.c b/src/data.c similarity index 73% rename from data.c rename to src/data.c index c44873e0d..bcf89d947 100644 --- a/data.c +++ b/src/data.c @@ -19,10 +19,63 @@ #include "storage/block.h" #include "storage/bufpage.h" #include "storage/checksum_impl.h" +#include +#include + +static size_t zlib_compress(void* dst, size_t dst_size, void const* src, size_t src_size) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, compress_level); + return rc == Z_OK ? compressed_size : rc; +} + +static size_t zlib_decompress(void* dst, size_t dst_size, void const* src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + return rc == Z_OK ? dest_len : rc; +} + +static size_t +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size); + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +static size_t +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + + typedef struct BackupPageHeader { BlockNumber block; /* block number */ + int32 compressed_size; } BackupPageHeader; /* Verify page's header */ @@ -61,8 +114,10 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupPageHeader header; off_t offset; DataPage page; /* used as read buffer */ - size_t write_buffer_size = sizeof(header) + BLCKSZ; - char write_buffer[write_buffer_size]; + DataPage compressed_page; /* used as read buffer */ + size_t write_buffer_size; + /* maximum size of write buffer */ + char write_buffer[BLCKSZ+sizeof(header)]; size_t read_len = 0; XLogRecPtr page_lsn; int try_checksum = 100; @@ -92,12 +147,6 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, * If after several attempts page header is still invalid, throw an error. * The same idea is applied to checksum verification. */ - - /* - * TODO Should we show a hint about possible false positives suggesting to - * decrease concurrent load? Or we can just copy this page and rely on - * xlog recovery, marking backup as untrusted. - */ if (!parse_page(&page, &page_lsn)) { int i; @@ -119,9 +168,8 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, /* Try to read and verify this page again several times. */ if (try_checksum) { - if (verbose) - elog(WARNING, "File: %s blknum %u have wrong page header, try again", - file->path, blknum); + elog(WARNING, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); usleep(100); continue; } @@ -150,9 +198,8 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, { if (try_checksum) { - if (verbose) - elog(WARNING, "File: %s blknum %u have wrong checksum, try again", - file->path, blknum); + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); usleep(100); } else @@ -164,9 +211,31 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, file->read_size += read_len; - memcpy(write_buffer, &header, sizeof(header)); - /* TODO implement block compression here? */ - memcpy(write_buffer + sizeof(header), page.data, BLCKSZ); + header.compressed_size = do_compress(compressed_page.data, sizeof(compressed_page.data), + page.data, sizeof(page.data), compress_alg); + + file->compress_alg = compress_alg; + + Assert (header.compressed_size <= BLCKSZ); + write_buffer_size = sizeof(header); + + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), compressed_page.data, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page.data, BLCKSZ); + write_buffer_size += header.compressed_size; + } + + /* Update CRC */ + COMP_CRC32C(*crc, &write_buffer, write_buffer_size); + /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) { @@ -177,10 +246,6 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, file->path, blknum, strerror(errno_tmp)); } - /* update CRC */ - COMP_CRC32C(*crc, &header, sizeof(header)); - COMP_CRC32C(*crc, page.data, BLCKSZ); - file->write_size += write_buffer_size; } @@ -188,6 +253,9 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. */ bool backup_data_file(const char *from_root, const char *to_root, @@ -275,6 +343,7 @@ backup_data_file(const char *from_root, const char *to_root, n_blocks_read++; } + pg_free(file->pagemap.bitmap); pg_free(iter); } @@ -293,15 +362,11 @@ backup_data_file(const char *from_root, const char *to_root, FIN_CRC32C(file->crc); - /* Treat empty file as not-datafile. TODO Why? */ - if (file->read_size == 0) - file->is_datafile = false; - /* * If we have pagemap then file can't be a zero size. * Otherwise, we will clear the last file. */ - if (n_blocks_read == n_blocks_skipped) + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) { if (remove(to_path) == -1) elog(ERROR, "cannot remove file \"%s\": %s", to_path, @@ -314,7 +379,6 @@ backup_data_file(const char *from_root, const char *to_root, /* * Restore compressed file that was backed up partly. - * TODO review */ static void restore_file_partly(const char *from_root,const char *to_root, pgFile *file) @@ -394,8 +458,6 @@ restore_file_partly(const char *from_root,const char *to_root, pgFile *file) write_size += read_len; } -// elog(LOG, "restore_file_partly(). %s write_size %lu, file->write_size %lu", -// file->path, write_size, file->write_size); /* update file permission */ if (chmod(to_path, file->mode) == -1) @@ -417,13 +479,10 @@ restore_compressed_file(const char *from_root, const char *to_root, pgFile *file) { - if (file->is_partial_copy == 0) + if (!file->is_partial_copy) copy_file(from_root, to_root, file); - else if (file->is_partial_copy == 1) - restore_file_partly(from_root, to_root, file); else - elog(ERROR, "restore_compressed_file(). Unknown is_partial_copy value %d", - file->is_partial_copy); + restore_file_partly(from_root, to_root, file); } /* @@ -470,7 +529,8 @@ restore_data_file(const char *from_root, for (blknum = 0; ; blknum++) { size_t read_len; - DataPage page; /* used as read buffer */ + DataPage compressed_page; /* used as read buffer */ + DataPage page; /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); @@ -484,48 +544,54 @@ restore_data_file(const char *from_root, "odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(ERROR, "cannot read block %u of \"%s\": %s", + elog(ERROR, "cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); } if (header.block < blknum) - elog(ERROR, "backup is broken at block %u", - blknum); + elog(ERROR, "backup is broken at block %u", blknum); + Assert(header.compressed_size <= BLCKSZ); - if (fread(page.data, 1, BLCKSZ, in) != BLCKSZ) - elog(ERROR, "cannot read block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); - /* update checksum because we are not save whole */ - if(backup->checksum_version) + if (header.compressed_size < BLCKSZ) { - bool is_zero_page = false; + size_t uncompressed_size = 0; - if(page.page_data.pd_upper == 0) - { - int i; - for(i = 0; i < BLCKSZ && page.data[i] == 0; i++); - if (i == BLCKSZ) - is_zero_page = true; - } + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + header.compressed_size, file->compress_alg); - /* skip calc checksum if zero page */ - if (!is_zero_page) - ((PageHeader) page.data)->pd_checksum = pg_checksum_page(page.data, file->segno * RELSEG_SIZE + header.block); + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page uncompressed to %ld bytes. != BLCKSZ", uncompressed_size); } /* - * Seek and write the restored page. Backup might have holes in - * differential backups. + * Seek and write the restored page. */ blknum = header.block; if (fseek(out, blknum * BLCKSZ, SEEK_SET) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); - if (fwrite(page.data, 1, sizeof(page), out) != sizeof(page)) - elog(ERROR, "cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } } /* update file permission */ @@ -543,20 +609,10 @@ restore_data_file(const char *from_root, fclose(out); } -/* If someone's want to use this function before correct - * generation values is set, he can look up for corresponding - * .cfm file in the file_list - */ -bool -is_compressed_data_file(pgFile *file) -{ - return (file->generation != -1); -} - /* - * Add check that file is not bigger than RELSEG_SIZE. - * WARNING compressed file can be exceed this limit. - * Add compression. + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. */ bool copy_file(const char *from_root, const char *to_root, pgFile *file) @@ -614,6 +670,8 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* copy content and calc CRC */ for (;;) { + read_len = 0; + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) break; @@ -629,8 +687,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* update CRC */ COMP_CRC32C(crc, buf, read_len); - file->write_size += sizeof(buf); - file->read_size += sizeof(buf); + file->read_size += read_len; } errno_tmp = errno; @@ -657,10 +714,10 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* update CRC */ COMP_CRC32C(crc, buf, read_len); - file->write_size += read_len; file->read_size += read_len; } + file->write_size = file->read_size; /* finish CRC calculation and store into pgFile */ FIN_CRC32C(crc); file->crc = crc; @@ -681,6 +738,105 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } +/* Almost like copy file, except the fact we don't calculate checksum */ +void +copy_wal_file(const char *from_path, const char *to_path) +{ + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[XLOG_BLCKSZ]; + struct stat st; + + /* open file for read */ + in = fopen(from_path, "r"); + if (in == NULL) + { + /* maybe deleted, it's not error */ + if (errno == ENOENT) + elog(ERROR, "cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + } + + /* open backup file for write */ + out = fopen(to_path, "w"); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", from_path, + strerror(errno)); + } + + if (st.st_size > XLOG_SEG_SIZE) + elog(ERROR, "Unexpected wal file size %s : %ld", from_path, st.st_size); + + /* copy content */ + for (;;) + { + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + from_path, strerror(errno_tmp)); + } + + /* copy odd part */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + } + + + /* update file permission. */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + fclose(in); + fclose(out); + +} + /* * Save part of the file into backup. * skip_size - size of the file in previous backup. We can skip it @@ -799,9 +955,7 @@ copy_file_partly(const char *from_root, const char *to_root, } /* add meta information needed for recovery */ - file->is_partial_copy = 1; - -// elog(LOG, "copy_file_partly(). %s file->write_size %lu", to_path, file->write_size); + file->is_partial_copy = true; fclose(in); fclose(out); diff --git a/delete.c b/src/delete.c similarity index 84% rename from delete.c rename to src/delete.c index 55118fc3c..f33dd3a55 100644 --- a/delete.c +++ b/src/delete.c @@ -15,8 +15,7 @@ #include static int pgBackupDeleteFiles(pgBackup *backup); -static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - bool delete_all); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli); int do_delete(time_t backup_id) @@ -108,7 +107,7 @@ do_delete(time_t backup_id) } } - delete_walfiles(oldest_lsn, oldest_tli, true); + delete_walfiles(oldest_lsn, oldest_tli); } /* cleanup */ @@ -131,7 +130,7 @@ do_retention_purge(void) size_t i; time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TimeLineID oldest_tli; + TimeLineID oldest_tli = 0; bool keep_next_backup = true; /* Do not delete first full backup */ bool backup_deleted = false; /* At least one backup was deleted */ @@ -200,7 +199,7 @@ do_retention_purge(void) } /* Purge WAL files */ - delete_walfiles(oldest_lsn, oldest_tli, true); + delete_walfiles(oldest_lsn, oldest_tli); /* Cleanup */ parray_walk(backup_list, pgBackupFree); @@ -280,13 +279,16 @@ pgBackupDeleteFiles(pgBackup *backup) } /* - * Delete WAL segments up to oldest_lsn. + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). * - * If oldest_lsn is invalid function exists. But if delete_all is true then - * WAL segements will be deleted anyway. + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. */ static void -delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli) { XLogSegNo targetSegNo; char oldestSegmentNeeded[MAXFNAMELEN]; @@ -297,9 +299,6 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) char min_wal_file[MAXPGPATH]; int rc; - if (XLogRecPtrIsInvalid(oldest_lsn) && !delete_all) - return; - max_wal_file[0] = '\0'; min_wal_file[0] = '\0'; @@ -356,8 +355,7 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) wal_file, strerror(errno)); break; } - if (verbose) - elog(LOG, "removed WAL segment \"%s\"", wal_file); + elog(LOG, "removed WAL segment \"%s\"", wal_file); if (max_wal_file[0] == '\0' || strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) @@ -370,9 +368,9 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) } } - if (!verbose && min_wal_file[0] != '\0') + if (min_wal_file[0] != '\0') elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); - if (!verbose && max_wal_file[0] != '\0') + if (max_wal_file[0] != '\0') elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); if (errno) @@ -386,3 +384,48 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, bool delete_all) elog(WARNING, "could not open archive location \"%s\": %s", arclog_path, strerror(errno)); } + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} \ No newline at end of file diff --git a/dir.c b/src/dir.c similarity index 95% rename from dir.c rename to src/dir.c index 720cf2e11..5aed3acd4 100644 --- a/dir.c +++ b/src/dir.c @@ -154,8 +154,10 @@ pgFileInit(const char *path) file->segno = 0; file->path = pgut_malloc(strlen(path) + 1); strcpy(file->path, path); /* enough buffer size guaranteed */ - file->generation = -1; - file->is_partial_copy = 0; + file->is_cfs = false; + file->generation = 0; + file->is_partial_copy = false; + file->compress_alg = NOT_DEFINED_COMPRESS; return file; } @@ -307,7 +309,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray *black_list = NULL; char path[MAXPGPATH]; - join_path_components(path, backup_path, PG_BLACK_LIST); + join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) { @@ -344,7 +346,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, } /* - * TODO Add comment, review + * TODO Add comment */ static void dir_list_file_internal(parray *files, const char *root, bool exclude, @@ -633,7 +635,7 @@ read_tablespace_map(parray *files, const char *backup_dir) path[MAXPGPATH]; pgFile *file; - if (sscanf(buf, "%s %s", link_name, path) != 2) + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) elog(ERROR, "invalid format found in \"%s\"", map_path); file = pgut_new(pgFile); @@ -671,20 +673,21 @@ print_file_list(FILE *out, const parray *files, const char *root) path = GetRelativePath(path, root); fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," - "\"is_datafile\":\"%u\", \"crc\":\"%u\"", + "\"is_datafile\":\"%u\", \"crc\":\"%u\", \"compress_alg\":\"%s\"", path, (unsigned long) file->write_size, file->mode, - file->is_datafile?1:0, file->crc); + file->is_datafile?1:0, file->crc, deparse_compress_alg(file->compress_alg)); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); - /* TODO What for do we write it to file? */ if (S_ISLNK(file->mode)) fprintf(out, ",\"linked\":\"%s\"", file->linked); #ifdef PGPRO_EE - fprintf(out, ",\"CFS_generation\":\"" UINT64_FORMAT "\",\"is_partial_copy\":\"%d\"", - file->generation, file->is_partial_copy); + if (file->is_cfs) + fprintf(out, ",\"is_cfs\":\"%u\" ,\"CFS_generation\":\"" UINT64_FORMAT "\"," + "\"is_partial_copy\":\"%u\"", + file->is_cfs?1:0, file->generation, file->is_partial_copy?1:0); #endif fprintf(out, "}\n"); } @@ -847,6 +850,7 @@ dir_read_file_list(const char *root, const char *file_txt) char path[MAXPGPATH]; char filepath[MAXPGPATH]; char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; uint64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, @@ -854,7 +858,8 @@ dir_read_file_list(const char *root, const char *file_txt) segno; #ifdef PGPRO_EE uint64 generation, - is_partial_copy; + is_partial_copy, + is_cfs; #endif pgFile *file; @@ -867,10 +872,12 @@ dir_read_file_list(const char *root, const char *file_txt) /* optional fields */ get_control_value(buf, "linked", linked, NULL, false); get_control_value(buf, "segno", NULL, &segno, false); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); #ifdef PGPRO_EE - get_control_value(buf, "CFS_generation", NULL, &generation, true); - get_control_value(buf, "is_partial_copy", NULL, &is_partial_copy, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "CFS_generation", NULL, &generation, false); + get_control_value(buf, "is_partial_copy", NULL, &is_partial_copy, false); #endif if (root) join_path_components(filepath, root, path); @@ -883,12 +890,14 @@ dir_read_file_list(const char *root, const char *file_txt) file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); if (linked[0]) file->linked = pgut_strdup(linked); file->segno = (int) segno; #ifdef PGPRO_EE + file->is_cfs = is_cfs ? true : false; file->generation = generation; - file->is_partial_copy = (int) is_partial_copy; + file->is_partial_copy = is_partial_copy ? true : false; #endif parray_append(files, file); diff --git a/fetch.c b/src/fetch.c similarity index 100% rename from fetch.c rename to src/fetch.c diff --git a/src/help.c b/src/help.c new file mode 100644 index 000000000..1f116528f --- /dev/null +++ b/src/help.c @@ -0,0 +1,339 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command. Try pg_probackup help\n")); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level=log-level]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--progress] [--delete-expired]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + + printf(_("\n %s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + + printf(_("\n %s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + + printf(_("\n %s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path -D pgdata-dir\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress] [--delete-expired]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" --archive-timeout wait timeout for WAL segment archiving\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9]\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -d, --dbname=DBNAME database to connect\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication in seconds\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-dir] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-dir [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --wal remove unnecessary wal files\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-dir --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level=log-level]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level=log-level controls which message levels are sent to the log\n")); + printf(_(" --log-filename=log-filename file names of the created log files which is treated as as strftime pattern\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" file names of the created log files for error messages\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory in which log files will be created\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" maximum size of an individual log file in kilobytes\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" maximum lifetime of an individual log file in minutes\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9]\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -d, --dbname=DBNAME database to connect\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name=host_name database server host of master\n")); + printf(_(" --master-port=port=port database server port of master\n")); + printf(_(" --master-user=user_name=user_name user name to connect to master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-dir --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-dir -D pgdata-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-dir location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} diff --git a/init.c b/src/init.c similarity index 53% rename from init.c rename to src/init.c index 9a85022d9..d92704cbe 100644 --- a/init.c +++ b/src/init.c @@ -12,6 +12,7 @@ #include #include +#include /* * selects function for scandir. @@ -30,15 +31,8 @@ do_init(void) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; - struct dirent **dp; int results; - pgBackupConfig *config = pgut_new(pgBackupConfig); - - /* PGDATA is always required */ - if (pgdata == NULL) - elog(ERROR, "Required parameter not specified: PGDATA " - "(-D, --pgdata)"); if (access(backup_path, F_OK) == 0) { @@ -47,20 +41,63 @@ do_init(void) elog(ERROR, "backup catalog already exist and it's not empty"); } - /* Read system_identifier from PGDATA */ - system_identifier = get_system_identifier(); - /* create backup catalog root directory */ dir_create_dir(backup_path, DIR_PERMISSION); - /* create directories for backup of online files */ + /* create backup catalog data directory */ join_path_components(path, backup_path, BACKUPS_DIR); dir_create_dir(path, DIR_PERMISSION); - /* Create "wal" directory */ + /* create backup catalog wal directory */ join_path_components(arclog_path_dir, backup_path, "wal"); dir_create_dir(arclog_path_dir, DIR_PERMISSION); + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + /* * Wite initial config. system-identifier and pgdata are set in * init subcommand and will never be updated. @@ -70,5 +107,6 @@ do_init(void) config->pgdata = pgdata; writeBackupCatalogConfigFile(config); + elog(INFO, "Instance '%s' successfully inited", instance_name); return 0; } diff --git a/parsexlog.c b/src/parsexlog.c similarity index 69% rename from parsexlog.c rename to src/parsexlog.c index f94baaeac..aa8de7ed9 100644 --- a/parsexlog.c +++ b/src/parsexlog.c @@ -110,6 +110,14 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, XLogSegNo endSegNo, nextSegNo = 0; + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + private.archivedir = archivedir; private.tli = tli; xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); @@ -123,6 +131,7 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, do { record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) { XLogRecPtr errptr; @@ -130,12 +139,23 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, errptr = startpoint ? startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", + elog(WARNING, "could not read WAL record at %X/%X: %s", (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(ERROR, "could not read WAL record at %X/%X", + elog(WARNING, "could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + if (!xlogexists) + elog(ERROR, "WAL segment \"%s\" is absent", xlogfpath); + else if (xlogreadfd != -1) + elog(ERROR, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); } extractPageInfo(xlogreader); @@ -143,7 +163,6 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, startpoint = InvalidXLogRecPtr; /* continue reading at next record */ XLByteToSeg(xlogreader->EndRecPtr, nextSegNo); - } while (nextSegNo <= endSegNo && xlogreader->EndRecPtr != endpoint); XLogReaderFree(xlogreader); @@ -155,6 +174,98 @@ extractPageMap(const char *archivedir, XLogRecPtr startpoint, TimeLineID tli, } } +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, + TimeLineID tli) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + private.archivedir = backup_xlog_path; + private.tli = tli; + + /* We will check it in the end */ + xlogfpath[0] = '\0'; + + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + if (xlogfpath[0] != 0) + { + /* XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status below. + */ + if (!xlogexists) + { + elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); + } + else if (xlogreadfd != -1) + { + elog(WARNING, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); + } + } + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + elog(ERROR, "there are not enough WAL records to restore from %X/%X to %X/%X", + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + XLogReaderFree(xlogreader); + if (xlogreadfd != -1) + { + close(xlogreadfd); + xlogreadfd = -1; + xlogexists = false; + } +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ void validate_wal(pgBackup *backup, const char *archivedir, @@ -163,6 +274,7 @@ validate_wal(pgBackup *backup, TimeLineID tli) { XLogRecPtr startpoint = backup->start_lsn; + char *backup_id; XLogRecord *record; XLogReaderState *xlogreader; char *errormsg; @@ -171,10 +283,63 @@ validate_wal(pgBackup *backup, TimestampTz last_time = 0; char last_timestamp[100], target_timestamp[100]; - bool all_wal = false, - got_endpoint = false; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + sprintf(backup_xlog_path, "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli); + + free(backup_id); + + /* + * If recovery target is provided check that we can restore backup to a + * recoverty target time or xid. + */ + if (!TransactionIdIsValid(target_xid) || target_time == 0) + { + /* Recoverty target is not given so exit */ + elog(INFO, "backup validation completed successfully"); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ private.archivedir = archivedir; + private.tli = tli; xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -183,6 +348,15 @@ validate_wal(pgBackup *backup, /* We will check it in the end */ xlogfpath[0] = '\0'; + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time)) + all_wal = true; + + startpoint = backup->stop_lsn; while (true) { bool timestamp_record; @@ -196,10 +370,6 @@ validate_wal(pgBackup *backup, break; } - /* Got WAL record at stop_lsn */ - if (xlogreader->ReadRecPtr == backup->stop_lsn) - got_endpoint = true; - timestamp_record = getRecordTimestamp(xlogreader, &last_time); if (XLogRecGetXid(xlogreader) != InvalidTransactionId) last_xid = XLogRecGetXid(xlogreader); @@ -230,54 +400,45 @@ validate_wal(pgBackup *backup, if (last_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), timestamptz_to_time_t(last_time)); - else - time2iso(last_timestamp, lengthof(last_timestamp), - backup->recovery_time); - if (last_xid == InvalidTransactionId) - last_xid = backup->recovery_xid; - /* There are all need WAL records */ + /* There are all needed WAL records */ if (all_wal) elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, last_timestamp, last_xid); - /* There are not need WAL records */ + /* Some needed WAL records are absent */ else { if (xlogfpath[0] != 0) { - /* XLOG reader couldnt read WAL segment */ + /* XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status below. + */ if (!xlogexists) + { elog(WARNING, "WAL segment \"%s\" is absent", xlogfpath); + } else if (xlogreadfd != -1) - elog(ERROR, "Possible WAL CORRUPTION." - "Error has occured during reading WAL segment \"%s\"", xlogfpath); + { + elog(WARNING, "Possible WAL CORRUPTION." + "Error has occured during reading WAL segment \"%s\"", xlogfpath); + } } - if (!got_endpoint) - elog(ERROR, "there are not enough WAL records to restore from %X/%X to %X/%X", - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn), - (uint32) (backup->stop_lsn >> 32), - (uint32) (backup->stop_lsn)); - else - { - if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); - - elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, - last_timestamp, last_xid); - - if (TransactionIdIsValid(target_xid) && target_time != 0) - elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, - target_timestamp, target_xid); - else if (TransactionIdIsValid(target_xid)) - elog(ERROR, "not enough WAL records to xid " XID_FMT, - target_xid); - else if (target_time != 0) - elog(ERROR, "not enough WAL records to time %s", - target_timestamp); - } + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); + + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); } /* clean */ @@ -305,6 +466,14 @@ read_recovery_info(const char *archivedir, TimeLineID tli, XLogPageReadPrivate private; bool res; + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + private.archivedir = archivedir; private.tli = tli; @@ -365,7 +534,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, } /* - * Check if WAL segment file 'wal_path' contains 'target_lsn'. + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. */ bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, @@ -376,6 +546,10 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, char *errormsg; bool res; + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + private.archivedir = archivedir; private.tli = target_tli; @@ -384,15 +558,7 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "out of memory"); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; - if (!res) - { - if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", - (uint32) (target_lsn >> 32), (uint32) (target_lsn), - errormsg); - - /* Didn't find 'target_lsn' and there is no error, return false */ - } + /* Didn't find 'target_lsn' and there is no error, return false */ XLogReaderFree(xlogreader); if (xlogreadfd != -1) @@ -413,9 +579,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, { XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; uint32 targetPageOff; - XLogSegNo targetSegNo; - XLByteToSeg(targetPagePtr, targetSegNo); targetPageOff = targetPagePtr % XLogSegSize; /* @@ -478,8 +642,6 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, return -1; } - Assert(targetSegNo == xlogreadsegno); - *pageTLI = private->tli; return XLOG_BLCKSZ; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 000000000..867df1057 --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,475 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" + +#include +#include +#include +#include +#include + +const char *PROGRAM_VERSION = "1.1.17"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +char *backup_id_string_param = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool from_replica = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = 300; /* default is 300 seconds */ +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = 300; /* default is 300 seconds */ + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_inclusive; +static TimeLineID target_tli; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = NOT_DEFINED_COMPRESS; +int compress_level = DEFAULT_COMPRESS_LEVEL; + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd; + +bool help = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string_param, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE }, + { 'b', 12, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 13, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 14, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 15, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 16, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 17, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, }, + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 30, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 31, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 32, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 33, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 34, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 35, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 37, "compress-level", &compress_level, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, + { 's', 41, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 42, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 43, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 44, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE }, + { 'u', 45, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 50, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 51, "instance", &instance_name, SOURCE_CMDLINE }, + /* archive-push options */ + { 's', 60, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 61, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char path[MAXPGPATH]; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackup_init(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG; + else if (strcmp(argv[1], "--help") == 0 + || strcmp(argv[1], "help") == 0 + || strcmp(argv[1], "-?") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { + if (argc == 2) + { + fprintf(stderr, "%s %s\n", PROGRAM_NAME, PROGRAM_VERSION); + exit(0); + } + else if (strcmp(argv[2], "--help") == 0) + help_command(argv[1]); + else + elog(ERROR, "Invalid arguments for \"%s\" subcommand", argv[1]); + } + else + elog(ERROR, "Unknown subcommand"); + } + + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help) + help_command(argv[2]); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT && backup_subcmd != SHOW && backup_subcmd != VALIDATE) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT && backup_subcmd != ADD_INSTANCE) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG) + { + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR); + } + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + + /* Set log path */ + if (log_filename || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, backup_path, "log"); + } + + /* Sanity check of --backup-id option */ + if (backup_id_string_param != NULL) + { + if (backup_subcmd != RESTORE + && backup_subcmd != VALIDATE + && backup_subcmd != DELETE + && backup_subcmd != SHOW) + elog(ERROR, "Cannot use -i (--backup-id) option together with the '%s' command", + argv[1]); + + current.backup_id = base36dec(backup_id_string_param); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id"); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (pgut_dbname != NULL) + dbname = pstrdup(pgut_dbname); + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (target_time != NULL && target_xid != NULL) + elog(ERROR, "You can't specify recovery-target-time and recovery-target-xid at the same time"); + + if (num_threads < 1) + num_threads = 1; + + if (backup_subcmd != SET_CONFIG) + { + if (compress_level != DEFAULT_COMPRESS_LEVEL + && compress_alg == NONE_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH: + return do_archive_push(wal_file_path, wal_file_name); + case ARCHIVE_GET: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE: + return do_add_instance(); + case DELETE_INSTANCE: + return do_delete_instance(); + case INIT: + return do_init(); + case BACKUP: + return do_backup(); + case RESTORE: + return do_restore_or_validate(current.backup_id, + target_time, target_xid, + target_inclusive, target_tli, + true); + case VALIDATE: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + target_time, target_xid, + target_inclusive, target_tli, + false); + case SHOW: + return do_show(current.backup_id); + case DELETE: + if (delete_expired && backup_id_string_param) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case SHOW_CONFIG: + if (argc > 6) + elog(ERROR, "show-config command doesn't accept any options except -B and --instance"); + return do_configure(true); + case SET_CONFIG: + if (argc == 5) + elog(ERROR, "set-config command requires at least one option"); + return do_configure(false); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level(pgut_option *opt, const char *arg) +{ + log_level = parse_log_level(arg); + log_level_defined = true; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} diff --git a/pg_probackup.h b/src/pg_probackup.h similarity index 83% rename from pg_probackup.h rename to src/pg_probackup.h index 7bf3b559e..6e2b42eca 100644 --- a/pg_probackup.h +++ b/src/pg_probackup.h @@ -13,23 +13,33 @@ #include "postgres_fe.h" #include -#include "libpq-fe.h" +#include -#include "pgut/pgut.h" +#ifndef WIN32 +#include +#endif + +#include "access/timeline.h" #include "access/xlogdefs.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" -#include "utils/pg_crc.h" -#include "parray.h" -#include "datapagemap.h" -#include "storage/bufpage.h" #include "storage/block.h" +#include "storage/bufpage.h" #include "storage/checksum.h" -#include "access/timeline.h" +#include "utils/pg_crc.h" -#ifndef WIN32 -#include -#endif +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) /* Directory/File names */ #define DATABASE_DIR "database" @@ -53,6 +63,14 @@ #define XID_FMT "%u" #endif +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + /* Information about single file (or dir) in backup */ typedef struct pgFile { @@ -69,11 +87,13 @@ typedef struct pgFile char *path; /* path of the file */ char *ptrack_path; /* path of the ptrack fork of the relation */ int segno; /* Segment number for ptrack */ - uint64 generation; /* Generation of the compressed file. Set to '-1' - * for non-compressed files. If generation has changed, - * we cannot backup compressed file partially. */ - int is_partial_copy; /* for compressed files. Set to '1' if backed up - * via copy_file_partly() */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + uint64 generation; /* Generation of the compressed file.If generation + * has changed, we cannot backup compressed file + * partially. Has no sense if (is_cfs == false). */ + bool is_partial_copy; /* If the file was backed up via copy_file_partly(). + * Only applies to is_cfs files. */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile uint32 lock; /* lock for synchronization of parallel threads */ datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ } pgFile; @@ -102,6 +122,10 @@ typedef enum BackupMode typedef enum ProbackupSubcmd { INIT = 0, + ARCHIVE_PUSH, + ARCHIVE_GET, + ADD_INSTANCE, + DELETE_INSTANCE, BACKUP, RESTORE, VALIDATE, @@ -111,6 +135,7 @@ typedef enum ProbackupSubcmd SHOW_CONFIG } ProbackupSubcmd; + /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 #define BYTES_INVALID (-1) @@ -124,8 +149,24 @@ typedef struct pgBackupConfig const char *pgport; const char *pguser; + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int log_level; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + uint32 retention_redundancy; uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; } pgBackupConfig; /* Information about single backup stored in backup.conf */ @@ -155,6 +196,8 @@ typedef struct pgBackup * BYTES_INVALID means nothing was backed up. */ int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; /* Fields needed for compatibility check */ uint32 block_size; @@ -217,33 +260,53 @@ extern int cfs_munmap(FileMap* map); #define XLogDataFromLSN(data, xlogid, xrecoff) \ sscanf(data, "%X/%X", xlogid, xrecoff) -/* in probackup.c */ - -/* path configuration */ +/* directory options */ extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; extern char *pgdata; extern char arclog_path[MAXPGPATH]; -/* current settings */ -extern pgBackup current; -extern ProbackupSubcmd backup_subcmd; - -extern bool smooth_checkpoint; +/* common options */ extern int num_threads; extern bool stream_wal; -extern bool from_replica; extern bool progress; + +/* backup options */ +extern bool smooth_checkpoint; +extern uint32 archive_timeout; +extern bool from_replica; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +extern uint32 replica_timeout; + +/* delete options */ extern bool delete_wal; extern bool delete_expired; extern bool apply_to_all; extern bool force_delete; -extern uint32 archive_timeout; - -extern uint64 system_identifier; +/* retention options */ extern uint32 retention_redundancy; extern uint32 retention_window; +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; + +#define DEFAULT_COMPRESS_LEVEL 6 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + /* in dir.c */ /* exclude directory list for $PGDATA file listing */ extern const char *pgdata_exclude_dir[]; @@ -275,6 +338,12 @@ extern void opt_tablespace_map(pgut_option *opt, const char *arg); /* in init.c */ extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + /* in configure.c */ extern int do_configure(bool show_only); @@ -289,6 +358,7 @@ extern int do_show(time_t requested_backup_id); /* in delete.c */ extern int do_delete(time_t backup_id); extern int do_retention_purge(void); +extern int do_delete_instance(void); /* in fetch.c */ extern char *slurpFile(const char *datadir, @@ -302,6 +372,7 @@ extern void help_command(char *command); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); /* in catalog.c */ extern pgBackup *read_backup(time_t timestamp); @@ -314,6 +385,8 @@ extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void pgBackupWriteBackupControlFile(pgBackup *backup); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); @@ -350,12 +423,12 @@ extern void restore_data_file(const char *from_root, const char *to_root, pgFile *file, pgBackup *backup); extern void restore_compressed_file(const char *from_root, const char *to_root, pgFile *file); -extern bool is_compressed_data_file(pgFile *file); extern bool backup_compressed_file_partially(pgFile *file, void *arg, size_t *skip_size); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void copy_wal_file(const char *from_root, const char *to_root); extern bool copy_file_partly(const char *from_root, const char *to_root, pgFile *file, size_t skip_size); @@ -389,7 +462,7 @@ extern XLogRecPtr get_last_ptrack_lsn(void); extern uint32 get_data_checksum_version(bool safe); extern char *base36enc(long unsigned int value); extern long unsigned int base36dec(const char *text); -extern uint64 get_system_identifier(void); +extern uint64 get_system_identifier(char *pgdata); extern pg_time_t timestamptz_to_time_t(TimestampTz t); extern void pgBackup_init(pgBackup *backup); diff --git a/restore.c b/src/restore.c similarity index 92% rename from restore.c rename to src/restore.c index 617cdb4fb..cc012a9d5 100644 --- a/restore.c +++ b/src/restore.c @@ -90,8 +90,8 @@ do_restore_or_validate(time_t target_backup_id, pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; - int dest_backup_index; - int base_full_backup_index; + int dest_backup_index = 0; + int base_full_backup_index = 0; char *action = is_restore ? "Restore":"Validate"; if (is_restore) @@ -122,7 +122,7 @@ do_restore_or_validate(time_t target_backup_id, timelines = readTimeLineHistory_probackup(target_tli); } - /* Find backup range we should restore. */ + /* Find backup range we should restore or validate. */ for (i = 0; i < parray_num(backups); i++) { current_backup = (pgBackup *) parray_get(backups, i); @@ -141,7 +141,7 @@ do_restore_or_validate(time_t target_backup_id, { if (current_backup->status != BACKUP_STATUS_OK) elog(ERROR, "Backup %s has status: %s", - base36enc(current_backup->status), status2str(current_backup->status)); + base36enc(current_backup->start_time), status2str(current_backup->status)); if (target_tli) { @@ -217,40 +217,43 @@ do_restore_or_validate(time_t target_backup_id, pgBackupValidate(backup); } + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, base_full_backup->tli); + + /* We ensured that all backups are valid, now restore if required */ if (is_restore) { + pgBackup *backup; for (i = base_full_backup_index; i >= dest_backup_index; i--) { - pgBackup *backup = (pgBackup *) parray_get(backups, i); + backup = (pgBackup *) parray_get(backups, i); if (backup->status == BACKUP_STATUS_OK) restore_backup(backup); else elog(ERROR, "backup %s is not valid", base36enc(backup->start_time)); } - } - /* - * Delete files which are not in dest backup file list. Files which were - * deleted between previous and current backup are not in the list. - */ - if (is_restore) - { - pgBackup *dest_backup = (pgBackup *) parray_get(backups, dest_backup_index); + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ if (dest_backup->backup_mode != BACKUP_MODE_FULL) remove_deleted_files(dest_backup); - } - if (!dest_backup->stream - || (target_time != NULL || target_xid != NULL)) - { - if (is_restore) + /* Create recovery.conf with given recovery target parameters */ + if (!dest_backup->stream + || (target_time != NULL || target_xid != NULL)) + { create_recovery_conf(target_backup_id, target_time, target_xid, target_inclusive, target_tli); - else - validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, base_full_backup->tli); + } } /* cleanup */ @@ -269,7 +272,7 @@ void restore_backup(pgBackup *backup) { char timestamp[100]; - char backup_path[MAXPGPATH]; + char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; @@ -296,9 +299,10 @@ restore_backup(pgBackup *backup) /* * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ - pgBackupGetPath(backup, backup_path, lengthof(backup_path), NULL); - restore_directories(pgdata, backup_path); + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + restore_directories(pgdata, this_backup_path); /* * Get list of files which need to be restored. @@ -332,8 +336,7 @@ restore_backup(pgBackup *backup) arg->files = files; arg->backup = backup; - if (verbose) - elog(LOG, "Start thread for num:%li", parray_num(files)); + elog(LOG, "Start thread for num:%li", parray_num(files)); restore_threads_args[i] = arg; pthread_create(&restore_threads[i], NULL, (void *(*)(void *)) restore_files, arg); @@ -350,7 +353,7 @@ restore_backup(pgBackup *backup) parray_walk(files, pgFileFree); parray_free(files); - if (verbose) + if (log_level <= LOG) { char *backup_id; @@ -371,11 +374,9 @@ remove_deleted_files(pgBackup *backup) { parray *files; parray *files_restored; - char database_path[MAXPGPATH]; char filelist_path[MAXPGPATH]; int i; - pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ files = dir_read_file_list(pgdata, filelist_path); @@ -395,7 +396,7 @@ remove_deleted_files(pgBackup *backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - if (verbose) + if (log_level <= LOG) elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); } } @@ -449,7 +450,7 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) /* Extract link name from relative path */ link_sep = first_dir_separator(link_ptr); - if (link_sep) + if (link_sep != NULL) { int len = link_sep - link_ptr; strncpy(link_name, link_ptr, len); @@ -483,8 +484,12 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) */ if (strcmp(dir_created, linked_path) == 0) { - /* Create rest of directories */ - if (link_sep && (link_sep + 1)) + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') goto create_directory; else continue; @@ -527,8 +532,11 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) /* Save linked directory */ set_tablespace_created(link_name, linked_path); - /* Create rest of directories */ - if (link_sep && (link_sep + 1)) + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') goto create_directory; continue; @@ -560,7 +568,7 @@ restore_directories(const char *pg_data_dir, const char *backup_dir) static void check_tablespace_mapping(pgBackup *backup) { - char backup_path[MAXPGPATH]; + char this_backup_path[MAXPGPATH]; parray *links; size_t i; TablespaceListCell *cell; @@ -568,10 +576,17 @@ check_tablespace_mapping(pgBackup *backup) links = parray_new(); - pgBackupGetPath(backup, backup_path, lengthof(backup_path), NULL); - read_tablespace_map(links, backup_path); + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); - elog(LOG, "check tablespace directories of backup %s", base36enc(backup->start_time)); + if (log_level <= LOG) + { + char *backup_id; + + backup_id = base36enc(backup->start_time); + elog(LOG, "check tablespace directories of backup %s", backup_id); + pfree(backup_id); + } /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ for (cell = tablespace_dirs.head; cell; cell = cell->next) @@ -665,10 +680,15 @@ restore_files(void *arg) continue; } - /* restore file */ + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ if (file->is_datafile) { - if (is_compressed_data_file(file)) + if (file->is_cfs) restore_compressed_file(from_root, pgdata, file); else restore_data_file(from_root, pgdata, file, arguments->backup); @@ -703,7 +723,8 @@ create_recovery_conf(time_t backup_id, fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); - fprintf(fp, "restore_command = 'cp %s/%%f %%p'\n", arclog_path); + fprintf(fp, "restore_command = 'pg_probackup archive-get -B %s --instance %s --wal-file-path %%p --wal-file-name %%f'\n", + backup_path, instance_name); if (target_time) fprintf(fp, "recovery_target_time = '%s'\n", target_time); @@ -866,7 +887,6 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) /* * Get recovery options in the string format, parse them * and fill up the pgRecoveryTarget structure. - * TODO move arguments parsing and validation to getopt. */ pgRecoveryTarget * parseRecoveryTargetOptions(const char *target_time, diff --git a/show.c b/src/show.c similarity index 71% rename from show.c rename to src/show.c index 980066020..92c6da3c0 100644 --- a/show.c +++ b/src/show.c @@ -10,19 +10,74 @@ #include "pg_probackup.h" #include +#include +#include +#include + static void show_backup_list(FILE *out, parray *backup_list); static void show_backup_detail(FILE *out, pgBackup *backup); +static int do_show_instance(time_t requested_backup_id); + +int +do_show(time_t requested_backup_id) +{ + + if (instance_name == NULL + && requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + fprintf(stdout, "\nBACKUP INSTANCE '%s'\n", instance_name); + do_show_instance(0); + } + return 0; + } + else + return do_show_instance(requested_backup_id); +} /* * If 'requested_backup_id' is INVALID_BACKUP_ID, show brief meta information - * about all backups in the backup catalog. + * about all backups in the backup instance. * If valid backup id is passed, show detailed meta information * about specified backup. */ - -int -do_show(time_t requested_backup_id) +static int +do_show_instance(time_t requested_backup_id) { if (requested_backup_id != INVALID_BACKUP_ID) { @@ -107,7 +162,6 @@ pretty_size(int64 size, char *buf, size_t len) } } -/* TODO Add comment */ static TimeLineID get_parent_tli(TimeLineID child_tli) { @@ -170,10 +224,11 @@ show_backup_list(FILE *out, parray *backup_list) { int i; + /* if you add new fields here, fix the header */ /* show header */ - fputs("====================================================================================================================\n", out); - fputs("ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); - fputs("====================================================================================================================\n", out); + fputs("===============================================================================================================================\n", out); + fputs(" Instance ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n", out); + fputs("===============================================================================================================================\n", out); for (i = 0; i < parray_num(backup_list); i++) { @@ -202,8 +257,8 @@ show_backup_list(FILE *out, parray *backup_list) parent_tli = get_parent_tli(backup->tli); backup_id = base36enc(backup->start_time); - fprintf(out, "%-6s %-19s %-6s %-7s %3d / %-3d %5s %6s %2X/%08X %2X/%08X %-8s\n", - backup_id, + fprintf(out, " %-11s %-6s %-19s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, backup_id, timestamp, pgBackupGetBackupMode(backup), backup->stream ? "STREAM": "ARCHIVE", diff --git a/status.c b/src/status.c similarity index 100% rename from status.c rename to src/status.c diff --git a/util.c b/src/util.c similarity index 95% rename from util.c rename to src/util.c index 0901f7c73..bf9b6db8e 100644 --- a/util.c +++ b/src/util.c @@ -27,7 +27,7 @@ base36enc(long unsigned int value) buffer[--offset] = base36[value % 36]; } while (value /= 36); - return strdup(&buffer[offset]); // warning: this must be free-d by the user + return strdup(&buffer[offset]); /* warning: this must be free-d by the user */ } long unsigned int @@ -75,7 +75,9 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } -/* TODO Add comment */ +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ XLogRecPtr get_last_ptrack_lsn(void) { @@ -114,14 +116,14 @@ get_current_timeline(bool safe) } uint64 -get_system_identifier(void) +get_system_identifier(char *pgdata_path) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata, "global/pg_control", &size, false); + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 000000000..ddb90d555 --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,532 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* Logger parameters */ + +int log_level = INFO; +bool log_level_defined = false; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, const char *fmt, va_list args) + pg_attribute_printf(2, 0); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging to file is in progress */ +static bool logging_to_file = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + case FATAL: + fputs("FATAL: ", stream); + break; + case PANIC: + fputs("PANIC: ", stream); + break; + default: + elog(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Logs to stderr or to log file and exit if ERROR or FATAL. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + + write_to_file = log_path[0] != '\0' && !logging_to_file && + (log_filename || error_log_filename); + + /* + * There is no need to lock if this is elog() from upper elog() and + * logging is not initialized. + */ + if (write_to_file) + pthread_mutex_lock(&log_file_mutex); + + write_to_error_log = + elevel >= ERROR && error_log_filename && write_to_file; + write_to_stderr = elevel >= ERROR || !write_to_file; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (log_filename && write_to_file) + { + logging_to_file = true; + + if (log_file == NULL) + open_logfile(&log_file, log_filename); + + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + + logging_to_file = false; + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + logging_to_file = true; + + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + logging_to_file = false; + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + if (write_to_file) + pthread_mutex_unlock(&log_file_mutex); + + /* Exit with code if it is an error */ + if (elevel > WARNING) + exit(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR or FATAL. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + else if (pg_strncasecmp("fatal", v, len) == 0) + return FATAL; + else if (pg_strncasecmp("panic", v, len) == 0) + return PANIC; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + case FATAL: + return "FATAL"; + case PANIC: + return "PANIC"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog(FATAL, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time)) + elog(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 000000000..5961dfb0d --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define FATAL 2 +#define PANIC 3 + +/* Logger parameters */ + +extern int log_level; +extern bool log_level_defined; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +extern int log_rotation_size; +extern int log_rotation_age; + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/parray.c b/src/utils/parray.c similarity index 99% rename from parray.c rename to src/utils/parray.c index f21310931..a9ba7c8e5 100644 --- a/parray.c +++ b/src/utils/parray.c @@ -7,7 +7,7 @@ *------------------------------------------------------------------------- */ -#include "pg_probackup.h" +#include "src/pg_probackup.h" /* members of struct parray are hidden from client. */ struct parray diff --git a/parray.h b/src/utils/parray.h similarity index 100% rename from parray.h rename to src/utils/parray.h diff --git a/pgut/pgut.c b/src/utils/pgut.c similarity index 81% rename from pgut/pgut.c rename to src/utils/pgut.c index a129e8776..032067256 100644 --- a/pgut/pgut.c +++ b/src/utils/pgut.c @@ -2,7 +2,8 @@ * * pgut.c * - * Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional * *------------------------------------------------------------------------- */ @@ -15,6 +16,7 @@ #include #include +#include "logger.h" #include "pgut.h" /* old gcc doesn't have LLONG_MAX. */ @@ -33,8 +35,6 @@ const char *host = NULL; const char *port = NULL; const char *username = NULL; char *password = NULL; -bool verbose = false; -bool quiet = false; bool prompt_password = true; /* Database connections */ @@ -46,16 +46,6 @@ static bool in_cleanup = false; static bool parse_pair(const char buffer[], char key[], char value[]); -typedef enum -{ - PG_DEBUG, - PG_PROGRESS, - PG_WARNING, - PG_FATAL -} eLogType; - -void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); - /* Connection routines */ static void init_cancel_handler(void); static void on_before_exec(PGconn *conn); @@ -65,6 +55,75 @@ static void on_cleanup(void); static void exit_or_abort(int exitcode); static const char *get_username(void); +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XSEGS, (1024 * 1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, + {"GB", OPTION_UNIT_XSEGS, (1024 * 1024) / (XLOG_SEG_SIZE / 1024)}, + {"MB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / (1024 * 1024))}, + {"kB", OPTION_UNIT_XSEGS, -(XLOG_SEG_SIZE / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + static size_t option_length(const pgut_option opts[]) { @@ -115,7 +174,7 @@ option_find(int c, pgut_option opts1[]) static void assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) { - const char *message; + const char *message; if (opt == NULL) { @@ -135,6 +194,8 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) } else { + pgut_optsrc orig_source = opt->source; + /* can be overwritten if non-command line source */ opt->source = src; @@ -177,10 +238,13 @@ assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) message = "a 64bit unsigned integer"; break; case 's': - if (opt->source != SOURCE_DEFAULT) + if (orig_source != SOURCE_DEFAULT) free(*(char **) opt->var); *(char **) opt->var = pgut_strdup(optarg); - return; + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; case 't': if (parse_time(optarg, opt->var)) return; @@ -481,6 +545,129 @@ parse_time(const char *value, time_t *time) return true; } +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + static char * longopts_to_optstring(const struct option opts[]) { @@ -828,15 +1015,24 @@ prompt_for_password(const char *username) PGconn * pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username, password); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login, const char *pwd) { PGconn *conn; + if (interrupted && !in_cleanup) elog(ERROR, "interrupted"); /* Start the connection. Loop until we have a password if requested by backend. */ for (;;) { - conn = PQsetdbLogin(host, port, NULL, NULL, dbname, username, password); + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, pwd); if (PQstatus(conn) == CONNECTION_OK) return conn; @@ -896,7 +1092,7 @@ pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (verbose) + if (log_level <= LOG) { int i; @@ -945,7 +1141,7 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (verbose) + if (log_level <= LOG) { int i; @@ -978,6 +1174,24 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int return true; } +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + int pgut_wait(int num, PGconn *connections[], struct timeval *timeout) { @@ -1032,94 +1246,6 @@ pgut_wait(int num, PGconn *connections[], struct timeval *timeout) return -1; } -/* - * elog - log to stderr and exit if ERROR or FATAL - */ -void -elog(int elevel, const char *fmt, ...) -{ - va_list args; - - if (!verbose && elevel <= LOG) - return; - if (quiet && elevel < WARNING) - return; - - switch (elevel) - { - case LOG: - fputs("LOG: ", stderr); - break; - case INFO: - fputs("INFO: ", stderr); - break; - case NOTICE: - fputs("NOTICE: ", stderr); - break; - case WARNING: - fputs("WARNING: ", stderr); - break; - case FATAL: - fputs("FATAL: ", stderr); - break; - case PANIC: - fputs("PANIC: ", stderr); - break; - default: - if (elevel >= ERROR) - fputs("ERROR: ", stderr); - break; - } - - va_start(args, fmt); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); - va_end(args); - - if (elevel > 0) - exit_or_abort(elevel); -} - -void pg_log(eLogType type, const char *fmt, ...) -{ - va_list args; - - if (!verbose && type <= PG_PROGRESS) - return; - if (quiet && type < PG_WARNING) - return; - - switch (type) - { - case PG_DEBUG: - fputs("DEBUG: ", stderr); - break; - case PG_PROGRESS: - fputs("PROGRESS: ", stderr); - break; - case PG_WARNING: - fputs("WARNING: ", stderr); - break; - case PG_FATAL: - fputs("FATAL: ", stderr); - break; - default: - if (type >= PG_FATAL) - fputs("ERROR: ", stderr); - break; - } - - va_start(args, fmt); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); - va_end(args); - - if (type > 0) - exit_or_abort(type); -} - #ifdef WIN32 static CRITICAL_SECTION cancelConnLock; #endif diff --git a/pgut/pgut.h b/src/utils/pgut.h similarity index 82% rename from pgut/pgut.h rename to src/utils/pgut.h index c19efe04a..5ec405894 100644 --- a/pgut/pgut.h +++ b/src/utils/pgut.h @@ -2,7 +2,8 @@ * * pgut.h * - * Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional * *------------------------------------------------------------------------- */ @@ -16,6 +17,8 @@ #include #include +#include "logger.h" + #if !defined(C_H) && !defined(__cplusplus) #ifndef bool typedef char bool; @@ -65,6 +68,22 @@ typedef struct pgut_option typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + /* * pgut client variables and functions */ @@ -83,8 +102,6 @@ extern const char *host; extern const char *port; extern const char *username; extern char *password; -extern bool verbose; -extern bool quiet; extern bool prompt_password; extern bool interrupted; @@ -99,9 +116,13 @@ extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); * Database connections */ extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login, + const char *pwd); extern void pgut_disconnect(PGconn *conn); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params); extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); extern const char *pgut_get_host(void); @@ -126,23 +147,6 @@ extern char *strdup_trim(const char *str); */ extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); -/* - * elog - */ -#define VERBOSE (-5) -#define LOG (-4) -#define INFO (-3) -#define NOTICE (-2) -#define WARNING (-1) -#define ERROR 1 -#define FATAL 2 -#define PANIC 3 - -#undef elog -extern void -elog(int elevel, const char *fmt, ...) -__attribute__((format(printf, 2, 3))); - /* * Assert */ @@ -189,6 +193,8 @@ extern bool parse_uint32(const char *value, uint32 *result); extern bool parse_int64(const char *value, int64 *result); extern bool parse_uint64(const char *value, uint64 *result); extern bool parse_time(const char *value, time_t *time); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); #define IsSpace(c) (isspace((unsigned char)(c))) #define IsAlpha(c) (isalpha((unsigned char)(c))) diff --git a/validate.c b/src/validate.c similarity index 54% rename from validate.c rename to src/validate.c index 0ae862284..0795179a5 100644 --- a/validate.c +++ b/src/validate.c @@ -12,8 +12,12 @@ #include #include +#include static void pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; typedef struct { @@ -36,8 +40,17 @@ pgBackupValidate(pgBackup *backup) validate_files_args *validate_threads_args[num_threads]; int i; + /* We need free() this later */ backup_id_string = base36enc(backup->start_time); + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + { + elog(INFO, "Backup %s has status %s. Skip validation.", + backup_id_string, status2str(backup->status)); + return; + } + elog(LOG, "Validate backup %s", backup_id_string); if (backup->backup_mode != BACKUP_MODE_FULL && @@ -123,8 +136,12 @@ pgBackupValidateFiles(void *arg) */ if (file->write_size == BYTES_INVALID) continue; - /* We don't compute checksums for compressed data, so skip them */ - if (file->generation != -1) + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) continue; /* print progress */ @@ -161,3 +178,127 @@ pgBackupValidateFiles(void *arg) } } } + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + elog(INFO, "Some backups are not valid"); + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + int i, j; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *base_full_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + if (backups == NULL) + elog(ERROR, "Failed to get backup list."); + + /* Valiate each backup along with its xlog files. */ + for (i = 0; i < parray_num(backups); i++) + { + char *backup_id; + + current_backup = (pgBackup *) parray_get(backups, i); + backup_id = base36enc(current_backup->start_time); + + elog(INFO, "Validate backup %s", backup_id); + + free(backup_id); + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + j = i+1; + do + { + base_full_backup = (pgBackup *) parray_get(backups, j); + j++; + } + while (base_full_backup->backup_mode != BACKUP_MODE_FULL + && j < parray_num(backups)); + } + else + base_full_backup = current_backup; + + pgBackupValidate(current_backup); + + /* There is no point in wal validation for corrupted backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, base_full_backup->tli); + } + + if (current_backup->status != BACKUP_STATUS_OK) + corrupted_backup_found = true; + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/__init__.py b/tests/__init__.py index 70b792cb8..e6094ad03 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,11 +5,17 @@ retention_test, ptrack_clean, ptrack_cluster, \ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ - ptrack_vacuum_full, ptrack_vacuum_truncate + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + pgpro688, false_positive, replica def load_tests(loader, tests, pattern): suite = unittest.TestSuite() + suite.addTests(loader.loadTestsFromModule(replica)) +# suite.addTests(loader.loadTestsFromModule(pgpro560)) +# suite.addTests(loader.loadTestsFromModule(pgpro589)) +# suite.addTests(loader.loadTestsFromModule(pgpro688)) +# suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(show_test)) @@ -29,3 +35,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) return suite + + +# ExpectedFailures are bugs, which should be fixed diff --git a/tests/backup_test.py b/tests/backup_test.py index 62c5ceffa..b68be8d29 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1,7 +1,8 @@ import unittest -from os import path, listdir +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from time import sleep +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all @@ -9,159 +10,210 @@ class BackupTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(BackupTest, self).__init__(*args, **kwargs) + self.module_name = 'backup' -# @classmethod -# def tearDownClass(cls): -# stop_all() -# @unittest.skip("123") + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # full backup mode - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + #with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: + # backup_log.write(self.backup_node(node, options=["--verbose"])) + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] - show_backup = self.show_pb(node)[0] - full_backup_id = show_backup['ID'] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("FULL")) # postmaster.pid and postmaster.opts shouldn't be copied excluded = True - backups_dir = path.join(self.backup_dir(node), "backups") - for backup in listdir(backups_dir): - db_dir = path.join(backups_dir, backup, "database") - for f in listdir(db_dir): - if path.isfile(path.join(db_dir, f)) and \ - (f == "postmaster.pid" or f == "postmaster.opts"): + db_dir = os.path.join(backup_dir, "backups", 'node', backup_id, "database") + for f in os.listdir(db_dir): + if os.path.isfile(os.path.join(db_dir, f)) \ + and (f == "postmaster.pid" or f == "postmaster.opts"): excluded = False self.assertEqual(excluded, True) # page backup mode - with open(path.join(node.logs_dir, "backup_page.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) + page_backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") # print self.show_pb(node) - show_backup = self.show_pb(node)[1] + show_backup = self.show_pb(backup_dir, 'node')[1] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("PAGE")) # Check parent backup self.assertEqual( - full_backup_id, - self.show_pb(node, id=show_backup['ID'])["parent-backup-id"]) + backup_id, + self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) # ptrack backup mode - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - show_backup = self.show_pb(node)[2] + show_backup = self.show_pb(backup_dir, 'node')[2] self.assertEqual(show_backup['Status'], six.b("OK")) self.assertEqual(show_backup['Mode'], six.b("PTRACK")) + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) + node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node' ,node, options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + node.stop() + + #@unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page-level backup without validated full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - with open(path.join(node.logs_dir, "backup.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose", "-C"])) + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) node.stop() -# @unittest.skip("123") - def test_page_backup_without_full(self): - """page-level backup without validated full backup""" + @unittest.expectedFailure + # Need to forcibly validate parent + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) try: - self.backup_pb(node, backup_type="page", options=["--verbose"]) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - pass - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("ERROR")) + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "-j", "4"])) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "-j", "4"])) - - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/backup/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) -# node.append_conf("pg_hba.conf", "local replication all trust") -# node.append_conf("pg_hba.conf", "host replication all 127.0.0.1/32 trust") + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - - with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log: - backup_log.write(self.backup_pb( - node, - backup_type="full", - options=["--verbose", "-j", "4", "--stream"] - )) - - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) - - with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log: - backup_log.write(self.backup_pb( - node, - backup_type="ptrack", - options=["--verbose", "-j", "4", "--stream"] - )) - self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) node.stop() diff --git a/tests/class_check.py b/tests/class_check.py deleted file mode 100644 index ad6c44af4..000000000 --- a/tests/class_check.py +++ /dev/null @@ -1,24 +0,0 @@ -class Base(object): - def __init__(self): - self.a = 10 - def func(self, arg1, arg2): - print 'Child {0}, a = {1}'.format(arg1, arg2) - - -class ChildA(Base): - def __init__(self): - Base.__init__(self) - b = 5 - c = b + self.a - print 'Child A, a = {0}'.format(c) - - -class ChildB(Base): - def __init__(self): - super(ChildB, self).__init__() - b = 6 - c = b + self.a - self.func('B', c) - -#ChildA() -ChildB() diff --git a/tests/class_check1.py b/tests/class_check1.py deleted file mode 100644 index 06759d7ba..000000000 --- a/tests/class_check1.py +++ /dev/null @@ -1,15 +0,0 @@ -class Foo(object): - def __init__(self, *value1, **value2): -# do something with the values - print 'I think something is being called here' -# print value1, value2 - - -class MyFoo(Foo): - def __init__(self, *args, **kwargs): -# do something else, don't care about the args - print args, kwargs - super(MyFoo, self).__init__(*args, **kwargs) - - -foo = MyFoo('Python', 2.7, stack='overflow', ololo='lalala') \ No newline at end of file diff --git a/tests/class_check2.py b/tests/class_check2.py deleted file mode 100644 index bf6d0a9f1..000000000 --- a/tests/class_check2.py +++ /dev/null @@ -1,23 +0,0 @@ -class Base(object): - def __init__(self): - self.a = 10 - self.b = 1 -# def func(self, arg1, arg2): -# print 'Child {0}, a = {1}'.format(arg1, arg2) - - -class ChildA(Base): - def __init__(self): - Base.__init__(self) - self.b = self.b + 1 - - -class ChildB(ChildA): - def __init__(self): - ChildA.__init__(self) - print 'b = {0}'.format(self.b) -# c = b + self.a - - -#ChildA() -ChildB() diff --git a/tests/delete_test.py b/tests/delete_test.py index 41ea70da7..e9c176f0f 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -1,7 +1,7 @@ import unittest -from os import path +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all import subprocess @@ -10,88 +10,120 @@ class DeleteTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(DeleteTest, self).__init__(*args, **kwargs) + self.module_name = 'delete' -# @classmethod -# def tearDownClass(cls): -# stop_all() -# @unittest.skip("123") + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure def test_delete_full_backups(self): """delete full backups""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init() - # full backup mode - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + # full backup + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) - show_backups = self.show_pb(node) + show_backups = self.show_pb(backup_dir, 'node') id_1 = show_backups[0]['ID'] + id_2 = show_backups[1]['ID'] id_3 = show_backups[2]['ID'] - self.delete_pb(node, show_backups[1]['ID']) - show_backups = self.show_pb(node) + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(show_backups[0]['ID'], id_1) self.assertEqual(show_backups[1]['ID'], id_3) node.stop() -# @unittest.skip("123") - def test_delete_increment(self): + def test_delete_increment_page(self): """delete increment and all after him""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/delete/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # full backup mode - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node) # page backup mode - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node, backup_type="page") # page backup mode - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) - + self.backup_node(backup_dir, 'node', node, backup_type="page") # full backup mode - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) + self.assertEqual(show_backups[0]['Status'], six.b("OK")) + self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) + self.assertEqual(show_backups[1]['Status'], six.b("OK")) + + node.stop() + + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() - show_backups = self.show_pb(node) + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(node, show_backups[1]['ID']) + self.delete_pb(backup_dir, 'node', show_backups[1]['ID']) - show_backups = self.show_pb(node) + show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 53b46eb4a..838dd4b27 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -5,36 +5,58 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup version - pg_probackup init -B backup-path -D pgdata-dir - - pg_probackup set-config -B backup-dir - [-d dbname] [-h host] [-p port] [-U username] - [--retention-redundancy=retention-redundancy]] + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level=log-level] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] - pg_probackup show-config -B backup-dir + pg_probackup show-config -B backup-dir --instance=instance_name - pg_probackup backup -B backup-path -b backup-mode - [-D pgdata-dir] [-C] [--stream [-S slot-name]] [--backup-pg-log] + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] [-j num-threads] [--archive-timeout=archive-timeout] - [--progress] [-q] [-v] [--delete-expired] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--progress] [--delete-expired] [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] - pg_probackup restore -B backup-dir - [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] - [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] - pg_probackup validate -B backup-dir - [-D pgdata-dir] [-i backup-id] [--progress] [-q] [-v] - [--time=time|--xid=xid [--inclusive=boolean]] - [--timeline=timeline] + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid [--inclusive=boolean]] + [--timeline=timeline] pg_probackup show -B backup-dir - [-i backup-id] + [--instance=instance_name [-i backup-id]] + + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name - pg_probackup delete -B backup-dir - [--wal] [-i backup-id | --expired] + pg_probackup del-instance -B backup-dir + --instance=instance_name Read the website for details. Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index adc3ad0d7..e88cb7c97 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.11 +pg_probackup 1.1.17 diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 000000000..71e2899ff --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,155 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(FalsePositive, self).__init__(*args, **kwargs) + self.module_name = 'false_positive' + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, restore it to node1, + check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/master".format(fname), + set_archiving=True, + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + master.start() + + self.assertEqual(self.init_pb(master), six.b("")) + id = self.backup_pb(master, backup_type='full', options=["--stream"]) + + node1 = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/node1".format(fname)) + node1.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + self.backup_pb(master, backup_type='page', options=["--stream"]) + self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir) + node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port)) + node1.start({"-t": "600"}) + + timeline_master = master.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual(timeline_master, timeline_node1, "Timelines on Master and Node1 should be equal. This is unexpected") + + archive_command_master = master.safe_psql("postgres", "show archive_command") + archive_command_node1 = node1.safe_psql("postgres", "show archive_command") + self.assertEqual(archive_command_master, archive_command_node1, "Archive command on Master and Node should be equal. This is unexpected") + + res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + # self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + if res == six.b(""): + self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + master.stop() + node1.stop() + + def pgpro688(self): + """ + make node with archiving, make backup, + get Recovery Time, validate to Recovery Time + Waiting PGPRO-688 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full') + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + # Uncommenting this section will make this test True Positive + #node.psql("postgres", "select pg_create_restore_point('123')") + #node.psql("postgres", "select txid_current()") + #node.psql("postgres", "select pg_switch_xlog()") + #### + + try: + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)]) + self.assertEqual(1, 0, 'Error is expected because We should not be able safely validate "Recovery Time" without wal record with timestamp') + except ProbackupException, e: + self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message) + + node.stop() + + def pgpro702_688(self): + """ + make node without archiving, make stream backup, + get Recovery Time, validate to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + id = self.backup_pb(node, backup_type='full', options=["--stream"]) + recovery_time = self.show_pb(node, id=id)['recovery-time'] + + self.assertIn(six.b("INFO: backup validation completed successfully on"), + self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) + + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + set_archiving=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + node.start() + self.assertEqual(self.init_pb(node), six.b("")) + self.backup_pb(node, backup_type='full') + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) + + + ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it + ##### We need archive-push ASAP + self.backup_pb(node, backup_type='full') + self.assertTrue('validation completed successfully' in self.validate_pb(node)) + ######## + node.stop() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 000000000..769446ad6 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py similarity index 81% rename from tests/ptrack_helpers.py rename to tests/helpers/ptrack_helpers.py index 40c77c1ad..b7a7cf3b0 100644 --- a/tests/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -7,6 +7,7 @@ from testgres import get_new_node import hashlib import re +import pwd idx_ptrack = { @@ -55,8 +56,6 @@ # You can lookup error message and cmdline in exception object attributes class ProbackupException(Exception): def __init__(self, message, cmd): -# print message -# self.message = repr(message).strip("'") self.message = message self.cmd = cmd #need that to make second raise @@ -136,25 +135,34 @@ def __init__(self, *args, **kwargs): self.test_env["LC_MESSAGES"] = "C" self.test_env["LC_TIME"] = "C" - self.dir_path = os.path.dirname(os.path.realpath(__file__)) + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir)) + self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs')) try: - os.makedirs(os.path.join(self.dir_path, "tmp_dirs")) + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) except: pass self.probackup_path = os.path.abspath(os.path.join( - self.dir_path, - "../pg_probackup" - )) + self.dir_path, "../pg_probackup")) + self.user = self.get_username() def arcwal_dir(self, node): return "%s/backup/wal" % node.base_dir - def backup_dir(self, node): - return os.path.abspath("%s/backup" % node.base_dir) + def backup_dir(self, node=None, path=None): + if node: + return os.path.abspath("{0}/backup".format(node.base_dir)) + if path: + return - def make_simple_node(self, base_dir=None, set_replication=False, - set_archiving=False, initdb_params=[], pg_options={}): - real_base_dir = os.path.join(self.dir_path, base_dir) + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) node = get_new_node('test', base_dir=real_base_dir) @@ -172,9 +180,6 @@ def make_simple_node(self, base_dir=None, set_replication=False, # Allow replication in pg_hba.conf if set_replication: node.set_replication_conf() - # Setup archiving for node - if set_archiving: - self.set_archiving_conf(node, self.arcwal_dir(node)) return node def create_tblspace_in_node(self, node, tblspc_name, cfs=False): @@ -299,7 +304,8 @@ def check_ptrack_clean(self, idx_dict, size): def run_pb(self, command, async=False): try: - #print ' '.join(map(str,[self.probackup_path] + command)) + self.cmd = [' '.join(map(str,[self.probackup_path] + command))] + print self.cmd if async is True: return subprocess.Popen( [self.probackup_path] + command, @@ -308,80 +314,93 @@ def run_pb(self, command, async=False): env=self.test_env ) else: - output = subprocess.check_output( + self.output = subprocess.check_output( [self.probackup_path] + command, stderr=subprocess.STDOUT, env=self.test_env ) if command[0] == 'backup': - if '-q' in command or '--quiet' in command: - return None - elif '-v' in command or '--verbose' in command: - return output - else: - # return backup ID - for line in output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - return line.split()[2] + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] else: - return output + return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output, e.cmd) + raise ProbackupException(e.output, self.cmd) - def init_pb(self, node): + def init_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) return self.run_pb([ "init", - "-B", self.backup_dir(node), + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, "-D", node.data_dir ]) - def clean_pb(self, node): - shutil.rmtree(self.backup_dir(node), ignore_errors=True) + def del_instance(self, backup_dir, instance, node): - def backup_pb(self, node=None, data_dir=None, backup_dir=None, backup_type="full", options=[], async=False): - if data_dir is None: - data_dir = node.data_dir - if backup_dir is None: - backup_dir = self.backup_dir(node) + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False): cmd_list = [ "backup", "-B", backup_dir, - "-D", data_dir, + "-D", node.data_dir, "-p", "%i" % node.port, - "-d", "postgres" + "-d", "postgres", + "--instance={0}".format(instance) ] if backup_type: cmd_list += ["-b", backup_type] return self.run_pb(cmd_list + options, async) - def restore_pb(self, node=None, backup_dir=None, data_dir=None, id=None, options=[]): + def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]): if data_dir is None: data_dir = node.data_dir - if backup_dir is None: - backup_dir = self.backup_dir(node) cmd_list = [ "restore", "-B", backup_dir, - "-D", data_dir + "-D", data_dir, + "--instance={0}".format(instance) ] - if id: - cmd_list += ["-i", id] + if backup_id: + cmd_list += ["-i", backup_id] return self.run_pb(cmd_list + options) - def show_pb(self, node, id=None, options=[], as_text=False): + def show_pb(self, backup_dir, instance=None, backup_id=None, options=[], as_text=False): + backup_list = [] specific_record = {} cmd_list = [ "show", - "-B", self.backup_dir(node), + "-B", backup_dir, ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] if as_text: # You should print it when calling as_text=true @@ -389,7 +408,7 @@ def show_pb(self, node, id=None, options=[], as_text=False): # get show result as list of lines show_splitted = self.run_pb(cmd_list + options).splitlines() - if id is None: + if instance is not None and backup_id is None: # cut header(ID, Mode, etc) from show as single string header = show_splitted[1:2][0] # cut backup records from show as single list with string for every backup record @@ -431,40 +450,46 @@ def show_pb(self, node, id=None, options=[], as_text=False): specific_record[name.strip()] = var return specific_record - def validate_pb(self, node, id=None, options=[]): + def validate_pb(self, backup_dir, instance=None, backup_id=None, options=[]): + cmd_list = [ "validate", - "-B", self.backup_dir(node), + "-B", backup_dir ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] - # print(cmd_list) return self.run_pb(cmd_list + options) - def delete_pb(self, node, id=None, options=[]): + def delete_pb(self, backup_dir, instance=None, backup_id=None, options=[]): cmd_list = [ "delete", - "-B", self.backup_dir(node), + "-B", backup_dir ] - if id: - cmd_list += ["-i", id] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] # print(cmd_list) return self.run_pb(cmd_list + options) - def delete_expired(self, node, options=[]): + def delete_expired(self, backup_dir, instance, options=[]): cmd_list = [ "delete", "--expired", - "-B", self.backup_dir(node), + "-B", backup_dir, + "--instance={0}".format(instance) ] return self.run_pb(cmd_list + options) - def show_config(self, node): + def show_config(self, backup_dir, instance): out_dict = {} cmd_list = [ "show-config", - "-B", self.backup_dir(node), + "-B", backup_dir, + "--instance={0}".format(instance) ] res = self.run_pb(cmd_list).splitlines() for line in res: @@ -485,9 +510,7 @@ def get_recovery_conf(self, node): out_dict[key.strip()] = value.strip(" '").replace("'\n", "") return out_dict - def set_archiving_conf(self, node, archive_dir=False, replica=False): - if not archive_dir: - archive_dir = self.arcwal_dir(node) + def set_archiving(self, backup_dir, instance, node, replica=False): if replica: archive_mode = 'always' @@ -506,8 +529,8 @@ def set_archiving_conf(self, node, archive_dir=False, replica=False): if os.name == 'posix': node.append_conf( "postgresql.auto.conf", - "archive_command = 'test ! -f {0}/%f && cp %p {0}/%f'".format(archive_dir) - ) + "archive_command = '{0} archive-push -B {1} --instance={2} --wal-file-path %p --wal-file-name %f'".format( + self.probackup_path, backup_dir, instance)) #elif os.name == 'nt': # node.append_conf( # "postgresql.auto.conf", @@ -536,3 +559,7 @@ def get_pgpro_edition(self, node): return str(var[0][0]) else: return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] diff --git a/tests/init_test.py b/tests/init_test.py index 173c8ffd3..b19f069dc 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -3,56 +3,70 @@ import os from os import path import six -from .ptrack_helpers import dir_files, ProbackupTest, ProbackupException +from helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException -#TODO class InitTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(InitTest, self).__init__(*args, **kwargs) + self.module_name = 'init' - def test_success_1(self): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): """Success normal init""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) - self.assertEqual(self.init_pb(node), six.b("")) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) + self.init_pb(backup_dir) self.assertEqual( - dir_files(self.backup_dir(node)), - ['backups', 'pg_probackup.conf', 'wal'] + dir_files(backup_dir), + ['backups', 'wal'] ) + self.add_instance(backup_dir, 'node', node) - def test_already_exist_2(self): + self.assertEqual("INFO: Instance 'node' successfully deleted\n", + self.del_instance(backup_dir, 'node', node), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + def test_already_exist(self): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) - self.init_pb(node) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) + self.init_pb(backup_dir) try: - self.init_pb(node) - # we should die here because exception is what we expect to happen - exit(1) + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: backup catalog already exist and it's not empty\n" - ) + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - def test_abs_path_3(self): + def test_abs_path(self): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/init/{0}".format(fname)) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) try: self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: -B, --backup-path must be an absolute path\n" - ) + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) if __name__ == '__main__': diff --git a/tests/option_test.py b/tests/option_test.py index d16d4fe0d..1114c1696 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -1,7 +1,7 @@ import unittest -from os import path +import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all @@ -9,178 +9,206 @@ class OptionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) + self.module_name = 'option' @classmethod def tearDownClass(cls): stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_help_1(self): """help options""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - with open(path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( self.run_pb(["--help"]), help_out.read() ) + # @unittest.skip("skip") def test_version_2(self): """help options""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - with open(path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertEqual( self.run_pb(["--version"]), version_out.read() ) + # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') try: self.run_pb(["backup", "-b", "full"]) - # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n' - ) + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") def test_options_4(self): """check options test""" fname = self.id().split(".")[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/option/{0}".format(fname)) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) + try: node.stop() except: pass - self.assertEqual(self.init_pb(node), six.b("")) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # backup command failure without backup mode option try: - self.run_pb(["backup", "-B", self.backup_dir(node), "-D", node.data_dir]) - # we should die here because exception is what we expect to happen - exit(1) + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: -# print e.message - self.assertEqual( - e.message, - 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n' - ) + self.assertEqual(e.message, + 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # backup command failure with invalid backup mode option try: - self.run_pb(["backup", "-b", "bad", "-B", self.backup_dir(node)]) - # we should die here because exception is what we expect to happen - exit(1) + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid backup-mode "bad"\n' - ) + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # delete failure without ID try: - self.run_pb(["delete", "-B", self.backup_dir(node)]) + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: required backup ID not specified\n' - ) + self.assertEqual(e.message, + 'ERROR: required backup ID not specified\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + #@unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) + + self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir))) + self.add_instance(backup_dir, 'node', node) node.start() # syntax error in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write(" = INFINITE\n") - try: - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: syntax error in " = INFINITE"\n' - ) + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # invalid value in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("BACKUP_MODE=\n") try: - self.backup_pb(node, backup_type=None), + self.backup_node(backup_dir, 'node', node, backup_type=None), # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid backup-mode ""\n' - ) + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # Command line parameters should override file values - self.assertEqual(self.init_pb(node), six.b("")) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy=1\n") self.assertEqual( - self.show_config(node)['retention-redundancy'], + self.show_config(backup_dir, 'node')['retention-redundancy'], six.b('1') ) # User cannot send --system-identifier parameter via command line try: - self.backup_pb(node, options=["--system-identifier", "123"]), + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: option system-identifier cannot be specified in command line\n' - ) + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # invalid value in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("SMOOTH_CHECKPOINT=FOO\n") try: - self.backup_pb(node), + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n" - ) + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) # invalid option in pg_probackup.conf - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("TIMELINEID=1\n") try: - self.backup_pb(node), + self.backup_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: invalid option "TIMELINEID"\n' - ) - - self.clean_pb(node) - self.assertEqual(self.init_pb(node), six.b("")) + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - node.stop() +# self.clean_pb(backup_dir) +# node.stop() diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 000000000..b0117f702 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,78 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(CheckSystemID, self).__init__(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.start() + + self.assertEqual(self.init_pb(node), six.b("")) + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_pb(node, backup_type='full', options=['--stream']) + assertEqual(1, 0, 'Error is expected because of control file loss') + except ProbackupException, e: + self.assertTrue( + 'ERROR: could not open file' and 'pg_control' in e.message, + 'Expected error is about control file loss') + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node1".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node2".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + self.assertEqual(self.init_pb(node1), six.b("")) + + try: + self.backup_pb(node1, data_dir=node2.data_dir, backup_type='full', options=['--stream']) + assertEqual(1, 0, 'Error is expected because of SYSTEM ID mismatch') + except ProbackupException, e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' and + 'but target backup directory system id is' in e.message, + 'Expected error is about SYSTEM ID mismatch') diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 000000000..00988d057 --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,97 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ArchiveCheck, self).__init__(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_mode(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check ERROR text + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() + + self.assertEqual(self.init_pb(node), six.b("")) + + try: + self.backup_pb(node, backup_type='full', options=['--archive-timeout=10']) + assertEqual(1, 0, 'Error is expected because of disabled archive_mode') + except ProbackupException, e: + self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n') + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node.append_conf("postgresql.auto.conf", "archive_mode = on") + node.append_conf("postgresql.auto.conf", "wal_level = archive") + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() + self.assertEqual(self.init_pb(node), six.b("")) + + try: + self.backup_pb( + node, backup_type='full', options=['--archive-timeout=10']) + assertEqual(1, 0, 'Error is expected because of missing archive wal segment with start_backup() LSN') + except ProbackupException, e: + self.assertTrue('INFO: wait for LSN' in e.message, "Expecting 'INFO: wait for LSN'") + self.assertTrue('ERROR: switched WAL segment' and 'could not be archived' in e.message, + "Expecting 'ERROR: switched WAL segment could not be archived'") + + id = self.show_pb(node)[0]['ID'] + self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status') + #print self.backup_dir(node) + file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path) + self.assertFalse(os.path.isfile(file), + '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) diff --git a/tests/pgpro688.py b/tests/pgpro688.py new file mode 100644 index 000000000..416d2cf5e --- /dev/null +++ b/tests/pgpro688.py @@ -0,0 +1,201 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import stop_all, get_username +import subprocess +from sys import exit, _getframe +import shutil +import time + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ReplicaTest, self).__init__(*args, **kwargs) + self.module_name = 'replica' + self.instance_master = 'master' + self.instance_replica = 'replica' + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_full_backup(self): + """make full stream backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, self.instance_master, master) + master.start() + + # Make empty Object 'replica' from new node + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) + replica_port = replica.port + replica.cleanup() + + # FULL STREAM backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # FULL STREAM backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) + + # Restore last backup from master to Replica directory + self.restore_node(backup_dir, self.instance_master, replica.data_dir) + # Set Replica + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) + replica.start({"-t": "600"}) + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Add instance replica + self.add_instance(backup_dir, self.instance_replica, replica) + + # FULL STREAM backup of replica + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ + '--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + + # Validate instance replica + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + def test_replica_archive_full_backup(self): + """make page archive backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} + ) + self.set_archiving(backup_dir, self.instance_master, master) + self.init_pb(backup_dir) + self.add_instance(backup_dir, self.instance_master, master) + master.start() + + # Make empty Object 'replica' from new node + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) + replica_port = replica.port + replica.cleanup() + + # FULL ARCHIVE backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='full') + # Create table t_heap + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # PAGE ARCHIVE backup of master + self.backup_node(backup_dir, self.instance_master, master, backup_type='page') + + # Restore last backup from master to Replica directory + self.restore_node(backup_dir, self.instance_master, replica.data_dir) + + # Set Replica + self.set_archiving(backup_dir, self.instance_replica, replica, replica=True) + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) + replica.start({"-t": "600"}) + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make FULL ARCHIVE backup from replica + self.add_instance(backup_dir, self.instance_replica, replica) + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ + '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + # Drop Table t_heap + after = master.execute("postgres", "drop table t_heap") + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i") + before = master.execute("postgres", "SELECT * FROM t_heap") + + # Make page backup from replica + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[ + '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, self.instance_replica) + self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) + + @unittest.skip("skip") + def test_replica_archive_full_backup_123(self): + """ + make full archive backup from replica + """ + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.start() + + replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname)) + replica_port = replica.port + replica.cleanup() + + self.assertEqual(self.init_pb(master), six.b("")) + self.backup_pb(node=master, backup_type='full', options=['--stream']) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + id = self.backup_pb(master, backup_type='page', options=['--stream']) + self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir) + + # Settings for Replica + replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + # Set Archiving for replica + self.set_archiving_conf(replica, replica=True) + + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', + "primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port)) + replica.start({"-t": "600"}) + # Replica Started + + # master.execute("postgres", "checkpoint") + + # Check replica + after = replica.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.assertEqual(self.init_pb(replica), six.b("")) + self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30']) + self.validate_pb(replica) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 597ea7e73..0880c0315 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -1,12 +1,14 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_clean' def teardown(self): stop_all() @@ -15,13 +17,16 @@ def teardown(self): # @unittest.expectedFailure def test_ptrack_clean(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - set_archiving=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -35,8 +40,7 @@ def test_ptrack_clean(self): i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) # Make full backup to clean every ptrack - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) for i in idx_ptrack: # get fork size and calculate it in pages @@ -52,7 +56,7 @@ def test_ptrack_clean(self): node.psql('postgres', 'update t_heap set text = md5(text), tsvector = md5(repeat(tsvector::text, 10))::tsvector;') node.psql('postgres', 'vacuum t_heap') - id = self.backup_pb(node, backup_type='ptrack', options=['-j100', '--stream']) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['-j100', '--stream']) node.psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -71,7 +75,7 @@ def test_ptrack_clean(self): node.psql('postgres', 'vacuum t_heap') # Make page backup to clean every ptrack - self.backup_pb(node, backup_type='page', options=['-j100']) + self.backup_node(backup_dir, 'node', node, backup_type='page', options=['-j100']) node.psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -85,8 +89,7 @@ def test_ptrack_clean(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - print self.show_pb(node, as_text=True) - self.clean_pb(node) + print self.show_pb(backup_dir, 'node', as_text=True) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index e4525bfdf..ff0fb6a21 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -1,26 +1,32 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_cluster' def teardown(self): # clean_all() stop_all() - # @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_cluster_btree(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -45,8 +51,7 @@ def test_ptrack_cluster_btree(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_btree') @@ -67,18 +72,19 @@ def test_ptrack_cluster_btree(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -103,8 +109,7 @@ def test_ptrack_cluster_spgist(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_spgist') @@ -125,18 +130,19 @@ def test_ptrack_cluster_spgist(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -161,8 +167,7 @@ def test_ptrack_cluster_brin(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_brin') @@ -183,18 +188,19 @@ def test_ptrack_cluster_brin(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -219,8 +225,7 @@ def test_ptrack_cluster_gist(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_gist') @@ -241,18 +246,19 @@ def test_ptrack_cluster_gist(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() - # @unittest.skip("123") def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -277,8 +283,7 @@ def test_ptrack_cluster_gin(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'cluster t_heap using t_gin') @@ -299,7 +304,6 @@ def test_ptrack_cluster_gin(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index d43282f1e..e35234a65 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -3,26 +3,32 @@ from testgres import get_new_node, stop_all import os from signal import SIGTERM -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack from time import sleep class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_move_to_tablespace' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_recovery(self): - fname = self.id().split(".")[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -52,7 +58,6 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 73e9e085a..24802697c 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -3,26 +3,32 @@ from testgres import get_new_node, stop_all import os from signal import SIGTERM -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack from time import sleep class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_move_to_tablespace' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_recovery(self): - fname = self.id().split(".")[3] - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/{0}".format(fname), + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table @@ -54,7 +60,6 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 484c5c508..f6b22b97e 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -1,12 +1,14 @@ import unittest +import os from sys import exit from testgres import get_new_node, stop_all -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum' def teardown(self): # clean_all() @@ -16,12 +18,15 @@ def teardown(self): # @unittest.expectedFailure def test_ptrack_vacuum(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -47,8 +52,7 @@ def test_ptrack_vacuum(self): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make full backup to clean every ptrack - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) @@ -74,7 +78,6 @@ def test_ptrack_vacuum(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 75d25909f..1a4d3fe54 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_bits_frozen' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_frozen", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -43,8 +50,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'vacuum freeze t_heap') node.psql('postgres', 'checkpoint') @@ -63,8 +69,6 @@ def test_ptrack_vacuum_bits_frozen(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index 4fc12419c..ca5db7050 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_bits_visibility' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_bits_visibility", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -43,8 +50,7 @@ def test_ptrack_vacuum_bits_visibility(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'vacuum t_heap') node.psql('postgres', 'checkpoint') @@ -63,8 +69,6 @@ def test_ptrack_vacuum_bits_visibility(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index 98af70be2..9d9d5051f 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -1,38 +1,33 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all #import os -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack - -# res = node.execute('postgres', 'show fsync') -# print res[0][0] -# res = node.execute('postgres', 'show wal_level') -# print res[0][0] -# a = ProbackupTest -# res = node.execute('postgres', 'select 1')` -# self.assertEqual(len(res), 1) -# self.assertEqual(res[0][0], 1) -# node.stop() -# a = self.backup_dir(node) +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_full' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir='tmp_dirs/ptrack/{0}'.format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -57,8 +52,7 @@ def test_ptrack_vacuum_full(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id%2 = 1') node.psql('postgres', 'vacuum full t_heap') @@ -78,8 +72,6 @@ def test_ptrack_vacuum_full(self): # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index eba15da4e..37dd99202 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -1,25 +1,32 @@ +import os import unittest from sys import exit from testgres import get_new_node, stop_all -from os import path, open, lseek, read, close, O_RDONLY -from .ptrack_helpers import ProbackupTest, idx_ptrack +from helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) + self.module_name = 'ptrack_vacuum_truncate' def teardown(self): # clean_all() stop_all() + # @unittest.skip("skip") + # @unittest.expectedFailure def test_ptrack_vacuum_truncate(self): - node = self.make_simple_node(base_dir="tmp_dirs/ptrack/test_ptrack_vacuum_truncate", + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, - initdb_params=['--data-checksums', '-A trust'], + initdb_params=['--data-checksums'], pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -44,8 +51,7 @@ def test_ptrack_vacuum_truncate(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.init_pb(node) - self.backup_pb(node, backup_type='full', options=['-j100', '--stream']) + self.backup_node(backup_dir, 'node', node, options=['-j100', '--stream']) node.psql('postgres', 'delete from t_heap where id > 128;') node.psql('postgres', 'vacuum t_heap') @@ -65,8 +71,6 @@ def test_ptrack_vacuum_truncate(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - - self.clean_pb(node) node.stop() if __name__ == '__main__': diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 000000000..9f3e90f82 --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,129 @@ +import unittest +import os +import six +from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +from testgres import stop_all +import subprocess +from sys import exit, _getframe +import shutil +import time + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(ReplicaTest, self).__init__(*args, **kwargs) + self.module_name = 'replica' + +# @classmethod +# def tearDownClass(cls): +# stop_all() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_full_backup(self): + """make full stream backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname)) + slave.cleanup() + + # FULL BACKUP + self.backup_node(backup_dir, 'master', master, backup_type='full', options=['--stream']) + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + #FULL BACKUP + self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.restore_node(backup_dir, 'master', slave) + + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + + slave.append_conf('recovery.conf', "standby_mode = 'on'") + slave.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port)) + slave.start({"-t": "600"}) + # Replica Ready + + # Check replica + after = slave.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.add_instance(backup_dir, 'slave', slave) + #time.sleep(2) + self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in + self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose', + '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])) + self.validate_pb(backup_dir, 'slave') + self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + + # @unittest.skip("skip") + def test_replica_archive_full_backup(self): + """make full archive backup from replica""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.start() + + slave = self.make_simple_node(base_dir="{0}/{1}/slave".format(self.module_name, fname)) + slave.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + + before = master.execute("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node(backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', slave) + + # Settings for Replica + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + # Set Archiving for replica + #self.set_archiving_conf( slave, replica=True) + self.set_archiving(backup_dir, 'slave', slave, replica=True) + + # Set Replica + slave.append_conf('postgresql.auto.conf', 'port = {0}'.format(slave.port)) + slave.append_conf('postgresql.auto.conf', 'hot_standby = on') + slave.append_conf('recovery.conf', "standby_mode = 'on'") + slave.append_conf('recovery.conf', + "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(self.user, master.port)) + slave.start({"-t": "600"}) + + # Check replica + after = slave.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Make backup from replica + self.add_instance(backup_dir, 'slave', slave) + self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300', + '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'slave') diff --git a/tests/restore_test.py b/tests/restore_test.py index ea35a0b17..1ef35c7dc 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,54 +1,57 @@ import unittest import os -from os import path import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from testgres import stop_all import subprocess from datetime import datetime import shutil +from sys import exit class RestoreTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(RestoreTest, self).__init__(*args, **kwargs) + self.module_name = 'restore' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_restore_full_to_latest(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) node.stop({"-m": "immediate"}) node.cleanup() # 1 - Test recovery from latest -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # 2 - Test that recovery.conf was created - recovery_conf = path.join(node.data_dir, "recovery.conf") - self.assertEqual(path.isfile(recovery_conf), True) + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) node.start({"-t": "600"}) @@ -57,37 +60,38 @@ def test_restore_full_to_latest(self): node.stop() + # @unittest.skip("skip") def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -96,31 +100,33 @@ def test_restore_full_page_to_latest(self): node.stop() + # @unittest.skip("skip") def test_restore_to_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")]) node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -128,17 +134,15 @@ def test_restore_to_timeline(self): pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="full") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", "--timeline=%i" % target_tli]) -# ) + # Correct Backup must be choosen for restore + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) @@ -150,36 +154,36 @@ def test_restore_to_timeline(self): node.stop() + # @unittest.skip("skip") def test_restore_to_time(self): - """recovery to target timeline""" + """recovery to target time""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init(scale=2) + node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - node.stop({"-m": "immediate"}) + node.stop() node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", '--time="%s"' % target_time]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -188,23 +192,26 @@ def test_restore_to_time(self): node.stop() + # @unittest.skip("skip") def test_restore_to_xid(self): """recovery to target xid""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -223,16 +230,14 @@ def test_restore_to_xid(self): # Enforce segment to be archived to ensure that recovery goes up to the # wanted point. There is no way to ensure that all segments needed have # been archived up to the xmin point saved earlier without that. - node.execute("postgres", "SELECT pg_switch_xlog()") + #node.execute("postgres", "SELECT pg_switch_xlog()") node.stop({"-m": "fast"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=["-j", "4", "--verbose", '--xid=%s' % target_xid]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -241,45 +246,38 @@ def test_restore_to_xid(self): node.stop() - def test_restore_full_ptrack(self): - """recovery to latest from full + ptrack backups""" + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.append_conf("postgresql.conf", "ptrack_enable = on") - node.restart() + node.pgbench_init(scale=2) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -288,52 +286,44 @@ def test_restore_full_ptrack(self): node.stop() - def test_restore_full_ptrack_ptrack(self): - """recovery to latest from full + ptrack + ptrack backups""" + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.append_conf("postgresql.conf", "ptrack_enable = on") - node.restart() + node.pgbench_init(scale=2) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack") before = node.execute("postgres", "SELECT * FROM pgbench_branches") node.stop({"-m": "immediate"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -342,42 +332,39 @@ def test_restore_full_ptrack_ptrack(self): node.stop() + # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "--stream"])) + self.backup_node(backup_dir, 'node', node, options=["--stream"]) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) before = node.execute("postgres", "SELECT * FROM pgbench_branches") - node.stop({"-m": "immediate"}) + node.stop() node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -386,28 +373,24 @@ def test_restore_full_ptrack_stream(self): node.stop() + # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): """recovery to latest from full + ptrack backups with loads when ptrack backup do""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.restart() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -415,8 +398,7 @@ def test_restore_full_ptrack_under_load(self): options=["-c", "4", "-T", "8"] ) - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) pgbench.wait() pgbench.stdout.close() @@ -428,12 +410,9 @@ def test_restore_full_ptrack_under_load(self): node.stop({"-m": "immediate"}) node.cleanup() - self.wrong_wal_clean(node, wal_segment_size) - -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -444,26 +423,23 @@ def test_restore_full_ptrack_under_load(self): node.stop() + # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): """recovery to latest from full + page backups with loads when full backup do""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - wal_segment_size = self.guc_wal_segment_size(node) - node.pgbench_init(scale=2) - is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'") - if not is_ptrack: - node.stop() - self.skipTest("ptrack not supported") - return - node.restart() + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -471,14 +447,12 @@ def test_restore_full_under_load_ptrack(self): options=["-c", "4", "-T", "8"] ) - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + self.backup_node(backup_dir, 'node', node) pgbench.wait() pgbench.stdout.close() - with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "--stream"])) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") @@ -487,39 +461,39 @@ def test_restore_full_under_load_ptrack(self): node.stop({"-m": "immediate"}) node.cleanup() - self.wrong_wal_clean(node, wal_segment_size) - -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, options=["-j", "4", "--verbose"]) -# ) + #self.wrong_wal_clean(node, wal_segment_size) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - node.stop() + # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): """recovery with target inclusive false""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -538,20 +512,15 @@ def test_restore_to_xid_inclusive(self): # Enforce segment to be archived to ensure that recovery goes up to the # wanted point. There is no way to ensure that all segments needed have # been archived up to the xmin point saved earlier without that. - node.execute("postgres", "SELECT pg_switch_xlog()") + # node.execute("postgres", "SELECT pg_switch_xlog()") node.stop({"-m": "fast"}) node.cleanup() -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete"), - self.restore_pb(node, - options=[ - "-j", "4", - "--verbose", - '--xid=%s' % target_xid, - "--inclusive=false"]) -# ) + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) @@ -561,19 +530,22 @@ def test_restore_to_xid_inclusive(self): node.stop() + # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # Create tablespace - tblspc_path = path.join(node.base_dir, "tblspc") + tblspc_path = os.path.join(node.base_dir, "tblspc") os.makedirs(tblspc_path) with node.connect("postgres") as con: con.connection.autocommit = True @@ -583,88 +555,88 @@ def test_restore_with_tablespace_mapping_1(self): con.execute("INSERT INTO test VALUES (1)") con.commit() - self.backup_pb(node) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) # 1 - Try to restore to existing directory node.stop() try: - self.restore_pb(node) + self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir) - ) + self.assertEqual(e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - shutil.rmtree(node.data_dir) + node.cleanup() try: - self.restore_pb(node) + self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen - exit(1) + self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - e.message, - 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path) - ) + self.assertEqual(e.message, + 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 3 - Restore using tablespace-mapping - tblspc_path_new = path.join(node.base_dir, "tblspc_new") -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) -# ) + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - id = node.execute("postgres", "SELECT id FROM test") - self.assertEqual(id[0][0], 1) + res = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(res[0][0], 1) # 4 - Restore using tablespace-mapping using page backup - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) with node.connect("postgres") as con: con.execute("INSERT INTO test VALUES (2)") con.commit() - self.backup_pb(node, backup_type="page") + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - show_pb = self.show_pb(node) + show_pb = self.show_pb(backup_dir, 'node') self.assertEqual(show_pb[1]['Status'], six.b("OK")) self.assertEqual(show_pb[2]['Status'], six.b("OK")) node.stop() node.cleanup() - tblspc_path_page = path.join(node.base_dir, "tblspc_page") -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]) -# ) + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - id = node.execute("postgres", "SELECT id FROM test OFFSET 1") - self.assertEqual(id[0][0], 2) + res = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(res[0][0], 2) node.stop() + # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/restore/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) # Full backup - self.backup_pb(node) - self.assertEqual(self.show_pb(node)[0]['Status'], six.b("OK")) + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) # Create tablespace - tblspc_path = path.join(node.base_dir, "tblspc") + tblspc_path = os.path.join(node.base_dir, "tblspc") os.makedirs(tblspc_path) with node.connect("postgres") as con: con.connection.autocommit = True @@ -674,9 +646,9 @@ def test_restore_with_tablespace_mapping_2(self): con.commit() # First page backup - self.backup_pb(node, backup_type="page") - self.assertEqual(self.show_pb(node)[1]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(node)[1]['Mode'], six.b("PAGE")) + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE")) # Create tablespace table with node.connect("postgres") as con: @@ -688,28 +660,119 @@ def test_restore_with_tablespace_mapping_2(self): con.commit() # Second page backup - self.backup_pb(node, backup_type="page") - self.assertEqual(self.show_pb(node)[2]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(node)[2]['Mode'], six.b("PAGE")) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE")) node.stop() node.cleanup() - tblspc_path_new = path.join(node.base_dir, "tblspc_new") -# exit(1) -# TODO WAITING FIX FOR RESTORE -# self.assertIn(six.b("INFO: restore complete."), - self.restore_pb(node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) -# ) + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - # Check tables + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() count = node.execute("postgres", "SELECT count(*) FROM tbl") self.assertEqual(count[0][0], 4) - count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """make node with archiving, make stream backup, make PITR to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) + self.assertEqual(True, node.status()) + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_stream_pitr(self): + """make node with archiving, make stream backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) + node.stop() + + # @unittest.skip("skip") + def test_archive_node_backup_archive_pitr_2(self): + """make node with archiving, make archive backup, create table t_heap, make pitr to Recovery Time, check that t_heap do not exists""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.psql("postgres", "create table t_heap(a int)") + node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.restore_node(backup_dir, 'node', node, + options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + res = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in res[2]) node.stop() diff --git a/tests/retention_test.py b/tests/retention_test.py index 265ed8dab..8be47b6f9 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -2,7 +2,7 @@ import os from datetime import datetime, timedelta from os import path, listdir -from .ptrack_helpers import ProbackupTest +from helpers.ptrack_helpers import ProbackupTest from testgres import stop_all @@ -10,39 +10,42 @@ class RetentionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(RetentionTest, self).__init__(*args, **kwargs) + self.module_name = 'retention' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") + # @unittest.skip("skip") + # @unittest.expectedFailure def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.init_pb(node) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") # Make backups to be purged - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") # Make backups to be keeped - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(len(self.show_pb(node)), 4) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - log = self.delete_expired(node) - self.assertEqual(len(self.show_pb(node)), 2) + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Check that WAL segments were deleted min_wal = None @@ -52,7 +55,7 @@ def test_retention_redundancy_1(self): min_wal = line[31:-1] elif line.startswith(b"INFO: removed max WAL segment"): max_wal = line[31:-1] - for wal_name in listdir(path.join(self.backup_dir(node), "wal")): + for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): wal_name_b = wal_name.encode('ascii') self.assertEqual(wal_name_b[8:] > min_wal[8:], True) @@ -64,40 +67,43 @@ def test_retention_redundancy_1(self): def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/retention/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.init_pb(node) - with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: + with open(os.path.join(backup_dir, 'backups', 'node', "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy = 1\n") conf.write("retention-window = 1\n") # Make backups to be purged - self.backup_pb(node) - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") # Make backup to be keeped - self.backup_pb(node) + self.backup_node(backup_dir, 'node', node) - backups = path.join(self.backup_dir(node), "backups") + backups = path.join(backup_dir, 'backups', 'node') days_delta = 5 for backup in listdir(backups): + if backup == 'pg_probackup.conf': + continue with open(path.join(backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 # Make backup to be keeped - self.backup_pb(node, backup_type="page") + self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(len(self.show_pb(node)), 4) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - self.delete_expired(node) - self.assertEqual(len(self.show_pb(node)), 2) + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) node.stop() diff --git a/tests/show_test.py b/tests/show_test.py index d5fdc9fc6..37b146e4a 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -2,7 +2,7 @@ import os from os import path import six -from .ptrack_helpers import ProbackupTest +from helpers.ptrack_helpers import ProbackupTest from testgres import stop_all @@ -10,46 +10,56 @@ class OptionTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) + self.module_name = 'show' @classmethod def tearDownClass(cls): stop_all() - def show_test_1(self): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): """Status DONE and OK""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname), - set_archiving=True, + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) self.assertEqual( - self.backup_pb(node, options=["--quiet"]), + self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]), None ) - self.assertIn(six.b("OK"), self.show_pb(node, as_text=True)) + self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True)) node.stop() + # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" fname = self.id().split('.')[3] - print '{0} started'.format(fname) - node = self.make_simple_node(base_dir="tmp_dirs/show/{0}".format(fname), - set_archiving=True, + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id_backup = self.backup_pb(node) - path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf") - os.remove(path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf")) + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) - self.validate_pb(node, id_backup) - self.assertIn(six.b("CORRUPT"), self.show_pb(node, as_text=True)) + self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True)) node.stop() diff --git a/tests/validate_test.py b/tests/validate_test.py index f3fdf9664..bef73cd77 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,41 +1,47 @@ import unittest import os import six -from .ptrack_helpers import ProbackupTest, ProbackupException +from helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta from testgres import stop_all import subprocess +from sys import exit +import re class ValidateTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ValidateTest, self).__init__(*args, **kwargs) + self.module_name = 'validate' @classmethod def tearDownClass(cls): stop_all() -# @unittest.skip("123") - def test_validate_wal_1(self): - """recovery to latest from full backup""" + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """make node with archiving, make archive backup, validate to both real and unreal values""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - with open(os.path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log: - backup_log.write(self.backup_pb(node, options=["--verbose"])) + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -45,33 +51,33 @@ def test_validate_wal_1(self): pgbench.wait() pgbench.stdout.close() - id_backup = self.show_pb(node)[0]['ID'] - target_time = self.show_pb(node)[0]['Recovery time'] - after_backup_time = datetime.now() + target_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) # Validate to real time - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--time='{0}'".format(target_time)])) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) try: - self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( - after_backup_time - timedelta(days=2))]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n') + self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) try: - self.validate_pb(node, options=["--time='{:%Y-%m-%d %H:%M:%S}'".format( - after_backup_time + timedelta(days=2))]) - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal time") + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'ERROR: not enough WAL records to time' in e.message - ) + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate to real xid target_xid = None @@ -81,61 +87,132 @@ def test_validate_wal_1(self): target_xid = res[0][0] node.execute("postgres", "SELECT pg_switch_xlog()") - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--xid=%s" % target_xid])) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 try: - self.validate_pb(node, options=["--xid=%d" % (int(target_xid) + 1000)]) - self.assertEqual(1, 0, "Error in validation is expected because of validation of unreal xid") + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'ERROR: not enough WAL records to xid' in e.message - ) + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate with backup ID - self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, id_backup)) + self.assertIn(six.b("INFO: backup validation completed successfully"), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, make archive backup, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) - # Validate broken WAL - wals_dir = os.path.join(self.backup_dir(node), "wal") + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(256) - f.write(six.b("blablabla")) + f.seek(42) + f.write(six.b("blablablaadssaaaaaaaaaaaaaaa")) f.close + # Simple validate try: - self.validate_pb(node, id_backup, options=['--xid=%s' % target_xid]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) + self.assertTrue('Possible WAL CORRUPTION' in e.message), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') + node.stop() + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + f = open(os.path.join(wals_dir, wal), "rb+") + f.seek(0) + f.write(six.b("blablabla")) + f.close + + # Validate to xid try: - self.validate_pb(node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of wal segment corruption") + self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertTrue(True, 'Possible WAL CORRUPTION' in e.message) + self.assertTrue('Possible WAL CORRUPTION' in e.message), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') node.stop() -# @unittest.skip("123") + # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): - """Loose segment which belong to some backup""" + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -144,30 +221,56 @@ def test_validate_wal_lost_segment_1(self): ) pgbench.wait() pgbench.stdout.close() - self.backup_pb(node, backup_type='full') + backup_id = self.backup_node(backup_dir, 'node', node) - wals_dir = os.path.join(self.backup_dir(node), "wal") + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - os.remove(os.path.join(self.backup_dir(node), "wal", wals[1])) + file = os.path.join(backup_dir, 'wal', 'node', wals[1]) + os.remove(file) try: - self.validate_pb(node) - self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance") + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException, e: - self.assertTrue('is absent' in e.message) + self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format( + file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"') + + # Be paranoid and run validate again + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException, e: + self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) node.stop() - @unittest.expectedFailure + # @unittest.skip("skip") def test_validate_wal_lost_segment_2(self): - """Loose segment located between backups """ + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/validate/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) + + self.backup_node(backup_dir, 'node', node) + + # make some wals node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -176,39 +279,24 @@ def test_validate_wal_lost_segment_2(self): ) pgbench.wait() pgbench.stdout.close() - self.backup_pb(node, backup_type='full') - # need to do that to find segment between(!) backups - node.psql("postgres", "CREATE TABLE t1(a int)") - node.psql("postgres", "SELECT pg_switch_xlog()") - node.psql("postgres", "CREATE TABLE t2(a int)") - node.psql("postgres", "SELECT pg_switch_xlog()") - - wals_dir = os.path.join(self.backup_dir(node), "wal") + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals = map(int, wals) + file = os.path.join(wals_dir, '0000000' + str(max(wals))) + os.remove(file) - # delete last wal segment - os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) - - # Need more accurate error message about loosing wal segment between backups try: - self.backup_pb(node, backup_type='page') - # we should die here because exception is what we expect to happen - exit(1) + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) except ProbackupException, e: - self.assertEqual( - True, - 'could not read WAL record' in e.message - ) - self.delete_pb(node, id=self.show_pb(node)[1]['ID']) - - - ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it - ##### We need archive-push ASAP - self.backup_pb(node, backup_type='full') - self.assertEqual(False, - 'validation completed successfully' in self.validate_pb(node)) - ######## + self.assertTrue('INFO: wait for LSN' + and 'in archived WAL segment' + and 'WARNING: could not read WAL record at' + and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"') node.stop() From 133ec30346ae7fad3c435849fd5c7b6c383e1cc4 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 3 Jul 2017 18:14:35 +0300 Subject: [PATCH 0004/2107] Version 2.0 --- README.md | 184 --------------- src/archive.c | 3 + src/backup.c | 3 + src/data.c | 7 +- src/parsexlog.c | 2 +- src/pg_probackup.c | 9 +- src/pg_probackup.h | 1 + src/restore.c | 1 + src/validate.c | 3 + tests/__init__.py | 14 +- tests/backup_test.py | 90 ++++---- tests/delete_test.py | 33 ++- tests/expected/option_version.out | 2 +- tests/false_positive.py | 179 +++++++++------ tests/helpers/ptrack_helpers.py | 153 ++++++------- tests/init_test.py | 32 +-- tests/option_test.py | 49 ++-- tests/pb_lib.py | 304 ------------------------- tests/pgpro560.py | 60 +++-- tests/pgpro589.py | 76 ++++--- tests/pgpro688.py | 201 ---------------- tests/ptrack_clean.py | 16 +- tests/ptrack_cluster.py | 28 +-- tests/ptrack_move_to_tablespace.py | 18 +- tests/ptrack_recovery.py | 27 +-- tests/ptrack_vacuum.py | 16 +- tests/ptrack_vacuum_bits_frozen.py | 13 +- tests/ptrack_vacuum_bits_visibility.py | 13 +- tests/ptrack_vacuum_full.py | 14 +- tests/ptrack_vacuum_truncate.py | 13 +- tests/replica.py | 23 +- tests/restore_test.py | 195 ++++++++++------ tests/retention_test.py | 34 ++- tests/show_test.py | 25 +- tests/validate_test.py | 77 ++++--- 35 files changed, 645 insertions(+), 1273 deletions(-) delete mode 100644 README.md delete mode 100644 tests/pb_lib.py delete mode 100644 tests/pgpro688.py diff --git a/README.md b/README.md deleted file mode 100644 index 360fc81cc..000000000 --- a/README.md +++ /dev/null @@ -1,184 +0,0 @@ -pg_probackup fork of pg_arman by Postgres Professional -======================================== - -pg_probackup is a backup and recovery manager for PostgreSQL servers able to do -differential and full backup as well as restore a cluster to a -state defined by a given recovery target. It is designed to perform -periodic backups of an existing PostgreSQL server, combined with WAL -archives to provide a way to recover a server in case of failure of -server because of a reason or another. Its differential backup -facility reduces the amount of data necessary to be taken between -two consecutive backups. - -Main features: -* incremental backup from WAL and PTRACK -* backup from replica -* multithreaded backup and restore -* autonomous backup without archive command (will need slot replication) - -Requirements: -* >=PostgreSQL 9.5 -* >=gcc 4.4 or >=clang 3.6 or >= XLC 12.1 -* pthread - -Download --------- - -The latest version of this software can be found on the project website at -https://github.com/postgrespro/pg_probackup. Original fork of pg_probackup can be -found at https://github.com/michaelpq/pg_arman. - -Installation ------------- - -Compiling pg_probackup requires a PostgreSQL installation to be in place -as well as a raw source tree. Pass the path to the PostgreSQL source tree -to make, in the top_srcdir variable: - - make USE_PGXS=1 top_srcdir= - -In addition, you must have pg_config in $PATH. - -The current version of pg_probackup is compatible with PostgreSQL 9.5 and -upper versions. - -Platforms ---------- - -pg_probackup has been tested on Linux and Unix-based platforms. - -Documentation -------------- - -All the documentation you can find [here](doc/pg_probackup.md). - -Regression tests ----------------- - -For tests you must have python 2.7 or python 3.3 and higher. Also good idea -is make virtual enviroment by `virtualenv`. -First of all you need to install `testgres` python module which contains useful -functions to start postgres clusters and make queries: - -``` -pip install testgres -``` - -To run tests execute: - -``` -python -m unittest tests -``` - -from current (root of project) directory. If you want to run a specific postgres build then -you should specify the path to your pg_config executable by setting PG_CONFIG -environment variable: -``` -export PG_CONFIG=/path/to/pg_config -``` - - -Block level incremental backup ------------------------------- - -Idea of block level incremental backup is that you may backup only blocks -changed since last full backup. It gives two major benefits: taking backups -faster and making backups smaller. - -The major question here is how to get the list of changed blocks. Since -each block contains LSN number, changed blocks could be retrieved by full scan -of all the blocks. But this approach consumes as much server IO as full -backup. - -This is why we implemented alternative approaches to retrieve -list of changed blocks. - -1. Scan WAL archive and extract changed blocks from it. However, shortcoming -of these approach is requirement to have WAL archive. - -2. Track bitmap of changes blocks inside PostgreSQL (ptrack). It introduces -some overhead to PostgreSQL performance. On our experiments it appears to be -less than 3%. - -These two approaches were implemented in this fork of pg_probackup. The second -approach requires [patch for PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836) or -[patch for PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21). - -Testing block level incremental backup --------------------------------------- - -You need to apply ptrack patch to [PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836) -or [PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21). -Or you can build and install [PGPRO9_5 or PGPRO9_6 branch of PostgreSQL](https://github.com/postgrespro/postgrespro). -Note that PGPRO branches currently contain old version of ptrack. - -### Retrieving changed blocks from WAL archive - -You need to enable WAL archive by adding following lines to postgresql.conf: - -``` -wal_level = archive -archive_mode = on -archive_command = 'test ! -f /home/postgres/backup/wal/%f && cp %p /home/postgres/backup/wal/%f' -``` - -Example backup (assuming PostgreSQL is running): -```bash -# Init pg_aramn backup folder -pg_probackup init -B /home/postgres/backup -# Make full backup with 2 thread and verbose mode. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b full -v -j 2 -# Show backups information -pg_probackup show -B /home/postgres/backup - -# Now you can insert or update some data in your database - -# Then start the incremental backup. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b page -v -j 2 -# You should see that increment is really small -pg_probackup show -B /home/postgres/backup -``` - -For restore after remove your pgdata you can use: -``` -pg_probackup restore -B /home/postgres/backup -D /home/postgres/pgdata -j 4 --verbose -``` - -### Retrieving changed blocks from ptrack - -The advantage of this approach is that you don't have to save WAL archive. You will need to enable ptrack in postgresql.conf (restart required). - -``` -ptrack_enable = on -``` - -Also, some WALs still need to be fetched in order to get consistent backup. pg_probackup can fetch them trough the streaming replication protocol. Thus, you also need to [enable streaming replication connection](https://wiki.postgresql.org/wiki/Streaming_Replication). - -Example backup (assuming PostgreSQL is running): -```bash -# Init pg_aramn backup folder -pg_probackup init -B /home/postgres/backup -# Make full backup with 2 thread and verbose mode. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b full -v -j 2 --stream -# Show backups information -pg_probackup show -B /home/postgres/backup - -# Now you can insert or update some data in your database - -# Then start the incremental backup. -pg_probackup backup -B /home/postgres/backup -D /home/postgres/pgdata -b ptrack -v -j 2 --stream -# You should see that increment is really small -pg_probackup show -B /home/postgres/backup -``` - -For restore after remove your pgdata you can use: -``` -pg_probackup restore -B /home/postgres/backup -D /home/postgres/pgdata -j 4 --verbose --stream -``` - -License -------- - -pg_probackup can be distributed under the PostgreSQL license. See COPYRIGHT -file for more information. pg_arman is a fork of the existing project -pg_rman, initially created and maintained by NTT and Itagaki Takahiro. diff --git a/src/archive.c b/src/archive.c index 1bb17643a..4b9cd18af 100644 --- a/src/archive.c +++ b/src/archive.c @@ -66,6 +66,9 @@ do_archive_push(char *wal_file_path, char *wal_file_name) join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + if (access(backup_wal_file_path, F_OK) != -1) + elog(ERROR, "file '%s', already exists.", backup_wal_file_path); + copy_wal_file(absolute_wal_file_path, backup_wal_file_path); elog(INFO, "pg_probackup archive-push completed successfully"); diff --git a/src/backup.c b/src/backup.c index e8cb56e1c..368f12e40 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1112,6 +1112,7 @@ pg_stop_backup(pgBackup *backup) backup_label, strerror(errno)); fwrite(PQgetvalue(res, 0, 1), 1, strlen(PQgetvalue(res, 0, 1)), fp); + fsync(fileno(fp)); fclose(fp); /* @@ -1139,6 +1140,7 @@ pg_stop_backup(pgBackup *backup) tablespace_map, strerror(errno)); fwrite(PQgetvalue(res, 0, 2), 1, strlen(PQgetvalue(res, 0, 2)), fp); + fsync(fileno(fp)); fclose(fp); file = pgFileNew(tablespace_map, true); @@ -1662,6 +1664,7 @@ write_backup_file_list(parray *files, const char *root) print_file_list(fp, files, root); + fsync(fileno(fp)); fclose(fp); } diff --git a/src/data.c b/src/data.c index bcf89d947..c6bcceee0 100644 --- a/src/data.c +++ b/src/data.c @@ -357,6 +357,7 @@ backup_data_file(const char *from_root, const char *to_root, strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); @@ -470,6 +471,7 @@ restore_file_partly(const char *from_root,const char *to_root, pgFile *file) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); } @@ -605,6 +607,7 @@ restore_data_file(const char *from_root, strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); } @@ -732,6 +735,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); @@ -832,9 +836,9 @@ copy_wal_file(const char *from_path, const char *to_path) strerror(errno_tmp)); } + fsync(fileno(out)); fclose(in); fclose(out); - } /* @@ -957,6 +961,7 @@ copy_file_partly(const char *from_root, const char *to_root, /* add meta information needed for recovery */ file->is_partial_copy = true; + fsync(fileno(out)); fclose(in); fclose(out); diff --git a/src/parsexlog.c b/src/parsexlog.c index aa8de7ed9..6aadfaea5 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -319,7 +319,7 @@ validate_wal(pgBackup *backup, * If recovery target is provided check that we can restore backup to a * recoverty target time or xid. */ - if (!TransactionIdIsValid(target_xid) || target_time == 0) + if (!TransactionIdIsValid(target_xid) && target_time == 0) { /* Recoverty target is not given so exit */ elog(INFO, "backup validation completed successfully"); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 867df1057..7547f6797 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,7 +17,7 @@ #include #include -const char *PROGRAM_VERSION = "1.1.17"; +const char *PROGRAM_VERSION = "2.0.0"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; @@ -72,6 +72,7 @@ uint32 retention_window = 0; /* compression options */ CompressAlg compress_alg = NOT_DEFINED_COMPRESS; int compress_level = DEFAULT_COMPRESS_LEVEL; +bool compress_shortcut = false; /* other options */ char *instance_name; @@ -132,6 +133,7 @@ static pgut_option options[] = /* compression options */ { 'f', 36, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, { 'u', 37, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 38, "compress", &compress_shortcut, SOURCE_CMDLINE }, /* logging options */ { 'f', 40, "log-level", opt_log_level, SOURCE_CMDLINE }, { 's', 41, "log-filename", &log_filename, SOURCE_CMDLINE }, @@ -353,6 +355,9 @@ main(int argc, char *argv[]) if (num_threads < 1) num_threads = 1; + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + if (backup_subcmd != SET_CONFIG) { if (compress_level != DEFAULT_COMPRESS_LEVEL @@ -405,7 +410,7 @@ main(int argc, char *argv[]) elog(ERROR, "show-config command doesn't accept any options except -B and --instance"); return do_configure(true); case SET_CONFIG: - if (argc == 5) + if (argc == 6) elog(ERROR, "set-config command requires at least one option"); return do_configure(false); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6e2b42eca..efcb928e4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -294,6 +294,7 @@ extern uint32 retention_window; /* compression options */ extern CompressAlg compress_alg; extern int compress_level; +extern bool compress_shortcut; #define DEFAULT_COMPRESS_LEVEL 6 diff --git a/src/restore.c b/src/restore.c index cc012a9d5..69a614dc0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -750,6 +750,7 @@ create_recovery_conf(time_t backup_id, if (target_tli) fprintf(fp, "recovery_target_timeline = '%u'\n", target_tli); + fsync(fileno(fp)); fclose(fp); } diff --git a/src/validate.c b/src/validate.c index 0795179a5..f9eeb3fa2 100644 --- a/src/validate.c +++ b/src/validate.c @@ -230,7 +230,10 @@ do_validate_all(void) } if (corrupted_backup_found) + { elog(INFO, "Some backups are not valid"); + return 1; + } else elog(INFO, "All backups are valid"); diff --git a/tests/__init__.py b/tests/__init__.py index e6094ad03..db736768d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,16 +6,11 @@ ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ - pgpro688, false_positive, replica + false_positive, replica def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - suite.addTests(loader.loadTestsFromModule(replica)) -# suite.addTests(loader.loadTestsFromModule(pgpro560)) -# suite.addTests(loader.loadTestsFromModule(pgpro589)) -# suite.addTests(loader.loadTestsFromModule(pgpro688)) -# suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(show_test)) @@ -33,8 +28,9 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(false_positive)) return suite - - -# ExpectedFailures are bugs, which should be fixed diff --git a/tests/backup_test.py b/tests/backup_test.py index b68be8d29..da31f25f7 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1,9 +1,7 @@ import unittest import os -import six from time import sleep -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException class BackupTest(ProbackupTest, unittest.TestCase): @@ -12,10 +10,6 @@ def __init__(self, *args, **kwargs): super(BackupTest, self).__init__(*args, **kwargs) self.module_name = 'backup' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-707 @@ -39,8 +33,8 @@ def test_backup_modes_archive(self): backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("FULL")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied excluded = True @@ -56,8 +50,8 @@ def test_backup_modes_archive(self): # print self.show_pb(node) show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("PAGE")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "PAGE") # Check parent backup self.assertEqual( @@ -68,15 +62,16 @@ def test_backup_modes_archive(self): self.backup_node(backup_dir, 'node', node, backup_type="ptrack") show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['Status'], six.b("OK")) - self.assertEqual(show_backup['Mode'], six.b("PTRACK")) + self.assertEqual(show_backup['Status'], "OK") + self.assertEqual(show_backup['Mode'], "PTRACK") # Check parent backup self.assertEqual( page_backup_id, self.show_pb(backup_dir, 'node', backup_id=show_backup['ID'])["parent-backup-id"]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_smooth_checkpoint(self): @@ -93,9 +88,12 @@ def test_smooth_checkpoint(self): node.start() self.backup_node(backup_dir, 'node' ,node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + #@unittest.skip("skip") def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" @@ -115,7 +113,7 @@ def test_incremental_backup_without_full(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -127,16 +125,17 @@ def test_incremental_backup_without_full(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) - node.stop() + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) - @unittest.expectedFailure - # Need to forcibly validate parent + # @unittest.expectedFailure def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] @@ -151,29 +150,37 @@ def test_incremental_backup_corrupt_full(self): node.start() backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + file = os.path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf") os.remove(file) try: - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.validate_pb(backup_dir, 'node') # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + self.assertEqual(1, 0, "Expecting Error because of validation of corrupted backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: - self.assertEqual(e.message, - 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + except ProbackupException as e: + self.assertTrue("INFO: Validate backups of the instance 'node'\n" in e.message + and 'WARNING: Backup file "{0}" is not found\n'.format(file) in e.message + and "WARNING: Backup {0} is corrupted\n".format(backup_id) in e.message + and "INFO: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd)) - sleep(1) + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + "ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n", + "\n Unexpected Error Message: {0}\n CMD: {1}".format(repr(e.message), self.cmd)) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("ERROR")) - node.stop() + # sleep(1) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_ptrack_threads(self): @@ -190,12 +197,13 @@ def test_ptrack_threads(self): node.start() self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_ptrack_threads_stream(self): @@ -213,7 +221,9 @@ def test_ptrack_threads_stream(self): self.backup_node(backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") self.backup_node(backup_dir, 'node', node, backup_type="ptrack", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) - node.stop() + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py index e9c176f0f..593bd83c1 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -1,8 +1,6 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess @@ -12,10 +10,6 @@ def __init__(self, *args, **kwargs): super(DeleteTest, self).__init__(*args, **kwargs) self.module_name = 'delete' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_full_backups(self): @@ -55,7 +49,8 @@ def test_delete_full_backups(self): self.assertEqual(show_backups[0]['ID'], id_1) self.assertEqual(show_backups[1]['ID'], id_3) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_delete_increment_page(self): """delete increment and all after him""" @@ -88,12 +83,13 @@ def test_delete_increment_page(self): show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[0]['Status'], six.b("OK")) - self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[1]['Status'], six.b("OK")) + self.assertEqual(show_backups[0]['Mode'], "FULL") + self.assertEqual(show_backups[0]['Status'], "OK") + self.assertEqual(show_backups[1]['Mode'], "FULL") + self.assertEqual(show_backups[1]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_delete_increment_ptrack(self): """delete increment and all after him""" @@ -126,9 +122,10 @@ def test_delete_increment_ptrack(self): show_backups = self.show_pb(backup_dir, 'node') self.assertEqual(len(show_backups), 2) - self.assertEqual(show_backups[0]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[0]['Status'], six.b("OK")) - self.assertEqual(show_backups[1]['Mode'], six.b("FULL")) - self.assertEqual(show_backups[1]['Status'], six.b("OK")) + self.assertEqual(show_backups[0]['Mode'], "FULL") + self.assertEqual(show_backups[0]['Status'], "OK") + self.assertEqual(show_backups[1]['Mode'], "FULL") + self.assertEqual(show_backups[1]['Status'], "OK") - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e88cb7c97..4f95ebcf9 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 1.1.17 +pg_probackup 2.0.0 diff --git a/tests/false_positive.py b/tests/false_positive.py index 71e2899ff..00477524e 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -1,11 +1,8 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class FalsePositive(ProbackupTest, unittest.TestCase): @@ -14,10 +11,6 @@ def __init__(self, *args, **kwargs): super(FalsePositive, self).__init__(*args, **kwargs) self.module_name = 'false_positive' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_pgpro561(self): @@ -26,62 +19,64 @@ def test_pgpro561(self): check that archiving is not successful on node1 """ fname = self.id().split('.')[3] - master = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/master".format(fname), - set_archiving=True, + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) - master.start() + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() - self.assertEqual(self.init_pb(master), six.b("")) - id = self.backup_pb(master, backup_type='full', options=["--stream"]) + backup_id = self.backup_node(backup_dir, 'node1', node1, options=["--stream"]) - node1 = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}/node1".format(fname)) - node1.cleanup() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname)) + node2.cleanup() - master.psql( + node1.psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - self.backup_pb(master, backup_type='page', options=["--stream"]) - self.restore_pb(backup_dir=self.backup_dir(master), data_dir=node1.data_dir) - node1.append_conf('postgresql.auto.conf', 'port = {0}'.format(node1.port)) - node1.start({"-t": "600"}) + self.backup_node(backup_dir, 'node1', node1, backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf('postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.start({"-t": "600"}) - timeline_master = master.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] - self.assertEqual(timeline_master, timeline_node1, "Timelines on Master and Node1 should be equal. This is unexpected") + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual(timeline_node1, timeline_node2, "Timelines on Master and Node1 should be equal. This is unexpected") - archive_command_master = master.safe_psql("postgres", "show archive_command") archive_command_node1 = node1.safe_psql("postgres", "show archive_command") - self.assertEqual(archive_command_master, archive_command_node1, "Archive command on Master and Node should be equal. This is unexpected") + archive_command_node2 = node2.safe_psql("postgres", "show archive_command") + self.assertEqual(archive_command_node1, archive_command_node2, "Archive command on Master and Node should be equal. This is unexpected") - res = node1.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") # self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) - if res == six.b(""): + if result == "": self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') - master.stop() - node1.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def pgpro688(self): - """ - make node with archiving, make backup, - get Recovery Time, validate to Recovery Time - Waiting PGPRO-688 - """ + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full') - recovery_time = self.show_pb(node, id=id)['recovery-time'] + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] # Uncommenting this section will make this test True Positive #node.psql("postgres", "select pg_create_restore_point('123')") @@ -89,46 +84,58 @@ def pgpro688(self): #node.psql("postgres", "select pg_switch_xlog()") #### - try: - self.validate_pb(node, options=["--time='{0}'".format(recovery_time)]) - self.assertEqual(1, 0, 'Error is expected because We should not be able safely validate "Recovery Time" without wal record with timestamp') - except ProbackupException, e: - self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message) + #try: + self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(recovery_time)]) + # we should die here because exception is what we expect to happen + # self.assertEqual(1, 0, "Expecting Error because it should not be possible safely validate 'Recovery Time' without wal record with timestamp.\n Output: {0} \n CMD: {1}".format( + # repr(self.output), self.cmd)) + # except ProbackupException as e: + # self.assertTrue('WARNING: recovery can be done up to time {0}'.format(recovery_time) in e.message, + # '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def pgpro702_688(self): - """ - make node without archiving, make stream backup, - get Recovery Time, validate to Recovery Time - """ + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - id = self.backup_pb(node, backup_type='full', options=["--stream"]) - recovery_time = self.show_pb(node, id=id)['recovery-time'] + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] self.assertIn(six.b("INFO: backup validation completed successfully on"), - self.validate_pb(node, options=["--time='{0}'".format(recovery_time)])) + self.validate_pb(backup_dir, 'node', node, options=["--time='{0}'".format(recovery_time)])) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure def test_validate_wal_lost_segment(self): """Loose segment located between backups. ExpectedFailure. This is BUG """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/false_positive/{0}".format(fname), - set_archiving=True, + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} ) - + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) - self.backup_pb(node, backup_type='full') + + self.backup_node(backup_dir, 'node', node) # make some wals node.pgbench_init(scale=2) @@ -141,15 +148,59 @@ def test_validate_wal_lost_segment(self): pgbench.stdout.close() # delete last wal segment - wals_dir = os.path.join(self.backup_dir(node), "wal") + wals_dir = os.path.join(backup_dir, "wal", 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals = map(int, wals) - os.remove(os.path.join(self.backup_dir(node), "wal", '0000000' + str(max(wals)))) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) ##### Hole Smokes, Batman! We just lost a wal segment and know nothing about it ##### We need archive-push ASAP - self.backup_pb(node, backup_type='full') - self.assertTrue('validation completed successfully' in self.validate_pb(node)) + self.backup_node(backup_dir, 'node', node) + self.assertFalse('validation completed successfully' in self.validate_pb(backup_dir, 'node')) ######## - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) + self.assertEqual(1, 0, "Expecting Error because page backup should not be possible without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: Valid backup on current timeline is not found. Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b7a7cf3b0..d27b31814 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,10 +1,10 @@ # you need os for unittest to work import os -from sys import exit +from sys import exit, argv, version_info import subprocess import shutil import six -from testgres import get_new_node +from testgres import get_new_node, clean_all import hashlib import re import pwd @@ -73,40 +73,6 @@ def dir_files(base_dir): out_list.sort() return out_list - -class ShowBackup(object): - def __init__(self, line): - self.counter = 0 - - print split_line - self.id = self.get_inc(split_line) - # TODO: parse to datetime - if len(split_line) == 12: - self.recovery_time = "%s %s" % (self.get_inc(split_line), self.get_inc(split_line)) - # if recovery time is '----' - else: - self.recovery_time = self.get_inc(split_line) - self.mode = self.get_inc(split_line) -# print self.mode - self.wal = self.get_inc(split_line) - self.cur_tli = self.get_inc(split_line) - # slash - self.counter += 1 - self.parent_tli = self.get_inc(split_line) - # TODO: parse to interval - self.time = self.get_inc(split_line) - # TODO: maybe rename to size? - self.data = self.get_inc(split_line) - self.start_lsn = self.get_inc(split_line) - self.stop_lsn = self.get_inc(split_line) - self.status = self.get_inc(split_line) - - def get_inc(self, split_line): -# self.counter += 1 -# return split_line[self.counter - 1] - return split_line - - class ProbackupTest(object): def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) @@ -134,7 +100,6 @@ def __init__(self, *args, **kwargs): self.test_env["LC_MESSAGES"] = "C" self.test_env["LC_TIME"] = "C" - self.helpers_path = os.path.dirname(os.path.realpath(__file__)) self.dir_path = os.path.abspath(os.path.join(self.helpers_path, os.pardir)) self.tmp_path = os.path.abspath(os.path.join(self.dir_path, 'tmp_dirs')) @@ -145,6 +110,10 @@ def __init__(self, *args, **kwargs): self.probackup_path = os.path.abspath(os.path.join( self.dir_path, "../pg_probackup")) self.user = self.get_username() + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False def arcwal_dir(self, node): return "%s/backup/wal" % node.base_dir @@ -228,10 +197,12 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size): byte_size_minus_header = byte_size - header_size file = os.open(file + '_ptrack', os.O_RDONLY) os.lseek(file, header_size, 0) - lot_of_bytes = os.read(file, byte_size_minus_header) - for byte in lot_of_bytes: + lots_of_bytes = os.read(file, byte_size_minus_header) + byte_list = [lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes))] + for byte in byte_list: + #byte_inverted = bin(int(byte, base=16))[2:][::-1] + #bits = (byte >> x) & 1 for x in range(7, -1, -1) byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] -# byte_to_bits = (byte >> x) & 1 for x in range(7, -1, -1) for bit in byte_inverted: if len(ptrack_bits_for_fork) < size: ptrack_bits_for_fork.append(int(bit)) @@ -249,9 +220,10 @@ def check_ptrack_sanity(self, idx_dict): # Page was not present before, meaning that relation got bigger # Ptrack should be equal to 1 if idx_dict['ptrack'][PageNum] != 1: - print 'Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Page Number {0} of type {1} was added, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False continue if PageNum not in idx_dict['new_pages']: @@ -266,29 +238,34 @@ def check_ptrack_sanity(self, idx_dict): if idx_dict['new_pages'][PageNum] != idx_dict['old_pages'][PageNum]: # Page has been changed, meaning that ptrack should be equal to 1 if idx_dict['ptrack'][PageNum] != 1: - print 'Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Page Number {0} of type {1} was changed, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) if PageNum == 0 and idx_dict['type'] == 'spgist': - print 'SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0' + if self.verbose: + print('SPGIST is a special snowflake, so don`t fret about losing ptrack for blknum 0') continue success = False else: # Page has not been changed, meaning that ptrack should be equal to 0 if idx_dict['ptrack'][PageNum] != 0: - print 'Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict - self.assertEqual(success, True) + if self.verbose: + print('Page Number {0} of type {1} was not changed, but ptrack value is {2}'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) + self.assertEqual(success, True, 'Ptrack of index {0} does not correspond to state of its pages.\n Gory Details: \n{1}'.format( + idx_dict['type'], idx_dict)) def check_ptrack_recovery(self, idx_dict): success = True size = idx_dict['size'] for PageNum in range(size): if idx_dict['ptrack'][PageNum] != 1: - print 'Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Recovery for Page Number {0} of Type {1} was conducted, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False self.assertEqual(success, True) @@ -296,29 +273,23 @@ def check_ptrack_clean(self, idx_dict, size): success = True for PageNum in range(size): if idx_dict['ptrack'][PageNum] != 0: - print 'Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format( - PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum]) - print idx_dict + if self.verbose: + print('Ptrack for Page Number {0} of Type {1} should be clean, but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], idx_dict['ptrack'][PageNum])) + print(idx_dict) success = False - self.assertEqual(success, True) + self.assertEqual(success, True, '') - def run_pb(self, command, async=False): + def run_pb(self, command): try: self.cmd = [' '.join(map(str,[self.probackup_path] + command))] - print self.cmd - if async is True: - return subprocess.Popen( - [self.probackup_path] + command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.test_env - ) - else: - self.output = subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) + if self.verbose: + print(self.cmd) + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") if command[0] == 'backup': # return backup ID for line in self.output.splitlines(): @@ -327,7 +298,7 @@ def run_pb(self, command, async=False): else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output, self.cmd) + raise ProbackupException(e.output.decode("utf-8"), self.cmd) def init_pb(self, backup_dir): @@ -358,12 +329,21 @@ def del_instance(self, backup_dir, instance, node): def clean_pb(self, backup_dir): shutil.rmtree(backup_dir, ignore_errors=True) - def backup_node(self, backup_dir, instance, node, backup_type="full", options=[], async=False): + def backup_node(self, backup_dir, instance, node=False, data_dir=False, backup_type="full", options=[]): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir cmd_list = [ "backup", "-B", backup_dir, - "-D", node.data_dir, + "-D", pgdata, "-p", "%i" % node.port, "-d", "postgres", "--instance={0}".format(instance) @@ -371,7 +351,7 @@ def backup_node(self, backup_dir, instance, node, backup_type="full", options=[] if backup_type: cmd_list += ["-b", backup_type] - return self.run_pb(cmd_list + options, async) + return self.run_pb(cmd_list + options) def restore_node(self, backup_dir, instance, node=False, data_dir=None, backup_id=None, options=[]): if data_dir is None: @@ -429,9 +409,9 @@ def show_pb(self, backup_dir, instance=None, backup_id=None, options=[], as_text if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print warning.format( + print(warning.format( header=header, body=body, - header_split=header_split, body_split=backup_record_split) + header_split=header_split, body_split=backup_record_split)) exit(1) new_dict = dict(zip(header_split, backup_record_split)) backup_list.append(new_dict) @@ -473,7 +453,6 @@ def delete_pb(self, backup_dir, instance=None, backup_id=None, options=[]): if backup_id: cmd_list += ["-i", backup_id] - # print(cmd_list) return self.run_pb(cmd_list + options) def delete_expired(self, backup_dir, instance, options=[]): @@ -563,3 +542,17 @@ def get_pgpro_edition(self, node): def get_username(self): """ Returns current user name """ return pwd.getpwuid(os.getuid())[0] + + def del_test_dir(self, module_name, fname): + """ Returns current user name """ + try: + clean_all() + except: + pass + + shutil.rmtree(os.path.join(self.tmp_path, self.module_name, fname), + ignore_errors=True) + try: + os.rmdir(os.path.join(self.tmp_path, self.module_name)) + except: + pass diff --git a/tests/init_test.py b/tests/init_test.py index b19f069dc..52699f4f3 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -1,9 +1,6 @@ -import unittest -from sys import exit import os -from os import path -import six -from helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException class InitTest(ProbackupTest, unittest.TestCase): @@ -25,20 +22,22 @@ def test_success(self): ['backups', 'wal'] ) self.add_instance(backup_dir, 'node', node) - - self.assertEqual("INFO: Instance 'node' successfully deleted\n", - self.del_instance(backup_dir, 'node', node), + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node', node), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) try: self.show_pb(backup_dir, 'node') self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: Instance 'node' does not exist in this backup catalog\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") def test_already_exist(self): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] @@ -49,25 +48,28 @@ def test_already_exist(self): self.show_pb(backup_dir, 'node') self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: Instance 'node' does not exist in this backup catalog\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + # @unittest.skip("skip") def test_abs_path(self): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname)) try: - self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: -B, --backup-path must be an absolute path\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py index 1114c1696..a05a0c44d 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -1,8 +1,6 @@ import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException class OptionTest(ProbackupTest, unittest.TestCase): @@ -11,10 +9,6 @@ def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) self.module_name = 'option' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_help_1(self): @@ -24,7 +18,7 @@ def test_help_1(self): with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( self.run_pb(["--help"]), - help_out.read() + help_out.read().decode("utf-8") ) # @unittest.skip("skip") @@ -35,7 +29,7 @@ def test_version_2(self): with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertEqual( self.run_pb(["--version"]), - version_out.read() + version_out.read().decode("utf-8") ) # @unittest.skip("skip") @@ -47,7 +41,7 @@ def test_without_backup_path_3(self): self.run_pb(["backup", "-b", "full"]) self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -59,7 +53,6 @@ def test_options_4(self): backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) - try: node.stop() except: @@ -73,7 +66,7 @@ def test_options_4(self): self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: --instance\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -83,7 +76,7 @@ def test_options_4(self): self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -93,23 +86,25 @@ def test_options_4(self): self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid backup-mode "bad"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # delete failure without ID try: self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: required backup ID not specified\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + #@unittest.skip("skip") def test_options_5(self): """check options test""" @@ -118,7 +113,8 @@ def test_options_5(self): node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), pg_options={'wal_level': 'replica', 'max_wal_senders': '2'}) - self.assertEqual(self.init_pb(backup_dir), six.b("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir))) + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) self.add_instance(backup_dir, 'node', node) node.start() @@ -131,7 +127,7 @@ def test_options_5(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: syntax error in " = INFINITE"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -149,7 +145,7 @@ def test_options_5(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid backup-mode ""\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -162,10 +158,7 @@ def test_options_5(self): with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: conf.write("retention-redundancy=1\n") - self.assertEqual( - self.show_config(backup_dir, 'node')['retention-redundancy'], - six.b('1') - ) + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') # User cannot send --system-identifier parameter via command line try: @@ -173,7 +166,7 @@ def test_options_5(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: option system-identifier cannot be specified in command line\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -187,7 +180,7 @@ def test_options_5(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -205,10 +198,10 @@ def test_options_5(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: invalid option "TIMELINEID"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) -# self.clean_pb(backup_dir) -# node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pb_lib.py b/tests/pb_lib.py deleted file mode 100644 index 9406e9b62..000000000 --- a/tests/pb_lib.py +++ /dev/null @@ -1,304 +0,0 @@ -import os -from os import path, listdir -import subprocess -import shutil -import six -from testgres import get_new_node - - -def dir_files(base_dir): - out_list = [] - for dir_name, subdir_list, file_list in os.walk(base_dir): - if dir_name != base_dir: - out_list.append(path.relpath(dir_name, base_dir)) - for fname in file_list: - out_list.append(path.relpath(path.join(dir_name, fname), base_dir)) - out_list.sort() - return out_list - - -class ShowBackup(object): - def __init__(self, split_line): - self.counter = 0 - - self.id = self.get_inc(split_line) - # TODO: parse to datetime - if len(split_line) == 12: - self.recovery_time = "%s %s" % (self.get_inc(split_line), - self.get_inc(split_line)) - # if recovery time is '----' - else: - self.recovery_time = self.get_inc(split_line) - self.mode = self.get_inc(split_line) - self.cur_tli = self.get_inc(split_line) - # slash - self.counter += 1 - self.parent_tli = self.get_inc(split_line) - # TODO: parse to interval - self.time = self.get_inc(split_line) - # TODO: maybe rename to size? - self.data = self.get_inc(split_line) - self.start_lsn = self.get_inc(split_line) - self.stop_lsn = self.get_inc(split_line) - self.status = self.get_inc(split_line) - - def get_inc(self, split_line): - self.counter += 1 - return split_line[self.counter - 1] - - -class ProbackupTest(object): - def __init__(self, *args, **kwargs): - super(ProbackupTest, self).__init__(*args, **kwargs) - self.test_env = os.environ.copy() - envs_list = [ - "LANGUAGE", - "LC_ALL", - "PGCONNECT_TIMEOUT", - "PGDATA", - "PGDATABASE", - "PGHOSTADDR", - "PGREQUIRESSL", - "PGSERVICE", - "PGSSLMODE", - "PGUSER", - "PGPORT", - "PGHOST" - ] - - for e in envs_list: - try: - del self.test_env[e] - except: - pass - - self.test_env["LC_MESSAGES"] = "C" - self.test_env["LC_TIME"] = "C" - - self.dir_path = path.dirname(os.path.realpath(__file__)) - try: - os.makedirs(path.join(self.dir_path, "tmp_dirs")) - except: - pass - self.probackup_path = os.path.abspath(path.join( - self.dir_path, - "../pg_probackup" - )) - - def arcwal_dir(self, node): - return "%s/backup/wal" % node.base_dir - - def backup_dir(self, node): - return os.path.abspath("%s/backup" % node.base_dir) - - def make_bnode(self, base_dir=None, allows_streaming=False, options={}): - real_base_dir = path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - node = get_new_node('test', base_dir=real_base_dir) - node.init(allows_streaming=allows_streaming) - - if not allows_streaming: - node.append_conf("postgresql.conf", "wal_level = hot_standby") - node.append_conf("postgresql.conf", "archive_mode = on") - node.append_conf( - "postgresql.conf", - """archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node)) - ) - - for key, value in six.iteritems(options): - node.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return node - - def make_bnode_replica(self, root_node, base_dir=None, options={}): - real_base_dir = path.join(self.dir_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - - root_node.backup("basebackup") - - replica = get_new_node("replica", base_dir=real_base_dir) - # replica.init_from_backup(root_node, "data_replica", has_streaming=True) - - # Move data from backup - backup_path = os.path.join(root_node.base_dir, "basebackup") - shutil.move(backup_path, replica.data_dir) - os.chmod(replica.data_dir, 0o0700) - - # Change port in config file - replica.append_conf( - "postgresql.conf", - "port = {}".format(replica.port) - ) - # Enable streaming - replica.enable_streaming(root_node) - - for key, value in six.iteritems(options): - replica.append_conf("postgresql.conf", "%s = %s" % (key, value)) - - return replica - - def run_pb(self, command): - try: - return subprocess.check_output( - [self.probackup_path] + command, - stderr=subprocess.STDOUT, - env=self.test_env - ) - except subprocess.CalledProcessError as err: - return err.output - - def init_pb(self, node): - return self.run_pb([ - "init", - "-B", self.backup_dir(node), - "-D", node.data_dir - ]) - - def clean_pb(self, node): - shutil.rmtree(self.backup_dir(node), ignore_errors=True) - - def backup_pb(self, node, backup_type="full", options=[]): - cmd_list = [ - "backup", - "-D", node.data_dir, - "-B", self.backup_dir(node), - "-p", "%i" % node.port, - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] - - return self.run_pb(cmd_list + options) - - def backup_pb_proc(self, node, backup_dir, backup_type="full", - stdout=None, stderr=None, options=[]): - cmd_list = [ - self.probackup_path, - "backup", - "-D", node.data_dir, - "-B", backup_dir, - "-p", "%i" % (node.port), - "-d", "postgres" - ] - if backup_type: - cmd_list += ["-b", backup_type] - - proc = subprocess.Popen( - cmd_list + options, - stdout=stdout, - stderr=stderr - ) - - return proc - - def restore_pb(self, node, id=None, options=[]): - cmd_list = [ - "-D", node.data_dir, - "-B", self.backup_dir(node), - "restore" - ] - if id: - cmd_list.append(id) - - # print(cmd_list) - return self.run_pb(cmd_list + options) - - def show_pb(self, node, id=None, options=[], as_text=False): - cmd_list = [ - "show", - "-B", self.backup_dir(node), - ] - if id: - cmd_list += [id] - - # print(cmd_list) - if as_text: - return self.run_pb(options + cmd_list) - elif id is None: - return [ShowBackup(line.split()) for line in self.run_pb(options + cmd_list).splitlines()[3:]] - else: - return dict([ - line.split(six.b("=")) - for line in self.run_pb(options + cmd_list).splitlines() - if line[0] != six.b("#")[0] - ]) - - def validate_pb(self, node, id=None, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "validate", - ] - if id: - cmd_list += [id] - - # print(cmd_list) - return self.run_pb(options + cmd_list) - - def delete_pb(self, node, id=None, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "delete", - ] - if id: - cmd_list += [id] - - # print(cmd_list) - return self.run_pb(options + cmd_list) - - def retention_purge_pb(self, node, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "retention", "purge", - ] - - return self.run_pb(options + cmd_list) - - def retention_show(self, node, options=[]): - cmd_list = [ - "-B", self.backup_dir(node), - "retention", "show", - ] - - return self.run_pb(options + cmd_list) - - def get_control_data(self, node): - pg_controldata = node.get_bin_path("pg_controldata") - out_data = {} - lines = subprocess.check_output( - [pg_controldata] + ["-D", node.data_dir], - stderr=subprocess.STDOUT, - env=self.test_env - ).splitlines() - for l in lines: - key, value = l.split(b":", maxsplit=1) - out_data[key.strip()] = value.strip() - return out_data - - def get_recovery_conf(self, node): - out_dict = {} - with open(path.join(node.data_dir, "recovery.conf"), "r") as recovery_conf: - for line in recovery_conf: - try: - key, value = line.split("=") - except: - continue - out_dict[key.strip()] = value.strip(" '").replace("'\n", "") - - return out_dict - - def wrong_wal_clean(self, node, wal_size): - wals_dir = path.join(self.backup_dir(node), "wal") - wals = [f for f in listdir(wals_dir) if path.isfile(path.join(wals_dir, f))] - wals.sort() - file_path = path.join(wals_dir, wals[-1]) - if path.getsize(file_path) != wal_size: - os.remove(file_path) - - def guc_wal_segment_size(self, node): - var = node.execute("postgres", "select setting from pg_settings where name = 'wal_segment_size'") - return int(var[0][0]) * self.guc_wal_block_size(node) - - def guc_wal_block_size(self, node): - var = node.execute("postgres", "select setting from pg_settings where name = 'wal_block_size'") - return int(var[0][0]) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index b0117f702..1654636e3 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -1,21 +1,15 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class CheckSystemID(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(CheckSystemID, self).__init__(*args, **kwargs) - - @classmethod - def tearDownClass(cls): - stop_all() + self.module_name = 'pgpro560' # @unittest.skip("skip") # @unittest.expectedFailure @@ -27,24 +21,32 @@ def test_pgpro560_control_file_loss(self): check that backup failed """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() - self.assertEqual(self.init_pb(node), six.b("")) file = os.path.join(node.base_dir,'data', 'global', 'pg_control') os.remove(file) try: - self.backup_pb(node, backup_type='full', options=['--stream']) - assertEqual(1, 0, 'Error is expected because of control file loss') - except ProbackupException, e: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( - 'ERROR: could not open file' and 'pg_control' in e.message, - 'Expected error is about control file loss') + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_pgpro560_systemid_mismatch(self): """ @@ -54,25 +56,33 @@ def test_pgpro560_systemid_mismatch(self): check that backup failed """ fname = self.id().split('.')[3] - node1 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node1".format(fname), + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) node1.start() - node2 = self.make_simple_node(base_dir="tmp_dirs/pgpro560/{0}/node2".format(fname), + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(self.module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) node2.start() - self.assertEqual(self.init_pb(node1), six.b("")) + + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) try: - self.backup_pb(node1, data_dir=node2.data_dir, backup_type='full', options=['--stream']) - assertEqual(1, 0, 'Error is expected because of SYSTEM ID mismatch') - except ProbackupException, e: + self.backup_node(backup_dir, 'node1', node1, data_dir=node2.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( - 'ERROR: Backup data directory was initialized for system id' and - 'but target backup directory system id is' in e.message, - 'Expected error is about SYSTEM ID mismatch') + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but target backup directory system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 00988d057..c15cd4aaf 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -1,21 +1,15 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit class ArchiveCheck(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(ArchiveCheck, self).__init__(*args, **kwargs) - - @classmethod - def tearDownClass(cls): - stop_all() + self.module_name = 'pgpro589' # @unittest.skip("skip") # @unittest.expectedFailure @@ -26,10 +20,13 @@ def test_archive_mode(self): check ERROR text """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) node.start() node.pgbench_init(scale=5) @@ -41,15 +38,17 @@ def test_archive_mode(self): pgbench.wait() pgbench.stdout.close() - path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() - - self.assertEqual(self.init_pb(node), six.b("")) - try: - self.backup_pb(node, backup_type='full', options=['--archive-timeout=10']) - assertEqual(1, 0, 'Error is expected because of disabled archive_mode') - except ProbackupException, e: - self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n') + self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of disabled archive_mode.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: Archiving must be enabled for archive backup\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_pgpro589(self): """ @@ -59,12 +58,16 @@ def test_pgpro589(self): check that no files where copied to backup catalogue """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="tmp_dirs/pgpro589/{0}/node".format(fname), + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node.append_conf("postgresql.auto.conf", "archive_mode = on") - node.append_conf("postgresql.auto.conf", "wal_level = archive") + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") node.start() @@ -76,22 +79,25 @@ def test_pgpro589(self): ) pgbench.wait() pgbench.stdout.close() - - path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip() - self.assertEqual(self.init_pb(node), six.b("")) + path = node.safe_psql("postgres", "select pg_relation_filepath('pgbench_accounts')").rstrip().decode("utf-8") try: - self.backup_pb( - node, backup_type='full', options=['--archive-timeout=10']) - assertEqual(1, 0, 'Error is expected because of missing archive wal segment with start_backup() LSN') - except ProbackupException, e: - self.assertTrue('INFO: wait for LSN' in e.message, "Expecting 'INFO: wait for LSN'") - self.assertTrue('ERROR: switched WAL segment' and 'could not be archived' in e.message, - "Expecting 'ERROR: switched WAL segment could not be archived'") + self.backup_node(backup_dir, 'node', node, options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of missing archive wal segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: wait for LSN' in e.message + and 'ERROR: switched WAL segment' in e.message + and 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - id = self.show_pb(node)[0]['ID'] - self.assertEqual('ERROR', self.show_pb(node, id=id)['status'], 'Backup should have ERROR status') - #print self.backup_dir(node) - file = os.path.join(self.backup_dir(node), 'backups', id, 'database', path) + backup_id = self.show_pb(backup_dir, 'node')[0]['ID'] + self.assertEqual('ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup should have ERROR status') + file = os.path.join(backup_dir, 'backups', 'node', backup_id, 'database', path) self.assertFalse(os.path.isfile(file), '\n Start LSN was not found in archive but datafiles where copied to backup catalogue.\n For example: {0}\n It is not optimal'.format(file)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/pgpro688.py b/tests/pgpro688.py deleted file mode 100644 index 416d2cf5e..000000000 --- a/tests/pgpro688.py +++ /dev/null @@ -1,201 +0,0 @@ -import unittest -import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from testgres import stop_all, get_username -import subprocess -from sys import exit, _getframe -import shutil -import time - - -class ReplicaTest(ProbackupTest, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(ReplicaTest, self).__init__(*args, **kwargs) - self.module_name = 'replica' - self.instance_master = 'master' - self.instance_replica = 'replica' - -# @classmethod -# def tearDownClass(cls): -# stop_all() - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_replica_stream_full_backup(self): - """make full stream backup from replica""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') - master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} - ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, self.instance_master, master) - master.start() - - # Make empty Object 'replica' from new node - replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) - replica_port = replica.port - replica.cleanup() - - # FULL STREAM backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # FULL STREAM backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full', options=['--stream']) - - # Restore last backup from master to Replica directory - self.restore_node(backup_dir, self.instance_master, replica.data_dir) - # Set Replica - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) - replica.start({"-t": "600"}) - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Add instance replica - self.add_instance(backup_dir, self.instance_replica, replica) - - # FULL STREAM backup of replica - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ - '--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - - # Validate instance replica - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - def test_replica_archive_full_backup(self): - """make page archive backup from replica""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') - master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '5min'} - ) - self.set_archiving(backup_dir, self.instance_master, master) - self.init_pb(backup_dir) - self.add_instance(backup_dir, self.instance_master, master) - master.start() - - # Make empty Object 'replica' from new node - replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(self.module_name, fname)) - replica_port = replica.port - replica.cleanup() - - # FULL ARCHIVE backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='full') - # Create table t_heap - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # PAGE ARCHIVE backup of master - self.backup_node(backup_dir, self.instance_master, master, backup_type='page') - - # Restore last backup from master to Replica directory - self.restore_node(backup_dir, self.instance_master, replica.data_dir) - - # Set Replica - self.set_archiving(backup_dir, self.instance_replica, replica, replica=True) - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user={0} port={1} sslmode=prefer sslcompression=1'".format(get_username(), master.port)) - replica.start({"-t": "600"}) - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Make FULL ARCHIVE backup from replica - self.add_instance(backup_dir, self.instance_replica, replica) - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='full', options=[ - '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - # Drop Table t_heap - after = master.execute("postgres", "drop table t_heap") - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,512) i") - before = master.execute("postgres", "SELECT * FROM t_heap") - - # Make page backup from replica - self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in - self.backup_node(backup_dir, self.instance_replica, replica, backup_type='page', options=[ - '--log-level=verbose', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)])) - self.validate_pb(backup_dir, self.instance_replica) - self.assertEqual('OK', self.show_pb(backup_dir, self.instance_replica)[0]['Status']) - - @unittest.skip("skip") - def test_replica_archive_full_backup_123(self): - """ - make full archive backup from replica - """ - fname = self.id().split('.')[3] - master = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/master".format(fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) - master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.start() - - replica = self.make_simple_node(base_dir="tmp_dirs/replica/{0}/replica".format(fname)) - replica_port = replica.port - replica.cleanup() - - self.assertEqual(self.init_pb(master), six.b("")) - self.backup_pb(node=master, backup_type='full', options=['--stream']) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") - - before = master.execute("postgres", "SELECT * FROM t_heap") - - id = self.backup_pb(master, backup_type='page', options=['--stream']) - self.restore_pb(backup_dir=self.backup_dir(master), data_dir=replica.data_dir) - - # Settings for Replica - replica.append_conf('postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - # Set Archiving for replica - self.set_archiving_conf(replica, replica=True) - - replica.append_conf('recovery.conf', "standby_mode = 'on'") - replica.append_conf('recovery.conf', - "primary_conninfo = 'user=gsmol port={0} sslmode=prefer sslcompression=1'".format(master.port)) - replica.start({"-t": "600"}) - # Replica Started - - # master.execute("postgres", "checkpoint") - - # Check replica - after = replica.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Make backup from replica - self.assertEqual(self.init_pb(replica), six.b("")) - self.backup_pb(replica, backup_type='full', options=['--archive-timeout=30']) - self.validate_pb(replica) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 0880c0315..54d7f8a46 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,9 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_clean' - def teardown(self): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): @@ -89,8 +84,5 @@ def test_ptrack_clean(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - print self.show_pb(backup_dir, 'node', as_text=True) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index ff0fb6a21..13ea76788 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_cluster' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_btree(self): @@ -72,7 +66,8 @@ def test_ptrack_cluster_btree(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_spgist(self): fname = self.id().split('.')[3] @@ -130,7 +125,8 @@ def test_ptrack_cluster_spgist(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_brin(self): fname = self.id().split('.')[3] @@ -188,7 +184,8 @@ def test_ptrack_cluster_brin(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_gist(self): fname = self.id().split('.')[3] @@ -246,7 +243,8 @@ def test_ptrack_cluster_gist(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) def test_ptrack_cluster_gin(self): fname = self.id().split('.')[3] @@ -304,7 +302,5 @@ def test_ptrack_cluster_gin(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index e35234a65..41a310eb5 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -1,10 +1,6 @@ -import unittest -from sys import exit -from testgres import get_new_node, stop_all import os -from signal import SIGTERM -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack -from time import sleep +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -12,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_move_to_tablespace' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery(self): @@ -58,7 +50,5 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 24802697c..abc7a7514 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -1,20 +1,18 @@ +import os import unittest from sys import exit -from testgres import get_new_node, stop_all -import os -from signal import SIGTERM -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack -from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) - self.module_name = 'ptrack_move_to_tablespace' + self.module_name = 'ptrack_recovery' - def teardown(self): - # clean_all() - stop_all() + # @classmethod + # def tearDownClass(cls): + # clean_all() + # shutil.rmtree(os.path.join(self.tmp_path, self.module_name), ignore_errors=True) # @unittest.skip("skip") # @unittest.expectedFailure @@ -45,12 +43,13 @@ def test_ptrack_recovery(self): # get path to heap and index files idx_ptrack[i]['path'] = self.get_fork_path(node, i) - print 'Killing postmaster. Losing Ptrack changes' + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)}) if not node.status(): node.start() else: - print "Die! Die! Why won't you die?... Why won't you die?" + print("Die! Die! Why won't you die?... Why won't you die?") exit(1) for i in idx_ptrack: @@ -60,7 +59,5 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index f6b22b97e..fa8d145f6 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -1,8 +1,6 @@ -import unittest import os -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): @@ -78,7 +72,5 @@ def test_ptrack_vacuum(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() - -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 1a4d3fe54..e41bb2653 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_bits_frozen' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): @@ -69,7 +63,6 @@ def test_ptrack_vacuum_bits_frozen(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index ca5db7050..a52325d7f 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_bits_visibility' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): @@ -69,7 +63,6 @@ def test_ptrack_vacuum_bits_visibility(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index 9d9d5051f..592821a06 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -1,9 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -#import os -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -11,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_full' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full(self): @@ -72,7 +65,6 @@ def test_ptrack_vacuum_full(self): # compare pages and check ptrack sanity, the most important part self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 37dd99202..3bd54b694 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -1,8 +1,6 @@ import os import unittest -from sys import exit -from testgres import get_new_node, stop_all -from helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack class SimpleTest(ProbackupTest, unittest.TestCase): @@ -10,10 +8,6 @@ def __init__(self, *args, **kwargs): super(SimpleTest, self).__init__(*args, **kwargs) self.module_name = 'ptrack_vacuum_truncate' - def teardown(self): - # clean_all() - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate(self): @@ -71,7 +65,6 @@ def test_ptrack_vacuum_truncate(self): # compare pages and check ptrack sanity self.check_ptrack_sanity(idx_ptrack[i]) - node.stop() -if __name__ == '__main__': - unittest.main() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/replica.py b/tests/replica.py index 9f3e90f82..b74afaad0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1,13 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit, _getframe -import shutil -import time class ReplicaTest(ProbackupTest, unittest.TestCase): @@ -16,10 +11,6 @@ def __init__(self, *args, **kwargs): super(ReplicaTest, self).__init__(*args, **kwargs) self.module_name = 'replica' -# @classmethod -# def tearDownClass(cls): -# stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_replica_stream_full_backup(self): @@ -65,16 +56,18 @@ def test_replica_stream_full_backup(self): # Make backup from replica self.add_instance(backup_dir, 'slave', slave) - #time.sleep(2) self.assertTrue('INFO: Wait end of WAL streaming' and 'completed' in self.backup_node(backup_dir, 'slave', slave, options=['--stream', '--log-level=verbose', '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])) self.validate_pb(backup_dir, 'slave') self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def test_replica_archive_full_backup(self): - """make full archive backup from replica""" + """make full archive backup from replica, set replica, make backup from replica""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') master = self.make_simple_node(base_dir="{0}/{1}/master".format(self.module_name, fname), @@ -127,3 +120,7 @@ def test_replica_archive_full_backup(self): self.backup_node(backup_dir, 'slave', slave, options=['--archive-timeout=300', '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)]) self.validate_pb(backup_dir, 'slave') + self.assertEqual('OK', self.show_pb(backup_dir, 'slave')[0]['Status']) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py index 1ef35c7dc..4715e2a76 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,12 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import stop_all +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess from datetime import datetime -import shutil -from sys import exit class RestoreTest(ProbackupTest, unittest.TestCase): @@ -15,10 +11,6 @@ def __init__(self, *args, **kwargs): super(RestoreTest, self).__init__(*args, **kwargs) self.module_name = 'restore' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_restore_full_to_latest(self): @@ -45,7 +37,7 @@ def test_restore_full_to_latest(self): node.cleanup() # 1 - Test recovery from latest - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -58,7 +50,8 @@ def test_restore_full_to_latest(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_page_to_latest(self): @@ -89,7 +82,7 @@ def test_restore_full_page_to_latest(self): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -98,10 +91,11 @@ def test_restore_full_page_to_latest(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) - # @unittest.skip("skip") - def test_restore_to_timeline(self): + #@unittest.skip("skip") + def test_restore_to_specific_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), @@ -120,11 +114,11 @@ def test_restore_to_timeline(self): backup_id = self.backup_node(backup_dir, 'node', node) - target_tli = int(node.get_control_data()[six.b("Latest checkpoint's TimeLineID")]) + target_tli = int(node.get_control_data()["Latest checkpoint's TimeLineID"]) node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -134,13 +128,13 @@ def test_restore_to_timeline(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type="full") + self.backup_node(backup_dir, 'node', node) node.stop({"-m": "immediate"}) node.cleanup() # Correct Backup must be choosen for restore - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", "--timeline={0}".format(target_tli)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -152,7 +146,8 @@ def test_restore_to_timeline(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_time(self): @@ -181,7 +176,7 @@ def test_restore_to_time(self): node.stop() node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(target_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -190,7 +185,8 @@ def test_restore_to_time(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_xid(self): @@ -235,7 +231,7 @@ def test_restore_to_xid(self): node.stop({"-m": "fast"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -244,7 +240,8 @@ def test_restore_to_xid(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): @@ -275,7 +272,7 @@ def test_restore_full_ptrack_archive(self): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -284,7 +281,8 @@ def test_restore_full_ptrack_archive(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_ptrack(self): @@ -321,7 +319,7 @@ def test_restore_ptrack(self): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -330,7 +328,8 @@ def test_restore_ptrack(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): @@ -362,7 +361,7 @@ def test_restore_full_ptrack_stream(self): node.stop() node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -371,7 +370,8 @@ def test_restore_full_ptrack_stream(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): @@ -410,7 +410,7 @@ def test_restore_full_ptrack_under_load(self): node.stop({"-m": "immediate"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -418,10 +418,10 @@ def test_restore_full_ptrack_under_load(self): bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): @@ -463,16 +463,17 @@ def test_restore_full_under_load_ptrack(self): node.cleanup() #self.wrong_wal_clean(node, wal_segment_size) - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) bbalance = node.execute("postgres", "SELECT sum(bbalance) FROM pgbench_branches") delta = node.execute("postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): @@ -501,9 +502,9 @@ def test_restore_to_xid_inclusive(self): before = node.execute("postgres", "SELECT * FROM pgbench_branches") with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() - target_xid = res[0][0] + target_xid = result[0][0] pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -517,7 +518,7 @@ def test_restore_to_xid_inclusive(self): node.stop({"-m": "fast"}) node.cleanup() - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -528,7 +529,8 @@ def test_restore_to_xid_inclusive(self): self.assertEqual(before, after) self.assertEqual(len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): @@ -556,7 +558,7 @@ def test_restore_with_tablespace_mapping_1(self): con.commit() backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") # 1 - Try to restore to existing directory node.stop() @@ -565,7 +567,7 @@ def test_restore_with_tablespace_mapping_1(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because restore destionation is not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: restore destination is not empty: "{0}"\n'.format(node.data_dir), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -577,20 +579,20 @@ def test_restore_with_tablespace_mapping_1(self): # we should die here because exception is what we expect to happen self.assertEqual(1, 0, "Expecting Error because restore tablespace destination is not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: restore tablespace destination is not empty: "{0}"\n'.format(tblspc_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 3 - Restore using tablespace-mapping tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - res = node.execute("postgres", "SELECT id FROM test") - self.assertEqual(res[0][0], 1) + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) # 4 - Restore using tablespace-mapping using page backup self.backup_node(backup_dir, 'node', node) @@ -600,22 +602,23 @@ def test_restore_with_tablespace_mapping_1(self): backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['Status'], six.b("OK")) - self.assertEqual(show_pb[2]['Status'], six.b("OK")) + self.assertEqual(show_pb[1]['Status'], "OK") + self.assertEqual(show_pb[2]['Status'], "OK") node.stop() node.cleanup() tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() - res = node.execute("postgres", "SELECT id FROM test OFFSET 1") - self.assertEqual(res[0][0], 2) + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): @@ -633,7 +636,7 @@ def test_restore_with_tablespace_mapping_2(self): # Full backup self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], six.b("OK")) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['Status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -647,8 +650,8 @@ def test_restore_with_tablespace_mapping_2(self): # First page backup self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], six.b("PAGE")) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['Mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: @@ -661,15 +664,15 @@ def test_restore_with_tablespace_mapping_2(self): # Second page backup backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], six.b("OK")) - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], six.b("PAGE")) + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['Mode'], "PAGE") node.stop() node.cleanup() tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start() @@ -678,9 +681,11 @@ def test_restore_with_tablespace_mapping_2(self): self.assertEqual(count[0][0], 4) count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - node.stop() - # @unittest.skip("skip") + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + #@unittest.skip("skip") def test_archive_node_backup_stream_restore_to_recovery_time(self): """make node with archiving, make stream backup, make PITR to Recovery Time""" fname = self.id().split('.')[3] @@ -692,6 +697,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.start() backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -700,18 +706,57 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): node.stop() node.cleanup() - recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) self.assertEqual(True, node.status()) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) + + + #@unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """make node with archiving, make stream backup, make PITR to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(self.module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + node.psql("postgres", "create table t_heap(a int)") + node.psql("postgres", "select pg_switch_xlog()") node.stop() + node.cleanup() + + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + node.start({"-t": "600"}) + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + self.assertEqual(True, node.status()) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_archive_node_backup_stream_pitr(self): @@ -734,16 +779,18 @@ def test_archive_node_backup_stream_pitr(self): recovery_time = self.show_pb(backup_dir, 'node', backup_id=backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) - node.stop() + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_archive_node_backup_archive_pitr_2(self): @@ -766,13 +813,15 @@ def test_archive_node_backup_archive_pitr_2(self): recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] - self.assertIn(six.b("INFO: Restore of backup {0} completed.".format(backup_id)), + self.assertIn("INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node(backup_dir, 'node', node, options=["-j", "4", '--time="{0}"'.format(recovery_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) node.start({"-t": "600"}) - res = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in res[2]) - node.stop() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py index 8be47b6f9..d7702793b 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -1,9 +1,7 @@ -import unittest import os +import unittest from datetime import datetime, timedelta -from os import path, listdir -from helpers.ptrack_helpers import ProbackupTest -from testgres import stop_all +from .helpers.ptrack_helpers import ProbackupTest class RetentionTest(ProbackupTest, unittest.TestCase): @@ -12,10 +10,6 @@ def __init__(self, *args, **kwargs): super(RetentionTest, self).__init__(*args, **kwargs) self.module_name = 'retention' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_1(self): @@ -51,17 +45,18 @@ def test_retention_redundancy_1(self): min_wal = None max_wal = None for line in log.splitlines(): - if line.startswith(b"INFO: removed min WAL segment"): + if line.startswith("INFO: removed min WAL segment"): min_wal = line[31:-1] - elif line.startswith(b"INFO: removed max WAL segment"): + elif line.startswith("INFO: removed max WAL segment"): max_wal = line[31:-1] - for wal_name in listdir(os.path.join(backup_dir, 'wal', 'node')): + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - wal_name_b = wal_name.encode('ascii') - self.assertEqual(wal_name_b[8:] > min_wal[8:], True) - self.assertEqual(wal_name_b[8:] > max_wal[8:], True) + #wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("123") def test_retention_window_2(self): @@ -87,12 +82,12 @@ def test_retention_window_2(self): # Make backup to be keeped self.backup_node(backup_dir, 'node', node) - backups = path.join(backup_dir, 'backups', 'node') + backups = os.path.join(backup_dir, 'backups', 'node') days_delta = 5 - for backup in listdir(backups): + for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue - with open(path.join(backups, backup, "backup.control"), "a") as conf: + with open(os.path.join(backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=days_delta))) days_delta -= 1 @@ -106,4 +101,5 @@ def test_retention_window_2(self): self.delete_expired(backup_dir, 'node') self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - node.stop() + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py index 37b146e4a..4a3df5012 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -1,9 +1,6 @@ -import unittest import os -from os import path -import six -from helpers.ptrack_helpers import ProbackupTest -from testgres import stop_all +import unittest +from .helpers.ptrack_helpers import ProbackupTest class OptionTest(ProbackupTest, unittest.TestCase): @@ -12,10 +9,6 @@ def __init__(self, *args, **kwargs): super(OptionTest, self).__init__(*args, **kwargs) self.module_name = 'show' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_show_1(self): @@ -36,8 +29,10 @@ def test_show_1(self): self.backup_node(backup_dir, 'node', node, options=["--log-level=panic"]), None ) - self.assertIn(six.b("OK"), self.show_pb(backup_dir, 'node', as_text=True)) - node.stop() + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_corrupt_2(self): @@ -57,9 +52,11 @@ def test_corrupt_2(self): backup_id = self.backup_node(backup_dir, 'node', node) # delete file which belong to backup - file = path.join(backup_dir, "backups", "node", backup_id.decode("utf-8"), "database", "postgresql.conf") + file = os.path.join(backup_dir, "backups", "node", backup_id, "database", "postgresql.conf") os.remove(file) self.validate_pb(backup_dir, 'node', backup_id) - self.assertIn(six.b("CORRUPT"), self.show_pb(backup_dir, as_text=True)) - node.stop() + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(self.module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index bef73cd77..20f823de9 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,12 +1,8 @@ -import unittest import os -import six -from helpers.ptrack_helpers import ProbackupTest, ProbackupException +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta -from testgres import stop_all import subprocess -from sys import exit -import re class ValidateTest(ProbackupTest, unittest.TestCase): @@ -15,10 +11,6 @@ def __init__(self, *args, **kwargs): super(ValidateTest, self).__init__(*args, **kwargs) self.module_name = 'validate' - @classmethod - def tearDownClass(cls): - stop_all() - # @unittest.skip("skip") # @unittest.expectedFailure def test_validate_wal_unreal_values(self): @@ -55,7 +47,7 @@ def test_validate_wal_unreal_values(self): after_backup_time = datetime.now().replace(second=0, microsecond=0) # Validate to real time - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(target_time)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -65,7 +57,7 @@ def test_validate_wal_unreal_values(self): self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_1)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertEqual(e.message, 'ERROR: Full backup satisfying target options is not found.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -75,7 +67,7 @@ def test_validate_wal_unreal_values(self): self.validate_pb(backup_dir, 'node', options=["--time='{0}'".format(unreal_time_2)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('ERROR: not enough WAL records to time' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -87,7 +79,7 @@ def test_validate_wal_unreal_values(self): target_xid = res[0][0] node.execute("postgres", "SELECT pg_switch_xlog()") - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) @@ -97,15 +89,18 @@ def test_validate_wal_unreal_values(self): self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('ERROR: not enough WAL records to xid' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Validate with backup ID - self.assertIn(six.b("INFO: backup validation completed successfully"), + self.assertIn("INFO: backup validation completed successfully", self.validate_pb(backup_dir, 'node', backup_id), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + # Clean after yourself + self.del_test_dir(self.module_name, fname) + # @unittest.skip("skip") def test_validate_corrupt_wal_1(self): """make archive node, make archive backup, corrupt all wal files, run validate, expect errors""" @@ -131,22 +126,25 @@ def test_validate_corrupt_wal_1(self): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: - f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(42) - f.write(six.b("blablablaadssaaaaaaaaaaaaaaa")) - f.close + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close # Simple validate try: self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('Possible WAL CORRUPTION' in e.message), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_corrupt_wal_2(self): @@ -178,22 +176,25 @@ def test_validate_corrupt_wal_2(self): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] wals.sort() for wal in wals: - f = open(os.path.join(wals_dir, wal), "rb+") - f.seek(0) - f.write(six.b("blablabla")) - f.close + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(0) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close # Validate to xid try: self.validate_pb(backup_dir, 'node', backup_id, options=['--xid={0}'.format(target_xid)]) self.assertEqual(1, 0, "Expecting Error because of wal segments corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertTrue('Possible WAL CORRUPTION' in e.message), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd) self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): @@ -232,7 +233,7 @@ def test_validate_wal_lost_segment_1(self): self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertIn('WARNING: WAL segment "{0}" is absent\nERROR: there are not enough WAL records to restore'.format( file), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -243,10 +244,12 @@ def test_validate_wal_lost_segment_1(self): self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of backup corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except ProbackupException, e: + except ProbackupException as e: self.assertIn('INFO: Backup {0} has status CORRUPT. Skip validation.\n'.format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) # @unittest.skip("skip") def test_validate_wal_lost_segment_2(self): @@ -291,12 +294,14 @@ def test_validate_wal_lost_segment_2(self): backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') self.assertEqual(1, 0, "Expecting Error because of wal segment disappearance.\n Output: {0} \n CMD: {1}".format( self.output, self.cmd)) - except ProbackupException, e: - self.assertTrue('INFO: wait for LSN' - and 'in archived WAL segment' - and 'WARNING: could not read WAL record at' + except ProbackupException as e: + self.assertTrue('INFO: wait for LSN' in e.message + and 'in archived WAL segment' in e.message + and 'WARNING: could not read WAL record at' in e.message and 'ERROR: WAL segment "{0}" is absent\n'.format(file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.assertEqual('ERROR', self.show_pb(backup_dir, 'node')[1]['Status'], 'Backup {0} should have STATUS "ERROR"') - node.stop() + + # Clean after yourself + self.del_test_dir(self.module_name, fname) From bcf09b3e1947aba3c643a46fd357019cacae714d Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 5 Jun 2018 09:47:49 +0300 Subject: [PATCH 0005/2107] Using option --extra-directory=/path/to/directory You can add files to backup from extra directory (excluding subdirectory) 1) The permissons of extra directory must allow to read/write in it 2) You can add only one directory 3) No additional subfolders creates in backup/database/ TODO: 1) to add more then one extra directory 2) make to backup extra directory subdirectories --- src/backup.c | 10 ++++++---- src/delete.c | 2 +- src/dir.c | 41 ++++++++++++++++++++++++++++------------- src/pg_probackup.c | 3 +++ src/pg_probackup.h | 9 +++++++-- src/restore.c | 4 +++- 6 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index a0b02fcf6..f00bf9506 100644 --- a/src/backup.c +++ b/src/backup.c @@ -603,11 +603,13 @@ do_backup_instance(void) if (is_remote_backup) get_remote_pgdata_filelist(backup_files_list); else - dir_list_file(backup_files_list, pgdata, true, true, false); + dir_list_file(backup_files_list, pgdata, true, true, false, false); /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); + dir_list_file(backup_files_list, extradir, true, true, false, true); + if (current.backup_mode != BACKUP_MODE_FULL) { elog(LOG, "current_tli:%X", current.tli); @@ -753,7 +755,7 @@ do_backup_instance(void) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false); for (i = 0; i < parray_num(xlog_files_list); i++) { @@ -1819,7 +1821,7 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true); + file = pgFileNew(backup_label, true, false); calc_file_checksum(file); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -1863,7 +1865,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true); + file = pgFileNew(tablespace_map, true, false); if (S_ISREG(file->mode)) calc_file_checksum(file); free(file->path); diff --git a/src/delete.c b/src/delete.c index f81fe70da..cd3c699aa 100644 --- a/src/delete.c +++ b/src/delete.c @@ -271,7 +271,7 @@ pgBackupDeleteFiles(pgBackup *backup) /* list files to be deleted */ files = parray_new(); pgBackupGetPath(backup, path, lengthof(path), NULL); - dir_list_file(files, path, false, true, true); + dir_list_file(files, path, false, true, true, false); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); diff --git a/src/dir.c b/src/dir.c index 8df3da3f8..dd7120f82 100644 --- a/src/dir.c +++ b/src/dir.c @@ -93,7 +93,7 @@ static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list); + bool omit_symlink, parray *black_list, bool is_extra); /* * Create directory, also create parent directories if necessary. @@ -123,7 +123,7 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink) +pgFileNew(const char *path, bool omit_symlink, bool is_extra) { struct stat st; pgFile *file; @@ -141,6 +141,8 @@ pgFileNew(const char *path, bool omit_symlink) file = pgFileInit(path); file->size = st.st_size; file->mode = st.st_mode; + file->is_extra = is_extra; + file->extradir = NULL; return file; } @@ -335,7 +337,7 @@ BlackListCompare(const void *str1, const void *str2) */ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, - bool add_root) + bool add_root, bool is_extra) { pgFile *file; parray *black_list = NULL; @@ -372,7 +374,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false); + file = pgFileNew(root, false, is_extra); if (file == NULL) return; @@ -384,7 +386,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (add_root) parray_append(files, file); - dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, is_extra); parray_qsort(files, pgFileComparePath); } @@ -573,7 +575,7 @@ dir_check_file(const char *root, pgFile *file) */ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list) + bool exclude, bool omit_symlink, parray *black_list, bool is_extra) { DIR *dir; struct dirent *dent; @@ -602,7 +604,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink); + file = pgFileNew(child, omit_symlink, is_extra); if (file == NULL) continue; @@ -648,7 +650,14 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, /* At least add the file */ if (S_ISREG(file->mode)) + { + if (is_extra) + { + file->extradir = pgut_strdup(file->path); + dirname(file->extradir); + } parray_append(files, file); + } /* * If the entry is a directory call dir_list_file_internal() @@ -656,7 +665,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, root, file, exclude, omit_symlink, - black_list); + black_list, is_extra); } if (errno && errno != ENOENT) @@ -736,7 +745,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false); + dir = pgFileNew(path, false, false); parray_append(files, dir); } @@ -819,10 +828,10 @@ print_file_list(FILE *out, const parray *files, const char *root) fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," - "\"compress_alg\":\"%s\"", + "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"", path, (unsigned long) file->write_size, file->mode, file->is_datafile?1:0, file->is_cfs?1:0, file->crc, - deparse_compress_alg(file->compress_alg)); + deparse_compress_alg(file->compress_alg), file->is_extra?1:0); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -999,6 +1008,7 @@ dir_read_file_list(const char *root, const char *file_txt) mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, + is_extra, crc, segno, n_blocks; @@ -1016,14 +1026,20 @@ dir_read_file_list(const char *root, const char *file_txt) get_control_value(buf, "segno", NULL, &segno, false); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); get_control_value(buf, "n_blocks", NULL, &n_blocks, false); + get_control_value(buf, "is_extra", NULL, &is_extra, false); if (root) - join_path_components(filepath, root, path); + if (is_extra) + join_path_components(filepath, root, basename(path)); + else + join_path_components(filepath, root, path); else strcpy(filepath, path); file = pgFileInit(filepath); + file->is_extra = is_extra ? true : false; + file->extradir = is_extra ? pgut_strdup(dirname(path)) : NULL; file->write_size = (size_t) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; @@ -1034,7 +1050,6 @@ dir_read_file_list(const char *root, const char *file_txt) file->linked = pgut_strdup(linked); file->segno = (int) segno; file->n_blocks = (int) n_blocks; - parray_append(files, file); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5d464171b..2b63d82d7 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -35,6 +35,8 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; +/* extra directory to backup */ +char *extradir = NULL; /* common options */ char *backup_id_string_param = NULL; int num_threads = 1; @@ -174,6 +176,7 @@ static pgut_option options[] = /* other options */ { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, + { 's', 152, "extra-directory", &extradir, SOURCE_CMDLINE }, /* archive-push options */ { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 30df34ceb..eda18ede9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -100,6 +100,8 @@ typedef struct pgFile int n_blocks; /* size of the file in blocks, readed during DELTA backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; + bool is_extra; + char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile uint32 lock; /* lock for synchronization of parallel threads */ @@ -298,6 +300,9 @@ extern char backup_instance_path[MAXPGPATH]; extern char *pgdata; extern char arclog_path[MAXPGPATH]; +/* extra directory to backup */ +extern char *extradir; + /* common options */ extern int num_threads; extern bool stream_wal; @@ -442,7 +447,7 @@ extern int pgBackupCompareIdDesc(const void *f1, const void *f2); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root); + bool omit_symlink, bool add_root, bool is_extra); extern void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); @@ -456,7 +461,7 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, bool is_extra); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); diff --git a/src/restore.c b/src/restore.c index 69bac841e..40c78896c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -461,7 +461,7 @@ remove_deleted_files(pgBackup *backup) /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, pgdata, true, true, false); + dir_list_file(files_restored, pgdata, true, true, false, false); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); @@ -768,6 +768,8 @@ restore_files(void *arg) elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) restore_data_file(from_root, pgdata, file, arguments->backup); + else if (file->is_extra) + copy_file(from_root, file->extradir, file); else copy_file(from_root, pgdata, file); From 8550fb0b8a8b650ef8252cc45a891b4c81666833 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Thu, 7 Jun 2018 19:13:09 +0300 Subject: [PATCH 0006/2107] Precommit --- src/backup.c | 9 ++++++--- src/catalog.c | 2 +- src/dir.c | 36 ++++++++++++++++++++++++++---------- src/pg_probackup.h | 5 +++-- src/validate.c | 4 +++- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/backup.c b/src/backup.c index f00bf9506..61bd720eb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -608,7 +608,7 @@ do_backup_instance(void) /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); - dir_list_file(backup_files_list, extradir, true, true, false, true); + dir_list_file(backup_files_list, extradir, true, true, true, true); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -670,11 +670,14 @@ do_backup_instance(void) char database_path[MAXPGPATH]; if (!is_remote_backup) - dir_name = GetRelativePath(file->path, pgdata); + if (file->is_extra) + dir_name = GetRelativePath(file->path, file->extradir); + else + dir_name = GetRelativePath(file->path, pgdata); else dir_name = file->path; - elog(VERBOSE, "Create directory \"%s\"", dir_name); + elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); diff --git a/src/catalog.c b/src/catalog.c index f5884f015..8c1dd72fa 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -356,7 +356,7 @@ pgBackupCreateDir(pgBackup *backup) { int i; char path[MAXPGPATH]; - char *subdirs[] = { DATABASE_DIR, NULL }; + char *subdirs[] = { DATABASE_DIR, EXTRA_DIR, NULL }; pgBackupGetPath(backup, path, lengthof(path), NULL); diff --git a/src/dir.c b/src/dir.c index dd7120f82..4af321ffc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -384,7 +384,11 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, return; } if (add_root) + { + file->extradir = dirname(pgut_strdup(file->path)); + elog(WARNING,"ADD_ROOT Name: %s Path: %s %s",file->name, file->path, file->extradir); parray_append(files, file); + } dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, is_extra); parray_qsort(files, pgFileComparePath); @@ -636,6 +640,13 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, continue; } + /* If it is extra dir, remember it */ + if (is_extra) + { + file->extradir = pgut_strdup(root); +// dirname(file->extradir); + } + /* We add the directory anyway */ if (S_ISDIR(file->mode)) parray_append(files, file); @@ -650,14 +661,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, /* At least add the file */ if (S_ISREG(file->mode)) - { - if (is_extra) - { - file->extradir = pgut_strdup(file->path); - dirname(file->extradir); - } parray_append(files, file); - } /* * If the entry is a directory call dir_list_file_internal() @@ -825,6 +829,10 @@ print_file_list(FILE *out, const parray *files, const char *root) /* omit root directory portion */ if (root && strstr(path, root) == path) path = GetRelativePath(path, root); + else if(file->is_extra) + path = GetRelativePath(path, file->extradir); + else + elog(WARNING,"NOT PGDATA and NOT ExtraDir %s", path); fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," @@ -833,6 +841,9 @@ print_file_list(FILE *out, const parray *files, const char *root) file->is_datafile?1:0, file->is_cfs?1:0, file->crc, deparse_compress_alg(file->compress_alg), file->is_extra?1:0); + if (file->extradir) + fprintf(out, ",\"extradir\":\"%s\"", file->extradir); + if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -985,7 +996,7 @@ get_control_value(const char *str, const char *name, * If root is not NULL, path will be absolute path. */ parray * -dir_read_file_list(const char *root, const char *file_txt) +dir_read_file_list(const char *root, const char *extra_path, const char *file_txt) { FILE *fp; parray *files; @@ -1004,6 +1015,7 @@ dir_read_file_list(const char *root, const char *file_txt) char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; + char extradir_str[MAXPGPATH]; uint64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, @@ -1030,7 +1042,7 @@ dir_read_file_list(const char *root, const char *file_txt) if (root) if (is_extra) - join_path_components(filepath, root, basename(path)); + join_path_components(filepath, extra_path, basename(path)); else join_path_components(filepath, root, path); else @@ -1039,7 +1051,11 @@ dir_read_file_list(const char *root, const char *file_txt) file = pgFileInit(filepath); file->is_extra = is_extra ? true : false; - file->extradir = is_extra ? pgut_strdup(dirname(path)) : NULL; + if (is_extra) + { + get_control_value(buf, "extradir", extradir_str, NULL, true); + file->extradir = extradir_str; + } file->write_size = (size_t) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index eda18ede9..32d44a7d9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -43,7 +43,7 @@ #define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) /* Directory/File names */ -#define DATABASE_DIR "database" +#define DATABASE_DIR "database" #define BACKUPS_DIR "backups" #if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" @@ -59,6 +59,7 @@ #define PG_BACKUP_LABEL_FILE "backup_label" #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define EXTRA_DIR "extradir" /* Direcotry/File permission */ #define DIR_PERMISSION (0700) @@ -454,7 +455,7 @@ extern void list_data_directories(parray *files, const char *path, extern void read_tablespace_map(parray *files, const char *backup_dir); extern void print_file_list(FILE *out, const parray *files, const char *root); -extern parray *dir_read_file_list(const char *root, const char *file_txt); +extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); diff --git a/src/validate.c b/src/validate.c index afc7f07d2..8f973a519 100644 --- a/src/validate.c +++ b/src/validate.c @@ -38,6 +38,7 @@ void pgBackupValidate(pgBackup *backup) { char base_path[MAXPGPATH]; + char extra_path[MAXPGPATH]; char path[MAXPGPATH]; parray *files; bool corrupted = false; @@ -70,8 +71,9 @@ pgBackupValidate(pgBackup *backup) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, path); + files = dir_read_file_list(base_path, extra_path, path); /* setup threads */ for (i = 0; i < parray_num(files); i++) From 15a2bea0a0cb3b62d77050594e535cd621bbfd07 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Thu, 7 Jun 2018 21:22:17 +0300 Subject: [PATCH 0007/2107] Backup one_extra_newplace validate pass --- src/backup.c | 30 +++++++++++++++++++++++------- src/data.c | 5 ++++- src/dir.c | 4 ++-- src/pg_probackup.h | 1 + src/restore.c | 8 ++++++-- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index 61bd720eb..01709f043 100644 --- a/src/backup.c +++ b/src/backup.c @@ -452,6 +452,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; + char extra_path[MAXPGPATH]; char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -516,7 +517,7 @@ do_backup_instance(void) pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ - prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path); /* If lsn is not NULL, only pages with higher lsn will be copied. */ prev_backup_start_lsn = prev_backup->start_lsn; @@ -551,8 +552,8 @@ do_backup_instance(void) strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); + pgBackupGetPath(¤t, database_path, lengthof(database_path),DATABASE_DIR); + pgBackupGetPath(¤t, extra_path, lengthof(extra_path), EXTRA_DIR); /* start stream replication */ if (stream_wal) @@ -667,7 +668,7 @@ do_backup_instance(void) { char dirpath[MAXPGPATH]; char *dir_name; - char database_path[MAXPGPATH]; +// char database_path[MAXPGPATH]; if (!is_remote_backup) if (file->is_extra) @@ -678,10 +679,13 @@ do_backup_instance(void) dir_name = file->path; elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); +// pgBackupGetPath(¤t, database_path, lengthof(database_path), +// DATABASE_DIR); - join_path_components(dirpath, database_path, dir_name); + if (file->is_extra) + join_path_components(dirpath, extra_path, dir_name); + else + join_path_components(dirpath, database_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } @@ -699,6 +703,7 @@ do_backup_instance(void) arg->from_root = pgdata; arg->to_root = database_path; + arg->extra = extra_path; arg->backup_files_list = backup_files_list; arg->prev_backup_filelist = prev_backup_filelist; arg->prev_backup_start_lsn = prev_backup_start_lsn; @@ -2097,6 +2102,17 @@ backup_files(void *arg) continue; } } + else if (file->is_extra) + { + if (!copy_file(arguments->from_root, + arguments->extra, + file)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied extra files to backup", file->path); + continue; + } + } else /* TODO: * Check if file exists in previous backup diff --git a/src/data.c b/src/data.c index a1bdb91a8..81ae99544 100644 --- a/src/data.c +++ b/src/data.c @@ -774,7 +774,10 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) } /* open backup file for write */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (file->is_extra) + join_path_components(to_path, to_root, file->path + strlen(file->extradir) + 1); + else + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, "w"); if (out == NULL) { diff --git a/src/dir.c b/src/dir.c index 4af321ffc..b7854cb7d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -644,7 +644,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, if (is_extra) { file->extradir = pgut_strdup(root); -// dirname(file->extradir); + dirname(file->extradir); } /* We add the directory anyway */ @@ -1042,7 +1042,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx if (root) if (is_extra) - join_path_components(filepath, extra_path, basename(path)); + join_path_components(filepath, extra_path, path); else join_path_components(filepath, root, path); else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 32d44a7d9..7db5db655 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -263,6 +263,7 @@ typedef struct { const char *from_root; const char *to_root; + const char *extra; parray *backup_files_list; parray *prev_backup_filelist; XLogRecPtr prev_backup_start_lsn; diff --git a/src/restore.c b/src/restore.c index 40c78896c..6d8e75edc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -359,6 +359,7 @@ restore_backup(pgBackup *backup) char timestamp[100]; char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; + char extra_path[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; int i; @@ -394,8 +395,9 @@ restore_backup(pgBackup *backup) * Get list of files which need to be restored. */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); + pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, list_path); + files = dir_read_file_list(database_path, extra_path, list_path); /* setup threads */ for (i = 0; i < parray_num(files); i++) @@ -452,11 +454,13 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; + char extra_path[MAXPGPATH]; int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); + pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(pgdata, filelist_path); + files = dir_read_file_list(pgdata, extra_path, filelist_path); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ From 17efc78c8c0f4073266ec0aaeb903aabf7d0d642 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Fri, 8 Jun 2018 18:55:29 +0300 Subject: [PATCH 0008/2107] More then one extradirs? separator is : --- src/backup.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/backup.c b/src/backup.c index 01709f043..d1d020572 100644 --- a/src/backup.c +++ b/src/backup.c @@ -453,6 +453,8 @@ do_backup_instance(void) int i; char database_path[MAXPGPATH]; char extra_path[MAXPGPATH]; + char **extradirs; + uint extradirs_len; char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -466,6 +468,19 @@ do_backup_instance(void) parray *prev_backup_filelist = NULL; elog(LOG, "Database backup start"); + extradirs_len = 0; + char *p; + p = strtok(extradir,":"); + extradirs = palloc((extradirs_len+2)*sizeof(char*)); + while(p!=NULL) + { + extradirs = repalloc(extradirs,(extradirs_len+1)*sizeof(char*)); + extradirs[i] = (char *)palloc(strlen(p) + 1); + strcpy(extradirs[extradirs_len],p); + elog(WARNING,"%s",extradirs[extradirs_len]); + extradirs_len++; + p=strtok(NULL,":"); + } /* Initialize size summary */ current.data_bytes = 0; @@ -611,6 +626,11 @@ do_backup_instance(void) dir_list_file(backup_files_list, extradir, true, true, true, true); +// char *extradirs[] = { DATABASE_DIR, EXTRA_DIR, NULL }; +// /* Append to backup list all files dirictories from extra dirictory option */ +// for (i = 0; extradirs[i]; i++) +// dir_list_file(backup_files_list, extradir, true, true, true, true); + if (current.backup_mode != BACKUP_MODE_FULL) { elog(LOG, "current_tli:%X", current.tli); From 00794b50bf27583c27135c6f315bf179f4271837 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Thu, 14 Jun 2018 12:26:10 +0300 Subject: [PATCH 0009/2107] Fix restore backup --- src/backup.c | 31 ++++++++++++++++--------------- src/data.c | 9 +++++---- src/dir.c | 2 +- src/restore.c | 41 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/backup.c b/src/backup.c index d1d020572..df9ede37c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -453,8 +453,7 @@ do_backup_instance(void) int i; char database_path[MAXPGPATH]; char extra_path[MAXPGPATH]; - char **extradirs; - uint extradirs_len; + char *extradirs[] = { NULL, NULL, NULL, NULL }; char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -462,24 +461,24 @@ do_backup_instance(void) pthread_t backup_threads[num_threads]; backup_files_args *backup_threads_args[num_threads]; bool backup_isok = true; + char *p; pgBackup *prev_backup = NULL; char prev_backup_filelist_path[MAXPGPATH]; parray *prev_backup_filelist = NULL; elog(LOG, "Database backup start"); - extradirs_len = 0; - char *p; + i = 0; p = strtok(extradir,":"); - extradirs = palloc((extradirs_len+2)*sizeof(char*)); while(p!=NULL) { - extradirs = repalloc(extradirs,(extradirs_len+1)*sizeof(char*)); extradirs[i] = (char *)palloc(strlen(p) + 1); - strcpy(extradirs[extradirs_len],p); - elog(WARNING,"%s",extradirs[extradirs_len]); - extradirs_len++; + strcpy(extradirs[i],p); + elog(WARNING,"%s",extradirs[i]); + i++; p=strtok(NULL,":"); + if (i==3) + break; } /* Initialize size summary */ @@ -624,12 +623,14 @@ do_backup_instance(void) /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); - dir_list_file(backup_files_list, extradir, true, true, true, true); +// dir_list_file(backup_files_list, extradir, true, true, true, true); -// char *extradirs[] = { DATABASE_DIR, EXTRA_DIR, NULL }; -// /* Append to backup list all files dirictories from extra dirictory option */ -// for (i = 0; extradirs[i]; i++) -// dir_list_file(backup_files_list, extradir, true, true, true, true); + /* Append to backup list all files dirictories from extra dirictory option */ + for (i = 0; extradirs[i]; i++) + { + elog(WARNING,"%s",extradirs[i]); + dir_list_file(backup_files_list, extradirs[i], true, true, true, true); + } if (current.backup_mode != BACKUP_MODE_FULL) { @@ -2124,7 +2125,7 @@ backup_files(void *arg) } else if (file->is_extra) { - if (!copy_file(arguments->from_root, + if (!copy_file(file->extradir, arguments->extra, file)) { diff --git a/src/data.c b/src/data.c index 81ae99544..6048cb094 100644 --- a/src/data.c +++ b/src/data.c @@ -774,10 +774,11 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) } /* open backup file for write */ - if (file->is_extra) - join_path_components(to_path, to_root, file->path + strlen(file->extradir) + 1); - else - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); +// if (file->is_extra) +// join_path_components(to_path, to_root, file->path + strlen(file->extradir) + 1); +// else +// join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, "w"); if (out == NULL) { diff --git a/src/dir.c b/src/dir.c index b7854cb7d..23a600d90 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1054,7 +1054,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx if (is_extra) { get_control_value(buf, "extradir", extradir_str, NULL, true); - file->extradir = extradir_str; + file->extradir = pgut_strdup(extradir_str); } file->write_size = (size_t) write_size; file->mode = (mode_t) mode; diff --git a/src/restore.c b/src/restore.c index 6d8e75edc..047e3f1a5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -389,7 +389,7 @@ restore_backup(pgBackup *backup) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - restore_directories(pgdata, this_backup_path); +// restore_directories(pgdata, this_backup_path); /* * Get list of files which need to be restored. @@ -399,13 +399,50 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); files = dir_read_file_list(database_path, extra_path, list_path); - /* setup threads */ + /* Restore directories in do_backup_instance way */ + parray_qsort(files, pgFileComparePath); + /* + * Make directories before backup + * and setup threads at the same time + */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; +// char database_path[MAXPGPATH]; + + if (file->is_extra) + dir_name = GetRelativePath(file->path, extra_path); + else + dir_name = GetRelativePath(file->path, database_path); + + elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); +// pgBackupGetPath(&backup, database_path, lengthof(database_path), +// DATABASE_DIR); + + if (file->is_extra) + join_path_components(dirpath, file->extradir, dir_name); + else + join_path_components(dirpath, pgdata, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } + + /* setup threads */ __sync_lock_release(&file->lock); } + /* setup threads */ +// for (i = 0; i < parray_num(files); i++) +// { +// pgFile *file = (pgFile *) parray_get(files, i); +// __sync_lock_release(&file->lock); +// } + /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { From 668d5e5696cd8b68e85a5012a84a832cc3490d22 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Thu, 14 Jun 2018 18:39:30 +0300 Subject: [PATCH 0010/2107] Cleanup --- src/backup.c | 5 ----- src/data.c | 4 ---- src/dir.c | 4 +--- src/restore.c | 19 +------------------ 4 files changed, 2 insertions(+), 30 deletions(-) diff --git a/src/backup.c b/src/backup.c index df9ede37c..9a38dc019 100644 --- a/src/backup.c +++ b/src/backup.c @@ -623,8 +623,6 @@ do_backup_instance(void) /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); -// dir_list_file(backup_files_list, extradir, true, true, true, true); - /* Append to backup list all files dirictories from extra dirictory option */ for (i = 0; extradirs[i]; i++) { @@ -689,7 +687,6 @@ do_backup_instance(void) { char dirpath[MAXPGPATH]; char *dir_name; -// char database_path[MAXPGPATH]; if (!is_remote_backup) if (file->is_extra) @@ -700,8 +697,6 @@ do_backup_instance(void) dir_name = file->path; elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); -// pgBackupGetPath(¤t, database_path, lengthof(database_path), -// DATABASE_DIR); if (file->is_extra) join_path_components(dirpath, extra_path, dir_name); diff --git a/src/data.c b/src/data.c index 6048cb094..a1bdb91a8 100644 --- a/src/data.c +++ b/src/data.c @@ -774,10 +774,6 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) } /* open backup file for write */ -// if (file->is_extra) -// join_path_components(to_path, to_root, file->path + strlen(file->extradir) + 1); -// else -// join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); out = fopen(to_path, "w"); if (out == NULL) diff --git a/src/dir.c b/src/dir.c index 23a600d90..fa2068c55 100644 --- a/src/dir.c +++ b/src/dir.c @@ -386,7 +386,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (add_root) { file->extradir = dirname(pgut_strdup(file->path)); - elog(WARNING,"ADD_ROOT Name: %s Path: %s %s",file->name, file->path, file->extradir); + elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s",file->name, file->path, file->extradir); parray_append(files, file); } @@ -831,8 +831,6 @@ print_file_list(FILE *out, const parray *files, const char *root) path = GetRelativePath(path, root); else if(file->is_extra) path = GetRelativePath(path, file->extradir); - else - elog(WARNING,"NOT PGDATA and NOT ExtraDir %s", path); fprintf(out, "{\"path\":\"%s\", \"size\":\"%lu\",\"mode\":\"%u\"," "\"is_datafile\":\"%u\", \"is_cfs\":\"%u\", \"crc\":\"%u\"," diff --git a/src/restore.c b/src/restore.c index 047e3f1a5..7fc140d1d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -357,7 +357,6 @@ void restore_backup(pgBackup *backup) { char timestamp[100]; - char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; char extra_path[MAXPGPATH]; char list_path[MAXPGPATH]; @@ -384,13 +383,6 @@ restore_backup(pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->start_time); elog(LOG, "restoring database from backup %s", timestamp); - /* - * Restore backup directories. - * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id - */ - pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); -// restore_directories(pgdata, this_backup_path); - /* * Get list of files which need to be restored. */ @@ -401,6 +393,7 @@ restore_backup(pgBackup *backup) /* Restore directories in do_backup_instance way */ parray_qsort(files, pgFileComparePath); + /* * Make directories before backup * and setup threads at the same time @@ -414,7 +407,6 @@ restore_backup(pgBackup *backup) { char dirpath[MAXPGPATH]; char *dir_name; -// char database_path[MAXPGPATH]; if (file->is_extra) dir_name = GetRelativePath(file->path, extra_path); @@ -422,8 +414,6 @@ restore_backup(pgBackup *backup) dir_name = GetRelativePath(file->path, database_path); elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); -// pgBackupGetPath(&backup, database_path, lengthof(database_path), -// DATABASE_DIR); if (file->is_extra) join_path_components(dirpath, file->extradir, dir_name); @@ -436,13 +426,6 @@ restore_backup(pgBackup *backup) __sync_lock_release(&file->lock); } - /* setup threads */ -// for (i = 0; i < parray_num(files); i++) -// { -// pgFile *file = (pgFile *) parray_get(files, i); -// __sync_lock_release(&file->lock); -// } - /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { From 3744048d4e851355fe70772c6d6684a83611d43b Mon Sep 17 00:00:00 2001 From: Ilya Skvortsov Date: Wed, 18 Jul 2018 15:19:39 +0300 Subject: [PATCH 0011/2107] Test start-time using --- tests/__init__.py | 4 +++- tests/time_stamp.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/time_stamp.py diff --git a/tests/__init__.py b/tests/__init__.py index aeeabf2a9..fff85fb51 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,8 @@ ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ - exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ + time_stamp def load_tests(loader, tests, pattern): @@ -43,6 +44,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(validate_test)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(time_stamp)) return suite diff --git a/tests/time_stamp.py b/tests/time_stamp.py new file mode 100644 index 000000000..8f0355524 --- /dev/null +++ b/tests/time_stamp.py @@ -0,0 +1,52 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'time_stamp' + +class CheckTimeStamp(ProbackupTest, unittest.TestCase): + + def test_start_time_format(self): + """Test backup ID changing after start-time editing in backup.control. + We should convert local time in UTC format""" + # Create simple node + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) + show_backup = self.show_pb(backup_dir, 'node') + + i = 0 + while i < 2: + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: + output = "" + for line in f: + if line.startswith('start-time') is True: + if i == 0: + output = output + str(line[:-5])+'+00\''+'\n' + else: + output = output + str(line[:-5]) + '\'' + '\n' + else: + output = output + str(line) + f.close() + + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: + fw.write(output) + fw.flush() + show_backup = show_backup + self.show_pb(backup_dir, 'node') + i += 1 + + self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") + + node.stop() + # Clean after yourself + self.del_test_dir(module_name, fname) From 378ad33033609b76a726039e85f6c0f4abbe9513 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 15 Aug 2018 15:04:43 +0300 Subject: [PATCH 0012/2107] PGPRO-1893: Add define FRONTEND to build Windows distro --- gen_probackup_project.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index b136907be..1752f3bf0 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -127,6 +127,7 @@ sub build_pgprobackup #vvs test my $probackup = $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddDefine('FRONTEND'); $probackup->AddFiles( 'contrib/pg_probackup/src', 'archive.c', From 820cd0fe052f41cd44ef75aad627dbe719938289 Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Fri, 19 Oct 2018 16:33:33 +0300 Subject: [PATCH 0013/2107] Applied patch for zero symbols in CSV for COPY FROM command --- .gitignore | 45 + .travis.yml | 7 + COPYRIGHT | 29 + Makefile | 87 + README.md | 100 + doit.cmd | 1 + doit96.cmd | 1 + gen_probackup_project.pl | 190 ++ msvs/pg_probackup.sln | 28 + msvs/template.pg_probackup.vcxproj | 212 ++ msvs/template.pg_probackup96.vcxproj | 210 ++ msvs/template.pg_probackup_2.vcxproj | 203 ++ src/archive.c | 113 + src/backup.c | 2701 ++++++++++++++++++++++++ src/catalog.c | 915 ++++++++ src/configure.c | 490 +++++ src/data.c | 1407 ++++++++++++ src/delete.c | 464 ++++ src/dir.c | 1491 +++++++++++++ src/fetch.c | 116 + src/help.c | 605 ++++++ src/init.c | 108 + src/merge.c | 526 +++++ src/parsexlog.c | 1039 +++++++++ src/pg_probackup.c | 634 ++++++ src/pg_probackup.h | 620 ++++++ src/restore.c | 919 ++++++++ src/show.c | 500 +++++ src/status.c | 118 ++ src/util.c | 349 +++ src/utils/json.c | 134 ++ src/utils/json.h | 33 + src/utils/logger.c | 621 ++++++ src/utils/logger.h | 54 + src/utils/parray.c | 196 ++ src/utils/parray.h | 35 + src/utils/pgut.c | 2417 +++++++++++++++++++++ src/utils/pgut.h | 238 +++ src/utils/thread.c | 102 + src/utils/thread.h | 35 + src/validate.c | 354 ++++ tests/Readme.md | 24 + tests/__init__.py | 69 + tests/archive.py | 833 ++++++++ tests/auth_test.py | 391 ++++ tests/backup_test.py | 522 +++++ tests/cfs_backup.py | 1161 ++++++++++ tests/cfs_restore.py | 450 ++++ tests/cfs_validate_backup.py | 25 + tests/compression.py | 496 +++++ tests/delete_test.py | 203 ++ tests/delta.py | 1265 +++++++++++ tests/exclude.py | 164 ++ tests/expected/option_help.out | 95 + tests/expected/option_version.out | 1 + tests/false_positive.py | 333 +++ tests/helpers/__init__.py | 2 + tests/helpers/cfs_helpers.py | 91 + tests/helpers/ptrack_helpers.py | 1300 ++++++++++++ tests/init_test.py | 99 + tests/logging.py | 0 tests/merge.py | 454 ++++ tests/option_test.py | 218 ++ tests/page.py | 641 ++++++ tests/pgpro560.py | 98 + tests/pgpro589.py | 80 + tests/ptrack.py | 1600 ++++++++++++++ tests/ptrack_clean.py | 253 +++ tests/ptrack_cluster.py | 268 +++ tests/ptrack_move_to_tablespace.py | 57 + tests/ptrack_recovery.py | 58 + tests/ptrack_truncate.py | 130 ++ tests/ptrack_vacuum.py | 152 ++ tests/ptrack_vacuum_bits_frozen.py | 136 ++ tests/ptrack_vacuum_bits_visibility.py | 67 + tests/ptrack_vacuum_full.py | 140 ++ tests/ptrack_vacuum_truncate.py | 142 ++ tests/replica.py | 293 +++ tests/restore_test.py | 1243 +++++++++++ tests/retention_test.py | 178 ++ tests/show_test.py | 203 ++ tests/validate_test.py | 1730 +++++++++++++++ travis/backup_restore.sh | 66 + win32build.pl | 240 +++ win32build96.pl | 240 +++ win32build_2.pl | 219 ++ 86 files changed, 34877 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 README.md create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 gen_probackup_project.pl create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/archive.c create mode 100644 src/backup.c create mode 100644 src/catalog.c create mode 100644 src/configure.c create mode 100644 src/data.c create mode 100644 src/delete.c create mode 100644 src/dir.c create mode 100644 src/fetch.c create mode 100644 src/help.c create mode 100644 src/init.c create mode 100644 src/merge.c create mode 100644 src/parsexlog.c create mode 100644 src/pg_probackup.c create mode 100644 src/pg_probackup.h create mode 100644 src/restore.c create mode 100644 src/show.c create mode 100644 src/status.c create mode 100644 src/util.c create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h create mode 100644 src/utils/parray.c create mode 100644 src/utils/parray.h create mode 100644 src/utils/pgut.c create mode 100644 src/utils/pgut.h create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 src/validate.c create mode 100644 tests/Readme.md create mode 100644 tests/__init__.py create mode 100644 tests/archive.py create mode 100644 tests/auth_test.py create mode 100644 tests/backup_test.py create mode 100644 tests/cfs_backup.py create mode 100644 tests/cfs_restore.py create mode 100644 tests/cfs_validate_backup.py create mode 100644 tests/compression.py create mode 100644 tests/delete_test.py create mode 100644 tests/delta.py create mode 100644 tests/exclude.py create mode 100644 tests/expected/option_help.out create mode 100644 tests/expected/option_version.out create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/cfs_helpers.py create mode 100644 tests/helpers/ptrack_helpers.py create mode 100644 tests/init_test.py create mode 100644 tests/logging.py create mode 100644 tests/merge.py create mode 100644 tests/option_test.py create mode 100644 tests/page.py create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/ptrack.py create mode 100644 tests/ptrack_clean.py create mode 100644 tests/ptrack_cluster.py create mode 100644 tests/ptrack_move_to_tablespace.py create mode 100644 tests/ptrack_recovery.py create mode 100644 tests/ptrack_truncate.py create mode 100644 tests/ptrack_vacuum.py create mode 100644 tests/ptrack_vacuum_bits_frozen.py create mode 100644 tests/ptrack_vacuum_bits_visibility.py create mode 100644 tests/ptrack_vacuum_full.py create mode 100644 tests/ptrack_vacuum_truncate.py create mode 100644 tests/replica.py create mode 100644 tests/restore_test.py create mode 100644 tests/retention_test.py create mode 100644 tests/show_test.py create mode 100644 tests/validate_test.py create mode 100644 travis/backup_restore.sh create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..02d1512ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Object files +*.o + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.app + +# Dependencies +.deps + +# Binaries +/pg_probackup + +# Generated by test suite +/regression.diffs +/regression.out +/results +/env +/tests/__pycache__/ +/tests/helpers/__pycache__/ +/tests/tmp_dirs/ +/tests/*pyc +/tests/helpers/*pyc + +# Extra files +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c +/src/walmethods.c +/src/walmethods.h diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..35b49ec5b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: required + +services: +- docker + +script: +- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 000000000..49d704724 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,29 @@ +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + +Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +Portions Copyright (c) 1994, The Regents of the University of California + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION + (NTT) nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..56ad1b019 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +PROGRAM = pg_probackup +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o src/utils/thread.o src/merge.o + +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +# !USE_PGXS +else +subdir=contrib/pg_probackup +top_builddir=../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif # USE_PGXS + +ifeq ($(top_srcdir),../..) + ifeq ($(LN_S),ln -s) + srchome=$(top_srcdir)/.. + endif +else +srchome=$(top_srcdir) +endif + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +INCLUDES += src/walmethods.h +endif + +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) +PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} + +all: checksrcdir $(INCLUDES); + +$(PROGRAM): $(OBJS) + +src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ +src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ +src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ +src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ +src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ +src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ +src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ +src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +endif + +ifeq ($(PORTNAME), aix) + CC=xlc_r +endif + +# This rule's only purpose is to give the user instructions on how to pass +# the path to PostgreSQL source tree to the makefile. +.PHONY: checksrcdir +checksrcdir: +ifndef top_srcdir + @echo "You must have PostgreSQL source tree available to compile." + @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" + @echo "variable: \"make top_srcdir=\"" + @exit 1 +endif diff --git a/README.md b/README.md new file mode 100644 index 000000000..1471d648f --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# pg_probackup + +`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. + +The utility is compatible with: +* PostgreSQL 9.5, 9.6, 10; + +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6 +* Postgres Pro Enterprise + +As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: +* Choosing between full and page-level incremental backups to speed up backup and recovery +* Implementing a single backup strategy for multi-server PostgreSQL clusters +* Automatic data consistency checks and on-demand backup validation without actual data recovery +* Managing backups in accordance with retention policy +* Running backup, restore, and validation processes on multiple parallel threads +* Storing backup data in a compressed state to save disk space +* Taking backups from a standby server to avoid extra load on the master server +* Extended logging settings +* Custom commands to simplify WAL log archiving + +To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. + +Using `pg_probackup`, you can take full or incremental backups: +* `Full` backups contain all the data files required to restore the database cluster from scratch. +* `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups: + * `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode could impose read-only I/O pressure equal to `Full` backup. + * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies: +* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). + +## Limitations + +`pg_probackup` currently has the following limitations: +* Creating backups from a remote server is currently not supported. +* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number. +* Microsoft Windows operating system is not supported. +* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. + +## Installation and Setup +### Linux Installation +```shell +#DEB Ubuntu|Debian Packages +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +apt-get install pg-probackup-{10,9.6,9.5} + +#DEB-SRC Packages +echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list +apt-get source pg-probackup-{10,9.6,9.5} + +#RPM Centos Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM RHEL Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM Oracle Linux Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#SRPM Packages +yumdownloader --source pg_probackup-{10,9.6,9.5} +``` + +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: + +```shell +make USE_PGXS=1 PG_CONFIG= top_srcdir= +``` + +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Documentation + +Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). + +## Licence + +This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/). + +## Feedback + +Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_probackup/issues) page. + +## Authors + +Postgres Professional, Moscow, Russia. + +## Credits + +`pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 000000000..b46e3b36d --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 000000000..94d242c99 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 000000000..3ea79e96c --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 000000000..2df4b4042 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 000000000..46a7b2c24 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,212 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 000000000..46e019ba4 --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 000000000..2fc101a42 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 000000000..953a68779 --- /dev/null +++ b/src/archive.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + bool is_compress = false; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " INT64_FORMAT " instead of " INT64_FORMAT, + wal_file_name, instance_name, config->system_identifier, system_id); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + + if (compress_alg == PGLZ_COMPRESS) + elog(ERROR, "pglz compression is not supported"); + +#ifdef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + is_compress = IsXLogFileName(wal_file_name); +#endif + + push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, + overwrite); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", + backup_wal_file_path, absolute_wal_file_path); + get_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} diff --git a/src/backup.c b/src/backup.c new file mode 100644 index 000000000..3aa36c98b --- /dev/null +++ b/src/backup.c @@ -0,0 +1,2701 @@ +/*------------------------------------------------------------------------- + * + * backup.c: backup DB cluster, archived WAL + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" +#include "libpq/pqsignal.h" +#include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" + +static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ +static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +const char *progname = "pg_probackup"; + +/* list of files contained in backup */ +static parray *backup_files_list = NULL; + +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + const char *basedir; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static int is_ptrack_enable = false; +bool is_ptrack_support = false; +bool is_checksum_enabled = false; +bool exclusive_backup = false; + +/* Backup connections */ +static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; +static PGconn *backup_conn_replication = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; +static char server_version_str[100] = ""; + +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; +/* Is pg_stop_backup() was sent */ +static bool pg_stop_backup_is_sent = false; + +/* + * Backup routines + */ +static void backup_cleanup(bool fatal, void *userdata); +static void backup_disconnect(bool fatal, void *userdata); + +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); + +static void do_backup_instance(void); + +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_switch_wal(PGconn *conn); +static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); + +//static void backup_list_file(parray *files, const char *root, ) +static void parse_backup_filelist_filenames(parray *files, const char *root); +static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); +static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); +static void make_pagemap_from_ptrack(parray *files); +static void *StreamLog(void *arg); + +static void get_remote_pgdata_filelist(parray *files); +static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); +static void remote_copy_file(PGconn *conn, pgFile* file); + +/* Ptrack functions */ +static void pg_ptrack_clear(void); +static bool pg_ptrack_support(void); +static bool pg_ptrack_enable(void); +static bool pg_checksum_enable(void); +static bool pg_is_in_recovery(void); +static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid); +static char *pg_ptrack_get_and_clear(Oid tablespace_oid, + Oid db_oid, + Oid rel_oid, + size_t *result_size); +static XLogRecPtr get_last_ptrack_lsn(void); + +/* Check functions */ +static void check_server_version(void); +static void check_system_identifiers(void); +static void confirm_block_size(const char *name, int blcksz); +static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); + +#define disconnect_and_exit(code) \ + { \ + if (conn != NULL) PQfinish(conn); \ + exit(code); \ + } + +/* Fill "files" with data about all the files to backup */ +static void +get_remote_pgdata_filelist(parray *files) +{ + PGresult *res; + int resultStatus; + int i; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, "FILE_BACKUP", PQerrorMessage(backup_conn_replication)); + + res = PQgetResult(backup_conn_replication); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + resultStatus = PQresultStatus(res); + PQclear(res); + elog(ERROR, "cannot start getting FILE_BACKUP filelist: %s, result_status %d", + PQerrorMessage(backup_conn_replication), resultStatus); + } + + if (PQntuples(res) < 1) + elog(ERROR, "%s: no data returned from server", PROGRAM_NAME); + + for (i = 0; i < PQntuples(res); i++) + { + ReceiveFileList(files, backup_conn_replication, res, i); + } + + res = PQgetResult(backup_conn_replication); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "%s: final receive failed: %s", + PROGRAM_NAME, PQerrorMessage(backup_conn_replication)); + } + + PQfinish(backup_conn_replication); +} + +/* + * workhorse for get_remote_pgdata_filelist(). + * Parse received message into pgFile structure. + */ +static void +ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) +{ + char filename[MAXPGPATH]; + pgoff_t current_len_left = 0; + bool basetablespace; + char *copybuf = NULL; + pgFile *pgfile; + + /* What for do we need this basetablespace field?? */ + basetablespace = PQgetisnull(res, rownum, 0); + if (basetablespace) + elog(LOG,"basetablespace"); + else + elog(LOG, "basetablespace %s", PQgetvalue(res, rownum, 1)); + + res = PQgetResult(conn); + + if (PQresultStatus(res) != PGRES_COPY_OUT) + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(conn)); + + while (1) + { + int r; + int filemode; + + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + + if (r == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + /* end of copy */ + if (r == -1) + break; + + /* This must be the header for a new file */ + if (r != 512) + elog(ERROR, "Invalid tar block header size: %d\n", r); + + current_len_left = read_tar_number(©buf[124], 12); + + /* Set permissions on the file */ + filemode = read_tar_number(©buf[100], 8); + + /* First part of header is zero terminated filename */ + snprintf(filename, sizeof(filename), "%s", copybuf); + + pgfile = pgFileInit(filename); + pgfile->size = current_len_left; + pgfile->mode |= filemode; + + if (filename[strlen(filename) - 1] == '/') + { + /* Symbolic link or directory has size zero */ + Assert (pgfile->size == 0); + /* Ends in a slash means directory or symlink to directory */ + if (copybuf[156] == '5') + { + /* Directory */ + pgfile->mode |= S_IFDIR; + } + else if (copybuf[156] == '2') + { + /* Symlink */ +#ifndef WIN32 + pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif + } + else + elog(ERROR, "Unrecognized link indicator \"%c\"\n", + copybuf[156]); + } + else + { + /* regular file */ + pgfile->mode |= S_IFREG; + } + + parray_append(files, pgfile); + } + + if (copybuf != NULL) + PQfreemem(copybuf); +} + +/* read one file via replication protocol + * and write it to the destination subdir in 'backup_path' */ +static void +remote_copy_file(PGconn *conn, pgFile* file) +{ + PGresult *res; + char *copybuf = NULL; + char buf[32768]; + FILE *out; + char database_path[MAXPGPATH]; + char to_path[MAXPGPATH]; + bool skip_padding = false; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + join_path_components(to_path, database_path, file->path); + + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + INIT_CRC32C(file->crc); + + /* read from stream and write to backup file */ + while (1) + { + int row_length; + int errno_tmp; + int write_buffer_size = 0; + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + row_length = PQgetCopyData(conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + if (row_length == -1) + break; + + if (!skip_padding) + { + write_buffer_size = Min(row_length, sizeof(buf)); + memcpy(buf, copybuf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); + + /* TODO calc checksum*/ + if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) + { + errno_tmp = errno; + /* oops */ + FIN_CRC32C(file->crc); + fclose(out); + PQfinish(conn); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + file->read_size += write_buffer_size; + } + if (file->read_size >= file->size) + { + skip_padding = true; + } + } + + res = PQgetResult(conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); + } + + file->write_size = (int64) file->read_size; + FIN_CRC32C(file->crc); + + fclose(out); +} + +/* + * Take a remote backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + */ +static void * +remote_backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; + + for (i = 0; i < n_backup_files_list; i++) + { + char *query_str; + PGresult *res; + char *copybuf = NULL; + pgFile *file; + int row_length; + + file = (pgFile *) parray_get(arguments->files_list, i); + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + file_backup_conn = pgut_connect_replication(pgut_dbname); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); + + if (PQsendQuery(file_backup_conn, query_str) == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, query_str, PQerrorMessage(file_backup_conn)); + + res = PQgetResult(file_backup_conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + PQfinish(file_backup_conn); + continue; + } + + if (PQresultStatus(res) != PGRES_COPY_OUT) + { + PQclear(res); + PQfinish(file_backup_conn); + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(file_backup_conn)); + } + + /* read the header of the file */ + row_length = PQgetCopyData(file_backup_conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(file_backup_conn)); + + /* end of copy TODO handle it */ + if (row_length == -1) + elog(ERROR, "Unexpected end of COPY data"); + + if(row_length != 512) + elog(ERROR, "Invalid tar block header size: %d\n", row_length); + file->size = read_tar_number(©buf[124], 12); + + /* receive the data from stream and write to backup file */ + remote_copy_file(file_backup_conn, file); + + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", + file->path, file->write_size); + PQfinish(file_backup_conn); + } + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Take a backup of a single postgresql instance. + * Move files from 'pgdata' to a subdirectory in 'backup_path'. + */ +static void +do_backup_instance(void) +{ + int i; + char database_path[MAXPGPATH]; + char dst_backup_path[MAXPGPATH]; + char label[1024]; + XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; + + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgBackup *prev_backup = NULL; + parray *prev_backup_filelist = NULL; + + elog(LOG, "Database backup start"); + + /* Initialize size summary */ + current.data_bytes = 0; + + /* Obtain current timeline */ + if (is_remote_backup) + { + char *sysidentifier; + TimeLineID starttli; + XLogRecPtr startpos; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + /* Check replication prorocol connection */ + if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) + elog(ERROR, "Failed to send command for remote backup"); + +// TODO implement the check +// if (&sysidentifier != system_identifier) +// elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", +// system_identifier, sysidentifier); + + current.tli = starttli; + + PQfinish(backup_conn_replication); + } + else + current.tli = get_current_timeline(false); + + /* + * In incremental backup mode ensure that already-validated + * backup on current timeline exists and get its filelist. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + { + parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; + + /* get list of backups already taken */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + if (prev_backup == NULL) + elog(ERROR, "Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one."); + parray_free(backup_list); + + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); + /* Files of previous backup needed by DELTA backup */ + prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + + /* If lsn is not NULL, only pages with higher lsn will be copied. */ + prev_backup_start_lsn = prev_backup->start_lsn; + current.parent_backup = prev_backup->start_time; + + pgBackupWriteBackupControlFile(¤t); + } + + /* + * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to + * stort_backup LSN of previous backup + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(); + + if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control " UINT64_FORMAT " differs from STOP LSN of previous backup " + UINT64_FORMAT ".\n" + "Create new full backup before an incremental one.", + ptrack_lsn, prev_backup->stop_lsn); + } + } + + /* Clear ptrack files for FULL and PAGE backup */ + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) + pg_ptrack_clear(); + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + pg_start_backup(label, smooth_checkpoint, ¤t); + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + stream_thread_arg.basedir = dst_backup_path; + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(pgut_dbname); + + if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) + { + PQfinish(stream_thread_arg.conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtaining start LSN position and current timeline ID + * at the same time, necessary if not valid data can be found in the + * existing output directory. + */ + if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL)) + { + PQfinish(stream_thread_arg.conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* By default there are some error */ + stream_thread_arg.ret = 1; + + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + } + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (is_remote_backup) + get_remote_pgdata_filelist(backup_files_list); + else + dir_list_file(backup_files_list, pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, pgdata); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(LOG, "current_tli:%X", current.tli); + elog(LOG, "prev_backup->start_lsn: %X/%X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); + elog(LOG, "current.start_lsn: %X/%X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* + * Build page mapping in incremental mode. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + { + /* + * Build the page map. Obtain information about changed pages + * reading WAL segments present in archives up to the point + * where this backup has started. + */ + extractPageMap(arclog_path, current.tli, xlog_seg_size, + prev_backup->start_lsn, current.start_lsn, + /* + * For backup from master wait for previous segment. + * For backup from replica wait for current segment. + */ + !current.from_replica, backup_files_list); + } + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + /* + * Build the page map from ptrack information. + */ + make_pagemap_from_ptrack(backup_files_list); + } + + /* + * Make directories before backup and setup threads at the same time + */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char database_path[MAXPGPATH]; + + if (!is_remote_backup) + dir_name = GetRelativePath(file->path, pgdata); + else + dir_name = file->path; + + elog(VERBOSE, "Create directory \"%s\"", dir_name); + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + join_path_components(dirpath, database_path, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } + + /* setup threads */ + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + elog(LOG, "Start transfering data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + if (!is_remote_backup) + pthread_create(&threads[i], NULL, backup_files, arg); + else + pthread_create(&threads[i], NULL, remote_backup_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(LOG, "Data files are transfered"); + else + elog(ERROR, "Data files transferring failed"); + + /* clean previous backup file list */ + if (prev_backup_filelist) + { + parray_walk(prev_backup_filelist, pgFileFree); + parray_free(prev_backup_filelist); + } + + /* Notify end of backup */ + pg_stop_backup(¤t); + + /* Add archived xlog files into the list of files of this backup */ + if (stream_wal) + { + parray *xlog_files_list; + char pg_xlog_path[MAXPGPATH]; + + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + + for (i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + + if (S_ISREG(file->mode)) + calc_file_checksum(file); + /* Remove file path root prefix*/ + if (strstr(file->path, database_path) == file->path) + { + char *ptr = file->path; + + file->path = pstrdup(GetRelativePath(ptr, database_path)); + free(ptr); + } + } + + /* Add xlog files into the list of backed up files */ + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + } + + /* Print the list of files to backup catalog */ + pgBackupWriteFileList(¤t, backup_files_list, pgdata); + + /* Compute summary of size of regular files in the backup */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; + + /* Count the amount of the data actually copied */ + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; + } + + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; +} + +/* + * Entry point of pg_probackup BACKUP subcommand. + */ +int +do_backup(time_t start_time) +{ + + /* PGDATA and BACKUP_MODE are always required */ + if (pgdata == NULL) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "required parameter not specified: BACKUP_MODE " + "(-b, --backup-mode)"); + + /* Create connection for PostgreSQL */ + backup_conn = pgut_connect(pgut_dbname); + pgut_atexit_push(backup_disconnect, NULL); + + current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(backup_conn)) + elog(ERROR, "Failed to retreive wal_segment_size"); +#endif + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + + /* Confirm data block size and xlog block size are compatible */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + current.from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + + /* TODO fix it for remote backup*/ + if (!is_remote_backup) + current.checksum_version = get_data_checksum_version(true); + + is_checksum_enabled = pg_checksum_enable(); + + if (is_checksum_enabled) + elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + "Data block corruption will be detected"); + else + elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " + "pg_probackup have no way to detect data block corruption without them. " + "Reinitialize PGDATA with option '--data-checksums'."); + + StrNCpy(current.server_version, server_version_str, + sizeof(current.server_version)); + current.stream = stream_wal; + + is_ptrack_support = pg_ptrack_support(); + if (is_ptrack_support) + { + is_ptrack_enable = pg_ptrack_enable(); + } + + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (!is_ptrack_support) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else + { + if(!is_ptrack_enable) + elog(ERROR, "Ptrack is disabled"); + } + } + + if (current.from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, master_db, master_user); + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + /* TODO fix it for remote backup */ + if (!is_remote_backup) + check_system_identifiers(); + + + /* Start backup. Update backup status. */ + current.status = BACKUP_STATUS_RUNNING; + current.start_time = start_time; + + /* Create backup directory and BACKUP_CONTROL_FILE */ + if (pgBackupCreateDir(¤t)) + elog(ERROR, "cannot create backup directory"); + pgBackupWriteBackupControlFile(¤t); + + elog(LOG, "Backup destination is initialized"); + + /* set the error processing function for the backup process */ + pgut_atexit_push(backup_cleanup, NULL); + + /* backup data */ + do_backup_instance(); + pgut_atexit_pop(backup_cleanup, NULL); + + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = xlog_seg_size * + (current.stop_lsn / xlog_seg_size - + current.start_lsn / xlog_seg_size + 1); + } + + /* Backup is done. Update backup status */ + current.end_time = time(NULL); + current.status = BACKUP_STATUS_DONE; + pgBackupWriteBackupControlFile(¤t); + + //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", + // current.data_bytes); + + pgBackupValidate(¤t); + + elog(INFO, "Backup %s completed", base36enc(current.start_time)); + + /* + * After successfil backup completion remove backups + * which are expired according to retention policies + */ + if (delete_expired || delete_wal) + do_retention_purge(); + + return 0; +} + +/* + * Confirm that this server version is supported + */ +static void +check_server_version(void) +{ + PGresult *res; + + /* confirm server version */ + server_version = PQserverVersion(backup_conn); + + if (server_version == 0) + elog(ERROR, "Unknown server version %d", server_version); + + if (server_version < 100000) + sprintf(server_version_str, "%d.%d", + server_version / 10000, + (server_version / 100) % 100); + else + sprintf(server_version_str, "%d", + server_version / 10000); + + if (server_version < 90500) + elog(ERROR, + "server version is %s, must be %s or higher", + server_version_str, "9.5"); + + if (current.from_replica && server_version < 90600) + elog(ERROR, + "server version is %s, must be %s or higher for backup from replica", + server_version_str, "9.6"); + + res = pgut_execute_extended(backup_conn, "SELECT pgpro_edition()", + 0, NULL, true, true); + + /* + * Check major version of connected PostgreSQL and major version of + * compiled PostgreSQL. + */ +#ifdef PGPRO_VERSION + if (PQresultStatus(res) == PGRES_FATAL_ERROR) + /* It seems we connected to PostgreSQL (not Postgres Pro) */ + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && + strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, + server_version_str, PQgetvalue(res, 0, 0)); +#else + if (PQresultStatus(res) != PGRES_FATAL_ERROR) + /* It seems we connected to Postgres Pro (not PostgreSQL) */ + elog(ERROR, "%s was built with PostgreSQL %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, + server_version_str, PQgetvalue(res, 0, 0)); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, server_version_str); +#endif + + PQclear(res); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600 || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK; +} + +/* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + * All system identifiers must be equal. + */ +static void +check_system_identifiers(void) +{ + uint64 system_id_conn; + uint64 system_id_pgdata; + + system_id_pgdata = get_system_identifier(pgdata); + system_id_conn = get_remote_system_identifier(backup_conn); + + if (system_id_conn != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but connected instance system id is " UINT64_FORMAT, + system_identifier, system_id_conn); + if (system_id_pgdata != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but target backup directory system id is " UINT64_FORMAT, + system_identifier, system_id_pgdata); +} + +/* + * Ensure that target backup database is initialized with + * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. + */ +static void +confirm_block_size(const char *name, int blcksz) +{ + PGresult *res; + char *endp; + int block_size; + + res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); + if (PQntuples(res) != 1 || PQnfields(res) != 1) + elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); + + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); + if ((endp && *endp) || block_size != blcksz) + elog(ERROR, + "%s(%d) is not compatible(%d expected)", + name, block_size, blcksz); + + PQclear(res); +} + +/* + * Notify start of backup to PostgreSQL server. + */ +static void +pg_start_backup(const char *label, bool smooth, pgBackup *backup) +{ + PGresult *res; + const char *params[2]; + uint32 xlogid; + uint32 xrecoff; + PGconn *conn; + + params[0] = label; + + /* For replica we call pg_start_backup() on master */ + conn = (backup->from_replica) ? master_conn : backup_conn; + + /* 2nd argument is 'fast'*/ + params[1] = smooth ? "false" : "true"; + if (!exclusive_backup) + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2, false)", + 2, + params); + else + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2)", + 2, + params); + + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + */ + pg_switch_wal(conn); + + if (!stream_wal) + { + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + /* In PAGE mode wait for current segment... */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + wait_wal_lsn(backup->start_lsn, false); + /* ...for others wait for previous segment */ + else + wait_wal_lsn(backup->start_lsn, true); + } + + /* Wait for start_lsn to be replayed by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(backup->start_lsn, true); +} + +/* + * Switch to a new WAL segment. It should be called only for master. + */ +static void +pg_switch_wal(PGconn *conn) +{ + PGresult *res; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); + PQclear(res); + + if (server_version >= 100000) + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL); + else + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL); + + PQclear(res); +} + +/* + * Check if the instance supports ptrack + * TODO Maybe we should rather check ptrack_version()? + */ +static bool +pg_ptrack_support(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, + "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + + res_db = pgut_execute(backup_conn, + "SELECT pg_catalog.ptrack_version()", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + + /* Now we support only ptrack versions upper than 1.5 */ + if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0) + { + elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); + PQclear(res_db); + return false; + } + + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_ptrack_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show ptrack_enable", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_checksum_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show data_checksums", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if target instance is replica */ +static bool +pg_is_in_recovery(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); + + if (PQgetvalue(res_db, 0, 0)[0] == 't') + { + PQclear(res_db); + return true; + } + PQclear(res_db); + return false; +} + +/* Clear ptrack files in all databases of the instance we connected to */ +static void +pg_ptrack_clear(void) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(dbname); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); +} + +static bool +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) +{ + char *params[2]; + char *dbname; + PGresult *res_db; + PGresult *res; + bool result; + + params[0] = palloc(64); + params[1] = palloc(64); + + sprintf(params[0], "%i", dbOid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return false; + + dbname = PQgetvalue(res_db, 0, 0); + + /* Always backup all files from template0 database */ + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return true; + } + PQclear(res_db); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); + + if (!parse_bool(PQgetvalue(res, 0, 0), &result)) + elog(ERROR, + "result of pg_ptrack_get_and_clear_db() is invalid: %s", + PQgetvalue(res, 0, 0)); + + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* Read and clear ptrack files of the target relation. + * Result is a bytea ptrack map of all segments of the target relation. + * case 1: we know a tablespace_oid, db_oid, and rel_filenode + * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) + * case 3: we know only rel_filenode (because file in pg_global) + */ +static char * +pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, + size_t *result_size) +{ + PGconn *tmp_conn; + PGresult *res_db, + *res; + char *params[2]; + char *result; + char *val; + + params[0] = palloc(64); + params[1] = palloc(64); + + /* regular file (not in directory 'global') */ + if (db_oid != 0) + { + char *dbname; + + sprintf(params[0], "%i", db_oid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return NULL; + + dbname = PQgetvalue(res_db, 0, 0); + + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return NULL; + } + + tmp_conn = pgut_connect(dbname); + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", + dbname, tablespace_oid, rel_filenode); + PQclear(res_db); + pgut_disconnect(tmp_conn); + } + /* file in directory 'global' */ + else + { + /* + * execute ptrack_get_and_clear for relation in pg_global + * Use backup_conn, cause we can do it from any database. + */ + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", + rel_filenode); + } + + val = PQgetvalue(res, 0, 0); + + /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. + * It should be fixed in future ptrack releases, but till then we + * can parse it. + */ + if (strcmp("x", val+1) == 0) + { + /* Ptrack file is missing */ + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_wal' directory. + * + * If 'wait_prev_segment' wait for previous segment. + */ +static void +wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment) +{ + TimeLineID tli; + XLogSegNo targetSegNo; + char wal_dir[MAXPGPATH], + wal_segment_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + bool file_exists = false; + uint32 try_count = 0, + timeout; + +#ifdef HAVE_LIBZ + char gz_wal_segment_path[MAXPGPATH]; +#endif + + tli = get_current_timeline(false); + + /* Compute the name of the WAL file containig requested LSN */ + GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); + if (wait_prev_segment) + targetSegNo--; + GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); + + if (stream_wal) + { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_path, wal_dir, wal_segment); + + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + if (wait_prev_segment) + elog(LOG, "Looking for segment: %s", wal_segment); + else + elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + +#ifdef HAVE_LIBZ + snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", + wal_segment_path); +#endif + + /* Wait until target LSN is archived or streamed */ + while (true) + { + if (!file_exists) + { + file_exists = fileExists(wal_segment_path); + + /* Try to find compressed WAL file */ + if (!file_exists) + { +#ifdef HAVE_LIBZ + file_exists = fileExists(gz_wal_segment_path); + if (file_exists) + elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); +#endif + } + else + elog(LOG, "Found WAL segment: %s", wal_segment_path); + } + + if (file_exists) + { + /* Do not check LSN for previous WAL segment */ + if (wait_prev_segment) + return; + + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli, + xlog_seg_size)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli, + xlog_seg_size))) + /* Target LSN was found */ + { + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return; + } + } + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for WAL archiving"); + try_count++; + + /* Inform user if WAL segment is absent in first attempt */ + if (try_count == 1) + { + if (wait_prev_segment) + elog(INFO, "Wait for WAL segment %s to be archived", + wal_segment_path); + else + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + } + + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + /* If WAL segment doesn't exist or we wait for previous segment */ + else + elog(ERROR, + "Switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } + } +} + +/* + * Wait for target 'lsn' on replica instance from master. + */ +static void +wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) +{ + uint32 try_count = 0; + + while (true) + { + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr replica_lsn; + + /* + * For lsn from pg_start_backup() we need it to be replayed on replica's + * data. + */ + if (is_start_backup) + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_replay_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_replay_location()", + 0, NULL); + } + /* + * For lsn from pg_stop_backup() we need it only to be received by + * replica and fsync()'ed on WAL segment. + */ + else + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()", + 0, NULL); + } + + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + replica_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* target lsn was replicated */ + if (replica_lsn >= lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for target LSN"); + try_count++; + + /* Inform user if target lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for target LSN %X/%X to be received by replica", + (uint32) (lsn >> 32), (uint32) lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Target LSN %X/%X could not be recevied by replica " + "in %d seconds", + (uint32) (lsn >> 32), (uint32) lsn, + replica_timeout); + } +} + +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(pgBackup *backup) +{ + PGconn *conn; + PGresult *res; + PGresult *tablespace_map_content = NULL; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + int pg_stop_backup_timeout = 0; + char path[MAXPGPATH]; + char backup_label[MAXPGPATH]; + FILE *fp; + pgFile *file; + size_t len; + char *val = NULL; + char *stop_backup_query = NULL; + + /* + * We will use this values if there are no transactions between start_lsn + * and stop_lsn. + */ + time_t recovery_time; + TransactionId recovery_xid; + + if (!backup_in_progress) + elog(ERROR, "backup is not in progress"); + + /* For replica we call pg_stop_backup() on master */ + conn = (current.from_replica) ? master_conn : backup_conn; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", + 0, NULL); + PQclear(res); + + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + + if (!current.from_replica) + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup->start_time)); + else + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + base36enc(backup->start_time)); + params[0] = name; + + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + + if (!pg_stop_backup_is_sent) + { + bool sent = false; + + if (!exclusive_backup) + { + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + */ + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + + } + else + { + + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn"; + } + + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + pg_stop_backup_is_sent = true; + if (!sent) + elog(ERROR, "Failed to send pg_stop_backup query"); + } + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + if (pg_stop_backup_is_sent && !in_cleanup) + { + while (1) + { + if (!PQconsumeInput(conn) || PQisBusy(conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); + + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(conn); + break; + } + } + + /* Check successfull execution of pg_stop_backup() */ + if (!res) + elog(ERROR, "pg_stop backup() failed"); + else + { + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + break; + default: + elog(ERROR, "query failed: %s query was: %s", + PQerrorMessage(conn), stop_backup_query); + } + elog(INFO, "pg_stop backup() successfully executed"); + } + + backup_in_progress = false; + + /* Extract timeline and LSN from results of pg_stop_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 2), &xlogid, &xrecoff); + /* Calculate LSN */ + stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) + { + Assert(PQnfields(res) >= 4); + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + + /* Write backup_label */ + join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); + fp = fopen(backup_label, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open backup label file \"%s\": %s", + backup_label, strerror(errno)); + + len = strlen(PQgetvalue(res, 0, 3)); + if (fwrite(PQgetvalue(res, 0, 3), 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write backup label file \"%s\": %s", + backup_label, strerror(errno)); + + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } + } + + if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) + elog(ERROR, + "result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(res, 0, 0)); + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) + elog(ERROR, + "result of current_timestamp is invalid: %s", + PQgetvalue(res, 0, 1)); + + /* Get content for tablespace_map from stop_backup results + * in case of non-exclusive backup + */ + if (!exclusive_backup) + val = PQgetvalue(res, 0, 4); + + /* Write tablespace_map */ + if (!exclusive_backup && val && strlen(val) > 0) + { + char tablespace_map[MAXPGPATH]; + + join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); + fp = fopen(tablespace_map, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + len = strlen(val); + if (fwrite(val, 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + if (backup_files_list) + { + file = pgFileNew(tablespace_map, true); + if (S_ISREG(file->mode)) + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_TABLESPACE_MAP_FILE); + parray_append(backup_files_list, file); + } + } + + if (tablespace_map_content) + PQclear(tablespace_map_content); + PQclear(res); + + if (stream_wal) + { + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); + } + } + + /* Fill in fields if that is the correct end of backup. */ + if (backup != NULL) + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + + /* Wait for stop_lsn to be received by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(stop_backup_lsn, false); + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn, false); + + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; + + backup->tli = get_current_timeline(false); + backup->stop_lsn = stop_backup_lsn; + + elog(LOG, "Getting the Recovery Time from WAL"); + + if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time, &backup->recovery_xid)) + { + backup->recovery_time = recovery_time; + backup->recovery_xid = recovery_xid; + } + } +} + +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * Notify end of backup to server when "backup_label" is in the root directory + * of the DB cluster. + * Also update backup status to ERROR when the backup is not finished. + */ +static void +backup_cleanup(bool fatal, void *userdata) +{ + /* + * Update status of backup in BACKUP_CONTROL_FILE to ERROR. + * end_time != 0 means backup finished + */ + if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) + { + elog(WARNING, "Backup %s is running, setting its status to ERROR", + base36enc(current.start_time)); + current.end_time = time(NULL); + current.status = BACKUP_STATUS_ERROR; + pgBackupWriteBackupControlFile(¤t); + } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(WARNING, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } +} + +/* + * Disconnect backup connection during quit pg_probackup. + */ +static void +backup_disconnect(bool fatal, void *userdata) +{ + pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); +} + +/* + * Take a backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. + * In incremental backup mode, copy only files or datafiles' pages changed after + * previous backup. + */ +static void * +backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + + /* backup a file */ + for (i = 0; i < n_backup_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + elog(VERBOSE, "Copying file: \"%s\" ", file->path); + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + file->write_size = BYTES_INVALID; + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to backup \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* We have already copied all directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char *relative; + pgFile key; + pgFile **prev_file; + + relative = GetRelativePath(file->path, arguments->from_root); + key.path = relative; + + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; + } + /* copy the file into backup */ + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + + /* backup block by block if datafile AND not compressed by cfs*/ + if (!backup_data_file(arguments, to_path, file, + arguments->prev_start_lsn, + current.backup_mode, + compress_alg, compress_level)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + } + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + file->path, file->write_size); + } + else + elog(LOG, "unexpected file type %d", buf.st_mode); + } + + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Extract information about files in backup_list parsing their names: + * - remove temp tables from the list + * - remove unlogged tables from the list (leave the _init fork) + * - set flags for database directories + * - set flags for datafiles + */ +static void +parse_backup_filelist_filenames(parray *files, const char *root) +{ + size_t i = 0; + Oid unlogged_file_reloid = 0; + + while (i < parray_num(files)) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *relative; + int sscanf_result; + + relative = GetRelativePath(file->path, root); + + if (S_ISREG(file->mode) && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) + { + /* + * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + * Legal only in case of 'pg_compression' + */ + if (strcmp(file->name, "pg_compression") == 0) + { + Oid tblspcOid; + Oid dbOid; + char tmp_rel_path[MAXPGPATH]; + /* + * Check that the file is located under + * TABLESPACE_VERSION_DIRECTORY + */ + sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u", + &tblspcOid, tmp_rel_path, &dbOid); + + /* Yes, it is */ + if (sscanf_result == 2 && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + set_cfs_datafiles(files, root, relative, i); + } + } + + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->forkName, "init") == 0) + { + /* + * Do not backup files of unlogged relations. + * scan filelist backward and exclude these files. + */ + int unlogged_file_num = i - 1; + pgFile *unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + + unlogged_file_reloid = file->relOid; + + while (unlogged_file_num >= 0 && + (unlogged_file_reloid != 0) && + (unlogged_file->relOid == unlogged_file_reloid)) + { + pgFileFree(unlogged_file); + parray_remove(files, unlogged_file_num); + + unlogged_file_num--; + i--; + + unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + } + } + } + + i++; + } +} + +/* If file is equal to pg_compression, then we consider this tablespace as + * cfs-compressed and should mark every file in this tablespace as cfs-file + * Setting is_cfs is done via going back through 'files' set every file + * that contain cfs_tablespace in his path as 'is_cfs' + * Goings back through array 'files' is valid option possible because of current + * sort rules: + * tblspcOid/TABLESPACE_VERSION_DIRECTORY + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + */ +static void +set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +{ + int len; + int p; + pgFile *prev_file; + char *cfs_tblspc_path; + char *relative_prev_file; + + cfs_tblspc_path = strdup(relative); + if(!cfs_tblspc_path) + elog(ERROR, "Out of memory"); + len = strlen("/pg_compression"); + cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; + elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + + for (p = (int) i; p >= 0; p--) + { + prev_file = (pgFile *) parray_get(files, (size_t) p); + relative_prev_file = GetRelativePath(prev_file->path, root); + + elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file); + + if (strstr(relative_prev_file, cfs_tblspc_path) != NULL) + { + if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + { + elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", + relative_prev_file, prev_file->name); + prev_file->is_cfs = true; + } + } + else + { + elog(VERBOSE, "Breaking on %s", relative_prev_file); + break; + } + } + free(cfs_tblspc_path); +} + +/* + * Find pgfile by given rnode in the backup_files_list + * and add given blkno to its pagemap. + */ +void +process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) +{ + char *path; + char *rel_path; + BlockNumber blkno_inseg; + int segno; + pgFile **file_item; + pgFile f; + + segno = blkno / RELSEG_SIZE; + blkno_inseg = blkno % RELSEG_SIZE; + + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + else + path = psprintf("%s/%s", pgdata, rel_path); + + pg_free(rel_path); + + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); + + /* + * If we don't have any record of this file in the file map, it means + * that it's a relation that did not have much activity since the last + * backup. We can safely ignore it. If it is a new relation file, the + * backup would simply copy it as-is. + */ + if (file_item) + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_lock(&backup_pagemap_mutex); + + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } + + pg_free(path); +} + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * NOTE we rely on the fact that provided parray is sorted by file->path. + */ +static void +make_pagemap_from_ptrack(parray *files) +{ + size_t i; + Oid dbOid_with_ptrack_init = 0; + Oid tblspcOid_with_ptrack_init = 0; + char *ptrack_nonparsed = NULL; + size_t ptrack_nonparsed_size = 0; + + elog(LOG, "Compiling pagemap"); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + size_t start_addr; + + /* + * If there is a ptrack_init file in the database, + * we must backup all its files, ignoring ptrack files for relations. + */ + if (file->is_database) + { + char *filename = strrchr(file->path, '/'); + + Assert(filename != NULL); + filename++; + + /* + * The function pg_ptrack_get_and_clear_db returns true + * if there was a ptrack_init file. + * Also ignore ptrack files for global tablespace, + * to avoid any possible specific errors. + */ + if ((file->tblspcOid == GLOBALTABLESPACE_OID) || + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid)) + { + dbOid_with_ptrack_init = file->dbOid; + tblspcOid_with_ptrack_init = file->tblspcOid; + } + } + + if (file->is_datafile) + { + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) + { + /* ignore ptrack if ptrack_init exists */ + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + file->pagemap_isabsent = true; + continue; + } + + /* get ptrack bitmap once for all segments of the file */ + if (file->segno == 0) + { + /* release previous value */ + pg_free(ptrack_nonparsed); + ptrack_nonparsed_size = 0; + + ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, + file->relOid, &ptrack_nonparsed_size); + } + + if (ptrack_nonparsed != NULL) + { + /* + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cutted out. + * Compute the beginning of the ptrack map related to this segment + * + * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 + * RELSEG_SIZE. Number of Pages per segment: 131072 + * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed + * to keep track on one relsegment: 16384 + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; + + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) + { + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + else + { + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } + } + else + { + /* + * If ptrack file is missing, try to copy the entire file. + * It can happen in two cases: + * - files were created by commands that bypass buffer manager + * and, correspondingly, ptrack mechanism. + * i.e. CREATE DATABASE + * - target relation was deleted. + */ + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + } + } + elog(LOG, "Pagemap compiled"); +// res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL, true); +// PQclear(pgut_execute(backup_conn, "CHECKPOINT;", 0, NULL, true)); +} + + +/* + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + XLogRecPtr startpos; + TimeLineID starttli; + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * We must use startpos as start_lsn from start_backup + */ + startpos = current.start_lsn; + starttli = current.tli; + + /* + * Always start streaming at the beginning of a segment + */ + startpos -= startpos % xlog_seg_size; + + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + + /* + * Start the replication + */ + elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = startpos; + ctl.timeline = starttli; + ctl.sysidentifier = NULL; + +#if PG_VERSION_NUM >= 100000 + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; +#else + ctl.basedir = (char *) stream_arg->basedir; +#endif + + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + + if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + elog(ERROR, "Problem in receivexlog"); + +#if PG_VERSION_NUM >= 100000 + if (!ctl.walmethod->finish()) + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); +#endif + } +#else + if(ReceiveXlogStream(stream_arg->conn, startpos, starttli, NULL, + (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + elog(ERROR, "Problem in receivexlog"); +#endif + + elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ +static XLogRecPtr +get_last_ptrack_lsn(void) + +{ + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr lsn; + + res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL); + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + return lsn; +} + +char * +pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, + Oid tblsOid, + Oid relOid, + BlockNumber blknum, + size_t *result_size) +{ + PGresult *res; + char *params[4]; + char *result; + + params[0] = palloc(64); + params[1] = palloc(64); + params[2] = palloc(64); + params[3] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", tblsOid); + sprintf(params[1], "%i", dbOid); + sprintf(params[2], "%i", relOid); + sprintf(params[3], "%u", blknum); + + if (arguments->backup_conn == NULL) + { + arguments->backup_conn = pgut_connect(pgut_dbname); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", + 4, (const char **)params, true); + + if (PQnfields(res) != 1) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + if (PQgetisnull(res, 0, 0)) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + + PQclear(res); + + pfree(params[0]); + pfree(params[1]); + pfree(params[2]); + pfree(params[3]); + + return result; +} diff --git a/src/catalog.c b/src/catalog.c new file mode 100644 index 000000000..f3f752779 --- /dev/null +++ b/src/catalog.c @@ -0,0 +1,915 @@ +/*------------------------------------------------------------------------- + * + * catalog.c: backup catalog operation + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; +static pgBackup *readBackupControlFile(const char *path); + +static bool exit_hook_registered = false; +static char lock_file[MAXPGPATH]; + +static void +unlink_lock_atexit(void) +{ + int res; + res = unlink(lock_file); + if (res != 0 && res != ENOENT) + elog(WARNING, "%s: %s", lock_file, strerror(errno)); +} + +/* + * Create a lockfile. + */ +void +catalog_lock(void) +{ + int fd; + char buffer[MAXPGPATH * 2 + 256]; + int ntries; + int len; + int encoded_pid; + pid_t my_pid, + my_p_pid; + + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); + + /* + * If the PID in the lockfile is our own PID or our parent's or + * grandparent's PID, then the file must be stale (probably left over from + * a previous system boot cycle). We need to check this because of the + * likelihood that a reboot will assign exactly the same PID as we had in + * the previous reboot, or one that's only one or two counts larger and + * hence the lockfile's PID now refers to an ancestor shell process. We + * allow pg_ctl to pass down its parent shell PID (our grandparent PID) + * via the environment variable PG_GRANDPARENT_PID; this is so that + * launching the postmaster via pg_ctl can be just as reliable as + * launching it directly. There is no provision for detecting + * further-removed ancestor processes, but if the init script is written + * carefully then all but the immediate parent shell will be root-owned + * processes and so the kill test will fail with EPERM. Note that we + * cannot get a false negative this way, because an existing postmaster + * would surely never launch a competing postmaster or pg_ctl process + * directly. + */ + my_pid = getpid(); +#ifndef WIN32 + my_p_pid = getppid(); +#else + + /* + * Windows hasn't got getppid(), but doesn't need it since it's not using + * real kill() either... + */ + my_p_pid = 0; +#endif + + /* + * We need a loop here because of race conditions. But don't loop forever + * (for example, a non-writable $backup_instance_path directory might cause a failure + * that won't go away). 100 tries seems like plenty. + */ + for (ntries = 0;; ntries++) + { + /* + * Try to create the lock file --- O_EXCL makes this atomic. + * + * Think not to make the file protection weaker than 0600. See + * comments below. + */ + fd = open(lock_file, O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) + break; /* Success; exit the retry loop */ + + /* + * Couldn't create the pid file. Probably it already exists. + */ + if ((errno != EEXIST && errno != EACCES) || ntries > 100) + elog(ERROR, "could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + + /* + * Read the file to get the old owner's PID. Note race condition + * here: file might have been deleted since we tried to create it. + */ + fd = open(lock_file, O_RDONLY, 0600); + if (fd < 0) + { + if (errno == ENOENT) + continue; /* race condition; try again */ + elog(ERROR, "could not open lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) + elog(ERROR, "could not read lock file \"%s\": %s", + lock_file, strerror(errno)); + close(fd); + + if (len == 0) + elog(ERROR, "lock file \"%s\" is empty", lock_file); + + buffer[len] = '\0'; + encoded_pid = atoi(buffer); + + if (encoded_pid <= 0) + elog(ERROR, "bogus data in lock file \"%s\": \"%s\"", + lock_file, buffer); + + /* + * Check to see if the other process still exists + * + * Per discussion above, my_pid, my_p_pid can be + * ignored as false matches. + * + * Normally kill() will fail with ESRCH if the given PID doesn't + * exist. + */ + if (encoded_pid != my_pid && encoded_pid != my_p_pid) + { + if (kill(encoded_pid, 0) == 0 || + (errno != ESRCH && errno != EPERM)) + elog(ERROR, "lock file \"%s\" already exists", lock_file); + } + + /* + * Looks like nobody's home. Unlink the file and try again to create + * it. Need a loop because of possible race condition against other + * would-be creators. + */ + if (unlink(lock_file) < 0) + elog(ERROR, "could not remove old lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Successfully created the file, now fill it. + */ + snprintf(buffer, sizeof(buffer), "%d\n", my_pid); + + errno = 0; + if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + /* if write didn't set errno, assume problem is no disk space */ + errno = save_errno ? save_errno : ENOSPC; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (fsync(fd) != 0) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (close(fd) != 0) + { + int save_errno = errno; + + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Arrange to unlink the lock file(s) at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(unlink_lock_atexit); + exit_hook_registered = true; + } +} + +/* + * Read backup meta information from BACKUP_CONTROL_FILE. + * If no backup matches, return NULL. + */ +pgBackup * +read_backup(time_t timestamp) +{ + pgBackup tmp; + char conf_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + + return readBackupControlFile(conf_path); +} + +/* + * Get backup_mode in string representation. + */ +const char * +pgBackupGetBackupMode(pgBackup *backup) +{ + return backupModes[backup->backup_mode]; +} + +static bool +IsDir(const char *dirpath, const char *entry) +{ + char path[MAXPGPATH]; + struct stat st; + + snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); +} + +/* + * Create list of backups. + * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. + * The list is sorted in order of descending start time. + * If valid backup id is passed only matching backup will be added to the list. + */ +parray * +catalog_get_backup_list(time_t requested_backup_id) +{ + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + pgBackup *backup = NULL; + int i; + + /* open backup instance backups directory */ + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) + { + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + strerror(errno)); + goto err_proc; + } + + /* scan the directory and list backups */ + backups = parray_new(); + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) + { + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + + /* skip not-directory entries and hidden entries */ + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') + continue; + + /* open subdirectory of specific backup */ + join_path_components(data_path, backup_instance_path, data_ent->d_name); + + /* read backup information from BACKUP_CONTROL_FILE */ + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + backup = readBackupControlFile(backup_conf_path); + + /* ignore corrupted backups */ + if (backup) + { + backup->backup_id = backup->start_time; + + if (requested_backup_id != INVALID_BACKUP_ID + && requested_backup_id != backup->start_time) + { + pgBackupFree(backup); + continue; + } + parray_append(backups, backup); + backup = NULL; + } + + if (errno && errno != ENOENT) + { + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); + goto err_proc; + } + } + if (errno) + { + elog(WARNING, "cannot read backup root directory \"%s\": %s", + backup_instance_path, strerror(errno)); + goto err_proc; + } + + closedir(data_dir); + data_dir = NULL; + + parray_qsort(backups, pgBackupCompareIdDesc); + + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + + return backups; + +err_proc: + if (data_dir) + closedir(data_dir); + if (backup) + pgBackupFree(backup); + if (backups) + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + + return NULL; +} + +/* + * Find the last completed backup on given timeline + */ +pgBackup * +catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK && backup->tli == tli) + return backup; + } + + return NULL; +} + +/* create backup directory in $BACKUP_PATH */ +int +pgBackupCreateDir(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char *subdirs[] = { DATABASE_DIR, NULL }; + + pgBackupGetPath(backup, path, lengthof(path), NULL); + + if (!dir_is_empty(path)) + elog(ERROR, "backup destination is not empty \"%s\"", path); + + dir_create_dir(path, DIR_PERMISSION); + + /* create directories for actual backup files */ + for (i = 0; subdirs[i]; i++) + { + pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + dir_create_dir(path, DIR_PERMISSION); + } + + return 0; +} + +/* + * Write information about backup.in to stream "out". + */ +void +pgBackupWriteControl(FILE *out, pgBackup *backup) +{ + char timestamp[100]; + + fprintf(out, "#Configuration\n"); + fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + + fprintf(out, "\n#Compatibility\n"); + fprintf(out, "block-size = %u\n", backup->block_size); + fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); + if (backup->server_version[0] != '\0') + fprintf(out, "server-version = %s\n", backup->server_version); + + fprintf(out, "\n#Result backup info\n"); + fprintf(out, "timelineid = %d\n", backup->tli); + /* LSN returned by pg_start_backup */ + fprintf(out, "start-lsn = %X/%X\n", + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn); + /* LSN returned by pg_stop_backup */ + fprintf(out, "stop-lsn = %X/%X\n", + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + fprintf(out, "start-time = '%s'\n", timestamp); + if (backup->end_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + fprintf(out, "end-time = '%s'\n", timestamp); + } + fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + fprintf(out, "recovery-time = '%s'\n", timestamp); + } + + /* + * Size of PGDATA directory. The size does not include size of related + * WAL segments in archive 'wal' directory. + */ + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + + if (backup->wal_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + + fprintf(out, "status = %s\n", status2str(backup->status)); + + /* 'parent_backup' is set if it is incremental backup */ + if (backup->parent_backup != 0) + fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + + /* print connection info except password */ + if (backup->primary_conninfo) + fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); +} + +/* create BACKUP_CONTROL_FILE */ +void +pgBackupWriteBackupControlFile(pgBackup *backup) +{ + FILE *fp = NULL; + char ini_path[MAXPGPATH]; + + pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONTROL_FILE); + fp = fopen(ini_path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path, + strerror(errno)); + + pgBackupWriteControl(fp, backup); + + fclose(fp); +} + +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + +/* + * Read BACKUP_CONTROL_FILE and create pgBackup. + * - Comment starts with ';'. + * - Do not care section. + */ +static pgBackup * +readBackupControlFile(const char *path) +{ + pgBackup *backup = pgut_new(pgBackup); + char *backup_mode = NULL; + char *start_lsn = NULL; + char *stop_lsn = NULL; + char *status = NULL; + char *parent_backup = NULL; + char *program_version = NULL; + char *server_version = NULL; + char *compress_alg = NULL; + int parsed_options; + + pgut_option options[] = + { + {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, + {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, + {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, + {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, + {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, + {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, + {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, + {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, + {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, + {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, + {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, + {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, + {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, + {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, + {'s', 0, "status", &status, SOURCE_FILE_STRICT}, + {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, + {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {0} + }; + + if (access(path, F_OK) != 0) + { + elog(WARNING, "Control file \"%s\" doesn't exist", path); + pgBackupFree(backup); + return NULL; + } + + pgBackupInit(backup); + parsed_options = pgut_readopt(path, options, WARNING, true); + + if (parsed_options == 0) + { + elog(WARNING, "Control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } + + if (backup_mode) + { + backup->backup_mode = parse_backup_mode(backup_mode); + free(backup_mode); + } + + if (start_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); + free(start_lsn); + } + + if (stop_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); + free(stop_lsn); + } + + if (status) + { + if (strcmp(status, "OK") == 0) + backup->status = BACKUP_STATUS_OK; + else if (strcmp(status, "ERROR") == 0) + backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; + else if (strcmp(status, "DELETING") == 0) + backup->status = BACKUP_STATUS_DELETING; + else if (strcmp(status, "DELETED") == 0) + backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DONE") == 0) + backup->status = BACKUP_STATUS_DONE; + else if (strcmp(status, "ORPHAN") == 0) + backup->status = BACKUP_STATUS_ORPHAN; + else if (strcmp(status, "CORRUPT") == 0) + backup->status = BACKUP_STATUS_CORRUPT; + else + elog(WARNING, "Invalid STATUS \"%s\"", status); + free(status); + } + + if (parent_backup) + { + backup->parent_backup = base36dec(parent_backup); + free(parent_backup); + } + + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + + if (server_version) + { + StrNCpy(backup->server_version, server_version, + sizeof(backup->server_version)); + pfree(server_version); + } + + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + + return backup; +} + +BackupMode +parse_backup_mode(const char *value) +{ + const char *v = value; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0 && pg_strncasecmp("full", v, len) == 0) + return BACKUP_MODE_FULL; + else if (len > 0 && pg_strncasecmp("page", v, len) == 0) + return BACKUP_MODE_DIFF_PAGE; + else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0) + return BACKUP_MODE_DIFF_PTRACK; + else if (len > 0 && pg_strncasecmp("delta", v, len) == 0) + return BACKUP_MODE_DIFF_DELTA; + + /* Backup mode is invalid, so leave with an error */ + elog(ERROR, "invalid backup-mode \"%s\"", value); + return BACKUP_MODE_INVALID; +} + +const char * +deparse_backup_mode(BackupMode mode) +{ + switch (mode) + { + case BACKUP_MODE_FULL: + return "full"; + case BACKUP_MODE_DIFF_PAGE: + return "page"; + case BACKUP_MODE_DIFF_PTRACK: + return "ptrack"; + case BACKUP_MODE_DIFF_DELTA: + return "delta"; + case BACKUP_MODE_INVALID: + return "invalid"; + } + + return NULL; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + +/* free pgBackup object */ +void +pgBackupFree(void *backup) +{ + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); +} + +/* Compare two pgBackup with their IDs (start time) in ascending order */ +int +pgBackupCompareId(const void *l, const void *r) +{ + pgBackup *lp = *(pgBackup **)l; + pgBackup *rp = *(pgBackup **)r; + + if (lp->start_time > rp->start_time) + return 1; + else if (lp->start_time < rp->start_time) + return -1; + else + return 0; +} + +/* Compare two pgBackup with their IDs in descending order */ +int +pgBackupCompareIdDesc(const void *l, const void *r) +{ + return -pgBackupCompareId(l, r); +} + +/* + * Construct absolute path of the backup directory. + * If subdir is not NULL, it will be appended after the path. + */ +void +pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) +{ + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, + base36enc(backup->start_time)); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1); + /* "subdir1" and "subdir2" is not NULL */ + else + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1, subdir2); + + make_native_path(path); +} + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 000000000..8b86e438c --- /dev/null +++ b/src/configure.c @@ -0,0 +1,490 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "utils/logger.h" + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + +static pgBackupConfig *cur_config = NULL; + +static PQExpBufferData show_buf; +static int32 json_level = 0; + +/* + * All this code needs refactoring. + */ + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + + if (replica_timeout) + config->replica_timeout = replica_timeout; + + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg) + config->compress_alg = compress_alg; + if (compress_level) + config->compress_level = compress_level; + + if (show_only) + show_configure(config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + +#if PG_VERSION_NUM >= 110000 + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; +#endif + + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; + + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; + config->error_log_filename = NULL; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; + + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; + + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + uint64 res; + const char *unit; + + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); +#if PG_VERSION_NUM >= 110000 + fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); +#endif + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Logging parameters:\n"); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); + + fprintf(out, "#Retention parameters:\n"); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 0, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 0, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 0, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 0, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 0, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, +#if PG_VERSION_NUM >= 110000 + {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR, true); + +#if PG_VERSION_NUM >= 110000 + if (!IsValidWalSegSize(config->xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); +#endif + + return config; +} + +/* + * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. + */ +uint32 +get_config_xlog_seg_size(void) +{ +#if PG_VERSION_NUM >= 110000 + char path[MAXPGPATH]; + uint32 seg_size; + pgut_option options[] = + { + {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, + {0} + }; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, false); + + if (!IsValidWalSegSize(seg_size)) + elog(ERROR, "Invalid WAL segment size %u", seg_size); + + return seg_size; + +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + cur_config->log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + cur_config->log_level_file = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + +#if PG_VERSION_NUM >= 110000 + json_add_key(buf, "xlog-seg-size", json_level, true); + appendPQExpBuffer(buf, "%u", config->xlog_seg_size); +#endif + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Logging parameters */ + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + { + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); + } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); + + /* Retention parameters */ + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/data.c b/src/data.c new file mode 100644 index 000000000..a66770bcf --- /dev/null +++ b/src/data.c @@ -0,0 +1,1407 @@ +/*------------------------------------------------------------------------- + * + * data.c: utils to parse and backup data pages + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum_impl.h" +#include + +#ifdef HAVE_LIBZ +#include +#endif + +#ifdef HAVE_LIBZ +/* Implementation of zlib compression method */ +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + + return rc == Z_OK ? compressed_size : rc; +} + +/* Implementation of zlib compression method */ +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + + return rc == Z_OK ? dest_len : rc; +} +#endif + +/* + * Compresses source into dest using algorithm. Returns the number of bytes + * written in the destination buffer, or -1 if compression fails. + */ +static int32 +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size, level); +#endif + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +/* + * Decompresses source into dest using algorithm. Returns the number of bytes + * decompressed in the destination buffer, or -1 if decompression fails. + */ +static int32 +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); +#endif + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + +/* + * When copying datafiles to backup we validate and compress them block + * by block. Thus special header is required for each data block. + */ +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + int32 compressed_size; +} BackupPageHeader; + +/* Special value for compressed_size field */ +#define PageIsTruncated -2 +#define SkipCurrentPage -3 + +/* Verify page's header */ +static bool +parse_page(Page page, XLogRecPtr *lsn) +{ + PageHeader phdr = (PageHeader) page; + + /* Get lsn from page header */ + *lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + if (PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special)) + return true; + + return false; +} + +/* Read one page from file directly accessing disk + * return value: + * 0 - if the page is not found + * 1 - if the page is found and valid + * -1 - if the page is found but invalid + */ +static int +read_page_from_file(pgFile *file, BlockNumber blknum, + FILE *in, Page page, XLogRecPtr *page_lsn) +{ + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; + + /* read the block */ + if (fseek(in, offset, SEEK_SET) != 0) + elog(ERROR, "File: %s, could not seek to block %u: %s", + file->path, blknum, strerror(errno)); + + read_len = fread(page, 1, BLCKSZ, in); + + if (read_len != BLCKSZ) + { + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(LOG, "File %s, block %u, file was truncated", + file->path, blknum); + return 0; + } + else + elog(WARNING, "File: %s, block %u, expected block size %d," + "but read %lu, try again", + file->path, blknum, BLCKSZ, read_len); + } + + /* + * If we found page with invalid header, at first check if it is zeroed, + * which is a valid state for page. If it is not, read it and check header + * again, because it's possible that we've read a partly flushed page. + * If after several attempts page header is still invalid, throw an error. + * The same idea is applied to checksum verification. + */ + if (!parse_page(page, page_lsn)) + { + int i; + /* Check if the page is zeroed. */ + for(i = 0; i < BLCKSZ && page[i] == 0; i++); + + /* Page is zeroed. No need to check header and checksum. */ + if (i == BLCKSZ) + { + elog(LOG, "File: %s blknum %u, empty page", file->path, blknum); + return 1; + } + + /* + * If page is not completely empty and we couldn't parse it, + * try again several times. If it didn't help, throw error + */ + elog(LOG, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); + return -1; + } + + /* Verify checksum */ + if(current.checksum_version) + { + /* + * If checksum is wrong, sleep a bit and then try again + * several times. If it didn't help, throw error + */ + if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) + != ((PageHeader) page)->pd_checksum) + { + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); + return -1; + } + else + { + /* page header and checksum are correct */ + return 1; + } + } + else + { + /* page header is correct and checksum check is disabled */ + return 1; + } +} + +/* + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated + */ +static int32 +prepare_page(backup_files_arg *arguments, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + BlockNumber blknum, BlockNumber nblocks, + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) +{ + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; + BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* + * Read the page and verify its header and checksum. + * Under high write load it's possible that we've read partly + * flushed page, so try several times before throwing an error. + */ + if (backup_mode != BACKUP_MODE_DIFF_PTRACK) + { + while(!page_is_valid && try_again) + { + int result = read_page_from_file(file, blknum, + in, page, &page_lsn); + + try_again--; + if (result == 0) + { + /* This block was truncated.*/ + page_is_truncated = true; + /* Page is not actually valid, but it is absent + * and we're not going to reread it or validate */ + page_is_valid = true; + } + + if (result == 1) + page_is_valid = true; + + /* + * If ptrack support is available use it to get invalid block + * instead of rereading it 99 times + */ + //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); + + if (result == -1 && is_ptrack_support) + { + elog(WARNING, "File %s, block %u, try to fetch via SQL", + file->path, blknum); + break; + } + } + /* + * If page is not valid after 100 attempts to read it + * throw an error. + */ + if(!page_is_valid && !is_ptrack_support) + elog(ERROR, "Data file checksum mismatch. Canceling backup"); + } + + if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) + { + size_t page_size = 0; + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + file->relOid, absolute_blknum, &page_size); + + if (ptrack_page == NULL) + { + /* This block was truncated.*/ + page_is_truncated = true; + } + else if (page_size != BLCKSZ) + { + free(ptrack_page); + elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + file->path, absolute_blknum, BLCKSZ, page_size); + } + else + { + /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. + * We must set checksum here, because it is outdated + * in the block recieved from shared buffers. + */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); + if (is_checksum_enabled) + ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + } + /* get lsn from page, provided by pg_ptrack_get_block() */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + !parse_page(page, &page_lsn)) + elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " + "Possible risk of a memory corruption"); + + } + + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + page_lsn < prev_backup_start_lsn) + { + elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); + (*n_skipped)++; + return SkipCurrentPage; + } + + if (page_is_truncated) + return PageIsTruncated; + + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page, + CompressAlg calg, int clevel) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ + header.compressed_size = do_compress(compressed_page, BLCKSZ, + page, BLCKSZ, calg, clevel); + + file->compress_alg = calg; + file->read_size += BLCKSZ; + Assert (header.compressed_size <= BLCKSZ); + + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } + } + + /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", + blknum, header.compressed_size, write_buffer_size); */ + + /* Update CRC */ + COMP_CRC32C(*crc, write_buffer, write_buffer_size); + + /* write data page */ + if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) + { + int errno_tmp = errno; + + fclose(in); + fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u : %s", + file->path, blknum, strerror(errno_tmp)); + } + + file->write_size += write_buffer_size; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +bool +backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) +{ + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if ((backup_mode == BACKUP_MODE_DIFF_PAGE || + backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want make + * incremental backup, so we should exit. + */ + elog(VERBOSE, "Skipping the unchanged file: %s", file->path); + return false; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + INIT_CRC32C(file->crc); + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(file->crc); + + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return false; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + /* open backup file for write */ + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open backup file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. + */ + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) + { + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + if (backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = n_blocks_read; + } + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ + else + { + datapagemap_iterator_t *iter; + iter = datapagemap_iterate(&file->pagemap); + while (datapagemap_next(iter, &blknum)) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + + pg_free(file->pagemap.bitmap); + pg_free(iter); + } + + /* update file permission */ + if (chmod(to_path, FILE_PERMISSION) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", file->path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write backup file \"%s\": %s", + to_path, strerror(errno)); + fclose(in); + + FIN_CRC32C(file->crc); + + /* + * If we have pagemap then file in the backup can't be a zero size. + * Otherwise, we will clear the last file. + */ + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) + { + if (remove(to_path) == -1) + elog(ERROR, "cannot remove file \"%s\": %s", to_path, + strerror(errno)); + return false; + } + + return true; +} + +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. + */ +void +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) +{ + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; + + /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ + if (file->write_size != BYTES_INVALID) + { + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "cannot open backup file \"%s\": %s", file->path, + strerror(errno)); + } + } + + /* + * Open backup file for write. We use "r+" at first to overwrite only + * modified pages for differential restore. If the file does not exist, + * re-open it with "w" to create an empty file. + */ + out = fopen(to_path, PG_BINARY_R "+"); + if (out == NULL && errno == ENOENT) + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open restore target file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + while (true) + { + off_t write_pos; + size_t read_len; + DataPage compressed_page; /* used as read buffer */ + DataPage page; + + /* File didn`t changed. Nothig to copy */ + if (file->write_size == BYTES_INVALID) + break; + + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + + /* read BackupPageHeader */ + read_len = fread(&header, 1, sizeof(header), in); + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "odd size page found at block %u of \"%s\"", + blknum, file->path); + else + elog(ERROR, "cannot read header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno_tmp)); + } + + if (header.block < blknum) + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); + + blknum = header.block; + + if (header.compressed_size == PageIsTruncated) + { + /* + * Backup contains information that this block was truncated. + * We need to truncate file to this length. + */ + truncate_from = blknum; + need_truncate = true; + break; + } + + Assert(header.compressed_size <= BLCKSZ); + + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); + + if (header.compressed_size != BLCKSZ) + { + int32 uncompressed_size = 0; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); + } + + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + + /* + * Seek and write the restored page. + */ + if (fseek(out, write_pos, SEEK_SET) < 0) + elog(ERROR, "cannot seek block %u of \"%s\": %s", + blknum, to_path, strerror(errno)); + + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + } + + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) + { + size_t file_size = 0; + + /* get file current size */ + fseek(out, 0, SEEK_END); + file_size = ftell(out); + + if (file_size > file->n_blocks * BLCKSZ) + { + truncate_from = file->n_blocks; + need_truncate = true; + } + } + + if (need_truncate) + { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), write_pos) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + + /* update file permission */ + if (chmod(to_path, file->mode) == -1) + { + int errno_tmp = errno; + + if (in) + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + if (in) + fclose(in); +} + +/* + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +bool +copy_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* open backup file for write */ + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + /* copy content and calc CRC */ + for (;;) + { + read_len = 0; + + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + file->write_size = (int64) file->read_size; + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + /* update file permission */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + fclose(in); + + return true; +} + +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* + * Copy file attributes + */ +static void +copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) +{ + struct stat st; + + if (stat(from_path, &st) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot stat file \"%s\": %s", + from_path, strerror(errno)); + } + + if (chmod(to_path, st.st_mode) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot change mode of file \"%s\": %s", + to_path, strerror(errno)); + } +} + +/* + * Copy WAL segment from pgdata to archive catalog with possible compression. + */ +void +push_wal_file(const char *from_path, const char *to_path, bool is_compress, + bool overwrite) +{ + FILE *in = NULL; + FILE *out=NULL; + char buf[XLOG_BLCKSZ]; + const char *to_path_p = to_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + +#ifdef HAVE_LIBZ + char gz_to_path[MAXPGPATH]; + gzFile gz_out = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + + /* open backup file for write */ +#ifdef HAVE_LIBZ + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + + if (!overwrite && fileExists(gz_to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + + gz_out = gzopen(to_path_temp, PG_BINARY_W); + if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) + elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", + compress_level, to_path_temp, get_gz_error(gz_out, errno)); + + to_path_p = gz_to_path; + } + else +#endif + { + if (!overwrite && fileExists(to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + + /* copy content */ + for (;;) + { + size_t read_len = 0; + + read_len = fread(buf, 1, sizeof(buf), in); + + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, + "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzwrite(gz_out, buf, read_len) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + } + + if (feof(in) || read_len == 0) + break; + } + +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzclose(gz_out) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + /* update file permission. */ + copy_meta(from_path, to_path_temp, true); + + if (rename(to_path_temp, to_path_p) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path_p, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_compress) + elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); +#endif +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + */ +void +get_wal_file(const char *from_path, const char *to_path) +{ + FILE *in = NULL; + FILE *out; + char buf[XLOG_BLCKSZ]; + const char *from_path_p = from_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + bool is_decompress = false; + +#ifdef HAVE_LIBZ + char gz_from_path[MAXPGPATH]; + gzFile gz_in = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + { +#ifdef HAVE_LIBZ + /* + * Maybe we need to decompress the file. Check it with .gz + * extension. + */ + snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); + gz_in = gzopen(gz_from_path, PG_BINARY_R); + if (gz_in == NULL) + { + if (errno == ENOENT) + { + /* There is no compressed file too, raise an error below */ + } + /* Cannot open compressed file for some reason */ + else + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); + } + else + { + /* Found compressed file */ + is_decompress = true; + from_path_p = gz_from_path; + } +#endif + /* Didn't find compressed file */ + if (!is_decompress) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); + } + + /* open backup file for write */ + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + /* copy content */ + for (;;) + { + size_t read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !gzeof(gz_in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + read_len = fread(buf, 1, sizeof(buf), in); + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, + strerror(errno_temp)); + } + } + + /* Check for EOF */ +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzeof(gz_in) || read_len == 0) + break; + } + else +#endif + { + if (feof(in) || read_len == 0) + break; + } + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzclose(gz_in) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + /* update file permission. */ + copy_meta(from_path_p, to_path_temp, true); + + if (rename(to_path_temp, to_path) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); +#endif +} + +/* + * Calculate checksum of various files which are not copied from PGDATA, + * but created in process of backup, such as stream XLOG files, + * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. + */ +bool +calc_file_checksum(pgFile *file) +{ + FILE *in; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + Assert(S_ISREG(file->mode)); + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + for (;;) + { + read_len = fread(buf, 1, sizeof(buf), in); + + if(read_len == 0) + break; + + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->write_size += read_len; + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + fclose(in); + + return true; +} diff --git a/src/delete.c b/src/delete.c new file mode 100644 index 000000000..de29d2cf0 --- /dev/null +++ b/src/delete.c @@ -0,0 +1,464 @@ +/*------------------------------------------------------------------------- + * + * delete.c: delete backup files. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +static int pgBackupDeleteFiles(pgBackup *backup); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size); + +int +do_delete(time_t backup_id) +{ + int i; + parray *backup_list, + *delete_list; + pgBackup *target_backup = NULL; + time_t parent_id = 0; + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get complete list of backups */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + if (backup_id != 0) + { + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->start_time == backup_id) + { + parray_append(delete_list, backup); + + /* + * Do not remove next backups, if target backup was finished + * incorrectly. + */ + if (backup->status == BACKUP_STATUS_ERROR) + break; + + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + target_backup = backup; + } + else if (target_backup) + { + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->parent_backup == parent_id) + { + /* Append to delete list increment backup */ + parray_append(delete_list, backup); + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + } + else + break; + } + } + + if (parray_num(delete_list) == 0) + elog(ERROR, "no backup found, cannot delete"); + + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + + pgBackupDeleteFiles(backup); + } + + parray_free(delete_list); + } + + /* Clean WAL segments */ + if (delete_wal) + { + Assert(target_backup); + + /* Find oldest LSN, used by backups */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK) + { + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + break; + } + } + + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + return 0; +} + +/* + * Remove backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + */ +int +do_retention_purge(void) +{ + parray *backup_list; + uint32 backup_num; + size_t i; + time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + bool keep_next_backup = true; /* Do not delete first full backup */ + bool backup_deleted = false; /* At least one backup was deleted */ + + if (delete_expired) + { + if (retention_redundancy > 0) + elog(LOG, "REDUNDANCY=%u", retention_redundancy); + if (retention_window > 0) + elog(LOG, "WINDOW=%u", retention_window); + + if (retention_redundancy == 0 + && retention_window == 0) + { + elog(WARNING, "Retention policy is not set"); + if (!delete_wal) + return 0; + } + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get a complete list of backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + if (parray_num(backup_list) == 0) + { + elog(INFO, "backup list is empty, purging won't be executed"); + return 0; + } + + /* Find target backups to be deleted */ + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) + { + backup_num = 0; + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + uint32 backup_num_evaluate = backup_num; + + /* Consider only validated and correct backups */ + if (backup->status != BACKUP_STATUS_OK) + continue; + /* + * When a valid full backup was found, we can delete the + * backup that is older than it using the number of generations. + */ + if (backup->backup_mode == BACKUP_MODE_FULL) + backup_num++; + + /* Evaluate retention_redundancy if this backup is eligible for removal */ + if (keep_next_backup || + retention_redundancy >= backup_num_evaluate + 1 || + (retention_window > 0 && backup->recovery_time >= days_threshold)) + { + /* Save LSN and Timeline to remove unnecessary WAL segments */ + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + + /* Save parent backup of this incremental backup */ + if (backup->backup_mode != BACKUP_MODE_FULL) + keep_next_backup = true; + /* + * Previous incremental backup was kept or this is first backup + * so do not delete this backup. + */ + else + keep_next_backup = false; + + continue; + } + + /* Delete backup and update status to DELETED */ + pgBackupDeleteFiles(backup); + backup_deleted = true; + } + } + + /* + * If oldest_lsn and oldest_tli weren`t set because previous step was skipped + * then set them now if we are going to purge WAL + */ + if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn))) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1); + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + } + + /* Be paranoid */ + if (XLogRecPtrIsInvalid(oldest_lsn)) + elog(ERROR, "Not going to purge WAL because LSN is invalid"); + + /* Purge WAL files */ + if (delete_wal) + { + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + if (backup_deleted) + elog(INFO, "Purging finished"); + else + elog(INFO, "Nothing to delete by retention policy"); + + return 0; +} + +/* + * Delete backup files of the backup and update the status of the backup to + * BACKUP_STATUS_DELETED. + */ +static int +pgBackupDeleteFiles(pgBackup *backup) +{ + size_t i; + char path[MAXPGPATH]; + char timestamp[100]; + parray *files; + + /* + * If the backup was deleted already, there is nothing to do. + */ + if (backup->status == BACKUP_STATUS_DELETED) + return 0; + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); + + /* + * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which + * the error occurs before deleting all backup files. + */ + backup->status = BACKUP_STATUS_DELETING; + pgBackupWriteBackupControlFile(backup); + + /* list files to be deleted */ + files = parray_new(); + pgBackupGetPath(backup, path, lengthof(path), NULL); + dir_list_file(files, path, false, true, true); + + /* delete leaf node first */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + /* print progress */ + elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + (unsigned long) parray_num(files), file->path); + + if (remove(file->path)) + { + elog(WARNING, "can't remove \"%s\": %s", file->path, + strerror(errno)); + parray_walk(files, pgFileFree); + parray_free(files); + + return 1; + } + } + + parray_walk(files, pgFileFree); + parray_free(files); + backup->status = BACKUP_STATUS_DELETED; + + return 0; +} + +/* + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). + * + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. + */ +static void +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size) +{ + XLogSegNo targetSegNo; + char oldestSegmentNeeded[MAXFNAMELEN]; + DIR *arcdir; + struct dirent *arcde; + char wal_file[MAXPGPATH]; + char max_wal_file[MAXPGPATH]; + char min_wal_file[MAXPGPATH]; + int rc; + + max_wal_file[0] = '\0'; + min_wal_file[0] = '\0'; + + if (!XLogRecPtrIsInvalid(oldest_lsn)) + { + GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); + GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, + xlog_seg_size); + + elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); + } + else + elog(LOG, "removing all WAL segments"); + + /* + * Now it is time to do the actual work and to remove all the segments + * not needed anymore. + */ + if ((arcdir = opendir(arclog_path)) != NULL) + { + while (errno = 0, (arcde = readdir(arcdir)) != NULL) + { + /* + * We ignore the timeline part of the WAL segment identifiers in + * deciding whether a segment is still needed. This ensures that + * we won't prematurely remove a segment from a parent timeline. + * We could probably be a little more proactive about removing + * segments of non-parent timelines, but that would be a whole lot + * more complicated. + * + * We use the alphanumeric sorting property of the filenames to + * decide which ones are earlier than the exclusiveCleanupFileName + * file. Note that this means files are not removed in the order + * they were originally written, in case this worries you. + * + * We also should not forget that WAL segment can be compressed. + */ + if (IsXLogFileName(arcde->d_name) || + IsPartialXLogFileName(arcde->d_name) || + IsBackupHistoryFileName(arcde->d_name) || + IsCompressedXLogFileName(arcde->d_name)) + { + if (XLogRecPtrIsInvalid(oldest_lsn) || + strncmp(arcde->d_name + 8, oldestSegmentNeeded + 8, 16) < 0) + { + /* + * Use the original file name again now, including any + * extension that might have been chopped off before testing + * the sequence. + */ + snprintf(wal_file, MAXPGPATH, "%s/%s", + arclog_path, arcde->d_name); + + rc = unlink(wal_file); + if (rc != 0) + { + elog(WARNING, "could not remove file \"%s\": %s", + wal_file, strerror(errno)); + break; + } + elog(LOG, "removed WAL segment \"%s\"", wal_file); + + if (max_wal_file[0] == '\0' || + strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) + strcpy(max_wal_file, arcde->d_name); + + if (min_wal_file[0] == '\0' || + strcmp(min_wal_file + 8, arcde->d_name + 8) > 0) + strcpy(min_wal_file, arcde->d_name); + } + } + } + + if (min_wal_file[0] != '\0') + elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); + if (max_wal_file[0] != '\0') + elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); + + if (errno) + elog(WARNING, "could not read archive location \"%s\": %s", + arclog_path, strerror(errno)); + if (closedir(arcdir)) + elog(WARNING, "could not close archive location \"%s\": %s", + arclog_path, strerror(errno)); + } + else + elog(WARNING, "could not open archive location \"%s\": %s", + arclog_path, strerror(errno)); +} + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} diff --git a/src/dir.c b/src/dir.c new file mode 100644 index 000000000..a08bd9343 --- /dev/null +++ b/src/dir.c @@ -0,0 +1,1491 @@ +/*------------------------------------------------------------------------- + * + * dir.c: directory operation utility. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" + +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +const char *pgdata_exclude_dir[] = +{ + PG_XLOG_DIR, + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + "pg_stat_tmp", + "pgsql_tmp", + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + "pg_dynshmem", + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL, /* pg_log will be set later */ + NULL +}; + +static char *pgdata_exclude_files[] = +{ + /* Skip auto conf temporary file. */ + "postgresql.auto.conf.tmp", + + /* Skip current log file temporary file */ + "current_logfiles.tmp", + "recovery.conf", + "postmaster.pid", + "postmaster.opts", + NULL +}; + +static char *pgdata_exclude_files_non_exclusive[] = +{ + /*skip in non-exclusive backup */ + "backup_label", + "tablespace_map", + NULL +}; + +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + +static int BlackListCompare(const void *str1, const void *str2); + +static bool dir_check_file(const char *root, pgFile *file); +static void dir_list_file_internal(parray *files, const char *root, + pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list); + +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + +/* + * Create directory, also create parent directories if necessary. + */ +int +dir_create_dir(const char *dir, mode_t mode) +{ + char parent[MAXPGPATH]; + + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); + + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode); + + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST) /* already exist */ + return 0; + elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + } + + return 0; +} + +pgFile * +pgFileNew(const char *path, bool omit_symlink) +{ + struct stat st; + pgFile *file; + + /* stat the file */ + if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + { + /* file not found is not an error case */ + if (errno == ENOENT) + return NULL; + elog(ERROR, "cannot stat file \"%s\": %s", path, + strerror(errno)); + } + + file = pgFileInit(path); + file->size = st.st_size; + file->mode = st.st_mode; + + return file; +} + +pgFile * +pgFileInit(const char *path) +{ + pgFile *file; + char *file_name; + + file = (pgFile *) pgut_malloc(sizeof(pgFile)); + + file->name = NULL; + + file->size = 0; + file->mode = 0; + file->read_size = 0; + file->write_size = 0; + file->crc = 0; + file->is_datafile = false; + file->linked = NULL; + file->pagemap.bitmap = NULL; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; + file->tblspcOid = 0; + file->dbOid = 0; + file->relOid = 0; + file->segno = 0; + file->is_database = false; + file->forkName = pgut_malloc(MAXPGPATH); + file->forkName[0] = '\0'; + + file->path = pgut_malloc(strlen(path) + 1); + strcpy(file->path, path); /* enough buffer size guaranteed */ + + /* Get file name from the path */ + file_name = strrchr(file->path, '/'); + if (file_name == NULL) + file->name = file->path; + else + { + file_name++; + file->name = file_name; + } + + file->is_cfs = false; + file->exists_in_prev = false; /* can change only in Incremental backup. */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; + file->compress_alg = NOT_DEFINED_COMPRESS; + return file; +} + +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +pgFileDelete(pgFile *file) +{ + if (S_ISDIR(file->mode)) + { + if (rmdir(file->path) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR, "cannot remove directory \"%s\": %s", + file->path, strerror(errno)); + } + return; + } + +delete_file: + if (remove(file->path) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "cannot remove file \"%s\": %s", file->path, + strerror(errno)); + } +} + +pg_crc32 +pgFileGetCRC(const char *file_path) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[1024]; + size_t len; + int errno_tmp; + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", + file_path, strerror(errno)); + + /* calc CRC of backup file */ + INIT_CRC32C(crc); + while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + COMP_CRC32C(crc, buf, len); + } + errno_tmp = errno; + if (!feof(fp)) + elog(WARNING, "cannot read \"%s\": %s", file_path, + strerror(errno_tmp)); + if (len > 0) + COMP_CRC32C(crc, buf, len); + FIN_CRC32C(crc); + + fclose(fp); + + return crc; +} + +void +pgFileFree(void *file) +{ + pgFile *file_ptr; + + if (file == NULL) + return; + + file_ptr = (pgFile *) file; + + if (file_ptr->linked) + free(file_ptr->linked); + + if (file_ptr->forkName) + free(file_ptr->forkName); + + free(file_ptr->path); + free(file); +} + +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileComparePath(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->path, f2p->path); +} + +/* Compare two pgFile with their path in descending order of ASCII code. */ +int +pgFileComparePathDesc(const void *f1, const void *f2) +{ + return -pgFileComparePath(f1, f2); +} + +/* Compare two pgFile with their linked directory path. */ +int +pgFileCompareLinked(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->linked, f2p->linked); +} + +/* Compare two pgFile with their size */ +int +pgFileCompareSize(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + if (f1p->size > f2p->size) + return 1; + else if (f1p->size < f2p->size) + return -1; + else + return 0; +} + +static int +BlackListCompare(const void *str1, const void *str2) +{ + return strcmp(*(char **) str1, *(char **) str2); +} + +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * When omit_symlink is true, symbolic link is ignored and only file or + * directory llnked to will be listed. + */ +void +dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, + bool add_root) +{ + pgFile *file; + parray *black_list = NULL; + char path[MAXPGPATH]; + + join_path_components(path, backup_instance_path, PG_BLACK_LIST); + /* List files with black list */ + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + { + FILE *black_list_file = NULL; + char buf[MAXPGPATH * 2]; + char black_item[MAXPGPATH * 2]; + + black_list = parray_new(); + black_list_file = fopen(path, PG_BINARY_R); + + if (black_list_file == NULL) + elog(ERROR, "cannot open black_list: %s", strerror(errno)); + + while (fgets(buf, lengthof(buf), black_list_file) != NULL) + { + join_path_components(black_item, pgdata, buf); + + if (black_item[strlen(black_item) - 1] == '\n') + black_item[strlen(black_item) - 1] = '\0'; + + if (black_item[0] == '#' || black_item[0] == '\0') + continue; + + parray_append(black_list, black_item); + } + + fclose(black_list_file); + parray_qsort(black_list, BlackListCompare); + } + + file = pgFileNew(root, false); + if (file == NULL) + return; + + if (!S_ISDIR(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + return; + } + if (add_root) + parray_append(files, file); + + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); +} + +/* + * Check file or directory. + * + * Check for exclude. + * Extract information about the file parsing its name. + * Skip files: + * - skip temp tables files + * - skip unlogged tables files + * Set flags for: + * - database directories + * - datafiles + */ +static bool +dir_check_file(const char *root, pgFile *file) +{ + const char *rel_path; + int i; + int sscanf_res; + + /* Check if we need to exclude file by name */ + if (S_ISREG(file->mode)) + { + if (!exclusive_backup) + { + for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) + if (strcmp(file->name, + pgdata_exclude_files_non_exclusive[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + + for (i = 0; pgdata_exclude_files[i]; i++) + if (strcmp(file->name, pgdata_exclude_files[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + /* + * If the directory name is in the exclude list, do not list the + * contents. + */ + else if (S_ISDIR(file->mode)) + { + /* + * If the item in the exclude list starts with '/', compare to + * the absolute path of the directory. Otherwise compare to the + * directory name portion. + */ + for (i = 0; pgdata_exclude_dir[i]; i++) + { + /* Full-path exclude*/ + if (pgdata_exclude_dir[i][0] == '/') + { + if (strcmp(file->path, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + } + + rel_path = GetRelativePath(file->path, root); + + /* + * Do not copy tablespaces twice. It may happen if the tablespace is located + * inside the PGDATA. + */ + if (S_ISDIR(file->mode) && + strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) + { + Oid tblspcOid; + char tmp_rel_path[MAXPGPATH]; + + /* + * Valid path for the tablespace is + * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + */ + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + return false; + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", + &tblspcOid, tmp_rel_path); + if (sscanf_res == 0) + return false; + } + + if (path_is_prefix_of_path("global", rel_path)) + { + file->tblspcOid = GLOBALTABLESPACE_OID; + + if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("base", rel_path)) + { + file->tblspcOid = DEFAULTTABLESPACE_OID; + + sscanf(rel_path, "base/%u/", &(file->dbOid)); + + if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) + file->is_database = true; + } + else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + + /* Do not backup ptrack_init files */ + if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + return false; + + /* + * Check files located inside database directories including directory + * 'global' + */ + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->name, "pg_internal.init") == 0) + return false; + /* Do not backup temp files */ + else if (file->name[0] == 't' && isdigit(file->name[1])) + return false; + else if (isdigit(file->name[0])) + { + char *fork_name; + int len; + char suffix[MAXPGPATH]; + + fork_name = strstr(file->name, "_"); + if (fork_name) + { + /* Auxiliary fork of the relfile */ + sscanf(file->name, "%u_%s", &(file->relOid), file->forkName); + + /* Do not backup ptrack files */ + if (strcmp(file->forkName, "ptrack") == 0) + return false; + } + else + { + len = strlen(file->name); + /* reloid.cfm */ + if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) + return true; + + sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), + &(file->segno), suffix); + if (sscanf_res == 0) + elog(ERROR, "Cannot parse file name \"%s\"", file->name); + else if (sscanf_res == 1 || sscanf_res == 2) + file->is_datafile = true; + } + } + } + + return true; +} + +/* + * List files in "root" directory. If "exclude" is true do not add into "files" + * files from pgdata_exclude_files and directories from pgdata_exclude_dir. + */ +static void +dir_list_file_internal(parray *files, const char *root, pgFile *parent, + bool exclude, bool omit_symlink, parray *black_list) +{ + DIR *dir; + struct dirent *dent; + + if (!S_ISDIR(parent->mode)) + elog(ERROR, "\"%s\" is not a directory", parent->path); + + /* Open directory and list contents */ + dir = opendir(parent->path); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* Maybe the directory was removed */ + return; + } + elog(ERROR, "cannot open directory \"%s\": %s", + parent->path, strerror(errno)); + } + + errno = 0; + while ((dent = readdir(dir))) + { + pgFile *file; + char child[MAXPGPATH]; + + join_path_components(child, parent->path, dent->d_name); + + file = pgFileNew(child, omit_symlink); + if (file == NULL) + continue; + + /* Skip entries point current dir or parent dir */ + if (S_ISDIR(file->mode) && + (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) + { + pgFileFree(file); + continue; + } + + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + pgFileFree(file); + continue; + } + + /* Skip if the directory is in black_list defined by user */ + if (black_list && parray_bsearch(black_list, file->path, + BlackListCompare)) + { + elog(LOG, "Skip \"%s\": it is in the user's black list", file->path); + pgFileFree(file); + continue; + } + + /* We add the directory anyway */ + if (S_ISDIR(file->mode)) + parray_append(files, file); + + if (exclude && !dir_check_file(root, file)) + { + if (S_ISREG(file->mode)) + pgFileFree(file); + /* Skip */ + continue; + } + + /* At least add the file */ + if (S_ISREG(file->mode)) + parray_append(files, file); + + /* + * If the entry is a directory call dir_list_file_internal() + * recursively. + */ + if (S_ISDIR(file->mode)) + dir_list_file_internal(files, root, file, exclude, omit_symlink, + black_list); + } + + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + closedir(dir); + elog(ERROR, "cannot read directory \"%s\": %s", + parent->path, strerror(errno_tmp)); + } + closedir(dir); +} + +/* + * List data directories excluding directories from + * pgdata_exclude_dir array. + * + * **is_root** is a little bit hack. We exclude only first level of directories + * and on the first level we check all files and directories. + */ +static void +list_data_directories(parray *files, const char *path, bool is_root, + bool exclude) +{ + DIR *dir; + struct dirent *dent; + int prev_errno; + bool has_child_dirs = false; + + /* open directory and list contents */ + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + bool skip = false; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + /* Check for exclude for the first level of listing */ + if (is_root && exclude) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++) + { + if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0) + { + skip = true; + break; + } + } + } + if (skip) + continue; + + has_child_dirs = true; + list_data_directories(files, child, false, exclude); + } + + /* List only full and last directories */ + if (!is_root && !has_child_dirs) + { + pgFile *dir; + + dir = pgFileNew(path, false); + parray_append(files, dir); + } + + prev_errno = errno; + closedir(dir); + + if (prev_errno && prev_errno != ENOENT) + elog(ERROR, "cannot read directory \"%s\": %s", + path, strerror(prev_errno)); +} + +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + +/* + * Read names of symbolik names of tablespaces with links to directories from + * tablespace_map or tablespace_map.txt. + */ +void +read_tablespace_map(parray *files, const char *backup_dir) +{ + FILE *fp; + char db_path[MAXPGPATH], + map_path[MAXPGPATH]; + char buf[MAXPGPATH * 2]; + + join_path_components(db_path, backup_dir, DATABASE_DIR); + join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path)) + { + elog(LOG, "there is no file tablespace_map"); + return; + } + + fp = fopen(map_path, "rt"); + if (fp == NULL) + elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + + while (fgets(buf, lengthof(buf), fp)) + { + char link_name[MAXPGPATH], + path[MAXPGPATH]; + pgFile *file; + + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) + elog(ERROR, "invalid format found in \"%s\"", map_path); + + file = pgut_new(pgFile); + memset(file, 0, sizeof(pgFile)); + + file->path = pgut_malloc(strlen(link_name) + 1); + strcpy(file->path, link_name); + + file->linked = pgut_malloc(strlen(path) + 1); + strcpy(file->linked, path); + + parray_append(files, file); + } + + parray_qsort(files, pgFileCompareLinked); + fclose(fp); +} + +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + +/* + * Print backup content list. + */ +void +print_file_list(FILE *out, const parray *files, const char *root) +{ + size_t i; + + /* print each file in the list */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = GetRelativePath(path, root); + + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " + "\"compress_alg\":\"%s\"", + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, + deparse_compress_alg(file->compress_alg)); + + if (file->is_datafile) + fprintf(out, ",\"segno\":\"%d\"", file->segno); + +#ifndef WIN32 + if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif + fprintf(out, ",\"linked\":\"%s\"", file->linked); + + if (file->n_blocks != BLOCKNUM_INVALID) + fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + + fprintf(out, "}\n"); + } +} + +/* Parsing states for get_control_value() */ +#define CONTROL_WAIT_NAME 1 +#define CONTROL_INNAME 2 +#define CONTROL_WAIT_COLON 3 +#define CONTROL_WAIT_VALUE 4 +#define CONTROL_INVALUE 5 +#define CONTROL_WAIT_NEXT_NAME 6 + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string if it is not NULL. If it + * is NULL the value will be returned to "value_int64" as int64. + * + * Returns true if the value was found in the line. + */ +static bool +get_control_value(const char *str, const char *name, + char *value_str, int64 *value_int64, bool is_mandatory) +{ + int state = CONTROL_WAIT_NAME; + char *name_ptr = (char *) name; + char *buf = (char *) str; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; + + /* Set default values */ + if (value_str) + *value_str = '\0'; + else if (value_int64) + *value_int64 = 0; + + while (*buf) + { + switch (state) + { + case CONTROL_WAIT_NAME: + if (*buf == '"') + state = CONTROL_INNAME; + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INNAME: + /* Found target field. Parse value. */ + if (*buf == '"') + state = CONTROL_WAIT_COLON; + /* Check next field */ + else if (*buf != *name_ptr) + { + name_ptr = (char *) name; + state = CONTROL_WAIT_NEXT_NAME; + } + else + name_ptr++; + break; + case CONTROL_WAIT_COLON: + if (*buf == ':') + state = CONTROL_WAIT_VALUE; + else if (!IsSpace(*buf)) + goto bad_format; + break; + case CONTROL_WAIT_VALUE: + if (*buf == '"') + { + state = CONTROL_INVALUE; + buf_int64_ptr = buf_int64; + } + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INVALUE: + /* Value was parsed, exit */ + if (*buf == '"') + { + if (value_str) + { + *value_str = '\0'; + } + else if (value_int64) + { + /* Length of buf_uint64 should not be greater than 31 */ + if (buf_int64_ptr - buf_int64 >= 32) + elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) + goto bad_format; + } + + return true; + } + else + { + if (value_str) + { + *value_str = *buf; + value_str++; + } + else + { + *buf_int64_ptr = *buf; + buf_int64_ptr++; + } + } + break; + case CONTROL_WAIT_NEXT_NAME: + if (*buf == ',') + state = CONTROL_WAIT_NAME; + break; + default: + /* Should not happen */ + break; + } + + buf++; + } + + /* There is no close quotes */ + if (state == CONTROL_INNAME || state == CONTROL_INVALUE) + goto bad_format; + + /* Did not find target field */ + if (is_mandatory) + elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + return false; + +bad_format: + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ +} + +/* + * Construct parray of pgFile from the backup content list. + * If root is not NULL, path will be absolute path. + */ +parray * +dir_read_file_list(const char *root, const char *file_txt) +{ + FILE *fp; + parray *files; + char buf[MAXPGPATH * 2]; + + fp = fopen(file_txt, "rt"); + if (fp == NULL) + elog(errno == ENOENT ? ERROR : ERROR, + "cannot open \"%s\": %s", file_txt, strerror(errno)); + + files = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char filepath[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + crc, + segno, + n_blocks; + pgFile *file; + + get_control_value(buf, "path", path, NULL, true); + get_control_value(buf, "size", NULL, &write_size, true); + get_control_value(buf, "mode", NULL, &mode, true); + get_control_value(buf, "is_datafile", NULL, &is_datafile, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "crc", NULL, &crc, true); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); + + if (root) + join_path_components(filepath, root, path); + else + strcpy(filepath, path); + + file = pgFileInit(filepath); + + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + file->linked = pgut_strdup(linked); + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + parray_append(files, file); + } + + fclose(fp); + return files; +} + +/* + * Check if directory empty. + */ +bool +dir_is_empty(const char *path) +{ + DIR *dir; + struct dirent *dir_ent; + + dir = opendir(path); + if (dir == NULL) + { + /* Directory in path doesn't exist */ + if (errno == ENOENT) + return true; + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + } + + errno = 0; + while ((dir_ent = readdir(dir))) + { + /* Skip entries point current dir or parent dir */ + if (strcmp(dir_ent->d_name, ".") == 0 || + strcmp(dir_ent->d_name, "..") == 0) + continue; + + /* Directory is not empty */ + closedir(dir); + return false; + } + if (errno) + elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + + closedir(dir); + + return true; +} + +/* + * Return true if the path is a existing regular file. + */ +bool +fileExists(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1 && errno == ENOENT) + return false; + else if (!S_ISREG(buf.st_mode)) + return false; + else + return true; +} + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/fetch.c b/src/fetch.c new file mode 100644 index 000000000..0d4dbdaaf --- /dev/null +++ b/src/fetch.c @@ -0,0 +1,116 @@ +/*------------------------------------------------------------------------- + * + * fetch.c + * Functions for fetching files from PostgreSQL data directory + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "catalog/catalog.h" + +#include +#include +#include +#include +#include +#include + +#include "pg_probackup.h" + +/* + * Read a file into memory. The file to be read is /. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) +{ + int fd; + char *buffer; + struct stat statbuf; + char fullpath[MAXPGPATH]; + int len; + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + if (fstat(fd, &statbuf) < 0) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (read(fd, buffer, len) != len) + { + if (safe) + return NULL; + else + elog(ERROR, "could not read file \"%s\": %s\n", + fullpath, strerror(errno)); + } + + close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + return buffer; +} + +/* + * Receive a single file as a malloc'd buffer. + */ +char * +fetchFile(PGconn *conn, const char *filename, size_t *filesize) +{ + PGresult *res; + char *result; + const char *params[1]; + int len; + + params[0] = filename; + res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", + 1, params, false, false); + + /* sanity check the result set */ + if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + filename); + + /* Read result to local variables */ + len = PQgetlength(res, 0, 0); + result = pg_malloc(len + 1); + memcpy(result, PQgetvalue(res, 0, 0), len); + result[len] = '\0'; + + PQclear(res); + *filesize = len; + + return result; +} diff --git a/src/help.c b/src/help.c new file mode 100644 index 000000000..dc9cc3d85 --- /dev/null +++ b/src/help.c @@ -0,0 +1,605 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_merge(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); +static void help_archive_push(void); +static void help_archive_get(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "archive-push") == 0) + help_archive_push(); + else if (strcmp(command, "archive-get") == 0) + help_archive_get(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); + + printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); + + printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); + + printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n")); + + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + printf(_(" --delete-wal remove redundant archived wal files\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress compress data files\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + + printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --recovery-target-action=pause|promote|shutdown\n")); + printf(_(" action the server should take once the recovery target is reached\n")); + printf(_(" (default: pause)\n")); + + printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); + printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-i backup-id | --expired] [--wal]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); + printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_merge(void) +{ + printf(_("%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} + +static void +help_archive_push(void) +{ + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_(" --overwrite overwrite archived WAL file\n")); +} + +static void +help_archive_get(void) +{ + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the archive\n")); +} diff --git a/src/init.c b/src/init.c new file mode 100644 index 000000000..cd559cb48 --- /dev/null +++ b/src/init.c @@ -0,0 +1,108 @@ +/*------------------------------------------------------------------------- + * + * init.c: - initialize backup catalog. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +/* + * Initialize backup catalog. + */ +int +do_init(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + int results; + + results = pg_check_dir(backup_path); + if (results == 4) /* exists and not empty*/ + elog(ERROR, "backup catalog already exist and it's not empty"); + else if (results == -1) /*trouble accessing directory*/ + { + int errno_tmp = errno; + elog(ERROR, "cannot open backup catalog directory \"%s\": %s", + backup_path, strerror(errno_tmp)); + } + + /* create backup catalog root directory */ + dir_create_dir(backup_path, DIR_PERMISSION); + + /* create backup catalog data directory */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* create backup catalog wal directory */ + join_path_components(arclog_path_dir, backup_path, "wal"); + dir_create_dir(arclog_path_dir, DIR_PERMISSION); + + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ + xlog_seg_size = get_xlog_seg_size(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + + /* + * Wite initial config. system-identifier and pgdata are set in + * init subcommand and will never be updated. + */ + pgBackupConfigInit(config); + config->system_identifier = system_identifier; + config->xlog_seg_size = xlog_seg_size; + config->pgdata = pgdata; + writeBackupCatalogConfigFile(config); + + elog(INFO, "Instance '%s' successfully inited", instance_name); + return 0; +} diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 000000000..979a1729f --- /dev/null +++ b/src/merge.c @@ -0,0 +1,526 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i, + num_files = parray_num(argument->files); + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/parsexlog.c b/src/parsexlog.c new file mode 100644 index 000000000..297269b68 --- /dev/null +++ b/src/parsexlog.c @@ -0,0 +1,1039 @@ +/*------------------------------------------------------------------------- + * + * parsexlog.c + * Functions for reading Write-Ahead-Log + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "commands/dbcommands_xlog.h" +#include "catalog/storage_xlog.h" +#include "access/transam.h" +#include "utils/thread.h" + +/* + * RmgrNames is an array of resource manager names, to make error messages + * a bit nicer. + */ +#if PG_VERSION_NUM >= 100000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ + name, +#else +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ + name, +#endif + +static const char *RmgrNames[RM_MAX_ID + 1] = { +#include "access/rmgrlist.h" +}; + +/* some from access/xact.h */ +/* + * XLOG allows to store some information in high 4 bits of log record xl_info + * field. We use 3 for the opcode, and one about an optional flag variable. + */ +#define XLOG_XACT_COMMIT 0x00 +#define XLOG_XACT_PREPARE 0x10 +#define XLOG_XACT_ABORT 0x20 +#define XLOG_XACT_COMMIT_PREPARED 0x30 +#define XLOG_XACT_ABORT_PREPARED 0x40 +#define XLOG_XACT_ASSIGNMENT 0x50 +/* free opcode 0x60 */ +/* free opcode 0x70 */ + +/* mask for filtering opcodes out of xl_info */ +#define XLOG_XACT_OPMASK 0x70 + +typedef struct xl_xact_commit +{ + TimestampTz xact_time; /* time of commit */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* xl_xact_dbinfo follows if XINFO_HAS_DBINFO */ + /* xl_xact_subxacts follows if XINFO_HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if XINFO_HAS_RELFILENODES */ + /* xl_xact_invals follows if XINFO_HAS_INVALS */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ + /* xl_xact_origin follows if XINFO_HAS_ORIGIN, stored unaligned! */ +} xl_xact_commit; + +typedef struct xl_xact_abort +{ + TimestampTz xact_time; /* time of abort */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* No db_info required */ + /* xl_xact_subxacts follows if HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if HAS_RELFILENODES */ + /* No invalidation messages needed. */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ +} xl_xact_abort; + +static void extractPageInfo(XLogReaderState *record); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); + +typedef struct XLogPageReadPrivate +{ + const char *archivedir; + TimeLineID tli; + uint32 xlog_seg_size; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif +} XLogPageReadPrivate; + +/* An argument for a thread function */ +typedef struct +{ + int thread_num; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + +static int SimpleXLogPageRead(XLogReaderState *xlogreader, + XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, + bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * extractPageMap() worker. + */ +static void * +doExtractPageMap(void *arg) +{ + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + char *errormsg; + + private_data = &extract_arg->private_data; +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; + + do + { + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) + { + private_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + private_data->xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (private_data->xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + GetXLogRecPtr(private_data->xlogsegno, 0, + private_data->xlog_seg_size, + extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(private_data, ERROR); + } + + extractPageInfo(xlogreader); + + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, + private_data->xlog_seg_size); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, bool prev_seg, + parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t *threads; + xlog_thread_arg *thread_args; + time_t start_time, + end_time; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, seg_size); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + time(&start_time); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, + seg_size, false); + thread_args[i].thread_num = i; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; + + threads_need++; + } + + /* Run threads */ + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + + pfree(threads); + pfree(thread_args); + + time(&end_time); + if (extract_isok) + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap compiling failed"); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, TimeLineID tli, + uint32 xlog_seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, + xlog_seg_size, true); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + PrintXLogCorruptionMsg(&private, WARNING); + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ +void +validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + const char *backup_id; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + TransactionId last_xid = InvalidTransactionId; + TimestampTz last_time = 0; + char last_timestamp[100], + target_timestamp[100]; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, + seg_size); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, + seg_size); + + if (backup->status == BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + return; + } + /* + * If recovery target is provided check that we can restore backup to a + * recovery target time or xid. + */ + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + { + /* Recovery target is not given so exit */ + elog(INFO, "Backup %s WAL segments are valid", backup_id); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, + true); + + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + all_wal = true; + + startpoint = backup->stop_lsn; + while (true) + { + bool timestamp_record; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + timestamp_record = getRecordTimestamp(xlogreader, &last_time); + if (XLogRecGetXid(xlogreader) != InvalidTransactionId) + last_xid = XLogRecGetXid(xlogreader); + + /* Check target xid */ + if (TransactionIdIsValid(target_xid) && target_xid == last_xid) + { + all_wal = true; + break; + } + /* Check target time */ + else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + { + all_wal = true; + break; + } + /* If there are no target xid and target time */ + else if (!TransactionIdIsValid(target_xid) && target_time == 0 && + xlogreader->ReadRecPtr == backup->stop_lsn) + { + all_wal = true; + /* We don't stop here. We want to get last_xid and last_time */ + } + + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (last_time > 0) + time2iso(last_timestamp, lengthof(last_timestamp), + timestamptz_to_time_t(last_time)); + + /* There are all needed WAL records */ + if (all_wal) + elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, + last_timestamp, last_xid); + /* Some needed WAL records are absent */ + else + { + PrintXLogCorruptionMsg(&private, WARNING); + + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); + + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Read from archived WAL segments latest recovery time and xid. All necessary + * segments present at archive folder. We waited **stop_lsn** in + * pg_stop_backup(). + */ +bool +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, TransactionId *recovery_xid) +{ + XLogRecPtr startpoint = stop_lsn; + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + bool res; + + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + + /* Read records from stop_lsn down to start_lsn */ + do + { + XLogRecord *record; + TimestampTz last_time = 0; + char *errormsg; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(ERROR, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(ERROR, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + } + + /* Read previous record */ + startpoint = record->xl_prev; + + if (getRecordTimestamp(xlogreader, &last_time)) + { + *recovery_time = timestamptz_to_time_t(last_time); + *recovery_xid = XLogRecGetXid(xlogreader); + + /* Found timestamp in WAL record 'record' */ + res = true; + goto cleanup; + } + } while (startpoint >= start_lsn); + + /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ + res = false; + +cleanup: + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +/* + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. + */ +bool +wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size) +{ + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + char *errormsg; + bool res; + + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, + true); + + res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; + /* Didn't find 'target_lsn' and there is no error, return false */ + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf) +{ + int errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &errnum); + if (errnum == Z_ERRNO) + return strerror(errno); + else + return errmsg; +} +#endif + +/* XLogreader callback function, to read a WAL page */ +static int +SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI) +{ + XLogPageReadPrivate *private_data; + uint32 targetPageOff; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + targetPageOff = targetPagePtr % private_data->xlog_seg_size; + + /* + * See if we need to switch to a new segment because the requested record + * is not in the currently open one. + */ + if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size)) + { + CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ + if (private_data->manual_switch) + { + private_data->need_switch = true; + return -1; + } + } + + GetXLogSegNo(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size); + + /* Try to switch to the next WAL segment */ + if (!private_data->xlogexists) + { + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, + private_data->xlog_seg_size); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); + + if (fileExists(private_data->xlogpath)) + { + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); + + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); + + if (private_data->xlogfile < 0) + { + elog(WARNING, "Could not open WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + /* Try to open compressed WAL segment */ + else + { + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) + { + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); + + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) + { + elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, strerror(errno)); + return -1; + } + } + } +#endif + + /* Exit without error if WAL segment doesn't exist */ + if (!private_data->xlogexists) + return -1; + } + + /* + * At this point, we have the right segment open. + */ + Assert(private_data->xlogexists); + + /* Read the requested page */ + if (private_data->xlogfile != -1) + { + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + { + elog(WARNING, "Could not seek in WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + else + { + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + { + elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + } +#endif + + *pageTLI = private_data->tli; + return XLOG_BLCKSZ; +} + +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlog_seg_size = xlog_seg_size; + private_data->xlogfile = -1; + + if (allocate_reader) + { +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + +/* + * Extract information about blocks modified in this record. + */ +static void +extractPageInfo(XLogReaderState *record) +{ + uint8 block_id; + RmgrId rmid = XLogRecGetRmid(record); + uint8 info = XLogRecGetInfo(record); + uint8 rminfo = info & ~XLR_INFO_MASK; + + /* Is this a special record type that I recognize? */ + + if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + { + /* + * New databases can be safely ignored. They would be completely + * copied if found. + */ + } + else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP) + { + /* + * An existing database was dropped. It is fine to ignore that + * they will be removed appropriately. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE) + { + /* + * We can safely ignore these. The file will be removed when + * combining the backups in the case of differential on. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE) + { + /* + * We can safely ignore these. When we compare the sizes later on, + * we'll notice that they differ, and copy the missing tail from + * source system. + */ + } + else if (info & XLR_SPECIAL_REL_UPDATE) + { + /* + * This record type modifies a relation file in some special way, but + * we don't recognize the type. That's bad - we don't know how to + * track that change. + */ + elog(ERROR, "WAL record modifies a relation, but record type is not recognized\n" + "lsn: %X/%X, rmgr: %s, info: %02X", + (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), + RmgrNames[rmid], info); + } + + for (block_id = 0; block_id <= record->max_block_id; block_id++) + { + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blkno; + + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + continue; + + /* We only care about the main fork; others are copied in toto */ + if (forknum != MAIN_FORKNUM) + continue; + + process_block_change(forknum, rnode, blkno); + } +} + +/* + * Extract timestamp from WAL record. + * + * If the record contains a timestamp, returns true, and saves the timestamp + * in *recordXtime. If the record type has no timestamp, returns false. + * Currently, only transaction commit/abort records and restore points contain + * timestamps. + */ +static bool +getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + uint8 xact_info = info & XLOG_XACT_OPMASK; + uint8 rmid = XLogRecGetRmid(record); + + if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT) + { + *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT || + xact_info == XLOG_XACT_COMMIT_PREPARED)) + { + *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT || + xact_info == XLOG_XACT_ABORT_PREPARED)) + { + *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time; + return true; + } + + return false; +} + diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 000000000..a39ea5a8c --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,634 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" +#include "utils/thread.h" + +#include +#include +#include +#include +#include +#include "pg_getopt.h" + +const char *PROGRAM_VERSION = "2.0.18"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +static char *backup_id_string = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; +#if PG_VERSION_NUM >= 100000 +char *replication_slot = NULL; +#endif + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool is_remote_backup = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_lsn; +static char *target_inclusive; +static TimeLineID target_tli; +static bool target_immediate; +static char *target_name = NULL; +static char *target_action = NULL; + +static pgRecoveryTarget *recovery_target_options = NULL; + +bool restore_as_replica = false; +bool restore_no_validate = false; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; +bool compress_shortcut = false; + + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ +#if PG_VERSION_NUM >= 110000 +uint32 xlog_seg_size = 0; +#else +uint32 xlog_seg_size = XLOG_SEG_SIZE; +#endif + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; +static bool file_overwrite = false; + +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd = NO_CMD; + +static bool help_opt = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); + +static void compress_init(void); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + { 'b', 12, "delete-wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 13, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 14, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 15, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* TODO not completed feature. Make it unavailiable from user level + { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, + { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, + { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 136, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 137, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 138, "compress", &compress_shortcut, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 140, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 141, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 142, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + { 'b', 'W', "password", &force_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, +#if PG_VERSION_NUM >= 110000 + { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive-push options */ + { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char *command = NULL, + *command_name; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackupInit(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + +#if PG_VERSION_NUM >= 110000 + /* + * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() + * later. + */ + WalSegSz = 0; +#endif + + /* + * Save main thread's tid. It is used call exit() in case of errors. + */ + main_tid = pthread_self(); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + + /* + * Make command string before getopt_long() will call. It permutes the + * content of argv. + */ + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) + { + int i, + len = 0, + allocated = 0; + + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) + { + int arglen = strlen(argv[i]); + + if (arglen + len > allocated) + { + allocated *= 2; + command = repalloc(command, allocated); + } + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; + } + + command[len] = '\0'; + } + + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help_opt) + help_command(command_name); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + canonicalize_path(backup_path); + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG_CMD) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, true); + } + + /* Initialize logger */ + init_logger(backup_path); + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + +#if PG_VERSION_NUM >= 110000 + /* Check xlog-seg-size option */ + if (instance_name && + backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); +#endif + + /* Sanity check of --backup-id option */ + if (backup_id_string != NULL) + { + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); + + current.backup_id = base36dec(backup_id_string); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) + { + /* parse all recovery target options into recovery_target_options structure */ + recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, + target_inclusive, target_tli, target_lsn, target_immediate, + target_name, target_action, restore_no_validate); + } + + if (num_threads < 1) + num_threads = 1; + + compress_init(); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH_CMD: + return do_archive_push(wal_file_path, wal_file_name, file_overwrite); + case ARCHIVE_GET_CMD: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE_CMD: + return do_add_instance(); + case DELETE_INSTANCE_CMD: + return do_delete_instance(); + case INIT_CMD: + return do_init(); + case BACKUP_CMD: + { + const char *backup_mode; + time_t start_time; + + start_time = time(NULL); + backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; + + elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", + PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, + stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + + return do_backup(start_time); + } + case RESTORE_CMD: + return do_restore_or_validate(current.backup_id, + recovery_target_options, + true); + case VALIDATE_CMD: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + recovery_target_options, + false); + case SHOW_CMD: + return do_show(current.backup_id); + case DELETE_CMD: + if (delete_expired && backup_id_string) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (!delete_expired && !delete_wal && !backup_id_string) + elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); + if (delete_wal && !delete_expired && !backup_id_string) + return do_retention_purge(); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case MERGE_CMD: + do_merge(current.backup_id); + break; + case SHOW_CONFIG_CMD: + return do_configure(true); + case SET_CONFIG_CMD: + return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + log_level_file = parse_log_level(arg); +} + +static void +opt_show_format(pgut_option *opt, const char *arg) +{ + const char *v = arg; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0) + { + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); + } + else + elog(ERROR, "Invalid show format \"%s\"", arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize compress and sanity checks for compress. + */ +static void +compress_init(void) +{ + /* Default algorithm is zlib */ + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + + if (backup_subcmd != SET_CONFIG_CMD) + { + if (compress_level != COMPRESS_LEVEL_DEFAULT + && compress_alg == NOT_DEFINED_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + if (compress_level == 0) + compress_alg = NOT_DEFINED_COMPRESS; + + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + { +#ifndef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + elog(ERROR, "This build does not support zlib compression"); + else +#endif + if (compress_alg == PGLZ_COMPRESS && num_threads > 1) + elog(ERROR, "Multithread backup does not support pglz compression"); + } +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h new file mode 100644 index 000000000..8f3a0fea1 --- /dev/null +++ b/src/pg_probackup.h @@ -0,0 +1,620 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.h: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_H +#define PG_PROBACKUP_H + +#include "postgres_fe.h" + +#include +#include + +#include "access/timeline.h" +#include "access/xlogdefs.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "utils/pg_crc.h" +#include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif + +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + +/* Directory/File names */ +#define DATABASE_DIR "database" +#define BACKUPS_DIR "backups" +#if PG_VERSION_NUM >= 100000 +#define PG_XLOG_DIR "pg_wal" +#else +#define PG_XLOG_DIR "pg_xlog" +#endif +#define PG_TBLSPC_DIR "pg_tblspc" +#define PG_GLOBAL_DIR "global" +#define BACKUP_CONTROL_FILE "backup.control" +#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" +#define BACKUP_CATALOG_PID "pg_probackup.pid" +#define DATABASE_FILE_LIST "backup_content.control" +#define PG_BACKUP_LABEL_FILE "backup_label" +#define PG_BLACK_LIST "black_list" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" + +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" +/* Direcotry/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + +/* 64-bit xid support for PGPRO_EE */ +#ifndef PGPRO_EE +#define XID_FMT "%u" +#endif + +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + +/* Information about single file (or dir) in backup */ +typedef struct pgFile +{ + char *name; /* file or directory name */ + mode_t mode; /* protection (file type and permission) */ + size_t size; /* size of the file */ + size_t read_size; /* size of the portion read (if only some pages are + backed up, it's different from size) */ + int64 write_size; /* size of the backed-up file. BYTES_INVALID means + that the file existed but was not backed up + because not modified since last backup. */ + /* we need int64 here to store '-1' value */ + pg_crc32 crc; /* CRC value of the file, regular file only */ + char *linked; /* path of the linked file */ + bool is_datafile; /* true if the file is PostgreSQL data file */ + char *path; /* absolute path of the file */ + Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ + Oid dbOid; /* dbOid extracted from path, if applicable */ + Oid relOid; /* relOid extracted from path, if applicable */ + char *forkName; /* forkName extracted from path, if applicable */ + int segno; /* Segment number for ptrack */ + int n_blocks; /* size of the file in blocks, readed during DELTA backup */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + bool is_database; + bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ +} pgFile; + +/* Special values of datapagemap_t bitmapsize */ +#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ + +/* Current state of backup */ +typedef enum BackupStatus +{ + BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ + BACKUP_STATUS_OK, /* completed backup */ + BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ + BACKUP_STATUS_DELETING, /* data files are being deleted */ + BACKUP_STATUS_DELETED, /* data files have been deleted */ + BACKUP_STATUS_DONE, /* completed but not validated yet */ + BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ +} BackupStatus; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID = 0, + BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ + BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ + BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD +} ProbackupSubcmd; + +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + + +/* special values of pgBackup fields */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) + +typedef struct pgBackupConfig +{ + uint64 system_identifier; + uint32 xlog_seg_size; + + char *pgdata; + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; + + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int archive_timeout; + + int log_level_console; + int log_level_file; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + + uint32 retention_redundancy; + uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; +} pgBackupConfig; + + +/* Information about single backup stored in backup.conf */ + + +typedef struct pgBackup pgBackup; + +struct pgBackup +{ + BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ + time_t backup_id; /* Identifier of the backup. + * Currently it's the same as start_time */ + BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ + TimeLineID tli; /* timeline of start and stop baskup lsns */ + XLogRecPtr start_lsn; /* backup's starting transaction log location */ + XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ + time_t start_time; /* since this moment backup has status + * BACKUP_STATUS_RUNNING */ + time_t end_time; /* the moment when backup was finished, or the moment + * when we realized that backup is broken */ + time_t recovery_time; /* Earliest moment for which you can restore + * the state of the database cluster using + * this backup */ + TransactionId recovery_xid; /* Earliest xid for which you can restore + * the state of the database cluster using + * this backup */ + /* + * Amount of raw data. For a full backup, this is the total amount of + * data while for a differential backup this is just the difference + * of data taken. + * BYTES_INVALID means nothing was backed up. + */ + int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; + + CompressAlg compress_alg; + int compress_level; + + /* Fields needed for compatibility check */ + uint32 block_size; + uint32 wal_block_size; + uint32 checksum_version; + + char program_version[100]; + char server_version[100]; + + bool stream; /* Was this backup taken in stream mode? + * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ + time_t parent_backup; /* Identifier of the previous backup. + * Which is basic backup for this + * incremental backup. */ + pgBackup *parent_backup_link; + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ +}; + +/* Recovery target for restore and validate subcommands */ +typedef struct pgRecoveryTarget +{ + bool time_specified; + time_t recovery_target_time; + /* add one more field in order to avoid deparsing recovery_target_time back */ + const char *target_time_string; + bool xid_specified; + TransactionId recovery_target_xid; + /* add one more field in order to avoid deparsing recovery_target_xid back */ + const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; + TimeLineID recovery_target_tli; + bool recovery_target_inclusive; + bool inclusive_specified; + bool recovery_target_immediate; + const char *recovery_target_name; + const char *recovery_target_action; + bool restore_no_validate; +} pgRecoveryTarget; + +/* Union to ease operations on relation pages */ +typedef union DataPage +{ + PageHeaderData page_data; + char data[BLCKSZ]; +} DataPage; + +typedef struct +{ + const char *from_root; + const char *to_root; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} backup_files_arg; + +/* + * return pointer that exceeds the length of prefix from character string. + * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". + */ +#define GetRelativePath(str, prefix) \ + ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) + +/* + * Return timeline, xlog ID and record offset from an LSN of the type + * 0/B000188, usual result from pg_stop_backup() and friends. + */ +#define XLogDataFromLSN(data, xlogid, xrecoff) \ + sscanf(data, "%X/%X", xlogid, xrecoff) + +#define IsCompressedXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) + +#if PG_VERSION_NUM >= 110000 +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#else +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo) +#endif + +/* directory options */ +extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; +extern char *pgdata; +extern char arclog_path[MAXPGPATH]; + +/* common options */ +extern int num_threads; +extern bool stream_wal; +extern bool progress; +#if PG_VERSION_NUM >= 100000 +/* In pre-10 'replication_slot' is defined in receivelog.h */ +extern char *replication_slot; +#endif + +/* backup options */ +extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 +extern uint32 archive_timeout; +extern bool is_remote_backup; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 +extern uint32 replica_timeout; + +extern bool is_ptrack_support; +extern bool is_checksum_enabled; +extern bool exclusive_backup; + +/* restore options */ +extern bool restore_as_replica; + +/* delete options */ +extern bool delete_wal; +extern bool delete_expired; +extern bool apply_to_all; +extern bool force_delete; + +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + +extern uint32 retention_redundancy; +extern uint32 retention_window; + +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; +extern bool compress_shortcut; + +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; +extern uint32 xlog_seg_size; + +/* show options */ +extern ShowFormat show_format; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + +/* in dir.c */ +/* exclude directory list for $PGDATA file listing */ +extern const char *pgdata_exclude_dir[]; + +/* in backup.c */ +extern int do_backup(time_t start_time); +extern BackupMode parse_backup_mode(const char *value); +extern const char *deparse_backup_mode(BackupMode mode); +extern void process_block_change(ForkNumber forknum, RelFileNode rnode, + BlockNumber blkno); + +extern char *pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, Oid tblsOid, Oid relOid, + BlockNumber blknum, + size_t *result_size); +/* in restore.c */ +extern int do_restore_or_validate(time_t target_backup_id, + pgRecoveryTarget *rt, + bool is_restore); +extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_recovery_target(const pgBackup *backup, + const pgRecoveryTarget *rt); +extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); +extern pgRecoveryTarget *parseRecoveryTargetOptions( + const char *target_time, const char *target_xid, + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); + +/* in merge.c */ +extern void do_merge(time_t backup_id); + +/* in init.c */ +extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name, + bool overwrite); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + + +/* in configure.c */ +extern int do_configure(bool show_only); +extern void pgBackupConfigInit(pgBackupConfig *config); +extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); +extern void writeBackupCatalogConfigFile(pgBackupConfig *config); +extern pgBackupConfig* readBackupCatalogConfigFile(void); + +extern uint32 get_config_xlog_seg_size(void); + +/* in show.c */ +extern int do_show(time_t requested_backup_id); + +/* in delete.c */ +extern int do_delete(time_t backup_id); +extern int do_retention_purge(void); +extern int do_delete_instance(void); + +/* in fetch.c */ +extern char *slurpFile(const char *datadir, + const char *path, + size_t *filesize, + bool safe); +extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); + +/* in help.c */ +extern void help_pg_probackup(void); +extern void help_command(char *command); + +/* in validate.c */ +extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); + +/* in catalog.c */ +extern pgBackup *read_backup(time_t timestamp); +extern const char *pgBackupGetBackupMode(pgBackup *backup); + +extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern pgBackup *catalog_get_last_data_backup(parray *backup_list, + TimeLineID tli); +extern void catalog_lock(void); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); +extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); +extern void pgBackupFree(void *backup); +extern int pgBackupCompareId(const void *f1, const void *f2); +extern int pgBackupCompareIdDesc(const void *f1, const void *f2); + +extern pgBackup* find_parent_backup(pgBackup *current_backup); + +/* in dir.c */ +extern void dir_list_file(parray *files, const char *root, bool exclude, + bool omit_symlink, bool add_root); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); + +extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); + +extern void print_file_list(FILE *out, const parray *files, const char *root); +extern parray *dir_read_file_list(const char *root, const char *file_txt); + +extern int dir_create_dir(const char *path, mode_t mode); +extern bool dir_is_empty(const char *path); + +extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); + +extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileInit(const char *path); +extern void pgFileDelete(pgFile *file); +extern void pgFileFree(void *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); +extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileComparePathDesc(const void *f1, const void *f2); +extern int pgFileCompareLinked(const void *f1, const void *f2); +extern int pgFileCompareSize(const void *f1, const void *f2); + +/* in data.c */ +extern bool backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); +extern void push_wal_file(const char *from_path, const char *to_path, + bool is_compress, bool overwrite); +extern void get_wal_file(const char *from_path, const char *to_path); + +extern bool calc_file_checksum(pgFile *file); + +/* parsexlog.c */ +extern void extractPageMap(const char *datadir, + TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool prev_seg, parray *backup_files_list); +extern void validate_wal(pgBackup *backup, + const char *archivedir, + time_t target_time, + TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size); +extern bool read_recovery_info(const char *archivedir, TimeLineID tli, + uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, + TransactionId *recovery_xid); +extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size); + +/* in util.c */ +extern TimeLineID get_current_timeline(bool safe); +extern void sanityChecks(void); +extern void time2iso(char *buf, size_t len, time_t time); +extern const char *status2str(BackupStatus status); +extern void remove_trailing_space(char *buf, int comment_mark); +extern void remove_not_digit(char *buf, size_t len, const char *str); +extern uint32 get_data_checksum_version(bool safe); +extern const char *base36enc(long unsigned int value); +extern char *base36enc_dup(long unsigned int value); +extern long unsigned int base36dec(const char *text); +extern uint64 get_system_identifier(char *pgdata); +extern uint64 get_remote_system_identifier(PGconn *conn); +extern uint32 get_xlog_seg_size(char *pgdata_path); +extern pg_time_t timestamptz_to_time_t(TimestampTz t); +extern int parse_server_version(char *server_version_str); + +/* in status.c */ +extern bool is_pg_running(void); + +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + +#endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c new file mode 100644 index 000000000..3396b6f61 --- /dev/null +++ b/src/restore.c @@ -0,0 +1,919 @@ +/*------------------------------------------------------------------------- + * + * restore.c: restore DB cluster and archived WAL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "catalog/pg_control.h" +#include "utils/logger.h" +#include "utils/thread.h" + +typedef struct +{ + parray *files; + pgBackup *backup; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} restore_files_arg; + +static void restore_backup(pgBackup *backup); +static void create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup); +static void *restore_files(void *arg); +static void remove_deleted_files(pgBackup *backup); + + +/* + * Entry point of pg_probackup RESTORE and VALIDATE subcommands. + */ +int +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) +{ + int i = 0; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *dest_backup = NULL; + pgBackup *base_full_backup = NULL; + pgBackup *corrupted_backup = NULL; + int dest_backup_index = 0; + int base_full_backup_index = 0; + int corrupted_backup_index = 0; + char *action = is_restore ? "Restore":"Validate"; + + if (is_restore) + { + if (pgdata == NULL) + elog(ERROR, + "required parameter not specified: PGDATA (-D, --pgdata)"); + /* Check if restore destination empty */ + if (!dir_is_empty(pgdata)) + elog(ERROR, "restore destination is not empty: \"%s\"", pgdata); + } + + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + + elog(LOG, "%s begin.", action); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find backup range we should restore or validate. */ + while ((i < parray_num(backups)) && !dest_backup) + { + current_backup = (pgBackup *) parray_get(backups, i); + i++; + + /* Skip all backups which started after target backup */ + if (target_backup_id && current_backup->start_time > target_backup_id) + continue; + + /* + * [PGPRO-1164] If BACKUP_ID is not provided for restore command, + * we must find the first valid(!) backup. + */ + + if (is_restore && + target_backup_id == INVALID_BACKUP_ID && + current_backup->status != BACKUP_STATUS_OK) + { + elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + continue; + } + + /* + * We found target backup. Check its status and + * ensure that it satisfies recovery target. + */ + if ((target_backup_id == current_backup->start_time + || target_backup_id == INVALID_BACKUP_ID)) + { + + /* backup is not ok, + * but in case of CORRUPT, ORPHAN or DONE revalidation can be done, + * in other cases throw an error. + */ + if (current_backup->status != BACKUP_STATUS_OK) + { + if (current_backup->status == BACKUP_STATUS_DONE || + current_backup->status == BACKUP_STATUS_ORPHAN || + current_backup->status == BACKUP_STATUS_CORRUPT) + elog(WARNING, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + } + + if (rt->recovery_target_tli) + { + parray *timelines; + + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); + /* Read timeline history files from archives */ + timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + + if (!satisfy_timeline(timelines, current_backup)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy target timeline", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target timeline */ + continue; + } + } + + if (!satisfy_recovery_target(current_backup, rt)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy restore options", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target options */ + continue; + } + + /* + * Backup is fine and satisfies all recovery options. + * Save it as dest_backup + */ + dest_backup = current_backup; + dest_backup_index = i-1; + } + } + + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) + { + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) + { + base_full_backup_index = i; + break; + } + } + } + + if (base_full_backup == NULL) + elog(ERROR, "Full backup satisfying target options is not found."); + + /* + * Ensure that directories provided in tablespace mapping are valid + * i.e. empty or not exist. + */ + if (is_restore) + check_tablespace_mapping(dest_backup); + + if (!is_restore || !rt->restore_no_validate) + { + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + + /* + * Validate backups from base_full_backup to dest_backup. + */ + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) + { + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli, xlog_seg_size); + + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; + + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } + } + } + } + + /* + * If dest backup is corrupted or was orphaned in previous check + * produce corresponding error message + */ + if (dest_backup->status == BACKUP_STATUS_OK) + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } + else if (dest_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + else if (dest_backup->status == BACKUP_STATUS_ORPHAN) + elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); + + /* We ensured that all backups are valid, now restore if required */ + if (is_restore) + { + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + + restore_backup(backup); + } + + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + remove_deleted_files(dest_backup); + + /* Create recovery.conf with given recovery target parameters */ + create_recovery_conf(target_backup_id, rt, dest_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + return 0; +} + +/* + * Restore one backup. + */ +void +restore_backup(pgBackup *backup) +{ + char timestamp[100]; + char this_backup_path[MAXPGPATH]; + char database_path[MAXPGPATH]; + char list_path[MAXPGPATH]; + parray *files; + int i; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg *threads_args; + bool restore_isok = true; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + elog(LOG, "restoring database from backup %s", timestamp); + + /* + * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id + */ + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + create_data_directories(pgdata, this_backup_path, true); + + /* + * Get list of files which need to be restored. + */ + pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); + pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); + files = dir_read_file_list(database_path, list_path); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_clear_flag(&file->lock); + } + + /* Restore files into target directory */ + for (i = 0; i < num_threads; i++) + { + restore_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->backup = backup; + /* By default there are some error */ + threads_args[i].ret = 1; + + elog(LOG, "Start thread for num:%li", parray_num(files)); + + pthread_create(&threads[i], NULL, restore_files, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + restore_isok = false; + } + if (!restore_isok) + elog(ERROR, "Data files restoring failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); +} + +/* + * Delete files which are not in backup's file list from target pgdata. + * It is necessary to restore incremental backup correctly. + * Files which were deleted between previous and current backup + * are not in the backup's filelist. + */ +static void +remove_deleted_files(pgBackup *backup) +{ + parray *files; + parray *files_restored; + char filelist_path[MAXPGPATH]; + int i; + + pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); + /* Read backup's filelist using target database path as base path */ + files = dir_read_file_list(pgdata, filelist_path); + parray_qsort(files, pgFileComparePathDesc); + + /* Get list of files actually existing in target database */ + files_restored = parray_new(); + dir_list_file(files_restored, pgdata, true, true, false); + /* To delete from leaf, sort in reversed order */ + parray_qsort(files_restored, pgFileComparePathDesc); + + for (i = 0; i < parray_num(files_restored); i++) + { + pgFile *file = (pgFile *) parray_get(files_restored, i); + + /* If the file is not in the file list, delete it */ + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); + } + } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + parray_walk(files_restored, pgFileFree); + parray_free(files_restored); +} + +/* + * Restore files into $PGDATA. + */ +static void * +restore_files(void *arg) +{ + int i; + restore_files_arg *arguments = (restore_files_arg *)arg; + + for (i = 0; i < parray_num(arguments->files); i++) + { + char from_root[MAXPGPATH]; + char *rel_path; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + pgBackupGetPath(arguments->backup, from_root, + lengthof(from_root), DATABASE_DIR); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during restore database"); + + rel_path = GetRelativePath(file->path,from_root); + + if (progress) + elog(LOG, "Progress: (%d/%lu). Process file %s ", + i + 1, (unsigned long) parray_num(arguments->files), rel_path); + + /* + * For PAGE and PTRACK backups skip files which haven't changed + * since previous backup and thus were not backed up. + * We cannot do the same when restoring DELTA backup because we need information + * about every file to correctly truncate them. + */ + if (file->write_size == BYTES_INVALID && + (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE + || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) + { + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + { + elog(VERBOSE, "directory, skip"); + continue; + } + + /* Do not restore tablespace_map file */ + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) + { + elog(VERBOSE, "skip tablespace_map"); + continue; + } + + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } + else + copy_file(from_root, pgdata, file); + + /* print size of restored file */ + if (file->write_size != BYTES_INVALID) + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files restoring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* Create recovery.conf with given recovery target parameters */ +static void +create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup) +{ + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; + + if (!backup->stream + || (rt->time_specified || rt->xid_specified)) + need_restore_conf = true; + + /* No need to generate recovery.conf at all. */ + if (!(need_restore_conf || restore_as_replica)) + return; + + elog(LOG, "----------------------------------------"); + elog(LOG, "creating recovery.conf"); + + snprintf(path, lengthof(path), "%s/recovery.conf", pgdata); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, + strerror(errno)); + + fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (need_restore_conf) + { + + fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " + "--wal-file-path %%p --wal-file-name %%f'\n", + PROGRAM_NAME, backup_path, instance_name); + + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->recovery_target_name) + fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + + if (rt->time_specified) + fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + + if (rt->xid_specified) + fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + + if (rt->recovery_target_immediate) + fprintf(fp, "recovery_target = 'immediate'\n"); + + if (rt->inclusive_specified) + fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->recovery_target_inclusive?"true":"false"); + + if (rt->recovery_target_tli) + fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + + if (rt->recovery_target_action) + fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + } + + if (restore_as_replica) + { + fprintf(fp, "standby_mode = 'on'\n"); + + if (backup->primary_conninfo) + fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + } + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, + strerror(errno)); +} + +/* + * Try to read a timeline's history file. + * + * If successful, return the list of component TLIs (the ancestor + * timelines followed by target timeline). If we cannot find the history file, + * assume that the timeline has no parents, and return a list of just the + * specified timeline ID. + * based on readTimeLineHistory() in timeline.c + */ +parray * +readTimeLineHistory_probackup(TimeLineID targetTLI) +{ + parray *result; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd = NULL; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + + /* Look for timeline history file in archlog_path */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + targetTLI); + + /* Timeline 1 does not have a history file */ + if (targetTLI != 1) + { + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* There is no history file for target timeline */ + elog(ERROR, "recovery target timeline %u does not exist", + targetTLI); + } + } + + result = parray_new(); + + /* + * Parse the file... + */ + while (fd && fgets(fline, sizeof(fline), fd) != NULL) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = fline; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(fline, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, + "syntax error in history file: %s. Expected a numeric timeline ID.", + fline); + } + if (nfields != 3) + elog(ERROR, + "syntax error in history file: %s. Expected a transaction log switchpoint location.", + fline); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, + "Timeline IDs must be in increasing sequence."); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + parray_insert(result, 0, entry); + + /* we ignore the remainder of each line */ + } + + if (fd) + fclose(fd); + + if (last_timeline && targetTLI <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + + /* append target timeline */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = targetTLI; + /* LSN in target timeline is valid */ + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; + parray_insert(result, 0, entry); + + return result; +} + +bool +satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) +{ + if (rt->xid_specified) + return backup->recovery_xid <= rt->recovery_target_xid; + + if (rt->time_specified) + return backup->recovery_time <= rt->recovery_target_time; + + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + + return true; +} + +bool +satisfy_timeline(const parray *timelines, const pgBackup *backup) +{ + int i; + + for (i = 0; i < parray_num(timelines); i++) + { + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + if (backup->tli == timeline->tli && + backup->stop_lsn < timeline->end) + return true; + } + return false; +} +/* + * Get recovery options in the string format, parse them + * and fill up the pgRecoveryTarget structure. + */ +pgRecoveryTarget * +parseRecoveryTargetOptions(const char *target_time, + const char *target_xid, + const char *target_inclusive, + TimeLineID target_tli, + const char *target_lsn, + bool target_immediate, + const char *target_name, + const char *target_action, + bool restore_no_validate) +{ + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; + XLogRecPtr dummy_lsn; + /* + * count the number of the mutually exclusive options which may specify + * recovery target. If final value > 1, throw an error. + */ + int recovery_target_specified = 0; + pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); + + /* fill all options with default values */ + rt->time_specified = false; + rt->xid_specified = false; + rt->inclusive_specified = false; + rt->lsn_specified = false; + rt->recovery_target_time = 0; + rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; + rt->target_time_string = NULL; + rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; + rt->recovery_target_inclusive = false; + rt->recovery_target_tli = 0; + rt->recovery_target_immediate = false; + rt->recovery_target_name = NULL; + rt->recovery_target_action = NULL; + rt->restore_no_validate = false; + + /* parse given options */ + if (target_time) + { + recovery_target_specified++; + rt->time_specified = true; + rt->target_time_string = target_time; + + if (parse_time(target_time, &dummy_time, false)) + rt->recovery_target_time = dummy_time; + else + elog(ERROR, "Invalid value of --time option %s", target_time); + } + + if (target_xid) + { + recovery_target_specified++; + rt->xid_specified = true; + rt->target_xid_string = target_xid; + +#ifdef PGPRO_EE + if (parse_uint64(target_xid, &dummy_xid, 0)) +#else + if (parse_uint32(target_xid, &dummy_xid, 0)) +#endif + rt->recovery_target_xid = dummy_xid; + else + elog(ERROR, "Invalid value of --xid option %s", target_xid); + } + + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + + if (target_inclusive) + { + rt->inclusive_specified = true; + if (parse_bool(target_inclusive, &dummy_bool)) + rt->recovery_target_inclusive = dummy_bool; + else + elog(ERROR, "Invalid value of --inclusive option %s", target_inclusive); + } + + rt->recovery_target_tli = target_tli; + if (target_immediate) + { + recovery_target_specified++; + rt->recovery_target_immediate = target_immediate; + } + + if (restore_no_validate) + { + rt->restore_no_validate = restore_no_validate; + } + + if (target_name) + { + recovery_target_specified++; + rt->recovery_target_name = target_name; + } + + if (target_action) + { + rt->recovery_target_action = target_action; + + if ((strcmp(target_action, "pause") != 0) + && (strcmp(target_action, "promote") != 0) + && (strcmp(target_action, "shutdown") != 0)) + elog(ERROR, "Invalid value of --recovery-target-action option %s", target_action); + } + else + { + /* Default recovery target action is pause */ + rt->recovery_target_action = "pause"; + } + + /* More than one mutually exclusive option was defined. */ + if (recovery_target_specified > 1) + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); + + /* If none of the options is defined, '--inclusive' option is meaningless */ + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) + elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); + + return rt; +} diff --git a/src/show.c b/src/show.c new file mode 100644 index 000000000..f240ce933 --- /dev/null +++ b/src/show.c @@ -0,0 +1,500 @@ +/*------------------------------------------------------------------------- + * + * show.c: show backup information. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static int32 json_level = 0; + +int +do_show(time_t requested_backup_id) +{ + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); + + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + + show_instance(INVALID_BACKUP_ID, true); + } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + + return 0; + } + else + return show_backup(requested_backup_id); +} + +static void +pretty_size(int64 size, char *buf, size_t len) +{ + int exp = 0; + + /* minus means the size is invalid */ + if (size < 0) + { + strncpy(buf, "----", len); + return; + } + + /* determine postfix */ + while (size > 9999) + { + ++exp; + size /= 1000; + } + + switch (exp) + { + case 0: + snprintf(buf, len, "%dB", (int) size); + break; + case 1: + snprintf(buf, len, "%dkB", (int) size); + break; + case 2: + snprintf(buf, len, "%dMB", (int) size); + break; + case 3: + snprintf(buf, len, "%dGB", (int) size); + break; + case 4: + snprintf(buf, len, "%dTB", (int) size); + break; + case 5: + snprintf(buf, len, "%dPB", (int) size); + break; + default: + strncpy(buf, "***", len); + break; + } +} + +static TimeLineID +get_parent_tli(TimeLineID child_tli) +{ + TimeLineID result = 0; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd; + + /* Timeline 1 does not have a history file and parent timeline */ + if (child_tli == 1) + return 0; + + /* Search history file in archives */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + child_tli); + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* Did not find history file, do not raise the error */ + return 0; + } + + /* + * Parse the file... + */ + while (fgets(fline, sizeof(fline), fd) != NULL) + { + /* skip leading whitespace and check for # comment */ + char *ptr; + char *endptr; + + for (ptr = fline; *ptr; ptr++) + { + if (!IsSpace(*ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + /* expect a numeric timeline ID as first field of line */ + result = (TimeLineID) strtoul(ptr, &endptr, 0); + if (endptr == ptr) + elog(ERROR, + "syntax error(timeline ID) in history file: %s", + fline); + } + + fclose(fd); + + /* TLI of the last line is parent TLI */ + return result; +} + +/* + * Initialize instance visualization. + */ +static void +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) +{ + int i; + + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + + /* if you add new fields here, fix the header */ + /* show header */ + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + + if (backup->recovery_time != (time_t) 0) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + if (backup->end_time != (time_t) 0) + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + + /* + * Calculate Data field, in the case of full backup this shows the + * total amount of data. For an differential backup, this size is only + * the difference of data accumulated. + */ + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + + /* Get parent timeline before printing */ + parent_tli = get_parent_tli(backup->tli); + + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level, true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level, true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level, true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level, true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; +} diff --git a/src/status.c b/src/status.c new file mode 100644 index 000000000..155a07f40 --- /dev/null +++ b/src/status.c @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * status.c + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + * Monitor status of a PostgreSQL server. + * + *------------------------------------------------------------------------- + */ + + +#include "postgres_fe.h" + +#include +#include +#include + +#include "pg_probackup.h" + +/* PID can be negative for standalone backend */ +typedef long pgpid_t; + +static pgpid_t get_pgpid(void); +static bool postmaster_is_alive(pid_t pid); + +/* + * get_pgpid + * + * Get PID of postmaster, by scanning postmaster.pid. + */ +static pgpid_t +get_pgpid(void) +{ + FILE *pidf; + long pid; + char pid_file[MAXPGPATH]; + + snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); + + pidf = fopen(pid_file, PG_BINARY_R); + if (pidf == NULL) + { + /* No pid file, not an error on startup */ + if (errno == ENOENT) + return 0; + else + { + elog(ERROR, "could not open PID file \"%s\": %s", + pid_file, strerror(errno)); + } + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + /* Is the file empty? */ + if (ftell(pidf) == 0 && feof(pidf)) + elog(ERROR, "the PID file \"%s\" is empty", + pid_file); + else + elog(ERROR, "invalid data in PID file \"%s\"\n", + pid_file); + } + fclose(pidf); + return (pgpid_t) pid; +} + +/* + * postmaster_is_alive + * + * Check whether postmaster is alive or not. + */ +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +/* + * is_pg_running + * + * + */ +bool +is_pg_running(void) +{ + pgpid_t pid; + + pid = get_pgpid(); + + /* 0 means no pid file */ + if (pid == 0) + return false; + + /* Case of a standalone backend */ + if (pid < 0) + pid = -pid; + + /* Check if postmaster is alive */ + return postmaster_is_alive((pid_t) pid); +} diff --git a/src/util.c b/src/util.c new file mode 100644 index 000000000..82814d11d --- /dev/null +++ b/src/util.c @@ -0,0 +1,349 @@ +/*------------------------------------------------------------------------- + * + * util.c: log messages to log file or stderr, and misc code. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include + +#include "storage/bufpage.h" +#if PG_VERSION_NUM >= 110000 +#include "streamutil.h" +#endif + +const char * +base36enc(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + static char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return &buffer[offset]; +} + +/* + * Same as base36enc(), but the result must be released by the user. + */ +char * +base36enc_dup(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return strdup(&buffer[offset]); +} + +long unsigned int +base36dec(const char *text) +{ + return strtoul(text, NULL, 36); +} + +static void +checkControlFile(ControlFileData *ControlFile) +{ + pg_crc32c crc; + + /* Calculate CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); + + /* Then compare it */ + if (!EQ_CRC32C(crc, ControlFile->crc)) + elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n" + "Either the file is corrupt, or it has a different layout than this program\n" + "is expecting. The results below are untrustworthy."); + + if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + elog(ERROR, "possible byte ordering mismatch\n" + "The byte ordering used to store the pg_control file might not match the one\n" + "used by this program. In that case the results below would be incorrect, and\n" + "the PostgreSQL installation would be incompatible with this data directory."); +} + +/* + * Verify control file contents in the buffer src, and copy it to *ControlFile. + */ +static void +digestControlFile(ControlFileData *ControlFile, char *src, size_t size) +{ +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + if (size != ControlFileSize) + elog(ERROR, "unexpected control file size %d, expected %d", + (int) size, ControlFileSize); + + memcpy(ControlFile, src, sizeof(ControlFileData)); + + /* Additional checks on control file */ + checkControlFile(ControlFile); +} + +/* + * Utility shared by backup and restore to fetch the current timeline + * used by a node. + */ +TimeLineID +get_current_timeline(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (safe && buffer == NULL) + return 0; + + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.checkPointCopy.ThisTimeLineID; +} + +uint64 +get_system_identifier(char *pgdata_path) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +} + +uint64 +get_remote_system_identifier(PGconn *conn) +{ +#if PG_VERSION_NUM >= 90600 + PGresult *res; + uint64 system_id_conn; + char *val; + + res = pgut_execute(conn, + "SELECT system_identifier FROM pg_catalog.pg_control_system()", + 0, NULL); + val = PQgetvalue(res, 0, 0); + if (!parse_uint64(val, &system_id_conn, 0)) + { + PQclear(res); + elog(ERROR, "%s is not system_identifier", val); + } + PQclear(res); + + return system_id_conn; +#else + char *buffer; + size_t size; + ControlFileData ControlFile; + + buffer = fetchFile(conn, "global/pg_control", &size); + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +#endif +} + +uint32 +get_xlog_seg_size(char *pgdata_path) +{ +#if PG_VERSION_NUM >= 110000 + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.xlog_seg_size; +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +uint32 +get_data_checksum_version(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.data_checksum_version; +} + + +/* + * Convert time_t value to ISO-8601 format string. Always set timezone offset. + */ +void +time2iso(char *buf, size_t len, time_t time) +{ + struct tm *ptm = gmtime(&time); + time_t gmt = mktime(ptm); + time_t offset; + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) + { + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); + } +} + +/* copied from timestamp.c */ +pg_time_t +timestamptz_to_time_t(TimestampTz t) +{ + pg_time_t result; + +#ifdef HAVE_INT64_TIMESTAMP + result = (pg_time_t) (t / USECS_PER_SEC + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#else + result = (pg_time_t) (t + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#endif + return result; +} + +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + +const char * +status2str(BackupStatus status) +{ + static const char *statusName[] = + { + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" + }; + if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) + return "UNKNOWN"; + + return statusName[status]; +} + +void +remove_trailing_space(char *buf, int comment_mark) +{ + int i; + char *last_char = NULL; + + for (i = 0; buf[i]; i++) + { + if (buf[i] == comment_mark || buf[i] == '\n' || buf[i] == '\r') + { + buf[i] = '\0'; + break; + } + } + for (i = 0; buf[i]; i++) + { + if (!isspace(buf[i])) + last_char = buf + i; + } + if (last_char != NULL) + *(last_char + 1) = '\0'; + +} + +void +remove_not_digit(char *buf, size_t len, const char *str) +{ + int i, j; + + for (i = 0, j = 0; str[i] && j < len; i++) + { + if (!isdigit(str[i])) + continue; + buf[j++] = str[i]; + } + buf[j] = '\0'; +} diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 000000000..3afbe9e70 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 000000000..cf5a70648 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 000000000..31669ed0b --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,621 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" +#include "pg_probackup.h" +#include "thread.h" + +/* Logger parameters */ + +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) + pg_attribute_printf(3, 0); +static void elog_stderr(int elevel, const char *fmt, ...) + pg_attribute_printf(2, 3); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging of the current thread is in progress */ +static bool loggin_in_progress = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +void +init_logger(const char *root_path) +{ + /* Set log path */ + if (log_level_file != LOG_OFF || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); + } +} + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case VERBOSE: + fputs("VERBOSE: ", stream); + break; + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Exit with code if it is an error. + * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup + * routines. + */ +static void +exit_if_necessary(int elevel) +{ + if (elevel > WARNING && !in_cleanup) + { + /* Interrupt other possible routines */ + interrupted = true; + + if (loggin_in_progress) + { + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); + } + + /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) +#ifdef WIN32 + ExitThread(elevel); +#else + pthread_exit(NULL); +#endif + else + exit(elevel); + } +} + +/* + * Logs to stderr or to log file and exit if ERROR. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; + write_to_error_log = elevel >= ERROR && error_log_filename && + log_path[0] != '\0'; + write_to_stderr = elevel >= log_level_console && !file_only; + + pthread_lock(&log_file_mutex); +#ifdef WIN32 + std_args = NULL; + error_args = NULL; +#endif + loggin_in_progress = true; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + if (write_to_file || write_to_error_log) + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_file) + { + if (log_file == NULL) + { + if (log_filename == NULL) + open_logfile(&log_file, LOG_FILENAME_DEFAULT); + else + open_logfile(&log_file, log_filename); + } + + fprintf(log_file, "%s: ", strfbuf); + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + fprintf(error_log_file, "%s: ", strfbuf); + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + exit_if_necessary(elevel); + + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); +} + +/* + * Log only to stderr. It is called only within elog_internal() when another + * logging already was started. + */ +static void +elog_stderr(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < ERROR) + return; + + va_start(args, fmt); + + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + va_end(args); + + exit_if_necessary(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Logs only to log file and exit if ERROR. + */ +void +elog_file(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, true, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("off", v, len) == 0) + return LOG_OFF; + else if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case LOG_OFF: + return "OFF"; + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog_stderr(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog_stderr(ERROR, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog_stderr(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time, 0)) + elog_stderr(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog_stderr(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 000000000..8643ad186 --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +#define LOG_NONE (-10) + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define LOG_OFF 10 + +/* Logger parameters */ + +extern int log_to_file; +extern int log_level_console; +extern int log_level_file; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 +extern int log_rotation_size; +extern int log_rotation_age; + +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern void init_logger(const char *root_path); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c new file mode 100644 index 000000000..a9ba7c8e5 --- /dev/null +++ b/src/utils/parray.c @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * + * parray.c: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "src/pg_probackup.h" + +/* members of struct parray are hidden from client. */ +struct parray +{ + void **data; /* poiter array, expanded if necessary */ + size_t alloced; /* number of elements allocated */ + size_t used; /* number of elements in use */ +}; + +/* + * Create new parray object. + * Never returns NULL. + */ +parray * +parray_new(void) +{ + parray *a = pgut_new(parray); + + a->data = NULL; + a->used = 0; + a->alloced = 0; + + parray_expand(a, 1024); + + return a; +} + +/* + * Expand array pointed by data to newsize. + * Elements in expanded area are initialized to NULL. + * Note: never returns NULL. + */ +void +parray_expand(parray *array, size_t newsize) +{ + void **p; + + /* already allocated */ + if (newsize <= array->alloced) + return; + + p = pgut_realloc(array->data, sizeof(void *) * newsize); + + /* initialize expanded area to NULL */ + memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); + + array->alloced = newsize; + array->data = p; +} + +void +parray_free(parray *array) +{ + if (array == NULL) + return; + free(array->data); + free(array); +} + +void +parray_append(parray *array, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + array->data[array->used++] = elem; +} + +void +parray_insert(parray *array, size_t index, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + memmove(array->data + index + 1, array->data + index, + (array->alloced - index - 1) * sizeof(void *)); + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; + else + array->used++; +} + +/* + * Concatinate two parray. + * parray_concat() appends the copy of the content of src to the end of dest. + */ +parray * +parray_concat(parray *dest, const parray *src) +{ + /* expand head array */ + parray_expand(dest, dest->used + src->used); + + /* copy content of src after content of dest */ + memcpy(dest->data + dest->used, src->data, src->used * sizeof(void *)); + dest->used += parray_num(src); + + return dest; +} + +void +parray_set(parray *array, size_t index, void *elem) +{ + if (index > array->alloced - 1) + parray_expand(array, index + 1); + + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; +} + +void * +parray_get(const parray *array, size_t index) +{ + if (index > array->alloced - 1) + return NULL; + return array->data[index]; +} + +void * +parray_remove(parray *array, size_t index) +{ + void *val; + + /* removing unused element */ + if (index > array->used) + return NULL; + + val = array->data[index]; + + /* Do not move if the last element was removed. */ + if (index < array->alloced - 1) + memmove(array->data + index, array->data + index + 1, + (array->alloced - index - 1) * sizeof(void *)); + + /* adjust used count */ + array->used--; + + return val; +} + +bool +parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + int i; + + for (i = 0; i < array->used; i++) + { + if (compare(&key, &array->data[i]) == 0) + { + parray_remove(array, i); + return true; + } + } + return false; +} + +size_t +parray_num(const parray *array) +{ + return array->used; +} + +void +parray_qsort(parray *array, int(*compare)(const void *, const void *)) +{ + qsort(array->data, array->used, sizeof(void *), compare); +} + +void +parray_walk(parray *array, void (*action)(void *)) +{ + int i; + for (i = 0; i < array->used; i++) + action(array->data[i]); +} + +void * +parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + return bsearch(&key, array->data, array->used, sizeof(void *), compare); +} diff --git a/src/utils/parray.h b/src/utils/parray.h new file mode 100644 index 000000000..833a6961b --- /dev/null +++ b/src/utils/parray.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * parray.h: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#ifndef PARRAY_H +#define PARRAY_H + +/* + * "parray" hold pointers to objects in a linear memory area. + * Client use "parray *" to access parray object. + */ +typedef struct parray parray; + +extern parray *parray_new(void); +extern void parray_expand(parray *array, size_t newnum); +extern void parray_free(parray *array); +extern void parray_append(parray *array, void *val); +extern void parray_insert(parray *array, size_t index, void *val); +extern parray *parray_concat(parray *head, const parray *tail); +extern void parray_set(parray *array, size_t index, void *val); +extern void *parray_get(const parray *array, size_t index); +extern void *parray_remove(parray *array, size_t index); +extern bool parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern size_t parray_num(const parray *array); +extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); +extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern void parray_walk(parray *array, void (*action)(void *)); + +#endif /* PARRAY_H */ + diff --git a/src/utils/pgut.c b/src/utils/pgut.c new file mode 100644 index 000000000..f341c6a44 --- /dev/null +++ b/src/utils/pgut.c @@ -0,0 +1,2417 @@ +/*------------------------------------------------------------------------- + * + * pgut.c + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "libpq/pqsignal.h" + +#include "getopt_long.h" +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* old gcc doesn't have LLONG_MAX. */ +#ifndef LLONG_MAX +#if defined(HAVE_LONG_INT_64) || !defined(HAVE_LONG_LONG_INT_64) +#define LLONG_MAX LONG_MAX +#else +#define LLONG_MAX INT64CONST(0x7FFFFFFFFFFFFFFF) +#endif +#endif + +#define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ +#define SECS_PER_MINUTE 60 +#define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 + +const char *PROGRAM_NAME = NULL; + +const char *pgut_dbname = NULL; +const char *host = NULL; +const char *port = NULL; +const char *username = NULL; +static char *password = NULL; +bool prompt_password = true; +bool force_password = false; + +/* Database connections */ +static PGcancel *volatile cancel_conn = NULL; + +/* Interrupted by SIGINT (Ctrl+C) ? */ +bool interrupted = false; +bool in_cleanup = false; +bool in_password = false; + +static bool parse_pair(const char buffer[], char key[], char value[]); + +/* Connection routines */ +static void init_cancel_handler(void); +static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); +static void on_after_exec(PGcancel *thread_cancel_conn); +static void on_interrupt(void); +static void on_cleanup(void); +static void exit_or_abort(int exitcode); +static const char *get_username(void); +static pqsigfunc oldhandler = NULL; + +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"KB", OPTION_UNIT_KB, 1}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + +static size_t +option_length(const pgut_option opts[]) +{ + size_t len; + + for (len = 0; opts && opts[len].type; len++) { } + + return len; +} + +static int +option_has_arg(char type) +{ + switch (type) + { + case 'b': + case 'B': + return no_argument; + default: + return required_argument; + } +} + +static void +option_copy(struct option dst[], const pgut_option opts[], size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + { + dst[i].name = opts[i].lname; + dst[i].has_arg = option_has_arg(opts[i].type); + dst[i].flag = NULL; + dst[i].val = opts[i].sname; + } +} + +static pgut_option * +option_find(int c, pgut_option opts1[]) +{ + size_t i; + + for (i = 0; opts1 && opts1[i].type; i++) + if (opts1[i].sname == c) + return &opts1[i]; + + return NULL; /* not found */ +} + +static void +assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) +{ + const char *message; + + if (opt == NULL) + { + fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); + exit_or_abort(ERROR); + } + + if (opt->source > src) + { + /* high prior value has been set already. */ + return; + } + /* Allow duplicate entries for function option */ + else if (src >= SOURCE_CMDLINE && opt->source >= src && opt->type != 'f') + { + message = "specified only once"; + } + else + { + pgut_optsrc orig_source = opt->source; + + /* can be overwritten if non-command line source */ + opt->source = src; + + switch (opt->type) + { + case 'b': + case 'B': + if (optarg == NULL) + { + *((bool *) opt->var) = (opt->type == 'b'); + return; + } + else if (parse_bool(optarg, (bool *) opt->var)) + { + return; + } + message = "a boolean"; + break; + case 'f': + ((pgut_optfn) opt->var)(opt, optarg); + return; + case 'i': + if (parse_int32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit signed integer"; + break; + case 'u': + if (parse_uint32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit unsigned integer"; + break; + case 'I': + if (parse_int64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit signed integer"; + break; + case 'U': + if (parse_uint64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit unsigned integer"; + break; + case 's': + if (orig_source != SOURCE_DEFAULT) + free(*(char **) opt->var); + *(char **) opt->var = pgut_strdup(optarg); + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; + case 't': + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) + return; + message = "a time"; + break; + default: + elog(ERROR, "invalid option type: %c", opt->type); + return; /* keep compiler quiet */ + } + } + + if (isprint(opt->sname)) + elog(ERROR, "option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "option --%s should be %s: '%s'", + opt->lname, message, optarg); +} + +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Unsigned variant of convert_to_base_unit() + */ +static bool +convert_to_base_unit_u(uint64 value, const char *unit, + int base_unit, uint64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Convert a value in some base unit to a human-friendly unit. The output + * unit is chosen so that it's the greatest unit that can represent the value + * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is + * converted to 1 MB, but 1025 is represented as 1025 kB. + */ +void +convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Unsigned variant of convert_from_base_unit() + */ +void +convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +static bool +parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Unsigned variant of parse_unit() + */ +static bool +parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit_u(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Try to interpret value as boolean value. Valid values are: true, + * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + */ +bool +parse_bool(const char *value, bool *result) +{ + return parse_bool_with_len(value, strlen(value), result); +} + +bool +parse_bool_with_len(const char *value, size_t len, bool *result) +{ + switch (*value) + { + case 't': + case 'T': + if (pg_strncasecmp(value, "true", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'f': + case 'F': + if (pg_strncasecmp(value, "false", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'y': + case 'Y': + if (pg_strncasecmp(value, "yes", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'n': + case 'N': + if (pg_strncasecmp(value, "no", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'o': + case 'O': + /* 'o' is not unique enough */ + if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = true; + return true; + } + else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = false; + return true; + } + break; + case '1': + if (len == 1) + { + if (result) + *result = true; + return true; + } + break; + case '0': + if (len == 1) + { + if (result) + *result = false; + return true; + } + break; + default: + break; + } + + if (result) + *result = false; /* suppress compiler warning */ + return false; +} + +/* + * Parse string as 32bit signed int. + * valid range: -2147483648 ~ 2147483647 + */ +bool +parse_int32(const char *value, int32 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = INT_MAX; + return true; + } + + errno = 0; + val = strtol(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (int64) ((int32) val)) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as 32bit unsigned int. + * valid range: 0 ~ 4294967295 (2^32-1) + */ +bool +parse_uint32(const char *value, uint32 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = UINT_MAX; + return true; + } + + errno = 0; + val = strtoul(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (uint64) ((uint32) val)) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as int64 + * valid range: -9223372036854775808 ~ 9223372036854775807 + */ +bool +parse_int64(const char *value, int64 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = LLONG_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtol(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoll(value, &endptr, 0); +#else + val = strtol(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as uint64 + * valid range: 0 ~ (2^64-1) + */ +bool +parse_uint64(const char *value, uint64 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { +#if defined(HAVE_LONG_INT_64) + *result = ULONG_MAX; +#elif defined(HAVE_LONG_LONG_INT_64) + *result = ULLONG_MAX; +#else + *result = ULONG_MAX; +#endif + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtoul(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoull(value, &endptr, 0); +#else + val = strtoul(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. + */ +bool +parse_time(const char *value, time_t *result, bool utc_default) +{ + size_t len; + int fields_num, + tz = 0, + i; + bool tz_set = false; + char *tmp; + struct tm tm; + char junk[2]; + + /* tmp = replace( value, !isalnum, ' ' ) */ + tmp = pgut_malloc(strlen(value) + + 1); + len = 0; + fields_num = 1; + + while (*value) + { + if (IsAlnum(*value)) + { + tmp[len++] = *value; + value++; + } + else if (fields_num < 6) + { + fields_num++; + tmp[len++] = ' '; + value++; + } + /* timezone field is 7th */ + else if ((*value == '-' || *value == '+') && fields_num == 6) + { + int hr, + min, + sec = 0; + char *cp; + + errno = 0; + hr = strtol(value + 1, &cp, 10); + if ((value + 1) == cp || errno == ERANGE) + return false; + + /* explicit delimiter? */ + if (*cp == ':') + { + errno = 0; + min = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + if (*cp == ':') + { + errno = 0; + sec = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + } + } + /* otherwise, might have run things together... */ + else if (*cp == '\0' && strlen(value) > 3) + { + min = hr % 100; + hr = hr / 100; + /* we could, but don't, support a run-together hhmmss format */ + } + else + min = 0; + + /* Range-check the values; see notes in datatype/timestamp.h */ + if (hr < 0 || hr > MAX_TZDISP_HOUR) + return false; + if (min < 0 || min >= MINS_PER_HOUR) + return false; + if (sec < 0 || sec >= SECS_PER_MINUTE) + return false; + + tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; + if (*value == '-') + tz = -tz; + + tz_set = true; + + fields_num++; + value = cp; + } + /* wrong format */ + else if (!IsSpace(*value)) + return false; + } + tmp[len] = '\0'; + + /* parse for "YYYY-MM-DD HH:MI:SS" */ + memset(&tm, 0, sizeof(tm)); + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); + free(tmp); + + if (i < 1 || 6 < i) + return false; + + /* adjust year */ + if (tm.tm_year < 100) + tm.tm_year += 2000 - 1900; + else if (tm.tm_year >= 1900) + tm.tm_year -= 1900; + + /* adjust month */ + if (i > 1) + tm.tm_mon -= 1; + + /* determine whether Daylight Saving Time is in effect */ + tm.tm_isdst = -1; + + *result = mktime(&tm); + + /* adjust time zone */ + if (tz_set || utc_default) + { + time_t ltime = time(NULL); + struct tm *ptm = gmtime(<ime); + time_t gmt = mktime(ptm); + time_t offset; + + /* UTC time */ + *result -= tz; + + /* Get local time */ + ptm = localtime(<ime); + offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); + + *result += offset; + } + + return true; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + +static char * +longopts_to_optstring(const struct option opts[], const size_t len) +{ + size_t i; + char *result; + char *s; + + result = pgut_malloc(len * 2 + 1); + + s = result; + for (i = 0; i < len; i++) + { + if (!isprint(opts[i].val)) + continue; + *s++ = opts[i].val; + if (opts[i].has_arg != no_argument) + *s++ = ':'; + } + *s = '\0'; + + return result; +} + +void +pgut_getopt_env(pgut_option options[]) +{ + size_t i; + + for (i = 0; options && options[i].type; i++) + { + pgut_option *opt = &options[i]; + const char *value = NULL; + + /* If option was already set do not check env */ + if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) + continue; + + if (strcmp(opt->lname, "pgdata") == 0) + value = getenv("PGDATA"); + if (strcmp(opt->lname, "port") == 0) + value = getenv("PGPORT"); + if (strcmp(opt->lname, "host") == 0) + value = getenv("PGHOST"); + if (strcmp(opt->lname, "username") == 0) + value = getenv("PGUSER"); + if (strcmp(opt->lname, "pgdatabase") == 0) + { + value = getenv("PGDATABASE"); + if (value == NULL) + value = getenv("PGUSER"); + if (value == NULL) + value = get_username(); + } + + if (value) + assign_option(opt, value, SOURCE_ENV); + } +} + +int +pgut_getopt(int argc, char **argv, pgut_option options[]) +{ + int c; + int optindex = 0; + char *optstring; + pgut_option *opt; + struct option *longopts; + size_t len; + + len = option_length(options); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); + option_copy(longopts, options, len); + + optstring = longopts_to_optstring(longopts, len); + + /* Assign named options */ + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + { + opt = option_find(c, options); + if (opt && opt->allowed < SOURCE_CMDLINE) + elog(ERROR, "option %s cannot be specified in command line", + opt->lname); + /* Check 'opt == NULL' is performed in assign_option() */ + assign_option(opt, optarg, SOURCE_CMDLINE); + } + + init_cancel_handler(); + atexit(on_cleanup); + + return optind; +} + +/* compare two strings ignore cases and ignore -_ */ +static bool +key_equals(const char *lhs, const char *rhs) +{ + for (; *lhs && *rhs; lhs++, rhs++) + { + if (strchr("-_ ", *lhs)) + { + if (!strchr("-_ ", *rhs)) + return false; + } + else if (ToLower(*lhs) != ToLower(*rhs)) + return false; + } + + return *lhs == '\0' && *rhs == '\0'; +} + +/* + * Get configuration from configuration file. + * Return number of parsed options + */ +int +pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) +{ + FILE *fp; + char buf[1024]; + char key[1024]; + char value[1024]; + int parsed_options = 0; + + if (!options) + return parsed_options; + + if ((fp = pgut_fopen(path, "rt", true)) == NULL) + return parsed_options; + + while (fgets(buf, lengthof(buf), fp)) + { + size_t i; + + for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) + buf[i - 1] = '\0'; + + if (parse_pair(buf, key, value)) + { + for (i = 0; options[i].type; i++) + { + pgut_option *opt = &options[i]; + + if (key_equals(key, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "option %s cannot be specified in file", opt->lname); + else if (opt->source <= SOURCE_FILE) + { + assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } + break; + } + } + if (strict && !options[i].type) + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); + } + } + + fclose(fp); + + return parsed_options; +} + +static const char * +skip_space(const char *str, const char *line) +{ + while (IsSpace(*str)) { str++; } + return str; +} + +static const char * +get_next_token(const char *src, char *dst, const char *line) +{ + const char *s; + int i; + int j; + + if ((s = skip_space(src, line)) == NULL) + return NULL; + + /* parse quoted string */ + if (*s == '\'') + { + s++; + for (i = 0, j = 0; s[i] != '\0'; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + dst[j] = '\b'; + break; + case 'f': + dst[j] = '\f'; + break; + case 'n': + dst[j] = '\n'; + break; + case 'r': + dst[j] = '\r'; + break; + case 't': + dst[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + dst[j] = ((char) octVal); + } + break; + default: + dst[j] = s[i]; + break; + } + } + else if (s[i] == '\'') + { + i++; + /* doubled quote becomes just one quote */ + if (s[i] == '\'') + dst[j] = s[i]; + else + break; + } + else + dst[j] = s[i]; + j++; + } + } + else + { + i = j = strcspn(s, "#\n\r\t\v"); + memcpy(dst, s, j); + } + + dst[j] = '\0'; + return s + i; +} + +static bool +parse_pair(const char buffer[], char key[], char value[]) +{ + const char *start; + const char *end; + + key[0] = value[0] = '\0'; + + /* + * parse key + */ + start = buffer; + if ((start = skip_space(start, buffer)) == NULL) + return false; + + end = start + strcspn(start, "=# \n\r\t\v"); + + /* skip blank buffer */ + if (end - start <= 0) + { + if (*start == '=') + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + /* key found */ + strncpy(key, start, end - start); + key[end - start] = '\0'; + + /* find key and value split char */ + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '=') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + start++; + + /* + * parse value + */ + if ((end = get_next_token(start, value, buffer)) == NULL) + return false; + + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '\0' && *start != '#') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + return true; +} + +/* + * Ask the user for a password; 'username' is the username the + * password is for, if one has been explicitly specified. + * Set malloc'd string to the global variable 'password'. + */ +static void +prompt_for_password(const char *username) +{ + in_password = true; + + if (password) + { + free(password); + password = NULL; + } + +#if PG_VERSION_NUM >= 100000 + password = (char *) pgut_malloc(sizeof(char) * 100 + 1); + if (username == NULL) + simple_prompt("Password: ", password, 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + simple_prompt(message, password, 100, false); + } +#else + if (username == NULL) + password = simple_prompt("Password: ", 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message, 100, false); + } +#endif + + in_password = false; +} + +/* + * Copied from pg_basebackup.c + * Escape a parameter value so that it can be used as part of a libpq + * connection string, e.g. in: + * + * application_name= + * + * The returned string is malloc'd. Return NULL on out-of-memory. + */ +static char * +escapeConnectionParameter(const char *src) +{ + bool need_quotes = false; + bool need_escaping = false; + const char *p; + char *dstbuf; + char *dst; + + /* + * First check if quoting is needed. Any quote (') or backslash (\) + * characters need to be escaped. Parameters are separated by whitespace, + * so any string containing whitespace characters need to be quoted. An + * empty string is represented by ''. + */ + if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) + need_escaping = true; + + for (p = src; *p; p++) + { + if (isspace((unsigned char) *p)) + { + need_quotes = true; + break; + } + } + + if (*src == '\0') + return pg_strdup("''"); + + if (!need_quotes && !need_escaping) + return pg_strdup(src); /* no quoting or escaping needed */ + + /* + * Allocate a buffer large enough for the worst case that all the source + * characters need to be escaped, plus quotes. + */ + dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); + + dst = dstbuf; + if (need_quotes) + *(dst++) = '\''; + for (; *src; src++) + { + if (*src == '\'' || *src == '\\') + *(dst++) = '\\'; + *(dst++) = *src; + } + if (need_quotes) + *(dst++) = '\''; + *dst = '\0'; + + return dstbuf; +} + +/* Construct a connection string for possible future use in recovery.conf */ +char * +pgut_get_conninfo_string(PGconn *conn) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBuffer buf = createPQExpBuffer(); + char *connstr; + bool firstkeyword = true; + char *escaped; + + connOptions = PQconninfo(conn); + if (connOptions == NULL) + elog(ERROR, "out of memory"); + + /* Construct a new connection string in key='value' format. */ + for (option = connOptions; option && option->keyword; option++) + { + /* + * Do not emit this setting if: - the setting is "replication", + * "dbname" or "fallback_application_name", since these would be + * overridden by the libpqwalreceiver module anyway. - not set or + * empty. + */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* do not print password into the file */ + if (strcmp(option->keyword, "password") == 0) + continue; + + if (!firstkeyword) + appendPQExpBufferChar(buf, ' '); + + firstkeyword = false; + + escaped = escapeConnectionParameter(option->val); + appendPQExpBuffer(buf, "%s=%s", option->keyword, escaped); + free(escaped); + } + + connstr = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + return connstr; +} + +PGconn * +pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login) +{ + PGconn *conn; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(login); + + /* Start the connection. Loop until we have a password if requested by backend. */ + for (;;) + { + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, password); + + if (PQstatus(conn) == CONNECTION_OK) + return conn; + + if (conn && PQconnectionNeedsPassword(conn) && prompt_password) + { + PQfinish(conn); + prompt_for_password(login); + + if (interrupted) + elog(ERROR, "interrupted"); + + if (password == NULL || password[0] == '\0') + elog(ERROR, "no password supplied"); + + continue; + } + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(conn)); + + PQfinish(conn); + return NULL; + } +} + +PGconn * +pgut_connect_replication(const char *dbname) +{ + return pgut_connect_replication_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser) +{ + PGconn *tmpconn; + int argcount = 7; /* dbname, replication, fallback_app_name, + * host, user, port, password */ + int i; + const char **keywords; + const char **values; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(pguser); + + i = 0; + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + + keywords[i] = "dbname"; + values[i] = "replication"; + i++; + keywords[i] = "replication"; + values[i] = "true"; + i++; + keywords[i] = "fallback_application_name"; + values[i] = PROGRAM_NAME; + i++; + + if (pghost) + { + keywords[i] = "host"; + values[i] = pghost; + i++; + } + if (pguser) + { + keywords[i] = "user"; + values[i] = pguser; + i++; + } + if (pgport) + { + keywords[i] = "port"; + values[i] = pgport; + i++; + } + + /* Use (or reuse, on a subsequent connection) password if we have it */ + if (password) + { + keywords[i] = "password"; + values[i] = password; + } + else + { + keywords[i] = NULL; + values[i] = NULL; + } + + for (;;) + { + tmpconn = PQconnectdbParams(keywords, values, true); + + + if (PQstatus(tmpconn) == CONNECTION_OK) + { + free(values); + free(keywords); + return tmpconn; + } + + if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) + { + PQfinish(tmpconn); + prompt_for_password(pguser); + keywords[i] = "password"; + values[i] = password; + continue; + } + + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(tmpconn)); + PQfinish(tmpconn); + free(values); + free(keywords); + return NULL; + } +} + + +void +pgut_disconnect(PGconn *conn) +{ + if (conn) + PQfinish(conn); +} + +/* set/get host and port for connecting standby server */ +const char * +pgut_get_host() +{ + return host; +} + +const char * +pgut_get_port() +{ + return port; +} + +void +pgut_set_host(const char *new_host) +{ + host = new_host; +} + +void +pgut_set_port(const char *new_port) +{ + port = new_port; +} + + +PGresult * +pgut_execute_parallel(PGconn* conn, + PGcancel* thread_cancel_conn, const char *query, + int nParams, const char **params, + bool text_result) +{ + PGresult *res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + //on_before_exec(conn, thread_cancel_conn); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + //on_after_exec(thread_cancel_conn); + + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +PGresult * +pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) +{ + return pgut_execute_extended(conn, query, nParams, params, true, false); +} + +PGresult * +pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error) +{ + PGresult *res; + ExecStatusType res_status; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + on_before_exec(conn, NULL); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + on_after_exec(NULL); + + res_status = PQresultStatus(res); + switch (res_status) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + if (ok_error && res_status == PGRES_FATAL_ERROR) + break; + + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +bool +pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel) +{ + int res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(elevel, "not connected"); + return false; + } + + if (nParams == 0) + res = PQsendQuery(conn, query); + else + res = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, 0); + + if (res != 1) + { + elog(elevel, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + return false; + } + + return true; +} + +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + +int +pgut_wait(int num, PGconn *connections[], struct timeval *timeout) +{ + /* all connections are busy. wait for finish */ + while (!interrupted) + { + int i; + fd_set mask; + int maxsock; + + FD_ZERO(&mask); + + maxsock = -1; + for (i = 0; i < num; i++) + { + int sock; + + if (connections[i] == NULL) + continue; + sock = PQsocket(connections[i]); + if (sock >= 0) + { + FD_SET(sock, &mask); + if (maxsock < sock) + maxsock = sock; + } + } + + if (maxsock == -1) + { + errno = ENOENT; + return -1; + } + + i = wait_for_sockets(maxsock + 1, &mask, timeout); + if (i == 0) + break; /* timeout */ + + for (i = 0; i < num; i++) + { + if (connections[i] && FD_ISSET(PQsocket(connections[i]), &mask)) + { + PQconsumeInput(connections[i]); + if (PQisBusy(connections[i])) + continue; + return i; + } + } + } + + errno = EINTR; + return -1; +} + +#ifdef WIN32 +static CRITICAL_SECTION cancelConnLock; +#endif + +/* + * on_before_exec + * + * Set cancel_conn to point to the current database connection. + */ +static void +on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + thread_cancel_conn = PQgetCancel(conn); + } + else + { + /* Free the old one if we have one */ + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + cancel_conn = PQgetCancel(conn); + } + +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * on_after_exec + * + * Free the current cancel connection, if any, and set to NULL. + */ +static void +on_after_exec(PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } + else + { + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * Handle interrupt signals by cancelling the current command. + */ +static void +on_interrupt(void) +{ + int save_errno = errno; + char errbuf[256]; + + /* Set interruped flag */ + interrupted = true; + + /* User promts password, call on_cleanup() byhand */ + if (in_password) + { + on_cleanup(); + + pqsignal(SIGINT, oldhandler); + kill(0, SIGINT); + } + + /* Send QueryCancel if we are processing a database query */ + if (!in_cleanup && cancel_conn != NULL && + PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + { + elog(WARNING, "Cancel request sent"); + } + + errno = save_errno; /* just in case the write changed it */ +} + +typedef struct pgut_atexit_item pgut_atexit_item; +struct pgut_atexit_item +{ + pgut_atexit_callback callback; + void *userdata; + pgut_atexit_item *next; +}; + +static pgut_atexit_item *pgut_atexit_stack = NULL; + +void +pgut_atexit_push(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + + AssertArg(callback != NULL); + + item = pgut_new(pgut_atexit_item); + item->callback = callback; + item->userdata = userdata; + item->next = pgut_atexit_stack; + + pgut_atexit_stack = item; +} + +void +pgut_atexit_pop(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + pgut_atexit_item **prev; + + for (item = pgut_atexit_stack, prev = &pgut_atexit_stack; + item; + prev = &item->next, item = item->next) + { + if (item->callback == callback && item->userdata == userdata) + { + *prev = item->next; + free(item); + break; + } + } +} + +static void +call_atexit_callbacks(bool fatal) +{ + pgut_atexit_item *item; + + for (item = pgut_atexit_stack; item; item = item->next) + item->callback(fatal, item->userdata); +} + +static void +on_cleanup(void) +{ + in_cleanup = true; + interrupted = false; + call_atexit_callbacks(false); +} + +static void +exit_or_abort(int exitcode) +{ + if (in_cleanup) + { + /* oops, error in cleanup*/ + call_atexit_callbacks(true); + abort(); + } + else + { + /* normal exit */ + exit(exitcode); + } +} + +/* + * Returns the current user name. + */ +static const char * +get_username(void) +{ + const char *ret; + +#ifndef WIN32 + struct passwd *pw; + + pw = getpwuid(geteuid()); + ret = (pw ? pw->pw_name : NULL); +#else + static char username[128]; /* remains after function exit */ + DWORD len = sizeof(username) - 1; + + if (GetUserName(username, &len)) + ret = username; + else + { + _dosmaperr(GetLastError()); + ret = NULL; + } +#endif + + if (ret == NULL) + elog(ERROR, "%s: could not get current user name: %s", + PROGRAM_NAME, strerror(errno)); + return ret; +} + +int +appendStringInfoFile(StringInfo str, FILE *fp) +{ + AssertArg(str != NULL); + AssertArg(fp != NULL); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = fread(str->data + str->len, 1, str->maxlen - str->len - 1, fp); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (ferror(fp) && errno != EINTR) + return errno; + } + return 0; +} + +int +appendStringInfoFd(StringInfo str, int fd) +{ + AssertArg(str != NULL); + AssertArg(fd != -1); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = read(fd, str->data + str->len, str->maxlen - str->len - 1); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (errno != EINTR) + return errno; + } + return 0; +} + +void * +pgut_malloc(size_t size) +{ + char *ret; + + if ((ret = malloc(size)) == NULL) + elog(ERROR, "could not allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +void * +pgut_realloc(void *p, size_t size) +{ + char *ret; + + if ((ret = realloc(p, size)) == NULL) + elog(ERROR, "could not re-allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +char * +pgut_strdup(const char *str) +{ + char *ret; + + if (str == NULL) + return NULL; + + if ((ret = strdup(str)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + return ret; +} + +char * +strdup_with_len(const char *str, size_t len) +{ + char *r; + + if (str == NULL) + return NULL; + + r = pgut_malloc(len + 1); + memcpy(r, str, len); + r[len] = '\0'; + return r; +} + +/* strdup but trim whitespaces at head and tail */ +char * +strdup_trim(const char *str) +{ + size_t len; + + if (str == NULL) + return NULL; + + while (IsSpace(str[0])) { str++; } + len = strlen(str); + while (len > 0 && IsSpace(str[len - 1])) { len--; } + + return strdup_with_len(str, len); +} + +FILE * +pgut_fopen(const char *path, const char *mode, bool missing_ok) +{ + FILE *fp; + + if ((fp = fopen(path, mode)) == NULL) + { + if (missing_ok && errno == ENOENT) + return NULL; + + elog(ERROR, "could not open file \"%s\": %s", + path, strerror(errno)); + } + + return fp; +} + +#ifdef WIN32 +static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); +#define select select_win32 +#endif + +int +wait_for_socket(int sock, struct timeval *timeout) +{ + fd_set fds; + + FD_ZERO(&fds); + FD_SET(sock, &fds); + return wait_for_sockets(sock + 1, &fds, timeout); +} + +int +wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) +{ + int i; + + for (;;) + { + i = select(nfds, fds, NULL, NULL, timeout); + if (i < 0) + { + if (interrupted) + elog(ERROR, "interrupted"); + else if (errno != EINTR) + elog(ERROR, "select failed: %s", strerror(errno)); + } + else + return i; + } +} + +#ifndef WIN32 +static void +handle_sigint(SIGNAL_ARGS) +{ + on_interrupt(); +} + +static void +init_cancel_handler(void) +{ + oldhandler = pqsignal(SIGINT, handle_sigint); +} +#else /* WIN32 */ + +/* + * Console control handler for Win32. Note that the control handler will + * execute on a *different thread* than the main one, so we need to do + * proper locking around those structures. + */ +static BOOL WINAPI +consoleHandler(DWORD dwCtrlType) +{ + if (dwCtrlType == CTRL_C_EVENT || + dwCtrlType == CTRL_BREAK_EVENT) + { + EnterCriticalSection(&cancelConnLock); + on_interrupt(); + LeaveCriticalSection(&cancelConnLock); + return TRUE; + } + else + /* Return FALSE for any signals not being handled */ + return FALSE; +} + +static void +init_cancel_handler(void) +{ + InitializeCriticalSection(&cancelConnLock); + + SetConsoleCtrlHandler(consoleHandler, TRUE); +} + +int +sleep(unsigned int seconds) +{ + Sleep(seconds * 1000); + return 0; +} + +int +usleep(unsigned int usec) +{ + Sleep((usec + 999) / 1000); /* rounded up */ + return 0; +} + +#undef select +static int +select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) +{ + struct timeval remain; + + if (timeout != NULL) + remain = *timeout; + else + { + remain.tv_usec = 0; + remain.tv_sec = LONG_MAX; /* infinite */ + } + + /* sleep only one second because Ctrl+C doesn't interrupt select. */ + while (remain.tv_sec > 0 || remain.tv_usec > 0) + { + int ret; + struct timeval onesec; + + if (remain.tv_sec > 0) + { + onesec.tv_sec = 1; + onesec.tv_usec = 0; + remain.tv_sec -= 1; + } + else + { + onesec.tv_sec = 0; + onesec.tv_usec = remain.tv_usec; + remain.tv_usec = 0; + } + + ret = select(nfds, readfds, writefds, exceptfds, &onesec); + if (ret != 0) + { + /* succeeded or error */ + return ret; + } + else if (interrupted) + { + errno = EINTR; + return 0; + } + } + + return 0; /* timeout */ +} + +#endif /* WIN32 */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h new file mode 100644 index 000000000..fedb99b01 --- /dev/null +++ b/src/utils/pgut.h @@ -0,0 +1,238 @@ +/*------------------------------------------------------------------------- + * + * pgut.h + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PGUT_H +#define PGUT_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +#include +#include + +#include "access/xlogdefs.h" +#include "logger.h" + +#if !defined(C_H) && !defined(__cplusplus) +#ifndef bool +typedef char bool; +#endif +#ifndef true +#define true ((bool) 1) +#endif +#ifndef false +#define false ((bool) 0) +#endif +#endif + +#define INFINITE_STR "INFINITE" + +typedef enum pgut_optsrc +{ + SOURCE_DEFAULT, + SOURCE_FILE_STRICT, + SOURCE_ENV, + SOURCE_FILE, + SOURCE_CMDLINE, + SOURCE_CONST +} pgut_optsrc; + +/* + * type: + * b: bool (true) + * B: bool (false) + * f: pgut_optfn + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string + * t: time_t + */ +typedef struct pgut_option +{ + char type; + uint8 sname; /* short name */ + const char *lname; /* long name */ + void *var; /* pointer to variable */ + pgut_optsrc allowed; /* allowed source */ + pgut_optsrc source; /* actual source */ + int flags; /* option unit */ +} pgut_option; + +typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); +typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); + +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + +/* + * pgut client variables and functions + */ +extern const char *PROGRAM_NAME; +extern const char *PROGRAM_VERSION; +extern const char *PROGRAM_URL; +extern const char *PROGRAM_EMAIL; + +extern void pgut_help(bool details); + +/* + * pgut framework variables and functions + */ +extern const char *pgut_dbname; +extern const char *host; +extern const char *port; +extern const char *username; +extern bool prompt_password; +extern bool force_password; + +extern bool interrupted; +extern bool in_cleanup; +extern bool in_password; /* User prompts password */ + +extern int pgut_getopt(int argc, char **argv, pgut_option options[]); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel, + bool strict); +extern void pgut_getopt_env(pgut_option options[]); +extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); +extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); + +/* + * Database connections + */ +extern char *pgut_get_conninfo_string(PGconn *conn); +extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login); +extern PGconn *pgut_connect_replication(const char *dbname); +extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser); +extern void pgut_disconnect(PGconn *conn); +extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, + const char **params); +extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error); +extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, + const char *query, int nParams, + const char **params, bool text_result); +extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); +extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); + +extern const char *pgut_get_host(void); +extern const char *pgut_get_port(void); +extern void pgut_set_host(const char *new_host); +extern void pgut_set_port(const char *new_port); + +/* + * memory allocators + */ +extern void *pgut_malloc(size_t size); +extern void *pgut_realloc(void *p, size_t size); +extern char *pgut_strdup(const char *str); +extern char *strdup_with_len(const char *str, size_t len); +extern char *strdup_trim(const char *str); + +#define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) + +/* + * file operations + */ +extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); + +/* + * Assert + */ +#undef Assert +#undef AssertArg +#undef AssertMacro + +#ifdef USE_ASSERT_CHECKING +#define Assert(x) assert(x) +#define AssertArg(x) assert(x) +#define AssertMacro(x) assert(x) +#else +#define Assert(x) ((void) 0) +#define AssertArg(x) ((void) 0) +#define AssertMacro(x) ((void) 0) +#endif + +/* + * StringInfo and string operations + */ +#define STRINGINFO_H + +#define StringInfoData PQExpBufferData +#define StringInfo PQExpBuffer +#define makeStringInfo createPQExpBuffer +#define initStringInfo initPQExpBuffer +#define freeStringInfo destroyPQExpBuffer +#define termStringInfo termPQExpBuffer +#define resetStringInfo resetPQExpBuffer +#define enlargeStringInfo enlargePQExpBuffer +#define printfStringInfo printfPQExpBuffer /* reset + append */ +#define appendStringInfo appendPQExpBuffer +#define appendStringInfoString appendPQExpBufferStr +#define appendStringInfoChar appendPQExpBufferChar +#define appendBinaryStringInfo appendBinaryPQExpBuffer + +extern int appendStringInfoFile(StringInfo str, FILE *fp); +extern int appendStringInfoFd(StringInfo str, int fd); + +extern bool parse_bool(const char *value, bool *result); +extern bool parse_bool_with_len(const char *value, size_t len, bool *result); +extern bool parse_int32(const char *value, int32 *result, int flags); +extern bool parse_uint32(const char *value, uint32 *result, int flags); +extern bool parse_int64(const char *value, int64 *result, int flags); +extern bool parse_uint64(const char *value, uint64 *result, int flags); +extern bool parse_time(const char *value, time_t *result, bool utc_default); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); + +extern void convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit); +extern void convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit); + +#define IsSpace(c) (isspace((unsigned char)(c))) +#define IsAlpha(c) (isalpha((unsigned char)(c))) +#define IsAlnum(c) (isalnum((unsigned char)(c))) +#define IsIdentHead(c) (IsAlpha(c) || (c) == '_') +#define IsIdentBody(c) (IsAlnum(c) || (c) == '_') +#define ToLower(c) (tolower((unsigned char)(c))) +#define ToUpper(c) (toupper((unsigned char)(c))) + +/* + * socket operations + */ +extern int wait_for_socket(int sock, struct timeval *timeout); +extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); + +#ifdef WIN32 +extern int sleep(unsigned int seconds); +extern int usleep(unsigned int usec); +#endif + +#endif /* PGUT_H */ diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 000000000..82c237641 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 +#include + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static long mutex_initlock = 0; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 000000000..064605331 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "postgres_fe.h" +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +extern int pthread_lock(pthread_mutex_t *mp); + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 000000000..bc82e8116 --- /dev/null +++ b/src/validate.c @@ -0,0 +1,354 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + pgBackupWriteBackupControlFile(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + crc = pgFileGetCRC(file->path); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + break; + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + current_backup = (pgBackup *) parray_get(backups, i); + + /* Valiate each backup along with its xlog files. */ + pgBackupValidate(current_backup); + + /* Ensure that the backup has valid list of parent backups */ + if (current_backup->status == BACKUP_STATUS_OK) + { + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + } + + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + int j; + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + base36enc(backup->start_time), current_backup_id); + } + } + free(current_backup_id); + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 000000000..31dfb6560 --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,24 @@ +[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) + +``` +Note: For now there are tests only for Linix +``` + + +``` +Check physical correctness of restored instances: + Apply this patch to disable HINT BITS: https://gist.github.com/gsmol/2bb34fd3ba31984369a72cc1c27a36b6 + export PG_PROBACKUP_PARANOIA=ON + +Check archive compression: + export ARCHIVE_COMPRESSION=ON + +Specify path to pg_probackup binary file. By default tests use /pg_probackup/ + export PGPROBACKUPBIN= + +Usage: + pip install testgres + pip install psycopg2 + export PG_CONFIG=/path/to/pg_config + python -m unittest [-v] tests[.specific_module][.class.test] +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..aeeabf2a9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,69 @@ +import unittest + +from . import init_test, option_test, show_test, \ + backup_test, delete_test, restore_test, validate_test, \ + retention_test, ptrack_clean, ptrack_cluster, \ + ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ + ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + false_positive, replica, compression, page, ptrack, archive, \ + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() +# suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(archive)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) +# suite.addTests(loader.loadTestsFromModule(cfs_restore)) +# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) +# suite.addTests(loader.loadTestsFromModule(logging)) + suite.addTests(loader.loadTestsFromModule(compression)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(exclude)) + suite.addTests(loader.loadTestsFromModule(false_positive)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page)) + suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_clean)) + suite.addTests(loader.loadTestsFromModule(ptrack_cluster)) + suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace)) + suite.addTests(loader.loadTestsFromModule(ptrack_recovery)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + + return suite + +# test_pgpro434_2 unexpected success +# ToDo: +# archive: +# discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work +# replica: +# backup should exit with correct error message if some master* option is missing +# --master* options shoukd not work when backuping master +# logging: +# https://jira.postgrespro.ru/browse/PGPRO-584 +# https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md +# ptrack: +# ptrack backup on replica should work correctly +# archive: +# immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored +# 10vanilla_1.3ptrack + +# 10vanilla+ +# 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py new file mode 100644 index 000000000..8b8eb71aa --- /dev/null +++ b/tests/archive.py @@ -0,0 +1,833 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script +from datetime import datetime, timedelta +import subprocess +from sys import exit +from time import sleep + + +module_name = 'archive' + + +class ArchiveTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_pgpro434_1(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector from " + "generate_series(0,100) i") + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node) + node.slow_start() + + # Recreate backup calagoue + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Make backup + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + # Restore Database + self.restore_node( + backup_dir, 'node', node, + options=["--recovery-target-action=promote"]) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro434_2(self): + """ + Check that timelines are correct. + WAITING PGPRO-1053 for --immediate + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FIRST TIMELINE + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + backup_id = self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select 100501 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + # SECOND TIMELIN + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 " + "from t_heap where id = 100501)")[0][0], + 'data after restore not equal to original data') + + node.safe_psql( + "postgres", + "insert into t_heap select 2 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(100,200) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select 100502 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # THIRD TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print( + node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + node.safe_psql( + "postgres", + "insert into t_heap select 3 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(200,300) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.safe_psql( + "postgres", + "insert into t_heap select 100503 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # FOURTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fourth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # FIFTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fifth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # SIXTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Sixth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 from t_heap where id > 100500)")[0][0], + 'data after restore not equal to original data') + + self.assertEqual( + result, + node.safe_psql( + "postgres", + "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro434_3(self): + """Check pg_stop_backup_timeout, needed backup_timeout""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + archive_script_path = os.path.join(backup_dir, 'archive_script.sh') + with open(archive_script_path, 'w+') as f: + f.write( + archive_script.format( + backup_dir=backup_dir, node_name='node', count_limit=2)) + + st = os.stat(archive_script_path) + os.chmod(archive_script_path, st.st_mode | 0o111) + node.append_conf( + 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( + archive_script_path)) + node.slow_start() + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=verbose", + "--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pg_stop_backup failed to answer.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: pg_stop_backup doesn't answer" in e.message and + "cancel it" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + os.remove(file) + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists_overwrite(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + self.set_archiving(backup_dir, 'node', node, overwrite=True) + node.reload() + self.switch_wal_segment(node) + sleep(2) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_replica_archive(self): + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'max_wal_size': '1GB'} + ) + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica, synchronous=True) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # ADD INSTANCE 'REPLICA' + + sleep(1) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=30', '--log-level-file=verbose', + '--master-host=localhost', '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_parallel_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=20', + '--log-level-file=verbose', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_concurrent_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node( + backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + # self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'master', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog(self): + """Test backup with pg_receivexlog wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, + node.safe_psql( + "postgres", "SELECT * FROM t_heap" + ), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_compression_pg10(self): + """Test backup with pg_receivewal compressed wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'node', node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) diff --git a/tests/auth_test.py b/tests/auth_test.py new file mode 100644 index 000000000..fc21a480d --- /dev/null +++ b/tests/auth_test.py @@ -0,0 +1,391 @@ +""" +The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. + - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 +""" + +import os +import unittest +import signal +import time + +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import StartNodeException + +module_name = 'auth_test' +skip_test = False + + +try: + from pexpect import * +except ImportError: + skip_test = True + + +class SimpleAuthTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_backup_via_unpriviledged_user(self): + """ + Make node, create unpriviledged user, try to + run a backups without EXECUTE rights on + certain functions + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + + time.sleep(1) + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied for function " + "pg_create_restore_point\nquery was: " + "SELECT pg_catalog.pg_create_restore_point($1)", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_create_restore_point(text) TO backup;") + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.get_version(node) < self.version_to_num('10.0'): + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_stop_backup(boolean, boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql("postgres", "CREATE DATABASE test1") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql( + "test1", "create table t1 as select generate_series(0,100)") + + node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") + node.restart() + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role to clear ptrack files\n" + "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role read ptrack files\n" + "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "ALTER ROLE backup REPLICATION") + + time.sleep(1) + + # FULL + self.backup_node( + backup_dir, 'node', node, + options=['-U', 'backup']) + + # PTRACK + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + +class AuthTest(unittest.TestCase): + pb = None + node = None + + @classmethod + def setUpClass(cls): + + super(AuthTest, cls).setUpClass() + + cls.pb = ProbackupTest() + cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + + cls.node = cls.pb.make_simple_node( + base_dir="{}/node".format(module_name), + set_replication=True, + initdb_params=['--data-checksums', '--auth-host=md5'], + pg_options={ + 'wal_level': 'replica' + } + ) + modify_pg_hba(cls.node) + + cls.pb.init_pb(cls.backup_dir) + cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) + cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + try: + cls.node.start() + except StartNodeException: + raise unittest.skip("Node hasn't started") + + cls.node.safe_psql("postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ + GRANT USAGE ON SCHEMA pg_catalog TO backup; \ + GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + + @classmethod + def tearDownClass(cls): + cls.node.cleanup() + cls.pb.del_test_dir(module_name, '') + + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): + self.cmd = ['backup', + '-B', self.backup_dir, + '--instance', self.node.name, + '-h', '127.0.0.1', + '-p', str(self.node.port), + '-U', 'backup', + '-b', 'FULL' + ] + + def tearDown(self): + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSWORD"] + + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSFILE"] + + try: + os.remove(self.pgpass_file) + except OSError: + pass + + def test_empty_password(self): + """ Test case: PGPB_AUTH03 - zero password length """ + try: + self.assertIn("ERROR: no password supplied", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_wrong_password(self): + """ Test case: PGPB_AUTH04 - incorrect password """ + try: + self.assertIn("password authentication failed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password(self): + """ Test case: PGPB_AUTH01 - correct password """ + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_ctrl_c_event(self): + """ Test case: PGPB_AUTH02 - send interrupt signal """ + try: + run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + except TIMEOUT: + self.fail("Error: CTRL+C event ignored") + + def test_pgpassfile_env(self): + """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ + path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(path, line) + self.pb.test_env["PGPASSFILE"] = path + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpass(self): + """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword(self): + """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + +def run_pb_with_auth(cmd, password=None, kill=False): + try: + with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + result = probackup.expect(u"Password for user .*:", 5) + if kill: + probackup.kill(signal.SIGINT) + elif result == 0: + probackup.sendline(password) + probackup.expect(EOF) + return probackup.before + else: + raise ExceptionPexpect("Other pexpect errors.") + except TIMEOUT: + raise TIMEOUT("Timeout error.") + except ExceptionPexpect: + raise ExceptionPexpect("Pexpect error.") + + +def modify_pg_hba(node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) + + +def create_pgpass(path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py new file mode 100644 index 000000000..1fa74643a --- /dev/null +++ b/tests/backup_test.py @@ -0,0 +1,522 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +module_name = 'backup' + + +class BackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # postmaster.pid and postmaster.opts shouldn't be copied + excluded = True + db_dir = os.path.join( + backup_dir, "backups", 'node', backup_id, "database") + + for f in os.listdir(db_dir): + if ( + os.path.isfile(os.path.join(db_dir, f)) and + ( + f == "postmaster.pid" or + f == "postmaster.opts" + ) + ): + excluded = False + self.assertEqual(excluded, True) + + # page backup mode + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + # print self.show_pb(node) + show_backup = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Check parent backup + self.assertEqual( + backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + show_backup = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") + + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smooth_checkpoint(self): + """full backup with smooth checkpoint""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + node.stop() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page-level backup without validated full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of validation of corrupted backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'\n" in e.message and + "WARNING: Backup file \"{0}\" is not found\n".format( + file) in e.message and + "WARNING: Backup {0} data files are corrupted\n".format( + backup_id) in e.message and + "WARNING: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads_stream(self): + """ptrack multi thread backup mode and stream""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_in_pgdata_pgpro_1376(self): + """PGPRO-1376 """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=( + os.path.join( + node.data_dir, 'somedirectory', '100500')) + ) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=(os.path.join(node.data_dir)) + ) + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace tblspace1 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of too many levels " + "of symbolic linking\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Too many levels of symbolic links' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap2") + node.safe_psql( + "postgres", + "drop tablespace tblspace2") + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content(node.data_dir) + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap1'::regclass::oid" + ).rstrip() + + list = [] + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file == relfilenode: + path = os.path.join(root, file) + list = list + [path] + + # We expect that relfilenode occures only once + if len(list) > 1: + message = "" + for string in list: + message = message + string + "\n" + self.assertEqual( + 1, 0, + "Following file copied twice by backup:\n {0}".format( + message) + ) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py new file mode 100644 index 000000000..412320327 --- /dev/null +++ b/tests/cfs_backup.py @@ -0,0 +1,1161 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join( + self.tmp_path, module_name, self.fname, 'backup') + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2', + 'shared_buffers': '200MB' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( + tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created " + "or it create without compressions" + ) + + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + + # --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace(self): + """Case: Check fullbackup empty compressed tablespace""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_stream(self): + """Case: Check fullbackup empty compressed tablespace with options stream""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """Case: Make full backup after created table in the tablespace""" + if not self.enterprise: + return + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "\n ERROR: {0}\n CMD: {1}".format( + repr(e.message), + repr(self.cmd) + ) + ) + return False + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format( + os.path.join(self.backup_dir, 'node', backup_id)) + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # --- Section: Incremental from fill tablespace --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,25) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap', tblspace_name) + ) + + full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap') + ) + + page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments_in_multiple_tablespaces(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + tblspace_name_1 = 'tblspace_name_1' + tblspace_name_2 = 'tblspace_name_2' + + self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) + self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_1', tblspace_name_1) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_2', tblspace_name_2) + ) + + full_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + full_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_1') + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_2') + ) + + page_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + page_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + full_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + page_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # --- Make backup with not valid data(broken .cfm) --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_cfm_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_file_pg_compression_from_tablespace_dir(self): + os.remove( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_data_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_cfm_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_data_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_file_pg_compression_into_tablespace_dir(self): + + corrupted_file = find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + +# # --- End ---# +# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') +# def tearDown(self): +# self.node.cleanup() +# self.del_test_dir(module_name, self.fname) + + +#class CfsBackupEncTest(CfsBackupNoEncTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py new file mode 100644 index 000000000..73553a305 --- /dev/null +++ b/tests/cfs_restore.py @@ -0,0 +1,450 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'cfs_restore' + +tblspace_name = 'cfs_tblspace' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreBase(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', +# 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + self.add_data_in_cluster() + + self.backup_id = None + try: + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + self.node.stop(["-m", "immediate"]) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + +class CfsRestoreNoencTest(CfsRestoreBase): + def add_data_in_cluster(self): + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = self.node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + + # --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in tablespace dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-j", "3", "-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + +#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() +# +# +#class CfsRestoreEncTest(CfsRestoreNoencTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py new file mode 100644 index 000000000..eea6f0e21 --- /dev/null +++ b/tests/cfs_validate_backup.py @@ -0,0 +1,25 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_validate_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/compression.py b/tests/compression.py new file mode 100644 index 000000000..aa2753821 --- /dev/null +++ b/tests/compression.py @@ -0,0 +1,496 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'compression' + + +class CompressionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_compression_stream_zlib(self): + """make archive node, make full and page stream backups, check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=[ + '--stream', + '--compress-algorithm=zlib']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--stream', '--compress-algorithm=zlib', + '--log-level-console=verbose', + '--log-level-file=verbose']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_zlib(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=["--compress-algorithm=zlib"]) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--compress-algorithm=zlib"]) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_stream_pglz(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_pglz(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_wrong_algorithm(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--compress-algorithm=bla-blah']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because compress-algorithm is invalid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: invalid compress algorithm value "bla-blah"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py new file mode 100644 index 000000000..4afb15ae0 --- /dev/null +++ b/tests/delete_test.py @@ -0,0 +1,203 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from sys import exit + + +module_name = 'delete' + + +class DeleteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_full_backups(self): + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_page(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_orphaned_wal_segments(self): + """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + # Check wals + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + original_wal_quantity = len(wals) + + # delete second full backup + self.delete_pb(backup_dir, 'node', backup_2_id) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + # try to delete wals for second backup + self.delete_pb(backup_dir, 'node', options=['--wal']) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # delete first full backup + self.delete_pb(backup_dir, 'node', backup_1_id) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + result = self.delete_pb(backup_dir, 'node', options=['--wal']) + # delete useless wals + self.assertTrue('INFO: removed min WAL segment' in result + and 'INFO: removed max WAL segment' in result) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # Check quantity, it should be lower than original + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + + # Delete last backup + self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertEqual (0, len(wals), "Number of wals should be equal to 0") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py new file mode 100644 index 000000000..40450016b --- /dev/null +++ b/tests/delta.py @@ -0,0 +1,1265 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import QueryException +import subprocess +import time + + +module_name = 'delta' + + +class DeltaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_2(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose", + "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_3(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10100000) i;" + ) + filepath = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')" + ).rstrip() + + self.backup_node(backup_dir, 'node', node) + + print(os.path.join(node.data_dir, filepath + '.1')) + os.unlink(os.path.join(node.data_dir, filepath + '.1')) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_stream(self): + """ + make archive node, take full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_archive(self): + """ + make archive node, take full and delta archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check delta and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init( + scale=100, + options=['--tablespace=somedata', '--no-vacuum']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # delta BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream']) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_full(self): + """ + make node, make full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take delta backup, + restore database and check it presense + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND DELTA BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exists_in_previous_backup(self): + """ + Make node, take full backup, create table, take page backup, + take delta backup, check that file is no fully copied to delta backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + filepath = node.safe_psql( + "postgres", + "SELECT pg_relation_filepath('t_heap')").rstrip() + self.backup_node( + backup_dir, + 'node', + node, + options=["--stream"]) + + # PAGE BACKUP + backup_id = self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + +# if self.paranoia: +# pgdata_page = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream", "--log-level-file=verbose"] + ) +# if self.paranoia: +# pgdata_delta = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) +# self.compare_pgdata( +# pgdata_page, pgdata_delta) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_delta(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # DELTA BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_delta(self): + """ + Make node, take full backup, create database, + take delta backup, alter database tablespace location, + take delta backup restore last delta backup. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql( + "postgres", + "create database db1 tablespace = 'somedata'") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter database db1 set tablespace somedata_new" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream", "--log-level-file=verbose"]) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude.py new file mode 100644 index 000000000..48b7889c7 --- /dev/null +++ b/tests/exclude.py @@ -0,0 +1,164 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'exclude' + + +class ExcludeTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_exclude_temp_tables(self): + """ + make node without archiving, create temp table, take full backup, + check that temp table not present in backup catalogue + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create temp table test as " + "select generate_series(0,50050000)::text") + conn.commit() + + temp_schema_name = conn.execute( + "SELECT nspname FROM pg_namespace " + "WHERE oid = pg_my_temp_schema()")[0][0] + conn.commit() + + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] + conn.commit() + + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] + conn.commit() + + heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] + conn.commit() + + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + conn.commit() + + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] + conn.commit() + + temp_table_filename = os.path.basename(heap_path) + temp_idx_filename = os.path.basename(index_path) + temp_toast_filename = os.path.basename(toast_path) + temp_idx_toast_filename = os.path.basename(toast_idx_path) + + self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file in [ + temp_table_filename, temp_table_filename + ".1", + temp_idx_filename, + temp_idx_filename + ".1", + temp_toast_filename, + temp_toast_filename + ".1", + temp_idx_toast_filename, + temp_idx_toast_filename + ".1" + ]: + self.assertEqual( + 1, 0, + "Found temp table file in backup catalogue.\n " + "Filepath: {0}".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_1(self): + """ + make node without archiving, create unlogged table, take full backup, + alter table to unlogged, take ptrack backup, restore ptrack backup, + check that PGDATA`s are physically the same + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + "shared_buffers": "10MB", + "fsync": "off", + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create unlogged table test as " + "select generate_series(0,5005000)::text") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + node.safe_psql('postgres', "alter table test set logged") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out new file mode 100644 index 000000000..35f584062 --- /dev/null +++ b/tests/expected/option_help.out @@ -0,0 +1,95 @@ + +pg_probackup - utility to manage backup/recovery of PostgreSQL database. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] + + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] + [-j num-threads] [--archive-timeout=archive-timeout] + [--progress] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--delete-expired] [--delete-wal] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--immediate] [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-as-replica] + [--no-validate] + + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-name=target-name] + [--timeline=timeline] + + pg_probackup show -B backup-dir + [--instance=instance_name [-i backup-id]] + [--format=format] + + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name + + pg_probackup del-instance -B backup-dir + --instance=instance_name + + pg_probackup archive-push -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [--compress [--compress-level=compress-level]] + [--overwrite] + + pg_probackup archive-get -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + +Read the website for details. +Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out new file mode 100644 index 000000000..35e212c3e --- /dev/null +++ b/tests/expected/option_version.out @@ -0,0 +1 @@ +pg_probackup 2.0.18 \ No newline at end of file diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 000000000..1884159b2 --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,333 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + + +module_name = 'false_positive' + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, "wal", 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + + # We just lost a wal segment and know nothing about it + self.backup_node(backup_dir, 'node', node) + self.assertTrue( + 'validation completed successfully' in self.validate_pb( + backup_dir, 'node')) + ######## + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", + backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + self.assertFalse( + True, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_1(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_2(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('pthread_create') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace " + "where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb._execute("delete breakpoints") + gdb.continue_execution_until_exit() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup ptrack_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_multiple_delete(self): + """delete multiple backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + self.delete_pb(backup_dir, 'node', options= + ["-i {0}".format(backup_1_id), "-i {0}".format(backup_2_id), "-i {0}".format(backup_3_id)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 000000000..ac64c4230 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py new file mode 100644 index 000000000..67e2b331b --- /dev/null +++ b/tests/helpers/cfs_helpers.py @@ -0,0 +1,91 @@ +import os +import re +import random +import string + + +def find_by_extensions(dirs=None, extensions=None): + """ + find_by_extensions(['path1','path2'],['.txt','.log']) + :return: + Return list of files include full path by file extensions + """ + files = [] + new_dirs = [] + + if dirs is not None and extensions is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.splitext(d)[1] in extensions: + files.append(d) + + if new_dirs: + files.extend(find_by_extensions(new_dirs, extensions)) + + return files + + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + +def find_by_name(dirs=None, filename=None): + files = [] + new_dirs = [] + + if dirs is not None and filename is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.basename(d) in filename: + files.append(d) + + if new_dirs: + files.extend(find_by_name(new_dirs, filename)) + + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py new file mode 100644 index 000000000..0d04d8983 --- /dev/null +++ b/tests/helpers/ptrack_helpers.py @@ -0,0 +1,1300 @@ +# you need os for unittest to work +import os +from sys import exit, argv, version_info +import subprocess +import shutil +import six +import testgres +import hashlib +import re +import pwd +import select +import psycopg2 +from time import sleep +import re +import json + +idx_ptrack = { + 't_heap': { + 'type': 'heap' + }, + 't_btree': { + 'type': 'btree', + 'column': 'text', + 'relation': 't_heap' + }, + 't_seq': { + 'type': 'seq', + 'column': 't_seq', + 'relation': 't_heap' + }, + 't_spgist': { + 'type': 'spgist', + 'column': 'text', + 'relation': 't_heap' + }, + 't_brin': { + 'type': 'brin', + 'column': 'text', + 'relation': 't_heap' + }, + 't_gist': { + 'type': 'gist', + 'column': 'tsvector', + 'relation': 't_heap' + }, + 't_gin': { + 'type': 'gin', + 'column': 'tsvector', + 'relation': 't_heap' + }, +} + +archive_script = """ +#!/bin/bash +count=$(ls {backup_dir}/test00* | wc -l) +if [ $count -ge {count_limit} ] +then + exit 1 +else + cp $1 {backup_dir}/wal/{node_name}/$2 + count=$((count+1)) + touch {backup_dir}/test00$count + exit 0 +fi +""" +warning = """ +Wrong splint in show_pb +Original Header: +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +def dir_files(base_dir): + out_list = [] + for dir_name, subdir_list, file_list in os.walk(base_dir): + if dir_name != base_dir: + out_list.append(os.path.relpath(dir_name, base_dir)) + for fname in file_list: + out_list.append( + os.path.relpath(os.path.join( + dir_name, fname), base_dir) + ) + out_list.sort() + return out_list + + +def is_enterprise(): + # pg_config --help + p = subprocess.Popen( + [os.environ['PG_CONFIG'], '--help'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if b'postgrespro.ru' in p.communicate()[0]: + return True + else: + return False + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +def slow_start(self, replica=False): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue + + +class ProbackupTest(object): + # Class attributes + enterprise = is_enterprise() + + def __init__(self, *args, **kwargs): + super(ProbackupTest, self).__init__(*args, **kwargs) + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False + + self.test_env = os.environ.copy() + envs_list = [ + "LANGUAGE", + "LC_ALL", + "PGCONNECT_TIMEOUT", + "PGDATA", + "PGDATABASE", + "PGHOSTADDR", + "PGREQUIRESSL", + "PGSERVICE", + "PGSSLMODE", + "PGUSER", + "PGPORT", + "PGHOST" + ] + + for e in envs_list: + try: + del self.test_env[e] + except: + pass + + self.test_env["LC_MESSAGES"] = "C" + self.test_env["LC_TIME"] = "C" + + self.paranoia = False + if 'PG_PROBACKUP_PARANOIA' in self.test_env: + if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON': + self.paranoia = True + + self.archive_compress = False + if 'ARCHIVE_COMPRESSION' in self.test_env: + if self.test_env['ARCHIVE_COMPRESSION'] == 'ON': + self.archive_compress = True + try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + cache_pg_config=False, + node_cleanup_full=False) + except: + pass + + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath( + os.path.join(self.helpers_path, os.pardir) + ) + self.tmp_path = os.path.abspath( + os.path.join(self.dir_path, 'tmp_dirs') + ) + try: + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) + except: + pass + + self.user = self.get_username() + self.probackup_path = None + if "PGPROBACKUPBIN" in self.test_env: + if ( + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + ): + self.probackup_path = self.test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBINDIR is not an executable file') + if not self.probackup_path: + self.probackup_path = os.path.abspath(os.path.join( + self.dir_path, "../pg_probackup")) + + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) + node.should_rm_dirs = True + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # Sane default parameters + node.append_conf("postgresql.auto.conf", "max_connections = 100") + node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") + node.append_conf("postgresql.auto.conf", "fsync = on") + node.append_conf("postgresql.auto.conf", "wal_level = logical") + node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + + node.append_conf( + "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf("postgresql.auto.conf", "log_statement = none") + node.append_conf("postgresql.auto.conf", "log_duration = on") + node.append_conf( + "postgresql.auto.conf", "log_min_duration_statement = 0") + node.append_conf("postgresql.auto.conf", "log_connections = on") + node.append_conf("postgresql.auto.conf", "log_disconnections = on") + + # Apply given parameters + for key, value in six.iteritems(pg_options): + node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + + # Allow replication in pg_hba.conf + if set_replication: + node.append_conf( + "pg_hba.conf", + "local replication all trust\n") + node.append_conf( + "postgresql.auto.conf", + "max_wal_senders = 10") + + return node + + def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): + res = node.execute( + "postgres", + "select exists" + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertFalse( + res[0][0], + 'Tablespace "{0}" already exists'.format(tblspc_name) + ) + + if not tblspc_path: + tblspc_path = os.path.join( + node.base_dir, '{0}'.format(tblspc_name)) + cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( + tblspc_name, tblspc_path) + if cfs: + cmd += " with (compression=true)" + + if not os.path.exists(tblspc_path): + os.makedirs(tblspc_path) + res = node.safe_psql("postgres", cmd) + # Check that tablespace was successfully created + # self.assertEqual( + # res[0], 0, + # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + + def get_tblspace_path(self, node, tblspc_name): + return os.path.join(node.base_dir, tblspc_name) + + def get_fork_size(self, node, fork_name): + return node.execute( + "postgres", + "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] + + def get_fork_path(self, node, fork_name): + return os.path.join( + node.base_dir, 'data', node.execute( + "postgres", + "select pg_relation_filepath('{0}')".format( + fork_name))[0][0] + ) + + def get_md5_per_page_for_fork(self, file, size_in_pages): + pages_per_segment = {} + md5_per_page = {} + nsegments = size_in_pages/131072 + if size_in_pages % 131072 != 0: + nsegments = nsegments + 1 + + size = size_in_pages + for segment_number in range(nsegments): + if size - 131072 > 0: + pages_per_segment[segment_number] = 131072 + else: + pages_per_segment[segment_number] = size + size = size - 131072 + + for segment_number in range(nsegments): + offset = 0 + if segment_number == 0: + file_desc = os.open(file, os.O_RDONLY) + start_page = 0 + end_page = pages_per_segment[segment_number] + else: + file_desc = os.open( + file+".{0}".format(segment_number), os.O_RDONLY + ) + start_page = max(md5_per_page)+1 + end_page = end_page + pages_per_segment[segment_number] + + for page in range(start_page, end_page): + md5_per_page[page] = hashlib.md5( + os.read(file_desc, 8192)).hexdigest() + offset += 8192 + os.lseek(file_desc, offset, 0) + os.close(file_desc) + + return md5_per_page + + def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): + + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 + ptrack_bits_for_fork = [] + + page_body_size = 8192-header_size + byte_size = os.path.getsize(file + '_ptrack') + npages = byte_size/8192 + if byte_size % 8192 != 0: + print('Ptrack page is not 8k aligned') + sys.exit(1) + + file = os.open(file + '_ptrack', os.O_RDONLY) + + for page in range(npages): + offset = 8192*page+header_size + os.lseek(file, offset, 0) + lots_of_bytes = os.read(file, page_body_size) + byte_list = [ + lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes)) + ] + for byte in byte_list: + # byte_inverted = bin(int(byte, base=16))[2:][::-1] + # bits = (byte >> x) & 1 for x in range(7, -1, -1) + byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] + for bit in byte_inverted: + # if len(ptrack_bits_for_fork) < size: + ptrack_bits_for_fork.append(int(bit)) + + os.close(file) + return ptrack_bits_for_fork + + def check_ptrack_sanity(self, idx_dict): + success = True + if idx_dict['new_size'] > idx_dict['old_size']: + size = idx_dict['new_size'] + else: + size = idx_dict['old_size'] + for PageNum in range(size): + if PageNum not in idx_dict['old_pages']: + # Page was not present before, meaning that relation got bigger + # Ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was added,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + # print(idx_dict) + success = False + continue + if PageNum not in idx_dict['new_pages']: + # Page is not present now, meaning that relation got smaller + # Ptrack should be equal to 0, + # We are not freaking out about false positive stuff + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was deleted,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + continue + + # Ok, all pages in new_pages that do not have + # corresponding page in old_pages are been dealt with. + # We can now safely proceed to comparing old and new pages + if idx_dict['new_pages'][ + PageNum] != idx_dict['old_pages'][PageNum]: + # Page has been changed, + # meaning that ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was changed,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + print( + "\n Old checksumm: {0}\n" + " New checksumm: {1}".format( + idx_dict['old_pages'][PageNum], + idx_dict['new_pages'][PageNum]) + ) + + if PageNum == 0 and idx_dict['type'] == 'spgist': + if self.verbose: + print( + 'SPGIST is a special snowflake, so don`t ' + 'fret about losing ptrack for blknum 0' + ) + continue + success = False + else: + # Page has not been changed, + # meaning that ptrack should be equal to 0 + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was not changed,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum] + ) + ) + + self.assertTrue( + success, 'Ptrack does not correspond to state' + ' of its own pages.\n Gory Details: \n{0}'.format( + idx_dict['type'], idx_dict + ) + ) + + def check_ptrack_recovery(self, idx_dict): + size = idx_dict['size'] + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 1: + self.assertTrue( + False, + 'Recovery for Page Number {0} of Type {1}' + ' was conducted, but ptrack value is {2}.' + ' THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def check_ptrack_clean(self, idx_dict, size): + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 0: + self.assertTrue( + False, + 'Ptrack for Page Number {0} of Type {1}' + ' should be clean, but ptrack value is {2}.' + '\n THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, + idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def run_pb(self, command, async=False, gdb=False): + try: + self.cmd = [' '.join(map(str, [self.probackup_path] + command))] + if self.verbose: + print(self.cmd) + if gdb: + return GDBobj([self.probackup_path] + command, self.verbose) + if async: + return subprocess.Popen( + self.cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + if command[0] == 'backup': + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + else: + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), self.cmd) + + def run_binary(self, command, async=False): + if self.verbose: + print([' '.join(map(str, command))]) + try: + if async: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), command) + + def init_pb(self, backup_dir): + + shutil.rmtree(backup_dir, ignore_errors=True) + return self.run_pb([ + "init", + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def del_instance(self, backup_dir, instance): + + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node( + self, backup_dir, instance, node, data_dir=False, + backup_type="full", options=[], async=False, gdb=False + ): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir + + cmd_list = [ + "backup", + "-B", backup_dir, + # "-D", pgdata, + "-p", "%i" % node.port, + "-d", "postgres", + "--instance={0}".format(instance) + ] + if backup_type: + cmd_list += ["-b", backup_type] + + return self.run_pb(cmd_list + options, async, gdb) + + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + + def restore_node( + self, backup_dir, instance, node=False, + data_dir=None, backup_id=None, options=[] + ): + if data_dir is None: + data_dir = node.data_dir + + cmd_list = [ + "restore", + "-B", backup_dir, + "-D", data_dir, + "--instance={0}".format(instance) + ] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def show_pb( + self, backup_dir, instance=None, backup_id=None, + options=[], as_text=False, as_json=True + ): + + backup_list = [] + specific_record = {} + cmd_list = [ + "show", + "-B", backup_dir, + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] + + if as_json: + cmd_list += ["--format=json"] + + if as_text: + # You should print it when calling as_text=true + return self.run_pb(cmd_list + options) + + # get show result as list of lines + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + return backup_list + else: + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record + + def validate_pb( + self, backup_dir, instance=None, + backup_id=None, options=[] + ): + + cmd_list = [ + "validate", + "-B", backup_dir + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_pb(self, backup_dir, instance, backup_id=None, options=[]): + cmd_list = [ + "delete", + "-B", backup_dir + ] + + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_expired(self, backup_dir, instance, options=[]): + cmd_list = [ + "delete", "--expired", "--wal", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + return self.run_pb(cmd_list + options) + + def show_config(self, backup_dir, instance): + out_dict = {} + cmd_list = [ + "show-config", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + res = self.run_pb(cmd_list).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(" = ")[::2] + out_dict[name] = var + return out_dict + + def get_recovery_conf(self, node): + out_dict = {} + with open( + os.path.join(node.data_dir, "recovery.conf"), "r" + ) as recovery_conf: + for line in recovery_conf: + try: + key, value = line.split("=") + except: + continue + out_dict[key.strip()] = value.strip(" '").replace("'\n", "") + return out_dict + + def set_archiving( + self, backup_dir, instance, node, replica=False, overwrite=False): + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + + # node.append_conf( + # "postgresql.auto.conf", + # "wal_level = archive" + # ) + node.append_conf( + "postgresql.auto.conf", + "archive_mode = {0}".format(archive_mode) + ) + archive_command = "{0} archive-push -B {1} --instance={2} ".format( + self.probackup_path, backup_dir, instance) + + if os.name == 'posix': + if self.archive_compress: + archive_command = archive_command + "--compress " + + if overwrite: + archive_command = archive_command + "--overwrite " + + archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + + node.append_conf( + "postgresql.auto.conf", + "archive_command = '{0}'".format( + archive_command)) + # elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) + + def set_replica( + self, master, replica, + replica_name='replica', + synchronous=False + ): + replica.append_conf( + "postgresql.auto.conf", "port = {0}".format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf( + "recovery.conf", + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name) + ) + if synchronous: + master.append_conf( + "postgresql.auto.conf", + "synchronous_standby_names='{0}'".format(replica_name) + ) + master.append_conf( + 'postgresql.auto.conf', + "synchronous_commit='remote_apply'" + ) + master.reload() + + def wrong_wal_clean(self, node, wal_size): + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) + ] + wals.sort() + file_path = os.path.join(wals_dir, wals[-1]) + if os.path.getsize(file_path) != wal_size: + os.remove(file_path) + + def guc_wal_segment_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_segment_size'" + ) + return int(var[0][0]) * self.guc_wal_block_size(node) + + def guc_wal_block_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_block_size'" + ) + return int(var[0][0]) + + def get_pgpro_edition(self, node): + if node.execute( + "postgres", + "select exists (select 1 from" + " pg_proc where proname = 'pgpro_edition')" + )[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] + + def version_to_num(self, version): + if not version: + return 0 + parts = version.split(".") + while len(parts) < 3: + parts.append("0") + num = 0 + for part in parts: + num = num * 100 + int(re.sub("[^\d]", "", part)) + return num + + def switch_wal_segment(self, node): + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") + else: + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") + sleep(1) + + def get_version(self, node): + return self.version_to_num( + testgres.get_pg_config()["VERSION"].split(" ")[1]) + + def get_bin_path(self, binary): + return testgres.get_bin_path(binary) + + def del_test_dir(self, module_name, fname): + """ Del testdir and optimistically try to del module dir""" + try: + testgres.clean_all() + except: + pass + + shutil.rmtree( + os.path.join( + self.tmp_path, + module_name, + fname + ), + ignore_errors=True + ) + try: + os.rmdir(os.path.join(self.tmp_path, module_name)) + except: + pass + + def pgdata_content(self, directory, ignore_ptrack=True): + """ return dict with directory content. " + " TAKE IT AFTER CHECKPOINT or BACKUP""" + dirs_to_ignore = [ + 'pg_xlog', 'pg_wal', 'pg_log', + 'pg_stat_tmp', 'pg_subtrans', 'pg_notify' + ] + files_to_ignore = [ + 'postmaster.pid', 'postmaster.opts', + 'pg_internal.init', 'postgresql.auto.conf', + 'backup_label', 'tablespace_map', 'recovery.conf', + 'ptrack_control', 'ptrack_init', 'pg_control' + ] +# suffixes_to_ignore = ( +# '_ptrack' +# ) + directory_dict = {} + directory_dict['pgdata'] = directory + directory_dict['files'] = {} + for root, dirs, files in os.walk(directory, followlinks=True): + dirs[:] = [d for d in dirs if d not in dirs_to_ignore] + for file in files: + if ( + file in files_to_ignore or + (ignore_ptrack and file.endswith('_ptrack')) + ): + continue + + file_fullpath = os.path.join(root, file) + file_relpath = os.path.relpath(file_fullpath, directory) + directory_dict['files'][file_relpath] = {'is_datafile': False} + directory_dict['files'][file_relpath]['md5'] = hashlib.md5( + open(file_fullpath, 'rb').read()).hexdigest() + + if file.isdigit(): + directory_dict['files'][file_relpath]['is_datafile'] = True + size_in_pages = os.path.getsize(file_fullpath)/8192 + directory_dict['files'][file_relpath][ + 'md5_per_page'] = self.get_md5_per_page_for_fork( + file_fullpath, size_in_pages + ) + + return directory_dict + + def compare_pgdata(self, original_pgdata, restored_pgdata): + """ return dict with directory content. DO IT BEFORE RECOVERY""" + fail = False + error_message = 'Restored PGDATA is not equal to original!\n' + for file in restored_pgdata['files']: + # File is present in RESTORED PGDATA + # but not present in ORIGINAL + # only backup_label is allowed + if file not in original_pgdata['files']: + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in original_pgdata['files']: + if file in restored_pgdata['files']: + + if ( + original_pgdata['files'][file]['md5'] != + restored_pgdata['files'][file]['md5'] + ): + fail = True + error_message += ( + '\nFile Checksumm mismatch.\n' + 'File_old: {0}\nChecksumm_old: {1}\n' + 'File_new: {2}\nChecksumm_new: {3}\n').format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['md5'], + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['md5'] + ) + + if original_pgdata['files'][file]['is_datafile']: + for page in original_pgdata['files'][file]['md5_per_page']: + if page not in restored_pgdata['files'][file]['md5_per_page']: + error_message += ( + '\n Page {0} dissappeared.\n ' + 'File: {1}\n').format( + page, + os.path.join( + restored_pgdata['pgdata'], + file + ) + ) + continue + + if original_pgdata['files'][file][ + 'md5_per_page'][page] != restored_pgdata[ + 'files'][file]['md5_per_page'][page]: + error_message += ( + '\n Page checksumm mismatch: {0}\n ' + ' PAGE Checksumm_old: {1}\n ' + ' PAGE Checksumm_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original_pgdata['files'][file][ + 'md5_per_page'][page], + restored_pgdata['files'][file][ + 'md5_per_page'][page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + for page in restored_pgdata['files'][file]['md5_per_page']: + if page not in original_pgdata['files'][file]['md5_per_page']: + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join( + restored_pgdata['pgdata'], file)) + + else: + error_message += ( + '\nFile dissappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True + self.assertFalse(fail, error_message) + + def get_async_connect(self, database=None, host=None, port=5432): + if not database: + database = 'postgres' + if not host: + host = '127.0.0.1' + + return psycopg2.connect( + database="postgres", + host='127.0.0.1', + port=port, + async=True + ) + + def wait(self, connection): + while True: + state = connection.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [connection.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([connection.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def gdb_attach(self, pid): + return GDBobj([str(pid)], self.verbose, attach=True) + + +class GdbException(Exception): + def __init__(self, message=False): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj(ProbackupTest): + def __init__(self, cmd, verbose, attach=False): + self.verbose = verbose + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ["gdb", "--version"], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + b"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + universal_newlines=True + ) + self.gdb_pid = self.proc.pid + + # discard data from pipe, + # is there a way to do it a less derpy way? + while True: + line = self.proc.stdout.readline() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def set_breakpoint(self, location): + result = self._execute('break ' + location) + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('^error') or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + running = False + for line in result: + if line.startswith('*running'): + running = True + break + if line.startswith('*stopped,reason="breakpoint-hit"'): + running = False + continue + if line.startswith('*stopped,reason="exited-normally"'): + running = False + continue + return running + + def continue_execution_until_exit(self): + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if ( + line.startswith('*stopped,reason="exited-normally"') or + line == '*stopped\n' + ): + return + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + running = False + for line in result: + if line.startswith('*running'): + running = True + if line.startswith('*stopped,reason="breakpoint-hit"'): + return 'breakpoint-hit' + if line.startswith('*stopped,reason="exited-normally"'): + return 'exited-normally' + if running: + return 'running' + + def stopped_in_breakpoint(self): + output = [] + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + return False + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(repr(line)) + if line == '^done\n' or line.startswith('*stopped'): + break + if running and line.startswith('*running'): + break + return output diff --git a/tests/init_test.py b/tests/init_test.py new file mode 100644 index 000000000..0b91dafa7 --- /dev/null +++ b/tests/init_test.py @@ -0,0 +1,99 @@ +import os +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException + + +module_name = 'init' + + +class InitTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): + """Success normal init""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + self.assertEqual( + dir_files(backup_dir), + ['backups', 'wal'] + ) + self.add_instance(backup_dir, 'node', node) + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node'), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Show non-existing instance + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Delete non-existing instance + try: + self.del_instance(backup_dir, 'node1') + self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node1' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Add instance without pgdata + try: + self.run_pb([ + "add-instance", + "--instance=node1", + "-B", backup_dir + ]) + self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_already_exist(self): + """Failure with backup catalog already existed""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_abs_path(self): + """failure with backup catalog should be given as absolute path""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + try: + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/logging.py b/tests/logging.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 000000000..1be3dd8b3 --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py new file mode 100644 index 000000000..8bd473fa9 --- /dev/null +++ b/tests/option_test.py @@ -0,0 +1,218 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'option' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_help_1(self): + """help options""" + self.maxDiff = None + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + + # @unittest.skip("skip") + def test_version_2(self): + """help options""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + self.assertIn( + version_out.read().decode("utf-8"), + self.run_pb(["--version"]) + ) + + # @unittest.skip("skip") + def test_without_backup_path_3(self): + """backup command failure without backup mode option""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + try: + self.run_pb(["backup", "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # @unittest.skip("skip") + def test_options_4(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure without backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure with invalid backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # delete failure without delete options + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--backup_id\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # delete failure without ID + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue("option requires an argument -- 'i'" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + pg_options={ + 'wal_level': 'logical', + 'max_wal_senders': '2'}) + + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) + self.add_instance(backup_dir, 'node', node) + + node.start() + + # syntax error in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write(" = INFINITE\n") + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("BACKUP_MODE=\n") + + try: + self.backup_node(backup_dir, 'node', node, backup_type=None), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Command line parameters should override file values + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy=1\n") + + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + + # User cannot send --system-identifier parameter via command line + try: + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("SMOOTH_CHECKPOINT=FOO\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid option in pg_probackup.conf + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: + conf.write("TIMELINEID=1\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py new file mode 100644 index 000000000..ef7122b68 --- /dev/null +++ b/tests/page.py @@ -0,0 +1,641 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + +module_name = 'page' + + +class PageBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, take second page backup, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=verbose']) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_stream(self): + """ + make archive node, take full and page stream backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_archive(self): + """ + make archive node, take full and page archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn("INFO: Restore of backup {0} completed.".format( + full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check page and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--log-level-file=verbose"]) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", + "--recovery-target-action=promote", + "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete_1(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 000000000..bf3345561 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,98 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro560' + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + try: + self.backup_node(backup_dir, 'node1', node2, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 000000000..bd40f16de --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,80 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro589' + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: Switched WAL segment' in e.message and + 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack.py new file mode 100644 index 000000000..c2d6abff3 --- /dev/null +++ b/tests/ptrack.py @@ -0,0 +1,1600 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +module_name = 'ptrack' + + +class PtrackTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_enable(self): + """make ptrack without full backup, should result in error""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack disabled.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_disable(self): + """ + Take full backup, disable ptrack restart postgresql, + enable ptrack, restart postgresql, take ptrack backup + which should fail + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DISABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.restart() + + # ENABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.restart() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack_enable was set to OFF at some" + " point after previous backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd + ) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_uncommited_xact(self): + """make ptrack backup while there is uncommited open transaction""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + con = node.connect("postgres") + con.execute( + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + pgdata = self.pgdata_content(node.data_dir) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_full(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_truncate(self): + """make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take ptrack backup, take second ptrack backup, + restore last ptrack backup and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, + ignore_ptrack=False + ) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_get_block(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_stream(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', + node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_archive(self): + """make archive node, make full and ptrack backups, + check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as" + " select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node(backup_dir, 'node', node) + full_target_time = self.show_pb( + backup_dir, 'node', full_backup_id)['recovery-time'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + ptrack_target_time = self.show_pb( + backup_dir, 'node', ptrack_backup_id)['recovery-time'] + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--recovery-target-action=promote", + "--time={0}".format(full_target_time)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=[ + "-j", "4", + "--time={0}".format(ptrack_target_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_pgpro417(self): + """Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': + 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql( + "postgres", + "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["--stream"]) + + start_lsn_full = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + start_lsn_ptrack = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + # SECOND PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_pgpro417(self): + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.delete_pb(backup_dir, 'node', backup_id) +# sys.exit(1) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0}\n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_full_pgpro417(self): + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector " + " from generate_series(0,100) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # SECOND FULL BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + + self.delete_pb(backup_dir, 'node', backup_id) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + "ERROR: LSN from ptrack_control" in e.message and + "Create new full backup before " + "an incremental one" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream", "--log-level-file=verbose"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_ptrack(self): + """Make node, create tablespace with table, take full backup, + alter tablespace location, take ptrack backup, restore database.""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # sys.exit(1) + # PTRACK BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + # node.stop() + # node.cleanup() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_ptrack(self): + """Make node, create tablespace with database," + " take full backup, alter tablespace location," + " take ptrack backup, restore database.""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE TABLESPACE + self.create_tblspace_in_node(node, 'somedata') + + # ALTER DATABASE + node.safe_psql( + "template1", + "alter database postgres set tablespace somedata") + + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", '--log-level-file=verbose']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + node.stop() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', + node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata'))]) + + # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to tablespace 'somedata' + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # Move table back to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # DROP TABLESPACE 'somedata' + node.safe_psql( + "postgres", "drop tablespace somedata") + # THIRD PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + tblspace = self.get_tblspace_path(node, 'somedata') + node.cleanup() + shutil.rmtree(tblspace, ignore_errors=True) + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + node.start() + + tblspc_exist = node.safe_psql( + "postgres", + "select exists(select 1 from " + "pg_tablespace where spcname = 'somedata')") + + if tblspc_exist.rstrip() == 't': + self.assertEqual( + 1, 0, + "Expecting Error because " + "tablespace 'somedata' should not be present") + + result_new = node.safe_psql("postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_alter_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + tblspc_path = self.get_tblspace_path(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to separate tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from t_heap") + + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + # GET PHYSICAL CONTENT FROM NODE + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore ptrack backup + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + # COMPARE LOGICAL CONTENT + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + restored_node.cleanup() + shutil.rmtree(tblspc_path_new, ignore_errors=True) + + # Move table to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore second ptrack backup and check table consistency + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_multiple_segments(self): + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PTRACK STUFF + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench(options=['-T', '150', '-c', '2', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts' + ) + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size'] + ) + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path'] + ) + self.check_ptrack_sanity(idx_ptrack) + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--log-level-file=verbose"] + ) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, + 'somedata_restored' + ) + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select * from pgbench_accounts" + ) + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_atexit_fail(self): + """ + Take backups of every available types and check that PTRACK is clean + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'max_connections': '15'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=[ + "--stream", "-j 30", + "--log-level-file=verbose"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are opening too many connections" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'setting its status to ERROR', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + self.assertEqual( + node.safe_psql( + "postgres", + "select * from pg_is_in_backup()").rstrip(), + "f") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py new file mode 100644 index 000000000..f4350af04 --- /dev/null +++ b/tests/ptrack_clean.py @@ -0,0 +1,253 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import time + + +module_name = 'ptrack_clean' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10', '--log-level-file=verbose']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean_replica(self): + """Take backups of every available types from master and check that PTRACK on replica is clean""" + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py new file mode 100644 index 000000000..784751ef6 --- /dev/null +++ b/tests/ptrack_cluster.py @@ -0,0 +1,268 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from time import sleep +from sys import exit + + +module_name = 'ptrack_cluster' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + #@unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_gist') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py new file mode 100644 index 000000000..98c209142 --- /dev/null +++ b/tests/ptrack_move_to_tablespace.py @@ -0,0 +1,57 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_move_to_tablespace' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Move table and indexes and make checkpoint + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'heap': + node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i)) + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i)) + node.safe_psql('postgres', 'checkpoint') + + # Check ptrack files + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'seq': + continue + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py new file mode 100644 index 000000000..8569ef592 --- /dev/null +++ b/tests/ptrack_recovery.py @@ -0,0 +1,58 @@ +import os +import unittest +from sys import exit +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_recovery' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py new file mode 100644 index 000000000..928608c4a --- /dev/null +++ b/tests/ptrack_truncate.py @@ -0,0 +1,130 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + replica.safe_psql('postgres', 'truncate t_heap') + replica.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py new file mode 100644 index 000000000..0409cae3f --- /dev/null +++ b/tests/ptrack_vacuum.py @@ -0,0 +1,152 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py new file mode 100644 index 000000000..f0cd3bbda --- /dev/null +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -0,0 +1,136 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_frozen' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py new file mode 100644 index 000000000..45a8d9b60 --- /dev/null +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -0,0 +1,67 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_visibility' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py new file mode 100644 index 000000000..ec12c9e27 --- /dev/null +++ b/tests/ptrack_vacuum_full.py @@ -0,0 +1,140 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_full' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py new file mode 100644 index 000000000..5c84c7e8f --- /dev/null +++ b/tests/ptrack_vacuum_truncate.py @@ -0,0 +1,142 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id > 128;') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id > 128;') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 000000000..d74c375c2 --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,293 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'replica' + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_ptrack_backup(self): + """ + make node, take full backup, restore it and make replica from it, + take full stream backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + # CREATE TABLE + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # take full backup and restore it + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + # Check data correctness on replica + replica.slow_start(replica=True) + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take PTRACK backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PTRACK BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_archive_page_backup(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data + # equal to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_make_replica_via_restore(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node( + backup_dir, 'master', replica, + options=['-R', '--recovery-target-action=promote']) + + # Settings for Replica + # self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py new file mode 100644 index 000000000..c33a1e299 --- /dev/null +++ b/tests/restore_test.py @@ -0,0 +1,1243 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +import time + + +module_name = 'restore' + + +class RestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_restore_full_to_latest(self): + """recovery to latest from full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # 1 - Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # 2 - Test that recovery.conf was created + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_page_to_latest(self): + """recovery to latest from full + page backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_specific_timeline(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_tli = int( + node.get_control_data()["Latest checkpoint's TimeLineID"]) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Correct Backup must be choosen for restore + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--timeline={0}".format(target_tli), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + recovery_target_timeline = self.get_recovery_conf( + node)["recovery_target_timeline"] + self.assertEqual(int(recovery_target_timeline), target_tli) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_time(self): + """recovery to target time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") + node.start() + + node.pgbench_init(scale=2) + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(target_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_inclusive(self): + """recovery to target xid""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--xid={0}'.format(target_xid), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_not_inclusive(self): + """recovery with target inclusive false""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = result[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + '--xid={0}'.format(target_xid), + "--inclusive=false", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_stream(self): + """recovery in stream mode to latest from full + ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_under_load(self): + """ + recovery to latest from full + ptrack backups + with loads when ptrack backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + pgbench.wait() + pgbench.stdout.close() + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_under_load_ptrack(self): + """ + recovery to latest from full + page backups + with loads when full backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + self.backup_node(backup_dir, 'node', node) + + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + + node.stop() + node.cleanup() + # self.wrong_wal_clean(node, wal_segment_size) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_1(self): + """recovery using tablespace-mapping option""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # 1 - Try to restore to existing directory + node.stop() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore destionation is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format( + node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 2 - Try to restore to existing tablespace directory + node.cleanup() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore tablespace destination ' + 'is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 3 - Restore using tablespace-mapping + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) + + # 4 - Restore using tablespace-mapping using page backup + self.backup_node(backup_dir, 'node', node) + with node.connect("postgres") as con: + con.execute("INSERT INTO test VALUES (2)") + con.commit() + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_pb = self.show_pb(backup_dir, 'node') + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") + + node.stop() + node.cleanup() + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_2(self): + """recovery using tablespace-mapping option and page backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Full backup + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # First page backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + + # Create tablespace table + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CHECKPOINT") + con.connection.autocommit = False + con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") + con.execute( + "INSERT INTO tbl1 SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Second page backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + + node.stop() + node.cleanup() + + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + count = node.execute("postgres", "SELECT count(*) FROM tbl1") + self.assertEqual(count[0][0], 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.safe_psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_pitr(self): + """ + make node with archiving, make stream backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_archive_pitr_2(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_restore_to_restore_point(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,10000)") + result = node.safe_psql( + "postgres", + "select * from t_heap") + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + node.safe_psql( + "postgres", + "create table t_heap_1 as select generate_series(0,10000)") + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "--recovery-target-name=savepoint", + "--recovery-target-action=promote"]) + + node.slow_start() + + result_new = node.safe_psql("postgres", "select * from t_heap") + res = node.psql("postgres", "select * from t_heap_1") + self.assertEqual( + res[0], 1, + "Table t_heap_1 should not exist in restored instance") + + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py new file mode 100644 index 000000000..652f7c39d --- /dev/null +++ b/tests/retention_test.py @@ -0,0 +1,178 @@ +import os +import unittest +from datetime import datetime, timedelta +from .helpers.ptrack_helpers import ProbackupTest + + +module_name = 'retention' + + +class RetentionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_1(self): + """purge backups using redundancy-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Check that WAL segments were deleted + min_wal = None + max_wal = None + for line in log.splitlines(): + if line.startswith("INFO: removed min WAL segment"): + min_wal = line[31:-1] + elif line.startswith("INFO: removed max WAL segment"): + max_wal = line[31:-1] + + if not min_wal: + self.assertTrue(False, "min_wal is empty") + + if not max_wal: + self.assertTrue(False, "max_wal is not set") + + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): + if not wal_name.endswith(".backup"): + # wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_window_2(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + conf.write("retention-window = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_wal(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=2']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py new file mode 100644 index 000000000..931da1844 --- /dev/null +++ b/tests/show_test.py @@ -0,0 +1,203 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'show' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_corrupt_2(self): + """Status CORRUPT""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup corrupted.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py new file mode 100644 index 000000000..ab091c578 --- /dev/null +++ b/tests/validate_test.py @@ -0,0 +1,1730 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'validate' + + +class ValidateTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """ + make node with archiving, make archive backup + validate to both real and unreal values + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + + pgbench.wait() + pgbench.stdout.close() + + target_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) + + # Validate to real time + self.assertIn( + "INFO: backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format( + unreal_time_1)]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) + try: + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate to real xid + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + self.switch_wal_segment(node) + + self.assertIn("INFO: backup validation completed successfully", + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 + try: + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate with backup ID + self.assertIn("INFO: Validating backup {0}".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} data files are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} WAL segments are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} is valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Validate of backup {0} completed".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backup(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, + run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file + file = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path) + with open(file, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message + and 'ERROR: Backup {0} is corrupt'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in FULL and PAGE1 backupd, run validate on PAGE1, + expect FULL and PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Corrupt some file in PAGE1 backup + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap_1) + with open(file_page1, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE1 + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_2) in e.message + and 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_3) in e.message + and 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups_1(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, + options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_page1) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_specific_target_corrupted_intermediate_backups(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE4 + target_xid = node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i RETURNING (xmin)")[0][0] + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb(backup_dir, 'node', + options=['--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_2) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_3) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_page(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in PAGE1 backup and run validate on instance, + expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + # PAGE3 + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups/node', backup_id_2, + 'database', file_path_t_heap1) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb( + backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_full) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(128) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_1(self): + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.backup_node(backup_dir, 'node', node) + + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) + os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: WAL segment \"{0}\" is absent".format( + file) in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') + + # Run validate again + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Revalidating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: not enough WAL records to xid' in e.message and + 'WARNING: recovery can be done up to time' in e.message and + "ERROR: not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_2(self): + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to this backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + file = os.path.join(wals_dir, max(wals)) + os.remove(file) + if self.archive_compress: + file = file[:-3] + + # Try to restore + try: + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for LSN' in e.message and + 'in archived WAL segment' in e.message and + 'WARNING: could not read WAL record at' in e.message and + 'ERROR: WAL segment "{0}" is absent\n'.format( + file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro702_688(self): + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(recovery_time)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro688(self): + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, + restore it to node1, check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() + + backup_id = self.backup_node( + backup_dir, 'node1', node1, options=["--stream"]) + + node2 = self.make_simple_node( + base_dir="{0}/{1}/node2".format(module_name, fname)) + node2.cleanup() + + node1.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node( + backup_dir, 'node1', node1, + backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.slow_start() + + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual( + timeline_node1, timeline_node2, + "Timelines on Master and Node1 should be equal. " + "This is unexpected") + + archive_command_node1 = node1.safe_psql( + "postgres", "show archive_command") + archive_command_node2 = node2.safe_psql( + "postgres", "show archive_command") + self.assertEqual( + archive_command_node1, archive_command_node2, + "Archive command on Master and Node should be equal. " + "This is unexpected") + + # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + # if result == "": + # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + self.switch_wal_segment(node1) + self.switch_wal_segment(node2) + time.sleep(5) + + log_file = os.path.join(node2.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse( + 'pg_probackup archive-push completed successfully' in log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and three page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption and run valudate again, check that + second full backup and his page backups are OK + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "alter system set archive_command = 'false'") + node.reload() + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=1s']) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + pass + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.set_archiving(backup_dir, 'node', node) + node.reload() + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + + os.rename(file_new, file) + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full_1(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and corrupt his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + second page should be CORRUPT + third page should be ORPHAN + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + os.rename(file_new, file) + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_page, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh new file mode 100644 index 000000000..7fe1cfd8f --- /dev/null +++ b/travis/backup_restore.sh @@ -0,0 +1,66 @@ +#!/bin/sh -ex + +# vars +export PGVERSION=9.5.4 +export PATH=$PATH:/usr/pgsql-9.5/bin +export PGUSER=pgbench +export PGDATABASE=pgbench +export PGDATA=/var/lib/pgsql/9.5/data +export BACKUP_PATH=/backups +export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog +export PGDATA2=/var/lib/pgsql/9.5/data2 +export PGBENCH_SCALE=100 +export PGBENCH_TIME=60 + +# prepare directory +cp -a /tests /build +pushd /build + +# download postgresql +yum install -y wget +wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz +tar xf postgresql.tar.gz + +# install pg_probackup +yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm +yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel +make top_srcdir=postgresql-$PGVERSION +make install top_srcdir=postgresql-$PGVERSION + +# initalize cluster and database +yum install -y postgresql95-server +su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" +cat < $PGDATA/pg_hba.conf +local all all trust +host all all 127.0.0.1/32 trust +local replication pgbench trust +host replication pgbench 127.0.0.1/32 trust +EOF +cat < $PGDATA/postgresql.auto.conf +max_wal_senders = 2 +wal_level = logical +wal_log_hints = on +EOF +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" +su postgres -c "createdb -U postgres $PGUSER" +su postgres -c "createuser -U postgres -a -d -E $PGUSER" +pgbench -i -s $PGBENCH_SCALE + +# Count current +COUNT=$(psql -Atc "select count(*) from pgbench_accounts") +pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & + +# create backup +pg_probackup init +pg_probackup backup -b full --disable-ptrack-clear --stream -v +pg_probackup show +sleep $PGBENCH_TIME + +# restore from backup +chown -R postgres:postgres $BACKUP_PATH +su postgres -c "pg_probackup restore -D $PGDATA2" + +# start backup server +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" +( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 000000000..148641812 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 000000000..c869e485b --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 000000000..a4f75553c --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From 8c5e8e0ccd3cbf65bd4f31bacdbeb7e2f01f36ba Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Oct 2018 12:22:27 +0300 Subject: [PATCH 0014/2107] minor spell fix in elog messages about LSN from future --- src/data.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index e8b7171fd..c3fcc2544 100644 --- a/src/data.c +++ b/src/data.c @@ -1501,8 +1501,8 @@ validate_one_page(Page page, pgFile *file, lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) - elog(WARNING, "File: %s, block %u, checksum is not enabled." - "page is from future: pageLSN %X/%X stopLSN %X/%X", + elog(WARNING, "File: %s, block %u, checksum is not enabled. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); else @@ -1515,8 +1515,8 @@ validate_one_page(Page page, pgFile *file, lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) - elog(WARNING, "File: %s, block %u, checksum is correct." - "page is from future: pageLSN %X/%X stopLSN %X/%X", + elog(WARNING, "File: %s, block %u, checksum is correct. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); else From 9804b79c91d1bc6a6f198b812d0b4d7548e5e65e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Oct 2018 12:47:21 +0300 Subject: [PATCH 0015/2107] bugfix: uninterrupt WAL reading --- src/parsexlog.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index c087c51ba..7f7365f5c 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -793,6 +793,10 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private_data = (XLogPageReadPrivate *) xlogreader->private_data; targetPageOff = targetPagePtr % private_data->xlog_seg_size; + if (interrupted) + elog(ERROR, "Thread [%d]: Interrupted during WAL reading", + private_data->thread_num); + /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. From b065f09cf3318f23b8f85ae7324145d8e9a9f52d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 29 Oct 2018 19:19:08 +0300 Subject: [PATCH 0016/2107] Release PGresult after query execution in pg_ptrack_clear() --- src/backup.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1d58d9f2b..fc8630700 100644 --- a/src/backup.c +++ b/src/backup.c @@ -538,8 +538,8 @@ do_backup_instance(void) } /* - * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to - * stort_backup LSN of previous backup + * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not + * equal to stop_lsn of previous backup. */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -1283,7 +1283,9 @@ pg_ptrack_clear(void) tblspcOid = atoi(PQgetvalue(res_db, i, 2)); tmp_conn = pgut_connect(dbname); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", + 0, NULL); + PQclear(res); sprintf(params[0], "%i", dbOid); sprintf(params[1], "%i", tblspcOid); @@ -1512,7 +1514,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (wait_prev_segment) elog(LOG, "Looking for segment: %s", wal_segment); else - elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + elog(LOG, "Looking for LSN: %X/%X in segment: %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment); #ifdef HAVE_LIBZ snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", @@ -2648,7 +2651,8 @@ get_last_ptrack_lsn(void) uint32 lsn_lo; XLogRecPtr lsn; - res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL); + res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", + 0, NULL); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); From 7ab07a5f49eacc2792d8eaff9a01f60aa84a7f79 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 30 Oct 2018 15:44:54 +0300 Subject: [PATCH 0017/2107] Fix tests - remove outdated comment about ptrack - stop compatibility tests from launching two times - make test_tablespace_in_pgdata_pgpro_1376 clean up after itself --- tests/__init__.py | 3 --- tests/backup_test.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index f72684695..3593f657f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,7 +21,6 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) # suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(compression)) - suite.addTests(loader.loadTestsFromModule(compatibility)) suite.addTests(loader.loadTestsFromModule(delete_test)) suite.addTests(loader.loadTestsFromModule(delta)) suite.addTests(loader.loadTestsFromModule(exclude)) @@ -62,8 +61,6 @@ def load_tests(loader, tests, pattern): # logging: # https://jira.postgrespro.ru/browse/PGPRO-584 # https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md -# ptrack: -# ptrack backup on replica should work correctly # archive: # immediate recovery and full recovery # backward compatibility: diff --git a/tests/backup_test.py b/tests/backup_test.py index 1fa74643a..b21aab9b4 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -520,3 +520,6 @@ def test_tablespace_in_pgdata_pgpro_1376(self): if self.paranoia: pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From bf1c8797aedeaebe9446ad79dfe18a79f74eb257 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 30 Oct 2018 16:21:38 +0300 Subject: [PATCH 0018/2107] Make pg_probackup support ptrack 1.7 --- src/backup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index fc8630700..0dd681fae 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1193,7 +1193,8 @@ pg_ptrack_support(void) /* Now we support only ptrack versions upper than 1.5 */ if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && - strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0) + strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "1.7") != 0) { elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); PQclear(res_db); From d29aa8b0b4ce8a5e3bd3c7b5dfbbf040d7029db5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 31 Oct 2018 09:47:53 +0300 Subject: [PATCH 0019/2107] PGPRO-2095: backup from replica without connection to master for PostgreSQL >= 9.6 --- src/backup.c | 81 +++++++++++++++++++++++++++++++++---------- src/pg_probackup.h | 1 + src/util.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 18 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0dd681fae..602ab823e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -475,6 +475,8 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + pgFile *pg_control = NULL; + elog(LOG, "Database backup start"); /* Initialize size summary */ @@ -754,9 +756,37 @@ do_backup_instance(void) parray_free(prev_backup_filelist); } + /* Copy pg_control in case of backup from replica >= 9.6 */ + if (current.from_replica && !exclusive_backup) + { + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); + + if (strcmp(tmp_file->name, "pg_control") == 0) + { + pg_control = tmp_file; + break; + } + } + + if (!pg_control) + elog(ERROR, "Failed to locate pg_control in copied files"); + + if (is_remote_backup) + remote_copy_file(NULL, pg_control); + else + if (!copy_file(pgdata, database_path, pg_control)) + elog(ERROR, "Failed to copy pg_control"); + } + + /* Notify end of backup */ pg_stop_backup(¤t); + if (current.from_replica && !exclusive_backup) + set_min_recovery_point(pg_control, database_path, current.stop_lsn); + /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { @@ -883,7 +913,7 @@ do_backup(time_t start_time) } } - if (current.from_replica) + if (current.from_replica && exclusive_backup) { /* Check master connection options */ if (master_host == NULL) @@ -1089,8 +1119,11 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; - /* For replica we call pg_start_backup() on master */ - conn = (backup->from_replica) ? master_conn : backup_conn; + /* For 9.5 replica we call pg_start_backup() on master */ + if (backup->from_replica && exclusive_backup) + conn = master_conn; + else + conn = backup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1118,16 +1151,21 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) PQclear(res); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE && + (!(backup->from_replica && !exclusive_backup))) /* * Switch to a new WAL segment. It is necessary to get archived WAL * segment, which includes start LSN of current backup. + * Don`t do this for replica backups unless it`s PG 9.5 */ pg_switch_wal(conn); + //elog(INFO, "START LSN: %X/%X", + // (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn)); + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ - wait_wal_lsn(backup->start_lsn, true, false); + wait_wal_lsn(backup->start_lsn, true, false); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream @@ -1669,7 +1707,7 @@ pg_stop_backup(pgBackup *backup) PGresult *tablespace_map_content = NULL; uint32 lsn_hi; uint32 lsn_lo; - XLogRecPtr restore_lsn = InvalidXLogRecPtr; + //XLogRecPtr restore_lsn = InvalidXLogRecPtr; int pg_stop_backup_timeout = 0; char path[MAXPGPATH]; char backup_label[MAXPGPATH]; @@ -1689,16 +1727,21 @@ pg_stop_backup(pgBackup *backup) if (!backup_in_progress) elog(ERROR, "backup is not in progress"); - /* For replica we call pg_stop_backup() on master */ - conn = (current.from_replica) ? master_conn : backup_conn; + /* For 9.5 replica we call pg_stop_backup() on master */ + if (current.from_replica && exclusive_backup) + conn = master_conn; + else + conn = backup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); - /* Create restore point */ - if (backup != NULL) + /* Create restore point + * only if it`s backup from master, or exclusive replica(wich connects to master) + */ + if (backup != NULL && (!current.from_replica || (current.from_replica && exclusive_backup))) { const char *params[1]; char name[1024]; @@ -1716,7 +1759,7 @@ pg_stop_backup(pgBackup *backup) /* Extract timeline and LSN from the result */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); /* Calculate LSN */ - restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + //restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; PQclear(res); } @@ -1830,10 +1873,10 @@ pg_stop_backup(pgBackup *backup) /* Calculate LSN */ stop_backup_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - if (!XRecOffIsValid(stop_backup_lsn)) - { - stop_backup_lsn = restore_lsn; - } + //if (!XRecOffIsValid(stop_backup_lsn)) + //{ + // stop_backup_lsn = restore_lsn; + //} if (!XRecOffIsValid(stop_backup_lsn)) elog(ERROR, "Invalid stop_backup_lsn value %X/%X", @@ -1939,7 +1982,7 @@ pg_stop_backup(pgBackup *backup) stream_xlog_path[MAXPGPATH]; /* Wait for stop_lsn to be received by replica */ - if (backup->from_replica) + if (current.from_replica) wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. @@ -1962,10 +2005,12 @@ pg_stop_backup(pgBackup *backup) elog(LOG, "Getting the Recovery Time from WAL"); + /* iterate over WAL from stop_backup lsn to start_backup lsn */ if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, backup->start_lsn, backup->stop_lsn, &backup->recovery_time, &backup->recovery_xid)) { + elog(LOG, "Failed to find Recovery Time in WAL. Forced to trust current_timestamp"); backup->recovery_time = recovery_time; backup->recovery_xid = recovery_xid; } @@ -2074,7 +2119,7 @@ backup_files(void *arg) elog(ERROR, "interrupted during backup"); if (progress) - elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_backup_files_list, file->path); /* stat file to check its current state */ @@ -2168,7 +2213,7 @@ backup_files(void *arg) file->path, file->write_size); } else - elog(LOG, "unexpected file type %d", buf.st_mode); + elog(WARNING, "unexpected file type %d", buf.st_mode); } /* Close connection */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e337771e6..b75bb5817 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -555,6 +555,7 @@ extern uint64 get_system_identifier(char *pgdata); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern uint32 get_xlog_seg_size(char *pgdata_path); +extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); diff --git a/src/util.c b/src/util.c index 4eefa7887..5f059c374 100644 --- a/src/util.c +++ b/src/util.c @@ -14,6 +14,8 @@ #include +#include + const char * base36enc(long unsigned int value) { @@ -100,6 +102,44 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } +/* + * Write ControlFile to pg_control + */ +static void +writeControlFile(ControlFileData *ControlFile, char *path) +{ + int fd; + char *buffer = NULL; + +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + /* copy controlFileSize */ + buffer = pg_malloc(ControlFileSize); + memcpy(buffer, &ControlFile, sizeof(ControlFileData)); + + /* Write pg_control */ + unlink(path); + fd = open(path, + O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR); + + if (fd < 0) + elog(ERROR, "Failed to open file: %s", path); + + if (write(fd, buffer, ControlFileSize) != ControlFileSize) + elog(ERROR, "Failed to overwrite file: %s", path); + + if (fsync(fd) != 0) + elog(ERROR, "Failed to fsync file: %s", path); + + pg_free(buffer); + close(fd); +} + /* * Utility shared by backup and restore to fetch the current timeline * used by a node. @@ -250,6 +290,52 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } +/* MinRecoveryPoint 'as-is' is not to be trusted + * Use STOP LSN instead + */ +void +set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + char fullpath[MAXPGPATH]; + + elog(LOG, "Setting minRecPoint to STOP LSN: %X/%X", + (uint32) (stop_backup_lsn >> 32), + (uint32) stop_backup_lsn); + + /* Path to pg_control in backup */ + snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); + + /* First fetch file... */ + buffer = slurpFile(backup_path, XLOG_CONTROL_FILE, &size, false); + if (buffer == NULL) + elog(ERROR, "ERROR"); + + digestControlFile(&ControlFile, buffer, size); + + ControlFile.minRecoveryPoint = stop_backup_lsn; + + /* Update checksum in pg_control header */ + INIT_CRC32C(ControlFile.crc); + COMP_CRC32C(ControlFile.crc, + (char *) &ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(ControlFile.crc); + + /* paranoia */ + checkControlFile(&ControlFile); + + /* update pg_control */ + writeControlFile(&ControlFile, fullpath); + + /* Update pg_control checksum in backup_list */ + file->crc = pgFileGetCRC(fullpath, false); + + pg_free(buffer); +} + /* * Convert time_t value to ISO-8601 format string. Always set timezone offset. From 96b4aa791d0797f973703965a9805fcd977257f5 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Wed, 31 Oct 2018 13:44:16 +0300 Subject: [PATCH 0020/2107] ICU: fix: run default collation tests during make (install)check-world Thanks to Alexander Lakhin for reporting this. --- .gitignore | 45 + .travis.yml | 7 + COPYRIGHT | 29 + Makefile | 87 + README.md | 100 + doit.cmd | 1 + doit96.cmd | 1 + gen_probackup_project.pl | 190 ++ msvs/pg_probackup.sln | 28 + msvs/template.pg_probackup.vcxproj | 212 ++ msvs/template.pg_probackup96.vcxproj | 210 ++ msvs/template.pg_probackup_2.vcxproj | 203 ++ src/archive.c | 113 + src/backup.c | 2701 ++++++++++++++++++++++++ src/catalog.c | 915 ++++++++ src/configure.c | 490 +++++ src/data.c | 1407 ++++++++++++ src/delete.c | 464 ++++ src/dir.c | 1491 +++++++++++++ src/fetch.c | 116 + src/help.c | 605 ++++++ src/init.c | 108 + src/merge.c | 526 +++++ src/parsexlog.c | 1039 +++++++++ src/pg_probackup.c | 634 ++++++ src/pg_probackup.h | 620 ++++++ src/restore.c | 920 ++++++++ src/show.c | 500 +++++ src/status.c | 118 ++ src/util.c | 349 +++ src/utils/json.c | 134 ++ src/utils/json.h | 33 + src/utils/logger.c | 621 ++++++ src/utils/logger.h | 54 + src/utils/parray.c | 196 ++ src/utils/parray.h | 35 + src/utils/pgut.c | 2417 +++++++++++++++++++++ src/utils/pgut.h | 238 +++ src/utils/thread.c | 102 + src/utils/thread.h | 35 + src/validate.c | 354 ++++ tests/Readme.md | 24 + tests/__init__.py | 69 + tests/archive.py | 833 ++++++++ tests/auth_test.py | 391 ++++ tests/backup_test.py | 522 +++++ tests/cfs_backup.py | 1161 ++++++++++ tests/cfs_restore.py | 450 ++++ tests/cfs_validate_backup.py | 25 + tests/compression.py | 496 +++++ tests/delete_test.py | 203 ++ tests/delta.py | 1265 +++++++++++ tests/exclude.py | 164 ++ tests/expected/option_help.out | 95 + tests/expected/option_version.out | 1 + tests/false_positive.py | 333 +++ tests/helpers/__init__.py | 2 + tests/helpers/cfs_helpers.py | 91 + tests/helpers/ptrack_helpers.py | 1300 ++++++++++++ tests/init_test.py | 99 + tests/logging.py | 0 tests/merge.py | 454 ++++ tests/option_test.py | 218 ++ tests/page.py | 641 ++++++ tests/pgpro560.py | 98 + tests/pgpro589.py | 80 + tests/ptrack.py | 1600 ++++++++++++++ tests/ptrack_clean.py | 253 +++ tests/ptrack_cluster.py | 268 +++ tests/ptrack_move_to_tablespace.py | 57 + tests/ptrack_recovery.py | 58 + tests/ptrack_truncate.py | 130 ++ tests/ptrack_vacuum.py | 152 ++ tests/ptrack_vacuum_bits_frozen.py | 136 ++ tests/ptrack_vacuum_bits_visibility.py | 67 + tests/ptrack_vacuum_full.py | 140 ++ tests/ptrack_vacuum_truncate.py | 142 ++ tests/replica.py | 293 +++ tests/restore_test.py | 1243 +++++++++++ tests/retention_test.py | 178 ++ tests/show_test.py | 203 ++ tests/validate_test.py | 1730 +++++++++++++++ travis/backup_restore.sh | 66 + win32build.pl | 240 +++ win32build96.pl | 240 +++ win32build_2.pl | 219 ++ 86 files changed, 34878 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 README.md create mode 100644 doit.cmd create mode 100644 doit96.cmd create mode 100644 gen_probackup_project.pl create mode 100644 msvs/pg_probackup.sln create mode 100644 msvs/template.pg_probackup.vcxproj create mode 100644 msvs/template.pg_probackup96.vcxproj create mode 100644 msvs/template.pg_probackup_2.vcxproj create mode 100644 src/archive.c create mode 100644 src/backup.c create mode 100644 src/catalog.c create mode 100644 src/configure.c create mode 100644 src/data.c create mode 100644 src/delete.c create mode 100644 src/dir.c create mode 100644 src/fetch.c create mode 100644 src/help.c create mode 100644 src/init.c create mode 100644 src/merge.c create mode 100644 src/parsexlog.c create mode 100644 src/pg_probackup.c create mode 100644 src/pg_probackup.h create mode 100644 src/restore.c create mode 100644 src/show.c create mode 100644 src/status.c create mode 100644 src/util.c create mode 100644 src/utils/json.c create mode 100644 src/utils/json.h create mode 100644 src/utils/logger.c create mode 100644 src/utils/logger.h create mode 100644 src/utils/parray.c create mode 100644 src/utils/parray.h create mode 100644 src/utils/pgut.c create mode 100644 src/utils/pgut.h create mode 100644 src/utils/thread.c create mode 100644 src/utils/thread.h create mode 100644 src/validate.c create mode 100644 tests/Readme.md create mode 100644 tests/__init__.py create mode 100644 tests/archive.py create mode 100644 tests/auth_test.py create mode 100644 tests/backup_test.py create mode 100644 tests/cfs_backup.py create mode 100644 tests/cfs_restore.py create mode 100644 tests/cfs_validate_backup.py create mode 100644 tests/compression.py create mode 100644 tests/delete_test.py create mode 100644 tests/delta.py create mode 100644 tests/exclude.py create mode 100644 tests/expected/option_help.out create mode 100644 tests/expected/option_version.out create mode 100644 tests/false_positive.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/cfs_helpers.py create mode 100644 tests/helpers/ptrack_helpers.py create mode 100644 tests/init_test.py create mode 100644 tests/logging.py create mode 100644 tests/merge.py create mode 100644 tests/option_test.py create mode 100644 tests/page.py create mode 100644 tests/pgpro560.py create mode 100644 tests/pgpro589.py create mode 100644 tests/ptrack.py create mode 100644 tests/ptrack_clean.py create mode 100644 tests/ptrack_cluster.py create mode 100644 tests/ptrack_move_to_tablespace.py create mode 100644 tests/ptrack_recovery.py create mode 100644 tests/ptrack_truncate.py create mode 100644 tests/ptrack_vacuum.py create mode 100644 tests/ptrack_vacuum_bits_frozen.py create mode 100644 tests/ptrack_vacuum_bits_visibility.py create mode 100644 tests/ptrack_vacuum_full.py create mode 100644 tests/ptrack_vacuum_truncate.py create mode 100644 tests/replica.py create mode 100644 tests/restore_test.py create mode 100644 tests/retention_test.py create mode 100644 tests/show_test.py create mode 100644 tests/validate_test.py create mode 100644 travis/backup_restore.sh create mode 100644 win32build.pl create mode 100644 win32build96.pl create mode 100644 win32build_2.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..02d1512ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Object files +*.o + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.app + +# Dependencies +.deps + +# Binaries +/pg_probackup + +# Generated by test suite +/regression.diffs +/regression.out +/results +/env +/tests/__pycache__/ +/tests/helpers/__pycache__/ +/tests/tmp_dirs/ +/tests/*pyc +/tests/helpers/*pyc + +# Extra files +/src/datapagemap.c +/src/datapagemap.h +/src/logging.h +/src/receivelog.c +/src/receivelog.h +/src/streamutil.c +/src/streamutil.h +/src/xlogreader.c +/src/walmethods.c +/src/walmethods.h diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..35b49ec5b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: required + +services: +- docker + +script: +- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 000000000..49d704724 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,29 @@ +Copyright (c) 2015-2017, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + +Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +Portions Copyright (c) 1994, The Regents of the University of California + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION + (NTT) nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..56ad1b019 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +PROGRAM = pg_probackup +OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \ + src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \ + src/pg_probackup.o src/restore.o src/show.o src/status.o \ + src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \ + src/xlogreader.o src/streamutil.o src/receivelog.o \ + src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \ + src/utils/json.o src/utils/thread.o src/merge.o + +EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \ + src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h + +INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +# !USE_PGXS +else +subdir=contrib/pg_probackup +top_builddir=../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif # USE_PGXS + +ifeq ($(top_srcdir),../..) + ifeq ($(LN_S),ln -s) + srchome=$(top_srcdir)/.. + endif +else +srchome=$(top_srcdir) +endif + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +INCLUDES += src/walmethods.h +endif + +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) +PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} + +all: checksrcdir $(INCLUDES); + +$(PROGRAM): $(OBJS) + +src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c + rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ +src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ +src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ +src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ +src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ +src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ +src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ + + +ifneq (,$(filter 10 11 12,$(MAJORVERSION))) +src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ +src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +endif + +ifeq ($(PORTNAME), aix) + CC=xlc_r +endif + +# This rule's only purpose is to give the user instructions on how to pass +# the path to PostgreSQL source tree to the makefile. +.PHONY: checksrcdir +checksrcdir: +ifndef top_srcdir + @echo "You must have PostgreSQL source tree available to compile." + @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" + @echo "variable: \"make top_srcdir=\"" + @exit 1 +endif diff --git a/README.md b/README.md new file mode 100644 index 000000000..1471d648f --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# pg_probackup + +`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. + +The utility is compatible with: +* PostgreSQL 9.5, 9.6, 10; + +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6 +* Postgres Pro Enterprise + +As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: +* Choosing between full and page-level incremental backups to speed up backup and recovery +* Implementing a single backup strategy for multi-server PostgreSQL clusters +* Automatic data consistency checks and on-demand backup validation without actual data recovery +* Managing backups in accordance with retention policy +* Running backup, restore, and validation processes on multiple parallel threads +* Storing backup data in a compressed state to save disk space +* Taking backups from a standby server to avoid extra load on the master server +* Extended logging settings +* Custom commands to simplify WAL log archiving + +To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. + +Using `pg_probackup`, you can take full or incremental backups: +* `Full` backups contain all the data files required to restore the database cluster from scratch. +* `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups: + * `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode could impose read-only I/O pressure equal to `Full` backup. + * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies: +* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). + +## Limitations + +`pg_probackup` currently has the following limitations: +* Creating backups from a remote server is currently not supported. +* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number. +* Microsoft Windows operating system is not supported. +* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. + +## Installation and Setup +### Linux Installation +```shell +#DEB Ubuntu|Debian Packages +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +apt-get install pg-probackup-{10,9.6,9.5} + +#DEB-SRC Packages +echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list +apt-get source pg-probackup-{10,9.6,9.5} + +#RPM Centos Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM RHEL Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#RPM Oracle Linux Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm +yum install pg_probackup-{10,9.6,9.5} + +#SRPM Packages +yumdownloader --source pg_probackup-{10,9.6,9.5} +``` + +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: + +```shell +make USE_PGXS=1 PG_CONFIG= top_srcdir= +``` + +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Documentation + +Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). + +## Licence + +This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/). + +## Feedback + +Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_probackup/issues) page. + +## Authors + +Postgres Professional, Moscow, Russia. + +## Credits + +`pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file diff --git a/doit.cmd b/doit.cmd new file mode 100644 index 000000000..b46e3b36d --- /dev/null +++ b/doit.cmd @@ -0,0 +1 @@ +perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd new file mode 100644 index 000000000..94d242c99 --- /dev/null +++ b/doit96.cmd @@ -0,0 +1 @@ +perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl new file mode 100644 index 000000000..3ea79e96c --- /dev/null +++ b/gen_probackup_project.pl @@ -0,0 +1,190 @@ +# -*-perl-*- hey - emacs - this is a perl file +BEGIN{ +use Cwd; +use File::Basename; + +my $pgsrc=""; +if (@ARGV==1) +{ + $pgsrc = shift @ARGV; + if($pgsrc == "--help"){ + print STDERR "Usage $0 pg-source-dir \n"; + print STDERR "Like this: \n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; + print STDERR "May be need input this before: \n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + exit 1; + } +} +else +{ + use Cwd qw(abs_path); + my $path = dirname(abs_path($0)); + chdir($path); + chdir("../.."); + $pgsrc = cwd(); +} + +chdir("$pgsrc/src/tools/msvc"); +push(@INC, "$pgsrc/src/tools/msvc"); +chdir("../../..") if (-d "../msvc" && -d "../../../src"); + +} + +use Win32; +use Carp; +use strict; +use warnings; + + +use Project; +use Solution; +use File::Copy; +use Config; +use VSObjectFactory; +use List::Util qw(first); + +use Exporter; +our (@ISA, @EXPORT_OK); +@ISA = qw(Exporter); +@EXPORT_OK = qw(Mkvcbuild); + +my $solution; +my $libpgport; +my $libpgcommon; +my $libpgfeutils; +my $postgres; +my $libpq; +my @unlink_on_exit; + + +use lib "src/tools/msvc"; + +use Mkvcbuild; + +# if (-e "src/tools/msvc/buildenv.pl") +# { +# do "src/tools/msvc/buildenv.pl"; +# } +# elsif (-e "./buildenv.pl") +# { +# do "./buildenv.pl"; +# } + +# set up the project +our $config; +do "config_default.pl"; +do "config.pl" if (-f "src/tools/msvc/config.pl"); + +# my $vcver = Mkvcbuild::mkvcbuild($config); +my $vcver = build_pgprobackup($config); + +# check what sort of build we are doing + +my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; +my $buildwhat = $ARGV[1] || ""; +if (uc($ARGV[0]) eq 'DEBUG') +{ + $bconf = "Debug"; +} +elsif (uc($ARGV[0]) ne "RELEASE") +{ + $buildwhat = $ARGV[0] || ""; +} + +# ... and do it +system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); + + +# report status + +my $status = $? >> 8; + +exit $status; + + + +sub build_pgprobackup +{ + our $config = shift; + + chdir('../../..') if (-d '../msvc' && -d '../../../src'); + die 'Must run from root or msvc directory' + unless (-d 'src/tools/msvc' && -d 'src'); + + # my $vsVersion = DetermineVisualStudioVersion(); + my $vsVersion = '12.00'; + + $solution = CreateSolution($vsVersion, $config); + + $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', + 'src/interfaces/libpq'); + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); + $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); + + #vvs test + my $probackup = + $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $probackup->AddFiles( + 'contrib/pg_probackup/src', + 'archive.c', + 'backup.c', + 'catalog.c', + 'configure.c', + 'data.c', + 'delete.c', + 'dir.c', + 'fetch.c', + 'help.c', + 'init.c', + 'parsexlog.c', + 'pg_probackup.c', + 'restore.c', + 'show.c', + 'status.c', + 'util.c', + 'validate.c' + ); + $probackup->AddFiles( + 'contrib/pg_probackup/src/utils', + 'json.c', + 'logger.c', + 'parray.c', + 'pgut.c', + 'thread.c' + ); + $probackup->AddFile('src/backend/access/transam/xlogreader.c'); + $probackup->AddFiles( + 'src/bin/pg_basebackup', + 'receivelog.c', + 'streamutil.c' + ); + + if (-e 'src/bin/pg_basebackup/walmethods.c') + { + $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + } + + $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + + $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + + $probackup->AddIncludeDir('src/bin/pg_basebackup'); + $probackup->AddIncludeDir('src/bin/pg_rewind'); + $probackup->AddIncludeDir('src/interfaces/libpq'); + $probackup->AddIncludeDir('src'); + $probackup->AddIncludeDir('src/port'); + + $probackup->AddIncludeDir('contrib/pg_probackup'); + $probackup->AddIncludeDir('contrib/pg_probackup/src'); + $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddLibrary('ws2_32.lib'); + + $probackup->Save(); + return $solution->{vcver}; + +} diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln new file mode 100644 index 000000000..2df4b4042 --- /dev/null +++ b/msvs/pg_probackup.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2013 for Windows Desktop +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 + {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj new file mode 100644 index 000000000..46a7b2c24 --- /dev/null +++ b/msvs/template.pg_probackup.vcxproj @@ -0,0 +1,212 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj new file mode 100644 index 000000000..46e019ba4 --- /dev/null +++ b/msvs/template.pg_probackup96.vcxproj @@ -0,0 +1,210 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) + @PGROOT@\lib;$(LibraryPath) + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj new file mode 100644 index 000000000..2fc101a42 --- /dev/null +++ b/msvs/template.pg_probackup_2.vcxproj @@ -0,0 +1,203 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4886B21A-D8CA-4A03-BADF-743B24C88327} + Win32Proj + pg_probackup + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + true + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + false + ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) + @PGROOT@\lib;@$(LibraryPath) + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + + + + Level3 + Disabled + _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + Level3 + + + MaxSpeed + true + true + _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + true + + + Console + true + true + true + @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) + libc;%(IgnoreSpecificDefaultLibraries) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/archive.c b/src/archive.c new file mode 100644 index 000000000..953a68779 --- /dev/null +++ b/src/archive.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * archive.c: - pg_probackup specific archive commands for archive backups. + * + * + * Portions Copyright (c) 2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +#include +#include + +/* + * pg_probackup specific archive command for archive backups + * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup + * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/system_id. + * Currently it just copies wal files to the new location. + * TODO: Planned options: list the arclog content, + * compute and validate checksums. + */ +int +do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + int64 system_id; + pgBackupConfig *config; + bool is_compress = false; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + /* verify that archive-push --instance parameter is valid */ + config = readBackupCatalogConfigFile(); + system_id = get_system_identifier(current_dir); + + if (config->pgdata == NULL) + elog(ERROR, "cannot read pg_probackup.conf for this instance"); + + if(system_id != config->system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " INT64_FORMAT " instead of " INT64_FORMAT, + wal_file_name, instance_name, config->system_identifier, system_id); + + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + dir_create_dir(arclog_path, DIR_PERMISSION); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + + if (compress_alg == PGLZ_COMPRESS) + elog(ERROR, "pglz compression is not supported"); + +#ifdef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + is_compress = IsXLogFileName(wal_file_name); +#endif + + push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, + overwrite); + elog(INFO, "pg_probackup archive-push completed successfully"); + + return 0; +} + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", + backup_wal_file_path, absolute_wal_file_path); + get_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; +} diff --git a/src/backup.c b/src/backup.c new file mode 100644 index 000000000..3aa36c98b --- /dev/null +++ b/src/backup.c @@ -0,0 +1,2701 @@ +/*------------------------------------------------------------------------- + * + * backup.c: backup DB cluster, archived WAL + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" +#include "libpq/pqsignal.h" +#include "pgtar.h" +#include "receivelog.h" +#include "storage/bufpage.h" +#include "streamutil.h" +#include "utils/thread.h" + +static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ +static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +const char *progname = "pg_probackup"; + +/* list of files contained in backup */ +static parray *backup_files_list = NULL; + +/* We need critical section for datapagemap_add() in case of using threads */ +static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + const char *basedir; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static int is_ptrack_enable = false; +bool is_ptrack_support = false; +bool is_checksum_enabled = false; +bool exclusive_backup = false; + +/* Backup connections */ +static PGconn *backup_conn = NULL; +static PGconn *master_conn = NULL; +static PGconn *backup_conn_replication = NULL; + +/* PostgreSQL server version from "backup_conn" */ +static int server_version = 0; +static char server_version_str[100] = ""; + +/* Is pg_start_backup() was executed */ +static bool backup_in_progress = false; +/* Is pg_stop_backup() was sent */ +static bool pg_stop_backup_is_sent = false; + +/* + * Backup routines + */ +static void backup_cleanup(bool fatal, void *userdata); +static void backup_disconnect(bool fatal, void *userdata); + +static void *backup_files(void *arg); +static void *remote_backup_files(void *arg); + +static void do_backup_instance(void); + +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_switch_wal(PGconn *conn); +static void pg_stop_backup(pgBackup *backup); +static int checkpoint_timeout(void); + +//static void backup_list_file(parray *files, const char *root, ) +static void parse_backup_filelist_filenames(parray *files, const char *root); +static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment); +static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); +static void make_pagemap_from_ptrack(parray *files); +static void *StreamLog(void *arg); + +static void get_remote_pgdata_filelist(parray *files); +static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); +static void remote_copy_file(PGconn *conn, pgFile* file); + +/* Ptrack functions */ +static void pg_ptrack_clear(void); +static bool pg_ptrack_support(void); +static bool pg_ptrack_enable(void); +static bool pg_checksum_enable(void); +static bool pg_is_in_recovery(void); +static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid); +static char *pg_ptrack_get_and_clear(Oid tablespace_oid, + Oid db_oid, + Oid rel_oid, + size_t *result_size); +static XLogRecPtr get_last_ptrack_lsn(void); + +/* Check functions */ +static void check_server_version(void); +static void check_system_identifiers(void); +static void confirm_block_size(const char *name, int blcksz); +static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); + +#define disconnect_and_exit(code) \ + { \ + if (conn != NULL) PQfinish(conn); \ + exit(code); \ + } + +/* Fill "files" with data about all the files to backup */ +static void +get_remote_pgdata_filelist(parray *files) +{ + PGresult *res; + int resultStatus; + int i; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, "FILE_BACKUP", PQerrorMessage(backup_conn_replication)); + + res = PQgetResult(backup_conn_replication); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + resultStatus = PQresultStatus(res); + PQclear(res); + elog(ERROR, "cannot start getting FILE_BACKUP filelist: %s, result_status %d", + PQerrorMessage(backup_conn_replication), resultStatus); + } + + if (PQntuples(res) < 1) + elog(ERROR, "%s: no data returned from server", PROGRAM_NAME); + + for (i = 0; i < PQntuples(res); i++) + { + ReceiveFileList(files, backup_conn_replication, res, i); + } + + res = PQgetResult(backup_conn_replication); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "%s: final receive failed: %s", + PROGRAM_NAME, PQerrorMessage(backup_conn_replication)); + } + + PQfinish(backup_conn_replication); +} + +/* + * workhorse for get_remote_pgdata_filelist(). + * Parse received message into pgFile structure. + */ +static void +ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) +{ + char filename[MAXPGPATH]; + pgoff_t current_len_left = 0; + bool basetablespace; + char *copybuf = NULL; + pgFile *pgfile; + + /* What for do we need this basetablespace field?? */ + basetablespace = PQgetisnull(res, rownum, 0); + if (basetablespace) + elog(LOG,"basetablespace"); + else + elog(LOG, "basetablespace %s", PQgetvalue(res, rownum, 1)); + + res = PQgetResult(conn); + + if (PQresultStatus(res) != PGRES_COPY_OUT) + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(conn)); + + while (1) + { + int r; + int filemode; + + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + + if (r == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + /* end of copy */ + if (r == -1) + break; + + /* This must be the header for a new file */ + if (r != 512) + elog(ERROR, "Invalid tar block header size: %d\n", r); + + current_len_left = read_tar_number(©buf[124], 12); + + /* Set permissions on the file */ + filemode = read_tar_number(©buf[100], 8); + + /* First part of header is zero terminated filename */ + snprintf(filename, sizeof(filename), "%s", copybuf); + + pgfile = pgFileInit(filename); + pgfile->size = current_len_left; + pgfile->mode |= filemode; + + if (filename[strlen(filename) - 1] == '/') + { + /* Symbolic link or directory has size zero */ + Assert (pgfile->size == 0); + /* Ends in a slash means directory or symlink to directory */ + if (copybuf[156] == '5') + { + /* Directory */ + pgfile->mode |= S_IFDIR; + } + else if (copybuf[156] == '2') + { + /* Symlink */ +#ifndef WIN32 + pgfile->mode |= S_IFLNK; +#else + pgfile->mode |= S_IFDIR; +#endif + } + else + elog(ERROR, "Unrecognized link indicator \"%c\"\n", + copybuf[156]); + } + else + { + /* regular file */ + pgfile->mode |= S_IFREG; + } + + parray_append(files, pgfile); + } + + if (copybuf != NULL) + PQfreemem(copybuf); +} + +/* read one file via replication protocol + * and write it to the destination subdir in 'backup_path' */ +static void +remote_copy_file(PGconn *conn, pgFile* file) +{ + PGresult *res; + char *copybuf = NULL; + char buf[32768]; + FILE *out; + char database_path[MAXPGPATH]; + char to_path[MAXPGPATH]; + bool skip_padding = false; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + join_path_components(to_path, database_path, file->path); + + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + INIT_CRC32C(file->crc); + + /* read from stream and write to backup file */ + while (1) + { + int row_length; + int errno_tmp; + int write_buffer_size = 0; + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + row_length = PQgetCopyData(conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); + + if (row_length == -1) + break; + + if (!skip_padding) + { + write_buffer_size = Min(row_length, sizeof(buf)); + memcpy(buf, copybuf, write_buffer_size); + COMP_CRC32C(file->crc, buf, write_buffer_size); + + /* TODO calc checksum*/ + if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) + { + errno_tmp = errno; + /* oops */ + FIN_CRC32C(file->crc); + fclose(out); + PQfinish(conn); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + file->read_size += write_buffer_size; + } + if (file->read_size >= file->size) + { + skip_padding = true; + } + } + + res = PQgetResult(conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); + } + + file->write_size = (int64) file->read_size; + FIN_CRC32C(file->crc); + + fclose(out); +} + +/* + * Take a remote backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + */ +static void * +remote_backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + PGconn *file_backup_conn = NULL; + + for (i = 0; i < n_backup_files_list; i++) + { + char *query_str; + PGresult *res; + char *copybuf = NULL; + pgFile *file; + int row_length; + + file = (pgFile *) parray_get(arguments->files_list, i); + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + file_backup_conn = pgut_connect_replication(pgut_dbname); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); + + if (PQsendQuery(file_backup_conn, query_str) == 0) + elog(ERROR,"%s: could not send replication command \"%s\": %s", + PROGRAM_NAME, query_str, PQerrorMessage(file_backup_conn)); + + res = PQgetResult(file_backup_conn); + + /* File is not found. That's normal. */ + if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + PQfinish(file_backup_conn); + continue; + } + + if (PQresultStatus(res) != PGRES_COPY_OUT) + { + PQclear(res); + PQfinish(file_backup_conn); + elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(file_backup_conn)); + } + + /* read the header of the file */ + row_length = PQgetCopyData(file_backup_conn, ©buf, 0); + + if (row_length == -2) + elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(file_backup_conn)); + + /* end of copy TODO handle it */ + if (row_length == -1) + elog(ERROR, "Unexpected end of COPY data"); + + if(row_length != 512) + elog(ERROR, "Invalid tar block header size: %d\n", row_length); + file->size = read_tar_number(©buf[124], 12); + + /* receive the data from stream and write to backup file */ + remote_copy_file(file_backup_conn, file); + + elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", + file->path, file->write_size); + PQfinish(file_backup_conn); + } + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Take a backup of a single postgresql instance. + * Move files from 'pgdata' to a subdirectory in 'backup_path'. + */ +static void +do_backup_instance(void) +{ + int i; + char database_path[MAXPGPATH]; + char dst_backup_path[MAXPGPATH]; + char label[1024]; + XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; + + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgBackup *prev_backup = NULL; + parray *prev_backup_filelist = NULL; + + elog(LOG, "Database backup start"); + + /* Initialize size summary */ + current.data_bytes = 0; + + /* Obtain current timeline */ + if (is_remote_backup) + { + char *sysidentifier; + TimeLineID starttli; + XLogRecPtr startpos; + + backup_conn_replication = pgut_connect_replication(pgut_dbname); + + /* Check replication prorocol connection */ + if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) + elog(ERROR, "Failed to send command for remote backup"); + +// TODO implement the check +// if (&sysidentifier != system_identifier) +// elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", +// system_identifier, sysidentifier); + + current.tli = starttli; + + PQfinish(backup_conn_replication); + } + else + current.tli = get_current_timeline(false); + + /* + * In incremental backup mode ensure that already-validated + * backup on current timeline exists and get its filelist. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + { + parray *backup_list; + char prev_backup_filelist_path[MAXPGPATH]; + + /* get list of backups already taken */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + if (prev_backup == NULL) + elog(ERROR, "Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one."); + parray_free(backup_list); + + pgBackupGetPath(prev_backup, prev_backup_filelist_path, + lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); + /* Files of previous backup needed by DELTA backup */ + prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + + /* If lsn is not NULL, only pages with higher lsn will be copied. */ + prev_backup_start_lsn = prev_backup->start_lsn; + current.parent_backup = prev_backup->start_time; + + pgBackupWriteBackupControlFile(¤t); + } + + /* + * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not equal to + * stort_backup LSN of previous backup + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(); + + if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control " UINT64_FORMAT " differs from STOP LSN of previous backup " + UINT64_FORMAT ".\n" + "Create new full backup before an incremental one.", + ptrack_lsn, prev_backup->stop_lsn); + } + } + + /* Clear ptrack files for FULL and PAGE backup */ + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) + pg_ptrack_clear(); + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + pg_start_backup(label, smooth_checkpoint, ¤t); + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* start stream replication */ + if (stream_wal) + { + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + dir_create_dir(dst_backup_path, DIR_PERMISSION); + + stream_thread_arg.basedir = dst_backup_path; + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(pgut_dbname); + + if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) + { + PQfinish(stream_thread_arg.conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtaining start LSN position and current timeline ID + * at the same time, necessary if not valid data can be found in the + * existing output directory. + */ + if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL)) + { + PQfinish(stream_thread_arg.conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* By default there are some error */ + stream_thread_arg.ret = 1; + + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + } + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (is_remote_backup) + get_remote_pgdata_filelist(backup_files_list); + else + dir_list_file(backup_files_list, pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, pgdata); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(LOG, "current_tli:%X", current.tli); + elog(LOG, "prev_backup->start_lsn: %X/%X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); + elog(LOG, "current.start_lsn: %X/%X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* + * Build page mapping in incremental mode. + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + { + /* + * Build the page map. Obtain information about changed pages + * reading WAL segments present in archives up to the point + * where this backup has started. + */ + extractPageMap(arclog_path, current.tli, xlog_seg_size, + prev_backup->start_lsn, current.start_lsn, + /* + * For backup from master wait for previous segment. + * For backup from replica wait for current segment. + */ + !current.from_replica, backup_files_list); + } + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + /* + * Build the page map from ptrack information. + */ + make_pagemap_from_ptrack(backup_files_list); + } + + /* + * Make directories before backup and setup threads at the same time + */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* if the entry was a directory, create it in the backup */ + if (S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char database_path[MAXPGPATH]; + + if (!is_remote_backup) + dir_name = GetRelativePath(file->path, pgdata); + else + dir_name = file->path; + + elog(VERBOSE, "Create directory \"%s\"", dir_name); + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + join_path_components(dirpath, database_path, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } + + /* setup threads */ + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (prev_backup_filelist) + parray_qsort(prev_backup_filelist, pgFileComparePath); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->prev_filelist = prev_backup_filelist; + arg->prev_start_lsn = prev_backup_start_lsn; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + elog(LOG, "Start transfering data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + if (!is_remote_backup) + pthread_create(&threads[i], NULL, backup_files, arg); + else + pthread_create(&threads[i], NULL, remote_backup_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(LOG, "Data files are transfered"); + else + elog(ERROR, "Data files transferring failed"); + + /* clean previous backup file list */ + if (prev_backup_filelist) + { + parray_walk(prev_backup_filelist, pgFileFree); + parray_free(prev_backup_filelist); + } + + /* Notify end of backup */ + pg_stop_backup(¤t); + + /* Add archived xlog files into the list of files of this backup */ + if (stream_wal) + { + parray *xlog_files_list; + char pg_xlog_path[MAXPGPATH]; + + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + + for (i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + + if (S_ISREG(file->mode)) + calc_file_checksum(file); + /* Remove file path root prefix*/ + if (strstr(file->path, database_path) == file->path) + { + char *ptr = file->path; + + file->path = pstrdup(GetRelativePath(ptr, database_path)); + free(ptr); + } + } + + /* Add xlog files into the list of backed up files */ + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + } + + /* Print the list of files to backup catalog */ + pgBackupWriteFileList(¤t, backup_files_list, pgdata); + + /* Compute summary of size of regular files in the backup */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + if (S_ISDIR(file->mode)) + current.data_bytes += 4096; + + /* Count the amount of the data actually copied */ + if (S_ISREG(file->mode)) + current.data_bytes += file->write_size; + } + + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; +} + +/* + * Entry point of pg_probackup BACKUP subcommand. + */ +int +do_backup(time_t start_time) +{ + + /* PGDATA and BACKUP_MODE are always required */ + if (pgdata == NULL) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "required parameter not specified: BACKUP_MODE " + "(-b, --backup-mode)"); + + /* Create connection for PostgreSQL */ + backup_conn = pgut_connect(pgut_dbname); + pgut_atexit_push(backup_disconnect, NULL); + + current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(backup_conn)) + elog(ERROR, "Failed to retreive wal_segment_size"); +#endif + + current.compress_alg = compress_alg; + current.compress_level = compress_level; + + /* Confirm data block size and xlog block size are compatible */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + current.from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + + /* TODO fix it for remote backup*/ + if (!is_remote_backup) + current.checksum_version = get_data_checksum_version(true); + + is_checksum_enabled = pg_checksum_enable(); + + if (is_checksum_enabled) + elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + "Data block corruption will be detected"); + else + elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " + "pg_probackup have no way to detect data block corruption without them. " + "Reinitialize PGDATA with option '--data-checksums'."); + + StrNCpy(current.server_version, server_version_str, + sizeof(current.server_version)); + current.stream = stream_wal; + + is_ptrack_support = pg_ptrack_support(); + if (is_ptrack_support) + { + is_ptrack_enable = pg_ptrack_enable(); + } + + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (!is_ptrack_support) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else + { + if(!is_ptrack_enable) + elog(ERROR, "Ptrack is disabled"); + } + } + + if (current.from_replica) + { + /* Check master connection options */ + if (master_host == NULL) + elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + + /* Create connection to master server */ + master_conn = pgut_connect_extended(master_host, master_port, master_db, master_user); + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + /* TODO fix it for remote backup */ + if (!is_remote_backup) + check_system_identifiers(); + + + /* Start backup. Update backup status. */ + current.status = BACKUP_STATUS_RUNNING; + current.start_time = start_time; + + /* Create backup directory and BACKUP_CONTROL_FILE */ + if (pgBackupCreateDir(¤t)) + elog(ERROR, "cannot create backup directory"); + pgBackupWriteBackupControlFile(¤t); + + elog(LOG, "Backup destination is initialized"); + + /* set the error processing function for the backup process */ + pgut_atexit_push(backup_cleanup, NULL); + + /* backup data */ + do_backup_instance(); + pgut_atexit_pop(backup_cleanup, NULL); + + /* compute size of wal files of this backup stored in the archive */ + if (!current.stream) + { + current.wal_bytes = xlog_seg_size * + (current.stop_lsn / xlog_seg_size - + current.start_lsn / xlog_seg_size + 1); + } + + /* Backup is done. Update backup status */ + current.end_time = time(NULL); + current.status = BACKUP_STATUS_DONE; + pgBackupWriteBackupControlFile(¤t); + + //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", + // current.data_bytes); + + pgBackupValidate(¤t); + + elog(INFO, "Backup %s completed", base36enc(current.start_time)); + + /* + * After successfil backup completion remove backups + * which are expired according to retention policies + */ + if (delete_expired || delete_wal) + do_retention_purge(); + + return 0; +} + +/* + * Confirm that this server version is supported + */ +static void +check_server_version(void) +{ + PGresult *res; + + /* confirm server version */ + server_version = PQserverVersion(backup_conn); + + if (server_version == 0) + elog(ERROR, "Unknown server version %d", server_version); + + if (server_version < 100000) + sprintf(server_version_str, "%d.%d", + server_version / 10000, + (server_version / 100) % 100); + else + sprintf(server_version_str, "%d", + server_version / 10000); + + if (server_version < 90500) + elog(ERROR, + "server version is %s, must be %s or higher", + server_version_str, "9.5"); + + if (current.from_replica && server_version < 90600) + elog(ERROR, + "server version is %s, must be %s or higher for backup from replica", + server_version_str, "9.6"); + + res = pgut_execute_extended(backup_conn, "SELECT pgpro_edition()", + 0, NULL, true, true); + + /* + * Check major version of connected PostgreSQL and major version of + * compiled PostgreSQL. + */ +#ifdef PGPRO_VERSION + if (PQresultStatus(res) == PGRES_FATAL_ERROR) + /* It seems we connected to PostgreSQL (not Postgres Pro) */ + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && + strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, + server_version_str, PQgetvalue(res, 0, 0)); +#else + if (PQresultStatus(res) != PGRES_FATAL_ERROR) + /* It seems we connected to Postgres Pro (not PostgreSQL) */ + elog(ERROR, "%s was built with PostgreSQL %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, + server_version_str, PQgetvalue(res, 0, 0)); + else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, server_version_str); +#endif + + PQclear(res); + + /* Do exclusive backup only for PostgreSQL 9.5 */ + exclusive_backup = server_version < 90600 || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK; +} + +/* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + * All system identifiers must be equal. + */ +static void +check_system_identifiers(void) +{ + uint64 system_id_conn; + uint64 system_id_pgdata; + + system_id_pgdata = get_system_identifier(pgdata); + system_id_conn = get_remote_system_identifier(backup_conn); + + if (system_id_conn != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but connected instance system id is " UINT64_FORMAT, + system_identifier, system_id_conn); + if (system_id_pgdata != system_identifier) + elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT + ", but target backup directory system id is " UINT64_FORMAT, + system_identifier, system_id_pgdata); +} + +/* + * Ensure that target backup database is initialized with + * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. + */ +static void +confirm_block_size(const char *name, int blcksz) +{ + PGresult *res; + char *endp; + int block_size; + + res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); + if (PQntuples(res) != 1 || PQnfields(res) != 1) + elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); + + block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); + if ((endp && *endp) || block_size != blcksz) + elog(ERROR, + "%s(%d) is not compatible(%d expected)", + name, block_size, blcksz); + + PQclear(res); +} + +/* + * Notify start of backup to PostgreSQL server. + */ +static void +pg_start_backup(const char *label, bool smooth, pgBackup *backup) +{ + PGresult *res; + const char *params[2]; + uint32 xlogid; + uint32 xrecoff; + PGconn *conn; + + params[0] = label; + + /* For replica we call pg_start_backup() on master */ + conn = (backup->from_replica) ? master_conn : backup_conn; + + /* 2nd argument is 'fast'*/ + params[1] = smooth ? "false" : "true"; + if (!exclusive_backup) + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2, false)", + 2, + params); + else + res = pgut_execute(conn, + "SELECT pg_catalog.pg_start_backup($1, $2)", + 2, + params); + + /* + * Set flag that pg_start_backup() was called. If an error will happen it + * is necessary to call pg_stop_backup() in backup_cleanup(). + */ + backup_in_progress = true; + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + /* + * Switch to a new WAL segment. It is necessary to get archived WAL + * segment, which includes start LSN of current backup. + */ + pg_switch_wal(conn); + + if (!stream_wal) + { + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + /* In PAGE mode wait for current segment... */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + wait_wal_lsn(backup->start_lsn, false); + /* ...for others wait for previous segment */ + else + wait_wal_lsn(backup->start_lsn, true); + } + + /* Wait for start_lsn to be replayed by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(backup->start_lsn, true); +} + +/* + * Switch to a new WAL segment. It should be called only for master. + */ +static void +pg_switch_wal(PGconn *conn) +{ + PGresult *res; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); + PQclear(res); + + if (server_version >= 100000) + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL); + else + res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL); + + PQclear(res); +} + +/* + * Check if the instance supports ptrack + * TODO Maybe we should rather check ptrack_version()? + */ +static bool +pg_ptrack_support(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, + "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + + res_db = pgut_execute(backup_conn, + "SELECT pg_catalog.ptrack_version()", + 0, NULL); + if (PQntuples(res_db) == 0) + { + PQclear(res_db); + return false; + } + + /* Now we support only ptrack versions upper than 1.5 */ + if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0) + { + elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); + PQclear(res_db); + return false; + } + + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_ptrack_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show ptrack_enable", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if ptrack is enabled in target instance */ +static bool +pg_checksum_enable(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "show data_checksums", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* Check if target instance is replica */ +static bool +pg_is_in_recovery(void) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); + + if (PQgetvalue(res_db, 0, 0)[0] == 't') + { + PQclear(res_db); + return true; + } + PQclear(res_db); + return false; +} + +/* Clear ptrack files in all databases of the instance we connected to */ +static void +pg_ptrack_clear(void) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(dbname); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); +} + +static bool +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) +{ + char *params[2]; + char *dbname; + PGresult *res_db; + PGresult *res; + bool result; + + params[0] = palloc(64); + params[1] = palloc(64); + + sprintf(params[0], "%i", dbOid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return false; + + dbname = PQgetvalue(res_db, 0, 0); + + /* Always backup all files from template0 database */ + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return true; + } + PQclear(res_db); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); + + if (!parse_bool(PQgetvalue(res, 0, 0), &result)) + elog(ERROR, + "result of pg_ptrack_get_and_clear_db() is invalid: %s", + PQgetvalue(res, 0, 0)); + + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* Read and clear ptrack files of the target relation. + * Result is a bytea ptrack map of all segments of the target relation. + * case 1: we know a tablespace_oid, db_oid, and rel_filenode + * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) + * case 3: we know only rel_filenode (because file in pg_global) + */ +static char * +pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, + size_t *result_size) +{ + PGconn *tmp_conn; + PGresult *res_db, + *res; + char *params[2]; + char *result; + char *val; + + params[0] = palloc(64); + params[1] = palloc(64); + + /* regular file (not in directory 'global') */ + if (db_oid != 0) + { + char *dbname; + + sprintf(params[0], "%i", db_oid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return NULL; + + dbname = PQgetvalue(res_db, 0, 0); + + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return NULL; + } + + tmp_conn = pgut_connect(dbname); + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", + dbname, tablespace_oid, rel_filenode); + PQclear(res_db); + pgut_disconnect(tmp_conn); + } + /* file in directory 'global' */ + else + { + /* + * execute ptrack_get_and_clear for relation in pg_global + * Use backup_conn, cause we can do it from any database. + */ + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", + rel_filenode); + } + + val = PQgetvalue(res, 0, 0); + + /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. + * It should be fixed in future ptrack releases, but till then we + * can parse it. + */ + if (strcmp("x", val+1) == 0) + { + /* Ptrack file is missing */ + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* + * Wait for target 'lsn'. + * + * If current backup started in archive mode wait for 'lsn' to be archived in + * archive 'wal' directory with WAL segment file. + * If current backup started in stream mode wait for 'lsn' to be streamed in + * 'pg_wal' directory. + * + * If 'wait_prev_segment' wait for previous segment. + */ +static void +wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment) +{ + TimeLineID tli; + XLogSegNo targetSegNo; + char wal_dir[MAXPGPATH], + wal_segment_path[MAXPGPATH]; + char wal_segment[MAXFNAMELEN]; + bool file_exists = false; + uint32 try_count = 0, + timeout; + +#ifdef HAVE_LIBZ + char gz_wal_segment_path[MAXPGPATH]; +#endif + + tli = get_current_timeline(false); + + /* Compute the name of the WAL file containig requested LSN */ + GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); + if (wait_prev_segment) + targetSegNo--; + GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); + + if (stream_wal) + { + pgBackupGetPath2(¤t, wal_dir, lengthof(wal_dir), + DATABASE_DIR, PG_XLOG_DIR); + join_path_components(wal_segment_path, wal_dir, wal_segment); + + timeout = (uint32) checkpoint_timeout(); + timeout = timeout + timeout * 0.1; + } + else + { + join_path_components(wal_segment_path, arclog_path, wal_segment); + timeout = archive_timeout; + } + + if (wait_prev_segment) + elog(LOG, "Looking for segment: %s", wal_segment); + else + elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + +#ifdef HAVE_LIBZ + snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", + wal_segment_path); +#endif + + /* Wait until target LSN is archived or streamed */ + while (true) + { + if (!file_exists) + { + file_exists = fileExists(wal_segment_path); + + /* Try to find compressed WAL file */ + if (!file_exists) + { +#ifdef HAVE_LIBZ + file_exists = fileExists(gz_wal_segment_path); + if (file_exists) + elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); +#endif + } + else + elog(LOG, "Found WAL segment: %s", wal_segment_path); + } + + if (file_exists) + { + /* Do not check LSN for previous WAL segment */ + if (wait_prev_segment) + return; + + /* + * A WAL segment found. Check LSN on it. + */ + if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli, + xlog_seg_size)) || + (!stream_wal && wal_contains_lsn(arclog_path, lsn, tli, + xlog_seg_size))) + /* Target LSN was found */ + { + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return; + } + } + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for WAL archiving"); + try_count++; + + /* Inform user if WAL segment is absent in first attempt */ + if (try_count == 1) + { + if (wait_prev_segment) + elog(INFO, "Wait for WAL segment %s to be archived", + wal_segment_path); + else + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + } + + if (timeout > 0 && try_count > timeout) + { + if (file_exists) + elog(ERROR, "WAL segment %s was archived, " + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + /* If WAL segment doesn't exist or we wait for previous segment */ + else + elog(ERROR, + "Switched WAL segment %s could not be archived in %d seconds", + wal_segment, timeout); + } + } +} + +/* + * Wait for target 'lsn' on replica instance from master. + */ +static void +wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) +{ + uint32 try_count = 0; + + while (true) + { + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr replica_lsn; + + /* + * For lsn from pg_start_backup() we need it to be replayed on replica's + * data. + */ + if (is_start_backup) + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_replay_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_replay_location()", + 0, NULL); + } + /* + * For lsn from pg_stop_backup() we need it only to be received by + * replica and fsync()'ed on WAL segment. + */ + else + { + if (server_version >= 100000) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()", + 0, NULL); + else + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()", + 0, NULL); + } + + /* Extract timeline and LSN from result */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + replica_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + PQclear(res); + + /* target lsn was replicated */ + if (replica_lsn >= lsn) + break; + + sleep(1); + if (interrupted) + elog(ERROR, "Interrupted during waiting for target LSN"); + try_count++; + + /* Inform user if target lsn is absent in first attempt */ + if (try_count == 1) + elog(INFO, "Wait for target LSN %X/%X to be received by replica", + (uint32) (lsn >> 32), (uint32) lsn); + + if (replica_timeout > 0 && try_count > replica_timeout) + elog(ERROR, "Target LSN %X/%X could not be recevied by replica " + "in %d seconds", + (uint32) (lsn >> 32), (uint32) lsn, + replica_timeout); + } +} + +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(pgBackup *backup) +{ + PGconn *conn; + PGresult *res; + PGresult *tablespace_map_content = NULL; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr restore_lsn = InvalidXLogRecPtr; + int pg_stop_backup_timeout = 0; + char path[MAXPGPATH]; + char backup_label[MAXPGPATH]; + FILE *fp; + pgFile *file; + size_t len; + char *val = NULL; + char *stop_backup_query = NULL; + + /* + * We will use this values if there are no transactions between start_lsn + * and stop_lsn. + */ + time_t recovery_time; + TransactionId recovery_xid; + + if (!backup_in_progress) + elog(ERROR, "backup is not in progress"); + + /* For replica we call pg_stop_backup() on master */ + conn = (current.from_replica) ? master_conn : backup_conn; + + /* Remove annoying NOTICE messages generated by backend */ + res = pgut_execute(conn, "SET client_min_messages = warning;", + 0, NULL); + PQclear(res); + + /* Create restore point */ + if (backup != NULL) + { + const char *params[1]; + char name[1024]; + + if (!current.from_replica) + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup->start_time)); + else + snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", + base36enc(backup->start_time)); + params[0] = name; + + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); + } + + /* + * send pg_stop_backup asynchronously because we could came + * here from backup_cleanup() after some error caused by + * postgres archive_command problem and in this case we will + * wait for pg_stop_backup() forever. + */ + + if (!pg_stop_backup_is_sent) + { + bool sent = false; + + if (!exclusive_backup) + { + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + */ + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + + } + else + { + + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn"; + } + + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + pg_stop_backup_is_sent = true; + if (!sent) + elog(ERROR, "Failed to send pg_stop_backup query"); + } + + /* + * Wait for the result of pg_stop_backup(), + * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + */ + if (pg_stop_backup_is_sent && !in_cleanup) + { + while (1) + { + if (!PQconsumeInput(conn) || PQisBusy(conn)) + { + pg_stop_backup_timeout++; + sleep(1); + + if (interrupted) + { + pgut_cancel(conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } + + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); + + /* + * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + { + pgut_cancel(conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", + PG_STOP_BACKUP_TIMEOUT); + } + } + else + { + res = PQgetResult(conn); + break; + } + } + + /* Check successfull execution of pg_stop_backup() */ + if (!res) + elog(ERROR, "pg_stop backup() failed"); + else + { + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + break; + default: + elog(ERROR, "query failed: %s query was: %s", + PQerrorMessage(conn), stop_backup_query); + } + elog(INFO, "pg_stop backup() successfully executed"); + } + + backup_in_progress = false; + + /* Extract timeline and LSN from results of pg_stop_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 2), &xlogid, &xrecoff); + /* Calculate LSN */ + stop_backup_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + if (!XRecOffIsValid(stop_backup_lsn)) + { + stop_backup_lsn = restore_lsn; + } + + if (!XRecOffIsValid(stop_backup_lsn)) + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) + { + Assert(PQnfields(res) >= 4); + pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + + /* Write backup_label */ + join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); + fp = fopen(backup_label, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open backup label file \"%s\": %s", + backup_label, strerror(errno)); + + len = strlen(PQgetvalue(res, 0, 3)); + if (fwrite(PQgetvalue(res, 0, 3), 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write backup label file \"%s\": %s", + backup_label, strerror(errno)); + + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (backup_files_list) + { + file = pgFileNew(backup_label, true); + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_BACKUP_LABEL_FILE); + parray_append(backup_files_list, file); + } + } + + if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) + elog(ERROR, + "result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(res, 0, 0)); + if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) + elog(ERROR, + "result of current_timestamp is invalid: %s", + PQgetvalue(res, 0, 1)); + + /* Get content for tablespace_map from stop_backup results + * in case of non-exclusive backup + */ + if (!exclusive_backup) + val = PQgetvalue(res, 0, 4); + + /* Write tablespace_map */ + if (!exclusive_backup && val && strlen(val) > 0) + { + char tablespace_map[MAXPGPATH]; + + join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); + fp = fopen(tablespace_map, PG_BINARY_W); + if (fp == NULL) + elog(ERROR, "can't open tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + len = strlen(val); + if (fwrite(val, 1, len, fp) != len || + fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "can't write tablespace map file \"%s\": %s", + tablespace_map, strerror(errno)); + + if (backup_files_list) + { + file = pgFileNew(tablespace_map, true); + if (S_ISREG(file->mode)) + calc_file_checksum(file); + free(file->path); + file->path = strdup(PG_TABLESPACE_MAP_FILE); + parray_append(backup_files_list, file); + } + } + + if (tablespace_map_content) + PQclear(tablespace_map_content); + PQclear(res); + + if (stream_wal) + { + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); + } + } + + /* Fill in fields if that is the correct end of backup. */ + if (backup != NULL) + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + + /* Wait for stop_lsn to be received by replica */ + if (backup->from_replica) + wait_replica_wal_lsn(stop_backup_lsn, false); + /* + * Wait for stop_lsn to be archived or streamed. + * We wait for stop_lsn in stream mode just in case. + */ + wait_wal_lsn(stop_backup_lsn, false); + + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; + + backup->tli = get_current_timeline(false); + backup->stop_lsn = stop_backup_lsn; + + elog(LOG, "Getting the Recovery Time from WAL"); + + if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time, &backup->recovery_xid)) + { + backup->recovery_time = recovery_time; + backup->recovery_xid = recovery_xid; + } + } +} + +/* + * Retreive checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(void) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * Notify end of backup to server when "backup_label" is in the root directory + * of the DB cluster. + * Also update backup status to ERROR when the backup is not finished. + */ +static void +backup_cleanup(bool fatal, void *userdata) +{ + /* + * Update status of backup in BACKUP_CONTROL_FILE to ERROR. + * end_time != 0 means backup finished + */ + if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) + { + elog(WARNING, "Backup %s is running, setting its status to ERROR", + base36enc(current.start_time)); + current.end_time = time(NULL); + current.status = BACKUP_STATUS_ERROR; + pgBackupWriteBackupControlFile(¤t); + } + + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(WARNING, "backup in progress, stop backup"); + pg_stop_backup(NULL); /* don't care stop_lsn on error case */ + } +} + +/* + * Disconnect backup connection during quit pg_probackup. + */ +static void +backup_disconnect(bool fatal, void *userdata) +{ + pgut_disconnect(backup_conn); + if (master_conn) + pgut_disconnect(master_conn); +} + +/* + * Take a backup of the PGDATA at a file level. + * Copy all directories and files listed in backup_files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. + * In incremental backup mode, copy only files or datafiles' pages changed after + * previous backup. + */ +static void * +backup_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + + /* backup a file */ + for (i = 0; i < n_backup_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + elog(VERBOSE, "Copying file: \"%s\" ", file->path); + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + file->write_size = BYTES_INVALID; + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to backup \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* We have already copied all directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char *relative; + pgFile key; + pgFile **prev_file; + + relative = GetRelativePath(file->path, arguments->from_root); + key.path = relative; + + prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, + &key, pgFileComparePath); + if (prev_file) + /* File exists in previous backup */ + file->exists_in_prev = true; + } + /* copy the file into backup */ + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + + /* backup block by block if datafile AND not compressed by cfs*/ + if (!backup_data_file(arguments, to_path, file, + arguments->prev_start_lsn, + current.backup_mode, + compress_alg, compress_level)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + } + /* TODO: + * Check if file exists in previous backup + * If exists: + * if mtime > start_backup_time of parent backup, + * copy file to backup + * if mtime < start_backup_time + * calculate crc, compare crc to old file + * if crc is the same -> skip file + */ + else if (!copy_file(arguments->from_root, arguments->to_root, file)) + { + file->write_size = BYTES_INVALID; + elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + file->path, file->write_size); + } + else + elog(LOG, "unexpected file type %d", buf.st_mode); + } + + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Extract information about files in backup_list parsing their names: + * - remove temp tables from the list + * - remove unlogged tables from the list (leave the _init fork) + * - set flags for database directories + * - set flags for datafiles + */ +static void +parse_backup_filelist_filenames(parray *files, const char *root) +{ + size_t i = 0; + Oid unlogged_file_reloid = 0; + + while (i < parray_num(files)) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *relative; + int sscanf_result; + + relative = GetRelativePath(file->path, root); + + if (S_ISREG(file->mode) && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) + { + /* + * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + * Legal only in case of 'pg_compression' + */ + if (strcmp(file->name, "pg_compression") == 0) + { + Oid tblspcOid; + Oid dbOid; + char tmp_rel_path[MAXPGPATH]; + /* + * Check that the file is located under + * TABLESPACE_VERSION_DIRECTORY + */ + sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u", + &tblspcOid, tmp_rel_path, &dbOid); + + /* Yes, it is */ + if (sscanf_result == 2 && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + set_cfs_datafiles(files, root, relative, i); + } + } + + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->forkName, "init") == 0) + { + /* + * Do not backup files of unlogged relations. + * scan filelist backward and exclude these files. + */ + int unlogged_file_num = i - 1; + pgFile *unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + + unlogged_file_reloid = file->relOid; + + while (unlogged_file_num >= 0 && + (unlogged_file_reloid != 0) && + (unlogged_file->relOid == unlogged_file_reloid)) + { + pgFileFree(unlogged_file); + parray_remove(files, unlogged_file_num); + + unlogged_file_num--; + i--; + + unlogged_file = (pgFile *) parray_get(files, + unlogged_file_num); + } + } + } + + i++; + } +} + +/* If file is equal to pg_compression, then we consider this tablespace as + * cfs-compressed and should mark every file in this tablespace as cfs-file + * Setting is_cfs is done via going back through 'files' set every file + * that contain cfs_tablespace in his path as 'is_cfs' + * Goings back through array 'files' is valid option possible because of current + * sort rules: + * tblspcOid/TABLESPACE_VERSION_DIRECTORY + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm + * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + */ +static void +set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +{ + int len; + int p; + pgFile *prev_file; + char *cfs_tblspc_path; + char *relative_prev_file; + + cfs_tblspc_path = strdup(relative); + if(!cfs_tblspc_path) + elog(ERROR, "Out of memory"); + len = strlen("/pg_compression"); + cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; + elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + + for (p = (int) i; p >= 0; p--) + { + prev_file = (pgFile *) parray_get(files, (size_t) p); + relative_prev_file = GetRelativePath(prev_file->path, root); + + elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file); + + if (strstr(relative_prev_file, cfs_tblspc_path) != NULL) + { + if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + { + elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", + relative_prev_file, prev_file->name); + prev_file->is_cfs = true; + } + } + else + { + elog(VERBOSE, "Breaking on %s", relative_prev_file); + break; + } + } + free(cfs_tblspc_path); +} + +/* + * Find pgfile by given rnode in the backup_files_list + * and add given blkno to its pagemap. + */ +void +process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) +{ + char *path; + char *rel_path; + BlockNumber blkno_inseg; + int segno; + pgFile **file_item; + pgFile f; + + segno = blkno / RELSEG_SIZE; + blkno_inseg = blkno % RELSEG_SIZE; + + rel_path = relpathperm(rnode, forknum); + if (segno > 0) + path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + else + path = psprintf("%s/%s", pgdata, rel_path); + + pg_free(rel_path); + + f.path = path; + /* backup_files_list should be sorted before */ + file_item = (pgFile **) parray_bsearch(backup_files_list, &f, + pgFileComparePath); + + /* + * If we don't have any record of this file in the file map, it means + * that it's a relation that did not have much activity since the last + * backup. We can safely ignore it. If it is a new relation file, the + * backup would simply copy it as-is. + */ + if (file_item) + { + /* We need critical section only we use more than one threads */ + if (num_threads > 1) + pthread_lock(&backup_pagemap_mutex); + + datapagemap_add(&(*file_item)->pagemap, blkno_inseg); + + if (num_threads > 1) + pthread_mutex_unlock(&backup_pagemap_mutex); + } + + pg_free(path); +} + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * NOTE we rely on the fact that provided parray is sorted by file->path. + */ +static void +make_pagemap_from_ptrack(parray *files) +{ + size_t i; + Oid dbOid_with_ptrack_init = 0; + Oid tblspcOid_with_ptrack_init = 0; + char *ptrack_nonparsed = NULL; + size_t ptrack_nonparsed_size = 0; + + elog(LOG, "Compiling pagemap"); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + size_t start_addr; + + /* + * If there is a ptrack_init file in the database, + * we must backup all its files, ignoring ptrack files for relations. + */ + if (file->is_database) + { + char *filename = strrchr(file->path, '/'); + + Assert(filename != NULL); + filename++; + + /* + * The function pg_ptrack_get_and_clear_db returns true + * if there was a ptrack_init file. + * Also ignore ptrack files for global tablespace, + * to avoid any possible specific errors. + */ + if ((file->tblspcOid == GLOBALTABLESPACE_OID) || + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid)) + { + dbOid_with_ptrack_init = file->dbOid; + tblspcOid_with_ptrack_init = file->tblspcOid; + } + } + + if (file->is_datafile) + { + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) + { + /* ignore ptrack if ptrack_init exists */ + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + file->pagemap_isabsent = true; + continue; + } + + /* get ptrack bitmap once for all segments of the file */ + if (file->segno == 0) + { + /* release previous value */ + pg_free(ptrack_nonparsed); + ptrack_nonparsed_size = 0; + + ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, + file->relOid, &ptrack_nonparsed_size); + } + + if (ptrack_nonparsed != NULL) + { + /* + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cutted out. + * Compute the beginning of the ptrack map related to this segment + * + * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 + * RELSEG_SIZE. Number of Pages per segment: 131072 + * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed + * to keep track on one relsegment: 16384 + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; + + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) + { + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + else + { + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } + } + else + { + /* + * If ptrack file is missing, try to copy the entire file. + * It can happen in two cases: + * - files were created by commands that bypass buffer manager + * and, correspondingly, ptrack mechanism. + * i.e. CREATE DATABASE + * - target relation was deleted. + */ + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + } + } + elog(LOG, "Pagemap compiled"); +// res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL, true); +// PQclear(pgut_execute(backup_conn, "CHECKPOINT;", 0, NULL, true)); +} + + +/* + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos > stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_timeout == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_timeout = checkpoint_timeout(); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + XLogRecPtr startpos; + TimeLineID starttli; + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * We must use startpos as start_lsn from start_backup + */ + startpos = current.start_lsn; + starttli = current.tli; + + /* + * Always start streaming at the beginning of a segment + */ + startpos -= startpos % xlog_seg_size; + + /* Initialize timeout */ + stream_stop_timeout = 0; + stream_stop_begin = 0; + + /* + * Start the replication + */ + elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), + (uint32) (startpos >> 32), (uint32) startpos, starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = startpos; + ctl.timeline = starttli; + ctl.sysidentifier = NULL; + +#if PG_VERSION_NUM >= 100000 + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; +#else + ctl.basedir = (char *) stream_arg->basedir; +#endif + + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + + if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + elog(ERROR, "Problem in receivexlog"); + +#if PG_VERSION_NUM >= 100000 + if (!ctl.walmethod->finish()) + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); +#endif + } +#else + if(ReceiveXlogStream(stream_arg->conn, startpos, starttli, NULL, + (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + elog(ERROR, "Problem in receivexlog"); +#endif + + elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ +static XLogRecPtr +get_last_ptrack_lsn(void) + +{ + PGresult *res; + uint32 xlogid; + uint32 xrecoff; + XLogRecPtr lsn; + + res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", 0, NULL); + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff); + /* Calculate LSN */ + lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + + PQclear(res); + return lsn; +} + +char * +pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, + Oid tblsOid, + Oid relOid, + BlockNumber blknum, + size_t *result_size) +{ + PGresult *res; + char *params[4]; + char *result; + + params[0] = palloc(64); + params[1] = palloc(64); + params[2] = palloc(64); + params[3] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", tblsOid); + sprintf(params[1], "%i", dbOid); + sprintf(params[2], "%i", relOid); + sprintf(params[3], "%u", blknum); + + if (arguments->backup_conn == NULL) + { + arguments->backup_conn = pgut_connect(pgut_dbname); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", + 4, (const char **)params, true); + + if (PQnfields(res) != 1) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + if (PQgetisnull(res, 0, 0)) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + + PQclear(res); + + pfree(params[0]); + pfree(params[1]); + pfree(params[2]); + pfree(params[3]); + + return result; +} diff --git a/src/catalog.c b/src/catalog.c new file mode 100644 index 000000000..f3f752779 --- /dev/null +++ b/src/catalog.c @@ -0,0 +1,915 @@ +/*------------------------------------------------------------------------- + * + * catalog.c: backup catalog operation + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; +static pgBackup *readBackupControlFile(const char *path); + +static bool exit_hook_registered = false; +static char lock_file[MAXPGPATH]; + +static void +unlink_lock_atexit(void) +{ + int res; + res = unlink(lock_file); + if (res != 0 && res != ENOENT) + elog(WARNING, "%s: %s", lock_file, strerror(errno)); +} + +/* + * Create a lockfile. + */ +void +catalog_lock(void) +{ + int fd; + char buffer[MAXPGPATH * 2 + 256]; + int ntries; + int len; + int encoded_pid; + pid_t my_pid, + my_p_pid; + + join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); + + /* + * If the PID in the lockfile is our own PID or our parent's or + * grandparent's PID, then the file must be stale (probably left over from + * a previous system boot cycle). We need to check this because of the + * likelihood that a reboot will assign exactly the same PID as we had in + * the previous reboot, or one that's only one or two counts larger and + * hence the lockfile's PID now refers to an ancestor shell process. We + * allow pg_ctl to pass down its parent shell PID (our grandparent PID) + * via the environment variable PG_GRANDPARENT_PID; this is so that + * launching the postmaster via pg_ctl can be just as reliable as + * launching it directly. There is no provision for detecting + * further-removed ancestor processes, but if the init script is written + * carefully then all but the immediate parent shell will be root-owned + * processes and so the kill test will fail with EPERM. Note that we + * cannot get a false negative this way, because an existing postmaster + * would surely never launch a competing postmaster or pg_ctl process + * directly. + */ + my_pid = getpid(); +#ifndef WIN32 + my_p_pid = getppid(); +#else + + /* + * Windows hasn't got getppid(), but doesn't need it since it's not using + * real kill() either... + */ + my_p_pid = 0; +#endif + + /* + * We need a loop here because of race conditions. But don't loop forever + * (for example, a non-writable $backup_instance_path directory might cause a failure + * that won't go away). 100 tries seems like plenty. + */ + for (ntries = 0;; ntries++) + { + /* + * Try to create the lock file --- O_EXCL makes this atomic. + * + * Think not to make the file protection weaker than 0600. See + * comments below. + */ + fd = open(lock_file, O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) + break; /* Success; exit the retry loop */ + + /* + * Couldn't create the pid file. Probably it already exists. + */ + if ((errno != EEXIST && errno != EACCES) || ntries > 100) + elog(ERROR, "could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + + /* + * Read the file to get the old owner's PID. Note race condition + * here: file might have been deleted since we tried to create it. + */ + fd = open(lock_file, O_RDONLY, 0600); + if (fd < 0) + { + if (errno == ENOENT) + continue; /* race condition; try again */ + elog(ERROR, "could not open lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) + elog(ERROR, "could not read lock file \"%s\": %s", + lock_file, strerror(errno)); + close(fd); + + if (len == 0) + elog(ERROR, "lock file \"%s\" is empty", lock_file); + + buffer[len] = '\0'; + encoded_pid = atoi(buffer); + + if (encoded_pid <= 0) + elog(ERROR, "bogus data in lock file \"%s\": \"%s\"", + lock_file, buffer); + + /* + * Check to see if the other process still exists + * + * Per discussion above, my_pid, my_p_pid can be + * ignored as false matches. + * + * Normally kill() will fail with ESRCH if the given PID doesn't + * exist. + */ + if (encoded_pid != my_pid && encoded_pid != my_p_pid) + { + if (kill(encoded_pid, 0) == 0 || + (errno != ESRCH && errno != EPERM)) + elog(ERROR, "lock file \"%s\" already exists", lock_file); + } + + /* + * Looks like nobody's home. Unlink the file and try again to create + * it. Need a loop because of possible race condition against other + * would-be creators. + */ + if (unlink(lock_file) < 0) + elog(ERROR, "could not remove old lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Successfully created the file, now fill it. + */ + snprintf(buffer, sizeof(buffer), "%d\n", my_pid); + + errno = 0; + if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + /* if write didn't set errno, assume problem is no disk space */ + errno = save_errno ? save_errno : ENOSPC; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (fsync(fd) != 0) + { + int save_errno = errno; + + close(fd); + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + if (close(fd) != 0) + { + int save_errno = errno; + + unlink(lock_file); + errno = save_errno; + elog(ERROR, "could not write lock file \"%s\": %s", + lock_file, strerror(errno)); + } + + /* + * Arrange to unlink the lock file(s) at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(unlink_lock_atexit); + exit_hook_registered = true; + } +} + +/* + * Read backup meta information from BACKUP_CONTROL_FILE. + * If no backup matches, return NULL. + */ +pgBackup * +read_backup(time_t timestamp) +{ + pgBackup tmp; + char conf_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + + return readBackupControlFile(conf_path); +} + +/* + * Get backup_mode in string representation. + */ +const char * +pgBackupGetBackupMode(pgBackup *backup) +{ + return backupModes[backup->backup_mode]; +} + +static bool +IsDir(const char *dirpath, const char *entry) +{ + char path[MAXPGPATH]; + struct stat st; + + snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); +} + +/* + * Create list of backups. + * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. + * The list is sorted in order of descending start time. + * If valid backup id is passed only matching backup will be added to the list. + */ +parray * +catalog_get_backup_list(time_t requested_backup_id) +{ + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + pgBackup *backup = NULL; + int i; + + /* open backup instance backups directory */ + data_dir = opendir(backup_instance_path); + if (data_dir == NULL) + { + elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + strerror(errno)); + goto err_proc; + } + + /* scan the directory and list backups */ + backups = parray_new(); + for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) + { + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + + /* skip not-directory entries and hidden entries */ + if (!IsDir(backup_instance_path, data_ent->d_name) + || data_ent->d_name[0] == '.') + continue; + + /* open subdirectory of specific backup */ + join_path_components(data_path, backup_instance_path, data_ent->d_name); + + /* read backup information from BACKUP_CONTROL_FILE */ + snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + backup = readBackupControlFile(backup_conf_path); + + /* ignore corrupted backups */ + if (backup) + { + backup->backup_id = backup->start_time; + + if (requested_backup_id != INVALID_BACKUP_ID + && requested_backup_id != backup->start_time) + { + pgBackupFree(backup); + continue; + } + parray_append(backups, backup); + backup = NULL; + } + + if (errno && errno != ENOENT) + { + elog(WARNING, "cannot read data directory \"%s\": %s", + data_ent->d_name, strerror(errno)); + goto err_proc; + } + } + if (errno) + { + elog(WARNING, "cannot read backup root directory \"%s\": %s", + backup_instance_path, strerror(errno)); + goto err_proc; + } + + closedir(data_dir); + data_dir = NULL; + + parray_qsort(backups, pgBackupCompareIdDesc); + + /* Link incremental backups with their ancestors.*/ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *curr = parray_get(backups, i); + + int j; + + if (curr->backup_mode == BACKUP_MODE_FULL) + continue; + + for (j = i+1; j < parray_num(backups); j++) + { + pgBackup *ancestor = parray_get(backups, j); + + if (ancestor->start_time == curr->parent_backup) + { + curr->parent_backup_link = ancestor; + /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), + base36enc_dup(ancestor->start_time), j); */ + break; + } + } + } + + return backups; + +err_proc: + if (data_dir) + closedir(data_dir); + if (backup) + pgBackupFree(backup); + if (backups) + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(ERROR, "Failed to get backup list"); + + return NULL; +} + +/* + * Find the last completed backup on given timeline + */ +pgBackup * +catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) +{ + int i; + pgBackup *backup = NULL; + + /* backup_list is sorted in order of descending ID */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK && backup->tli == tli) + return backup; + } + + return NULL; +} + +/* create backup directory in $BACKUP_PATH */ +int +pgBackupCreateDir(pgBackup *backup) +{ + int i; + char path[MAXPGPATH]; + char *subdirs[] = { DATABASE_DIR, NULL }; + + pgBackupGetPath(backup, path, lengthof(path), NULL); + + if (!dir_is_empty(path)) + elog(ERROR, "backup destination is not empty \"%s\"", path); + + dir_create_dir(path, DIR_PERMISSION); + + /* create directories for actual backup files */ + for (i = 0; subdirs[i]; i++) + { + pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + dir_create_dir(path, DIR_PERMISSION); + } + + return 0; +} + +/* + * Write information about backup.in to stream "out". + */ +void +pgBackupWriteControl(FILE *out, pgBackup *backup) +{ + char timestamp[100]; + + fprintf(out, "#Configuration\n"); + fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fprintf(out, "compress-alg = %s\n", + deparse_compress_alg(backup->compress_alg)); + fprintf(out, "compress-level = %d\n", backup->compress_level); + fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + + fprintf(out, "\n#Compatibility\n"); + fprintf(out, "block-size = %u\n", backup->block_size); + fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fprintf(out, "program-version = %s\n", PROGRAM_VERSION); + if (backup->server_version[0] != '\0') + fprintf(out, "server-version = %s\n", backup->server_version); + + fprintf(out, "\n#Result backup info\n"); + fprintf(out, "timelineid = %d\n", backup->tli); + /* LSN returned by pg_start_backup */ + fprintf(out, "start-lsn = %X/%X\n", + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn); + /* LSN returned by pg_stop_backup */ + fprintf(out, "stop-lsn = %X/%X\n", + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + fprintf(out, "start-time = '%s'\n", timestamp); + if (backup->end_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + fprintf(out, "end-time = '%s'\n", timestamp); + } + fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + fprintf(out, "recovery-time = '%s'\n", timestamp); + } + + /* + * Size of PGDATA directory. The size does not include size of related + * WAL segments in archive 'wal' directory. + */ + if (backup->data_bytes != BYTES_INVALID) + fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + + if (backup->wal_bytes != BYTES_INVALID) + fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + + fprintf(out, "status = %s\n", status2str(backup->status)); + + /* 'parent_backup' is set if it is incremental backup */ + if (backup->parent_backup != 0) + fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + + /* print connection info except password */ + if (backup->primary_conninfo) + fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); +} + +/* create BACKUP_CONTROL_FILE */ +void +pgBackupWriteBackupControlFile(pgBackup *backup) +{ + FILE *fp = NULL; + char ini_path[MAXPGPATH]; + + pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONTROL_FILE); + fp = fopen(ini_path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path, + strerror(errno)); + + pgBackupWriteControl(fp, backup); + + fclose(fp); +} + +/* + * Output the list of files to backup catalog DATABASE_FILE_LIST + */ +void +pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +{ + FILE *fp; + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_file_list(fp, files, root); + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); +} + +/* + * Read BACKUP_CONTROL_FILE and create pgBackup. + * - Comment starts with ';'. + * - Do not care section. + */ +static pgBackup * +readBackupControlFile(const char *path) +{ + pgBackup *backup = pgut_new(pgBackup); + char *backup_mode = NULL; + char *start_lsn = NULL; + char *stop_lsn = NULL; + char *status = NULL; + char *parent_backup = NULL; + char *program_version = NULL; + char *server_version = NULL; + char *compress_alg = NULL; + int parsed_options; + + pgut_option options[] = + { + {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, + {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, + {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, + {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, + {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, + {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, + {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, + {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, + {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, + {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, + {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, + {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, + {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, + {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, + {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, + {'s', 0, "status", &status, SOURCE_FILE_STRICT}, + {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, + {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, + {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, + {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {0} + }; + + if (access(path, F_OK) != 0) + { + elog(WARNING, "Control file \"%s\" doesn't exist", path); + pgBackupFree(backup); + return NULL; + } + + pgBackupInit(backup); + parsed_options = pgut_readopt(path, options, WARNING, true); + + if (parsed_options == 0) + { + elog(WARNING, "Control file \"%s\" is empty", path); + pgBackupFree(backup); + return NULL; + } + + if (backup->start_time == 0) + { + elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); + pgBackupFree(backup); + return NULL; + } + + if (backup_mode) + { + backup->backup_mode = parse_backup_mode(backup_mode); + free(backup_mode); + } + + if (start_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); + free(start_lsn); + } + + if (stop_lsn) + { + uint32 xlogid; + uint32 xrecoff; + + if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) + backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); + free(stop_lsn); + } + + if (status) + { + if (strcmp(status, "OK") == 0) + backup->status = BACKUP_STATUS_OK; + else if (strcmp(status, "ERROR") == 0) + backup->status = BACKUP_STATUS_ERROR; + else if (strcmp(status, "RUNNING") == 0) + backup->status = BACKUP_STATUS_RUNNING; + else if (strcmp(status, "MERGING") == 0) + backup->status = BACKUP_STATUS_MERGING; + else if (strcmp(status, "DELETING") == 0) + backup->status = BACKUP_STATUS_DELETING; + else if (strcmp(status, "DELETED") == 0) + backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DONE") == 0) + backup->status = BACKUP_STATUS_DONE; + else if (strcmp(status, "ORPHAN") == 0) + backup->status = BACKUP_STATUS_ORPHAN; + else if (strcmp(status, "CORRUPT") == 0) + backup->status = BACKUP_STATUS_CORRUPT; + else + elog(WARNING, "Invalid STATUS \"%s\"", status); + free(status); + } + + if (parent_backup) + { + backup->parent_backup = base36dec(parent_backup); + free(parent_backup); + } + + if (program_version) + { + StrNCpy(backup->program_version, program_version, + sizeof(backup->program_version)); + pfree(program_version); + } + + if (server_version) + { + StrNCpy(backup->server_version, server_version, + sizeof(backup->server_version)); + pfree(server_version); + } + + if (compress_alg) + backup->compress_alg = parse_compress_alg(compress_alg); + + return backup; +} + +BackupMode +parse_backup_mode(const char *value) +{ + const char *v = value; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0 && pg_strncasecmp("full", v, len) == 0) + return BACKUP_MODE_FULL; + else if (len > 0 && pg_strncasecmp("page", v, len) == 0) + return BACKUP_MODE_DIFF_PAGE; + else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0) + return BACKUP_MODE_DIFF_PTRACK; + else if (len > 0 && pg_strncasecmp("delta", v, len) == 0) + return BACKUP_MODE_DIFF_DELTA; + + /* Backup mode is invalid, so leave with an error */ + elog(ERROR, "invalid backup-mode \"%s\"", value); + return BACKUP_MODE_INVALID; +} + +const char * +deparse_backup_mode(BackupMode mode) +{ + switch (mode) + { + case BACKUP_MODE_FULL: + return "full"; + case BACKUP_MODE_DIFF_PAGE: + return "page"; + case BACKUP_MODE_DIFF_PTRACK: + return "ptrack"; + case BACKUP_MODE_DIFF_DELTA: + return "delta"; + case BACKUP_MODE_INVALID: + return "invalid"; + } + + return NULL; +} + +CompressAlg +parse_compress_alg(const char *arg) +{ + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*arg)) + arg++; + len = strlen(arg); + + if (len == 0) + elog(ERROR, "compress algrorithm is empty"); + + if (pg_strncasecmp("zlib", arg, len) == 0) + return ZLIB_COMPRESS; + else if (pg_strncasecmp("pglz", arg, len) == 0) + return PGLZ_COMPRESS; + else if (pg_strncasecmp("none", arg, len) == 0) + return NONE_COMPRESS; + else + elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + + return NOT_DEFINED_COMPRESS; +} + +const char* +deparse_compress_alg(int alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return "none"; + case ZLIB_COMPRESS: + return "zlib"; + case PGLZ_COMPRESS: + return "pglz"; + } + + return NULL; +} + +/* + * Fill pgBackup struct with default values. + */ +void +pgBackupInit(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + backup->backup_mode = BACKUP_MODE_INVALID; + backup->status = BACKUP_STATUS_INVALID; + backup->tli = 0; + backup->start_lsn = 0; + backup->stop_lsn = 0; + backup->start_time = (time_t) 0; + backup->end_time = (time_t) 0; + backup->recovery_xid = 0; + backup->recovery_time = (time_t) 0; + + backup->data_bytes = BYTES_INVALID; + backup->wal_bytes = BYTES_INVALID; + + backup->compress_alg = COMPRESS_ALG_DEFAULT; + backup->compress_level = COMPRESS_LEVEL_DEFAULT; + + backup->block_size = BLCKSZ; + backup->wal_block_size = XLOG_BLCKSZ; + backup->checksum_version = 0; + + backup->stream = false; + backup->from_replica = false; + backup->parent_backup = INVALID_BACKUP_ID; + backup->parent_backup_link = NULL; + backup->primary_conninfo = NULL; + backup->program_version[0] = '\0'; + backup->server_version[0] = '\0'; +} + +/* + * Copy backup metadata from **src** into **dst**. + */ +void +pgBackupCopy(pgBackup *dst, pgBackup *src) +{ + pfree(dst->primary_conninfo); + + memcpy(dst, src, sizeof(pgBackup)); + + if (src->primary_conninfo) + dst->primary_conninfo = pstrdup(src->primary_conninfo); +} + +/* free pgBackup object */ +void +pgBackupFree(void *backup) +{ + pgBackup *b = (pgBackup *) backup; + + pfree(b->primary_conninfo); + pfree(backup); +} + +/* Compare two pgBackup with their IDs (start time) in ascending order */ +int +pgBackupCompareId(const void *l, const void *r) +{ + pgBackup *lp = *(pgBackup **)l; + pgBackup *rp = *(pgBackup **)r; + + if (lp->start_time > rp->start_time) + return 1; + else if (lp->start_time < rp->start_time) + return -1; + else + return 0; +} + +/* Compare two pgBackup with their IDs in descending order */ +int +pgBackupCompareIdDesc(const void *l, const void *r) +{ + return -pgBackupCompareId(l, r); +} + +/* + * Construct absolute path of the backup directory. + * If subdir is not NULL, it will be appended after the path. + */ +void +pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) +{ + pgBackupGetPath2(backup, path, len, subdir, NULL); +} + +/* + * Construct absolute path of the backup directory. + * Append "subdir1" and "subdir2" to the backup directory. + */ +void +pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) +{ + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, + base36enc(backup->start_time)); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1); + /* "subdir1" and "subdir2" is not NULL */ + else + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1, subdir2); + + make_native_path(path); +} + +/* Find parent base FULL backup for current backup using parent_backup_link, + * return NULL if not found + */ +pgBackup* +find_parent_backup(pgBackup *current_backup) +{ + pgBackup *base_full_backup = NULL; + base_full_backup = current_backup; + + while (base_full_backup->backup_mode != BACKUP_MODE_FULL) + { + /* + * If we haven't found parent for incremental backup, + * mark it and all depending backups as orphaned + */ + if (base_full_backup->parent_backup_link == NULL + || (base_full_backup->status != BACKUP_STATUS_OK + && base_full_backup->status != BACKUP_STATUS_DONE)) + { + pgBackup *orphaned_backup = current_backup; + + while (orphaned_backup != NULL) + { + orphaned_backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(orphaned_backup); + if (base_full_backup->parent_backup_link == NULL) + elog(WARNING, "Backup %s is orphaned because its parent backup is not found", + base36enc(orphaned_backup->start_time)); + else + elog(WARNING, "Backup %s is orphaned because its parent backup is corrupted", + base36enc(orphaned_backup->start_time)); + + orphaned_backup = orphaned_backup->parent_backup_link; + } + + base_full_backup = NULL; + break; + } + + base_full_backup = base_full_backup->parent_backup_link; + } + + return base_full_backup; +} diff --git a/src/configure.c b/src/configure.c new file mode 100644 index 000000000..8b86e438c --- /dev/null +++ b/src/configure.c @@ -0,0 +1,490 @@ +/*------------------------------------------------------------------------- + * + * configure.c: - manage backup catalog. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "utils/logger.h" + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); + +static void show_configure_start(void); +static void show_configure_end(void); +static void show_configure(pgBackupConfig *config); + +static void show_configure_json(pgBackupConfig *config); + +static pgBackupConfig *cur_config = NULL; + +static PQExpBufferData show_buf; +static int32 json_level = 0; + +/* + * All this code needs refactoring. + */ + +/* Set configure options */ +int +do_configure(bool show_only) +{ + pgBackupConfig *config = readBackupCatalogConfigFile(); + if (pgdata) + config->pgdata = pgdata; + if (pgut_dbname) + config->pgdatabase = pgut_dbname; + if (host) + config->pghost = host; + if (port) + config->pgport = port; + if (username) + config->pguser = username; + + if (master_host) + config->master_host = master_host; + if (master_port) + config->master_port = master_port; + if (master_db) + config->master_db = master_db; + if (master_user) + config->master_user = master_user; + + if (replica_timeout) + config->replica_timeout = replica_timeout; + + if (archive_timeout) + config->archive_timeout = archive_timeout; + + if (log_level_console) + config->log_level_console = log_level_console; + if (log_level_file) + config->log_level_file = log_level_file; + if (log_filename) + config->log_filename = log_filename; + if (error_log_filename) + config->error_log_filename = error_log_filename; + if (log_directory) + config->log_directory = log_directory; + if (log_rotation_size) + config->log_rotation_size = log_rotation_size; + if (log_rotation_age) + config->log_rotation_age = log_rotation_age; + + if (retention_redundancy) + config->retention_redundancy = retention_redundancy; + if (retention_window) + config->retention_window = retention_window; + + if (compress_alg) + config->compress_alg = compress_alg; + if (compress_level) + config->compress_level = compress_level; + + if (show_only) + show_configure(config); + else + writeBackupCatalogConfigFile(config); + + return 0; +} + +void +pgBackupConfigInit(pgBackupConfig *config) +{ + config->system_identifier = 0; + +#if PG_VERSION_NUM >= 110000 + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; +#endif + + config->pgdata = NULL; + config->pgdatabase = NULL; + config->pghost = NULL; + config->pgport = NULL; + config->pguser = NULL; + + config->master_host = NULL; + config->master_port = NULL; + config->master_db = NULL; + config->master_user = NULL; + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; + + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; + + config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; + config->log_level_file = LOG_LEVEL_FILE_DEFAULT; + config->log_filename = LOG_FILENAME_DEFAULT; + config->error_log_filename = NULL; + config->log_directory = LOG_DIRECTORY_DEFAULT; + config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; + config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; + + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; + + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; +} + +void +writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +{ + uint64 res; + const char *unit; + + fprintf(out, "#Backup instance info\n"); + fprintf(out, "PGDATA = %s\n", config->pgdata); + fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); +#if PG_VERSION_NUM >= 110000 + fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); +#endif + + fprintf(out, "#Connection parameters:\n"); + if (config->pgdatabase) + fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); + if (config->pghost) + fprintf(out, "PGHOST = %s\n", config->pghost); + if (config->pgport) + fprintf(out, "PGPORT = %s\n", config->pgport); + if (config->pguser) + fprintf(out, "PGUSER = %s\n", config->pguser); + + fprintf(out, "#Replica parameters:\n"); + if (config->master_host) + fprintf(out, "master-host = %s\n", config->master_host); + if (config->master_port) + fprintf(out, "master-port = %s\n", config->master_port); + if (config->master_db) + fprintf(out, "master-db = %s\n", config->master_db); + if (config->master_user) + fprintf(out, "master-user = %s\n", config->master_user); + + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Archive parameters:\n"); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); + + fprintf(out, "#Logging parameters:\n"); + fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); + fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); + fprintf(out, "log-filename = %s\n", config->log_filename); + if (config->error_log_filename) + fprintf(out, "error-log-filename = %s\n", config->error_log_filename); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); + else + fprintf(out, "log-directory = %s\n", config->log_directory); + /* Convert values from base unit */ + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); + + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); + + fprintf(out, "#Retention parameters:\n"); + fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); + fprintf(out, "retention-window = %u\n", config->retention_window); + + fprintf(out, "#Compression parameters:\n"); + + fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); + fprintf(out, "compress-level = %d\n", config->compress_level); +} + +void +writeBackupCatalogConfigFile(pgBackupConfig *config) +{ + char path[MAXPGPATH]; + FILE *fp; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot create %s: %s", + BACKUP_CATALOG_CONF_FILE, strerror(errno)); + + writeBackupCatalogConfig(fp, config); + fclose(fp); +} + + +pgBackupConfig* +readBackupCatalogConfigFile(void) +{ + pgBackupConfig *config = pgut_new(pgBackupConfig); + char path[MAXPGPATH]; + + pgut_option options[] = + { + /* retention options */ + { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, + { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, + /* compression options */ + { 'f', 0, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 0, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, + /* logging options */ + { 'f', 0, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 0, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 0, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, + { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, + { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, + { 'u', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* connection options */ + { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, + { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, + { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, + { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, + { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, + /* replica options */ + { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, + { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, + { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, + { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* other options */ + { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, +#if PG_VERSION_NUM >= 110000 + {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive options */ + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + {0} + }; + + cur_config = config; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + + pgBackupConfigInit(config); + pgut_readopt(path, options, ERROR, true); + +#if PG_VERSION_NUM >= 110000 + if (!IsValidWalSegSize(config->xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); +#endif + + return config; +} + +/* + * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. + */ +uint32 +get_config_xlog_seg_size(void) +{ +#if PG_VERSION_NUM >= 110000 + char path[MAXPGPATH]; + uint32 seg_size; + pgut_option options[] = + { + {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, + {0} + }; + + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, false); + + if (!IsValidWalSegSize(seg_size)) + elog(ERROR, "Invalid WAL segment size %u", seg_size); + + return seg_size; + +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + cur_config->log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + cur_config->log_level_file = parse_log_level(arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + cur_config->compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize configure visualization. + */ +static void +show_configure_start(void) +{ + if (show_format == SHOW_PLAIN) + return; + + /* For now we need buffer only for JSON format */ + json_level = 0; + initPQExpBuffer(&show_buf); +} + +/* + * Finalize configure visualization. + */ +static void +show_configure_end(void) +{ + if (show_format == SHOW_PLAIN) + return; + else + appendPQExpBufferChar(&show_buf, '\n'); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show configure information of pg_probackup. + */ +static void +show_configure(pgBackupConfig *config) +{ + show_configure_start(); + + if (show_format == SHOW_PLAIN) + writeBackupCatalogConfig(stdout, config); + else + show_configure_json(config); + + show_configure_end(); +} + +/* + * Json output. + */ + +static void +show_configure_json(pgBackupConfig *config) +{ + PQExpBuffer buf = &show_buf; + uint64 res; + const char *unit; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "pgdata", config->pgdata, json_level, false); + + json_add_key(buf, "system-identifier", json_level, true); + appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); + +#if PG_VERSION_NUM >= 110000 + json_add_key(buf, "xlog-seg-size", json_level, true); + appendPQExpBuffer(buf, "%u", config->xlog_seg_size); +#endif + + /* Connection parameters */ + if (config->pgdatabase) + json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); + if (config->pghost) + json_add_value(buf, "pghost", config->pghost, json_level, true); + if (config->pgport) + json_add_value(buf, "pgport", config->pgport, json_level, true); + if (config->pguser) + json_add_value(buf, "pguser", config->pguser, json_level, true); + + /* Replica parameters */ + if (config->master_host) + json_add_value(buf, "master-host", config->master_host, json_level, + true); + if (config->master_port) + json_add_value(buf, "master-port", config->master_port, json_level, + true); + if (config->master_db) + json_add_value(buf, "master-db", config->master_db, json_level, true); + if (config->master_user) + json_add_value(buf, "master-user", config->master_user, json_level, + true); + + json_add_key(buf, "replica-timeout", json_level, true); + convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Archive parameters */ + json_add_key(buf, "archive-timeout", json_level, true); + convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); + + /* Logging parameters */ + json_add_value(buf, "log-level-console", + deparse_log_level(config->log_level_console), json_level, + true); + json_add_value(buf, "log-level-file", + deparse_log_level(config->log_level_file), json_level, + true); + json_add_value(buf, "log-filename", config->log_filename, json_level, + true); + if (config->error_log_filename) + json_add_value(buf, "error-log-filename", config->error_log_filename, + json_level, true); + + if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) + { + char log_directory_fullpath[MAXPGPATH]; + + sprintf(log_directory_fullpath, "%s/%s", + backup_path, config->log_directory); + + json_add_value(buf, "log-directory", log_directory_fullpath, + json_level, true); + } + else + json_add_value(buf, "log-directory", config->log_directory, + json_level, true); + + json_add_key(buf, "log-rotation-size", json_level, true); + convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); + + json_add_key(buf, "log-rotation-age", json_level, true); + convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_S, + &res, &unit); + appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); + + /* Retention parameters */ + json_add_key(buf, "retention-redundancy", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_redundancy); + + json_add_key(buf, "retention-window", json_level, true); + appendPQExpBuffer(buf, "%u", config->retention_window); + + /* Compression parameters */ + json_add_value(buf, "compress-algorithm", + deparse_compress_alg(config->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", config->compress_level); + + json_add(buf, JT_END_OBJECT, &json_level); +} diff --git a/src/data.c b/src/data.c new file mode 100644 index 000000000..a66770bcf --- /dev/null +++ b/src/data.c @@ -0,0 +1,1407 @@ +/*------------------------------------------------------------------------- + * + * data.c: utils to parse and backup data pages + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "libpq/pqsignal.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum_impl.h" +#include + +#ifdef HAVE_LIBZ +#include +#endif + +#ifdef HAVE_LIBZ +/* Implementation of zlib compression method */ +static int32 +zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, + int level) +{ + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); + + return rc == Z_OK ? compressed_size : rc; +} + +/* Implementation of zlib compression method */ +static int32 +zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) +{ + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); + + return rc == Z_OK ? dest_len : rc; +} +#endif + +/* + * Compresses source into dest using algorithm. Returns the number of bytes + * written in the destination buffer, or -1 if compression fails. + */ +static int32 +do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_compress(dst, dst_size, src, src_size, level); +#endif + case PGLZ_COMPRESS: + return pglz_compress(src, src_size, dst, PGLZ_strategy_always); + } + + return -1; +} + +/* + * Decompresses source into dest using algorithm. Returns the number of bytes + * decompressed in the destination buffer, or -1 if decompression fails. + */ +static int32 +do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg) +{ + switch (alg) + { + case NONE_COMPRESS: + case NOT_DEFINED_COMPRESS: + return -1; +#ifdef HAVE_LIBZ + case ZLIB_COMPRESS: + return zlib_decompress(dst, dst_size, src, src_size); +#endif + case PGLZ_COMPRESS: + return pglz_decompress(src, src_size, dst, dst_size); + } + + return -1; +} + +/* + * When copying datafiles to backup we validate and compress them block + * by block. Thus special header is required for each data block. + */ +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + int32 compressed_size; +} BackupPageHeader; + +/* Special value for compressed_size field */ +#define PageIsTruncated -2 +#define SkipCurrentPage -3 + +/* Verify page's header */ +static bool +parse_page(Page page, XLogRecPtr *lsn) +{ + PageHeader phdr = (PageHeader) page; + + /* Get lsn from page header */ + *lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + if (PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special)) + return true; + + return false; +} + +/* Read one page from file directly accessing disk + * return value: + * 0 - if the page is not found + * 1 - if the page is found and valid + * -1 - if the page is found but invalid + */ +static int +read_page_from_file(pgFile *file, BlockNumber blknum, + FILE *in, Page page, XLogRecPtr *page_lsn) +{ + off_t offset = blknum * BLCKSZ; + size_t read_len = 0; + + /* read the block */ + if (fseek(in, offset, SEEK_SET) != 0) + elog(ERROR, "File: %s, could not seek to block %u: %s", + file->path, blknum, strerror(errno)); + + read_len = fread(page, 1, BLCKSZ, in); + + if (read_len != BLCKSZ) + { + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(LOG, "File %s, block %u, file was truncated", + file->path, blknum); + return 0; + } + else + elog(WARNING, "File: %s, block %u, expected block size %d," + "but read %lu, try again", + file->path, blknum, BLCKSZ, read_len); + } + + /* + * If we found page with invalid header, at first check if it is zeroed, + * which is a valid state for page. If it is not, read it and check header + * again, because it's possible that we've read a partly flushed page. + * If after several attempts page header is still invalid, throw an error. + * The same idea is applied to checksum verification. + */ + if (!parse_page(page, page_lsn)) + { + int i; + /* Check if the page is zeroed. */ + for(i = 0; i < BLCKSZ && page[i] == 0; i++); + + /* Page is zeroed. No need to check header and checksum. */ + if (i == BLCKSZ) + { + elog(LOG, "File: %s blknum %u, empty page", file->path, blknum); + return 1; + } + + /* + * If page is not completely empty and we couldn't parse it, + * try again several times. If it didn't help, throw error + */ + elog(LOG, "File: %s blknum %u have wrong page header, try again", + file->path, blknum); + return -1; + } + + /* Verify checksum */ + if(current.checksum_version) + { + /* + * If checksum is wrong, sleep a bit and then try again + * several times. If it didn't help, throw error + */ + if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) + != ((PageHeader) page)->pd_checksum) + { + elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + file->path, blknum); + return -1; + } + else + { + /* page header and checksum are correct */ + return 1; + } + } + else + { + /* page header is correct and checksum check is disabled */ + return 1; + } +} + +/* + * Retrieves a page taking the backup mode into account + * and writes it into argument "page". Argument "page" + * should be a pointer to allocated BLCKSZ of bytes. + * + * Prints appropriate warnings/errors/etc into log. + * Returns 0 if page was successfully retrieved + * SkipCurrentPage(-3) if we need to skip this page + * PageIsTruncated(-2) if the page was truncated + */ +static int32 +prepare_page(backup_files_arg *arguments, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + BlockNumber blknum, BlockNumber nblocks, + FILE *in, int *n_skipped, + BackupMode backup_mode, + Page page) +{ + XLogRecPtr page_lsn = 0; + int try_again = 100; + bool page_is_valid = false; + bool page_is_truncated = false; + BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during backup"); + + /* + * Read the page and verify its header and checksum. + * Under high write load it's possible that we've read partly + * flushed page, so try several times before throwing an error. + */ + if (backup_mode != BACKUP_MODE_DIFF_PTRACK) + { + while(!page_is_valid && try_again) + { + int result = read_page_from_file(file, blknum, + in, page, &page_lsn); + + try_again--; + if (result == 0) + { + /* This block was truncated.*/ + page_is_truncated = true; + /* Page is not actually valid, but it is absent + * and we're not going to reread it or validate */ + page_is_valid = true; + } + + if (result == 1) + page_is_valid = true; + + /* + * If ptrack support is available use it to get invalid block + * instead of rereading it 99 times + */ + //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); + + if (result == -1 && is_ptrack_support) + { + elog(WARNING, "File %s, block %u, try to fetch via SQL", + file->path, blknum); + break; + } + } + /* + * If page is not valid after 100 attempts to read it + * throw an error. + */ + if(!page_is_valid && !is_ptrack_support) + elog(ERROR, "Data file checksum mismatch. Canceling backup"); + } + + if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) + { + size_t page_size = 0; + Page ptrack_page = NULL; + ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + file->relOid, absolute_blknum, &page_size); + + if (ptrack_page == NULL) + { + /* This block was truncated.*/ + page_is_truncated = true; + } + else if (page_size != BLCKSZ) + { + free(ptrack_page); + elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + file->path, absolute_blknum, BLCKSZ, page_size); + } + else + { + /* + * We need to copy the page that was successfully + * retreieved from ptrack into our output "page" parameter. + * We must set checksum here, because it is outdated + * in the block recieved from shared buffers. + */ + memcpy(page, ptrack_page, BLCKSZ); + free(ptrack_page); + if (is_checksum_enabled) + ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + } + /* get lsn from page, provided by pg_ptrack_get_block() */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + !parse_page(page, &page_lsn)) + elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " + "Possible risk of a memory corruption"); + + } + + if (backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev && + !page_is_truncated && + page_lsn < prev_backup_start_lsn) + { + elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); + (*n_skipped)++; + return SkipCurrentPage; + } + + if (page_is_truncated) + return PageIsTruncated; + + return 0; +} + +static void +compress_and_backup_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, pg_crc32 *crc, + int page_state, Page page, + CompressAlg calg, int clevel) +{ + BackupPageHeader header; + size_t write_buffer_size = sizeof(header); + char write_buffer[BLCKSZ+sizeof(header)]; + char compressed_page[BLCKSZ]; + + if(page_state == SkipCurrentPage) + return; + + header.block = blknum; + header.compressed_size = page_state; + + if(page_state == PageIsTruncated) + { + /* + * The page was truncated. Write only header + * to know that we must truncate restored file + */ + memcpy(write_buffer, &header, sizeof(header)); + } + else + { + /* The page was not truncated, so we need to compress it */ + header.compressed_size = do_compress(compressed_page, BLCKSZ, + page, BLCKSZ, calg, clevel); + + file->compress_alg = calg; + file->read_size += BLCKSZ; + Assert (header.compressed_size <= BLCKSZ); + + /* The page was successfully compressed. */ + if (header.compressed_size > 0) + { + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); + } + /* Nonpositive value means that compression failed. Write it as is. */ + else + { + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; + } + } + + /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", + blknum, header.compressed_size, write_buffer_size); */ + + /* Update CRC */ + COMP_CRC32C(*crc, write_buffer, write_buffer_size); + + /* write data page */ + if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) + { + int errno_tmp = errno; + + fclose(in); + fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u : %s", + file->path, blknum, strerror(errno_tmp)); + } + + file->write_size += write_buffer_size; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +bool +backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel) +{ + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int n_blocks_read = 0; + int page_state; + char curr_page[BLCKSZ]; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if ((backup_mode == BACKUP_MODE_DIFF_PAGE || + backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want make + * incremental backup, so we should exit. + */ + elog(VERBOSE, "Skipping the unchanged file: %s", file->path); + return false; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + INIT_CRC32C(file->crc); + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(file->crc); + + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return false; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + /* open backup file for write */ + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open backup file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * We will enter here if backup_mode is FULL or DELTA. + */ + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev) + { + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + if (backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = n_blocks_read; + } + /* + * If page map is not empty we scan only changed blocks. + * + * We will enter here if backup_mode is PAGE or PTRACK. + */ + else + { + datapagemap_iterator_t *iter; + iter = datapagemap_iterate(&file->pagemap); + while (datapagemap_next(iter, &blknum)) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } + + pg_free(file->pagemap.bitmap); + pg_free(iter); + } + + /* update file permission */ + if (chmod(to_path, FILE_PERMISSION) == -1) + { + int errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", file->path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write backup file \"%s\": %s", + to_path, strerror(errno)); + fclose(in); + + FIN_CRC32C(file->crc); + + /* + * If we have pagemap then file in the backup can't be a zero size. + * Otherwise, we will clear the last file. + */ + if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) + { + if (remove(to_path) == -1) + elog(ERROR, "cannot remove file \"%s\": %s", to_path, + strerror(errno)); + return false; + } + + return true; +} + +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. + */ +void +restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, + bool write_header) +{ + FILE *in = NULL; + FILE *out = NULL; + BackupPageHeader header; + BlockNumber blknum = 0, + truncate_from = 0; + bool need_truncate = false; + + /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ + if (file->write_size != BYTES_INVALID) + { + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "cannot open backup file \"%s\": %s", file->path, + strerror(errno)); + } + } + + /* + * Open backup file for write. We use "r+" at first to overwrite only + * modified pages for differential restore. If the file does not exist, + * re-open it with "w" to create an empty file. + */ + out = fopen(to_path, PG_BINARY_R "+"); + if (out == NULL && errno == ENOENT) + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open restore target file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + while (true) + { + off_t write_pos; + size_t read_len; + DataPage compressed_page; /* used as read buffer */ + DataPage page; + + /* File didn`t changed. Nothig to copy */ + if (file->write_size == BYTES_INVALID) + break; + + /* + * We need to truncate result file if data file in a incremental backup + * less than data file in a full backup. We know it thanks to n_blocks. + * + * It may be equal to -1, then we don't want to truncate the result + * file. + */ + if (file->n_blocks != BLOCKNUM_INVALID && + (blknum + 1) > file->n_blocks) + { + truncate_from = blknum; + need_truncate = true; + break; + } + + /* read BackupPageHeader */ + read_len = fread(&header, 1, sizeof(header), in); + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "odd size page found at block %u of \"%s\"", + blknum, file->path); + else + elog(ERROR, "cannot read header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno_tmp)); + } + + if (header.block < blknum) + elog(ERROR, "backup is broken at file->path %s block %u", + file->path, blknum); + + blknum = header.block; + + if (header.compressed_size == PageIsTruncated) + { + /* + * Backup contains information that this block was truncated. + * We need to truncate file to this length. + */ + truncate_from = blknum; + need_truncate = true; + break; + } + + Assert(header.compressed_size <= BLCKSZ); + + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + blknum, file->path, read_len, header.compressed_size); + + if (header.compressed_size != BLCKSZ) + { + int32 uncompressed_size = 0; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + MAXALIGN(header.compressed_size), + file->compress_alg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); + } + + write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : + blknum * BLCKSZ; + + /* + * Seek and write the restored page. + */ + if (fseek(out, write_pos, SEEK_SET) < 0) + elog(ERROR, "cannot seek block %u of \"%s\": %s", + blknum, to_path, strerror(errno)); + + if (write_header) + { + if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + elog(ERROR, "cannot write header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + + if (header.compressed_size < BLCKSZ) + { + if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + /* if page wasn't compressed, we've read full block */ + if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + elog(ERROR, "cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + } + + /* + * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do + * But during DELTA backup we read every file in PGDATA and thus DELTA backup + * knows exact size of every file at the time of backup. + * So when restoring file from DELTA backup we, knowning it`s size at + * a time of a backup, can truncate file to this size. + */ + if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) + { + size_t file_size = 0; + + /* get file current size */ + fseek(out, 0, SEEK_END); + file_size = ftell(out); + + if (file_size > file->n_blocks * BLCKSZ) + { + truncate_from = file->n_blocks; + need_truncate = true; + } + } + + if (need_truncate) + { + off_t write_pos; + + write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : + truncate_from * BLCKSZ; + + /* + * Truncate file to this length. + */ + if (ftruncate(fileno(out), write_pos) != 0) + elog(ERROR, "cannot truncate \"%s\": %s", + file->path, strerror(errno)); + elog(INFO, "Delta truncate file %s to block %u", + file->path, truncate_from); + } + + /* update file permission */ + if (chmod(to_path, file->mode) == -1) + { + int errno_tmp = errno; + + if (in) + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + if (in) + fclose(in); +} + +/* + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +bool +copy_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + FILE *in; + FILE *out; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* open backup file for write */ + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + out = fopen(to_path, PG_BINARY_W); + if (out == NULL) + { + int errno_tmp = errno; + fclose(in); + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno_tmp)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + /* copy content and calc CRC */ + for (;;) + { + read_len = 0; + + if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + break; + + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + fclose(out); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_tmp = errno; + /* oops */ + fclose(in); + fclose(out); + elog(ERROR, "cannot write to \"%s\": %s", to_path, + strerror(errno_tmp)); + } + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->read_size += read_len; + } + + file->write_size = (int64) file->read_size; + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + /* update file permission */ + if (chmod(to_path, st.st_mode) == -1) + { + errno_tmp = errno; + fclose(in); + fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno_tmp)); + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + fclose(in); + + return true; +} + +/* + * Move file from one backup to another. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +move_file(const char *from_root, const char *to_root, pgFile *file) +{ + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + if (rename(file->path, to_path) == -1) + elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", + file->path, to_path, strerror(errno)); +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* + * Copy file attributes + */ +static void +copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) +{ + struct stat st; + + if (stat(from_path, &st) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot stat file \"%s\": %s", + from_path, strerror(errno)); + } + + if (chmod(to_path, st.st_mode) == -1) + { + if (unlink_on_error) + unlink(to_path); + elog(ERROR, "Cannot change mode of file \"%s\": %s", + to_path, strerror(errno)); + } +} + +/* + * Copy WAL segment from pgdata to archive catalog with possible compression. + */ +void +push_wal_file(const char *from_path, const char *to_path, bool is_compress, + bool overwrite) +{ + FILE *in = NULL; + FILE *out=NULL; + char buf[XLOG_BLCKSZ]; + const char *to_path_p = to_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + +#ifdef HAVE_LIBZ + char gz_to_path[MAXPGPATH]; + gzFile gz_out = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + + /* open backup file for write */ +#ifdef HAVE_LIBZ + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + + if (!overwrite && fileExists(gz_to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + + gz_out = gzopen(to_path_temp, PG_BINARY_W); + if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) + elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", + compress_level, to_path_temp, get_gz_error(gz_out, errno)); + + to_path_p = gz_to_path; + } + else +#endif + { + if (!overwrite && fileExists(to_path)) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + + /* copy content */ + for (;;) + { + size_t read_len = 0; + + read_len = fread(buf, 1, sizeof(buf), in); + + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, + "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzwrite(gz_out, buf, read_len) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + } + + if (feof(in) || read_len == 0) + break; + } + +#ifdef HAVE_LIBZ + if (is_compress) + { + if (gzclose(gz_out) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + /* update file permission. */ + copy_meta(from_path, to_path_temp, true); + + if (rename(to_path_temp, to_path_p) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path_p, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_compress) + elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); +#endif +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + */ +void +get_wal_file(const char *from_path, const char *to_path) +{ + FILE *in = NULL; + FILE *out; + char buf[XLOG_BLCKSZ]; + const char *from_path_p = from_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + bool is_decompress = false; + +#ifdef HAVE_LIBZ + char gz_from_path[MAXPGPATH]; + gzFile gz_in = NULL; +#endif + + /* open file for read */ + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) + { +#ifdef HAVE_LIBZ + /* + * Maybe we need to decompress the file. Check it with .gz + * extension. + */ + snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); + gz_in = gzopen(gz_from_path, PG_BINARY_R); + if (gz_in == NULL) + { + if (errno == ENOENT) + { + /* There is no compressed file too, raise an error below */ + } + /* Cannot open compressed file for some reason */ + else + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); + } + else + { + /* Found compressed file */ + is_decompress = true; + from_path_p = gz_from_path; + } +#endif + /* Didn't find compressed file */ + if (!is_decompress) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); + } + + /* open backup file for write */ + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fopen(to_path_temp, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + /* copy content */ + for (;;) + { + size_t read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !gzeof(gz_in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + read_len = fread(buf, 1, sizeof(buf), in); + if (ferror(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, + strerror(errno_temp)); + } + } + + /* Check for EOF */ +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzeof(gz_in) || read_len == 0) + break; + } + else +#endif + { + if (feof(in) || read_len == 0) + break; + } + } + + if (fflush(out) != 0 || + fsync(fileno(out)) != 0 || + fclose(out)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (gzclose(gz_in) != 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + if (fclose(in)) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + /* update file permission. */ + copy_meta(from_path_p, to_path_temp, true); + + if (rename(to_path_temp, to_path) < 0) + { + errno_temp = errno; + unlink(to_path_temp); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); +#endif +} + +/* + * Calculate checksum of various files which are not copied from PGDATA, + * but created in process of backup, such as stream XLOG files, + * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. + */ +bool +calc_file_checksum(pgFile *file) +{ + FILE *in; + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + struct stat st; + pg_crc32 crc; + + Assert(S_ISREG(file->mode)); + INIT_CRC32C(crc); + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_CRC32C(crc); + file->crc = crc; + + /* maybe deleted, it's not error */ + if (errno == ENOENT) + return false; + + elog(ERROR, "cannot open source file \"%s\": %s", file->path, + strerror(errno)); + } + + /* stat source file to change mode of destination file */ + if (fstat(fileno(in), &st) == -1) + { + fclose(in); + elog(ERROR, "cannot stat \"%s\": %s", file->path, + strerror(errno)); + } + + for (;;) + { + read_len = fread(buf, 1, sizeof(buf), in); + + if(read_len == 0) + break; + + /* update CRC */ + COMP_CRC32C(crc, buf, read_len); + + file->write_size += read_len; + file->read_size += read_len; + } + + errno_tmp = errno; + if (!feof(in)) + { + fclose(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ + FIN_CRC32C(crc); + file->crc = crc; + + fclose(in); + + return true; +} diff --git a/src/delete.c b/src/delete.c new file mode 100644 index 000000000..de29d2cf0 --- /dev/null +++ b/src/delete.c @@ -0,0 +1,464 @@ +/*------------------------------------------------------------------------- + * + * delete.c: delete backup files. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +static int pgBackupDeleteFiles(pgBackup *backup); +static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size); + +int +do_delete(time_t backup_id) +{ + int i; + parray *backup_list, + *delete_list; + pgBackup *target_backup = NULL; + time_t parent_id = 0; + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get complete list of backups */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + if (backup_id != 0) + { + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->start_time == backup_id) + { + parray_append(delete_list, backup); + + /* + * Do not remove next backups, if target backup was finished + * incorrectly. + */ + if (backup->status == BACKUP_STATUS_ERROR) + break; + + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + target_backup = backup; + } + else if (target_backup) + { + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->parent_backup == parent_id) + { + /* Append to delete list increment backup */ + parray_append(delete_list, backup); + /* Save backup id to retreive increment backups */ + parent_id = backup->start_time; + } + else + break; + } + } + + if (parray_num(delete_list) == 0) + elog(ERROR, "no backup found, cannot delete"); + + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + + pgBackupDeleteFiles(backup); + } + + parray_free(delete_list); + } + + /* Clean WAL segments */ + if (delete_wal) + { + Assert(target_backup); + + /* Find oldest LSN, used by backups */ + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + + if (backup->status == BACKUP_STATUS_OK) + { + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + break; + } + } + + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + return 0; +} + +/* + * Remove backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + */ +int +do_retention_purge(void) +{ + parray *backup_list; + uint32 backup_num; + size_t i; + time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + bool keep_next_backup = true; /* Do not delete first full backup */ + bool backup_deleted = false; /* At least one backup was deleted */ + + if (delete_expired) + { + if (retention_redundancy > 0) + elog(LOG, "REDUNDANCY=%u", retention_redundancy); + if (retention_window > 0) + elog(LOG, "WINDOW=%u", retention_window); + + if (retention_redundancy == 0 + && retention_window == 0) + { + elog(WARNING, "Retention policy is not set"); + if (!delete_wal) + return 0; + } + } + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get a complete list of backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + if (parray_num(backup_list) == 0) + { + elog(INFO, "backup list is empty, purging won't be executed"); + return 0; + } + + /* Find target backups to be deleted */ + if (delete_expired && + (retention_redundancy > 0 || retention_window > 0)) + { + backup_num = 0; + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + uint32 backup_num_evaluate = backup_num; + + /* Consider only validated and correct backups */ + if (backup->status != BACKUP_STATUS_OK) + continue; + /* + * When a valid full backup was found, we can delete the + * backup that is older than it using the number of generations. + */ + if (backup->backup_mode == BACKUP_MODE_FULL) + backup_num++; + + /* Evaluate retention_redundancy if this backup is eligible for removal */ + if (keep_next_backup || + retention_redundancy >= backup_num_evaluate + 1 || + (retention_window > 0 && backup->recovery_time >= days_threshold)) + { + /* Save LSN and Timeline to remove unnecessary WAL segments */ + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + + /* Save parent backup of this incremental backup */ + if (backup->backup_mode != BACKUP_MODE_FULL) + keep_next_backup = true; + /* + * Previous incremental backup was kept or this is first backup + * so do not delete this backup. + */ + else + keep_next_backup = false; + + continue; + } + + /* Delete backup and update status to DELETED */ + pgBackupDeleteFiles(backup); + backup_deleted = true; + } + } + + /* + * If oldest_lsn and oldest_tli weren`t set because previous step was skipped + * then set them now if we are going to purge WAL + */ + if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn))) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1); + oldest_lsn = backup->start_lsn; + oldest_tli = backup->tli; + } + + /* Be paranoid */ + if (XLogRecPtrIsInvalid(oldest_lsn)) + elog(ERROR, "Not going to purge WAL because LSN is invalid"); + + /* Purge WAL files */ + if (delete_wal) + { + delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + if (backup_deleted) + elog(INFO, "Purging finished"); + else + elog(INFO, "Nothing to delete by retention policy"); + + return 0; +} + +/* + * Delete backup files of the backup and update the status of the backup to + * BACKUP_STATUS_DELETED. + */ +static int +pgBackupDeleteFiles(pgBackup *backup) +{ + size_t i; + char path[MAXPGPATH]; + char timestamp[100]; + parray *files; + + /* + * If the backup was deleted already, there is nothing to do. + */ + if (backup->status == BACKUP_STATUS_DELETED) + return 0; + + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + + elog(INFO, "delete: %s %s", + base36enc(backup->start_time), timestamp); + + /* + * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which + * the error occurs before deleting all backup files. + */ + backup->status = BACKUP_STATUS_DELETING; + pgBackupWriteBackupControlFile(backup); + + /* list files to be deleted */ + files = parray_new(); + pgBackupGetPath(backup, path, lengthof(path), NULL); + dir_list_file(files, path, false, true, true); + + /* delete leaf node first */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + /* print progress */ + elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + (unsigned long) parray_num(files), file->path); + + if (remove(file->path)) + { + elog(WARNING, "can't remove \"%s\": %s", file->path, + strerror(errno)); + parray_walk(files, pgFileFree); + parray_free(files); + + return 1; + } + } + + parray_walk(files, pgFileFree); + parray_free(files); + backup->status = BACKUP_STATUS_DELETED; + + return 0; +} + +/* + * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups + * was deleted and so oldest_lsn is invalid). + * + * oldest_lsn - if valid, function deletes WAL segments, which contain lsn + * older than oldest_lsn. If it is invalid function deletes all WAL segments. + * oldest_tli - is used to construct oldest WAL segment in addition to + * oldest_lsn. + */ +static void +delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, + uint32 xlog_seg_size) +{ + XLogSegNo targetSegNo; + char oldestSegmentNeeded[MAXFNAMELEN]; + DIR *arcdir; + struct dirent *arcde; + char wal_file[MAXPGPATH]; + char max_wal_file[MAXPGPATH]; + char min_wal_file[MAXPGPATH]; + int rc; + + max_wal_file[0] = '\0'; + min_wal_file[0] = '\0'; + + if (!XLogRecPtrIsInvalid(oldest_lsn)) + { + GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); + GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, + xlog_seg_size); + + elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); + } + else + elog(LOG, "removing all WAL segments"); + + /* + * Now it is time to do the actual work and to remove all the segments + * not needed anymore. + */ + if ((arcdir = opendir(arclog_path)) != NULL) + { + while (errno = 0, (arcde = readdir(arcdir)) != NULL) + { + /* + * We ignore the timeline part of the WAL segment identifiers in + * deciding whether a segment is still needed. This ensures that + * we won't prematurely remove a segment from a parent timeline. + * We could probably be a little more proactive about removing + * segments of non-parent timelines, but that would be a whole lot + * more complicated. + * + * We use the alphanumeric sorting property of the filenames to + * decide which ones are earlier than the exclusiveCleanupFileName + * file. Note that this means files are not removed in the order + * they were originally written, in case this worries you. + * + * We also should not forget that WAL segment can be compressed. + */ + if (IsXLogFileName(arcde->d_name) || + IsPartialXLogFileName(arcde->d_name) || + IsBackupHistoryFileName(arcde->d_name) || + IsCompressedXLogFileName(arcde->d_name)) + { + if (XLogRecPtrIsInvalid(oldest_lsn) || + strncmp(arcde->d_name + 8, oldestSegmentNeeded + 8, 16) < 0) + { + /* + * Use the original file name again now, including any + * extension that might have been chopped off before testing + * the sequence. + */ + snprintf(wal_file, MAXPGPATH, "%s/%s", + arclog_path, arcde->d_name); + + rc = unlink(wal_file); + if (rc != 0) + { + elog(WARNING, "could not remove file \"%s\": %s", + wal_file, strerror(errno)); + break; + } + elog(LOG, "removed WAL segment \"%s\"", wal_file); + + if (max_wal_file[0] == '\0' || + strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) + strcpy(max_wal_file, arcde->d_name); + + if (min_wal_file[0] == '\0' || + strcmp(min_wal_file + 8, arcde->d_name + 8) > 0) + strcpy(min_wal_file, arcde->d_name); + } + } + } + + if (min_wal_file[0] != '\0') + elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); + if (max_wal_file[0] != '\0') + elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); + + if (errno) + elog(WARNING, "could not read archive location \"%s\": %s", + arclog_path, strerror(errno)); + if (closedir(arcdir)) + elog(WARNING, "could not close archive location \"%s\": %s", + arclog_path, strerror(errno)); + } + else + elog(WARNING, "could not open archive location \"%s\": %s", + arclog_path, strerror(errno)); +} + + +/* Delete all backup files and wal files of given instance. */ +int +do_delete_instance(void) +{ + parray *backup_list; + int i; + char instance_config_path[MAXPGPATH]; + + /* Delete all backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackupDeleteFiles(backup); + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + /* Delete all wal files. */ + delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); + + /* Delete backup instance config file */ + join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (remove(instance_config_path)) + { + elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + strerror(errno)); + } + + /* Delete instance root directories */ + if (rmdir(backup_instance_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + if (rmdir(arclog_path) != 0) + elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + strerror(errno)); + + elog(INFO, "Instance '%s' successfully deleted", instance_name); + return 0; +} diff --git a/src/dir.c b/src/dir.c new file mode 100644 index 000000000..a08bd9343 --- /dev/null +++ b/src/dir.c @@ -0,0 +1,1491 @@ +/*------------------------------------------------------------------------- + * + * dir.c: directory operation utility. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" +#include "datapagemap.h" + +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +const char *pgdata_exclude_dir[] = +{ + PG_XLOG_DIR, + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + "pg_stat_tmp", + "pgsql_tmp", + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + "pg_dynshmem", + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL, /* pg_log will be set later */ + NULL +}; + +static char *pgdata_exclude_files[] = +{ + /* Skip auto conf temporary file. */ + "postgresql.auto.conf.tmp", + + /* Skip current log file temporary file */ + "current_logfiles.tmp", + "recovery.conf", + "postmaster.pid", + "postmaster.opts", + NULL +}; + +static char *pgdata_exclude_files_non_exclusive[] = +{ + /*skip in non-exclusive backup */ + "backup_label", + "tablespace_map", + NULL +}; + +/* Tablespace mapping structures */ + +typedef struct TablespaceListCell +{ + struct TablespaceListCell *next; + char old_dir[MAXPGPATH]; + char new_dir[MAXPGPATH]; +} TablespaceListCell; + +typedef struct TablespaceList +{ + TablespaceListCell *head; + TablespaceListCell *tail; +} TablespaceList; + +typedef struct TablespaceCreatedListCell +{ + struct TablespaceCreatedListCell *next; + char link_name[MAXPGPATH]; + char linked_dir[MAXPGPATH]; +} TablespaceCreatedListCell; + +typedef struct TablespaceCreatedList +{ + TablespaceCreatedListCell *head; + TablespaceCreatedListCell *tail; +} TablespaceCreatedList; + +static int BlackListCompare(const void *str1, const void *str2); + +static bool dir_check_file(const char *root, pgFile *file); +static void dir_list_file_internal(parray *files, const char *root, + pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list); + +static void list_data_directories(parray *files, const char *path, bool is_root, + bool exclude); + +/* Tablespace mapping */ +static TablespaceList tablespace_dirs = {NULL, NULL}; +static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; + +/* + * Create directory, also create parent directories if necessary. + */ +int +dir_create_dir(const char *dir, mode_t mode) +{ + char parent[MAXPGPATH]; + + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); + + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode); + + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST) /* already exist */ + return 0; + elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + } + + return 0; +} + +pgFile * +pgFileNew(const char *path, bool omit_symlink) +{ + struct stat st; + pgFile *file; + + /* stat the file */ + if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + { + /* file not found is not an error case */ + if (errno == ENOENT) + return NULL; + elog(ERROR, "cannot stat file \"%s\": %s", path, + strerror(errno)); + } + + file = pgFileInit(path); + file->size = st.st_size; + file->mode = st.st_mode; + + return file; +} + +pgFile * +pgFileInit(const char *path) +{ + pgFile *file; + char *file_name; + + file = (pgFile *) pgut_malloc(sizeof(pgFile)); + + file->name = NULL; + + file->size = 0; + file->mode = 0; + file->read_size = 0; + file->write_size = 0; + file->crc = 0; + file->is_datafile = false; + file->linked = NULL; + file->pagemap.bitmap = NULL; + file->pagemap.bitmapsize = PageBitmapIsEmpty; + file->pagemap_isabsent = false; + file->tblspcOid = 0; + file->dbOid = 0; + file->relOid = 0; + file->segno = 0; + file->is_database = false; + file->forkName = pgut_malloc(MAXPGPATH); + file->forkName[0] = '\0'; + + file->path = pgut_malloc(strlen(path) + 1); + strcpy(file->path, path); /* enough buffer size guaranteed */ + + /* Get file name from the path */ + file_name = strrchr(file->path, '/'); + if (file_name == NULL) + file->name = file->path; + else + { + file_name++; + file->name = file_name; + } + + file->is_cfs = false; + file->exists_in_prev = false; /* can change only in Incremental backup. */ + /* Number of blocks readed during backup */ + file->n_blocks = BLOCKNUM_INVALID; + file->compress_alg = NOT_DEFINED_COMPRESS; + return file; +} + +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +pgFileDelete(pgFile *file) +{ + if (S_ISDIR(file->mode)) + { + if (rmdir(file->path) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR, "cannot remove directory \"%s\": %s", + file->path, strerror(errno)); + } + return; + } + +delete_file: + if (remove(file->path) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "cannot remove file \"%s\": %s", file->path, + strerror(errno)); + } +} + +pg_crc32 +pgFileGetCRC(const char *file_path) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[1024]; + size_t len; + int errno_tmp; + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", + file_path, strerror(errno)); + + /* calc CRC of backup file */ + INIT_CRC32C(crc); + while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + COMP_CRC32C(crc, buf, len); + } + errno_tmp = errno; + if (!feof(fp)) + elog(WARNING, "cannot read \"%s\": %s", file_path, + strerror(errno_tmp)); + if (len > 0) + COMP_CRC32C(crc, buf, len); + FIN_CRC32C(crc); + + fclose(fp); + + return crc; +} + +void +pgFileFree(void *file) +{ + pgFile *file_ptr; + + if (file == NULL) + return; + + file_ptr = (pgFile *) file; + + if (file_ptr->linked) + free(file_ptr->linked); + + if (file_ptr->forkName) + free(file_ptr->forkName); + + free(file_ptr->path); + free(file); +} + +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileComparePath(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->path, f2p->path); +} + +/* Compare two pgFile with their path in descending order of ASCII code. */ +int +pgFileComparePathDesc(const void *f1, const void *f2) +{ + return -pgFileComparePath(f1, f2); +} + +/* Compare two pgFile with their linked directory path. */ +int +pgFileCompareLinked(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->linked, f2p->linked); +} + +/* Compare two pgFile with their size */ +int +pgFileCompareSize(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + if (f1p->size > f2p->size) + return 1; + else if (f1p->size < f2p->size) + return -1; + else + return 0; +} + +static int +BlackListCompare(const void *str1, const void *str2) +{ + return strcmp(*(char **) str1, *(char **) str2); +} + +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * When omit_symlink is true, symbolic link is ignored and only file or + * directory llnked to will be listed. + */ +void +dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, + bool add_root) +{ + pgFile *file; + parray *black_list = NULL; + char path[MAXPGPATH]; + + join_path_components(path, backup_instance_path, PG_BLACK_LIST); + /* List files with black list */ + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + { + FILE *black_list_file = NULL; + char buf[MAXPGPATH * 2]; + char black_item[MAXPGPATH * 2]; + + black_list = parray_new(); + black_list_file = fopen(path, PG_BINARY_R); + + if (black_list_file == NULL) + elog(ERROR, "cannot open black_list: %s", strerror(errno)); + + while (fgets(buf, lengthof(buf), black_list_file) != NULL) + { + join_path_components(black_item, pgdata, buf); + + if (black_item[strlen(black_item) - 1] == '\n') + black_item[strlen(black_item) - 1] = '\0'; + + if (black_item[0] == '#' || black_item[0] == '\0') + continue; + + parray_append(black_list, black_item); + } + + fclose(black_list_file); + parray_qsort(black_list, BlackListCompare); + } + + file = pgFileNew(root, false); + if (file == NULL) + return; + + if (!S_ISDIR(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + return; + } + if (add_root) + parray_append(files, file); + + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); +} + +/* + * Check file or directory. + * + * Check for exclude. + * Extract information about the file parsing its name. + * Skip files: + * - skip temp tables files + * - skip unlogged tables files + * Set flags for: + * - database directories + * - datafiles + */ +static bool +dir_check_file(const char *root, pgFile *file) +{ + const char *rel_path; + int i; + int sscanf_res; + + /* Check if we need to exclude file by name */ + if (S_ISREG(file->mode)) + { + if (!exclusive_backup) + { + for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) + if (strcmp(file->name, + pgdata_exclude_files_non_exclusive[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + + for (i = 0; pgdata_exclude_files[i]; i++) + if (strcmp(file->name, pgdata_exclude_files[i]) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding file: %s", file->name); + return false; + } + } + /* + * If the directory name is in the exclude list, do not list the + * contents. + */ + else if (S_ISDIR(file->mode)) + { + /* + * If the item in the exclude list starts with '/', compare to + * the absolute path of the directory. Otherwise compare to the + * directory name portion. + */ + for (i = 0; pgdata_exclude_dir[i]; i++) + { + /* Full-path exclude*/ + if (pgdata_exclude_dir[i][0] == '/') + { + if (strcmp(file->path, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + { + elog(VERBOSE, "Excluding directory content: %s", + file->name); + return false; + } + } + } + + rel_path = GetRelativePath(file->path, root); + + /* + * Do not copy tablespaces twice. It may happen if the tablespace is located + * inside the PGDATA. + */ + if (S_ISDIR(file->mode) && + strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) + { + Oid tblspcOid; + char tmp_rel_path[MAXPGPATH]; + + /* + * Valid path for the tablespace is + * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + */ + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + return false; + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", + &tblspcOid, tmp_rel_path); + if (sscanf_res == 0) + return false; + } + + if (path_is_prefix_of_path("global", rel_path)) + { + file->tblspcOid = GLOBALTABLESPACE_OID; + + if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("base", rel_path)) + { + file->tblspcOid = DEFAULTTABLESPACE_OID; + + sscanf(rel_path, "base/%u/", &(file->dbOid)); + + if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) + file->is_database = true; + } + else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + + /* Do not backup ptrack_init files */ + if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + return false; + + /* + * Check files located inside database directories including directory + * 'global' + */ + if (S_ISREG(file->mode) && file->tblspcOid != 0 && + file->name && file->name[0]) + { + if (strcmp(file->name, "pg_internal.init") == 0) + return false; + /* Do not backup temp files */ + else if (file->name[0] == 't' && isdigit(file->name[1])) + return false; + else if (isdigit(file->name[0])) + { + char *fork_name; + int len; + char suffix[MAXPGPATH]; + + fork_name = strstr(file->name, "_"); + if (fork_name) + { + /* Auxiliary fork of the relfile */ + sscanf(file->name, "%u_%s", &(file->relOid), file->forkName); + + /* Do not backup ptrack files */ + if (strcmp(file->forkName, "ptrack") == 0) + return false; + } + else + { + len = strlen(file->name); + /* reloid.cfm */ + if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) + return true; + + sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), + &(file->segno), suffix); + if (sscanf_res == 0) + elog(ERROR, "Cannot parse file name \"%s\"", file->name); + else if (sscanf_res == 1 || sscanf_res == 2) + file->is_datafile = true; + } + } + } + + return true; +} + +/* + * List files in "root" directory. If "exclude" is true do not add into "files" + * files from pgdata_exclude_files and directories from pgdata_exclude_dir. + */ +static void +dir_list_file_internal(parray *files, const char *root, pgFile *parent, + bool exclude, bool omit_symlink, parray *black_list) +{ + DIR *dir; + struct dirent *dent; + + if (!S_ISDIR(parent->mode)) + elog(ERROR, "\"%s\" is not a directory", parent->path); + + /* Open directory and list contents */ + dir = opendir(parent->path); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* Maybe the directory was removed */ + return; + } + elog(ERROR, "cannot open directory \"%s\": %s", + parent->path, strerror(errno)); + } + + errno = 0; + while ((dent = readdir(dir))) + { + pgFile *file; + char child[MAXPGPATH]; + + join_path_components(child, parent->path, dent->d_name); + + file = pgFileNew(child, omit_symlink); + if (file == NULL) + continue; + + /* Skip entries point current dir or parent dir */ + if (S_ISDIR(file->mode) && + (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) + { + pgFileFree(file); + continue; + } + + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) + { + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + pgFileFree(file); + continue; + } + + /* Skip if the directory is in black_list defined by user */ + if (black_list && parray_bsearch(black_list, file->path, + BlackListCompare)) + { + elog(LOG, "Skip \"%s\": it is in the user's black list", file->path); + pgFileFree(file); + continue; + } + + /* We add the directory anyway */ + if (S_ISDIR(file->mode)) + parray_append(files, file); + + if (exclude && !dir_check_file(root, file)) + { + if (S_ISREG(file->mode)) + pgFileFree(file); + /* Skip */ + continue; + } + + /* At least add the file */ + if (S_ISREG(file->mode)) + parray_append(files, file); + + /* + * If the entry is a directory call dir_list_file_internal() + * recursively. + */ + if (S_ISDIR(file->mode)) + dir_list_file_internal(files, root, file, exclude, omit_symlink, + black_list); + } + + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + closedir(dir); + elog(ERROR, "cannot read directory \"%s\": %s", + parent->path, strerror(errno_tmp)); + } + closedir(dir); +} + +/* + * List data directories excluding directories from + * pgdata_exclude_dir array. + * + * **is_root** is a little bit hack. We exclude only first level of directories + * and on the first level we check all files and directories. + */ +static void +list_data_directories(parray *files, const char *path, bool is_root, + bool exclude) +{ + DIR *dir; + struct dirent *dent; + int prev_errno; + bool has_child_dirs = false; + + /* open directory and list contents */ + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + bool skip = false; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + /* Check for exclude for the first level of listing */ + if (is_root && exclude) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++) + { + if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0) + { + skip = true; + break; + } + } + } + if (skip) + continue; + + has_child_dirs = true; + list_data_directories(files, child, false, exclude); + } + + /* List only full and last directories */ + if (!is_root && !has_child_dirs) + { + pgFile *dir; + + dir = pgFileNew(path, false); + parray_append(files, dir); + } + + prev_errno = errno; + closedir(dir); + + if (prev_errno && prev_errno != ENOENT) + elog(ERROR, "cannot read directory \"%s\": %s", + path, strerror(prev_errno)); +} + +/* + * Save create directory path into memory. We can use it in next page restore to + * not raise the error "restore tablespace destination is not empty" in + * create_data_directories(). + */ +static void +set_tablespace_created(const char *link, const char *dir) +{ + TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); + + strcpy(cell->link_name, link); + strcpy(cell->linked_dir, dir); + cell->next = NULL; + + if (tablespace_created_dirs.tail) + tablespace_created_dirs.tail->next = cell; + else + tablespace_created_dirs.head = cell; + tablespace_created_dirs.tail = cell; +} + +/* + * Retrieve tablespace path, either relocated or original depending on whether + * -T was passed or not. + * + * Copy of function get_tablespace_mapping() from pg_basebackup.c. + */ +static const char * +get_tablespace_mapping(const char *dir) +{ + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(dir, cell->old_dir) == 0) + return cell->new_dir; + + return dir; +} + +/* + * Is directory was created when symlink was created in restore_directories(). + */ +static const char * +get_tablespace_created(const char *link) +{ + TablespaceCreatedListCell *cell; + + for (cell = tablespace_created_dirs.head; cell; cell = cell->next) + if (strcmp(link, cell->link_name) == 0) + return cell->linked_dir; + + return NULL; +} + +/* + * Split argument into old_dir and new_dir and append to tablespace mapping + * list. + * + * Copy of function tablespace_list_append() from pg_basebackup.c. + */ +void +opt_tablespace_map(pgut_option *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid tablespace mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", + cell->new_dir); + + if (tablespace_dirs.tail) + tablespace_dirs.tail->next = cell; + else + tablespace_dirs.head = cell; + tablespace_dirs.tail = cell; +} + +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + */ +void +create_data_directories(const char *data_dir, const char *backup_dir, + bool extract_tablespaces) +{ + parray *dirs, + *links = NULL; + size_t i; + char backup_database_dir[MAXPGPATH], + to_path[MAXPGPATH]; + + dirs = parray_new(); + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + } + + join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); + list_data_directories(dirs, backup_database_dir, true, false); + + elog(LOG, "restore directories and symlinks..."); + + for (i = 0; i < parray_num(dirs); i++) + { + pgFile *dir = (pgFile *) parray_get(dirs, i); + char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); + + Assert(S_ISDIR(dir->mode)); + + /* Try to create symlink and linked directory if necessary */ + if (extract_tablespaces && + path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + { + char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + *link_sep, + *tmp_ptr; + char link_name[MAXPGPATH]; + pgFile **link; + + /* Extract link name from relative path */ + link_sep = first_dir_separator(link_ptr); + if (link_sep != NULL) + { + int len = link_sep - link_ptr; + strncpy(link_name, link_ptr, len); + link_name[len] = '\0'; + } + else + goto create_directory; + + tmp_ptr = dir->path; + dir->path = link_name; + /* Search only by symlink name without path */ + link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); + dir->path = tmp_ptr; + + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + const char *dir_created; + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + /* Check if linked directory was created earlier */ + dir_created = get_tablespace_created(link_name); + if (dir_created) + { + /* + * If symlink and linked directory were created do not + * create it second time. + */ + if (strcmp(dir_created, linked_path) == 0) + { + /* + * Create rest of directories. + * First check is there any directory name after + * separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + else + continue; + } + else + elog(ERROR, "tablespace directory \"%s\" of page backup does not " + "match with previous created tablespace directory \"%s\" of symlink \"%s\"", + linked_path, dir_created, link_name); + } + + /* + * This check was done in check_tablespace_mapping(). But do + * it again. + */ + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + + if (link_sep) + elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + linked_path, + (int) (link_sep - relative_ptr), relative_ptr); + else + elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + linked_path, relative_ptr); + + /* Firstly, create linked directory */ + dir_create_dir(linked_path, DIR_PERMISSION); + + join_path_components(to_path, data_dir, PG_TBLSPC_DIR); + /* Create pg_tblspc directory just in case */ + dir_create_dir(to_path, DIR_PERMISSION); + + /* Secondly, create link */ + join_path_components(to_path, to_path, link_name); + if (symlink(linked_path, to_path) < 0) + elog(ERROR, "could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + /* Save linked directory */ + set_tablespace_created(link_name, linked_path); + + /* + * Create rest of directories. + * First check is there any directory name after separator. + */ + if (link_sep != NULL && *(link_sep + 1) != '\0') + goto create_directory; + + continue; + } + } + +create_directory: + elog(LOG, "create directory \"%s\"", relative_ptr); + + /* This is not symlink, create directory */ + join_path_components(to_path, data_dir, relative_ptr); + dir_create_dir(to_path, DIR_PERMISSION); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } + + parray_walk(dirs, pgFileFree); + parray_free(dirs); +} + +/* + * Read names of symbolik names of tablespaces with links to directories from + * tablespace_map or tablespace_map.txt. + */ +void +read_tablespace_map(parray *files, const char *backup_dir) +{ + FILE *fp; + char db_path[MAXPGPATH], + map_path[MAXPGPATH]; + char buf[MAXPGPATH * 2]; + + join_path_components(db_path, backup_dir, DATABASE_DIR); + join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path)) + { + elog(LOG, "there is no file tablespace_map"); + return; + } + + fp = fopen(map_path, "rt"); + if (fp == NULL) + elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + + while (fgets(buf, lengthof(buf), fp)) + { + char link_name[MAXPGPATH], + path[MAXPGPATH]; + pgFile *file; + + if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) + elog(ERROR, "invalid format found in \"%s\"", map_path); + + file = pgut_new(pgFile); + memset(file, 0, sizeof(pgFile)); + + file->path = pgut_malloc(strlen(link_name) + 1); + strcpy(file->path, link_name); + + file->linked = pgut_malloc(strlen(path) + 1); + strcpy(file->linked, path); + + parray_append(files, file); + } + + parray_qsort(files, pgFileCompareLinked); + fclose(fp); +} + +/* + * Check that all tablespace mapping entries have correct linked directory + * paths. Linked directories must be empty or do not exist. + * + * If tablespace-mapping option is supplied, all OLDDIR entries must have + * entries in tablespace_map file. + */ +void +check_tablespace_mapping(pgBackup *backup) +{ + char this_backup_path[MAXPGPATH]; + parray *links; + size_t i; + TablespaceListCell *cell; + pgFile *tmp_file = pgut_new(pgFile); + + links = parray_new(); + + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, this_backup_path); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + tmp_file->linked = cell->old_dir; + + if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) + elog(ERROR, "--tablespace-mapping option's old directory " + "doesn't have an entry in tablespace_map file: \"%s\"", + cell->old_dir); + } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(links); i++) + { + pgFile *link = (pgFile *) parray_get(links, i); + const char *linked_path = link->linked; + TablespaceListCell *cell; + + for (cell = tablespace_dirs.head; cell; cell = cell->next) + if (strcmp(link->linked, cell->old_dir) == 0) + { + linked_path = cell->new_dir; + break; + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + linked_path); + + if (!dir_is_empty(linked_path)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + } + + free(tmp_file); + parray_walk(links, pgFileFree); + parray_free(links); +} + +/* + * Print backup content list. + */ +void +print_file_list(FILE *out, const parray *files, const char *root) +{ + size_t i; + + /* print each file in the list */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = GetRelativePath(path, root); + + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " + "\"compress_alg\":\"%s\"", + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, + deparse_compress_alg(file->compress_alg)); + + if (file->is_datafile) + fprintf(out, ",\"segno\":\"%d\"", file->segno); + +#ifndef WIN32 + if (S_ISLNK(file->mode)) +#else + if (pgwin32_is_junction(file->path)) +#endif + fprintf(out, ",\"linked\":\"%s\"", file->linked); + + if (file->n_blocks != BLOCKNUM_INVALID) + fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + + fprintf(out, "}\n"); + } +} + +/* Parsing states for get_control_value() */ +#define CONTROL_WAIT_NAME 1 +#define CONTROL_INNAME 2 +#define CONTROL_WAIT_COLON 3 +#define CONTROL_WAIT_VALUE 4 +#define CONTROL_INVALUE 5 +#define CONTROL_WAIT_NEXT_NAME 6 + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string if it is not NULL. If it + * is NULL the value will be returned to "value_int64" as int64. + * + * Returns true if the value was found in the line. + */ +static bool +get_control_value(const char *str, const char *name, + char *value_str, int64 *value_int64, bool is_mandatory) +{ + int state = CONTROL_WAIT_NAME; + char *name_ptr = (char *) name; + char *buf = (char *) str; + char buf_int64[32], /* Buffer for "value_int64" */ + *buf_int64_ptr = buf_int64; + + /* Set default values */ + if (value_str) + *value_str = '\0'; + else if (value_int64) + *value_int64 = 0; + + while (*buf) + { + switch (state) + { + case CONTROL_WAIT_NAME: + if (*buf == '"') + state = CONTROL_INNAME; + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INNAME: + /* Found target field. Parse value. */ + if (*buf == '"') + state = CONTROL_WAIT_COLON; + /* Check next field */ + else if (*buf != *name_ptr) + { + name_ptr = (char *) name; + state = CONTROL_WAIT_NEXT_NAME; + } + else + name_ptr++; + break; + case CONTROL_WAIT_COLON: + if (*buf == ':') + state = CONTROL_WAIT_VALUE; + else if (!IsSpace(*buf)) + goto bad_format; + break; + case CONTROL_WAIT_VALUE: + if (*buf == '"') + { + state = CONTROL_INVALUE; + buf_int64_ptr = buf_int64; + } + else if (IsAlpha(*buf)) + goto bad_format; + break; + case CONTROL_INVALUE: + /* Value was parsed, exit */ + if (*buf == '"') + { + if (value_str) + { + *value_str = '\0'; + } + else if (value_int64) + { + /* Length of buf_uint64 should not be greater than 31 */ + if (buf_int64_ptr - buf_int64 >= 32) + elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + + *buf_int64_ptr = '\0'; + if (!parse_int64(buf_int64, value_int64, 0)) + goto bad_format; + } + + return true; + } + else + { + if (value_str) + { + *value_str = *buf; + value_str++; + } + else + { + *buf_int64_ptr = *buf; + buf_int64_ptr++; + } + } + break; + case CONTROL_WAIT_NEXT_NAME: + if (*buf == ',') + state = CONTROL_WAIT_NAME; + break; + default: + /* Should not happen */ + break; + } + + buf++; + } + + /* There is no close quotes */ + if (state == CONTROL_INNAME || state == CONTROL_INVALUE) + goto bad_format; + + /* Did not find target field */ + if (is_mandatory) + elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); + return false; + +bad_format: + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); + return false; /* Make compiler happy */ +} + +/* + * Construct parray of pgFile from the backup content list. + * If root is not NULL, path will be absolute path. + */ +parray * +dir_read_file_list(const char *root, const char *file_txt) +{ + FILE *fp; + parray *files; + char buf[MAXPGPATH * 2]; + + fp = fopen(file_txt, "rt"); + if (fp == NULL) + elog(errno == ENOENT ? ERROR : ERROR, + "cannot open \"%s\": %s", file_txt, strerror(errno)); + + files = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char filepath[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + crc, + segno, + n_blocks; + pgFile *file; + + get_control_value(buf, "path", path, NULL, true); + get_control_value(buf, "size", NULL, &write_size, true); + get_control_value(buf, "mode", NULL, &mode, true); + get_control_value(buf, "is_datafile", NULL, &is_datafile, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "crc", NULL, &crc, true); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); + + if (root) + join_path_components(filepath, root, path); + else + strcpy(filepath, path); + + file = pgFileInit(filepath); + + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + + /* + * Optional fields + */ + + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + file->linked = pgut_strdup(linked); + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + parray_append(files, file); + } + + fclose(fp); + return files; +} + +/* + * Check if directory empty. + */ +bool +dir_is_empty(const char *path) +{ + DIR *dir; + struct dirent *dir_ent; + + dir = opendir(path); + if (dir == NULL) + { + /* Directory in path doesn't exist */ + if (errno == ENOENT) + return true; + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + } + + errno = 0; + while ((dir_ent = readdir(dir))) + { + /* Skip entries point current dir or parent dir */ + if (strcmp(dir_ent->d_name, ".") == 0 || + strcmp(dir_ent->d_name, "..") == 0) + continue; + + /* Directory is not empty */ + closedir(dir); + return false; + } + if (errno) + elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + + closedir(dir); + + return true; +} + +/* + * Return true if the path is a existing regular file. + */ +bool +fileExists(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1 && errno == ENOENT) + return false; + else if (!S_ISREG(buf.st_mode)) + return false; + else + return true; +} + +size_t +pgFileSize(const char *path) +{ + struct stat buf; + + if (stat(path, &buf) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + return buf.st_size; +} diff --git a/src/fetch.c b/src/fetch.c new file mode 100644 index 000000000..0d4dbdaaf --- /dev/null +++ b/src/fetch.c @@ -0,0 +1,116 @@ +/*------------------------------------------------------------------------- + * + * fetch.c + * Functions for fetching files from PostgreSQL data directory + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "catalog/catalog.h" + +#include +#include +#include +#include +#include +#include + +#include "pg_probackup.h" + +/* + * Read a file into memory. The file to be read is /. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) +{ + int fd; + char *buffer; + struct stat statbuf; + char fullpath[MAXPGPATH]; + int len; + snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + + if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + if (fstat(fd, &statbuf) < 0) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (read(fd, buffer, len) != len) + { + if (safe) + return NULL; + else + elog(ERROR, "could not read file \"%s\": %s\n", + fullpath, strerror(errno)); + } + + close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + return buffer; +} + +/* + * Receive a single file as a malloc'd buffer. + */ +char * +fetchFile(PGconn *conn, const char *filename, size_t *filesize) +{ + PGresult *res; + char *result; + const char *params[1]; + int len; + + params[0] = filename; + res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", + 1, params, false, false); + + /* sanity check the result set */ + if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + filename); + + /* Read result to local variables */ + len = PQgetlength(res, 0, 0); + result = pg_malloc(len + 1); + memcpy(result, PQgetvalue(res, 0, 0), len); + result[len] = '\0'; + + PQclear(res); + *filesize = len; + + return result; +} diff --git a/src/help.c b/src/help.c new file mode 100644 index 000000000..dc9cc3d85 --- /dev/null +++ b/src/help.c @@ -0,0 +1,605 @@ +/*------------------------------------------------------------------------- + * + * help.c + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "pg_probackup.h" + +static void help_init(void); +static void help_backup(void); +static void help_restore(void); +static void help_validate(void); +static void help_show(void); +static void help_delete(void); +static void help_merge(void); +static void help_set_config(void); +static void help_show_config(void); +static void help_add_instance(void); +static void help_del_instance(void); +static void help_archive_push(void); +static void help_archive_get(void); + +void +help_command(char *command) +{ + if (strcmp(command, "init") == 0) + help_init(); + else if (strcmp(command, "backup") == 0) + help_backup(); + else if (strcmp(command, "restore") == 0) + help_restore(); + else if (strcmp(command, "validate") == 0) + help_validate(); + else if (strcmp(command, "show") == 0) + help_show(); + else if (strcmp(command, "delete") == 0) + help_delete(); + else if (strcmp(command, "merge") == 0) + help_merge(); + else if (strcmp(command, "set-config") == 0) + help_set_config(); + else if (strcmp(command, "show-config") == 0) + help_show_config(); + else if (strcmp(command, "add-instance") == 0) + help_add_instance(); + else if (strcmp(command, "del-instance") == 0) + help_del_instance(); + else if (strcmp(command, "archive-push") == 0) + help_archive_push(); + else if (strcmp(command, "archive-get") == 0) + help_archive_get(); + else if (strcmp(command, "--help") == 0 + || strcmp(command, "help") == 0 + || strcmp(command, "-?") == 0 + || strcmp(command, "--version") == 0 + || strcmp(command, "version") == 0 + || strcmp(command, "-V") == 0) + printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); + else + printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); + exit(0); +} + +void +help_pg_probackup(void) +{ + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + + printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + + printf(_("\n %s version\n"), PROGRAM_NAME); + + printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + + printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + + printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n")); + + printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n")); + + printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); + + printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); + printf(_(" [--timeline=timeline]\n")); + + printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n")); + + printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); + + printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n")); + + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n")); + + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + + if ((PROGRAM_URL || PROGRAM_EMAIL)) + { + printf("\n"); + if (PROGRAM_URL) + printf("Read the website for details. <%s>\n", PROGRAM_URL); + if (PROGRAM_EMAIL) + printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + } + exit(0); +} + +static void +help_init(void) +{ + printf(_("%s init -B backup-path\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); +} + +static void +help_backup(void) +{ + printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); + printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); + printf(_(" [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); + printf(_(" --stream stream the transaction log and include it in the backup\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); + printf(_(" --delete-wal remove redundant archived wal files\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress compress data files\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); +} + +static void +help_restore(void) +{ + printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" -i, --backup-id=backup-id backup to restore\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + + printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --recovery-target-action=pause|promote|shutdown\n")); + printf(_(" action the server should take once the recovery target is reached\n")); + printf(_(" (default: pause)\n")); + + printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); + printf(_(" to ease setting up a standby server\n")); + printf(_(" --no-validate disable backup validation during restore\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_validate(void) +{ + printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--timeline=timeline]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to validate\n")); + + printf(_(" --progress show progress\n")); + printf(_(" --time=time time stamp up to which recovery will proceed\n")); + printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-name=target-name\n")); + printf(_(" the named restore point to which recovery will proceed\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_show(void) +{ + printf(_("%s show -B backup-path\n"), PROGRAM_NAME); + printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_delete(void) +{ + printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-i backup-id | --expired] [--wal]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to delete\n")); + printf(_(" --expired delete backups expired according to current\n")); + printf(_(" retention policy\n")); + printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_merge(void) +{ + printf(_("%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to merge\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); +} + +static void +help_set_config(void) +{ + printf(_("%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--log-level-console=log-level-console]\n")); + printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-filename=log-filename]\n")); + printf(_(" [--error-log-filename=error-log-filename]\n")); + printf(_(" [--log-directory=log-directory]\n")); + printf(_(" [--log-rotation-size=log-rotation-size]\n")); + printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); + printf(_(" [--master-port=port] [--master-user=user_name]\n")); + printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--archive-timeout=timeout]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + + printf(_("\n Compression options:\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master\n")); + printf(_(" --master-db=db_name database to connect to master\n")); + printf(_(" --master-host=host_name database server host of master\n")); + printf(_(" --master-port=port database server port of master\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); +} + +static void +help_show_config(void) +{ + printf(_("%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [--format=format]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); +} + +static void +help_add_instance(void) +{ + printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance_name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" --instance=instance_name name of the new instance\n")); +} + +static void +help_del_instance(void) +{ + printf(_("%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); +} + +static void +help_archive_push(void) +{ + printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--compress]\n")); + printf(_(" [--compress-algorithm=compress-algorithm]\n")); + printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--overwrite]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress-algorithm=compress-algorithm\n")); + printf(_(" available options: 'zlib','none'\n")); + printf(_(" --compress-level=compress-level\n")); + printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_(" --overwrite overwrite archived WAL file\n")); +} + +static void +help_archive_get(void) +{ + printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" --wal-file-name=wal-file-name\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path name of the WAL file on the server\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" name of the WAL file to retrieve from the archive\n")); +} diff --git a/src/init.c b/src/init.c new file mode 100644 index 000000000..cd559cb48 --- /dev/null +++ b/src/init.c @@ -0,0 +1,108 @@ +/*------------------------------------------------------------------------- + * + * init.c: - initialize backup catalog. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +/* + * Initialize backup catalog. + */ +int +do_init(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + int results; + + results = pg_check_dir(backup_path); + if (results == 4) /* exists and not empty*/ + elog(ERROR, "backup catalog already exist and it's not empty"); + else if (results == -1) /*trouble accessing directory*/ + { + int errno_tmp = errno; + elog(ERROR, "cannot open backup catalog directory \"%s\": %s", + backup_path, strerror(errno_tmp)); + } + + /* create backup catalog root directory */ + dir_create_dir(backup_path, DIR_PERMISSION); + + /* create backup catalog data directory */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir_create_dir(path, DIR_PERMISSION); + + /* create backup catalog wal directory */ + join_path_components(arclog_path_dir, backup_path, "wal"); + dir_create_dir(arclog_path_dir, DIR_PERMISSION); + + elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + return 0; +} + +int +do_add_instance(void) +{ + char path[MAXPGPATH]; + char arclog_path_dir[MAXPGPATH]; + struct stat st; + pgBackupConfig *config = pgut_new(pgBackupConfig); + + /* PGDATA is always required */ + if (pgdata == NULL) + elog(ERROR, "Required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Read system_identifier from PGDATA */ + system_identifier = get_system_identifier(pgdata); + /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ + xlog_seg_size = get_xlog_seg_size(pgdata); + + /* Ensure that all root directories already exist */ + if (access(backup_path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", backup_path); + + join_path_components(path, backup_path, BACKUPS_DIR); + if (access(path, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", path); + + join_path_components(arclog_path_dir, backup_path, "wal"); + if (access(arclog_path_dir, F_OK) != 0) + elog(ERROR, "%s directory does not exist.", arclog_path_dir); + + /* Create directory for data files of this specific instance */ + if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", backup_instance_path); + dir_create_dir(backup_instance_path, DIR_PERMISSION); + + /* + * Create directory for wal files of this specific instance. + * Existence check is extra paranoid because if we don't have such a + * directory in data dir, we shouldn't have it in wal as well. + */ + if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", arclog_path); + dir_create_dir(arclog_path, DIR_PERMISSION); + + /* + * Wite initial config. system-identifier and pgdata are set in + * init subcommand and will never be updated. + */ + pgBackupConfigInit(config); + config->system_identifier = system_identifier; + config->xlog_seg_size = xlog_seg_size; + config->pgdata = pgdata; + writeBackupCatalogConfigFile(config); + + elog(INFO, "Instance '%s' successfully inited", instance_name); + return 0; +} diff --git a/src/merge.c b/src/merge.c new file mode 100644 index 000000000..979a1729f --- /dev/null +++ b/src/merge.c @@ -0,0 +1,526 @@ +/*------------------------------------------------------------------------- + * + * merge.c: merge FULL and incremental backups + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +typedef struct +{ + parray *to_files; + parray *files; + + pgBackup *to_backup; + pgBackup *from_backup; + + const char *to_root; + const char *from_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} merge_files_arg; + +static void merge_backups(pgBackup *backup, pgBackup *next_backup); +static void *merge_files(void *arg); + +/* + * Implementation of MERGE command. + * + * - Find target and its parent full backup + * - Merge data files of target, parent and and intermediate backups + * - Remove unnecessary files, which doesn't exist in the target backup anymore + */ +void +do_merge(time_t backup_id) +{ + parray *backups; + pgBackup *dest_backup = NULL; + pgBackup *full_backup = NULL; + time_t prev_parent = INVALID_BACKUP_ID; + int i; + int dest_backup_idx = 0; + int full_backup_idx = 0; + + if (backup_id == INVALID_BACKUP_ID) + elog(ERROR, "required parameter is not specified: --backup-id"); + + if (instance_name == NULL) + elog(ERROR, "required parameter is not specified: --instance"); + + elog(LOG, "Merge started"); + + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find destination and parent backups */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time > backup_id) + continue; + else if (backup->start_time == backup_id && !dest_backup) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + if (backup->backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Backup %s if full backup", + base36enc(backup->start_time)); + + dest_backup = backup; + dest_backup_idx = i; + } + else + { + Assert(dest_backup); + + if (backup->start_time != prev_parent) + continue; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Skipping backup %s, because it has non-valid status: %s", + base36enc(backup->start_time), status2str(backup->status)); + + /* If we already found dest_backup, look for full backup */ + if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) + { + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Parent full backup %s for the given backup %s has status: %s", + base36enc_dup(backup->start_time), + base36enc_dup(dest_backup->start_time), + status2str(backup->status)); + + full_backup = backup; + full_backup_idx = i; + + /* Found target and full backups, so break the loop */ + break; + } + } + + prev_parent = backup->parent_backup; + } + + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + if (full_backup == NULL) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(backup_id)); + + Assert(full_backup_idx != dest_backup_idx); + + /* + * Found target and full backups, merge them and intermediate backups + */ + for (i = full_backup_idx; i > dest_backup_idx; i--) + { + pgBackup *to_backup = (pgBackup *) parray_get(backups, i); + pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + + merge_backups(to_backup, from_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(LOG, "Merge completed"); +} + +/* + * Merge two backups data files using threads. + * - move instance files from from_backup to to_backup + * - remove unnecessary directories and files from to_backup + * - update metadata of from_backup, it becames FULL backup + */ +static void +merge_backups(pgBackup *to_backup, pgBackup *from_backup) +{ + char *to_backup_id = base36enc_dup(to_backup->start_time), + *from_backup_id = base36enc_dup(from_backup->start_time); + char to_backup_path[MAXPGPATH], + to_database_path[MAXPGPATH], + from_backup_path[MAXPGPATH], + from_database_path[MAXPGPATH], + control_file[MAXPGPATH]; + parray *files, + *to_files; + pthread_t *threads; + merge_files_arg *threads_args; + int i; + bool merge_isok = true; + + elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + to_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + pgBackupWriteBackupControlFile(from_backup); + + /* + * Make backup paths. + */ + pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); + pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), + DATABASE_DIR); + pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); + pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), + DATABASE_DIR); + + create_data_directories(to_database_path, from_backup_path, false); + + /* + * Get list of files which will be modified or removed. + */ + pgBackupGetPath(to_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + to_files = dir_read_file_list(from_database_path, /* Use from_database_path + * so root path will be + * equal with 'files' */ + control_file); + /* To delete from leaf, sort in reversed order */ + parray_qsort(to_files, pgFileComparePathDesc); + /* + * Get list of files which need to be moved. + */ + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + files = dir_read_file_list(from_database_path, control_file); + /* sort by size for load balancing */ + parray_qsort(files, pgFileCompareSize); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + + /* Setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_init_flag(&file->lock); + } + + for (i = 0; i < num_threads; i++) + { + merge_files_arg *arg = &(threads_args[i]); + + arg->to_files = to_files; + arg->files = files; + arg->to_backup = to_backup; + arg->from_backup = from_backup; + arg->to_root = to_database_path; + arg->from_root = from_database_path; + /* By default there are some error */ + arg->ret = 1; + + elog(VERBOSE, "Start thread: %d", i); + + pthread_create(&threads[i], NULL, merge_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + merge_isok = false; + } + if (!merge_isok) + elog(ERROR, "Data files merging failed"); + + /* + * Files were copied into to_backup and deleted from from_backup. Remove + * remaining directories from from_backup. + */ + parray_qsort(files, pgFileComparePathDesc); + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (rmdir(file->path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + file->path, strerror(errno)); + } + if (rmdir(from_database_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_database_path, strerror(errno)); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + pgBackupGetPath(from_backup, control_file, lengthof(control_file), + BACKUP_CONTROL_FILE); + if (unlink(control_file)) + elog(ERROR, "Could not remove file \"%s\": %s", + control_file, strerror(errno)); + + if (rmdir(from_backup_path)) + elog(ERROR, "Could not remove directory \"%s\": %s", + from_backup_path, strerror(errno)); + + /* + * Delete files which are not in from_backup file list. + */ + for (i = 0; i < parray_num(to_files); i++) + { + pgFile *file = (pgFile *) parray_get(to_files, i); + + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + elog(LOG, "Deleted \"%s\"", file->path); + } + } + + /* + * Rename FULL backup directory. + */ + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Update to_backup metadata. + */ + pgBackupCopy(to_backup, from_backup); + /* Correct metadata */ + to_backup->backup_mode = BACKUP_MODE_FULL; + to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; + } + /* compute size of wal files of this backup stored in the archive */ + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); + else + to_backup->wal_bytes = BYTES_INVALID; + + pgBackupWriteFileList(to_backup, files, from_database_path); + pgBackupWriteBackupControlFile(to_backup); + + /* Cleanup */ + pfree(threads_args); + pfree(threads); + + parray_walk(to_files, pgFileFree); + parray_free(to_files); + + parray_walk(files, pgFileFree); + parray_free(files); + + pfree(to_backup_id); + pfree(from_backup_id); +} + +/* + * Thread worker of merge_backups(). + */ +static void * +merge_files(void *arg) +{ + merge_files_arg *argument = (merge_files_arg *) arg; + pgBackup *to_backup = argument->to_backup; + pgBackup *from_backup = argument->from_backup; + char tmp_file_path[MAXPGPATH]; + int i, + num_files = parray_num(argument->files); + int to_root_len = strlen(argument->to_root); + + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + join_path_components(tmp_file_path, argument->to_root, "tmp"); + + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(argument->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "Interrupted during merging backups"); + + if (progress) + elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); + + /* + * Skip files which haven't changed since previous backup. But in case + * of DELTA backup we should consider n_blocks to truncate the target + * backup. + */ + if (file->write_size == BYTES_INVALID && + file->n_blocks == -1) + { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed in PAGE backup, retreive its + * write_size from previous FULL backup. + */ + if (S_ISREG(file->mode)) + { + pgFile **res_file; + + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + if (res_file && *res_file) + { + file->compress_alg = (*res_file)->compress_alg; + file->write_size = (*res_file)->write_size; + file->crc = (*res_file)->crc; + } + } + + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + /* + * Move the file. We need to decompress it and compress again if + * necessary. + */ + elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + file->path, file->is_database, file->is_cfs); + + if (file->is_datafile && !file->is_cfs) + { + char to_path_tmp[MAXPGPATH]; /* Path of target file */ + + join_path_components(to_path_tmp, argument->to_root, + file->path + to_root_len + 1); + + /* + * We need more complicate algorithm if target file exists and it is + * compressed. + */ + if (to_backup->compress_alg == PGLZ_COMPRESS || + to_backup->compress_alg == ZLIB_COMPRESS) + { + char *prev_path; + + /* Start the magic */ + + /* + * Merge files: + * - decompress first file + * - decompress second file and merge with first decompressed file + * - compress result file + */ + + elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", + tmp_file_path); + + prev_path = file->path; + /* + * We need to decompress target file only if it exists. + */ + if (fileExists(to_path_tmp)) + { + /* + * file->path points to the file in from_root directory. But we + * need the file in directory to_root. + */ + file->path = to_path_tmp; + + /* Decompress first/target file */ + restore_data_file(tmp_file_path, file, false, false); + + file->path = prev_path; + } + /* Merge second/source file with first/target file */ + restore_data_file(tmp_file_path, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + + elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + argument->to_root); + + /* Again we need change path */ + file->path = tmp_file_path; + /* backup_data_file() requires file size to calculate nblocks */ + file->size = pgFileSize(file->path); + /* Now we can compress the file */ + backup_data_file(NULL, /* We shouldn't need 'arguments' here */ + to_path_tmp, file, + to_backup->start_lsn, + to_backup->backup_mode, + to_backup->compress_alg, + to_backup->compress_level); + + file->path = prev_path; + + /* We can remove temporary file now */ + if (unlink(tmp_file_path)) + elog(ERROR, "Could not remove temporary file \"%s\": %s", + tmp_file_path, strerror(errno)); + } + /* + * Otherwise merging algorithm is simpler. + */ + else + { + /* We can merge in-place here */ + restore_data_file(to_path_tmp, file, + from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + true); + + /* + * We need to calculate write_size, restore_data_file() doesn't + * do that. + */ + file->write_size = pgFileSize(to_path_tmp); + file->crc = pgFileGetCRC(to_path_tmp); + } + pgFileDelete(file); + } + else + move_file(argument->from_root, argument->to_root, file); + + if (file->write_size != BYTES_INVALID) + elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files merging is successful */ + argument->ret = 0; + + return NULL; +} diff --git a/src/parsexlog.c b/src/parsexlog.c new file mode 100644 index 000000000..297269b68 --- /dev/null +++ b/src/parsexlog.c @@ -0,0 +1,1039 @@ +/*------------------------------------------------------------------------- + * + * parsexlog.c + * Functions for reading Write-Ahead-Log + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "commands/dbcommands_xlog.h" +#include "catalog/storage_xlog.h" +#include "access/transam.h" +#include "utils/thread.h" + +/* + * RmgrNames is an array of resource manager names, to make error messages + * a bit nicer. + */ +#if PG_VERSION_NUM >= 100000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ + name, +#else +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ + name, +#endif + +static const char *RmgrNames[RM_MAX_ID + 1] = { +#include "access/rmgrlist.h" +}; + +/* some from access/xact.h */ +/* + * XLOG allows to store some information in high 4 bits of log record xl_info + * field. We use 3 for the opcode, and one about an optional flag variable. + */ +#define XLOG_XACT_COMMIT 0x00 +#define XLOG_XACT_PREPARE 0x10 +#define XLOG_XACT_ABORT 0x20 +#define XLOG_XACT_COMMIT_PREPARED 0x30 +#define XLOG_XACT_ABORT_PREPARED 0x40 +#define XLOG_XACT_ASSIGNMENT 0x50 +/* free opcode 0x60 */ +/* free opcode 0x70 */ + +/* mask for filtering opcodes out of xl_info */ +#define XLOG_XACT_OPMASK 0x70 + +typedef struct xl_xact_commit +{ + TimestampTz xact_time; /* time of commit */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* xl_xact_dbinfo follows if XINFO_HAS_DBINFO */ + /* xl_xact_subxacts follows if XINFO_HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if XINFO_HAS_RELFILENODES */ + /* xl_xact_invals follows if XINFO_HAS_INVALS */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ + /* xl_xact_origin follows if XINFO_HAS_ORIGIN, stored unaligned! */ +} xl_xact_commit; + +typedef struct xl_xact_abort +{ + TimestampTz xact_time; /* time of abort */ + + /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ + /* No db_info required */ + /* xl_xact_subxacts follows if HAS_SUBXACT */ + /* xl_xact_relfilenodes follows if HAS_RELFILENODES */ + /* No invalidation messages needed. */ + /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ +} xl_xact_abort; + +static void extractPageInfo(XLogReaderState *record); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); + +typedef struct XLogPageReadPrivate +{ + const char *archivedir; + TimeLineID tli; + uint32 xlog_seg_size; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + +#ifdef HAVE_LIBZ + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; +#endif +} XLogPageReadPrivate; + +/* An argument for a thread function */ +typedef struct +{ + int thread_num; + XLogPageReadPrivate private_data; + + XLogRecPtr startpoint; + XLogRecPtr endpoint; + XLogSegNo endSegNo; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} xlog_thread_arg; + +static int SimpleXLogPageRead(XLogReaderState *xlogreader, + XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI); +static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, + const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, + bool allocate_reader); +static void CleanupXLogPageRead(XLogReaderState *xlogreader); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, + int elevel); + +static XLogSegNo nextSegNoToRead = 0; +static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* + * extractPageMap() worker. + */ +static void * +doExtractPageMap(void *arg) +{ + xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; + XLogPageReadPrivate *private_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + char *errormsg; + + private_data = &extract_arg->private_data; +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Start LSN of thread %d: %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + /* Switch WAL segment manually below without using SimpleXLogPageRead() */ + private_data->manual_switch = true; + + do + { + XLogRecord *record; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * Try to switch to the next WAL segment. Usually + * SimpleXLogPageRead() does it by itself. But here we need to do it + * manually to support threads. + */ + if (private_data->need_switch) + { + private_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(nextSegNoToRead); + private_data->xlogsegno = nextSegNoToRead; + nextSegNoToRead++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We reach the end */ + if (private_data->xlogsegno > extract_arg->endSegNo) + break; + + /* Adjust next record position */ + GetXLogRecPtr(private_data->xlogsegno, 0, + private_data->xlog_seg_size, + extract_arg->startpoint); + /* Skip over the page header */ + extract_arg->startpoint = XLogFindNextRecord(xlogreader, + extract_arg->startpoint); + + elog(VERBOSE, "Thread %d switched to LSN %X/%X", + extract_arg->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); + + continue; + } + + errptr = extract_arg->startpoint ? + extract_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(private_data, ERROR); + } + + extractPageInfo(xlogreader); + + /* continue reading at next record */ + extract_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, + private_data->xlog_seg_size); + } while (nextSegNo <= extract_arg->endSegNo && + xlogreader->EndRecPtr < extract_arg->endpoint); + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + extract_arg->ret = 0; + return NULL; +} + +/* + * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the + * given timeline. Collect data blocks touched by the WAL records into a page map. + * + * If **prev_segno** is true then read all segments up to **endpoint** segment + * minus one. Else read all segments up to **endpoint** segment. + * + * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * file. + */ +void +extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, bool prev_seg, + parray *files) +{ + int i; + int threads_need = 0; + XLogSegNo endSegNo; + bool extract_isok = true; + pthread_t *threads; + xlog_thread_arg *thread_args; + time_t start_time, + end_time; + + elog(LOG, "Compiling pagemap"); + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, seg_size); + if (prev_seg) + endSegNo--; + + nextSegNoToRead = 0; + time(&start_time); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, + seg_size, false); + thread_args[i].thread_num = i; + + thread_args[i].startpoint = startpoint; + thread_args[i].endpoint = endpoint; + thread_args[i].endSegNo = endSegNo; + /* By default there is some error */ + thread_args[i].ret = 1; + + /* Adjust startpoint to the next thread */ + if (nextSegNoToRead == 0) + GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); + + nextSegNoToRead++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (nextSegNoToRead > endSegNo) + break; + GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); + /* Skip over the page header */ + startpoint += SizeOfXLogLongPHD; + + threads_need++; + } + + /* Run threads */ + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i); + pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + extract_isok = false; + } + + pfree(threads); + pfree(thread_args); + + time(&end_time); + if (extract_isok) + elog(LOG, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap compiling failed"); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent state. + */ +static void +validate_backup_wal_from_start_to_stop(pgBackup *backup, + char *backup_xlog_path, TimeLineID tli, + uint32 xlog_seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + bool got_endpoint = false; + + xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, + xlog_seg_size, true); + + while (true) + { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + /* Got WAL record at stop_lsn */ + if (xlogreader->ReadRecPtr == backup->stop_lsn) + { + got_endpoint = true; + break; + } + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (!got_endpoint) + { + PrintXLogCorruptionMsg(&private, WARNING); + + /* + * If we don't have WAL between start_lsn and stop_lsn, + * the backup is definitely corrupted. Update its status. + */ + backup->status = BACKUP_STATUS_CORRUPT; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "There are not enough WAL records to consistenly restore " + "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", + base36enc(backup->start_time), + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn), + (uint32) (backup->stop_lsn >> 32), + (uint32) (backup->stop_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Ensure that the backup has all wal files needed for recovery to consistent + * state. And check if we have in archive all files needed to restore the backup + * up to the given recovery target. + */ +void +validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size) +{ + XLogRecPtr startpoint = backup->start_lsn; + const char *backup_id; + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + XLogPageReadPrivate private; + TransactionId last_xid = InvalidTransactionId; + TimestampTz last_time = 0; + char last_timestamp[100], + target_timestamp[100]; + bool all_wal = false; + char backup_xlog_path[MAXPGPATH]; + + /* We need free() this later */ + backup_id = base36enc(backup->start_time); + + if (!XRecOffIsValid(backup->start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", + (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), + backup_id); + + if (!XRecOffIsValid(backup->stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", + (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), + backup_id); + + /* + * Check that the backup has all wal files needed + * for recovery to consistent state. + */ + if (backup->stream) + { + snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", + backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + + validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, + seg_size); + } + else + validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, + seg_size); + + if (backup->status == BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + return; + } + /* + * If recovery target is provided check that we can restore backup to a + * recovery target time or xid. + */ + if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + { + /* Recovery target is not given so exit */ + elog(INFO, "Backup %s WAL segments are valid", backup_id); + return; + } + + /* + * If recovery target is provided, ensure that archive files exist in + * archive directory. + */ + if (dir_is_empty(archivedir)) + elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); + + /* + * Check if we have in archive all files needed to restore backup + * up to the given recovery target. + * In any case we cannot restore to the point before stop_lsn. + */ + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, + true); + + /* We can restore at least up to the backup end */ + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + last_xid = backup->recovery_xid; + + if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + || (target_time != 0 && backup->recovery_time >= target_time) + || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + all_wal = true; + + startpoint = backup->stop_lsn; + while (true) + { + bool timestamp_record; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + if (errormsg) + elog(WARNING, "%s", errormsg); + + break; + } + + timestamp_record = getRecordTimestamp(xlogreader, &last_time); + if (XLogRecGetXid(xlogreader) != InvalidTransactionId) + last_xid = XLogRecGetXid(xlogreader); + + /* Check target xid */ + if (TransactionIdIsValid(target_xid) && target_xid == last_xid) + { + all_wal = true; + break; + } + /* Check target time */ + else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + { + all_wal = true; + break; + } + /* If there are no target xid and target time */ + else if (!TransactionIdIsValid(target_xid) && target_time == 0 && + xlogreader->ReadRecPtr == backup->stop_lsn) + { + all_wal = true; + /* We don't stop here. We want to get last_xid and last_time */ + } + + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ + } + + if (last_time > 0) + time2iso(last_timestamp, lengthof(last_timestamp), + timestamptz_to_time_t(last_time)); + + /* There are all needed WAL records */ + if (all_wal) + elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, + last_timestamp, last_xid); + /* Some needed WAL records are absent */ + else + { + PrintXLogCorruptionMsg(&private, WARNING); + + elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, + last_timestamp, last_xid); + + if (target_time > 0) + time2iso(target_timestamp, lengthof(target_timestamp), + target_time); + if (TransactionIdIsValid(target_xid) && target_time != 0) + elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + target_timestamp, target_xid); + else if (TransactionIdIsValid(target_xid)) + elog(ERROR, "not enough WAL records to xid " XID_FMT, + target_xid); + else if (target_time != 0) + elog(ERROR, "not enough WAL records to time %s", + target_timestamp); + else if (XRecOffIsValid(target_lsn)) + elog(ERROR, "not enough WAL records to lsn %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + } + + /* clean */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); +} + +/* + * Read from archived WAL segments latest recovery time and xid. All necessary + * segments present at archive folder. We waited **stop_lsn** in + * pg_stop_backup(). + */ +bool +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, TransactionId *recovery_xid) +{ + XLogRecPtr startpoint = stop_lsn; + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + bool res; + + if (!XRecOffIsValid(start_lsn)) + elog(ERROR, "Invalid start_lsn value %X/%X", + (uint32) (start_lsn >> 32), (uint32) (start_lsn)); + + if (!XRecOffIsValid(stop_lsn)) + elog(ERROR, "Invalid stop_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + + /* Read records from stop_lsn down to start_lsn */ + do + { + XLogRecord *record; + TimestampTz last_time = 0; + char *errormsg; + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(ERROR, "could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(ERROR, "could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + } + + /* Read previous record */ + startpoint = record->xl_prev; + + if (getRecordTimestamp(xlogreader, &last_time)) + { + *recovery_time = timestamptz_to_time_t(last_time); + *recovery_xid = XLogRecGetXid(xlogreader); + + /* Found timestamp in WAL record 'record' */ + res = true; + goto cleanup; + } + } while (startpoint >= start_lsn); + + /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ + res = false; + +cleanup: + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +/* + * Check if there is a WAL segment file in 'archivedir' which contains + * 'target_lsn'. + */ +bool +wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size) +{ + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + char *errormsg; + bool res; + + if (!XRecOffIsValid(target_lsn)) + elog(ERROR, "Invalid target_lsn value %X/%X", + (uint32) (target_lsn >> 32), (uint32) (target_lsn)); + + xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, + true); + + res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; + /* Didn't find 'target_lsn' and there is no error, return false */ + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf) +{ + int errnum; + const char *errmsg; + + errmsg = gzerror(gzf, &errnum); + if (errnum == Z_ERRNO) + return strerror(errno); + else + return errmsg; +} +#endif + +/* XLogreader callback function, to read a WAL page */ +static int +SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, + int reqLen, XLogRecPtr targetRecPtr, char *readBuf, + TimeLineID *pageTLI) +{ + XLogPageReadPrivate *private_data; + uint32 targetPageOff; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + targetPageOff = targetPagePtr % private_data->xlog_seg_size; + + /* + * See if we need to switch to a new segment because the requested record + * is not in the currently open one. + */ + if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size)) + { + CleanupXLogPageRead(xlogreader); + /* + * Do not switch to next WAL segment in this function. Currently it is + * manually switched only in doExtractPageMap(). + */ + if (private_data->manual_switch) + { + private_data->need_switch = true; + return -1; + } + } + + GetXLogSegNo(targetPagePtr, private_data->xlogsegno, + private_data->xlog_seg_size); + + /* Try to switch to the next WAL segment */ + if (!private_data->xlogexists) + { + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, + private_data->xlog_seg_size); + snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", + private_data->archivedir, xlogfname); + + if (fileExists(private_data->xlogpath)) + { + elog(LOG, "Opening WAL segment \"%s\"", private_data->xlogpath); + + private_data->xlogexists = true; + private_data->xlogfile = open(private_data->xlogpath, + O_RDONLY | PG_BINARY, 0); + + if (private_data->xlogfile < 0) + { + elog(WARNING, "Could not open WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + /* Try to open compressed WAL segment */ + else + { + snprintf(private_data->gz_xlogpath, + sizeof(private_data->gz_xlogpath), "%s.gz", + private_data->xlogpath); + if (fileExists(private_data->gz_xlogpath)) + { + elog(LOG, "Opening compressed WAL segment \"%s\"", + private_data->gz_xlogpath); + + private_data->xlogexists = true; + private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, + "rb"); + if (private_data->gz_xlogfile == NULL) + { + elog(WARNING, "Could not open compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, strerror(errno)); + return -1; + } + } + } +#endif + + /* Exit without error if WAL segment doesn't exist */ + if (!private_data->xlogexists) + return -1; + } + + /* + * At this point, we have the right segment open. + */ + Assert(private_data->xlogexists); + + /* Read the requested page */ + if (private_data->xlogfile != -1) + { + if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + { + elog(WARNING, "Could not seek in WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + + if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from WAL segment \"%s\": %s", + private_data->xlogpath, strerror(errno)); + return -1; + } + } +#ifdef HAVE_LIBZ + else + { + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + { + elog(WARNING, "Could not seek in compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Could not read from compressed WAL segment \"%s\": %s", + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + } +#endif + + *pageTLI = private_data->tli; + return XLOG_BLCKSZ; +} + +/* + * Initialize WAL segments reading. + */ +static XLogReaderState * +InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, + TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +{ + XLogReaderState *xlogreader = NULL; + + MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); + private_data->archivedir = archivedir; + private_data->tli = tli; + private_data->xlog_seg_size = xlog_seg_size; + private_data->xlogfile = -1; + + if (allocate_reader) + { +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(xlog_seg_size, + &SimpleXLogPageRead, private_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "out of memory"); + } + + return xlogreader; +} + +/* + * Cleanup after WAL segment reading. + */ +static void +CleanupXLogPageRead(XLogReaderState *xlogreader) +{ + XLogPageReadPrivate *private_data; + + private_data = (XLogPageReadPrivate *) xlogreader->private_data; + if (private_data->xlogfile >= 0) + { + close(private_data->xlogfile); + private_data->xlogfile = -1; + } +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + { + gzclose(private_data->gz_xlogfile); + private_data->gz_xlogfile = NULL; + } +#endif + private_data->xlogexists = false; +} + +static void +PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +{ + if (private_data->xlogpath[0] != 0) + { + /* + * XLOG reader couldn't read WAL segment. + * We throw a WARNING here to be able to update backup status. + */ + if (!private_data->xlogexists) + elog(elevel, "WAL segment \"%s\" is absent", private_data->xlogpath); + else if (private_data->xlogfile != -1) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->xlogpath); +#ifdef HAVE_LIBZ + else if (private_data->gz_xlogfile != NULL) + elog(elevel, "Possible WAL corruption. " + "Error has occured during reading WAL segment \"%s\"", + private_data->gz_xlogpath); +#endif + } +} + +/* + * Extract information about blocks modified in this record. + */ +static void +extractPageInfo(XLogReaderState *record) +{ + uint8 block_id; + RmgrId rmid = XLogRecGetRmid(record); + uint8 info = XLogRecGetInfo(record); + uint8 rminfo = info & ~XLR_INFO_MASK; + + /* Is this a special record type that I recognize? */ + + if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + { + /* + * New databases can be safely ignored. They would be completely + * copied if found. + */ + } + else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP) + { + /* + * An existing database was dropped. It is fine to ignore that + * they will be removed appropriately. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE) + { + /* + * We can safely ignore these. The file will be removed when + * combining the backups in the case of differential on. + */ + } + else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE) + { + /* + * We can safely ignore these. When we compare the sizes later on, + * we'll notice that they differ, and copy the missing tail from + * source system. + */ + } + else if (info & XLR_SPECIAL_REL_UPDATE) + { + /* + * This record type modifies a relation file in some special way, but + * we don't recognize the type. That's bad - we don't know how to + * track that change. + */ + elog(ERROR, "WAL record modifies a relation, but record type is not recognized\n" + "lsn: %X/%X, rmgr: %s, info: %02X", + (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), + RmgrNames[rmid], info); + } + + for (block_id = 0; block_id <= record->max_block_id; block_id++) + { + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blkno; + + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + continue; + + /* We only care about the main fork; others are copied in toto */ + if (forknum != MAIN_FORKNUM) + continue; + + process_block_change(forknum, rnode, blkno); + } +} + +/* + * Extract timestamp from WAL record. + * + * If the record contains a timestamp, returns true, and saves the timestamp + * in *recordXtime. If the record type has no timestamp, returns false. + * Currently, only transaction commit/abort records and restore points contain + * timestamps. + */ +static bool +getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + uint8 xact_info = info & XLOG_XACT_OPMASK; + uint8 rmid = XLogRecGetRmid(record); + + if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT) + { + *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT || + xact_info == XLOG_XACT_COMMIT_PREPARED)) + { + *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time; + return true; + } + else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT || + xact_info == XLOG_XACT_ABORT_PREPARED)) + { + *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time; + return true; + } + + return false; +} + diff --git a/src/pg_probackup.c b/src/pg_probackup.c new file mode 100644 index 000000000..a39ea5a8c --- /dev/null +++ b/src/pg_probackup.c @@ -0,0 +1,634 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.c: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "streamutil.h" +#include "utils/thread.h" + +#include +#include +#include +#include +#include +#include "pg_getopt.h" + +const char *PROGRAM_VERSION = "2.0.18"; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; + +/* directory options */ +char *backup_path = NULL; +char *pgdata = NULL; +/* + * path or to the data files in the backup catalog + * $BACKUP_PATH/backups/instance_name + */ +char backup_instance_path[MAXPGPATH]; +/* + * path or to the wal files in the backup catalog + * $BACKUP_PATH/wal/instance_name + */ +char arclog_path[MAXPGPATH] = ""; + +/* common options */ +static char *backup_id_string = NULL; +int num_threads = 1; +bool stream_wal = false; +bool progress = false; +#if PG_VERSION_NUM >= 100000 +char *replication_slot = NULL; +#endif + +/* backup options */ +bool backup_logs = false; +bool smooth_checkpoint; +bool is_remote_backup = false; +/* Wait timeout for WAL segment archiving */ +uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; +const char *master_db = NULL; +const char *master_host = NULL; +const char *master_port= NULL; +const char *master_user = NULL; +uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; + +/* restore options */ +static char *target_time; +static char *target_xid; +static char *target_lsn; +static char *target_inclusive; +static TimeLineID target_tli; +static bool target_immediate; +static char *target_name = NULL; +static char *target_action = NULL; + +static pgRecoveryTarget *recovery_target_options = NULL; + +bool restore_as_replica = false; +bool restore_no_validate = false; + +/* delete options */ +bool delete_wal = false; +bool delete_expired = false; +bool apply_to_all = false; +bool force_delete = false; + +/* retention options */ +uint32 retention_redundancy = 0; +uint32 retention_window = 0; + +/* compression options */ +CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; +int compress_level = COMPRESS_LEVEL_DEFAULT; +bool compress_shortcut = false; + + +/* other options */ +char *instance_name; +uint64 system_identifier = 0; + +/* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ +#if PG_VERSION_NUM >= 110000 +uint32 xlog_seg_size = 0; +#else +uint32 xlog_seg_size = XLOG_SEG_SIZE; +#endif + +/* archive push options */ +static char *wal_file_path; +static char *wal_file_name; +static bool file_overwrite = false; + +/* show options */ +ShowFormat show_format = SHOW_PLAIN; + +/* current settings */ +pgBackup current; +ProbackupSubcmd backup_subcmd = NO_CMD; + +static bool help_opt = false; + +static void opt_backup_mode(pgut_option *opt, const char *arg); +static void opt_log_level_console(pgut_option *opt, const char *arg); +static void opt_log_level_file(pgut_option *opt, const char *arg); +static void opt_compress_alg(pgut_option *opt, const char *arg); +static void opt_show_format(pgut_option *opt, const char *arg); + +static void compress_init(void); + +static pgut_option options[] = +{ + /* directory options */ + { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, + { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + /* common options */ + { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, + { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, + { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, + /* backup options */ + { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, + { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + { 'b', 12, "delete-wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 13, "delete-expired", &delete_expired, SOURCE_CMDLINE }, + { 's', 14, "master-db", &master_db, SOURCE_CMDLINE, }, + { 's', 15, "master-host", &master_host, SOURCE_CMDLINE, }, + { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, + { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, + { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + /* TODO not completed feature. Make it unavailiable from user level + { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + /* restore options */ + { 's', 20, "time", &target_time, SOURCE_CMDLINE }, + { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, + { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, + { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, + { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, + { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, + { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + /* delete options */ + { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, + { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, + { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, + /* TODO not implemented yet */ + { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, + /* retention options */ + { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, + { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, + /* compression options */ + { 'f', 136, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, + { 'u', 137, "compress-level", &compress_level, SOURCE_CMDLINE }, + { 'b', 138, "compress", &compress_shortcut, SOURCE_CMDLINE }, + /* logging options */ + { 'f', 140, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, + { 'f', 141, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, + { 's', 142, "log-filename", &log_filename, SOURCE_CMDLINE }, + { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, + { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, + { 'u', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, + { 'u', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + /* connection options */ + { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, + { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, + { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, + { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, + { 'b', 'W', "password", &force_password, SOURCE_CMDLINE }, + /* other options */ + { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, + { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, +#if PG_VERSION_NUM >= 110000 + { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, +#endif + /* archive-push options */ + { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, + { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, + { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + /* show options */ + { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, + { 0 } +}; + +/* + * Entry point of pg_probackup command. + */ +int +main(int argc, char *argv[]) +{ + char *command = NULL, + *command_name; + /* Check if backup_path is directory. */ + struct stat stat_buf; + int rc; + + /* initialize configuration */ + pgBackupInit(¤t); + + PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], "pgscripts"); + +#if PG_VERSION_NUM >= 110000 + /* + * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() + * later. + */ + WalSegSz = 0; +#endif + + /* + * Save main thread's tid. It is used call exit() in case of errors. + */ + main_tid = pthread_self(); + + /* Parse subcommands and non-subcommand options */ + if (argc > 1) + { + if (strcmp(argv[1], "archive-push") == 0) + backup_subcmd = ARCHIVE_PUSH_CMD; + else if (strcmp(argv[1], "archive-get") == 0) + backup_subcmd = ARCHIVE_GET_CMD; + else if (strcmp(argv[1], "add-instance") == 0) + backup_subcmd = ADD_INSTANCE_CMD; + else if (strcmp(argv[1], "del-instance") == 0) + backup_subcmd = DELETE_INSTANCE_CMD; + else if (strcmp(argv[1], "init") == 0) + backup_subcmd = INIT_CMD; + else if (strcmp(argv[1], "backup") == 0) + backup_subcmd = BACKUP_CMD; + else if (strcmp(argv[1], "restore") == 0) + backup_subcmd = RESTORE_CMD; + else if (strcmp(argv[1], "validate") == 0) + backup_subcmd = VALIDATE_CMD; + else if (strcmp(argv[1], "delete") == 0) + backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "merge") == 0) + backup_subcmd = MERGE_CMD; + else if (strcmp(argv[1], "show") == 0) + backup_subcmd = SHOW_CMD; + else if (strcmp(argv[1], "set-config") == 0) + backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "show-config") == 0) + backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-?") == 0 || + strcmp(argv[1], "help") == 0) + { + if (argc > 2) + help_command(argv[2]); + else + help_pg_probackup(); + } + else if (strcmp(argv[1], "--version") == 0 + || strcmp(argv[1], "version") == 0 + || strcmp(argv[1], "-V") == 0) + { +#ifdef PGPRO_VERSION + fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stderr, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif + exit(0); + } + else + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + } + + if (backup_subcmd == NO_CMD) + elog(ERROR, "No subcommand specified"); + + /* + * Make command string before getopt_long() will call. It permutes the + * content of argv. + */ + command_name = pstrdup(argv[1]); + if (backup_subcmd == BACKUP_CMD || + backup_subcmd == RESTORE_CMD || + backup_subcmd == VALIDATE_CMD || + backup_subcmd == DELETE_CMD || + backup_subcmd == MERGE_CMD) + { + int i, + len = 0, + allocated = 0; + + allocated = sizeof(char) * MAXPGPATH; + command = (char *) palloc(allocated); + + for (i = 0; i < argc; i++) + { + int arglen = strlen(argv[i]); + + if (arglen + len > allocated) + { + allocated *= 2; + command = repalloc(command, allocated); + } + + strncpy(command + len, argv[i], arglen); + len += arglen; + command[len++] = ' '; + } + + command[len] = '\0'; + } + + optind += 1; + /* Parse command line arguments */ + pgut_getopt(argc, argv, options); + + if (help_opt) + help_command(command_name); + + /* backup_path is required for all pg_probackup commands except help */ + if (backup_path == NULL) + { + /* + * If command line argument is not set, try to read BACKUP_PATH + * from environment variable + */ + backup_path = getenv("BACKUP_PATH"); + if (backup_path == NULL) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + } + canonicalize_path(backup_path); + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + + /* Option --instance is required for all commands except init and show */ + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD) + { + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + } + + /* + * If --instance option was passed, construct paths for backup data and + * xlog files of this backup instance. + */ + if (instance_name) + { + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + + /* + * Ensure that requested backup instance exists. + * for all commands except init, which doesn't take this parameter + * and add-instance which creates new instance. + */ + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + { + if (access(backup_instance_path, F_OK) != 0) + elog(ERROR, "Instance '%s' does not exist in this backup catalog", + instance_name); + } + } + + /* + * Read options from env variables or from config file, + * unless we're going to set them via set-config. + */ + if (instance_name && backup_subcmd != SET_CONFIG_CMD) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + pgut_getopt_env(options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + pgut_readopt(path, options, ERROR, true); + } + + /* Initialize logger */ + init_logger(backup_path); + + /* + * We have read pgdata path from command line or from configuration file. + * Ensure that pgdata is an absolute path. + */ + if (pgdata != NULL && !is_absolute_path(pgdata)) + elog(ERROR, "-D, --pgdata must be an absolute path"); + +#if PG_VERSION_NUM >= 110000 + /* Check xlog-seg-size option */ + if (instance_name && + backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); +#endif + + /* Sanity check of --backup-id option */ + if (backup_id_string != NULL) + { + if (backup_subcmd != RESTORE_CMD && + backup_subcmd != VALIDATE_CMD && + backup_subcmd != DELETE_CMD && + backup_subcmd != MERGE_CMD && + backup_subcmd != SHOW_CMD) + elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", + command_name); + + current.backup_id = base36dec(backup_id_string); + if (current.backup_id == 0) + elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); + } + + /* Setup stream options. They are used in streamutil.c. */ + if (host != NULL) + dbhost = pstrdup(host); + if (port != NULL) + dbport = pstrdup(port); + if (username != NULL) + dbuser = pstrdup(username); + + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = "pg_log"; + } + + if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) + { + /* parse all recovery target options into recovery_target_options structure */ + recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, + target_inclusive, target_tli, target_lsn, target_immediate, + target_name, target_action, restore_no_validate); + } + + if (num_threads < 1) + num_threads = 1; + + compress_init(); + + /* do actual operation */ + switch (backup_subcmd) + { + case ARCHIVE_PUSH_CMD: + return do_archive_push(wal_file_path, wal_file_name, file_overwrite); + case ARCHIVE_GET_CMD: + return do_archive_get(wal_file_path, wal_file_name); + case ADD_INSTANCE_CMD: + return do_add_instance(); + case DELETE_INSTANCE_CMD: + return do_delete_instance(); + case INIT_CMD: + return do_init(); + case BACKUP_CMD: + { + const char *backup_mode; + time_t start_time; + + start_time = time(NULL); + backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; + + elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", + PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, + stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + + return do_backup(start_time); + } + case RESTORE_CMD: + return do_restore_or_validate(current.backup_id, + recovery_target_options, + true); + case VALIDATE_CMD: + if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + return do_validate_all(); + else + return do_restore_or_validate(current.backup_id, + recovery_target_options, + false); + case SHOW_CMD: + return do_show(current.backup_id); + case DELETE_CMD: + if (delete_expired && backup_id_string) + elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + if (!delete_expired && !delete_wal && !backup_id_string) + elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); + if (delete_wal && !delete_expired && !backup_id_string) + return do_retention_purge(); + if (delete_expired) + return do_retention_purge(); + else + return do_delete(current.backup_id); + case MERGE_CMD: + do_merge(current.backup_id); + break; + case SHOW_CONFIG_CMD: + return do_configure(true); + case SET_CONFIG_CMD: + return do_configure(false); + case NO_CMD: + /* Should not happen */ + elog(ERROR, "Unknown subcommand"); + } + + return 0; +} + +static void +opt_backup_mode(pgut_option *opt, const char *arg) +{ + current.backup_mode = parse_backup_mode(arg); +} + +static void +opt_log_level_console(pgut_option *opt, const char *arg) +{ + log_level_console = parse_log_level(arg); +} + +static void +opt_log_level_file(pgut_option *opt, const char *arg) +{ + log_level_file = parse_log_level(arg); +} + +static void +opt_show_format(pgut_option *opt, const char *arg) +{ + const char *v = arg; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0) + { + if (pg_strncasecmp("plain", v, len) == 0) + show_format = SHOW_PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + show_format = SHOW_JSON; + else + elog(ERROR, "Invalid show format \"%s\"", arg); + } + else + elog(ERROR, "Invalid show format \"%s\"", arg); +} + +static void +opt_compress_alg(pgut_option *opt, const char *arg) +{ + compress_alg = parse_compress_alg(arg); +} + +/* + * Initialize compress and sanity checks for compress. + */ +static void +compress_init(void) +{ + /* Default algorithm is zlib */ + if (compress_shortcut) + compress_alg = ZLIB_COMPRESS; + + if (backup_subcmd != SET_CONFIG_CMD) + { + if (compress_level != COMPRESS_LEVEL_DEFAULT + && compress_alg == NOT_DEFINED_COMPRESS) + elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + } + + if (compress_level < 0 || compress_level > 9) + elog(ERROR, "--compress-level value must be in the range from 0 to 9"); + + if (compress_level == 0) + compress_alg = NOT_DEFINED_COMPRESS; + + if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + { +#ifndef HAVE_LIBZ + if (compress_alg == ZLIB_COMPRESS) + elog(ERROR, "This build does not support zlib compression"); + else +#endif + if (compress_alg == PGLZ_COMPRESS && num_threads > 1) + elog(ERROR, "Multithread backup does not support pglz compression"); + } +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h new file mode 100644 index 000000000..8f3a0fea1 --- /dev/null +++ b/src/pg_probackup.h @@ -0,0 +1,620 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup.h: Backup/Recovery manager for PostgreSQL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_H +#define PG_PROBACKUP_H + +#include "postgres_fe.h" + +#include +#include + +#include "access/timeline.h" +#include "access/xlogdefs.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "utils/pg_crc.h" +#include "common/relpath.h" +#include "port.h" + +#ifdef FRONTEND +#undef FRONTEND + #include "port/atomics.h" +#define FRONTEND +#endif + +#include "utils/parray.h" +#include "utils/pgut.h" + +#include "datapagemap.h" + +# define PG_STOP_BACKUP_TIMEOUT 300 +/* + * Macro needed to parse ptrack. + * NOTE Keep those values syncronised with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + +/* Directory/File names */ +#define DATABASE_DIR "database" +#define BACKUPS_DIR "backups" +#if PG_VERSION_NUM >= 100000 +#define PG_XLOG_DIR "pg_wal" +#else +#define PG_XLOG_DIR "pg_xlog" +#endif +#define PG_TBLSPC_DIR "pg_tblspc" +#define PG_GLOBAL_DIR "global" +#define BACKUP_CONTROL_FILE "backup.control" +#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" +#define BACKUP_CATALOG_PID "pg_probackup.pid" +#define DATABASE_FILE_LIST "backup_content.control" +#define PG_BACKUP_LABEL_FILE "backup_label" +#define PG_BLACK_LIST "black_list" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" + +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" +/* Direcotry/File permission */ +#define DIR_PERMISSION (0700) +#define FILE_PERMISSION (0600) + +/* 64-bit xid support for PGPRO_EE */ +#ifndef PGPRO_EE +#define XID_FMT "%u" +#endif + +typedef enum CompressAlg +{ + NOT_DEFINED_COMPRESS = 0, + NONE_COMPRESS, + PGLZ_COMPRESS, + ZLIB_COMPRESS, +} CompressAlg; + +/* Information about single file (or dir) in backup */ +typedef struct pgFile +{ + char *name; /* file or directory name */ + mode_t mode; /* protection (file type and permission) */ + size_t size; /* size of the file */ + size_t read_size; /* size of the portion read (if only some pages are + backed up, it's different from size) */ + int64 write_size; /* size of the backed-up file. BYTES_INVALID means + that the file existed but was not backed up + because not modified since last backup. */ + /* we need int64 here to store '-1' value */ + pg_crc32 crc; /* CRC value of the file, regular file only */ + char *linked; /* path of the linked file */ + bool is_datafile; /* true if the file is PostgreSQL data file */ + char *path; /* absolute path of the file */ + Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ + Oid dbOid; /* dbOid extracted from path, if applicable */ + Oid relOid; /* relOid extracted from path, if applicable */ + char *forkName; /* forkName extracted from path, if applicable */ + int segno; /* Segment number for ptrack */ + int n_blocks; /* size of the file in blocks, readed during DELTA backup */ + bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + bool is_database; + bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ +} pgFile; + +/* Special values of datapagemap_t bitmapsize */ +#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ + +/* Current state of backup */ +typedef enum BackupStatus +{ + BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ + BACKUP_STATUS_OK, /* completed backup */ + BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ + BACKUP_STATUS_RUNNING, /* running backup */ + BACKUP_STATUS_MERGING, /* merging backups */ + BACKUP_STATUS_DELETING, /* data files are being deleted */ + BACKUP_STATUS_DELETED, /* data files have been deleted */ + BACKUP_STATUS_DONE, /* completed but not validated yet */ + BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ +} BackupStatus; + +typedef enum BackupMode +{ + BACKUP_MODE_INVALID = 0, + BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ + BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ + BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ + BACKUP_MODE_FULL /* full backup */ +} BackupMode; + +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SHOW_CONFIG_CMD +} ProbackupSubcmd; + +typedef enum ShowFormat +{ + SHOW_PLAIN, + SHOW_JSON +} ShowFormat; + + +/* special values of pgBackup fields */ +#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ +#define BYTES_INVALID (-1) +#define BLOCKNUM_INVALID (-1) + +typedef struct pgBackupConfig +{ + uint64 system_identifier; + uint32 xlog_seg_size; + + char *pgdata; + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; + + const char *master_host; + const char *master_port; + const char *master_db; + const char *master_user; + int replica_timeout; + + int archive_timeout; + + int log_level_console; + int log_level_file; + char *log_filename; + char *error_log_filename; + char *log_directory; + int log_rotation_size; + int log_rotation_age; + + uint32 retention_redundancy; + uint32 retention_window; + + CompressAlg compress_alg; + int compress_level; +} pgBackupConfig; + + +/* Information about single backup stored in backup.conf */ + + +typedef struct pgBackup pgBackup; + +struct pgBackup +{ + BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ + time_t backup_id; /* Identifier of the backup. + * Currently it's the same as start_time */ + BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ + TimeLineID tli; /* timeline of start and stop baskup lsns */ + XLogRecPtr start_lsn; /* backup's starting transaction log location */ + XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ + time_t start_time; /* since this moment backup has status + * BACKUP_STATUS_RUNNING */ + time_t end_time; /* the moment when backup was finished, or the moment + * when we realized that backup is broken */ + time_t recovery_time; /* Earliest moment for which you can restore + * the state of the database cluster using + * this backup */ + TransactionId recovery_xid; /* Earliest xid for which you can restore + * the state of the database cluster using + * this backup */ + /* + * Amount of raw data. For a full backup, this is the total amount of + * data while for a differential backup this is just the difference + * of data taken. + * BYTES_INVALID means nothing was backed up. + */ + int64 data_bytes; + /* Size of WAL files in archive needed to restore this backup */ + int64 wal_bytes; + + CompressAlg compress_alg; + int compress_level; + + /* Fields needed for compatibility check */ + uint32 block_size; + uint32 wal_block_size; + uint32 checksum_version; + + char program_version[100]; + char server_version[100]; + + bool stream; /* Was this backup taken in stream mode? + * i.e. does it include all needed WAL files? */ + bool from_replica; /* Was this backup taken from replica */ + time_t parent_backup; /* Identifier of the previous backup. + * Which is basic backup for this + * incremental backup. */ + pgBackup *parent_backup_link; + char *primary_conninfo; /* Connection parameters of the backup + * in the format suitable for recovery.conf */ +}; + +/* Recovery target for restore and validate subcommands */ +typedef struct pgRecoveryTarget +{ + bool time_specified; + time_t recovery_target_time; + /* add one more field in order to avoid deparsing recovery_target_time back */ + const char *target_time_string; + bool xid_specified; + TransactionId recovery_target_xid; + /* add one more field in order to avoid deparsing recovery_target_xid back */ + const char *target_xid_string; + bool lsn_specified; + XLogRecPtr recovery_target_lsn; + /* add one more field in order to avoid deparsing recovery_target_lsn back */ + const char *target_lsn_string; + TimeLineID recovery_target_tli; + bool recovery_target_inclusive; + bool inclusive_specified; + bool recovery_target_immediate; + const char *recovery_target_name; + const char *recovery_target_action; + bool restore_no_validate; +} pgRecoveryTarget; + +/* Union to ease operations on relation pages */ +typedef union DataPage +{ + PageHeaderData page_data; + char data[BLCKSZ]; +} DataPage; + +typedef struct +{ + const char *from_root; + const char *to_root; + + parray *files_list; + parray *prev_filelist; + XLogRecPtr prev_start_lsn; + + PGconn *backup_conn; + PGcancel *cancel_conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} backup_files_arg; + +/* + * return pointer that exceeds the length of prefix from character string. + * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". + */ +#define GetRelativePath(str, prefix) \ + ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) + +/* + * Return timeline, xlog ID and record offset from an LSN of the type + * 0/B000188, usual result from pg_stop_backup() and friends. + */ +#define XLogDataFromLSN(data, xlogid, xrecoff) \ + sscanf(data, "%X/%X", xlogid, xrecoff) + +#define IsCompressedXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) + +#if PG_VERSION_NUM >= 110000 +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#else +#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteToSeg(xlrp, logSegNo) +#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ + XLogSegNoOffsetToRecPtr(segno, offset, dest) +#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFileName(fname, tli, logSegNo) +#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ + XLByteInSeg(xlrp, logSegNo) +#endif + +/* directory options */ +extern char *backup_path; +extern char backup_instance_path[MAXPGPATH]; +extern char *pgdata; +extern char arclog_path[MAXPGPATH]; + +/* common options */ +extern int num_threads; +extern bool stream_wal; +extern bool progress; +#if PG_VERSION_NUM >= 100000 +/* In pre-10 'replication_slot' is defined in receivelog.h */ +extern char *replication_slot; +#endif + +/* backup options */ +extern bool smooth_checkpoint; +#define ARCHIVE_TIMEOUT_DEFAULT 300 +extern uint32 archive_timeout; +extern bool is_remote_backup; +extern const char *master_db; +extern const char *master_host; +extern const char *master_port; +extern const char *master_user; +#define REPLICA_TIMEOUT_DEFAULT 300 +extern uint32 replica_timeout; + +extern bool is_ptrack_support; +extern bool is_checksum_enabled; +extern bool exclusive_backup; + +/* restore options */ +extern bool restore_as_replica; + +/* delete options */ +extern bool delete_wal; +extern bool delete_expired; +extern bool apply_to_all; +extern bool force_delete; + +/* retention options. 0 disables the option */ +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + +extern uint32 retention_redundancy; +extern uint32 retention_window; + +/* compression options */ +extern CompressAlg compress_alg; +extern int compress_level; +extern bool compress_shortcut; + +#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS +#define COMPRESS_LEVEL_DEFAULT 1 + +extern CompressAlg parse_compress_alg(const char *arg); +extern const char* deparse_compress_alg(int alg); +/* other options */ +extern char *instance_name; +extern uint64 system_identifier; +extern uint32 xlog_seg_size; + +/* show options */ +extern ShowFormat show_format; + +/* current settings */ +extern pgBackup current; +extern ProbackupSubcmd backup_subcmd; + +/* in dir.c */ +/* exclude directory list for $PGDATA file listing */ +extern const char *pgdata_exclude_dir[]; + +/* in backup.c */ +extern int do_backup(time_t start_time); +extern BackupMode parse_backup_mode(const char *value); +extern const char *deparse_backup_mode(BackupMode mode); +extern void process_block_change(ForkNumber forknum, RelFileNode rnode, + BlockNumber blkno); + +extern char *pg_ptrack_get_block(backup_files_arg *arguments, + Oid dbOid, Oid tblsOid, Oid relOid, + BlockNumber blknum, + size_t *result_size); +/* in restore.c */ +extern int do_restore_or_validate(time_t target_backup_id, + pgRecoveryTarget *rt, + bool is_restore); +extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_recovery_target(const pgBackup *backup, + const pgRecoveryTarget *rt); +extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); +extern pgRecoveryTarget *parseRecoveryTargetOptions( + const char *target_time, const char *target_xid, + const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + bool target_immediate, const char *target_name, + const char *target_action, bool restore_no_validate); + +/* in merge.c */ +extern void do_merge(time_t backup_id); + +/* in init.c */ +extern int do_init(void); +extern int do_add_instance(void); + +/* in archive.c */ +extern int do_archive_push(char *wal_file_path, char *wal_file_name, + bool overwrite); +extern int do_archive_get(char *wal_file_path, char *wal_file_name); + + +/* in configure.c */ +extern int do_configure(bool show_only); +extern void pgBackupConfigInit(pgBackupConfig *config); +extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); +extern void writeBackupCatalogConfigFile(pgBackupConfig *config); +extern pgBackupConfig* readBackupCatalogConfigFile(void); + +extern uint32 get_config_xlog_seg_size(void); + +/* in show.c */ +extern int do_show(time_t requested_backup_id); + +/* in delete.c */ +extern int do_delete(time_t backup_id); +extern int do_retention_purge(void); +extern int do_delete_instance(void); + +/* in fetch.c */ +extern char *slurpFile(const char *datadir, + const char *path, + size_t *filesize, + bool safe); +extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); + +/* in help.c */ +extern void help_pg_probackup(void); +extern void help_command(char *command); + +/* in validate.c */ +extern void pgBackupValidate(pgBackup* backup); +extern int do_validate_all(void); + +/* in catalog.c */ +extern pgBackup *read_backup(time_t timestamp); +extern const char *pgBackupGetBackupMode(pgBackup *backup); + +extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern pgBackup *catalog_get_last_data_backup(parray *backup_list, + TimeLineID tli); +extern void catalog_lock(void); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteBackupControlFile(pgBackup *backup); +extern void pgBackupWriteFileList(pgBackup *backup, parray *files, + const char *root); + +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); +extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupInit(pgBackup *backup); +extern void pgBackupCopy(pgBackup *dst, pgBackup *src); +extern void pgBackupFree(void *backup); +extern int pgBackupCompareId(const void *f1, const void *f2); +extern int pgBackupCompareIdDesc(const void *f1, const void *f2); + +extern pgBackup* find_parent_backup(pgBackup *current_backup); + +/* in dir.c */ +extern void dir_list_file(parray *files, const char *root, bool exclude, + bool omit_symlink, bool add_root); +extern void create_data_directories(const char *data_dir, + const char *backup_dir, + bool extract_tablespaces); + +extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void check_tablespace_mapping(pgBackup *backup); + +extern void print_file_list(FILE *out, const parray *files, const char *root); +extern parray *dir_read_file_list(const char *root, const char *file_txt); + +extern int dir_create_dir(const char *path, mode_t mode); +extern bool dir_is_empty(const char *path); + +extern bool fileExists(const char *path); +extern size_t pgFileSize(const char *path); + +extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileInit(const char *path); +extern void pgFileDelete(pgFile *file); +extern void pgFileFree(void *file); +extern pg_crc32 pgFileGetCRC(const char *file_path); +extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileComparePathDesc(const void *f1, const void *f2); +extern int pgFileCompareLinked(const void *f1, const void *f2); +extern int pgFileCompareSize(const void *f1, const void *f2); + +/* in data.c */ +extern bool backup_data_file(backup_files_arg* arguments, + const char *to_path, pgFile *file, + XLogRecPtr prev_backup_start_lsn, + BackupMode backup_mode, + CompressAlg calg, int clevel); +extern void restore_data_file(const char *to_path, + pgFile *file, bool allow_truncate, + bool write_header); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern void move_file(const char *from_root, const char *to_root, pgFile *file); +extern void push_wal_file(const char *from_path, const char *to_path, + bool is_compress, bool overwrite); +extern void get_wal_file(const char *from_path, const char *to_path); + +extern bool calc_file_checksum(pgFile *file); + +/* parsexlog.c */ +extern void extractPageMap(const char *datadir, + TimeLineID tli, uint32 seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool prev_seg, parray *backup_files_list); +extern void validate_wal(pgBackup *backup, + const char *archivedir, + time_t target_time, + TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 seg_size); +extern bool read_recovery_info(const char *archivedir, TimeLineID tli, + uint32 seg_size, + XLogRecPtr start_lsn, XLogRecPtr stop_lsn, + time_t *recovery_time, + TransactionId *recovery_xid); +extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, + TimeLineID target_tli, uint32 seg_size); + +/* in util.c */ +extern TimeLineID get_current_timeline(bool safe); +extern void sanityChecks(void); +extern void time2iso(char *buf, size_t len, time_t time); +extern const char *status2str(BackupStatus status); +extern void remove_trailing_space(char *buf, int comment_mark); +extern void remove_not_digit(char *buf, size_t len, const char *str); +extern uint32 get_data_checksum_version(bool safe); +extern const char *base36enc(long unsigned int value); +extern char *base36enc_dup(long unsigned int value); +extern long unsigned int base36dec(const char *text); +extern uint64 get_system_identifier(char *pgdata); +extern uint64 get_remote_system_identifier(PGconn *conn); +extern uint32 get_xlog_seg_size(char *pgdata_path); +extern pg_time_t timestamptz_to_time_t(TimestampTz t); +extern int parse_server_version(char *server_version_str); + +/* in status.c */ +extern bool is_pg_running(void); + +#ifdef WIN32 +#ifdef _DEBUG +#define lseek _lseek +#define open _open +#define fstat _fstat +#define read _read +#define close _close +#define write _write +#define mkdir(dir,mode) _mkdir(dir) +#endif +#endif + +#endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c new file mode 100644 index 000000000..acd794c8b --- /dev/null +++ b/src/restore.c @@ -0,0 +1,920 @@ +/*------------------------------------------------------------------------- + * + * restore.c: restore DB cluster and archived WAL. + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include +#include + +#include "catalog/pg_control.h" +#include "utils/logger.h" +#include "utils/thread.h" + +typedef struct +{ + parray *files; + pgBackup *backup; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} restore_files_arg; + +static void restore_backup(pgBackup *backup); +static void create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup); +static void *restore_files(void *arg); +static void remove_deleted_files(pgBackup *backup); + + +/* + * Entry point of pg_probackup RESTORE and VALIDATE subcommands. + */ +int +do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, + bool is_restore) +{ + int i = 0; + parray *backups; + pgBackup *current_backup = NULL; + pgBackup *dest_backup = NULL; + pgBackup *base_full_backup = NULL; + pgBackup *corrupted_backup = NULL; + int dest_backup_index = 0; + int base_full_backup_index = 0; + int corrupted_backup_index = 0; + char *action = is_restore ? "Restore":"Validate"; + + if (is_restore) + { + if (pgdata == NULL) + elog(ERROR, + "required parameter not specified: PGDATA (-D, --pgdata)"); + /* Check if restore destination empty */ + if (!dir_is_empty(pgdata)) + elog(ERROR, "restore destination is not empty: \"%s\"", pgdata); + } + + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + + elog(LOG, "%s begin.", action); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Find backup range we should restore or validate. */ + while ((i < parray_num(backups)) && !dest_backup) + { + current_backup = (pgBackup *) parray_get(backups, i); + i++; + + /* Skip all backups which started after target backup */ + if (target_backup_id && current_backup->start_time > target_backup_id) + continue; + + /* + * [PGPRO-1164] If BACKUP_ID is not provided for restore command, + * we must find the first valid(!) backup. + */ + + if (is_restore && + target_backup_id == INVALID_BACKUP_ID && + current_backup->status != BACKUP_STATUS_OK) + { + elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + continue; + } + + /* + * We found target backup. Check its status and + * ensure that it satisfies recovery target. + */ + if ((target_backup_id == current_backup->start_time + || target_backup_id == INVALID_BACKUP_ID)) + { + + /* backup is not ok, + * but in case of CORRUPT, ORPHAN or DONE revalidation can be done, + * in other cases throw an error. + */ + if (current_backup->status != BACKUP_STATUS_OK) + { + if (current_backup->status == BACKUP_STATUS_DONE || + current_backup->status == BACKUP_STATUS_ORPHAN || + current_backup->status == BACKUP_STATUS_CORRUPT) + elog(WARNING, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(current_backup->start_time), status2str(current_backup->status)); + } + + if (rt->recovery_target_tli) + { + parray *timelines; + + elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); + /* Read timeline history files from archives */ + timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + + if (!satisfy_timeline(timelines, current_backup)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy target timeline", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target timeline */ + continue; + } + } + + if (!satisfy_recovery_target(current_backup, rt)) + { + if (target_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "target backup %s does not satisfy restore options", + base36enc(target_backup_id)); + else + /* Try to find another backup that satisfies target options */ + continue; + } + + /* + * Backup is fine and satisfies all recovery options. + * Save it as dest_backup + */ + dest_backup = current_backup; + dest_backup_index = i-1; + } + } + + if (dest_backup == NULL) + elog(ERROR, "Backup satisfying target options is not found."); + + /* If we already found dest_backup, look for full backup. */ + if (dest_backup) + { + base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* + * We have found full backup by link, + * now we need to walk the list to find its index. + * + * TODO I think we should rewrite it someday to use double linked list + * and avoid relying on sort order anymore. + */ + for (i = dest_backup_index; i < parray_num(backups); i++) + { + pgBackup * temp_backup = (pgBackup *) parray_get(backups, i); + if (temp_backup->start_time == base_full_backup->start_time) + { + base_full_backup_index = i; + break; + } + } + } + + if (base_full_backup == NULL) + elog(ERROR, "Full backup satisfying target options is not found."); + + /* + * Ensure that directories provided in tablespace mapping are valid + * i.e. empty or not exist. + */ + if (is_restore) + check_tablespace_mapping(dest_backup); + + if (!is_restore || !rt->restore_no_validate) + { + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + + /* + * Validate backups from base_full_backup to dest_backup. + */ + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + pgBackupValidate(backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (backup->status == BACKUP_STATUS_CORRUPT) + { + corrupted_backup = backup; + corrupted_backup_index = i; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ + } + /* There is no point in wal validation + * if there is corrupted backup between base_backup and dest_backup + */ + if (!corrupted_backup) + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->recovery_target_time, + rt->recovery_target_xid, rt->recovery_target_lsn, + base_full_backup->tli, xlog_seg_size); + + /* Set every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (corrupted_backup) + { + for (i = corrupted_backup_index - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + /* Mark incremental OK backup as orphan */ + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + char *backup_id, + *corrupted_backup_id; + + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + backup_id = base36enc_dup(backup->start_time); + corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + backup_id, corrupted_backup_id); + + free(backup_id); + free(corrupted_backup_id); + } + } + } + } + + /* + * If dest backup is corrupted or was orphaned in previous check + * produce corresponding error message + */ + if (dest_backup->status == BACKUP_STATUS_OK) + { + if (rt->restore_no_validate) + elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + else + elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + } + else if (dest_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + else if (dest_backup->status == BACKUP_STATUS_ORPHAN) + elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); + + /* We ensured that all backups are valid, now restore if required */ + if (is_restore) + { + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), dest_backup->server_version); + + restore_backup(backup); + } + + /* + * Delete files which are not in dest backup file list. Files which were + * deleted between previous and current backup are not in the list. + */ + if (dest_backup->backup_mode != BACKUP_MODE_FULL) + remove_deleted_files(dest_backup); + + /* Create recovery.conf with given recovery target parameters */ + create_recovery_conf(target_backup_id, rt, dest_backup); + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); + + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + return 0; +} + +/* + * Restore one backup. + */ +void +restore_backup(pgBackup *backup) +{ + char timestamp[100]; + char this_backup_path[MAXPGPATH]; + char database_path[MAXPGPATH]; + char list_path[MAXPGPATH]; + parray *files; + int i; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg *threads_args; + bool restore_isok = true; + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + elog(LOG, "restoring database from backup %s", timestamp); + + /* + * Restore backup directories. + * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id + */ + pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + create_data_directories(pgdata, this_backup_path, true); + + /* + * Get list of files which need to be restored. + */ + pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); + pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); + files = dir_read_file_list(database_path, list_path); + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pg_atomic_clear_flag(&file->lock); + } + + /* Restore files into target directory */ + for (i = 0; i < num_threads; i++) + { + restore_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->backup = backup; + /* By default there are some error */ + threads_args[i].ret = 1; + + elog(LOG, "Start thread for num:%li", parray_num(files)); + + pthread_create(&threads[i], NULL, restore_files, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + restore_isok = false; + } + if (!restore_isok) + elog(ERROR, "Data files restoring failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); +} + +/* + * Delete files which are not in backup's file list from target pgdata. + * It is necessary to restore incremental backup correctly. + * Files which were deleted between previous and current backup + * are not in the backup's filelist. + */ +static void +remove_deleted_files(pgBackup *backup) +{ + parray *files; + parray *files_restored; + char filelist_path[MAXPGPATH]; + int i; + + pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); + /* Read backup's filelist using target database path as base path */ + files = dir_read_file_list(pgdata, filelist_path); + parray_qsort(files, pgFileComparePathDesc); + + /* Get list of files actually existing in target database */ + files_restored = parray_new(); + dir_list_file(files_restored, pgdata, true, true, false); + /* To delete from leaf, sort in reversed order */ + parray_qsort(files_restored, pgFileComparePathDesc); + + for (i = 0; i < parray_num(files_restored); i++) + { + pgFile *file = (pgFile *) parray_get(files_restored, i); + + /* If the file is not in the file list, delete it */ + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + { + pgFileDelete(file); + if (log_level_console <= LOG || log_level_file <= LOG) + elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); + } + } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + parray_walk(files_restored, pgFileFree); + parray_free(files_restored); +} + +/* + * Restore files into $PGDATA. + */ +static void * +restore_files(void *arg) +{ + int i; + restore_files_arg *arguments = (restore_files_arg *)arg; + + for (i = 0; i < parray_num(arguments->files); i++) + { + char from_root[MAXPGPATH]; + char *rel_path; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + pgBackupGetPath(arguments->backup, from_root, + lengthof(from_root), DATABASE_DIR); + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during restore database"); + + rel_path = GetRelativePath(file->path,from_root); + + if (progress) + elog(LOG, "Progress: (%d/%lu). Process file %s ", + i + 1, (unsigned long) parray_num(arguments->files), rel_path); + + /* + * For PAGE and PTRACK backups skip files which haven't changed + * since previous backup and thus were not backed up. + * We cannot do the same when restoring DELTA backup because we need information + * about every file to correctly truncate them. + */ + if (file->write_size == BYTES_INVALID && + (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE + || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) + { + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + continue; + } + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + { + elog(VERBOSE, "directory, skip"); + continue; + } + + /* Do not restore tablespace_map file */ + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) + { + elog(VERBOSE, "skip tablespace_map"); + continue; + } + + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + file->path, file->is_datafile?1:0, file->is_cfs?1:0); + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, pgdata, + file->path + strlen(from_root) + 1); + restore_data_file(to_path, file, + arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, + false); + } + else + copy_file(from_root, pgdata, file); + + /* print size of restored file */ + if (file->write_size != BYTES_INVALID) + elog(LOG, "Restored file %s : " INT64_FORMAT " bytes", + file->path, file->write_size); + } + + /* Data files restoring is successful */ + arguments->ret = 0; + + return NULL; +} + +/* Create recovery.conf with given recovery target parameters */ +static void +create_recovery_conf(time_t backup_id, + pgRecoveryTarget *rt, + pgBackup *backup) +{ + char path[MAXPGPATH]; + FILE *fp; + bool need_restore_conf = false; + + if (!backup->stream + || (rt->time_specified || rt->xid_specified)) + need_restore_conf = true; + + /* No need to generate recovery.conf at all. */ + if (!(need_restore_conf || restore_as_replica)) + return; + + elog(LOG, "----------------------------------------"); + elog(LOG, "creating recovery.conf"); + + snprintf(path, lengthof(path), "%s/recovery.conf", pgdata); + fp = fopen(path, "wt"); + if (fp == NULL) + elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, + strerror(errno)); + + fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (need_restore_conf) + { + + fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " + "--wal-file-path %%p --wal-file-name %%f'\n", + PROGRAM_NAME, backup_path, instance_name); + + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->recovery_target_name) + fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + + if (rt->time_specified) + fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + + if (rt->xid_specified) + fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + + if (rt->recovery_target_lsn) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + + if (rt->recovery_target_immediate) + fprintf(fp, "recovery_target = 'immediate'\n"); + + if (rt->inclusive_specified) + fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->recovery_target_inclusive?"true":"false"); + + if (rt->recovery_target_tli) + fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + + if (rt->recovery_target_action) + fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + } + + if (restore_as_replica) + { + fprintf(fp, "standby_mode = 'on'\n"); + + if (backup->primary_conninfo) + fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + } + + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, + strerror(errno)); +} + +/* + * Try to read a timeline's history file. + * + * If successful, return the list of component TLIs (the ancestor + * timelines followed by target timeline). If we cannot find the history file, + * assume that the timeline has no parents, and return a list of just the + * specified timeline ID. + * based on readTimeLineHistory() in timeline.c + */ +parray * +readTimeLineHistory_probackup(TimeLineID targetTLI) +{ + parray *result; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd = NULL; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + + /* Look for timeline history file in archlog_path */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + targetTLI); + + /* Timeline 1 does not have a history file */ + if (targetTLI != 1) + { + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* There is no history file for target timeline */ + elog(ERROR, "recovery target timeline %u does not exist", + targetTLI); + } + } + + result = parray_new(); + + /* + * Parse the file... + */ + while (fd && fgets(fline, sizeof(fline), fd) != NULL) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = fline; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(fline, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, + "syntax error in history file: %s. Expected a numeric timeline ID.", + fline); + } + if (nfields != 3) + elog(ERROR, + "syntax error in history file: %s. Expected a transaction log switchpoint location.", + fline); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, + "Timeline IDs must be in increasing sequence."); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + parray_insert(result, 0, entry); + + /* we ignore the remainder of each line */ + } + + if (fd) + fclose(fd); + + if (last_timeline && targetTLI <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + + /* append target timeline */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = targetTLI; + /* LSN in target timeline is valid */ + /* TODO ensure that -1UL --> -1L fix is correct */ + entry->end = (uint32) (-1L << 32) | -1L; + parray_insert(result, 0, entry); + + return result; +} + +bool +satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) +{ + if (rt->xid_specified) + return backup->recovery_xid <= rt->recovery_target_xid; + + if (rt->time_specified) + return backup->recovery_time <= rt->recovery_target_time; + + if (rt->lsn_specified) + return backup->stop_lsn <= rt->recovery_target_lsn; + + return true; +} + +bool +satisfy_timeline(const parray *timelines, const pgBackup *backup) +{ + int i; + + for (i = 0; i < parray_num(timelines); i++) + { + TimeLineHistoryEntry *timeline; + + timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + if (backup->tli == timeline->tli && + backup->stop_lsn < timeline->end) + return true; + } + return false; +} +/* + * Get recovery options in the string format, parse them + * and fill up the pgRecoveryTarget structure. + */ +pgRecoveryTarget * +parseRecoveryTargetOptions(const char *target_time, + const char *target_xid, + const char *target_inclusive, + TimeLineID target_tli, + const char *target_lsn, + bool target_immediate, + const char *target_name, + const char *target_action, + bool restore_no_validate) +{ + time_t dummy_time; + TransactionId dummy_xid; + bool dummy_bool; + XLogRecPtr dummy_lsn; + /* + * count the number of the mutually exclusive options which may specify + * recovery target. If final value > 1, throw an error. + */ + int recovery_target_specified = 0; + pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); + + /* fill all options with default values */ + rt->time_specified = false; + rt->xid_specified = false; + rt->inclusive_specified = false; + rt->lsn_specified = false; + rt->recovery_target_time = 0; + rt->recovery_target_xid = 0; + rt->recovery_target_lsn = InvalidXLogRecPtr; + rt->target_time_string = NULL; + rt->target_xid_string = NULL; + rt->target_lsn_string = NULL; + rt->recovery_target_inclusive = false; + rt->recovery_target_tli = 0; + rt->recovery_target_immediate = false; + rt->recovery_target_name = NULL; + rt->recovery_target_action = NULL; + rt->restore_no_validate = false; + + /* parse given options */ + if (target_time) + { + recovery_target_specified++; + rt->time_specified = true; + rt->target_time_string = target_time; + + if (parse_time(target_time, &dummy_time, false)) + rt->recovery_target_time = dummy_time; + else + elog(ERROR, "Invalid value of --time option %s", target_time); + } + + if (target_xid) + { + recovery_target_specified++; + rt->xid_specified = true; + rt->target_xid_string = target_xid; + +#ifdef PGPRO_EE + if (parse_uint64(target_xid, &dummy_xid, 0)) +#else + if (parse_uint32(target_xid, &dummy_xid, 0)) +#endif + rt->recovery_target_xid = dummy_xid; + else + elog(ERROR, "Invalid value of --xid option %s", target_xid); + } + + if (target_lsn) + { + recovery_target_specified++; + rt->lsn_specified = true; + rt->target_lsn_string = target_lsn; + if (parse_lsn(target_lsn, &dummy_lsn)) + rt->recovery_target_lsn = dummy_lsn; + else + elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + } + + if (target_inclusive) + { + rt->inclusive_specified = true; + if (parse_bool(target_inclusive, &dummy_bool)) + rt->recovery_target_inclusive = dummy_bool; + else + elog(ERROR, "Invalid value of --inclusive option %s", target_inclusive); + } + + rt->recovery_target_tli = target_tli; + if (target_immediate) + { + recovery_target_specified++; + rt->recovery_target_immediate = target_immediate; + } + + if (restore_no_validate) + { + rt->restore_no_validate = restore_no_validate; + } + + if (target_name) + { + recovery_target_specified++; + rt->recovery_target_name = target_name; + } + + if (target_action) + { + rt->recovery_target_action = target_action; + + if ((strcmp(target_action, "pause") != 0) + && (strcmp(target_action, "promote") != 0) + && (strcmp(target_action, "shutdown") != 0)) + elog(ERROR, "Invalid value of --recovery-target-action option %s", target_action); + } + else + { + /* Default recovery target action is pause */ + rt->recovery_target_action = "pause"; + } + + /* More than one mutually exclusive option was defined. */ + if (recovery_target_specified > 1) + elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); + + /* If none of the options is defined, '--inclusive' option is meaningless */ + if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) + elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); + + return rt; +} diff --git a/src/show.c b/src/show.c new file mode 100644 index 000000000..f240ce933 --- /dev/null +++ b/src/show.c @@ -0,0 +1,500 @@ +/*------------------------------------------------------------------------- + * + * show.c: show backup information. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include +#include + +#include "pqexpbuffer.h" + +#include "utils/json.h" + + +static void show_instance_start(void); +static void show_instance_end(void); +static void show_instance(time_t requested_backup_id, bool show_name); +static int show_backup(time_t requested_backup_id); + +static void show_instance_plain(parray *backup_list, bool show_name); +static void show_instance_json(parray *backup_list); + +static PQExpBufferData show_buf; +static bool first_instance = true; +static int32 json_level = 0; + +int +do_show(time_t requested_backup_id) +{ + if (instance_name == NULL && + requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You must specify --instance to use --backup_id option"); + + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); + + show_instance_start(); + + while (errno = 0, (dent = readdir(dir)) != NULL) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + + show_instance(INVALID_BACKUP_ID, true); + } + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + show_instance_end(); + + return 0; + } + else if (requested_backup_id == INVALID_BACKUP_ID || + show_format == SHOW_JSON) + { + show_instance_start(); + show_instance(requested_backup_id, false); + show_instance_end(); + + return 0; + } + else + return show_backup(requested_backup_id); +} + +static void +pretty_size(int64 size, char *buf, size_t len) +{ + int exp = 0; + + /* minus means the size is invalid */ + if (size < 0) + { + strncpy(buf, "----", len); + return; + } + + /* determine postfix */ + while (size > 9999) + { + ++exp; + size /= 1000; + } + + switch (exp) + { + case 0: + snprintf(buf, len, "%dB", (int) size); + break; + case 1: + snprintf(buf, len, "%dkB", (int) size); + break; + case 2: + snprintf(buf, len, "%dMB", (int) size); + break; + case 3: + snprintf(buf, len, "%dGB", (int) size); + break; + case 4: + snprintf(buf, len, "%dTB", (int) size); + break; + case 5: + snprintf(buf, len, "%dPB", (int) size); + break; + default: + strncpy(buf, "***", len); + break; + } +} + +static TimeLineID +get_parent_tli(TimeLineID child_tli) +{ + TimeLineID result = 0; + char path[MAXPGPATH]; + char fline[MAXPGPATH]; + FILE *fd; + + /* Timeline 1 does not have a history file and parent timeline */ + if (child_tli == 1) + return 0; + + /* Search history file in archives */ + snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, + child_tli); + fd = fopen(path, "rt"); + if (fd == NULL) + { + if (errno != ENOENT) + elog(ERROR, "could not open file \"%s\": %s", path, + strerror(errno)); + + /* Did not find history file, do not raise the error */ + return 0; + } + + /* + * Parse the file... + */ + while (fgets(fline, sizeof(fline), fd) != NULL) + { + /* skip leading whitespace and check for # comment */ + char *ptr; + char *endptr; + + for (ptr = fline; *ptr; ptr++) + { + if (!IsSpace(*ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + /* expect a numeric timeline ID as first field of line */ + result = (TimeLineID) strtoul(ptr, &endptr, 0); + if (endptr == ptr) + elog(ERROR, + "syntax error(timeline ID) in history file: %s", + fline); + } + + fclose(fd); + + /* TLI of the last line is parent TLI */ + return result; +} + +/* + * Initialize instance visualization. + */ +static void +show_instance_start(void) +{ + initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + return; + + first_instance = true; + json_level = 0; + + appendPQExpBufferChar(&show_buf, '['); + json_level++; +} + +/* + * Finalize instance visualization. + */ +static void +show_instance_end(void) +{ + if (show_format == SHOW_JSON) + appendPQExpBufferStr(&show_buf, "\n]\n"); + + fputs(show_buf.data, stdout); + termPQExpBuffer(&show_buf); +} + +/* + * Show brief meta information about all backups in the backup instance. + */ +static void +show_instance(time_t requested_backup_id, bool show_name) +{ + parray *backup_list; + + backup_list = catalog_get_backup_list(requested_backup_id); + + if (show_format == SHOW_PLAIN) + show_instance_plain(backup_list, show_name); + else if (show_format == SHOW_JSON) + show_instance_json(backup_list); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* + * Show detailed meta information about specified backup. + */ +static int +show_backup(time_t requested_backup_id) +{ + pgBackup *backup; + + backup = read_backup(requested_backup_id); + if (backup == NULL) + { + elog(INFO, "Requested backup \"%s\" is not found.", + /* We do not need free base36enc's result, we exit anyway */ + base36enc(requested_backup_id)); + /* This is not error */ + return 0; + } + + if (show_format == SHOW_PLAIN) + pgBackupWriteControl(stdout, backup); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + /* cleanup */ + pgBackupFree(backup); + + return 0; +} + +/* + * Plain output. + */ + +/* + * Show instance backups in plain format. + */ +static void +show_instance_plain(parray *backup_list, bool show_name) +{ + int i; + + if (show_name) + printfPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); + + /* if you add new fields here, fix the header */ + /* show header */ + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + appendPQExpBufferStr(&show_buf, + " Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status \n"); + appendPQExpBufferStr(&show_buf, + "============================================================================================================================================\n"); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char duration[20] = "----"; + char data_bytes_str[10] = "----"; + + if (backup->recovery_time != (time_t) 0) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + if (backup->end_time != (time_t) 0) + snprintf(duration, lengthof(duration), "%.*lfs", 0, + difftime(backup->end_time, backup->start_time)); + + /* + * Calculate Data field, in the case of full backup this shows the + * total amount of data. For an differential backup, this size is only + * the difference of data accumulated. + */ + pretty_size(backup->data_bytes, data_bytes_str, + lengthof(data_bytes_str)); + + /* Get parent timeline before printing */ + parent_tli = get_parent_tli(backup->tli); + + appendPQExpBuffer(&show_buf, + " %-11s %-8s %-6s %-22s %-6s %-7s %3d / %-3d %5s %6s %2X/%-8X %2X/%-8X %-8s\n", + instance_name, + (backup->server_version[0] ? backup->server_version : "----"), + base36enc(backup->start_time), + timestamp, + pgBackupGetBackupMode(backup), + backup->stream ? "STREAM": "ARCHIVE", + backup->tli, + parent_tli, + duration, + data_bytes_str, + (uint32) (backup->start_lsn >> 32), + (uint32) backup->start_lsn, + (uint32) (backup->stop_lsn >> 32), + (uint32) backup->stop_lsn, + status2str(backup->status)); + } +} + +/* + * Json output. + */ + +/* + * Show instance backups in json format. + */ +static void +show_instance_json(parray *backup_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, false); + json_add_key(buf, "backups", json_level, true); + + /* + * List backups. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + if (i != 0) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + false); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level, true); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level, true); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level, true); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level, true); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level, true); + parent_tli = get_parent_tli(backup->tli); + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level, true); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level, true); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; +} diff --git a/src/status.c b/src/status.c new file mode 100644 index 000000000..155a07f40 --- /dev/null +++ b/src/status.c @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * status.c + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * + * Monitor status of a PostgreSQL server. + * + *------------------------------------------------------------------------- + */ + + +#include "postgres_fe.h" + +#include +#include +#include + +#include "pg_probackup.h" + +/* PID can be negative for standalone backend */ +typedef long pgpid_t; + +static pgpid_t get_pgpid(void); +static bool postmaster_is_alive(pid_t pid); + +/* + * get_pgpid + * + * Get PID of postmaster, by scanning postmaster.pid. + */ +static pgpid_t +get_pgpid(void) +{ + FILE *pidf; + long pid; + char pid_file[MAXPGPATH]; + + snprintf(pid_file, lengthof(pid_file), "%s/postmaster.pid", pgdata); + + pidf = fopen(pid_file, PG_BINARY_R); + if (pidf == NULL) + { + /* No pid file, not an error on startup */ + if (errno == ENOENT) + return 0; + else + { + elog(ERROR, "could not open PID file \"%s\": %s", + pid_file, strerror(errno)); + } + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + /* Is the file empty? */ + if (ftell(pidf) == 0 && feof(pidf)) + elog(ERROR, "the PID file \"%s\" is empty", + pid_file); + else + elog(ERROR, "invalid data in PID file \"%s\"\n", + pid_file); + } + fclose(pidf); + return (pgpid_t) pid; +} + +/* + * postmaster_is_alive + * + * Check whether postmaster is alive or not. + */ +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +/* + * is_pg_running + * + * + */ +bool +is_pg_running(void) +{ + pgpid_t pid; + + pid = get_pgpid(); + + /* 0 means no pid file */ + if (pid == 0) + return false; + + /* Case of a standalone backend */ + if (pid < 0) + pid = -pid; + + /* Check if postmaster is alive */ + return postmaster_is_alive((pid_t) pid); +} diff --git a/src/util.c b/src/util.c new file mode 100644 index 000000000..82814d11d --- /dev/null +++ b/src/util.c @@ -0,0 +1,349 @@ +/*------------------------------------------------------------------------- + * + * util.c: log messages to log file or stderr, and misc code. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include + +#include "storage/bufpage.h" +#if PG_VERSION_NUM >= 110000 +#include "streamutil.h" +#endif + +const char * +base36enc(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + static char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return &buffer[offset]; +} + +/* + * Same as base36enc(), but the result must be released by the user. + */ +char * +base36enc_dup(long unsigned int value) +{ + const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ + char buffer[14]; + unsigned int offset = sizeof(buffer); + + buffer[--offset] = '\0'; + do { + buffer[--offset] = base36[value % 36]; + } while (value /= 36); + + return strdup(&buffer[offset]); +} + +long unsigned int +base36dec(const char *text) +{ + return strtoul(text, NULL, 36); +} + +static void +checkControlFile(ControlFileData *ControlFile) +{ + pg_crc32c crc; + + /* Calculate CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); + + /* Then compare it */ + if (!EQ_CRC32C(crc, ControlFile->crc)) + elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n" + "Either the file is corrupt, or it has a different layout than this program\n" + "is expecting. The results below are untrustworthy."); + + if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + elog(ERROR, "possible byte ordering mismatch\n" + "The byte ordering used to store the pg_control file might not match the one\n" + "used by this program. In that case the results below would be incorrect, and\n" + "the PostgreSQL installation would be incompatible with this data directory."); +} + +/* + * Verify control file contents in the buffer src, and copy it to *ControlFile. + */ +static void +digestControlFile(ControlFileData *ControlFile, char *src, size_t size) +{ +#if PG_VERSION_NUM >= 100000 + int ControlFileSize = PG_CONTROL_FILE_SIZE; +#else + int ControlFileSize = PG_CONTROL_SIZE; +#endif + + if (size != ControlFileSize) + elog(ERROR, "unexpected control file size %d, expected %d", + (int) size, ControlFileSize); + + memcpy(ControlFile, src, sizeof(ControlFileData)); + + /* Additional checks on control file */ + checkControlFile(ControlFile); +} + +/* + * Utility shared by backup and restore to fetch the current timeline + * used by a node. + */ +TimeLineID +get_current_timeline(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (safe && buffer == NULL) + return 0; + + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.checkPointCopy.ThisTimeLineID; +} + +uint64 +get_system_identifier(char *pgdata_path) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +} + +uint64 +get_remote_system_identifier(PGconn *conn) +{ +#if PG_VERSION_NUM >= 90600 + PGresult *res; + uint64 system_id_conn; + char *val; + + res = pgut_execute(conn, + "SELECT system_identifier FROM pg_catalog.pg_control_system()", + 0, NULL); + val = PQgetvalue(res, 0, 0); + if (!parse_uint64(val, &system_id_conn, 0)) + { + PQclear(res); + elog(ERROR, "%s is not system_identifier", val); + } + PQclear(res); + + return system_id_conn; +#else + char *buffer; + size_t size; + ControlFileData ControlFile; + + buffer = fetchFile(conn, "global/pg_control", &size); + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.system_identifier; +#endif +} + +uint32 +get_xlog_seg_size(char *pgdata_path) +{ +#if PG_VERSION_NUM >= 110000 + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.xlog_seg_size; +#else + return (uint32) XLOG_SEG_SIZE; +#endif +} + +uint32 +get_data_checksum_version(bool safe) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.data_checksum_version; +} + + +/* + * Convert time_t value to ISO-8601 format string. Always set timezone offset. + */ +void +time2iso(char *buf, size_t len, time_t time) +{ + struct tm *ptm = gmtime(&time); + time_t gmt = mktime(ptm); + time_t offset; + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) + { + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); + } +} + +/* copied from timestamp.c */ +pg_time_t +timestamptz_to_time_t(TimestampTz t) +{ + pg_time_t result; + +#ifdef HAVE_INT64_TIMESTAMP + result = (pg_time_t) (t / USECS_PER_SEC + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#else + result = (pg_time_t) (t + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); +#endif + return result; +} + +/* Parse string representation of the server version */ +int +parse_server_version(char *server_version_str) +{ + int nfields; + int result = 0; + int major_version = 0; + int minor_version = 0; + + nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); + if (nfields == 2) + { + /* Server version lower than 10 */ + if (major_version > 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000 + minor_version * 100; + } + else if (nfields == 1) + { + if (major_version < 10) + elog(ERROR, "Server version format doesn't match major version %d", major_version); + result = major_version * 10000; + } + else + elog(ERROR, "Unknown server version format"); + + return result; +} + +const char * +status2str(BackupStatus status) +{ + static const char *statusName[] = + { + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" + }; + if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) + return "UNKNOWN"; + + return statusName[status]; +} + +void +remove_trailing_space(char *buf, int comment_mark) +{ + int i; + char *last_char = NULL; + + for (i = 0; buf[i]; i++) + { + if (buf[i] == comment_mark || buf[i] == '\n' || buf[i] == '\r') + { + buf[i] = '\0'; + break; + } + } + for (i = 0; buf[i]; i++) + { + if (!isspace(buf[i])) + last_char = buf + i; + } + if (last_char != NULL) + *(last_char + 1) = '\0'; + +} + +void +remove_not_digit(char *buf, size_t len, const char *str) +{ + int i, j; + + for (i = 0, j = 0; str[i] && j < len; i++) + { + if (!isdigit(str[i])) + continue; + buf[j++] = str[i]; + } + buf[j] = '\0'; +} diff --git a/src/utils/json.c b/src/utils/json.c new file mode 100644 index 000000000..3afbe9e70 --- /dev/null +++ b/src/utils/json.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * json.c: - make json document. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "json.h" + +static void json_add_indent(PQExpBuffer buf, int32 level); +static void json_add_escaped(PQExpBuffer buf, const char *str); + +/* + * Start or end json token. Currently it is a json object or array. + * + * Function modifies level value and adds indent if it appropriate. + */ +void +json_add(PQExpBuffer buf, JsonToken type, int32 *level) +{ + switch (type) + { + case JT_BEGIN_ARRAY: + appendPQExpBufferChar(buf, '['); + *level += 1; + break; + case JT_END_ARRAY: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, ']'); + break; + case JT_BEGIN_OBJECT: + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '{'); + *level += 1; + break; + case JT_END_OBJECT: + *level -= 1; + if (*level == 0) + appendPQExpBufferChar(buf, '\n'); + else + json_add_indent(buf, *level); + appendPQExpBufferChar(buf, '}'); + break; + default: + break; + } +} + +/* + * Add json object's key. If it isn't first key we need to add a comma. + */ +void +json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +{ + if (add_comma) + appendPQExpBufferChar(buf, ','); + json_add_indent(buf, level); + + json_add_escaped(buf, name); + appendPQExpBufferStr(buf, ": "); +} + +/* + * Add json object's key and value. If it isn't first key we need to add a + * comma. + */ +void +json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma) +{ + json_add_key(buf, name, level, add_comma); + json_add_escaped(buf, value); +} + +static void +json_add_indent(PQExpBuffer buf, int32 level) +{ + uint16 i; + + if (level == 0) + return; + + appendPQExpBufferChar(buf, '\n'); + for (i = 0; i < level; i++) + appendPQExpBufferStr(buf, " "); +} + +static void +json_add_escaped(PQExpBuffer buf, const char *str) +{ + const char *p; + + appendPQExpBufferChar(buf, '"'); + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < ' ') + appendPQExpBuffer(buf, "\\u%04x", (int) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } + appendPQExpBufferChar(buf, '"'); +} diff --git a/src/utils/json.h b/src/utils/json.h new file mode 100644 index 000000000..cf5a70648 --- /dev/null +++ b/src/utils/json.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * json.h: - prototypes of json output functions. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_JSON_H +#define PROBACKUP_JSON_H + +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +/* + * Json document tokens. + */ +typedef enum +{ + JT_BEGIN_ARRAY, + JT_END_ARRAY, + JT_BEGIN_OBJECT, + JT_END_OBJECT +} JsonToken; + +extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, + bool add_comma); +extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, + int32 level, bool add_comma); + +#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c new file mode 100644 index 000000000..31669ed0b --- /dev/null +++ b/src/utils/logger.c @@ -0,0 +1,621 @@ +/*------------------------------------------------------------------------- + * + * logger.c: - log events into log file or stderr. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "pgut.h" +#include "pg_probackup.h" +#include "thread.h" + +/* Logger parameters */ + +int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; +int log_level_file = LOG_LEVEL_FILE_DEFAULT; + +char *log_filename = NULL; +char *error_log_filename = NULL; +char *log_directory = NULL; +/* + * If log_path is empty logging is not initialized. + * We will log only into stderr + */ +char log_path[MAXPGPATH] = ""; + +/* Maximum size of an individual log file in kilobytes */ +int log_rotation_size = 0; +/* Maximum lifetime of an individual log file in minutes */ +int log_rotation_age = 0; + +/* Implementation for logging.h */ + +typedef enum +{ + PG_DEBUG, + PG_PROGRESS, + PG_WARNING, + PG_FATAL +} eLogType; + +void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); + +static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) + pg_attribute_printf(3, 0); +static void elog_stderr(int elevel, const char *fmt, ...) + pg_attribute_printf(2, 3); + +/* Functions to work with log files */ +static void open_logfile(FILE **file, const char *filename_format); +static void release_logfile(void); +static char *logfile_getname(const char *format, time_t timestamp); +static FILE *logfile_open(const char *filename, const char *mode); + +/* Static variables */ + +static FILE *log_file = NULL; +static FILE *error_log_file = NULL; + +static bool exit_hook_registered = false; +/* Logging of the current thread is in progress */ +static bool loggin_in_progress = false; + +static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; + +void +init_logger(const char *root_path) +{ + /* Set log path */ + if (log_level_file != LOG_OFF || error_log_filename) + { + if (log_directory) + strcpy(log_path, log_directory); + else + join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); + } +} + +static void +write_elevel(FILE *stream, int elevel) +{ + switch (elevel) + { + case VERBOSE: + fputs("VERBOSE: ", stream); + break; + case LOG: + fputs("LOG: ", stream); + break; + case INFO: + fputs("INFO: ", stream); + break; + case NOTICE: + fputs("NOTICE: ", stream); + break; + case WARNING: + fputs("WARNING: ", stream); + break; + case ERROR: + fputs("ERROR: ", stream); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + +/* + * Exit with code if it is an error. + * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup + * routines. + */ +static void +exit_if_necessary(int elevel) +{ + if (elevel > WARNING && !in_cleanup) + { + /* Interrupt other possible routines */ + interrupted = true; + + if (loggin_in_progress) + { + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); + } + + /* If this is not the main thread then don't call exit() */ + if (main_tid != pthread_self()) +#ifdef WIN32 + ExitThread(elevel); +#else + pthread_exit(NULL); +#endif + else + exit(elevel); + } +} + +/* + * Logs to stderr or to log file and exit if ERROR. + * + * Actual implementation for elog() and pg_log(). + */ +static void +elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +{ + bool write_to_file, + write_to_error_log, + write_to_stderr; + va_list error_args, + std_args; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + + write_to_file = elevel >= log_level_file && log_path[0] != '\0'; + write_to_error_log = elevel >= ERROR && error_log_filename && + log_path[0] != '\0'; + write_to_stderr = elevel >= log_level_console && !file_only; + + pthread_lock(&log_file_mutex); +#ifdef WIN32 + std_args = NULL; + error_args = NULL; +#endif + loggin_in_progress = true; + + /* We need copy args only if we need write to error log file */ + if (write_to_error_log) + va_copy(error_args, args); + /* + * We need copy args only if we need write to stderr. But do not copy args + * if we need to log only to stderr. + */ + if (write_to_stderr && write_to_file) + va_copy(std_args, args); + + if (write_to_file || write_to_error_log) + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + + /* + * Write message to log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_file) + { + if (log_file == NULL) + { + if (log_filename == NULL) + open_logfile(&log_file, LOG_FILENAME_DEFAULT); + else + open_logfile(&log_file, log_filename); + } + + fprintf(log_file, "%s: ", strfbuf); + write_elevel(log_file, elevel); + + vfprintf(log_file, fmt, args); + fputc('\n', log_file); + fflush(log_file); + } + + /* + * Write error message to error log file. + * Do not write to file if this error was raised during write previous + * message. + */ + if (write_to_error_log) + { + if (error_log_file == NULL) + open_logfile(&error_log_file, error_log_filename); + + fprintf(error_log_file, "%s: ", strfbuf); + write_elevel(error_log_file, elevel); + + vfprintf(error_log_file, fmt, error_args); + fputc('\n', error_log_file); + fflush(error_log_file); + + va_end(error_args); + } + + /* + * Write to stderr if the message was not written to log file. + * Write to stderr if the message level is greater than WARNING anyway. + */ + if (write_to_stderr) + { + write_elevel(stderr, elevel); + if (write_to_file) + vfprintf(stderr, fmt, std_args); + else + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + if (write_to_file) + va_end(std_args); + } + + exit_if_necessary(elevel); + + loggin_in_progress = false; + pthread_mutex_unlock(&log_file_mutex); +} + +/* + * Log only to stderr. It is called only within elog_internal() when another + * logging already was started. + */ +static void +elog_stderr(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < ERROR) + return; + + va_start(args, fmt); + + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); + + va_end(args); + + exit_if_necessary(elevel); +} + +/* + * Logs to stderr or to log file and exit if ERROR. + */ +void +elog(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Logs only to log file and exit if ERROR. + */ +void +elog_file(int elevel, const char *fmt, ...) +{ + va_list args; + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, true, fmt, args); + va_end(args); +} + +/* + * Implementation of pg_log() from logging.h. + */ +void +pg_log(eLogType type, const char *fmt, ...) +{ + va_list args; + int elevel = INFO; + + /* Transform logging level from eLogType to utils/logger.h levels */ + switch (type) + { + case PG_DEBUG: + elevel = LOG; + break; + case PG_PROGRESS: + elevel = INFO; + break; + case PG_WARNING: + elevel = WARNING; + break; + case PG_FATAL: + elevel = ERROR; + break; + default: + elog(ERROR, "invalid logging level: %d", type); + break; + } + + /* + * Do not log message if severity level is less than log_level. + * It is the little optimisation to put it here not in elog_internal(). + */ + if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + return; + + va_start(args, fmt); + elog_internal(elevel, false, fmt, args); + va_end(args); +} + +/* + * Parses string representation of log level. + */ +int +parse_log_level(const char *level) +{ + const char *v = level; + size_t len; + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-level is empty"); + + if (pg_strncasecmp("off", v, len) == 0) + return LOG_OFF; + else if (pg_strncasecmp("verbose", v, len) == 0) + return VERBOSE; + else if (pg_strncasecmp("log", v, len) == 0) + return LOG; + else if (pg_strncasecmp("info", v, len) == 0) + return INFO; + else if (pg_strncasecmp("notice", v, len) == 0) + return NOTICE; + else if (pg_strncasecmp("warning", v, len) == 0) + return WARNING; + else if (pg_strncasecmp("error", v, len) == 0) + return ERROR; + + /* Log level is invalid */ + elog(ERROR, "invalid log-level \"%s\"", level); + return 0; +} + +/* + * Converts integer representation of log level to string. + */ +const char * +deparse_log_level(int level) +{ + switch (level) + { + case LOG_OFF: + return "OFF"; + case VERBOSE: + return "VERBOSE"; + case LOG: + return "LOG"; + case INFO: + return "INFO"; + case NOTICE: + return "NOTICE"; + case WARNING: + return "WARNING"; + case ERROR: + return "ERROR"; + default: + elog(ERROR, "invalid log-level %d", level); + } + + return NULL; +} + +/* + * Construct logfile name using timestamp information. + * + * Result is palloc'd. + */ +static char * +logfile_getname(const char *format, time_t timestamp) +{ + char *filename; + size_t len; + struct tm *tm = localtime(×tamp); + + if (log_path[0] == '\0') + elog_stderr(ERROR, "logging path is not set"); + + filename = (char *) palloc(MAXPGPATH); + + snprintf(filename, MAXPGPATH, "%s/", log_path); + + len = strlen(filename); + + /* Treat log_filename as a strftime pattern */ + if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); + + return filename; +} + +/* + * Open a new log file. + */ +static FILE * +logfile_open(const char *filename, const char *mode) +{ + FILE *fh; + + /* + * Create log directory if not present; ignore errors + */ + mkdir(log_path, S_IRWXU); + + fh = fopen(filename, mode); + + if (fh) + setvbuf(fh, NULL, PG_IOLBF, 0); + else + { + int save_errno = errno; + + elog_stderr(ERROR, "could not open log file \"%s\": %s", + filename, strerror(errno)); + errno = save_errno; + } + + return fh; +} + +/* + * Open the log file. + */ +static void +open_logfile(FILE **file, const char *filename_format) +{ + char *filename; + char control[MAXPGPATH]; + struct stat st; + FILE *control_file; + time_t cur_time = time(NULL); + bool rotation_requested = false, + logfile_exists = false; + + filename = logfile_getname(filename_format, cur_time); + + /* "log_path" was checked in logfile_getname() */ + snprintf(control, MAXPGPATH, "%s.rotation", filename); + + if (stat(filename, &st) == -1) + { + if (errno == ENOENT) + { + /* There is no file "filename" and rotation does not need */ + goto logfile_open; + } + else + elog_stderr(ERROR, "cannot stat log file \"%s\": %s", + filename, strerror(errno)); + } + /* Found log file "filename" */ + logfile_exists = true; + + /* First check for rotation */ + if (log_rotation_size > 0 || log_rotation_age > 0) + { + /* Check for rotation by age */ + if (log_rotation_age > 0) + { + struct stat control_st; + + if (stat(control, &control_st) == -1) + { + if (errno != ENOENT) + elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); + } + else + { + char buf[1024]; + + control_file = fopen(control, "r"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + if (fgets(buf, lengthof(buf), control_file)) + { + time_t creation_time; + + if (!parse_int64(buf, (int64 *) &creation_time, 0)) + elog_stderr(ERROR, "rotation file \"%s\" has wrong " + "creation timestamp \"%s\"", + control, buf); + /* Parsed creation time */ + + rotation_requested = (cur_time - creation_time) > + /* convert to seconds */ + log_rotation_age * 60; + } + else + elog_stderr(ERROR, "cannot read creation timestamp from " + "rotation file \"%s\"", control); + + fclose(control_file); + } + } + + /* Check for rotation by size */ + if (!rotation_requested && log_rotation_size > 0) + rotation_requested = st.st_size >= + /* convert to bytes */ + log_rotation_size * 1024L; + } + +logfile_open: + if (rotation_requested) + *file = logfile_open(filename, "w"); + else + *file = logfile_open(filename, "a"); + pfree(filename); + + /* Rewrite rotation control file */ + if (rotation_requested || !logfile_exists) + { + time_t timestamp = time(NULL); + + control_file = fopen(control, "w"); + if (control_file == NULL) + elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", + control, strerror(errno)); + + fprintf(control_file, "%ld", timestamp); + + fclose(control_file); + } + + /* + * Arrange to close opened file at proc_exit. + */ + if (!exit_hook_registered) + { + atexit(release_logfile); + exit_hook_registered = true; + } +} + +/* + * Closes opened file. + */ +static void +release_logfile(void) +{ + if (log_file) + { + fclose(log_file); + log_file = NULL; + } + if (error_log_file) + { + fclose(error_log_file); + error_log_file = NULL; + } +} diff --git a/src/utils/logger.h b/src/utils/logger.h new file mode 100644 index 000000000..8643ad186 --- /dev/null +++ b/src/utils/logger.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * logger.h: - prototypes of logger functions. + * + * Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef LOGGER_H +#define LOGGER_H + +#include "postgres_fe.h" + +#define LOG_NONE (-10) + +/* Log level */ +#define VERBOSE (-5) +#define LOG (-4) +#define INFO (-3) +#define NOTICE (-2) +#define WARNING (-1) +#define ERROR 1 +#define LOG_OFF 10 + +/* Logger parameters */ + +extern int log_to_file; +extern int log_level_console; +extern int log_level_file; + +extern char *log_filename; +extern char *error_log_filename; +extern char *log_directory; +extern char log_path[MAXPGPATH]; + +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 +extern int log_rotation_size; +extern int log_rotation_age; + +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF + +#undef elog +extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); + +extern void init_logger(const char *root_path); + +extern int parse_log_level(const char *level); +extern const char *deparse_log_level(int level); + +#endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c new file mode 100644 index 000000000..a9ba7c8e5 --- /dev/null +++ b/src/utils/parray.c @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * + * parray.c: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#include "src/pg_probackup.h" + +/* members of struct parray are hidden from client. */ +struct parray +{ + void **data; /* poiter array, expanded if necessary */ + size_t alloced; /* number of elements allocated */ + size_t used; /* number of elements in use */ +}; + +/* + * Create new parray object. + * Never returns NULL. + */ +parray * +parray_new(void) +{ + parray *a = pgut_new(parray); + + a->data = NULL; + a->used = 0; + a->alloced = 0; + + parray_expand(a, 1024); + + return a; +} + +/* + * Expand array pointed by data to newsize. + * Elements in expanded area are initialized to NULL. + * Note: never returns NULL. + */ +void +parray_expand(parray *array, size_t newsize) +{ + void **p; + + /* already allocated */ + if (newsize <= array->alloced) + return; + + p = pgut_realloc(array->data, sizeof(void *) * newsize); + + /* initialize expanded area to NULL */ + memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); + + array->alloced = newsize; + array->data = p; +} + +void +parray_free(parray *array) +{ + if (array == NULL) + return; + free(array->data); + free(array); +} + +void +parray_append(parray *array, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + array->data[array->used++] = elem; +} + +void +parray_insert(parray *array, size_t index, void *elem) +{ + if (array->used + 1 > array->alloced) + parray_expand(array, array->alloced * 2); + + memmove(array->data + index + 1, array->data + index, + (array->alloced - index - 1) * sizeof(void *)); + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; + else + array->used++; +} + +/* + * Concatinate two parray. + * parray_concat() appends the copy of the content of src to the end of dest. + */ +parray * +parray_concat(parray *dest, const parray *src) +{ + /* expand head array */ + parray_expand(dest, dest->used + src->used); + + /* copy content of src after content of dest */ + memcpy(dest->data + dest->used, src->data, src->used * sizeof(void *)); + dest->used += parray_num(src); + + return dest; +} + +void +parray_set(parray *array, size_t index, void *elem) +{ + if (index > array->alloced - 1) + parray_expand(array, index + 1); + + array->data[index] = elem; + + /* adjust used count */ + if (array->used < index + 1) + array->used = index + 1; +} + +void * +parray_get(const parray *array, size_t index) +{ + if (index > array->alloced - 1) + return NULL; + return array->data[index]; +} + +void * +parray_remove(parray *array, size_t index) +{ + void *val; + + /* removing unused element */ + if (index > array->used) + return NULL; + + val = array->data[index]; + + /* Do not move if the last element was removed. */ + if (index < array->alloced - 1) + memmove(array->data + index, array->data + index + 1, + (array->alloced - index - 1) * sizeof(void *)); + + /* adjust used count */ + array->used--; + + return val; +} + +bool +parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + int i; + + for (i = 0; i < array->used; i++) + { + if (compare(&key, &array->data[i]) == 0) + { + parray_remove(array, i); + return true; + } + } + return false; +} + +size_t +parray_num(const parray *array) +{ + return array->used; +} + +void +parray_qsort(parray *array, int(*compare)(const void *, const void *)) +{ + qsort(array->data, array->used, sizeof(void *), compare); +} + +void +parray_walk(parray *array, void (*action)(void *)) +{ + int i; + for (i = 0; i < array->used; i++) + action(array->data[i]); +} + +void * +parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + return bsearch(&key, array->data, array->used, sizeof(void *), compare); +} diff --git a/src/utils/parray.h b/src/utils/parray.h new file mode 100644 index 000000000..833a6961b --- /dev/null +++ b/src/utils/parray.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * parray.h: pointer array collection. + * + * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * + *------------------------------------------------------------------------- + */ + +#ifndef PARRAY_H +#define PARRAY_H + +/* + * "parray" hold pointers to objects in a linear memory area. + * Client use "parray *" to access parray object. + */ +typedef struct parray parray; + +extern parray *parray_new(void); +extern void parray_expand(parray *array, size_t newnum); +extern void parray_free(parray *array); +extern void parray_append(parray *array, void *val); +extern void parray_insert(parray *array, size_t index, void *val); +extern parray *parray_concat(parray *head, const parray *tail); +extern void parray_set(parray *array, size_t index, void *val); +extern void *parray_get(const parray *array, size_t index); +extern void *parray_remove(parray *array, size_t index); +extern bool parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern size_t parray_num(const parray *array); +extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); +extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern void parray_walk(parray *array, void (*action)(void *)); + +#endif /* PARRAY_H */ + diff --git a/src/utils/pgut.c b/src/utils/pgut.c new file mode 100644 index 000000000..f341c6a44 --- /dev/null +++ b/src/utils/pgut.c @@ -0,0 +1,2417 @@ +/*------------------------------------------------------------------------- + * + * pgut.c + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "libpq/pqsignal.h" + +#include "getopt_long.h" +#include +#include +#include + +#include "logger.h" +#include "pgut.h" + +/* old gcc doesn't have LLONG_MAX. */ +#ifndef LLONG_MAX +#if defined(HAVE_LONG_INT_64) || !defined(HAVE_LONG_LONG_INT_64) +#define LLONG_MAX LONG_MAX +#else +#define LLONG_MAX INT64CONST(0x7FFFFFFFFFFFFFFF) +#endif +#endif + +#define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ +#define SECS_PER_MINUTE 60 +#define MINS_PER_HOUR 60 +#define MAXPG_LSNCOMPONENT 8 + +const char *PROGRAM_NAME = NULL; + +const char *pgut_dbname = NULL; +const char *host = NULL; +const char *port = NULL; +const char *username = NULL; +static char *password = NULL; +bool prompt_password = true; +bool force_password = false; + +/* Database connections */ +static PGcancel *volatile cancel_conn = NULL; + +/* Interrupted by SIGINT (Ctrl+C) ? */ +bool interrupted = false; +bool in_cleanup = false; +bool in_password = false; + +static bool parse_pair(const char buffer[], char key[], char value[]); + +/* Connection routines */ +static void init_cancel_handler(void); +static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); +static void on_after_exec(PGcancel *thread_cancel_conn); +static void on_interrupt(void); +static void on_cleanup(void); +static void exit_or_abort(int exitcode); +static const char *get_username(void); +static pqsigfunc oldhandler = NULL; + +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"KB", OPTION_UNIT_KB, 1}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + +static size_t +option_length(const pgut_option opts[]) +{ + size_t len; + + for (len = 0; opts && opts[len].type; len++) { } + + return len; +} + +static int +option_has_arg(char type) +{ + switch (type) + { + case 'b': + case 'B': + return no_argument; + default: + return required_argument; + } +} + +static void +option_copy(struct option dst[], const pgut_option opts[], size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + { + dst[i].name = opts[i].lname; + dst[i].has_arg = option_has_arg(opts[i].type); + dst[i].flag = NULL; + dst[i].val = opts[i].sname; + } +} + +static pgut_option * +option_find(int c, pgut_option opts1[]) +{ + size_t i; + + for (i = 0; opts1 && opts1[i].type; i++) + if (opts1[i].sname == c) + return &opts1[i]; + + return NULL; /* not found */ +} + +static void +assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) +{ + const char *message; + + if (opt == NULL) + { + fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); + exit_or_abort(ERROR); + } + + if (opt->source > src) + { + /* high prior value has been set already. */ + return; + } + /* Allow duplicate entries for function option */ + else if (src >= SOURCE_CMDLINE && opt->source >= src && opt->type != 'f') + { + message = "specified only once"; + } + else + { + pgut_optsrc orig_source = opt->source; + + /* can be overwritten if non-command line source */ + opt->source = src; + + switch (opt->type) + { + case 'b': + case 'B': + if (optarg == NULL) + { + *((bool *) opt->var) = (opt->type == 'b'); + return; + } + else if (parse_bool(optarg, (bool *) opt->var)) + { + return; + } + message = "a boolean"; + break; + case 'f': + ((pgut_optfn) opt->var)(opt, optarg); + return; + case 'i': + if (parse_int32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit signed integer"; + break; + case 'u': + if (parse_uint32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit unsigned integer"; + break; + case 'I': + if (parse_int64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit signed integer"; + break; + case 'U': + if (parse_uint64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit unsigned integer"; + break; + case 's': + if (orig_source != SOURCE_DEFAULT) + free(*(char **) opt->var); + *(char **) opt->var = pgut_strdup(optarg); + if (strcmp(optarg,"") != 0) + return; + message = "a valid string. But provided: "; + break; + case 't': + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) + return; + message = "a time"; + break; + default: + elog(ERROR, "invalid option type: %c", opt->type); + return; /* keep compiler quiet */ + } + } + + if (isprint(opt->sname)) + elog(ERROR, "option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "option --%s should be %s: '%s'", + opt->lname, message, optarg); +} + +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Unsigned variant of convert_to_base_unit() + */ +static bool +convert_to_base_unit_u(uint64 value, const char *unit, + int base_unit, uint64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + *base_value = value * table[i].multiplier; + return true; + } + } + return false; +} + +/* + * Convert a value in some base unit to a human-friendly unit. The output + * unit is chosen so that it's the greatest unit that can represent the value + * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is + * converted to 1 MB, but 1025 is represented as 1025 kB. + */ +void +convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Unsigned variant of convert_from_base_unit() + */ +void +convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +static bool +parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Unsigned variant of parse_unit() + */ +static bool +parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit_u(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Try to interpret value as boolean value. Valid values are: true, + * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + */ +bool +parse_bool(const char *value, bool *result) +{ + return parse_bool_with_len(value, strlen(value), result); +} + +bool +parse_bool_with_len(const char *value, size_t len, bool *result) +{ + switch (*value) + { + case 't': + case 'T': + if (pg_strncasecmp(value, "true", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'f': + case 'F': + if (pg_strncasecmp(value, "false", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'y': + case 'Y': + if (pg_strncasecmp(value, "yes", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'n': + case 'N': + if (pg_strncasecmp(value, "no", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'o': + case 'O': + /* 'o' is not unique enough */ + if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = true; + return true; + } + else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = false; + return true; + } + break; + case '1': + if (len == 1) + { + if (result) + *result = true; + return true; + } + break; + case '0': + if (len == 1) + { + if (result) + *result = false; + return true; + } + break; + default: + break; + } + + if (result) + *result = false; /* suppress compiler warning */ + return false; +} + +/* + * Parse string as 32bit signed int. + * valid range: -2147483648 ~ 2147483647 + */ +bool +parse_int32(const char *value, int32 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = INT_MAX; + return true; + } + + errno = 0; + val = strtol(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (int64) ((int32) val)) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as 32bit unsigned int. + * valid range: 0 ~ 4294967295 (2^32-1) + */ +bool +parse_uint32(const char *value, uint32 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = UINT_MAX; + return true; + } + + errno = 0; + val = strtoul(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE || val != (uint64) ((uint32) val)) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as int64 + * valid range: -9223372036854775808 ~ 9223372036854775807 + */ +bool +parse_int64(const char *value, int64 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = LLONG_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtol(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoll(value, &endptr, 0); +#else + val = strtol(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as uint64 + * valid range: 0 ~ (2^64-1) + */ +bool +parse_uint64(const char *value, uint64 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { +#if defined(HAVE_LONG_INT_64) + *result = ULONG_MAX; +#elif defined(HAVE_LONG_LONG_INT_64) + *result = ULLONG_MAX; +#else + *result = ULONG_MAX; +#endif + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtoul(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoull(value, &endptr, 0); +#else + val = strtoul(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. + */ +bool +parse_time(const char *value, time_t *result, bool utc_default) +{ + size_t len; + int fields_num, + tz = 0, + i; + bool tz_set = false; + char *tmp; + struct tm tm; + char junk[2]; + + /* tmp = replace( value, !isalnum, ' ' ) */ + tmp = pgut_malloc(strlen(value) + + 1); + len = 0; + fields_num = 1; + + while (*value) + { + if (IsAlnum(*value)) + { + tmp[len++] = *value; + value++; + } + else if (fields_num < 6) + { + fields_num++; + tmp[len++] = ' '; + value++; + } + /* timezone field is 7th */ + else if ((*value == '-' || *value == '+') && fields_num == 6) + { + int hr, + min, + sec = 0; + char *cp; + + errno = 0; + hr = strtol(value + 1, &cp, 10); + if ((value + 1) == cp || errno == ERANGE) + return false; + + /* explicit delimiter? */ + if (*cp == ':') + { + errno = 0; + min = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + if (*cp == ':') + { + errno = 0; + sec = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + } + } + /* otherwise, might have run things together... */ + else if (*cp == '\0' && strlen(value) > 3) + { + min = hr % 100; + hr = hr / 100; + /* we could, but don't, support a run-together hhmmss format */ + } + else + min = 0; + + /* Range-check the values; see notes in datatype/timestamp.h */ + if (hr < 0 || hr > MAX_TZDISP_HOUR) + return false; + if (min < 0 || min >= MINS_PER_HOUR) + return false; + if (sec < 0 || sec >= SECS_PER_MINUTE) + return false; + + tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; + if (*value == '-') + tz = -tz; + + tz_set = true; + + fields_num++; + value = cp; + } + /* wrong format */ + else if (!IsSpace(*value)) + return false; + } + tmp[len] = '\0'; + + /* parse for "YYYY-MM-DD HH:MI:SS" */ + memset(&tm, 0, sizeof(tm)); + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); + free(tmp); + + if (i < 1 || 6 < i) + return false; + + /* adjust year */ + if (tm.tm_year < 100) + tm.tm_year += 2000 - 1900; + else if (tm.tm_year >= 1900) + tm.tm_year -= 1900; + + /* adjust month */ + if (i > 1) + tm.tm_mon -= 1; + + /* determine whether Daylight Saving Time is in effect */ + tm.tm_isdst = -1; + + *result = mktime(&tm); + + /* adjust time zone */ + if (tz_set || utc_default) + { + time_t ltime = time(NULL); + struct tm *ptm = gmtime(<ime); + time_t gmt = mktime(ptm); + time_t offset; + + /* UTC time */ + *result -= tz; + + /* Get local time */ + ptm = localtime(<ime); + offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); + + *result += offset; + } + + return true; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + +static char * +longopts_to_optstring(const struct option opts[], const size_t len) +{ + size_t i; + char *result; + char *s; + + result = pgut_malloc(len * 2 + 1); + + s = result; + for (i = 0; i < len; i++) + { + if (!isprint(opts[i].val)) + continue; + *s++ = opts[i].val; + if (opts[i].has_arg != no_argument) + *s++ = ':'; + } + *s = '\0'; + + return result; +} + +void +pgut_getopt_env(pgut_option options[]) +{ + size_t i; + + for (i = 0; options && options[i].type; i++) + { + pgut_option *opt = &options[i]; + const char *value = NULL; + + /* If option was already set do not check env */ + if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) + continue; + + if (strcmp(opt->lname, "pgdata") == 0) + value = getenv("PGDATA"); + if (strcmp(opt->lname, "port") == 0) + value = getenv("PGPORT"); + if (strcmp(opt->lname, "host") == 0) + value = getenv("PGHOST"); + if (strcmp(opt->lname, "username") == 0) + value = getenv("PGUSER"); + if (strcmp(opt->lname, "pgdatabase") == 0) + { + value = getenv("PGDATABASE"); + if (value == NULL) + value = getenv("PGUSER"); + if (value == NULL) + value = get_username(); + } + + if (value) + assign_option(opt, value, SOURCE_ENV); + } +} + +int +pgut_getopt(int argc, char **argv, pgut_option options[]) +{ + int c; + int optindex = 0; + char *optstring; + pgut_option *opt; + struct option *longopts; + size_t len; + + len = option_length(options); + longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); + option_copy(longopts, options, len); + + optstring = longopts_to_optstring(longopts, len); + + /* Assign named options */ + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + { + opt = option_find(c, options); + if (opt && opt->allowed < SOURCE_CMDLINE) + elog(ERROR, "option %s cannot be specified in command line", + opt->lname); + /* Check 'opt == NULL' is performed in assign_option() */ + assign_option(opt, optarg, SOURCE_CMDLINE); + } + + init_cancel_handler(); + atexit(on_cleanup); + + return optind; +} + +/* compare two strings ignore cases and ignore -_ */ +static bool +key_equals(const char *lhs, const char *rhs) +{ + for (; *lhs && *rhs; lhs++, rhs++) + { + if (strchr("-_ ", *lhs)) + { + if (!strchr("-_ ", *rhs)) + return false; + } + else if (ToLower(*lhs) != ToLower(*rhs)) + return false; + } + + return *lhs == '\0' && *rhs == '\0'; +} + +/* + * Get configuration from configuration file. + * Return number of parsed options + */ +int +pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) +{ + FILE *fp; + char buf[1024]; + char key[1024]; + char value[1024]; + int parsed_options = 0; + + if (!options) + return parsed_options; + + if ((fp = pgut_fopen(path, "rt", true)) == NULL) + return parsed_options; + + while (fgets(buf, lengthof(buf), fp)) + { + size_t i; + + for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) + buf[i - 1] = '\0'; + + if (parse_pair(buf, key, value)) + { + for (i = 0; options[i].type; i++) + { + pgut_option *opt = &options[i]; + + if (key_equals(key, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "option %s cannot be specified in file", opt->lname); + else if (opt->source <= SOURCE_FILE) + { + assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } + break; + } + } + if (strict && !options[i].type) + elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); + } + } + + fclose(fp); + + return parsed_options; +} + +static const char * +skip_space(const char *str, const char *line) +{ + while (IsSpace(*str)) { str++; } + return str; +} + +static const char * +get_next_token(const char *src, char *dst, const char *line) +{ + const char *s; + int i; + int j; + + if ((s = skip_space(src, line)) == NULL) + return NULL; + + /* parse quoted string */ + if (*s == '\'') + { + s++; + for (i = 0, j = 0; s[i] != '\0'; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + dst[j] = '\b'; + break; + case 'f': + dst[j] = '\f'; + break; + case 'n': + dst[j] = '\n'; + break; + case 'r': + dst[j] = '\r'; + break; + case 't': + dst[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + dst[j] = ((char) octVal); + } + break; + default: + dst[j] = s[i]; + break; + } + } + else if (s[i] == '\'') + { + i++; + /* doubled quote becomes just one quote */ + if (s[i] == '\'') + dst[j] = s[i]; + else + break; + } + else + dst[j] = s[i]; + j++; + } + } + else + { + i = j = strcspn(s, "#\n\r\t\v"); + memcpy(dst, s, j); + } + + dst[j] = '\0'; + return s + i; +} + +static bool +parse_pair(const char buffer[], char key[], char value[]) +{ + const char *start; + const char *end; + + key[0] = value[0] = '\0'; + + /* + * parse key + */ + start = buffer; + if ((start = skip_space(start, buffer)) == NULL) + return false; + + end = start + strcspn(start, "=# \n\r\t\v"); + + /* skip blank buffer */ + if (end - start <= 0) + { + if (*start == '=') + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + /* key found */ + strncpy(key, start, end - start); + key[end - start] = '\0'; + + /* find key and value split char */ + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '=') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + start++; + + /* + * parse value + */ + if ((end = get_next_token(start, value, buffer)) == NULL) + return false; + + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '\0' && *start != '#') + { + elog(ERROR, "syntax error in \"%s\"", buffer); + return false; + } + + return true; +} + +/* + * Ask the user for a password; 'username' is the username the + * password is for, if one has been explicitly specified. + * Set malloc'd string to the global variable 'password'. + */ +static void +prompt_for_password(const char *username) +{ + in_password = true; + + if (password) + { + free(password); + password = NULL; + } + +#if PG_VERSION_NUM >= 100000 + password = (char *) pgut_malloc(sizeof(char) * 100 + 1); + if (username == NULL) + simple_prompt("Password: ", password, 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + simple_prompt(message, password, 100, false); + } +#else + if (username == NULL) + password = simple_prompt("Password: ", 100, false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message, 100, false); + } +#endif + + in_password = false; +} + +/* + * Copied from pg_basebackup.c + * Escape a parameter value so that it can be used as part of a libpq + * connection string, e.g. in: + * + * application_name= + * + * The returned string is malloc'd. Return NULL on out-of-memory. + */ +static char * +escapeConnectionParameter(const char *src) +{ + bool need_quotes = false; + bool need_escaping = false; + const char *p; + char *dstbuf; + char *dst; + + /* + * First check if quoting is needed. Any quote (') or backslash (\) + * characters need to be escaped. Parameters are separated by whitespace, + * so any string containing whitespace characters need to be quoted. An + * empty string is represented by ''. + */ + if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) + need_escaping = true; + + for (p = src; *p; p++) + { + if (isspace((unsigned char) *p)) + { + need_quotes = true; + break; + } + } + + if (*src == '\0') + return pg_strdup("''"); + + if (!need_quotes && !need_escaping) + return pg_strdup(src); /* no quoting or escaping needed */ + + /* + * Allocate a buffer large enough for the worst case that all the source + * characters need to be escaped, plus quotes. + */ + dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); + + dst = dstbuf; + if (need_quotes) + *(dst++) = '\''; + for (; *src; src++) + { + if (*src == '\'' || *src == '\\') + *(dst++) = '\\'; + *(dst++) = *src; + } + if (need_quotes) + *(dst++) = '\''; + *dst = '\0'; + + return dstbuf; +} + +/* Construct a connection string for possible future use in recovery.conf */ +char * +pgut_get_conninfo_string(PGconn *conn) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBuffer buf = createPQExpBuffer(); + char *connstr; + bool firstkeyword = true; + char *escaped; + + connOptions = PQconninfo(conn); + if (connOptions == NULL) + elog(ERROR, "out of memory"); + + /* Construct a new connection string in key='value' format. */ + for (option = connOptions; option && option->keyword; option++) + { + /* + * Do not emit this setting if: - the setting is "replication", + * "dbname" or "fallback_application_name", since these would be + * overridden by the libpqwalreceiver module anyway. - not set or + * empty. + */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* do not print password into the file */ + if (strcmp(option->keyword, "password") == 0) + continue; + + if (!firstkeyword) + appendPQExpBufferChar(buf, ' '); + + firstkeyword = false; + + escaped = escapeConnectionParameter(option->val); + appendPQExpBuffer(buf, "%s=%s", option->keyword, escaped); + free(escaped); + } + + connstr = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + return connstr; +} + +PGconn * +pgut_connect(const char *dbname) +{ + return pgut_connect_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login) +{ + PGconn *conn; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(login); + + /* Start the connection. Loop until we have a password if requested by backend. */ + for (;;) + { + conn = PQsetdbLogin(pghost, pgport, NULL, NULL, + dbname, login, password); + + if (PQstatus(conn) == CONNECTION_OK) + return conn; + + if (conn && PQconnectionNeedsPassword(conn) && prompt_password) + { + PQfinish(conn); + prompt_for_password(login); + + if (interrupted) + elog(ERROR, "interrupted"); + + if (password == NULL || password[0] == '\0') + elog(ERROR, "no password supplied"); + + continue; + } + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(conn)); + + PQfinish(conn); + return NULL; + } +} + +PGconn * +pgut_connect_replication(const char *dbname) +{ + return pgut_connect_replication_extended(host, port, dbname, username); +} + +PGconn * +pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser) +{ + PGconn *tmpconn; + int argcount = 7; /* dbname, replication, fallback_app_name, + * host, user, port, password */ + int i; + const char **keywords; + const char **values; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + if (force_password && !prompt_password) + elog(ERROR, "You cannot specify --password and --no-password options together"); + + if (!password && force_password) + prompt_for_password(pguser); + + i = 0; + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + + keywords[i] = "dbname"; + values[i] = "replication"; + i++; + keywords[i] = "replication"; + values[i] = "true"; + i++; + keywords[i] = "fallback_application_name"; + values[i] = PROGRAM_NAME; + i++; + + if (pghost) + { + keywords[i] = "host"; + values[i] = pghost; + i++; + } + if (pguser) + { + keywords[i] = "user"; + values[i] = pguser; + i++; + } + if (pgport) + { + keywords[i] = "port"; + values[i] = pgport; + i++; + } + + /* Use (or reuse, on a subsequent connection) password if we have it */ + if (password) + { + keywords[i] = "password"; + values[i] = password; + } + else + { + keywords[i] = NULL; + values[i] = NULL; + } + + for (;;) + { + tmpconn = PQconnectdbParams(keywords, values, true); + + + if (PQstatus(tmpconn) == CONNECTION_OK) + { + free(values); + free(keywords); + return tmpconn; + } + + if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) + { + PQfinish(tmpconn); + prompt_for_password(pguser); + keywords[i] = "password"; + values[i] = password; + continue; + } + + elog(ERROR, "could not connect to database %s: %s", + dbname, PQerrorMessage(tmpconn)); + PQfinish(tmpconn); + free(values); + free(keywords); + return NULL; + } +} + + +void +pgut_disconnect(PGconn *conn) +{ + if (conn) + PQfinish(conn); +} + +/* set/get host and port for connecting standby server */ +const char * +pgut_get_host() +{ + return host; +} + +const char * +pgut_get_port() +{ + return port; +} + +void +pgut_set_host(const char *new_host) +{ + host = new_host; +} + +void +pgut_set_port(const char *new_port) +{ + port = new_port; +} + + +PGresult * +pgut_execute_parallel(PGconn* conn, + PGcancel* thread_cancel_conn, const char *query, + int nParams, const char **params, + bool text_result) +{ + PGresult *res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + //on_before_exec(conn, thread_cancel_conn); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + //on_after_exec(thread_cancel_conn); + + switch (PQresultStatus(res)) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +PGresult * +pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) +{ + return pgut_execute_extended(conn, query, nParams, params, true, false); +} + +PGresult * +pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error) +{ + PGresult *res; + ExecStatusType res_status; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(ERROR, "not connected"); + return NULL; + } + + on_before_exec(conn, NULL); + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + on_after_exec(NULL); + + res_status = PQresultStatus(res); + switch (res_status) + { + case PGRES_TUPLES_OK: + case PGRES_COMMAND_OK: + case PGRES_COPY_IN: + break; + default: + if (ok_error && res_status == PGRES_FATAL_ERROR) + break; + + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + break; + } + + return res; +} + +bool +pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel) +{ + int res; + + if (interrupted && !in_cleanup) + elog(ERROR, "interrupted"); + + /* write query to elog if verbose */ + if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + { + int i; + + if (strchr(query, '\n')) + elog(VERBOSE, "(query)\n%s", query); + else + elog(VERBOSE, "(query) %s", query); + for (i = 0; i < nParams; i++) + elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); + } + + if (conn == NULL) + { + elog(elevel, "not connected"); + return false; + } + + if (nParams == 0) + res = PQsendQuery(conn, query); + else + res = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, 0); + + if (res != 1) + { + elog(elevel, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + return false; + } + + return true; +} + +void +pgut_cancel(PGconn* conn) +{ + PGcancel *cancel_conn = PQgetCancel(conn); + char errbuf[256]; + + if (cancel_conn != NULL) + { + if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + elog(WARNING, "Cancel request sent"); + else + elog(WARNING, "Cancel request failed"); + } + + if (cancel_conn) + PQfreeCancel(cancel_conn); +} + +int +pgut_wait(int num, PGconn *connections[], struct timeval *timeout) +{ + /* all connections are busy. wait for finish */ + while (!interrupted) + { + int i; + fd_set mask; + int maxsock; + + FD_ZERO(&mask); + + maxsock = -1; + for (i = 0; i < num; i++) + { + int sock; + + if (connections[i] == NULL) + continue; + sock = PQsocket(connections[i]); + if (sock >= 0) + { + FD_SET(sock, &mask); + if (maxsock < sock) + maxsock = sock; + } + } + + if (maxsock == -1) + { + errno = ENOENT; + return -1; + } + + i = wait_for_sockets(maxsock + 1, &mask, timeout); + if (i == 0) + break; /* timeout */ + + for (i = 0; i < num; i++) + { + if (connections[i] && FD_ISSET(PQsocket(connections[i]), &mask)) + { + PQconsumeInput(connections[i]); + if (PQisBusy(connections[i])) + continue; + return i; + } + } + } + + errno = EINTR; + return -1; +} + +#ifdef WIN32 +static CRITICAL_SECTION cancelConnLock; +#endif + +/* + * on_before_exec + * + * Set cancel_conn to point to the current database connection. + */ +static void +on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + thread_cancel_conn = PQgetCancel(conn); + } + else + { + /* Free the old one if we have one */ + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + + cancel_conn = PQgetCancel(conn); + } + +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * on_after_exec + * + * Free the current cancel connection, if any, and set to NULL. + */ +static void +on_after_exec(PGcancel *thread_cancel_conn) +{ + PGcancel *old; + + if (in_cleanup) + return; /* forbid cancel during cleanup */ + +#ifdef WIN32 + EnterCriticalSection(&cancelConnLock); +#endif + + if (thread_cancel_conn) + { + //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); + old = thread_cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + thread_cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } + else + { + old = cancel_conn; + + /* be sure handle_sigint doesn't use pointer while freeing */ + cancel_conn = NULL; + + if (old != NULL) + PQfreeCancel(old); + } +#ifdef WIN32 + LeaveCriticalSection(&cancelConnLock); +#endif +} + +/* + * Handle interrupt signals by cancelling the current command. + */ +static void +on_interrupt(void) +{ + int save_errno = errno; + char errbuf[256]; + + /* Set interruped flag */ + interrupted = true; + + /* User promts password, call on_cleanup() byhand */ + if (in_password) + { + on_cleanup(); + + pqsignal(SIGINT, oldhandler); + kill(0, SIGINT); + } + + /* Send QueryCancel if we are processing a database query */ + if (!in_cleanup && cancel_conn != NULL && + PQcancel(cancel_conn, errbuf, sizeof(errbuf))) + { + elog(WARNING, "Cancel request sent"); + } + + errno = save_errno; /* just in case the write changed it */ +} + +typedef struct pgut_atexit_item pgut_atexit_item; +struct pgut_atexit_item +{ + pgut_atexit_callback callback; + void *userdata; + pgut_atexit_item *next; +}; + +static pgut_atexit_item *pgut_atexit_stack = NULL; + +void +pgut_atexit_push(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + + AssertArg(callback != NULL); + + item = pgut_new(pgut_atexit_item); + item->callback = callback; + item->userdata = userdata; + item->next = pgut_atexit_stack; + + pgut_atexit_stack = item; +} + +void +pgut_atexit_pop(pgut_atexit_callback callback, void *userdata) +{ + pgut_atexit_item *item; + pgut_atexit_item **prev; + + for (item = pgut_atexit_stack, prev = &pgut_atexit_stack; + item; + prev = &item->next, item = item->next) + { + if (item->callback == callback && item->userdata == userdata) + { + *prev = item->next; + free(item); + break; + } + } +} + +static void +call_atexit_callbacks(bool fatal) +{ + pgut_atexit_item *item; + + for (item = pgut_atexit_stack; item; item = item->next) + item->callback(fatal, item->userdata); +} + +static void +on_cleanup(void) +{ + in_cleanup = true; + interrupted = false; + call_atexit_callbacks(false); +} + +static void +exit_or_abort(int exitcode) +{ + if (in_cleanup) + { + /* oops, error in cleanup*/ + call_atexit_callbacks(true); + abort(); + } + else + { + /* normal exit */ + exit(exitcode); + } +} + +/* + * Returns the current user name. + */ +static const char * +get_username(void) +{ + const char *ret; + +#ifndef WIN32 + struct passwd *pw; + + pw = getpwuid(geteuid()); + ret = (pw ? pw->pw_name : NULL); +#else + static char username[128]; /* remains after function exit */ + DWORD len = sizeof(username) - 1; + + if (GetUserName(username, &len)) + ret = username; + else + { + _dosmaperr(GetLastError()); + ret = NULL; + } +#endif + + if (ret == NULL) + elog(ERROR, "%s: could not get current user name: %s", + PROGRAM_NAME, strerror(errno)); + return ret; +} + +int +appendStringInfoFile(StringInfo str, FILE *fp) +{ + AssertArg(str != NULL); + AssertArg(fp != NULL); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = fread(str->data + str->len, 1, str->maxlen - str->len - 1, fp); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (ferror(fp) && errno != EINTR) + return errno; + } + return 0; +} + +int +appendStringInfoFd(StringInfo str, int fd) +{ + AssertArg(str != NULL); + AssertArg(fd != -1); + + for (;;) + { + int rc; + + if (str->maxlen - str->len < 2 && enlargeStringInfo(str, 1024) == 0) + return errno = ENOMEM; + + rc = read(fd, str->data + str->len, str->maxlen - str->len - 1); + if (rc == 0) + break; + else if (rc > 0) + { + str->len += rc; + str->data[str->len] = '\0'; + } + else if (errno != EINTR) + return errno; + } + return 0; +} + +void * +pgut_malloc(size_t size) +{ + char *ret; + + if ((ret = malloc(size)) == NULL) + elog(ERROR, "could not allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +void * +pgut_realloc(void *p, size_t size) +{ + char *ret; + + if ((ret = realloc(p, size)) == NULL) + elog(ERROR, "could not re-allocate memory (%lu bytes): %s", + (unsigned long) size, strerror(errno)); + return ret; +} + +char * +pgut_strdup(const char *str) +{ + char *ret; + + if (str == NULL) + return NULL; + + if ((ret = strdup(str)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + return ret; +} + +char * +strdup_with_len(const char *str, size_t len) +{ + char *r; + + if (str == NULL) + return NULL; + + r = pgut_malloc(len + 1); + memcpy(r, str, len); + r[len] = '\0'; + return r; +} + +/* strdup but trim whitespaces at head and tail */ +char * +strdup_trim(const char *str) +{ + size_t len; + + if (str == NULL) + return NULL; + + while (IsSpace(str[0])) { str++; } + len = strlen(str); + while (len > 0 && IsSpace(str[len - 1])) { len--; } + + return strdup_with_len(str, len); +} + +FILE * +pgut_fopen(const char *path, const char *mode, bool missing_ok) +{ + FILE *fp; + + if ((fp = fopen(path, mode)) == NULL) + { + if (missing_ok && errno == ENOENT) + return NULL; + + elog(ERROR, "could not open file \"%s\": %s", + path, strerror(errno)); + } + + return fp; +} + +#ifdef WIN32 +static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); +#define select select_win32 +#endif + +int +wait_for_socket(int sock, struct timeval *timeout) +{ + fd_set fds; + + FD_ZERO(&fds); + FD_SET(sock, &fds); + return wait_for_sockets(sock + 1, &fds, timeout); +} + +int +wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) +{ + int i; + + for (;;) + { + i = select(nfds, fds, NULL, NULL, timeout); + if (i < 0) + { + if (interrupted) + elog(ERROR, "interrupted"); + else if (errno != EINTR) + elog(ERROR, "select failed: %s", strerror(errno)); + } + else + return i; + } +} + +#ifndef WIN32 +static void +handle_sigint(SIGNAL_ARGS) +{ + on_interrupt(); +} + +static void +init_cancel_handler(void) +{ + oldhandler = pqsignal(SIGINT, handle_sigint); +} +#else /* WIN32 */ + +/* + * Console control handler for Win32. Note that the control handler will + * execute on a *different thread* than the main one, so we need to do + * proper locking around those structures. + */ +static BOOL WINAPI +consoleHandler(DWORD dwCtrlType) +{ + if (dwCtrlType == CTRL_C_EVENT || + dwCtrlType == CTRL_BREAK_EVENT) + { + EnterCriticalSection(&cancelConnLock); + on_interrupt(); + LeaveCriticalSection(&cancelConnLock); + return TRUE; + } + else + /* Return FALSE for any signals not being handled */ + return FALSE; +} + +static void +init_cancel_handler(void) +{ + InitializeCriticalSection(&cancelConnLock); + + SetConsoleCtrlHandler(consoleHandler, TRUE); +} + +int +sleep(unsigned int seconds) +{ + Sleep(seconds * 1000); + return 0; +} + +int +usleep(unsigned int usec) +{ + Sleep((usec + 999) / 1000); /* rounded up */ + return 0; +} + +#undef select +static int +select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) +{ + struct timeval remain; + + if (timeout != NULL) + remain = *timeout; + else + { + remain.tv_usec = 0; + remain.tv_sec = LONG_MAX; /* infinite */ + } + + /* sleep only one second because Ctrl+C doesn't interrupt select. */ + while (remain.tv_sec > 0 || remain.tv_usec > 0) + { + int ret; + struct timeval onesec; + + if (remain.tv_sec > 0) + { + onesec.tv_sec = 1; + onesec.tv_usec = 0; + remain.tv_sec -= 1; + } + else + { + onesec.tv_sec = 0; + onesec.tv_usec = remain.tv_usec; + remain.tv_usec = 0; + } + + ret = select(nfds, readfds, writefds, exceptfds, &onesec); + if (ret != 0) + { + /* succeeded or error */ + return ret; + } + else if (interrupted) + { + errno = EINTR; + return 0; + } + } + + return 0; /* timeout */ +} + +#endif /* WIN32 */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h new file mode 100644 index 000000000..fedb99b01 --- /dev/null +++ b/src/utils/pgut.h @@ -0,0 +1,238 @@ +/*------------------------------------------------------------------------- + * + * pgut.h + * + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2017-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PGUT_H +#define PGUT_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +#include +#include + +#include "access/xlogdefs.h" +#include "logger.h" + +#if !defined(C_H) && !defined(__cplusplus) +#ifndef bool +typedef char bool; +#endif +#ifndef true +#define true ((bool) 1) +#endif +#ifndef false +#define false ((bool) 0) +#endif +#endif + +#define INFINITE_STR "INFINITE" + +typedef enum pgut_optsrc +{ + SOURCE_DEFAULT, + SOURCE_FILE_STRICT, + SOURCE_ENV, + SOURCE_FILE, + SOURCE_CMDLINE, + SOURCE_CONST +} pgut_optsrc; + +/* + * type: + * b: bool (true) + * B: bool (false) + * f: pgut_optfn + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string + * t: time_t + */ +typedef struct pgut_option +{ + char type; + uint8 sname; /* short name */ + const char *lname; /* long name */ + void *var; /* pointer to variable */ + pgut_optsrc allowed; /* allowed source */ + pgut_optsrc source; /* actual source */ + int flags; /* option unit */ +} pgut_option; + +typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); +typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); + +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + +/* + * pgut client variables and functions + */ +extern const char *PROGRAM_NAME; +extern const char *PROGRAM_VERSION; +extern const char *PROGRAM_URL; +extern const char *PROGRAM_EMAIL; + +extern void pgut_help(bool details); + +/* + * pgut framework variables and functions + */ +extern const char *pgut_dbname; +extern const char *host; +extern const char *port; +extern const char *username; +extern bool prompt_password; +extern bool force_password; + +extern bool interrupted; +extern bool in_cleanup; +extern bool in_password; /* User prompts password */ + +extern int pgut_getopt(int argc, char **argv, pgut_option options[]); +extern int pgut_readopt(const char *path, pgut_option options[], int elevel, + bool strict); +extern void pgut_getopt_env(pgut_option options[]); +extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); +extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); + +/* + * Database connections + */ +extern char *pgut_get_conninfo_string(PGconn *conn); +extern PGconn *pgut_connect(const char *dbname); +extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, + const char *dbname, const char *login); +extern PGconn *pgut_connect_replication(const char *dbname); +extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport, + const char *dbname, const char *pguser); +extern void pgut_disconnect(PGconn *conn); +extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, + const char **params); +extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, + const char **params, bool text_result, bool ok_error); +extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, + const char *query, int nParams, + const char **params, bool text_result); +extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); +extern void pgut_cancel(PGconn* conn); +extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); + +extern const char *pgut_get_host(void); +extern const char *pgut_get_port(void); +extern void pgut_set_host(const char *new_host); +extern void pgut_set_port(const char *new_port); + +/* + * memory allocators + */ +extern void *pgut_malloc(size_t size); +extern void *pgut_realloc(void *p, size_t size); +extern char *pgut_strdup(const char *str); +extern char *strdup_with_len(const char *str, size_t len); +extern char *strdup_trim(const char *str); + +#define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) + +/* + * file operations + */ +extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); + +/* + * Assert + */ +#undef Assert +#undef AssertArg +#undef AssertMacro + +#ifdef USE_ASSERT_CHECKING +#define Assert(x) assert(x) +#define AssertArg(x) assert(x) +#define AssertMacro(x) assert(x) +#else +#define Assert(x) ((void) 0) +#define AssertArg(x) ((void) 0) +#define AssertMacro(x) ((void) 0) +#endif + +/* + * StringInfo and string operations + */ +#define STRINGINFO_H + +#define StringInfoData PQExpBufferData +#define StringInfo PQExpBuffer +#define makeStringInfo createPQExpBuffer +#define initStringInfo initPQExpBuffer +#define freeStringInfo destroyPQExpBuffer +#define termStringInfo termPQExpBuffer +#define resetStringInfo resetPQExpBuffer +#define enlargeStringInfo enlargePQExpBuffer +#define printfStringInfo printfPQExpBuffer /* reset + append */ +#define appendStringInfo appendPQExpBuffer +#define appendStringInfoString appendPQExpBufferStr +#define appendStringInfoChar appendPQExpBufferChar +#define appendBinaryStringInfo appendBinaryPQExpBuffer + +extern int appendStringInfoFile(StringInfo str, FILE *fp); +extern int appendStringInfoFd(StringInfo str, int fd); + +extern bool parse_bool(const char *value, bool *result); +extern bool parse_bool_with_len(const char *value, size_t len, bool *result); +extern bool parse_int32(const char *value, int32 *result, int flags); +extern bool parse_uint32(const char *value, uint32 *result, int flags); +extern bool parse_int64(const char *value, int64 *result, int flags); +extern bool parse_uint64(const char *value, uint64 *result, int flags); +extern bool parse_time(const char *value, time_t *result, bool utc_default); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); + +extern void convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit); +extern void convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit); + +#define IsSpace(c) (isspace((unsigned char)(c))) +#define IsAlpha(c) (isalpha((unsigned char)(c))) +#define IsAlnum(c) (isalnum((unsigned char)(c))) +#define IsIdentHead(c) (IsAlpha(c) || (c) == '_') +#define IsIdentBody(c) (IsAlnum(c) || (c) == '_') +#define ToLower(c) (tolower((unsigned char)(c))) +#define ToUpper(c) (toupper((unsigned char)(c))) + +/* + * socket operations + */ +extern int wait_for_socket(int sock, struct timeval *timeout); +extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); + +#ifdef WIN32 +extern int sleep(unsigned int seconds); +extern int usleep(unsigned int usec); +#endif + +#endif /* PGUT_H */ diff --git a/src/utils/thread.c b/src/utils/thread.c new file mode 100644 index 000000000..82c237641 --- /dev/null +++ b/src/utils/thread.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * thread.c: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "thread.h" + +pthread_t main_tid = 0; + +#ifdef WIN32 +#include + +typedef struct win32_pthread +{ + HANDLE handle; + void *(*routine) (void *); + void *arg; + void *result; +} win32_pthread; + +static long mutex_initlock = 0; + +static unsigned __stdcall +win32_pthread_run(void *arg) +{ + win32_pthread *th = (win32_pthread *)arg; + + th->result = th->routine(th->arg); + + return 0; +} + +int +pthread_create(pthread_t *thread, + pthread_attr_t *attr, + void *(*start_routine) (void *), + void *arg) +{ + int save_errno; + win32_pthread *th; + + th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); + th->routine = start_routine; + th->arg = arg; + th->result = NULL; + + th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); + if (th->handle == NULL) + { + save_errno = errno; + free(th); + return save_errno; + } + + *thread = th; + return 0; +} + +int +pthread_join(pthread_t th, void **thread_return) +{ + if (th == NULL || th->handle == NULL) + return errno = EINVAL; + + if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) + { + _dosmaperr(GetLastError()); + return errno; + } + + if (thread_return) + *thread_return = th->result; + + CloseHandle(th->handle); + free(th); + return 0; +} + +#endif /* WIN32 */ + +int +pthread_lock(pthread_mutex_t *mp) +{ +#ifdef WIN32 + if (*mp == NULL) + { + while (InterlockedExchange(&mutex_initlock, 1) == 1) + /* loop, another thread own the lock */ ; + if (*mp == NULL) + { + if (pthread_mutex_init(mp, NULL)) + return -1; + } + InterlockedExchange(&mutex_initlock, 0); + } +#endif + return pthread_mutex_lock(mp); +} diff --git a/src/utils/thread.h b/src/utils/thread.h new file mode 100644 index 000000000..064605331 --- /dev/null +++ b/src/utils/thread.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * thread.h: - multi-platform pthread implementations. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef PROBACKUP_THREAD_H +#define PROBACKUP_THREAD_H + +#ifdef WIN32 +#include "postgres_fe.h" +#include "port/pthread-win32.h" + +/* Use native win32 threads on Windows */ +typedef struct win32_pthread *pthread_t; +typedef int pthread_attr_t; + +#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } +#define PTHREAD_ONCE_INIT false + +extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); +extern int pthread_join(pthread_t th, void **thread_return); +#else +/* Use platform-dependent pthread capability */ +#include +#endif + +extern pthread_t main_tid; + +extern int pthread_lock(pthread_mutex_t *mp); + +#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 000000000..bc82e8116 --- /dev/null +++ b/src/validate.c @@ -0,0 +1,354 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2017, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + pgBackupWriteBackupControlFile(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + crc = pgFileGetCRC(file->path); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + break; + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + current_backup = (pgBackup *) parray_get(backups, i); + + /* Valiate each backup along with its xlog files. */ + pgBackupValidate(current_backup); + + /* Ensure that the backup has valid list of parent backups */ + if (current_backup->status == BACKUP_STATUS_OK) + { + pgBackup *base_full_backup = current_backup; + + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + base_full_backup = find_parent_backup(current_backup); + + if (base_full_backup == NULL) + elog(ERROR, "Valid full backup for backup %s is not found.", + base36enc(current_backup->start_time)); + } + + /* Validate corresponding WAL files */ + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + } + + /* Mark every incremental backup between corrupted backup and nearest FULL backup as orphans */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + int j; + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (backup->backup_mode == BACKUP_MODE_FULL) + break; + if (backup->status != BACKUP_STATUS_OK) + continue; + else + { + backup->status = BACKUP_STATUS_ORPHAN; + pgBackupWriteBackupControlFile(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s is corrupted", + base36enc(backup->start_time), current_backup_id); + } + } + free(current_backup_id); + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 000000000..31dfb6560 --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,24 @@ +[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) + +``` +Note: For now there are tests only for Linix +``` + + +``` +Check physical correctness of restored instances: + Apply this patch to disable HINT BITS: https://gist.github.com/gsmol/2bb34fd3ba31984369a72cc1c27a36b6 + export PG_PROBACKUP_PARANOIA=ON + +Check archive compression: + export ARCHIVE_COMPRESSION=ON + +Specify path to pg_probackup binary file. By default tests use /pg_probackup/ + export PGPROBACKUPBIN= + +Usage: + pip install testgres + pip install psycopg2 + export PG_CONFIG=/path/to/pg_config + python -m unittest [-v] tests[.specific_module][.class.test] +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..aeeabf2a9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,69 @@ +import unittest + +from . import init_test, option_test, show_test, \ + backup_test, delete_test, restore_test, validate_test, \ + retention_test, ptrack_clean, ptrack_cluster, \ + ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \ + ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ + ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ + false_positive, replica, compression, page, ptrack, archive, \ + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() +# suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(archive)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) +# suite.addTests(loader.loadTestsFromModule(cfs_restore)) +# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) +# suite.addTests(loader.loadTestsFromModule(logging)) + suite.addTests(loader.loadTestsFromModule(compression)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(exclude)) + suite.addTests(loader.loadTestsFromModule(false_positive)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page)) + suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_clean)) + suite.addTests(loader.loadTestsFromModule(ptrack_cluster)) + suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace)) + suite.addTests(loader.loadTestsFromModule(ptrack_recovery)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) + suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(replica)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + + return suite + +# test_pgpro434_2 unexpected success +# ToDo: +# archive: +# discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work +# replica: +# backup should exit with correct error message if some master* option is missing +# --master* options shoukd not work when backuping master +# logging: +# https://jira.postgrespro.ru/browse/PGPRO-584 +# https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md +# ptrack: +# ptrack backup on replica should work correctly +# archive: +# immediate recovery and full recovery +# backward compatibility: +# previous version catalog must be readable by newer version +# incremental chain from previous version can be continued +# backups from previous version can be restored +# 10vanilla_1.3ptrack + +# 10vanilla+ +# 9.6vanilla_1.3ptrack + diff --git a/tests/archive.py b/tests/archive.py new file mode 100644 index 000000000..8b8eb71aa --- /dev/null +++ b/tests/archive.py @@ -0,0 +1,833 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script +from datetime import datetime, timedelta +import subprocess +from sys import exit +from time import sleep + + +module_name = 'archive' + + +class ArchiveTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_pgpro434_1(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector from " + "generate_series(0,100) i") + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node) + node.slow_start() + + # Recreate backup calagoue + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Make backup + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-file=verbose"]) + node.cleanup() + + # Restore Database + self.restore_node( + backup_dir, 'node', node, + options=["--recovery-target-action=promote"]) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro434_2(self): + """ + Check that timelines are correct. + WAITING PGPRO-1053 for --immediate + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FIRST TIMELINE + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + backup_id = self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select 100501 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + # SECOND TIMELIN + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 " + "from t_heap where id = 100501)")[0][0], + 'data after restore not equal to original data') + + node.safe_psql( + "postgres", + "insert into t_heap select 2 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(100,200) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select 100502 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # THIRD TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print( + node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + node.safe_psql( + "postgres", + "insert into t_heap select 3 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(200,300) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.safe_psql( + "postgres", + "insert into t_heap select 100503 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # FOURTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fourth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # FIFTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fifth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # SIXTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Sixth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 from t_heap where id > 100500)")[0][0], + 'data after restore not equal to original data') + + self.assertEqual( + result, + node.safe_psql( + "postgres", + "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro434_3(self): + """Check pg_stop_backup_timeout, needed backup_timeout""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + archive_script_path = os.path.join(backup_dir, 'archive_script.sh') + with open(archive_script_path, 'w+') as f: + f.write( + archive_script.format( + backup_dir=backup_dir, node_name='node', count_limit=2)) + + st = os.stat(archive_script_path) + os.chmod(archive_script_path, st.st_mode | 0o111) + node.append_conf( + 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( + archive_script_path)) + node.slow_start() + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=verbose", + "--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pg_stop_backup failed to answer.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: pg_stop_backup doesn't answer" in e.message and + "cancel it" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + os.remove(file) + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_arhive_push_file_exists_overwrite(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + file = os.path.join(wals_dir, '000000010000000000000001.gz') + else: + file = os.path.join(wals_dir, '000000010000000000000001') + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content and + 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + self.set_archiving(backup_dir, 'node', node, overwrite=True) + node.reload() + self.switch_wal_segment(node) + sleep(2) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_replica_archive(self): + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'max_wal_size': '1GB'} + ) + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica, synchronous=True) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # ADD INSTANCE 'REPLICA' + + sleep(1) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=30', '--log-level-file=verbose', + '--master-host=localhost', '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_parallel_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=20', + '--log-level-file=verbose', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)] + ) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_concurrent_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node( + backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=True) + # ADD INSTANCE REPLICA + # self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node( + backup_dir, 'master', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog(self): + """Test backup with pg_receivexlog wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, + node.safe_psql( + "postgres", "SELECT * FROM t_heap" + ), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_compression_pg10(self): + """Test backup with pg_receivewal compressed wal delivary method""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') + ], async=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'node', node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) diff --git a/tests/auth_test.py b/tests/auth_test.py new file mode 100644 index 000000000..fc21a480d --- /dev/null +++ b/tests/auth_test.py @@ -0,0 +1,391 @@ +""" +The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. + - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 +""" + +import os +import unittest +import signal +import time + +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import StartNodeException + +module_name = 'auth_test' +skip_test = False + + +try: + from pexpect import * +except ImportError: + skip_test = True + + +class SimpleAuthTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_backup_via_unpriviledged_user(self): + """ + Make node, create unpriviledged user, try to + run a backups without EXECUTE rights on + certain functions + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + + time.sleep(1) + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied for function " + "pg_create_restore_point\nquery was: " + "SELECT pg_catalog.pg_create_restore_point($1)", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_create_restore_point(text) TO backup;") + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on EXECUTE.") + except ProbackupException as e: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.get_version(node) < self.version_to_num('10.0'): + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_stop_backup(boolean, boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql("postgres", "CREATE DATABASE test1") + + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + + node.safe_psql( + "test1", "create table t1 as select generate_series(0,100)") + + node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") + node.restart() + + try: + self.backup_node( + backup_dir, 'node', node, options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role to clear ptrack files\n" + "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + time.sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + self.assertEqual( + 1, 0, + "Expecting Error due to missing grant on clearing ptrack_files.") + except ProbackupException as e: + self.assertIn( + "ERROR: must be superuser or replication role read ptrack files\n" + "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "ALTER ROLE backup REPLICATION") + + time.sleep(1) + + # FULL + self.backup_node( + backup_dir, 'node', node, + options=['-U', 'backup']) + + # PTRACK + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + +class AuthTest(unittest.TestCase): + pb = None + node = None + + @classmethod + def setUpClass(cls): + + super(AuthTest, cls).setUpClass() + + cls.pb = ProbackupTest() + cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + + cls.node = cls.pb.make_simple_node( + base_dir="{}/node".format(module_name), + set_replication=True, + initdb_params=['--data-checksums', '--auth-host=md5'], + pg_options={ + 'wal_level': 'replica' + } + ) + modify_pg_hba(cls.node) + + cls.pb.init_pb(cls.backup_dir) + cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) + cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + try: + cls.node.start() + except StartNodeException: + raise unittest.skip("Node hasn't started") + + cls.node.safe_psql("postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ + GRANT USAGE ON SCHEMA pg_catalog TO backup; \ + GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ + GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + + @classmethod + def tearDownClass(cls): + cls.node.cleanup() + cls.pb.del_test_dir(module_name, '') + + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): + self.cmd = ['backup', + '-B', self.backup_dir, + '--instance', self.node.name, + '-h', '127.0.0.1', + '-p', str(self.node.port), + '-U', 'backup', + '-b', 'FULL' + ] + + def tearDown(self): + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSWORD"] + + if "PGPASSWORD" in self.pb.test_env.keys(): + del self.pb.test_env["PGPASSFILE"] + + try: + os.remove(self.pgpass_file) + except OSError: + pass + + def test_empty_password(self): + """ Test case: PGPB_AUTH03 - zero password length """ + try: + self.assertIn("ERROR: no password supplied", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_wrong_password(self): + """ Test case: PGPB_AUTH04 - incorrect password """ + try: + self.assertIn("password authentication failed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password(self): + """ Test case: PGPB_AUTH01 - correct password """ + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_right_password_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertIn("completed", + str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) + ) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_ctrl_c_event(self): + """ Test case: PGPB_AUTH02 - send interrupt signal """ + try: + run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + except TIMEOUT: + self.fail("Error: CTRL+C event ignored") + + def test_pgpassfile_env(self): + """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ + path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(path, line) + self.pb.test_env["PGPASSFILE"] = path + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpass(self): + """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + create_pgpass(self.pgpass_file, line) + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword(self): + """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + def test_pgpassword_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + create_pgpass(self.pgpass_file, line) + self.pb.test_env["PGPASSWORD"] = "password" + try: + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + except ProbackupException as e: + self.fail(e) + + +def run_pb_with_auth(cmd, password=None, kill=False): + try: + with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + result = probackup.expect(u"Password for user .*:", 5) + if kill: + probackup.kill(signal.SIGINT) + elif result == 0: + probackup.sendline(password) + probackup.expect(EOF) + return probackup.before + else: + raise ExceptionPexpect("Other pexpect errors.") + except TIMEOUT: + raise TIMEOUT("Timeout error.") + except ExceptionPexpect: + raise ExceptionPexpect("Pexpect error.") + + +def modify_pg_hba(node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) + + +def create_pgpass(path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py new file mode 100644 index 000000000..1fa74643a --- /dev/null +++ b/tests/backup_test.py @@ -0,0 +1,522 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +module_name = 'backup' + + +class BackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # postmaster.pid and postmaster.opts shouldn't be copied + excluded = True + db_dir = os.path.join( + backup_dir, "backups", 'node', backup_id, "database") + + for f in os.listdir(db_dir): + if ( + os.path.isfile(os.path.join(db_dir, f)) and + ( + f == "postmaster.pid" or + f == "postmaster.opts" + ) + ): + excluded = False + self.assertEqual(excluded, True) + + # page backup mode + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + # print self.show_pb(node) + show_backup = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Check parent backup + self.assertEqual( + backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + show_backup = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PTRACK") + + # Check parent backup + self.assertEqual( + page_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup['id'])["parent-backup-id"]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smooth_checkpoint(self): + """full backup with smooth checkpoint""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + node.stop() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page-level backup without validated full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of validation of corrupted backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'\n" in e.message and + "WARNING: Backup file \"{0}\" is not found\n".format( + file) in e.message and + "WARNING: Backup {0} data files are corrupted\n".format( + backup_id) in e.message and + "WARNING: Some backups are not valid\n" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_threads_stream(self): + """ptrack multi thread backup mode and stream""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_in_pgdata_pgpro_1376(self): + """PGPRO-1376 """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=( + os.path.join( + node.data_dir, 'somedirectory', '100500')) + ) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=(os.path.join(node.data_dir)) + ) + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace tblspace1 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of too many levels " + "of symbolic linking\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Too many levels of symbolic links' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap2") + node.safe_psql( + "postgres", + "drop tablespace tblspace2") + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content(node.data_dir) + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap1'::regclass::oid" + ).rstrip() + + list = [] + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file == relfilenode: + path = os.path.join(root, file) + list = list + [path] + + # We expect that relfilenode occures only once + if len(list) > 1: + message = "" + for string in list: + message = message + string + "\n" + self.assertEqual( + 1, 0, + "Following file copied twice by backup:\n {0}".format( + message) + ) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py new file mode 100644 index 000000000..412320327 --- /dev/null +++ b/tests/cfs_backup.py @@ -0,0 +1,1161 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join( + self.tmp_path, module_name, self.fname, 'backup') + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2', + 'shared_buffers': '200MB' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( + tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not created " + "or it create without compressions" + ) + + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + + # --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace(self): + """Case: Check fullbackup empty compressed tablespace""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_stream(self): + """Case: Check fullbackup empty compressed tablespace with options stream""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """Case: Make full backup after created table in the tablespace""" + if not self.enterprise: + return + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "\n ERROR: {0}\n CMD: {1}".format( + repr(e.message), + repr(self.cmd) + ) + ) + return False + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format( + os.path.join(self.backup_dir, 'node', backup_id)) + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # --- Section: Incremental from fill tablespace --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,25) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap', tblspace_name) + ) + + full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap') + ) + + page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments_in_multiple_tablespaces(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + tblspace_name_1 = 'tblspace_name_1' + tblspace_name_2 = 'tblspace_name_2' + + self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) + self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_1', tblspace_name_1) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_2', tblspace_name_2) + ) + + full_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + full_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_1') + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_2') + ) + + page_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + page_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, options=["-j", "4"]) + self.node.start() + self.assertEqual( + full_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + full_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_1), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name_2), + ignore_errors=True) + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, options=["-j", "4"]) + self.node.start() + self.assertEqual( + page_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + page_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # --- Make backup with not valid data(broken .cfm) --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_cfm_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_file_pg_compression_from_tablespace_dir(self): + os.remove( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_data_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_cfm_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_data_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_file_pg_compression_into_tablespace_dir(self): + + corrupted_file = find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + +# # --- End ---# +# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') +# def tearDown(self): +# self.node.cleanup() +# self.del_test_dir(module_name, self.fname) + + +#class CfsBackupEncTest(CfsBackupNoEncTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py new file mode 100644 index 000000000..73553a305 --- /dev/null +++ b/tests/cfs_restore.py @@ -0,0 +1,450 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'cfs_restore' + +tblspace_name = 'cfs_tblspace' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreBase(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', +# 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + 'max_wal_senders': '2' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.start() + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + self.add_data_in_cluster() + + self.backup_id = None + try: + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + def add_data_in_cluster(self): + pass + + def tearDown(self): + self.node.cleanup() + self.del_test_dir(module_name, self.fname) + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + self.node.stop(["-m", "immediate"]) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ) + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + +class CfsRestoreNoencTest(CfsRestoreBase): + def add_data_in_cluster(self): + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = self.node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + + # --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in tablespace dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + self.node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + self.node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', self.node_new, backup_id=self.backup_id, options=['-j', '5']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node_new.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + self.node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-j", "3", "-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + +#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() +# +# +#class CfsRestoreEncTest(CfsRestoreNoencTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py new file mode 100644 index 000000000..eea6f0e21 --- /dev/null +++ b/tests/cfs_validate_backup.py @@ -0,0 +1,25 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_validate_backup' +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/compression.py b/tests/compression.py new file mode 100644 index 000000000..aa2753821 --- /dev/null +++ b/tests/compression.py @@ -0,0 +1,496 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'compression' + + +class CompressionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_compression_stream_zlib(self): + """make archive node, make full and page stream backups, check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=[ + '--stream', + '--compress-algorithm=zlib']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--stream', '--compress-algorithm=zlib', + '--log-level-console=verbose', + '--log-level-file=verbose']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_zlib(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=["--compress-algorithm=zlib"]) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--compress-algorithm=zlib"]) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_stream_pglz(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_archive_pglz(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--compress-algorithm=pglz']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=ptrack_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_compression_wrong_algorithm(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--compress-algorithm=bla-blah']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because compress-algorithm is invalid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: invalid compress algorithm value "bla-blah"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delete_test.py b/tests/delete_test.py new file mode 100644 index 000000000..4afb15ae0 --- /dev/null +++ b/tests/delete_test.py @@ -0,0 +1,203 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from sys import exit + + +module_name = 'delete' + + +class DeleteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_full_backups(self): + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_page(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_orphaned_wal_segments(self): + """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + # Check wals + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + original_wal_quantity = len(wals) + + # delete second full backup + self.delete_pb(backup_dir, 'node', backup_2_id) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + # try to delete wals for second backup + self.delete_pb(backup_dir, 'node', options=['--wal']) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # delete first full backup + self.delete_pb(backup_dir, 'node', backup_1_id) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + result = self.delete_pb(backup_dir, 'node', options=['--wal']) + # delete useless wals + self.assertTrue('INFO: removed min WAL segment' in result + and 'INFO: removed max WAL segment' in result) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # Check quantity, it should be lower than original + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + + # Delete last backup + self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + self.assertEqual (0, len(wals), "Number of wals should be equal to 0") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py new file mode 100644 index 000000000..40450016b --- /dev/null +++ b/tests/delta.py @@ -0,0 +1,1265 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import QueryException +import subprocess +import time + + +module_name = 'delta' + + +class DeltaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_2(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose", + "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_3(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10100000) i;" + ) + filepath = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')" + ).rstrip() + + self.backup_node(backup_dir, 'node', node) + + print(os.path.join(node.data_dir, filepath + '.1')) + os.unlink(os.path.join(node.data_dir, filepath + '.1')) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-j", "1", + "--log-level-file=verbose" + ] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_stream(self): + """ + make archive node, take full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_archive(self): + """ + make archive node, take full and delta archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check delta and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init( + scale=100, + options=['--tablespace=somedata', '--no-vacuum']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # delta BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream']) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_vacuum_full(self): + """ + make node, make full and delta stream backups, + restore them and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take delta backup, + restore database and check it presense + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND DELTA BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exists_in_previous_backup(self): + """ + Make node, take full backup, create table, take page backup, + take delta backup, check that file is no fully copied to delta backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + filepath = node.safe_psql( + "postgres", + "SELECT pg_relation_filepath('t_heap')").rstrip() + self.backup_node( + backup_dir, + 'node', + node, + options=["--stream"]) + + # PAGE BACKUP + backup_id = self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + +# if self.paranoia: +# pgdata_page = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream", "--log-level-file=verbose"] + ) +# if self.paranoia: +# pgdata_delta = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) +# self.compare_pgdata( +# pgdata_page, pgdata_delta) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", "--log-level-file=verbose", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_delta(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # DELTA BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_delta(self): + """ + Make node, take full backup, create database, + take delta backup, alter database tablespace location, + take delta backup restore last delta backup. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql( + "postgres", + "create database db1 tablespace = 'somedata'") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter database db1 set tablespace somedata_new" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_1(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream", "--log-level-file=verbose"]) + + # open log file and check + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) + f.close + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', + "Backup Status should be OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_corruption_heal_via_ptrack_2(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + node.start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of page " + "corruption in PostgreSQL instance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', + "Backup Status should be ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude.py new file mode 100644 index 000000000..48b7889c7 --- /dev/null +++ b/tests/exclude.py @@ -0,0 +1,164 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'exclude' + + +class ExcludeTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_exclude_temp_tables(self): + """ + make node without archiving, create temp table, take full backup, + check that temp table not present in backup catalogue + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create temp table test as " + "select generate_series(0,50050000)::text") + conn.commit() + + temp_schema_name = conn.execute( + "SELECT nspname FROM pg_namespace " + "WHERE oid = pg_my_temp_schema()")[0][0] + conn.commit() + + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] + conn.commit() + + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] + conn.commit() + + heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] + conn.commit() + + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + conn.commit() + + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] + conn.commit() + + temp_table_filename = os.path.basename(heap_path) + temp_idx_filename = os.path.basename(index_path) + temp_toast_filename = os.path.basename(toast_path) + temp_idx_toast_filename = os.path.basename(toast_idx_path) + + self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file in [ + temp_table_filename, temp_table_filename + ".1", + temp_idx_filename, + temp_idx_filename + ".1", + temp_toast_filename, + temp_toast_filename + ".1", + temp_idx_toast_filename, + temp_idx_toast_filename + ".1" + ]: + self.assertEqual( + 1, 0, + "Found temp table file in backup catalogue.\n " + "Filepath: {0}".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_1(self): + """ + make node without archiving, create unlogged table, take full backup, + alter table to unlogged, take ptrack backup, restore ptrack backup, + check that PGDATA`s are physically the same + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + "shared_buffers": "10MB", + "fsync": "off", + 'ptrack_enable': 'on'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create unlogged table test as " + "select generate_series(0,5005000)::text") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + node.safe_psql('postgres', "alter table test set logged") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out new file mode 100644 index 000000000..35f584062 --- /dev/null +++ b/tests/expected/option_help.out @@ -0,0 +1,95 @@ + +pg_probackup - utility to manage backup/recovery of PostgreSQL database. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-dir --instance=instance_name + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup show-config -B backup-dir --instance=instance_name + [--format=format] + + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-C] [--stream [-S slot-name]] [--backup-pg-log] + [-j num-threads] [--archive-timeout=archive-timeout] + [--progress] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--delete-expired] [--delete-wal] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--master-db=db_name] [--master-host=host_name] + [--master-port=port] [--master-user=user_name] + [--replica-timeout=timeout] + + pg_probackup restore -B backup-dir --instance=instance_name + [-D pgdata-dir] [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--immediate] [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-as-replica] + [--no-validate] + + pg_probackup validate -B backup-dir [--instance=instance_name] + [-i backup-id] [--progress] + [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-name=target-name] + [--timeline=timeline] + + pg_probackup show -B backup-dir + [--instance=instance_name [-i backup-id]] + [--format=format] + + pg_probackup delete -B backup-dir --instance=instance_name + [--wal] [-i backup-id | --expired] + + pg_probackup merge -B backup-dir --instance=instance_name + -i backup-id + + pg_probackup add-instance -B backup-dir -D pgdata-dir + --instance=instance_name + + pg_probackup del-instance -B backup-dir + --instance=instance_name + + pg_probackup archive-push -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [--compress [--compress-level=compress-level]] + [--overwrite] + + pg_probackup archive-get -B backup-dir --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + +Read the website for details. +Report bugs to . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out new file mode 100644 index 000000000..35e212c3e --- /dev/null +++ b/tests/expected/option_version.out @@ -0,0 +1 @@ +pg_probackup 2.0.18 \ No newline at end of file diff --git a/tests/false_positive.py b/tests/false_positive.py new file mode 100644 index 000000000..1884159b2 --- /dev/null +++ b/tests/false_positive.py @@ -0,0 +1,333 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + + +module_name = 'false_positive' + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_wal_lost_segment(self): + """Loose segment located between backups. ExpectedFailure. This is BUG """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, "wal", 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + + # We just lost a wal segment and know nothing about it + self.backup_node(backup_dir, 'node', node) + self.assertTrue( + 'validation completed successfully' in self.validate_pb( + backup_dir, 'node')) + ######## + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", + backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + self.assertFalse( + True, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_1(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_ptrack_concurrent_get_and_clear_2(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('pthread_create') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + tablespace_oid = node.safe_psql( + "postgres", + "select oid from pg_tablespace " + "where spcname = 'pg_default'").rstrip() + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap'::regclass::oid").rstrip() + + node.safe_psql( + "postgres", + "SELECT pg_ptrack_get_and_clear({0}, {1})".format( + tablespace_oid, relfilenode)) + + gdb._execute("delete breakpoints") + gdb.continue_execution_until_exit() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup ptrack_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_multiple_delete(self): + """delete multiple backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + self.delete_pb(backup_dir, 'node', options= + ["-i {0}".format(backup_1_id), "-i {0}".format(backup_2_id), "-i {0}".format(backup_3_id)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 000000000..ac64c4230 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +#from . import * \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py new file mode 100644 index 000000000..67e2b331b --- /dev/null +++ b/tests/helpers/cfs_helpers.py @@ -0,0 +1,91 @@ +import os +import re +import random +import string + + +def find_by_extensions(dirs=None, extensions=None): + """ + find_by_extensions(['path1','path2'],['.txt','.log']) + :return: + Return list of files include full path by file extensions + """ + files = [] + new_dirs = [] + + if dirs is not None and extensions is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.splitext(d)[1] in extensions: + files.append(d) + + if new_dirs: + files.extend(find_by_extensions(new_dirs, extensions)) + + return files + + +def find_by_pattern(dirs=None, pattern=None): + """ + find_by_pattern(['path1','path2'],'^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + files = [] + new_dirs = [] + + if dirs is not None and pattern is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if re.match(pattern,d): + files.append(d) + + if new_dirs: + files.extend(find_by_pattern(new_dirs, pattern)) + + return files + + +def find_by_name(dirs=None, filename=None): + files = [] + new_dirs = [] + + if dirs is not None and filename is not None: + for d in dirs: + try: + new_dirs += [os.path.join(d, f) for f in os.listdir(d)] + except OSError: + if os.path.basename(d) in filename: + files.append(d) + + if new_dirs: + files.extend(find_by_name(new_dirs, filename)) + + return files + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + except OSError: + return False + + try: + with open(filename, "rb+") as f: + f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) + f.write(random_string(0.1*file_size)) + f.close() + except OSError: + return False + + return True + + +def random_string(n): + a = string.ascii_letters + string.digits + return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py new file mode 100644 index 000000000..0d04d8983 --- /dev/null +++ b/tests/helpers/ptrack_helpers.py @@ -0,0 +1,1300 @@ +# you need os for unittest to work +import os +from sys import exit, argv, version_info +import subprocess +import shutil +import six +import testgres +import hashlib +import re +import pwd +import select +import psycopg2 +from time import sleep +import re +import json + +idx_ptrack = { + 't_heap': { + 'type': 'heap' + }, + 't_btree': { + 'type': 'btree', + 'column': 'text', + 'relation': 't_heap' + }, + 't_seq': { + 'type': 'seq', + 'column': 't_seq', + 'relation': 't_heap' + }, + 't_spgist': { + 'type': 'spgist', + 'column': 'text', + 'relation': 't_heap' + }, + 't_brin': { + 'type': 'brin', + 'column': 'text', + 'relation': 't_heap' + }, + 't_gist': { + 'type': 'gist', + 'column': 'tsvector', + 'relation': 't_heap' + }, + 't_gin': { + 'type': 'gin', + 'column': 'tsvector', + 'relation': 't_heap' + }, +} + +archive_script = """ +#!/bin/bash +count=$(ls {backup_dir}/test00* | wc -l) +if [ $count -ge {count_limit} ] +then + exit 1 +else + cp $1 {backup_dir}/wal/{node_name}/$2 + count=$((count+1)) + touch {backup_dir}/test00$count + exit 0 +fi +""" +warning = """ +Wrong splint in show_pb +Original Header: +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +def dir_files(base_dir): + out_list = [] + for dir_name, subdir_list, file_list in os.walk(base_dir): + if dir_name != base_dir: + out_list.append(os.path.relpath(dir_name, base_dir)) + for fname in file_list: + out_list.append( + os.path.relpath(os.path.join( + dir_name, fname), base_dir) + ) + out_list.sort() + return out_list + + +def is_enterprise(): + # pg_config --help + p = subprocess.Popen( + [os.environ['PG_CONFIG'], '--help'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if b'postgrespro.ru' in p.communicate()[0]: + return True + else: + return False + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +def slow_start(self, replica=False): + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # raise_operational_error=False) + + self.start() + if not replica: + while True: + try: + self.poll_query_until( + "postgres", + "SELECT not pg_is_in_recovery()") + break + except Exception as e: + continue + else: + self.poll_query_until( + "postgres", + "SELECT pg_is_in_recovery()") + +# while True: +# try: +# self.poll_query_until( +# "postgres", +# "SELECT pg_is_in_recovery()") +# break +# except ProbackupException as e: +# continue + + +class ProbackupTest(object): + # Class attributes + enterprise = is_enterprise() + + def __init__(self, *args, **kwargs): + super(ProbackupTest, self).__init__(*args, **kwargs) + if '-v' in argv or '--verbose' in argv: + self.verbose = True + else: + self.verbose = False + + self.test_env = os.environ.copy() + envs_list = [ + "LANGUAGE", + "LC_ALL", + "PGCONNECT_TIMEOUT", + "PGDATA", + "PGDATABASE", + "PGHOSTADDR", + "PGREQUIRESSL", + "PGSERVICE", + "PGSSLMODE", + "PGUSER", + "PGPORT", + "PGHOST" + ] + + for e in envs_list: + try: + del self.test_env[e] + except: + pass + + self.test_env["LC_MESSAGES"] = "C" + self.test_env["LC_TIME"] = "C" + + self.paranoia = False + if 'PG_PROBACKUP_PARANOIA' in self.test_env: + if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON': + self.paranoia = True + + self.archive_compress = False + if 'ARCHIVE_COMPRESSION' in self.test_env: + if self.test_env['ARCHIVE_COMPRESSION'] == 'ON': + self.archive_compress = True + try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + cache_pg_config=False, + node_cleanup_full=False) + except: + pass + + self.helpers_path = os.path.dirname(os.path.realpath(__file__)) + self.dir_path = os.path.abspath( + os.path.join(self.helpers_path, os.pardir) + ) + self.tmp_path = os.path.abspath( + os.path.join(self.dir_path, 'tmp_dirs') + ) + try: + os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) + except: + pass + + self.user = self.get_username() + self.probackup_path = None + if "PGPROBACKUPBIN" in self.test_env: + if ( + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + ): + self.probackup_path = self.test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBINDIR is not an executable file') + if not self.probackup_path: + self.probackup_path = os.path.abspath(os.path.join( + self.dir_path, "../pg_probackup")) + + def make_simple_node( + self, + base_dir=None, + set_replication=False, + initdb_params=[], + pg_options={}): + + real_base_dir = os.path.join(self.tmp_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = testgres.get_new_node('test', base_dir=real_base_dir) + # bound method slow_start() to 'node' class instance + node.slow_start = slow_start.__get__(node) + node.should_rm_dirs = True + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # Sane default parameters + node.append_conf("postgresql.auto.conf", "max_connections = 100") + node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") + node.append_conf("postgresql.auto.conf", "fsync = on") + node.append_conf("postgresql.auto.conf", "wal_level = logical") + node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + + node.append_conf( + "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf("postgresql.auto.conf", "log_statement = none") + node.append_conf("postgresql.auto.conf", "log_duration = on") + node.append_conf( + "postgresql.auto.conf", "log_min_duration_statement = 0") + node.append_conf("postgresql.auto.conf", "log_connections = on") + node.append_conf("postgresql.auto.conf", "log_disconnections = on") + + # Apply given parameters + for key, value in six.iteritems(pg_options): + node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + + # Allow replication in pg_hba.conf + if set_replication: + node.append_conf( + "pg_hba.conf", + "local replication all trust\n") + node.append_conf( + "postgresql.auto.conf", + "max_wal_senders = 10") + + return node + + def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): + res = node.execute( + "postgres", + "select exists" + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertFalse( + res[0][0], + 'Tablespace "{0}" already exists'.format(tblspc_name) + ) + + if not tblspc_path: + tblspc_path = os.path.join( + node.base_dir, '{0}'.format(tblspc_name)) + cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( + tblspc_name, tblspc_path) + if cfs: + cmd += " with (compression=true)" + + if not os.path.exists(tblspc_path): + os.makedirs(tblspc_path) + res = node.safe_psql("postgres", cmd) + # Check that tablespace was successfully created + # self.assertEqual( + # res[0], 0, + # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + + def get_tblspace_path(self, node, tblspc_name): + return os.path.join(node.base_dir, tblspc_name) + + def get_fork_size(self, node, fork_name): + return node.execute( + "postgres", + "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] + + def get_fork_path(self, node, fork_name): + return os.path.join( + node.base_dir, 'data', node.execute( + "postgres", + "select pg_relation_filepath('{0}')".format( + fork_name))[0][0] + ) + + def get_md5_per_page_for_fork(self, file, size_in_pages): + pages_per_segment = {} + md5_per_page = {} + nsegments = size_in_pages/131072 + if size_in_pages % 131072 != 0: + nsegments = nsegments + 1 + + size = size_in_pages + for segment_number in range(nsegments): + if size - 131072 > 0: + pages_per_segment[segment_number] = 131072 + else: + pages_per_segment[segment_number] = size + size = size - 131072 + + for segment_number in range(nsegments): + offset = 0 + if segment_number == 0: + file_desc = os.open(file, os.O_RDONLY) + start_page = 0 + end_page = pages_per_segment[segment_number] + else: + file_desc = os.open( + file+".{0}".format(segment_number), os.O_RDONLY + ) + start_page = max(md5_per_page)+1 + end_page = end_page + pages_per_segment[segment_number] + + for page in range(start_page, end_page): + md5_per_page[page] = hashlib.md5( + os.read(file_desc, 8192)).hexdigest() + offset += 8192 + os.lseek(file_desc, offset, 0) + os.close(file_desc) + + return md5_per_page + + def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): + + if self.get_pgpro_edition(node) == 'enterprise': + header_size = 48 + else: + header_size = 24 + ptrack_bits_for_fork = [] + + page_body_size = 8192-header_size + byte_size = os.path.getsize(file + '_ptrack') + npages = byte_size/8192 + if byte_size % 8192 != 0: + print('Ptrack page is not 8k aligned') + sys.exit(1) + + file = os.open(file + '_ptrack', os.O_RDONLY) + + for page in range(npages): + offset = 8192*page+header_size + os.lseek(file, offset, 0) + lots_of_bytes = os.read(file, page_body_size) + byte_list = [ + lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes)) + ] + for byte in byte_list: + # byte_inverted = bin(int(byte, base=16))[2:][::-1] + # bits = (byte >> x) & 1 for x in range(7, -1, -1) + byte_inverted = bin(ord(byte))[2:].rjust(8, '0')[::-1] + for bit in byte_inverted: + # if len(ptrack_bits_for_fork) < size: + ptrack_bits_for_fork.append(int(bit)) + + os.close(file) + return ptrack_bits_for_fork + + def check_ptrack_sanity(self, idx_dict): + success = True + if idx_dict['new_size'] > idx_dict['old_size']: + size = idx_dict['new_size'] + else: + size = idx_dict['old_size'] + for PageNum in range(size): + if PageNum not in idx_dict['old_pages']: + # Page was not present before, meaning that relation got bigger + # Ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was added,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + # print(idx_dict) + success = False + continue + if PageNum not in idx_dict['new_pages']: + # Page is not present now, meaning that relation got smaller + # Ptrack should be equal to 0, + # We are not freaking out about false positive stuff + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was deleted,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + continue + + # Ok, all pages in new_pages that do not have + # corresponding page in old_pages are been dealt with. + # We can now safely proceed to comparing old and new pages + if idx_dict['new_pages'][ + PageNum] != idx_dict['old_pages'][PageNum]: + # Page has been changed, + # meaning that ptrack should be equal to 1 + if idx_dict['ptrack'][PageNum] != 1: + if self.verbose: + print( + 'Page Number {0} of type {1} was changed,' + ' but ptrack value is {2}. THIS IS BAD'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum]) + ) + print( + "\n Old checksumm: {0}\n" + " New checksumm: {1}".format( + idx_dict['old_pages'][PageNum], + idx_dict['new_pages'][PageNum]) + ) + + if PageNum == 0 and idx_dict['type'] == 'spgist': + if self.verbose: + print( + 'SPGIST is a special snowflake, so don`t ' + 'fret about losing ptrack for blknum 0' + ) + continue + success = False + else: + # Page has not been changed, + # meaning that ptrack should be equal to 0 + if idx_dict['ptrack'][PageNum] != 0: + if self.verbose: + print( + 'Page Number {0} of type {1} was not changed,' + ' but ptrack value is {2}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum] + ) + ) + + self.assertTrue( + success, 'Ptrack does not correspond to state' + ' of its own pages.\n Gory Details: \n{0}'.format( + idx_dict['type'], idx_dict + ) + ) + + def check_ptrack_recovery(self, idx_dict): + size = idx_dict['size'] + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 1: + self.assertTrue( + False, + 'Recovery for Page Number {0} of Type {1}' + ' was conducted, but ptrack value is {2}.' + ' THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def check_ptrack_clean(self, idx_dict, size): + for PageNum in range(size): + if idx_dict['ptrack'][PageNum] != 0: + self.assertTrue( + False, + 'Ptrack for Page Number {0} of Type {1}' + ' should be clean, but ptrack value is {2}.' + '\n THIS IS BAD\n IDX_DICT: {3}'.format( + PageNum, + idx_dict['type'], + idx_dict['ptrack'][PageNum], + idx_dict + ) + ) + + def run_pb(self, command, async=False, gdb=False): + try: + self.cmd = [' '.join(map(str, [self.probackup_path] + command))] + if self.verbose: + print(self.cmd) + if gdb: + return GDBobj([self.probackup_path] + command, self.verbose) + if async: + return subprocess.Popen( + self.cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + [self.probackup_path] + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + if command[0] == 'backup': + # return backup ID + for line in self.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + else: + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), self.cmd) + + def run_binary(self, command, async=False): + if self.verbose: + print([' '.join(map(str, command))]) + try: + if async: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.test_env + ) + else: + self.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=self.test_env + ).decode("utf-8") + return self.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode("utf-8"), command) + + def init_pb(self, backup_dir): + + shutil.rmtree(backup_dir, ignore_errors=True) + return self.run_pb([ + "init", + "-B", backup_dir + ]) + + def add_instance(self, backup_dir, instance, node): + + return self.run_pb([ + "add-instance", + "--instance={0}".format(instance), + "-B", backup_dir, + "-D", node.data_dir + ]) + + def del_instance(self, backup_dir, instance): + + return self.run_pb([ + "del-instance", + "--instance={0}".format(instance), + "-B", backup_dir + ]) + + def clean_pb(self, backup_dir): + shutil.rmtree(backup_dir, ignore_errors=True) + + def backup_node( + self, backup_dir, instance, node, data_dir=False, + backup_type="full", options=[], async=False, gdb=False + ): + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if node: + pgdata = node.data_dir + + if data_dir: + pgdata = data_dir + + cmd_list = [ + "backup", + "-B", backup_dir, + # "-D", pgdata, + "-p", "%i" % node.port, + "-d", "postgres", + "--instance={0}".format(instance) + ] + if backup_type: + cmd_list += ["-b", backup_type] + + return self.run_pb(cmd_list + options, async, gdb) + + def merge_backup(self, backup_dir, instance, backup_id): + cmd_list = [ + "merge", + "-B", backup_dir, + "--instance={0}".format(instance), + "-i", backup_id + ] + + return self.run_pb(cmd_list) + + def restore_node( + self, backup_dir, instance, node=False, + data_dir=None, backup_id=None, options=[] + ): + if data_dir is None: + data_dir = node.data_dir + + cmd_list = [ + "restore", + "-B", backup_dir, + "-D", data_dir, + "--instance={0}".format(instance) + ] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def show_pb( + self, backup_dir, instance=None, backup_id=None, + options=[], as_text=False, as_json=True + ): + + backup_list = [] + specific_record = {} + cmd_list = [ + "show", + "-B", backup_dir, + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if backup_id: + cmd_list += ["-i", backup_id] + + if as_json: + cmd_list += ["--format=json"] + + if as_text: + # You should print it when calling as_text=true + return self.run_pb(cmd_list + options) + + # get show result as list of lines + if as_json: + data = json.loads(self.run_pb(cmd_list + options)) + # print(data) + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + return backup_list + else: + show_splitted = self.run_pb(cmd_list + options).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(" +", header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(" +", backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(" = ")[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + return specific_record + + def validate_pb( + self, backup_dir, instance=None, + backup_id=None, options=[] + ): + + cmd_list = [ + "validate", + "-B", backup_dir + ] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_pb(self, backup_dir, instance, backup_id=None, options=[]): + cmd_list = [ + "delete", + "-B", backup_dir + ] + + cmd_list += ["--instance={0}".format(instance)] + if backup_id: + cmd_list += ["-i", backup_id] + + return self.run_pb(cmd_list + options) + + def delete_expired(self, backup_dir, instance, options=[]): + cmd_list = [ + "delete", "--expired", "--wal", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + return self.run_pb(cmd_list + options) + + def show_config(self, backup_dir, instance): + out_dict = {} + cmd_list = [ + "show-config", + "-B", backup_dir, + "--instance={0}".format(instance) + ] + res = self.run_pb(cmd_list).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(" = ")[::2] + out_dict[name] = var + return out_dict + + def get_recovery_conf(self, node): + out_dict = {} + with open( + os.path.join(node.data_dir, "recovery.conf"), "r" + ) as recovery_conf: + for line in recovery_conf: + try: + key, value = line.split("=") + except: + continue + out_dict[key.strip()] = value.strip(" '").replace("'\n", "") + return out_dict + + def set_archiving( + self, backup_dir, instance, node, replica=False, overwrite=False): + + if replica: + archive_mode = 'always' + node.append_conf('postgresql.auto.conf', 'hot_standby = on') + else: + archive_mode = 'on' + + # node.append_conf( + # "postgresql.auto.conf", + # "wal_level = archive" + # ) + node.append_conf( + "postgresql.auto.conf", + "archive_mode = {0}".format(archive_mode) + ) + archive_command = "{0} archive-push -B {1} --instance={2} ".format( + self.probackup_path, backup_dir, instance) + + if os.name == 'posix': + if self.archive_compress: + archive_command = archive_command + "--compress " + + if overwrite: + archive_command = archive_command + "--overwrite " + + archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + + node.append_conf( + "postgresql.auto.conf", + "archive_command = '{0}'".format( + archive_command)) + # elif os.name == 'nt': + # node.append_conf( + # "postgresql.auto.conf", + # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) + # ) + + def set_replica( + self, master, replica, + replica_name='replica', + synchronous=False + ): + replica.append_conf( + "postgresql.auto.conf", "port = {0}".format(replica.port)) + replica.append_conf('postgresql.auto.conf', 'hot_standby = on') + replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf( + "recovery.conf", + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name) + ) + if synchronous: + master.append_conf( + "postgresql.auto.conf", + "synchronous_standby_names='{0}'".format(replica_name) + ) + master.append_conf( + 'postgresql.auto.conf', + "synchronous_commit='remote_apply'" + ) + master.reload() + + def wrong_wal_clean(self, node, wal_size): + wals_dir = os.path.join(self.backup_dir(node), "wal") + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) + ] + wals.sort() + file_path = os.path.join(wals_dir, wals[-1]) + if os.path.getsize(file_path) != wal_size: + os.remove(file_path) + + def guc_wal_segment_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_segment_size'" + ) + return int(var[0][0]) * self.guc_wal_block_size(node) + + def guc_wal_block_size(self, node): + var = node.execute( + "postgres", + "select setting from pg_settings where name = 'wal_block_size'" + ) + return int(var[0][0]) + + def get_pgpro_edition(self, node): + if node.execute( + "postgres", + "select exists (select 1 from" + " pg_proc where proname = 'pgpro_edition')" + )[0][0]: + var = node.execute("postgres", "select pgpro_edition()") + return str(var[0][0]) + else: + return False + + def get_username(self): + """ Returns current user name """ + return pwd.getpwuid(os.getuid())[0] + + def version_to_num(self, version): + if not version: + return 0 + parts = version.split(".") + while len(parts) < 3: + parts.append("0") + num = 0 + for part in parts: + num = num * 100 + int(re.sub("[^\d]", "", part)) + return num + + def switch_wal_segment(self, node): + """ + Execute pg_switch_wal/xlog() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + if self.version_to_num( + node.safe_psql("postgres", "show server_version") + ) >= self.version_to_num('10.0'): + node.safe_psql("postgres", "select pg_switch_wal()") + else: + node.safe_psql("postgres", "select pg_switch_xlog()") + else: + if self.version_to_num( + node.execute("show server_version")[0][0] + ) >= self.version_to_num('10.0'): + node.execute("select pg_switch_wal()") + else: + node.execute("select pg_switch_xlog()") + sleep(1) + + def get_version(self, node): + return self.version_to_num( + testgres.get_pg_config()["VERSION"].split(" ")[1]) + + def get_bin_path(self, binary): + return testgres.get_bin_path(binary) + + def del_test_dir(self, module_name, fname): + """ Del testdir and optimistically try to del module dir""" + try: + testgres.clean_all() + except: + pass + + shutil.rmtree( + os.path.join( + self.tmp_path, + module_name, + fname + ), + ignore_errors=True + ) + try: + os.rmdir(os.path.join(self.tmp_path, module_name)) + except: + pass + + def pgdata_content(self, directory, ignore_ptrack=True): + """ return dict with directory content. " + " TAKE IT AFTER CHECKPOINT or BACKUP""" + dirs_to_ignore = [ + 'pg_xlog', 'pg_wal', 'pg_log', + 'pg_stat_tmp', 'pg_subtrans', 'pg_notify' + ] + files_to_ignore = [ + 'postmaster.pid', 'postmaster.opts', + 'pg_internal.init', 'postgresql.auto.conf', + 'backup_label', 'tablespace_map', 'recovery.conf', + 'ptrack_control', 'ptrack_init', 'pg_control' + ] +# suffixes_to_ignore = ( +# '_ptrack' +# ) + directory_dict = {} + directory_dict['pgdata'] = directory + directory_dict['files'] = {} + for root, dirs, files in os.walk(directory, followlinks=True): + dirs[:] = [d for d in dirs if d not in dirs_to_ignore] + for file in files: + if ( + file in files_to_ignore or + (ignore_ptrack and file.endswith('_ptrack')) + ): + continue + + file_fullpath = os.path.join(root, file) + file_relpath = os.path.relpath(file_fullpath, directory) + directory_dict['files'][file_relpath] = {'is_datafile': False} + directory_dict['files'][file_relpath]['md5'] = hashlib.md5( + open(file_fullpath, 'rb').read()).hexdigest() + + if file.isdigit(): + directory_dict['files'][file_relpath]['is_datafile'] = True + size_in_pages = os.path.getsize(file_fullpath)/8192 + directory_dict['files'][file_relpath][ + 'md5_per_page'] = self.get_md5_per_page_for_fork( + file_fullpath, size_in_pages + ) + + return directory_dict + + def compare_pgdata(self, original_pgdata, restored_pgdata): + """ return dict with directory content. DO IT BEFORE RECOVERY""" + fail = False + error_message = 'Restored PGDATA is not equal to original!\n' + for file in restored_pgdata['files']: + # File is present in RESTORED PGDATA + # but not present in ORIGINAL + # only backup_label is allowed + if file not in original_pgdata['files']: + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in original_pgdata['files']: + if file in restored_pgdata['files']: + + if ( + original_pgdata['files'][file]['md5'] != + restored_pgdata['files'][file]['md5'] + ): + fail = True + error_message += ( + '\nFile Checksumm mismatch.\n' + 'File_old: {0}\nChecksumm_old: {1}\n' + 'File_new: {2}\nChecksumm_new: {3}\n').format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['md5'], + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['md5'] + ) + + if original_pgdata['files'][file]['is_datafile']: + for page in original_pgdata['files'][file]['md5_per_page']: + if page not in restored_pgdata['files'][file]['md5_per_page']: + error_message += ( + '\n Page {0} dissappeared.\n ' + 'File: {1}\n').format( + page, + os.path.join( + restored_pgdata['pgdata'], + file + ) + ) + continue + + if original_pgdata['files'][file][ + 'md5_per_page'][page] != restored_pgdata[ + 'files'][file]['md5_per_page'][page]: + error_message += ( + '\n Page checksumm mismatch: {0}\n ' + ' PAGE Checksumm_old: {1}\n ' + ' PAGE Checksumm_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original_pgdata['files'][file][ + 'md5_per_page'][page], + restored_pgdata['files'][file][ + 'md5_per_page'][page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + for page in restored_pgdata['files'][file]['md5_per_page']: + if page not in original_pgdata['files'][file]['md5_per_page']: + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join( + restored_pgdata['pgdata'], file)) + + else: + error_message += ( + '\nFile dissappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True + self.assertFalse(fail, error_message) + + def get_async_connect(self, database=None, host=None, port=5432): + if not database: + database = 'postgres' + if not host: + host = '127.0.0.1' + + return psycopg2.connect( + database="postgres", + host='127.0.0.1', + port=port, + async=True + ) + + def wait(self, connection): + while True: + state = connection.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [connection.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([connection.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + + def gdb_attach(self, pid): + return GDBobj([str(pid)], self.verbose, attach=True) + + +class GdbException(Exception): + def __init__(self, message=False): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj(ProbackupTest): + def __init__(self, cmd, verbose, attach=False): + self.verbose = verbose + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ["gdb", "--version"], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + b"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + universal_newlines=True + ) + self.gdb_pid = self.proc.pid + + # discard data from pipe, + # is there a way to do it a less derpy way? + while True: + line = self.proc.stdout.readline() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def set_breakpoint(self, location): + result = self._execute('break ' + location) + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('^error') or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + running = False + for line in result: + if line.startswith('*running'): + running = True + break + if line.startswith('*stopped,reason="breakpoint-hit"'): + running = False + continue + if line.startswith('*stopped,reason="exited-normally"'): + running = False + continue + return running + + def continue_execution_until_exit(self): + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if ( + line.startswith('*stopped,reason="exited-normally"') or + line == '*stopped\n' + ): + return + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + running = False + for line in result: + if line.startswith('*running'): + running = True + if line.startswith('*stopped,reason="breakpoint-hit"'): + return 'breakpoint-hit' + if line.startswith('*stopped,reason="exited-normally"'): + return 'exited-normally' + if running: + return 'running' + + def stopped_in_breakpoint(self): + output = [] + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + return False + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + + while True: + line = self.proc.stdout.readline() + output += [line] + if self.verbose: + print(repr(line)) + if line == '^done\n' or line.startswith('*stopped'): + break + if running and line.startswith('*running'): + break + return output diff --git a/tests/init_test.py b/tests/init_test.py new file mode 100644 index 000000000..0b91dafa7 --- /dev/null +++ b/tests/init_test.py @@ -0,0 +1,99 @@ +import os +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException + + +module_name = 'init' + + +class InitTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): + """Success normal init""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + self.assertEqual( + dir_files(backup_dir), + ['backups', 'wal'] + ) + self.add_instance(backup_dir, 'node', node) + self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node'), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Show non-existing instance + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Delete non-existing instance + try: + self.del_instance(backup_dir, 'node1') + self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node1' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Add instance without pgdata + try: + self.run_pb([ + "add-instance", + "--instance=node1", + "-B", backup_dir + ]) + self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_already_exist(self): + """Failure with backup catalog already existed""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + self.init_pb(backup_dir) + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: Instance 'node' does not exist in this backup catalog\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_abs_path(self): + """failure with backup catalog should be given as absolute path""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + try: + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: -B, --backup-path must be an absolute path\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/logging.py b/tests/logging.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/merge.py b/tests/merge.py new file mode 100644 index 000000000..1be3dd8b3 --- /dev/null +++ b/tests/merge.py @@ -0,0 +1,454 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest + +module_name = "merge" + + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node(backup_dir, "node", node, backup_type="page") + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"] + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Some test here + """ + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/option_test.py b/tests/option_test.py new file mode 100644 index 000000000..8bd473fa9 --- /dev/null +++ b/tests/option_test.py @@ -0,0 +1,218 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'option' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_help_1(self): + """help options""" + self.maxDiff = None + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + + # @unittest.skip("skip") + def test_version_2(self): + """help options""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + self.assertIn( + version_out.read().decode("utf-8"), + self.run_pb(["--version"]) + ) + + # @unittest.skip("skip") + def test_without_backup_path_3(self): + """backup command failure without backup mode option""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + try: + self.run_pb(["backup", "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # @unittest.skip("skip") + def test_options_4(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: required parameter not specified: --instance\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure without backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure with invalid backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode "bad"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # delete failure without delete options + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--backup_id\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # delete failure without ID + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue("option requires an argument -- 'i'" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_options_5(self): + """check options test""" + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + pg_options={ + 'wal_level': 'logical', + 'max_wal_senders': '2'}) + + self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), + self.init_pb(backup_dir)) + self.add_instance(backup_dir, 'node', node) + + node.start() + + # syntax error in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write(" = INFINITE\n") + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: syntax error in " = INFINITE"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("BACKUP_MODE=\n") + + try: + self.backup_node(backup_dir, 'node', node, backup_type=None), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid backup-mode ""\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Command line parameters should override file values + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy=1\n") + + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + + # User cannot send --system-identifier parameter via command line + try: + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: option system-identifier cannot be specified in command line\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # invalid value in pg_probackup.conf + with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf.write("SMOOTH_CHECKPOINT=FOO\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid option in pg_probackup.conf + pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(pbconf_path, "a") as conf: + conf.write("TIMELINEID=1\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual(e.message, + 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py new file mode 100644 index 000000000..ef7122b68 --- /dev/null +++ b/tests/page.py @@ -0,0 +1,641 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + +module_name = 'page' + + +class PageBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, take second page backup, + restore last page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=verbose']) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace), + "--recovery-target-action=promote"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_stream(self): + """ + make archive node, take full and page stream backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_archive(self): + """ + make archive node, take full and page archive backups, + restore them and check data correctness + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn("INFO: Restore of backup {0} completed.".format( + full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check page and data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--log-level-file=verbose"]) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", + "--recovery-target-action=promote", + "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_delete_1(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + self.del_test_dir(module_name, fname) + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.start() + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560.py new file mode 100644 index 000000000..bf3345561 --- /dev/null +++ b/tests/pgpro560.py @@ -0,0 +1,98 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro560' + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: could not open file' in e.message + and 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node1.start() + node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + node2.start() + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + try: + self.backup_node(backup_dir, 'node1', node2, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Backup data directory was initialized for system id' in e.message + and 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py new file mode 100644 index 000000000..bd40f16de --- /dev/null +++ b/tests/pgpro589.py @@ -0,0 +1,80 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +module_name = 'pgpro589' + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneus archive_command + node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + node.start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: Switched WAL segment' in e.message and + 'could not be archived' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack.py new file mode 100644 index 000000000..c2d6abff3 --- /dev/null +++ b/tests/ptrack.py @@ -0,0 +1,1600 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +module_name = 'ptrack' + + +class PtrackTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_enable(self): + """make ptrack without full backup, should result in error""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack disabled.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_disable(self): + """ + Take full backup, disable ptrack restart postgresql, + enable ptrack, restart postgresql, take ptrack backup + which should fail + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DISABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.restart() + + # ENABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.restart() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack_enable was set to OFF at some" + " point after previous backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd + ) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_uncommited_xact(self): + """make ptrack backup while there is uncommited open transaction""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + con = node.connect("postgres") + con.execute( + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + pgdata = self.pgdata_content(node.data_dir) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_full(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + # create async connection + conn = self.get_async_connect(port=node.port) + + self.wait(conn) + + acurs = conn.cursor() + acurs.execute("select pg_backend_pid()") + + self.wait(conn) + pid = acurs.fetchall()[0][0] + print(pid) + + gdb = self.gdb_attach(pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + if not gdb.continue_execution_until_running(): + print('Failed gdb continue') + exit(1) + + acurs.execute("VACUUM FULL t_heap") + + if gdb.stopped_in_breakpoint(): + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_truncate(self): + """make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take ptrack backup, take second ptrack backup, + restore last ptrack backup and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, + ignore_ptrack=False + ) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_get_block(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '300s', + 'ptrack_enable': 'on' + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i" + ) + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'], + gdb=True + ) + + gdb.set_breakpoint('make_pagemap_from_ptrack') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream'] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap") + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_stream(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', + node, backup_type='ptrack', + options=['--stream', '--log-level-file=verbose'] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=["-j", "4", "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_archive(self): + """make archive node, make full and ptrack backups, + check data correctness in restored instance""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as" + " select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node(backup_dir, 'node', node) + full_target_time = self.show_pb( + backup_dir, 'node', full_backup_id)['recovery-time'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + ptrack_target_time = self.show_pb( + backup_dir, 'node', ptrack_backup_id)['recovery-time'] + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--recovery-target-action=promote", + "--time={0}".format(full_target_time)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=[ + "-j", "4", + "--time={0}".format(ptrack_target_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_pgpro417(self): + """Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': + 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql( + "postgres", + "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["--stream"]) + + start_lsn_full = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + start_lsn_ptrack = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + # SECOND PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_pgpro417(self): + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.delete_pb(backup_dir, 'node', backup_id) +# sys.exit(1) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0}\n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_full_pgpro417(self): + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector " + " from generate_series(0,100) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # SECOND FULL BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + + self.delete_pb(backup_dir, 'node', backup_id) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + "ERROR: LSN from ptrack_control" in e.message and + "Create new full backup before " + "an incremental one" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream", "--log-level-file=verbose"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_ptrack(self): + """Make node, create tablespace with table, take full backup, + alter tablespace location, take ptrack backup, restore database.""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new" + ) + + # sys.exit(1) + # PTRACK BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + # node.stop() + # node.cleanup() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ), + "--recovery-target-action=promote" + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_ptrack(self): + """Make node, create tablespace with database," + " take full backup, alter tablespace location," + " take ptrack backup, restore database.""" + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE TABLESPACE + self.create_tblspace_in_node(node, 'somedata') + + # ALTER DATABASE + node.safe_psql( + "template1", + "alter database postgres set tablespace somedata") + + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", '--log-level-file=verbose']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + node.stop() + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', + node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata'))]) + + # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', + 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to tablespace 'somedata' + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # Move table back to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # DROP TABLESPACE 'somedata' + node.safe_psql( + "postgres", "drop tablespace somedata") + # THIRD PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + tblspace = self.get_tblspace_path(node, 'somedata') + node.cleanup() + shutil.rmtree(tblspace, ignore_errors=True) + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + node.start() + + tblspc_exist = node.safe_psql( + "postgres", + "select exists(select 1 from " + "pg_tablespace where spcname = 'somedata')") + + if tblspc_exist.rstrip() == 't': + self.assertEqual( + 1, 0, + "Expecting Error because " + "tablespace 'somedata' should not be present") + + result_new = node.safe_psql("postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_alter_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', + 'autovacuum': 'off'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + tblspc_path = self.get_tblspace_path(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to separate tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from t_heap") + + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + # GET PHYSICAL CONTENT FROM NODE + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore ptrack backup + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + # COMPARE LOGICAL CONTENT + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + restored_node.cleanup() + shutil.rmtree(tblspc_path_new, ignore_errors=True) + + # Move table to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "--log-level-file=verbose"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore second ptrack backup and check table consistency + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_multiple_segments(self): + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'ptrack_enable': 'on', 'fsync': 'off', + 'autovacuum': 'off', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PTRACK STUFF + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench(options=['-T', '150', '-c', '2', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts' + ) + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size'] + ) + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path'] + ) + self.check_ptrack_sanity(idx_ptrack) + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--log-level-file=verbose"] + ) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir="{0}/{1}/restored_node".format(module_name, fname)) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, + 'somedata_restored' + ) + + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + restored_node.append_conf( + "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select * from pgbench_accounts" + ) + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_atexit_fail(self): + """ + Take backups of every available types and check that PTRACK is clean + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'max_connections': '15'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=[ + "--stream", "-j 30", + "--log-level-file=verbose"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are opening too many connections" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'setting its status to ERROR', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + self.assertEqual( + node.safe_psql( + "postgres", + "select * from pg_is_in_backup()").rstrip(), + "f") + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py new file mode 100644 index 000000000..f4350af04 --- /dev/null +++ b/tests/ptrack_clean.py @@ -0,0 +1,253 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +import time + + +module_name = 'ptrack_clean' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10', '--log-level-file=verbose']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean_replica(self): + """Take backups of every available types from master and check that PTRACK on replica is clean""" + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py new file mode 100644 index 000000000..784751ef6 --- /dev/null +++ b/tests/ptrack_cluster.py @@ -0,0 +1,268 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from time import sleep +from sys import exit + + +module_name = 'ptrack_cluster' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + #@unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, nextval('t_seq') as t_seq, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_gist') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py new file mode 100644 index 000000000..98c209142 --- /dev/null +++ b/tests/ptrack_move_to_tablespace.py @@ -0,0 +1,57 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_move_to_tablespace' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Move table and indexes and make checkpoint + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'heap': + node.safe_psql('postgres', 'alter table {0} set tablespace somedata;'.format(i)) + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql('postgres', 'alter index {0} set tablespace somedata'.format(i)) + node.safe_psql('postgres', 'checkpoint') + + # Check ptrack files + for i in idx_ptrack: + if idx_ptrack[i]['type'] == 'seq': + continue + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py new file mode 100644 index 000000000..8569ef592 --- /dev/null +++ b/tests/ptrack_recovery.py @@ -0,0 +1,58 @@ +import os +import unittest +from sys import exit +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_recovery' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql("postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text,md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py new file mode 100644 index 000000000..928608c4a --- /dev/null +++ b/tests/ptrack_truncate.py @@ -0,0 +1,130 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + replica.safe_psql('postgres', 'truncate t_heap') + replica.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py new file mode 100644 index 000000000..0409cae3f --- /dev/null +++ b/tests/ptrack_vacuum.py @@ -0,0 +1,152 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py new file mode 100644 index 000000000..f0cd3bbda --- /dev/null +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -0,0 +1,136 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_frozen' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py new file mode 100644 index 000000000..45a8d9b60 --- /dev/null +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -0,0 +1,67 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_bits_visibility' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py new file mode 100644 index 000000000..ec12c9e27 --- /dev/null +++ b/tests/ptrack_vacuum_full.py @@ -0,0 +1,140 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_full' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,127) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take FULL backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py new file mode 100644 index 000000000..5c84c7e8f --- /dev/null +++ b/tests/ptrack_vacuum_truncate.py @@ -0,0 +1,142 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack + + +module_name = 'ptrack_vacuum_truncate' + + +class SimpleTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql("postgres", "create index {0} on {1} using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node(backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id > 128;') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'ptrack_enable': 'on', 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.start() + + # Create table and indexes + master.safe_psql( + "postgres", + "create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql("postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Take PTRACK backup to clean every ptrack + self.backup_node(backup_dir, 'replica', replica, options=['-j10', + '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + + master.safe_psql('postgres', 'delete from t_heap where id > 128;') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + self.check_ptrack_sanity(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py new file mode 100644 index 000000000..d74c375c2 --- /dev/null +++ b/tests/replica.py @@ -0,0 +1,293 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'replica' + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_ptrack_backup(self): + """ + make node, take full backup, restore it and make replica from it, + take full stream backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} + ) + master.start() + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + # CREATE TABLE + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # take full backup and restore it + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + # Check data correctness on replica + replica.slow_start(replica=True) + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take PTRACK backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PTRACK BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_archive_page_backup(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data + # equal to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname)) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + options=[ + '--archive-timeout=300', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_make_replica_via_restore(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node( + backup_dir, 'master', replica, + options=['-R', '--recovery-target-action=promote']) + + # Settings for Replica + # self.set_replica(master, replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/restore_test.py b/tests/restore_test.py new file mode 100644 index 000000000..c33a1e299 --- /dev/null +++ b/tests/restore_test.py @@ -0,0 +1,1243 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +import time + + +module_name = 'restore' + + +class RestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_restore_full_to_latest(self): + """recovery to latest from full backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # 1 - Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # 2 - Test that recovery.conf was created + recovery_conf = os.path.join(node.data_dir, "recovery.conf") + self.assertEqual(os.path.isfile(recovery_conf), True) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_page_to_latest(self): + """recovery to latest from full + page backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_specific_timeline(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_tli = int( + node.get_control_data()["Latest checkpoint's TimeLineID"]) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Correct Backup must be choosen for restore + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--timeline={0}".format(target_tli), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + recovery_target_timeline = self.get_recovery_conf( + node)["recovery_target_timeline"] + self.assertEqual(int(recovery_target_timeline), target_tli) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_time(self): + """recovery to target time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") + node.start() + + node.pgbench_init(scale=2) + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(target_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_inclusive(self): + """recovery to target xid""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--xid={0}'.format(target_xid), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_xid_not_inclusive(self): + """recovery with target inclusive false""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = result[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + '--xid={0}'.format(target_xid), + "--inclusive=false", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + if self.get_version(node) < self.version_to_num('10.0'): + self.del_test_dir(module_name, fname) + return + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_stream(self): + """recovery in stream mode to latest from full + ptrack backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_ptrack_under_load(self): + """ + recovery to latest from full + ptrack backups + with loads when ptrack backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + pgbench.wait() + pgbench.stdout.close() + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_full_under_load_ptrack(self): + """ + recovery to latest from full + page backups + with loads when full backup do + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + self.backup_node(backup_dir, 'node', node) + + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + + node.stop() + node.cleanup() + # self.wrong_wal_clean(node, wal_segment_size) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_1(self): + """recovery using tablespace-mapping option""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on', + 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # 1 - Try to restore to existing directory + node.stop() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore destionation is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore destination is not empty: "{0}"\n'.format( + node.data_dir), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 2 - Try to restore to existing tablespace directory + node.cleanup() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: restore tablespace destination ' + 'is not empty: "{0}"\n'.format(tblspc_path), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 3 - Restore using tablespace-mapping + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) + + # 4 - Restore using tablespace-mapping using page backup + self.backup_node(backup_dir, 'node', node) + with node.connect("postgres") as con: + con.execute("INSERT INTO test VALUES (2)") + con.commit() + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_pb = self.show_pb(backup_dir, 'node') + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") + + node.stop() + node.cleanup() + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_2(self): + """recovery using tablespace-mapping option and page backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # Full backup + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # First page backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + + # Create tablespace table + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CHECKPOINT") + con.connection.autocommit = False + con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") + con.execute( + "INSERT INTO tbl1 SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Second page backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + + node.stop() + node.cleanup() + + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new), + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + count = node.execute("postgres", "SELECT count(*) FROM tbl1") + self.assertEqual(count[0][0], 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.safe_psql("postgres", "select pg_switch_xlog()") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_pitr(self): + """ + make node with archiving, make stream backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_archive_pitr_2(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_restore_to_restore_point(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,10000)") + result = node.safe_psql( + "postgres", + "select * from t_heap") + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + node.safe_psql( + "postgres", + "create table t_heap_1 as select generate_series(0,10000)") + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "--recovery-target-name=savepoint", + "--recovery-target-action=promote"]) + + node.slow_start() + + result_new = node.safe_psql("postgres", "select * from t_heap") + res = node.psql("postgres", "select * from t_heap_1") + self.assertEqual( + res[0], 1, + "Table t_heap_1 should not exist in restored instance") + + self.assertEqual(result, result_new) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/retention_test.py b/tests/retention_test.py new file mode 100644 index 000000000..652f7c39d --- /dev/null +++ b/tests/retention_test.py @@ -0,0 +1,178 @@ +import os +import unittest +from datetime import datetime, timedelta +from .helpers.ptrack_helpers import ProbackupTest + + +module_name = 'retention' + + +class RetentionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_1(self): + """purge backups using redundancy-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + log = self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Check that WAL segments were deleted + min_wal = None + max_wal = None + for line in log.splitlines(): + if line.startswith("INFO: removed min WAL segment"): + min_wal = line[31:-1] + elif line.startswith("INFO: removed max WAL segment"): + max_wal = line[31:-1] + + if not min_wal: + self.assertTrue(False, "min_wal is empty") + + if not max_wal: + self.assertTrue(False, "max_wal is not set") + + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): + if not wal_name.endswith(".backup"): + # wal_name_b = wal_name.encode('ascii') + self.assertEqual(wal_name[8:] > min_wal[8:], True) + self.assertEqual(wal_name[8:] > max_wal[8:], True) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_window_2(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + conf.write("retention-window = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + self.delete_expired(backup_dir, 'node') + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# @unittest.skip("123") + def test_retention_wal(self): + """purge backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=2']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/show_test.py b/tests/show_test.py new file mode 100644 index 000000000..931da1844 --- /dev/null +++ b/tests/show_test.py @@ -0,0 +1,203 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'show' + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=panic"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_corrupt_2(self): + """Status CORRUPT""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup corrupted.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + self.assertIn('control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + self.assertIn('control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py new file mode 100644 index 000000000..ab091c578 --- /dev/null +++ b/tests/validate_test.py @@ -0,0 +1,1730 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from sys import exit +import time + + +module_name = 'validate' + + +class ValidateTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """ + make node with archiving, make archive backup + validate to both real and unreal values + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + + pgbench.wait() + pgbench.stdout.close() + + target_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) + + # Validate to real time + self.assertIn( + "INFO: backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(target_time)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format( + unreal_time_1)]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Full backup satisfying target options is not found.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) + try: + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(unreal_time_2)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal time.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate to real xid + target_xid = None + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + self.switch_wal_segment(node) + + self.assertIn("INFO: backup validation completed successfully", + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 + try: + self.validate_pb(backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + self.assertEqual(1, 0, "Expecting Error because of validation to unreal xid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('ERROR: not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Validate with backup ID + self.assertIn("INFO: Validating backup {0}".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} data files are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} WAL segments are valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Backup {0} is valid".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + self.assertIn("INFO: Validate of backup {0} completed".format(backup_id), + self.validate_pb(backup_dir, 'node', backup_id), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backup(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, + run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file + file = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path) + with open(file, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message + and 'ERROR: Backup {0} is corrupt'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups(self): + """make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in FULL and PAGE1 backupd, run validate on PAGE1, + expect FULL and PAGE1 to gain status CORRUPT and PAGE2 get status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Corrupt some file in PAGE1 backup + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_2, 'database', file_path_t_heap_1) + with open(file_page1, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE1 + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id_2, + options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue('INFO: Validating parents for backup {0}'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_2) in e.message + and 'WARNING: Backup {0} is orphaned because his parent'.format(backup_id_3) in e.message + and 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups_1(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, + options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_page1) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_specific_target_corrupted_intermediate_backups(self): + """make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 backups, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(10000,20000) i") + backup_id_4 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE4 + target_xid = node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i RETURNING (xmin)")[0][0] + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + backup_id_6 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_7 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join(backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join(backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb(backup_dir, 'node', + options=['--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_2) in e.message + and 'INFO: Backup {0} data files are valid'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_3) in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_page1) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent {1} is corrupted'.format(backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_page(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in PAGE1 backup and run validate on instance, + expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + # PAGE3 + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups/node', backup_id_2, + 'database', file_path_t_heap1) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb( + backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} is corrupted'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file "{0}"'.format( + file_full) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(128) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_1(self): + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.backup_node(backup_dir, 'node', node) + + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) + os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: WAL segment \"{0}\" is absent".format( + file) in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') + + # Run validate again + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Revalidating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--log-level-console=verbose", + "--xid={0}".format(target_xid)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: not enough WAL records to xid' in e.message and + 'WARNING: recovery can be done up to time' in e.message and + "ERROR: not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_2(self): + """ + make node with archiving + make archive backup + delete from archive wal segment which DO NOT belong to this backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + fname = self.id().split('.')[3] + node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + file = os.path.join(wals_dir, max(wals)) + os.remove(file) + if self.archive_compress: + file = file[:-3] + + # Try to restore + try: + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for LSN' in e.message and + 'in archived WAL segment' in e.message and + 'WARNING: could not read WAL record at' in e.message and + 'ERROR: WAL segment "{0}" is absent\n'.format( + file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro702_688(self): + """make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(recovery_time)]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro688(self): + """make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb(backup_dir, 'node', backup_id)['recovery-time'] + + self.validate_pb(backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, + restore it to node1, check that archiving is not successful on node1 + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir="{0}/{1}/node1".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.start() + + backup_id = self.backup_node( + backup_dir, 'node1', node1, options=["--stream"]) + + node2 = self.make_simple_node( + base_dir="{0}/{1}/node2".format(module_name, fname)) + node2.cleanup() + + node1.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node( + backup_dir, 'node1', node1, + backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.slow_start() + + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual( + timeline_node1, timeline_node2, + "Timelines on Master and Node1 should be equal. " + "This is unexpected") + + archive_command_node1 = node1.safe_psql( + "postgres", "show archive_command") + archive_command_node2 = node2.safe_psql( + "postgres", "show archive_command") + self.assertEqual( + archive_command_node1, archive_command_node2, + "Archive command on Master and Node should be equal. " + "This is unexpected") + + # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + # if result == "": + # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + self.switch_wal_segment(node1) + self.switch_wal_segment(node2) + time.sleep(5) + + log_file = os.path.join(node2.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'INFO: pg_probackup archive-push from' in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse( + 'pg_probackup archive-push completed successfully' in log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and three page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption and run valudate again, check that + second full backup and his page backups are OK + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "alter system set archive_command = 'false'") + node.reload() + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=1s']) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + pass + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.set_archiving(backup_dir, 'node', node) + node.reload() + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + + os.rename(file_new, file) + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_corrupted_full_1(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and corrupt his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + second page should be CORRUPT + third page should be ORPHAN + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + os.rename(file_new, file) + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_page, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_file_size_corruption_no_validate(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + # initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], async=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open(os.path.join(backup_dir, "backups", 'node', backup_id, "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue("ERROR: Data files restoring failed" in e.message, repr(e.message)) + print "\nExpected error: \n" + e.message + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh new file mode 100644 index 000000000..7fe1cfd8f --- /dev/null +++ b/travis/backup_restore.sh @@ -0,0 +1,66 @@ +#!/bin/sh -ex + +# vars +export PGVERSION=9.5.4 +export PATH=$PATH:/usr/pgsql-9.5/bin +export PGUSER=pgbench +export PGDATABASE=pgbench +export PGDATA=/var/lib/pgsql/9.5/data +export BACKUP_PATH=/backups +export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog +export PGDATA2=/var/lib/pgsql/9.5/data2 +export PGBENCH_SCALE=100 +export PGBENCH_TIME=60 + +# prepare directory +cp -a /tests /build +pushd /build + +# download postgresql +yum install -y wget +wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz +tar xf postgresql.tar.gz + +# install pg_probackup +yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm +yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel +make top_srcdir=postgresql-$PGVERSION +make install top_srcdir=postgresql-$PGVERSION + +# initalize cluster and database +yum install -y postgresql95-server +su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" +cat < $PGDATA/pg_hba.conf +local all all trust +host all all 127.0.0.1/32 trust +local replication pgbench trust +host replication pgbench 127.0.0.1/32 trust +EOF +cat < $PGDATA/postgresql.auto.conf +max_wal_senders = 2 +wal_level = logical +wal_log_hints = on +EOF +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" +su postgres -c "createdb -U postgres $PGUSER" +su postgres -c "createuser -U postgres -a -d -E $PGUSER" +pgbench -i -s $PGBENCH_SCALE + +# Count current +COUNT=$(psql -Atc "select count(*) from pgbench_accounts") +pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & + +# create backup +pg_probackup init +pg_probackup backup -b full --disable-ptrack-clear --stream -v +pg_probackup show +sleep $PGBENCH_TIME + +# restore from backup +chown -R postgres:postgres $BACKUP_PATH +su postgres -c "pg_probackup restore -D $PGDATA2" + +# start backup server +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" +su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" +( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/win32build.pl b/win32build.pl new file mode 100644 index 000000000..148641812 --- /dev/null +++ b/win32build.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build96.pl b/win32build96.pl new file mode 100644 index 000000000..c869e485b --- /dev/null +++ b/win32build96.pl @@ -0,0 +1,240 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +our $libpath32=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@ADDLIBS32\@/$libpath32/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary32 +{ + $inc = shift; + if ($libpath32 ne '') + { + $libpath32 .= ';'; + } + $libpath32 .= $inc; + +} +sub AddLibrary64 +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} + +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + AddLibrary32($config->{icu} . '\lib\icuin.lib'); + AddLibrary32($config->{icu} . '\lib\icuuc.lib'); + AddLibrary32($config->{icu} . '\lib\icudt.lib'); + AddLibrary64($config->{icu} . '\lib64\icuin.lib'); + AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary64($config->{icu} . '\lib64\icudt.lib'); + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + # AddLibrary($config->{libedit} . "\\" . + # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); + AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); + + + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); + AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); + AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; + } + # return $proj; +} + + + + diff --git a/win32build_2.pl b/win32build_2.pl new file mode 100644 index 000000000..a4f75553c --- /dev/null +++ b/win32build_2.pl @@ -0,0 +1,219 @@ +#!/usr/bin/perl +use JSON; +our $repack_version; +our $pgdir; +our $pgsrc; +if (@ARGV!=2) { + print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; + exit 1; +} + + +our $liblist=""; + + +$pgdir = shift @ARGV; +$pgsrc = shift @ARGV if @ARGV; + + +our $arch = $ENV{'ARCH'} || "x64"; +$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); +$arch='x64' if $arch eq 'X64'; + +$conffile = $pgsrc."/tools/msvc/config.pl"; + + +die 'Could not find config.pl' + unless (-f $conffile); + +our $config; +do $conffile; + + +if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { + print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; + exit 1; +} +our $includepath=""; +our $libpath=""; +AddProject(); + +print "\n\n"; +print $libpath."\n"; +print $includepath."\n"; + +# open F,"<","META.json" or die "Cannot open META.json: $!\n"; +# { +# local $/ = undef; +# $decoded = decode_json(); +# $repack_version= $decoded->{'version'}; +# } + +# substitute new path in the project files + + + +preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); + +exit 0; + + +sub preprocess_project { + my $in = shift; + my $out = shift; + our $pgdir; + our $adddir; + my $libs; + if (defined $adddir) { + $libs ="$adddir;"; + } else{ + $libs =""; + } + open IN,"<",$in or die "Cannot open $in: $!\n"; + open OUT,">",$out or die "Cannot open $out: $!\n"; + +# $includepath .= ";"; +# $libpath .= ";"; + + while () { + s/\@PGROOT\@/$pgdir/g; + s/\@ADDLIBS\@/$libpath/g; + s/\@PGSRC\@/$pgsrc/g; + s/\@ADDINCLUDE\@/$includepath/g; + + + print OUT $_; + } + close IN; + close OUT; + +} + + + +# my sub +sub AddLibrary +{ + $inc = shift; + if ($libpath ne '') + { + $libpath .= ';'; + } + $libpath .= $inc; + +} +sub AddIncludeDir +{ + # my ($self, $inc) = @_; + $inc = shift; + if ($includepath ne '') + { + $includepath .= ';'; + } + $includepath .= $inc; + +} + +sub AddProject +{ + # my ($self, $name, $type, $folder, $initialdir) = @_; + + if ($config->{zlib}) + { + AddIncludeDir($config->{zlib} . '\include'); + AddLibrary($config->{zlib} . '\lib\zdll.lib'); + } + if ($config->{openssl}) + { + AddIncludeDir($config->{openssl} . '\include'); + if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") + { + AddLibrary( + $config->{openssl} . '\lib\VC\ssleay32.lib', 1); + AddLibrary( + $config->{openssl} . '\lib\VC\libeay32.lib', 1); + } + else + { + # We don't expect the config-specific library to be here, + # so don't ask for it in last parameter + AddLibrary( + $config->{openssl} . '\lib\ssleay32.lib', 0); + AddLibrary( + $config->{openssl} . '\lib\libeay32.lib', 0); + } + } + if ($config->{nls}) + { + AddIncludeDir($config->{nls} . '\include'); + AddLibrary($config->{nls} . '\lib\libintl.lib'); + } + if ($config->{gss}) + { + AddIncludeDir($config->{gss} . '\inc\krb5'); + AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); + AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); + AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); + } + if ($config->{iconv}) + { + AddIncludeDir($config->{iconv} . '\include'); + AddLibrary($config->{iconv} . '\lib\iconv.lib'); + } + if ($config->{icu}) + { + AddIncludeDir($config->{icu} . '\include'); + if ($arch eq 'Win32') + { + AddLibrary($config->{icu} . '\lib\icuin.lib'); + AddLibrary($config->{icu} . '\lib\icuuc.lib'); + AddLibrary($config->{icu} . '\lib\icudt.lib'); + } + else + { + AddLibrary($config->{icu} . '\lib64\icuin.lib'); + AddLibrary($config->{icu} . '\lib64\icuuc.lib'); + AddLibrary($config->{icu} . '\lib64\icudt.lib'); + } + } + if ($config->{xml}) + { + AddIncludeDir($config->{xml} . '\include'); + AddIncludeDir($config->{xml} . '\include\libxml2'); + AddLibrary($config->{xml} . '\lib\libxml2.lib'); + } + if ($config->{xslt}) + { + AddIncludeDir($config->{xslt} . '\include'); + AddLibrary($config->{xslt} . '\lib\libxslt.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{uuid}) + { + AddIncludeDir($config->{uuid} . '\include'); + AddLibrary($config->{uuid} . '\lib\uuid.lib'); + } + if ($config->{libedit}) + { + AddIncludeDir($config->{libedit} . '\include'); + AddLibrary($config->{libedit} . "\\" . + ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); + } + if ($config->{zstd}) + { + AddIncludeDir($config->{zstd}); + AddLibrary($config->{zstd}. "\\". + ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") + ); + } + # return $proj; +} + + + + From 3769efdcb2c2cf1d88261d2d5e382d6b766e48a6 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 2 Nov 2018 15:08:33 +0300 Subject: [PATCH 0021/2107] Update copyrights and rename COPYRIGHT to LICENSE --- COPYRIGHT | 29 ----------------------------- LICENSE | 22 ++++++++++++++++++++++ README.md | 6 +++--- src/archive.c | 2 +- src/backup.c | 2 +- src/catalog.c | 2 +- src/data.c | 2 +- src/delete.c | 2 +- src/dir.c | 2 +- src/fetch.c | 2 +- src/help.c | 2 +- src/init.c | 2 +- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 +- src/restore.c | 2 +- src/util.c | 2 +- src/utils/logger.c | 2 +- src/utils/logger.h | 2 +- src/utils/pgut.c | 6 +++--- src/utils/pgut.h | 4 ++-- src/validate.c | 2 +- 21 files changed, 46 insertions(+), 53 deletions(-) delete mode 100644 COPYRIGHT create mode 100644 LICENSE diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index 49d704724..000000000 --- a/COPYRIGHT +++ /dev/null @@ -1,29 +0,0 @@ -Copyright (c) 2015-2017, Postgres Professional -Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - -Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group -Portions Copyright (c) 1994, The Regents of the University of California - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION - (NTT) nor the names of its contributors may be used to endorse or - promote products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..3969eaa89 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2015-2018, Postgres Professional +Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + +Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +Portions Copyright (c) 1994, The Regents of the University of California + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose, without fee, and without a written agreement +is hereby granted, provided that the above copyright notice and this +paragraph and the following two paragraphs appear in all copies. + +IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR +DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING +LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +DOCUMENTATION, EVEN IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS +ON AN "AS IS" BASIS, AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO +PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/README.md b/README.md index 73936ae1e..53945eb88 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The utility is compatible with: `PTRACK` backup support provided via following options: * vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* Postgres Pro Standard 9.5, 9.6, 10 +* Postgres Pro Standard 9.5, 9.6, 10, 11 * Postgres Pro Enterprise 9.5, 9.6, 10 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: @@ -38,7 +38,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * Creating backups from a remote server is currently not supported. -* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number. +* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. * Microsoft Windows operating system is not supported. * Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. @@ -85,7 +85,7 @@ Currently the latest documentation can be found at [Postgres Pro Enterprise docu ## Licence -This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/). +This module available under the [license](LICENSE) similar to [PostgreSQL](https://www.postgresql.org/about/licence/). ## Feedback diff --git a/src/archive.c b/src/archive.c index e26d17b6b..2953b89e8 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2017, Postgres Professional + * Portions Copyright (c) 2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/backup.c b/src/backup.c index 0dd681fae..b2aed3186 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3,7 +3,7 @@ * backup.c: backup DB cluster, archived WAL * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/catalog.c b/src/catalog.c index 74d8ee908..41676f4c9 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -3,7 +3,7 @@ * catalog.c: backup catalog operation * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/data.c b/src/data.c index c3fcc2544..a8a9e4de7 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/delete.c b/src/delete.c index 9d1c3867d..c5f16af7d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -3,7 +3,7 @@ * delete.c: delete backup files. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/dir.c b/src/dir.c index 1a93c1fee..9e55c8217 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,7 +3,7 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/fetch.c b/src/fetch.c index 17e77025a..988ce2834 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -3,7 +3,7 @@ * fetch.c * Functions for fetching files from PostgreSQL data directory * - * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ diff --git a/src/help.c b/src/help.c index f534f3967..1cf6d4041 100644 --- a/src/help.c +++ b/src/help.c @@ -2,7 +2,7 @@ * * help.c * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/init.c b/src/init.c index d8e238fdf..75b50e4f0 100644 --- a/src/init.c +++ b/src/init.c @@ -3,7 +3,7 @@ * init.c: - initialize backup catalog. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 113e8901f..cf51fd107 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -3,7 +3,7 @@ * pg_probackup.c: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e337771e6..7bd87e56c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/restore.c b/src/restore.c index 9c87cd39f..c08e647cc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/util.c b/src/util.c index 4eefa7887..8d4974ca9 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 563d20277..4cdbf7211 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -2,7 +2,7 @@ * * logger.c: - log events into log file or stderr. * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.h b/src/utils/logger.h index e1feb86c6..15ec38f1b 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -2,7 +2,7 @@ * * logger.h: - prototypes of logger functions. * - * Copyright (c) 2017-2017, Postgres Professional + * Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ec3fd8bbb..b789a326e 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2017, Postgres Professional + * Portions Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ @@ -1655,8 +1655,8 @@ pgut_disconnect(PGconn *conn) PGresult * -pgut_execute_parallel(PGconn* conn, - PGcancel* thread_cancel_conn, const char *query, +pgut_execute_parallel(PGconn* conn, + PGcancel* thread_cancel_conn, const char *query, int nParams, const char **params, bool text_result) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 9aac75cac..6e9a50fda 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2017, Postgres Professional + * Portions Copyright (c) 2017-2018, Postgres Professional * *------------------------------------------------------------------------- */ @@ -115,7 +115,7 @@ extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params); extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, const char **params, bool text_result, bool ok_error); -extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, +extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, const char *query, int nParams, const char **params, bool text_result); extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); diff --git a/src/validate.c b/src/validate.c index f61bcb79f..4fa7d78b6 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,7 +3,7 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2017, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ From 078d341bb8990ec934d011314ef9f4a8162208aa Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Fri, 2 Nov 2018 15:47:15 +0300 Subject: [PATCH 0022/2107] pg_control: fix reading of pg_control of vanilla in pg_controldata Do not change the CRC as this makes its check senseless. Instead, read the pg_control of vanilla to the pg_control strcucture of vanilla (ControlFileDataOriginal) and print its values as is done in vanilla. Thanks to Arthur Zakirov for reporting this. Also: fix checking of byte ordering mismatch of pg_control in pg_probackup. --- src/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index 4eefa7887..27a0ca244 100644 --- a/src/util.c +++ b/src/util.c @@ -71,7 +71,8 @@ checkControlFile(ControlFileData *ControlFile) "Either the file is corrupt, or it has a different layout than this program\n" "is expecting. The results below are untrustworthy."); - if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) + if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) && + ControlFile->pg_control_version / 65536 != 0) elog(ERROR, "possible byte ordering mismatch\n" "The byte ordering used to store the pg_control file might not match the one\n" "used by this program. In that case the results below would be incorrect, and\n" From 050d0150d0e4274d50d6bc02b912ce8844e26bac Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 2 Nov 2018 20:24:09 +0300 Subject: [PATCH 0023/2107] Implement remote backup though SSH --- Makefile | 2 +- src/backup.c | 23 ++++++----- src/catalog.c | 70 +++++++++++++++++----------------- src/data.c | 95 +++++++++++++++++++++------------------------- src/dir.c | 20 +++++----- src/init.c | 2 +- src/merge.c | 2 +- src/pg_probackup.c | 56 +++++++++++++++++++-------- src/pg_probackup.h | 23 ++++++----- src/restore.c | 2 +- src/utils/pgut.c | 5 ++- 11 files changed, 162 insertions(+), 138 deletions(-) diff --git a/Makefile b/Makefile index 55d4428c0..f3e14b5da 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PROGRAM = pg_probackup # utils OBJS = src/utils/json.o src/utils/logger.o src/utils/parray.o \ - src/utils/pgut.o src/utils/thread.o + src/utils/pgut.o src/utils/thread.o src/utils/remote.o src/utils/file.o OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ diff --git a/src/backup.c b/src/backup.c index 0dd681fae..5abdc12c8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -22,6 +22,7 @@ #include #include "utils/thread.h" +#include "utils/file.h" #define PG_STOP_BACKUP_TIMEOUT 300 @@ -572,7 +573,7 @@ do_backup_instance(void) if (stream_wal) { join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); - dir_create_dir(dst_backup_path, DIR_PERMISSION); + fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); stream_thread_arg.basedir = dst_backup_path; @@ -689,7 +690,7 @@ do_backup_instance(void) DATABASE_DIR); join_path_components(dirpath, database_path, dir_name); - dir_create_dir(dirpath, DIR_PERMISSION); + fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } /* setup threads */ @@ -1847,16 +1848,15 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fopen(backup_label, PG_BINARY_W); + fp = fio_open(backup_label, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); len = strlen(PQgetvalue(res, 0, 3)); - if (fwrite(PQgetvalue(res, 0, 3), 1, len, fp) != len || - fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) + if (fio_write(fp, PQgetvalue(res, 0, 3), len) != len || + fio_flush(fp) != 0 || + fio_close(fp)) elog(ERROR, "can't write backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1895,16 +1895,15 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fopen(tablespace_map, PG_BINARY_W); + fp = fio_open(tablespace_map, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); len = strlen(val); - if (fwrite(val, 1, len, fp) != len || - fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) + if (fio_write(fp, val, len) != len || + fio_flush(fp) != 0 || + fio_close(fp)) elog(ERROR, "can't write tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); diff --git a/src/catalog.c b/src/catalog.c index 74d8ee908..fae199c62 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -8,13 +8,14 @@ *------------------------------------------------------------------------- */ -#include "pg_probackup.h" - #include #include #include #include +#include "pg_probackup.h" +#include "utils/file.h" + static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); @@ -413,13 +414,13 @@ pgBackupCreateDir(pgBackup *backup) if (!dir_is_empty(path)) elog(ERROR, "backup destination is not empty \"%s\"", path); - dir_create_dir(path, DIR_PERMISSION); + fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); /* create directories for actual backup files */ for (i = 0; subdirs[i]; i++) { pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); - dir_create_dir(path, DIR_PERMISSION); + fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); } return 0; @@ -433,46 +434,46 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) { char timestamp[100]; - fprintf(out, "#Configuration\n"); - fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); - fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); - fprintf(out, "compress-alg = %s\n", + fio_printf(out, "#Configuration\n"); + fio_printf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fio_printf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fio_printf(out, "compress-alg = %s\n", deparse_compress_alg(backup->compress_alg)); - fprintf(out, "compress-level = %d\n", backup->compress_level); - fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + fio_printf(out, "compress-level = %d\n", backup->compress_level); + fio_printf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); - fprintf(out, "\n#Compatibility\n"); - fprintf(out, "block-size = %u\n", backup->block_size); - fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); - fprintf(out, "checksum-version = %u\n", backup->checksum_version); + fio_printf(out, "\n#Compatibility\n"); + fio_printf(out, "block-size = %u\n", backup->block_size); + fio_printf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fio_printf(out, "checksum-version = %u\n", backup->checksum_version); if (backup->program_version[0] != '\0') - fprintf(out, "program-version = %s\n", backup->program_version); + fio_printf(out, "program-version = %s\n", backup->program_version); if (backup->server_version[0] != '\0') - fprintf(out, "server-version = %s\n", backup->server_version); + fio_printf(out, "server-version = %s\n", backup->server_version); - fprintf(out, "\n#Result backup info\n"); - fprintf(out, "timelineid = %d\n", backup->tli); + fio_printf(out, "\n#Result backup info\n"); + fio_printf(out, "timelineid = %d\n", backup->tli); /* LSN returned by pg_start_backup */ - fprintf(out, "start-lsn = %X/%X\n", + fio_printf(out, "start-lsn = %X/%X\n", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); /* LSN returned by pg_stop_backup */ - fprintf(out, "stop-lsn = %X/%X\n", + fio_printf(out, "stop-lsn = %X/%X\n", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); time2iso(timestamp, lengthof(timestamp), backup->start_time); - fprintf(out, "start-time = '%s'\n", timestamp); + fio_printf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->end_time); - fprintf(out, "end-time = '%s'\n", timestamp); + fio_printf(out, "end-time = '%s'\n", timestamp); } - fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + fio_printf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - fprintf(out, "recovery-time = '%s'\n", timestamp); + fio_printf(out, "recovery-time = '%s'\n", timestamp); } /* @@ -480,20 +481,20 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) * WAL segments in archive 'wal' directory. */ if (backup->data_bytes != BYTES_INVALID) - fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + fio_printf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); if (backup->wal_bytes != BYTES_INVALID) - fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + fio_printf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); - fprintf(out, "status = %s\n", status2str(backup->status)); + fio_printf(out, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ if (backup->parent_backup != 0) - fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + fio_printf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); /* print connection info except password */ if (backup->primary_conninfo) - fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); + fio_printf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); } /* @@ -506,14 +507,14 @@ write_backup(pgBackup *backup) char conf_path[MAXPGPATH]; pgBackupGetPath(backup, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); - fp = fopen(conf_path, "wt"); + fp = fio_open(conf_path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, strerror(errno)); pgBackupWriteControl(fp, backup); - fclose(fp); + fio_close(fp); } /* @@ -527,16 +528,15 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - fp = fopen(path, "wt"); + fp = fio_open(path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "cannot open file list \"%s\": %s", path, strerror(errno)); print_file_list(fp, files, root); - if (fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) + if (fio_flush(fp) != 0 || + fio_close(fp)) elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); } diff --git a/src/data.c b/src/data.c index c3fcc2544..05644a88b 100644 --- a/src/data.c +++ b/src/data.c @@ -13,7 +13,7 @@ #include "storage/checksum.h" #include "storage/checksum_impl.h" #include - +#include "utils/file.h" #include #ifdef HAVE_LIBZ @@ -419,12 +419,12 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size); /* write data page */ - if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) + if (fio_write(out, write_buffer, write_buffer_size) != write_buffer_size) { int errno_tmp = errno; fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", file->path, blknum, strerror(errno_tmp)); } @@ -512,7 +512,7 @@ backup_data_file(backup_files_arg* arguments, nblocks = file->size/BLCKSZ; /* open backup file for write */ - out = fopen(to_path, PG_BINARY_W); + out = fio_open(to_path, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) { int errno_tmp = errno; @@ -571,18 +571,17 @@ backup_data_file(backup_files_arg* arguments, } /* update file permission */ - if (chmod(to_path, FILE_PERMISSION) == -1) + if (fio_chmod(to_path, FILE_PERMISSION, FIO_BACKUP_HOST) == -1) { int errno_tmp = errno; fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot change mode of \"%s\": %s", file->path, strerror(errno_tmp)); } - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fio_flush(out) != 0 || + fio_close(out)) elog(ERROR, "cannot write backup file \"%s\": %s", to_path, strerror(errno)); fclose(in); @@ -639,9 +638,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ - out = fopen(to_path, PG_BINARY_R "+"); - if (out == NULL && errno == ENOENT) - out = fopen(to_path, PG_BINARY_W); + out = fio_open(to_path, PG_BINARY_W "+", FIO_DB_HOST); if (out == NULL) { int errno_tmp = errno; @@ -737,27 +734,27 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, /* * Seek and write the restored page. */ - if (fseek(out, write_pos, SEEK_SET) < 0) + if (fio_seek(out, write_pos) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); if (write_header) { - if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) + if (fio_write(out, &header, sizeof(header)) != sizeof(header)) elog(ERROR, "cannot write header of block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } if (header.compressed_size < BLCKSZ) { - if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) + if (fio_write(out, page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } else { /* if page wasn't compressed, we've read full block */ - if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) + if (fio_write(out, compressed_page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } @@ -775,7 +772,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, size_t file_size = 0; /* get file current size */ - fseek(out, 0, SEEK_END); + fio_seek(out, 0); file_size = ftell(out); if (file_size > file->n_blocks * BLCKSZ) @@ -795,7 +792,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, /* * Truncate file to this length. */ - if (ftruncate(fileno(out), write_pos) != 0) + if (fio_truncate(out, write_pos) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); elog(VERBOSE, "Delta truncate file %s to block %u", @@ -803,20 +800,19 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } /* update file permission */ - if (chmod(to_path, file->mode) == -1) + if (fio_chmod(to_path, file->mode, FIO_DB_HOST) == -1) { int errno_tmp = errno; if (in) fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); } - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fio_flush(out) != 0 || + fio_close(out)) elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); if (in) fclose(in); @@ -862,7 +858,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fopen(to_path, PG_BINARY_W); + out = fio_open(to_path, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) { int errno_tmp = errno; @@ -875,7 +871,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if (fstat(fileno(in), &st) == -1) { fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot stat \"%s\": %s", file->path, strerror(errno)); } @@ -888,12 +884,12 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) break; - if (fwrite(buf, 1, read_len, out) != read_len) + if (fio_write(out, buf, read_len) != read_len) { errno_tmp = errno; /* oops */ fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -907,7 +903,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if (!feof(in)) { fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot read backup mode file \"%s\": %s", file->path, strerror(errno_tmp)); } @@ -915,12 +911,12 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* copy odd part. */ if (read_len > 0) { - if (fwrite(buf, 1, read_len, out) != read_len) + if (fio_write(out, buf, read_len) != read_len) { errno_tmp = errno; /* oops */ fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -936,18 +932,17 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->crc = crc; /* update file permission */ - if (chmod(to_path, st.st_mode) == -1) + if (fio_chmod(to_path, st.st_mode, FIO_BACKUP_HOST) == -1) { errno_tmp = errno; fclose(in); - fclose(out); + fio_close(out); elog(ERROR, "cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); } - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fio_flush(out) != 0 || + fio_close(out)) elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); fclose(in); @@ -992,7 +987,7 @@ get_gz_error(gzFile gzf, int errnum) * Copy file attributes */ static void -copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) +copy_meta(const char *from_path, const char *to_path, bool unlink_on_error, fio_location location) { struct stat st; @@ -1004,7 +999,7 @@ copy_meta(const char *from_path, const char *to_path, bool unlink_on_error) from_path, strerror(errno)); } - if (chmod(to_path, st.st_mode) == -1) + if (fio_chmod(to_path, st.st_mode, location) == -1) { if (unlink_on_error) unlink(to_path); @@ -1064,7 +1059,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, PG_BINARY_W); + out = fio_open(to_path_temp, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1102,7 +1097,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fwrite(buf, 1, read_len, out) != read_len) + if (fio_write(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1130,9 +1125,8 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fio_flush(out) != 0 || + fio_close(out)) { errno_temp = errno; unlink(to_path_temp); @@ -1150,9 +1144,9 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, } /* update file permission. */ - copy_meta(from_path, to_path_temp, true); + copy_meta(from_path, to_path_temp, true, FIO_BACKUP_HOST); - if (rename(to_path_temp, to_path_p) < 0) + if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) { errno_temp = errno; unlink(to_path_temp); @@ -1223,7 +1217,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, PG_BINARY_W); + out = fio_open(to_path_temp, PG_BINARY_W, FIO_DB_HOST); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1260,7 +1254,7 @@ get_wal_file(const char *from_path, const char *to_path) if (read_len > 0) { - if (fwrite(buf, 1, read_len, out) != read_len) + if (fio_write(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1284,9 +1278,8 @@ get_wal_file(const char *from_path, const char *to_path) } } - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fio_flush(out) != 0 || + fio_close(out)) { errno_temp = errno; unlink(to_path_temp); @@ -1318,9 +1311,9 @@ get_wal_file(const char *from_path, const char *to_path) } /* update file permission. */ - copy_meta(from_path_p, to_path_temp, true); + copy_meta(from_path_p, to_path_temp, true, FIO_DB_HOST); - if (rename(to_path_temp, to_path) < 0) + if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) { errno_temp = errno; unlink(to_path_temp); diff --git a/src/dir.c b/src/dir.c index 1a93c1fee..9509a2f50 100644 --- a/src/dir.c +++ b/src/dir.c @@ -9,6 +9,8 @@ */ #include "pg_probackup.h" +#include "utils/file.h" + #if PG_VERSION_NUM < 110000 #include "catalog/catalog.h" @@ -926,7 +928,7 @@ opt_tablespace_map(pgut_option *opt, const char *arg) */ void create_data_directories(const char *data_dir, const char *backup_dir, - bool extract_tablespaces) + bool extract_tablespaces, fio_location location) { parray *dirs, *links = NULL; @@ -1026,11 +1028,11 @@ create_data_directories(const char *data_dir, const char *backup_dir, linked_path, relative_ptr); /* Firstly, create linked directory */ - dir_create_dir(linked_path, DIR_PERMISSION); + fio_mkdir(linked_path, DIR_PERMISSION, location); join_path_components(to_path, data_dir, PG_TBLSPC_DIR); /* Create pg_tblspc directory just in case */ - dir_create_dir(to_path, DIR_PERMISSION); + fio_mkdir(to_path, DIR_PERMISSION, location); /* Secondly, create link */ join_path_components(to_path, to_path, link_name); @@ -1057,7 +1059,7 @@ create_data_directories(const char *data_dir, const char *backup_dir, /* This is not symlink, create directory */ join_path_components(to_path, data_dir, relative_ptr); - dir_create_dir(to_path, DIR_PERMISSION); + fio_mkdir(to_path, DIR_PERMISSION, location); } if (extract_tablespaces) @@ -1204,7 +1206,7 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + fio_printf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\"", @@ -1213,19 +1215,19 @@ print_file_list(FILE *out, const parray *files, const char *root) deparse_compress_alg(file->compress_alg)); if (file->is_datafile) - fprintf(out, ",\"segno\":\"%d\"", file->segno); + fio_printf(out, ",\"segno\":\"%d\"", file->segno); #ifndef WIN32 if (S_ISLNK(file->mode)) #else if (pgwin32_is_junction(file->path)) #endif - fprintf(out, ",\"linked\":\"%s\"", file->linked); + fio_printf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) - fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + fio_printf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); - fprintf(out, "}\n"); + fio_printf(out, "}\n"); } } diff --git a/src/init.c b/src/init.c index d8e238fdf..e5e5cb1b6 100644 --- a/src/init.c +++ b/src/init.c @@ -93,7 +93,7 @@ do_add_instance(void) dir_create_dir(arclog_path, DIR_PERMISSION); /* - * Wite initial config. system-identifier and pgdata are set in + * Write initial config. system-identifier and pgdata are set in * init subcommand and will never be updated. */ pgBackupConfigInit(config); diff --git a/src/merge.c b/src/merge.c index 2464199f2..2873eb5c4 100644 --- a/src/merge.c +++ b/src/merge.c @@ -182,7 +182,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), DATABASE_DIR); - create_data_directories(to_database_path, from_backup_path, false); + create_data_directories(to_database_path, from_backup_path, false, FIO_BACKUP_HOST); /* * Get list of files which will be modified or removed. diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 113e8901f..124625c72 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -12,6 +12,7 @@ #include "pg_getopt.h" #include "streamutil.h" +#include "utils/file.h" #include @@ -65,7 +66,6 @@ char *replication_slot = NULL; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -bool is_remote_backup = false; /* Wait timeout for WAL segment archiving */ uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; const char *master_db = NULL; @@ -73,6 +73,10 @@ const char *master_host = NULL; const char *master_port= NULL; const char *master_user = NULL; uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; +char *ssh_host; +char *ssh_port; +bool is_remote_agent; +bool is_remote_backup; /* restore options */ static char *target_time; @@ -165,17 +169,19 @@ static pgut_option options[] = { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, - /* TODO not completed feature. Make it unavailiable from user level - { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + { 's', 19, "ssh-host", &ssh_host, SOURCE_CMDLINE, }, + { 's', 20, "ssh-port", &ssh_port, SOURCE_CMDLINE, }, + { 'b', 21, "agent", &is_remote_agent, SOURCE_CMDLINE, }, + { 'b', 22, "remote", &is_remote_backup, SOURCE_CMDLINE, }, /* restore options */ - { 's', 20, "time", &target_time, SOURCE_CMDLINE }, - { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, - { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, - { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, + { 's', 30, "time", &target_time, SOURCE_CMDLINE }, + { 's', 31, "xid", &target_xid, SOURCE_CMDLINE }, + { 's', 32, "inclusive", &target_inclusive, SOURCE_CMDLINE }, + { 'u', 33, "timeline", &target_tli, SOURCE_CMDLINE }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, - { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, - { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, - { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, + { 'b', 34, "immediate", &target_immediate, SOURCE_CMDLINE }, + { 's', 35, "recovery-target-name", &target_name, SOURCE_CMDLINE }, + { 's', 36, "recovery-target-action", &target_action, SOURCE_CMDLINE }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, @@ -372,10 +378,28 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); - /* Ensure that backup_path is a path to a directory */ - rc = stat(backup_path, &stat_buf); - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); + if (ssh_host != NULL && + (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD)) + { + if (is_remote_agent) { + if (backup_subcmd != BACKUP_CMD) { + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; + } + fio_redirect(STDIN_FILENO, STDOUT_FILENO); + } else { + /* Execute remote probackup */ + remote_execute(argc, argv, backup_subcmd == BACKUP_CMD); + } + } + + if (!is_remote_agent) + { + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + } /* command was initialized for a few commands */ if (command) @@ -409,7 +433,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && !is_remote_agent) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -523,7 +547,7 @@ main(int argc, char *argv[]) elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + stream_wal ? "true" : "false", ssh_host ? "true" : "false"); return do_backup(start_time); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e337771e6..a3e76b95b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -10,23 +10,23 @@ #ifndef PG_PROBACKUP_H #define PG_PROBACKUP_H -#include "postgres_fe.h" -#include "libpq-fe.h" - -#include "access/xlog_internal.h" -#include "utils/pg_crc.h" +#include +#include +#include +#include #ifdef FRONTEND #undef FRONTEND -#include "port/atomics.h" +#include #define FRONTEND #else -#include "port/atomics.h" +#include #endif #include "utils/logger.h" #include "utils/parray.h" #include "utils/pgut.h" +#include "utils/file.h" #include "datapagemap.h" @@ -325,11 +325,14 @@ extern char *replication_slot; extern bool smooth_checkpoint; #define ARCHIVE_TIMEOUT_DEFAULT 300 extern uint32 archive_timeout; -extern bool is_remote_backup; +extern char *ssh_port; +extern char *ssh_host; extern const char *master_db; extern const char *master_host; extern const char *master_port; extern const char *master_user; +extern bool is_remote_backup; +extern bool is_remote_agent; #define REPLICA_TIMEOUT_DEFAULT 300 extern uint32 replica_timeout; @@ -472,6 +475,7 @@ extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); +extern void remote_execute(int argc, char *argv[], bool do_backup); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS #define COMPRESS_LEVEL_DEFAULT 1 @@ -484,7 +488,8 @@ extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root); extern void create_data_directories(const char *data_dir, const char *backup_dir, - bool extract_tablespaces); + bool extract_tablespaces, + fio_location location); extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(pgut_option *opt, const char *arg); diff --git a/src/restore.c b/src/restore.c index 9c87cd39f..f798675b5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -445,7 +445,7 @@ restore_backup(pgBackup *backup) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - create_data_directories(pgdata, this_backup_path, true); + create_data_directories(pgdata, this_backup_path, true, FIO_DB_HOST); /* * Get list of files which need to be restored. diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ec3fd8bbb..ba4b077e6 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -19,6 +19,7 @@ #include "pgut.h" #include "logger.h" +#include "file.h" #define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ #define SECS_PER_MINUTE 60 @@ -1147,7 +1148,7 @@ pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) if (!options) return parsed_options; - if ((fp = pgut_fopen(path, "rt", true)) == NULL) + if ((fp = fio_open_stream(path, FIO_BACKUP_HOST)) == NULL) return parsed_options; while (fgets(buf, lengthof(buf), fp)) @@ -1181,7 +1182,7 @@ pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) } } - fclose(fp); + fio_close_stream(fp); return parsed_options; } From ca5c4b09c7493c3b96eb2dfb1d177ce10bba11b8 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 2 Nov 2018 20:31:48 +0300 Subject: [PATCH 0024/2107] Add missed files --- src/utils/file.c | 435 +++++++++++++++++++++++++++++++++++++++++++++ src/utils/file.h | 58 ++++++ src/utils/remote.c | 83 +++++++++ 3 files changed, 576 insertions(+) create mode 100644 src/utils/file.c create mode 100644 src/utils/file.h create mode 100644 src/utils/remote.c diff --git a/src/utils/file.c b/src/utils/file.c new file mode 100644 index 000000000..7febd8261 --- /dev/null +++ b/src/utils/file.c @@ -0,0 +1,435 @@ +#include +#include +#include +#include + +#include "pg_probackup.h" +#include "file.h" + +#define PRINTF_BUF_SIZE 1024 + +static pthread_mutex_t fio_mutex = PTHREAD_MUTEX_INITIALIZER; +static unsigned long fio_fdset = 0; +static void* fio_stdin_buffer; +static int fio_stdout = 0; +static int fio_stdin = 0; + +void fio_redirect(int in, int out) +{ + fio_stdin = in; + fio_stdout = out; +} + +static bool fio_is_remote_file(FILE* fd) +{ + return (size_t)fd <= FIO_FDMAX; +} + +static bool fio_is_remote(fio_location location) +{ + return (location == FIO_BACKUP_HOST && is_remote_agent) + || (location == FIO_DB_HOST && ssh_host != NULL); +} + +static ssize_t fio_read(int fd, void* buf, size_t size) +{ + size_t offs = 0; + while (offs < size) + { + ssize_t rc = read(fd, (char*)buf + offs, size - offs); + if (rc < 0) { + if (errno == EINTR) { + continue; + } + return rc; + } + offs += rc; + } + return size; +} + +FILE* fio_open_stream(char const* path, fio_location location) +{ + FILE* f; + if (fio_is_remote(location)) + { + fio_header hdr; + hdr.cop = FIO_READ; + hdr.size = strlen(path) + 1; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, path, hdr.size), hdr.size); + + IO_CHECK(fio_read(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > 0) + { + Assert(fio_stdin_buffer == NULL); + fio_stdin_buffer = malloc(hdr.size); + IO_CHECK(fio_read(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); + f = fmemopen(fio_stdin_buffer, hdr.size, "r"); + } + else + { + f = NULL; + } + } + else + { + f = fopen(path, "rt"); + } + return f; +} + +int fio_close_stream(FILE* f) +{ + if (fio_stdin_buffer) + { + free(fio_stdin_buffer); + fio_stdin_buffer = NULL; + } + return fclose(f); +} + +FILE* fio_open(char const* path, char const* mode, fio_location location) +{ + FILE* f; + if (fio_is_remote(location)) + { + int i; + fio_header hdr; + unsigned long mask; + + SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + mask = fio_fdset; + for (i = 0; (mask & 1) != 0; i++, mask >>= 1); + if (i == FIO_FDMAX) { + return NULL; + } + hdr.cop = strchr(mode,'+') ? FIO_OPEN_EXISTED : FIO_OPEN_NEW; + hdr.handle = i; + hdr.size = strlen(path) + 1; + fio_fdset |= 1 << i; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, path, hdr.size), hdr.size); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + + f = (FILE*)(size_t)(i + 1); + } + else + { + f = fopen(path, mode); + } + return f; +} + +int fio_printf(FILE* f, char const* format, ...) +{ + int rc; + va_list args; + va_start (args, format); + if (fio_stdout) + { + char buf[PRINTF_BUF_SIZE]; +#ifdef HAS_VSNPRINTF + rc = vsnprintf(buf, sizeof(buf), format, args); +#else + rc = vsprintf(buf, format, args); +#endif + if (rc > 0) { + fio_write(f, buf, rc); + } + } + else + { + rc = vfprintf(f, format, args); + } + va_end (args); + return rc; +} + +int fio_flush(FILE* f) +{ + int rc = 0; + if (!fio_is_remote_file(f)) + { + rc = fflush(f); + if (rc == 0) { + rc = fsync(fileno(f)); + } + } + return rc; +} + +int fio_close(FILE* f) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + + hdr.cop = FIO_CLOSE; + hdr.handle = (size_t)f - 1; + hdr.size = 0; + fio_fdset &= ~(1 << hdr.handle); + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return fclose(f); + } +} + +int fio_truncate(FILE* f, off_t size) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + + hdr.cop = FIO_TRUNCATE; + hdr.handle = (size_t)f - 1; + hdr.size = 0; + hdr.arg = size; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return ftruncate(fileno(f), size); + } +} + +int fio_seek(FILE* f, off_t offs) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + + hdr.cop = FIO_SEEK; + hdr.handle = (size_t)f - 1; + hdr.size = 0; + hdr.arg = offs; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return fseek(f, offs, SEEK_SET); + } +} + +size_t fio_write(FILE* f, void const* buf, size_t size) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + + hdr.cop = FIO_WRITE; + hdr.handle = (size_t)f - 1; + hdr.size = size; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, buf, size), size); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return size; + } + else + { + return fwrite(buf, 1, size, f); + } +} + +int fio_rename(char const* old_path, char const* new_path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t old_path_len = strlen(old_path) + 1; + size_t new_path_len = strlen(new_path) + 1; + hdr.cop = FIO_RENAME; + hdr.handle = -1; + hdr.size = old_path_len + new_path_len; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, old_path, old_path_len), old_path_len); + IO_CHECK(write(fio_stdout, new_path, new_path_len), new_path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return rename(old_path, new_path); + } +} + +int fio_unlink(char const* path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_UNLINK; + hdr.handle = -1; + hdr.size = path_len; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return unlink(path); + } +} + +int fio_mkdir(char const* path, int mode, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_MKDIR; + hdr.handle = -1; + hdr.size = path_len; + hdr.arg = mode; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return dir_create_dir(path, mode); + } +} + +int fio_chmod(char const* path, int mode, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_CHMOD; + hdr.handle = -1; + hdr.size = path_len; + hdr.arg = mode; + + IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(write(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + return 0; + } + else + { + return chmod(path, mode); + } +} + +static void fio_send_file(int out, char const* path) +{ + int fd = open(path, O_RDONLY); + fio_header hdr; + void* buf = NULL; + + hdr.size = 0; + hdr.cop = FIO_READ; + + if (fd >= 0) + { + off_t size = lseek(fd, 0, SEEK_END); + buf = malloc(size); + lseek(fd, 0, SEEK_SET); + IO_CHECK(fio_read(fd, buf, size), size); + hdr.size = size; + SYS_CHECK(close(fd)); + } + IO_CHECK(write(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (buf) + { + IO_CHECK(write(out, buf, hdr.size), hdr.size); + free(buf); + } +} + +void fio_communicate(int in, int out) +{ + int fd[FIO_FDMAX]; + char buf[BLCKSZ*2]; /* need more space for page header */ + fio_header hdr; + int rc; + + while ((rc = read(in, &hdr, sizeof hdr)) == sizeof(hdr)) { + if (hdr.size != 0) { + Assert(hdr.size < sizeof(buf)); + IO_CHECK(fio_read(in, buf, hdr.size), hdr.size); + } + switch (hdr.cop) { + case FIO_READ: + fio_send_file(out, buf); + break; + case FIO_OPEN_NEW: + SYS_CHECK(fd[hdr.handle] = open(buf, O_RDWR|O_CREAT|O_TRUNC, 0777)); + break; + case FIO_OPEN_EXISTED: + SYS_CHECK(fd[hdr.handle] = open(buf, O_RDWR|O_CREAT, 0777)); + break; + case FIO_CLOSE: + SYS_CHECK(close(fd[hdr.handle])); + break; + case FIO_WRITE: + IO_CHECK(write(fd[hdr.handle], buf, hdr.size), hdr.size); + break; + case FIO_RENAME: + SYS_CHECK(rename(buf, buf + strlen(buf))); + break; + case FIO_UNLINK: + SYS_CHECK(unlink(buf)); + break; + case FIO_MKDIR: + SYS_CHECK(dir_create_dir(buf, hdr.arg)); + break; + case FIO_CHMOD: + SYS_CHECK(chmod(buf, hdr.arg)); + break; + case FIO_SEEK: + SYS_CHECK(lseek(fd[hdr.handle], hdr.arg, SEEK_SET)); + break; + case FIO_TRUNCATE: + SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); + break; + default: + Assert(false); + } + } + + if (rc != 0) { + perror("read"); + exit(EXIT_FAILURE); + } +} + diff --git a/src/utils/file.h b/src/utils/file.h new file mode 100644 index 000000000..c5ec6118d --- /dev/null +++ b/src/utils/file.h @@ -0,0 +1,58 @@ +#ifndef __FILE__H__ +#define __FILE__H__ + +#include + +typedef enum +{ + FIO_OPEN_NEW, + FIO_OPEN_EXISTED, + FIO_CLOSE, + FIO_WRITE, + FIO_RENAME, + FIO_UNLINK, + FIO_MKDIR, + FIO_CHMOD, + FIO_SEEK, + FIO_TRUNCATE, + FIO_READ +} fio_operations; + +typedef enum +{ + FIO_DB_HOST, + FIO_BACKUP_HOST +} fio_location; + +#define FIO_FDMAX 64 + +#define SYS_CHECK(cmd) do if ((cmd) < 0) { perror(#cmd); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "Receive %d bytes instead of %d\n", _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) + +typedef struct { + unsigned cop : 4; + unsigned handle : 8; + unsigned size : 20; + unsigned arg; +} fio_header; + +extern void fio_redirect(int in, int out); +extern void fio_communicate(int in, int out); + +extern FILE* fio_open(char const* name, char const* mode, fio_location location); +extern int fio_close(FILE* f); +extern size_t fio_write(FILE* f, void const* buf, size_t size); +extern int fio_printf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); +extern int fio_flush(FILE* f); +extern int fio_seek(FILE* f, off_t offs); +extern int fio_truncate(FILE* f, off_t size); +extern int fio_rename(char const* old_path, char const* new_path, fio_location location); +extern int fio_unlink(char const* path, fio_location location); +extern int fio_mkdir(char const* path, int mode, fio_location location); +extern int fio_chmod(char const* path, int mode, fio_location location); + +extern FILE* fio_open_stream(char const* name, fio_location location); +extern int fio_close_stream(FILE* f); + +#endif + diff --git a/src/utils/remote.c b/src/utils/remote.c new file mode 100644 index 000000000..f7b3f4c5b --- /dev/null +++ b/src/utils/remote.c @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include + +#include "pg_probackup.h" +#include "file.h" + +#define MAX_CMDLINE_LENGTH 4096 + +static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) +{ + size_t len = strlen(src); + if (dst + len + 1 >= buf_size) { + fprintf(stderr, "Too long command line\n"); + exit(EXIT_FAILURE); + } + buf[dst] = ' '; + memcpy(&buf[dst+1], src, len); + return dst + len + 1; +} + +void remote_execute(int argc, char* argv[], bool listen) +{ + char cmd[MAX_CMDLINE_LENGTH]; + size_t dst = 0; + char* ssh_argv[6]; + int ssh_argc; + int i; + int outfd[2]; + int infd[2]; + pid_t pid; + + ssh_argc = 0; + ssh_argv[ssh_argc++] = (char*)"ssh"; + if (ssh_port != 0) { + ssh_argv[ssh_argc++] = (char*)"-p"; + ssh_argv[ssh_argc++] = ssh_port; + } + ssh_argv[ssh_argc++] = ssh_host; + ssh_argv[ssh_argc++] = cmd+1; + ssh_argv[ssh_argc] = NULL; + + for (i = 0; i < argc; i++) { + dst = append_option(cmd, sizeof(cmd), dst, argv[i]); + } + dst = append_option(cmd, sizeof(cmd), dst, "--agent"); + cmd[dst] = '\0'; + + SYS_CHECK(pipe(infd)); + SYS_CHECK(pipe(outfd)); + + SYS_CHECK(pid = fork()); + + if (pid == 0) { /* child */ + SYS_CHECK(close(STDIN_FILENO)); + SYS_CHECK(close(STDOUT_FILENO)); + + SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); + SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + + SYS_CHECK(close(infd[0])); + SYS_CHECK(close(infd[1])); + SYS_CHECK(close(outfd[0])); + SYS_CHECK(close(outfd[1])); + + SYS_CHECK(execvp(ssh_argv[0], ssh_argv)); + } else { + SYS_CHECK(close(outfd[0])); /* These are being used by the child */ + SYS_CHECK(close(infd[1])); + + if (listen) { + int status; + fio_communicate(infd[0], outfd[1]); + SYS_CHECK(wait(&status)); + exit(status); + } else { + fio_redirect(infd[0], outfd[1]); /* write to stdout */ + } + } +} + From be4c4be4c2083d74bfb33e737143e460bd2e74a1 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 4 Nov 2018 10:02:26 +0300 Subject: [PATCH 0025/2107] Fix rermote backup protocol --- .gitignore | 2 - Makefile | 17 +-- src/backup.c | 17 ++- src/catalog.c | 92 ++++++------ src/data.c | 68 ++++----- src/dir.c | 10 +- src/fetch.c | 9 +- src/parsexlog.c | 10 +- src/utils/file.c | 382 ++++++++++++++++++++++++++++++++++++---------- src/utils/file.h | 52 ++++--- src/utils/pgut.c | 17 --- src/utils/pgut.h | 5 - src/walmethods.c | 384 +++++++++++++++++++++++++++++++++++++++++++++++ src/walmethods.h | 92 ++++++++++++ 14 files changed, 914 insertions(+), 243 deletions(-) create mode 100644 src/walmethods.c create mode 100644 src/walmethods.h diff --git a/.gitignore b/.gitignore index 4fc21d916..da388019c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,5 +42,3 @@ /src/streamutil.c /src/streamutil.h /src/xlogreader.c -/src/walmethods.c -/src/walmethods.h diff --git a/Makefile b/Makefile index f3e14b5da..75f58f8fd 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,13 @@ OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \ # borrowed files OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ - src/xlogreader.o + src/xlogreader.o src/walmethods.o EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c -INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h +INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h src/walmethods.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -39,12 +39,6 @@ else srchome=$(top_srcdir) endif -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -OBJS += src/walmethods.o -EXTRA_CLEAN += src/walmethods.c src/walmethods.h -INCLUDES += src/walmethods.h -endif - PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} @@ -73,13 +67,6 @@ src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ -src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ -endif - ifeq ($(PORTNAME), aix) CC=xlc_r endif diff --git a/src/backup.c b/src/backup.c index 5abdc12c8..9ef93fb9f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1822,6 +1822,7 @@ pg_stop_backup(pgBackup *backup) PQerrorMessage(conn), stop_backup_query); } elog(INFO, "pg_stop backup() successfully executed"); + sleep(20); } backup_in_progress = false; @@ -1848,15 +1849,15 @@ pg_stop_backup(pgBackup *backup) /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fio_open(backup_label, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(backup_label, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "can't open backup label file \"%s\": %s", backup_label, strerror(errno)); len = strlen(PQgetvalue(res, 0, 3)); - if (fio_write(fp, PQgetvalue(res, 0, 3), len) != len || - fio_flush(fp) != 0 || - fio_close(fp)) + if (fio_fwrite(fp, PQgetvalue(res, 0, 3), len) != len || + fio_fflush(fp) != 0 || + fio_fclose(fp)) elog(ERROR, "can't write backup label file \"%s\": %s", backup_label, strerror(errno)); @@ -1895,15 +1896,15 @@ pg_stop_backup(pgBackup *backup) char tablespace_map[MAXPGPATH]; join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fio_open(tablespace_map, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(tablespace_map, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "can't open tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); len = strlen(val); - if (fio_write(fp, val, len) != len || - fio_flush(fp) != 0 || - fio_close(fp)) + if (fio_fwrite(fp, val, len) != len || + fio_fflush(fp) != 0 || + fio_fclose(fp)) elog(ERROR, "can't write tablespace map file \"%s\": %s", tablespace_map, strerror(errno)); diff --git a/src/catalog.c b/src/catalog.c index fae199c62..bf373bac6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -26,8 +26,8 @@ static void unlink_lock_atexit(void) { int res; - res = unlink(lock_file); - if (res != 0 && res != ENOENT) + res = fio_unlink(lock_file, FIO_BACKUP_HOST); + if (res != 0 && errno != ENOENT) elog(WARNING, "%s: %s", lock_file, strerror(errno)); } @@ -90,7 +90,7 @@ catalog_lock(void) * Think not to make the file protection weaker than 0600. See * comments below. */ - fd = open(lock_file, O_RDWR | O_CREAT | O_EXCL, 0600); + fd = fio_open(lock_file, O_RDWR | O_CREAT | O_EXCL, FIO_BACKUP_HOST); if (fd >= 0) break; /* Success; exit the retry loop */ @@ -105,7 +105,7 @@ catalog_lock(void) * Read the file to get the old owner's PID. Note race condition * here: file might have been deleted since we tried to create it. */ - fd = open(lock_file, O_RDONLY, 0600); + fd = fio_open(lock_file, O_RDONLY, FIO_BACKUP_HOST); if (fd < 0) { if (errno == ENOENT) @@ -113,10 +113,10 @@ catalog_lock(void) elog(ERROR, "could not open lock file \"%s\": %s", lock_file, strerror(errno)); } - if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) + if ((len = fio_read(fd, buffer, sizeof(buffer) - 1)) < 0) elog(ERROR, "could not read lock file \"%s\": %s", lock_file, strerror(errno)); - close(fd); + fio_close(fd); if (len == 0) elog(ERROR, "lock file \"%s\" is empty", lock_file); @@ -149,7 +149,7 @@ catalog_lock(void) * it. Need a loop because of possible race condition against other * would-be creators. */ - if (unlink(lock_file) < 0) + if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) elog(ERROR, "could not remove old lock file \"%s\": %s", lock_file, strerror(errno)); } @@ -160,32 +160,32 @@ catalog_lock(void) snprintf(buffer, sizeof(buffer), "%d\n", my_pid); errno = 0; - if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) + if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) { int save_errno = errno; - close(fd); - unlink(lock_file); + fio_close(fd); + fio_unlink(lock_file, FIO_BACKUP_HOST); /* if write didn't set errno, assume problem is no disk space */ errno = save_errno ? save_errno : ENOSPC; elog(ERROR, "could not write lock file \"%s\": %s", lock_file, strerror(errno)); } - if (fsync(fd) != 0) + if (fio_flush(fd) != 0) { int save_errno = errno; - close(fd); - unlink(lock_file); + fio_close(fd); + fio_unlink(lock_file, FIO_BACKUP_HOST); errno = save_errno; elog(ERROR, "could not write lock file \"%s\": %s", lock_file, strerror(errno)); } - if (close(fd) != 0) + if (fio_close(fd) != 0) { int save_errno = errno; - unlink(lock_file); + fio_unlink(lock_file, FIO_BACKUP_HOST); errno = save_errno; elog(ERROR, "could not write lock file \"%s\": %s", lock_file, strerror(errno)); @@ -434,46 +434,46 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) { char timestamp[100]; - fio_printf(out, "#Configuration\n"); - fio_printf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); - fio_printf(out, "stream = %s\n", backup->stream ? "true" : "false"); - fio_printf(out, "compress-alg = %s\n", + fio_fprintf(out, "#Configuration\n"); + fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fio_fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); + fio_fprintf(out, "compress-alg = %s\n", deparse_compress_alg(backup->compress_alg)); - fio_printf(out, "compress-level = %d\n", backup->compress_level); - fio_printf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); + fio_fprintf(out, "compress-level = %d\n", backup->compress_level); + fio_fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); - fio_printf(out, "\n#Compatibility\n"); - fio_printf(out, "block-size = %u\n", backup->block_size); - fio_printf(out, "xlog-block-size = %u\n", backup->wal_block_size); - fio_printf(out, "checksum-version = %u\n", backup->checksum_version); + fio_fprintf(out, "\n#Compatibility\n"); + fio_fprintf(out, "block-size = %u\n", backup->block_size); + fio_fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); + fio_fprintf(out, "checksum-version = %u\n", backup->checksum_version); if (backup->program_version[0] != '\0') - fio_printf(out, "program-version = %s\n", backup->program_version); + fio_fprintf(out, "program-version = %s\n", backup->program_version); if (backup->server_version[0] != '\0') - fio_printf(out, "server-version = %s\n", backup->server_version); + fio_fprintf(out, "server-version = %s\n", backup->server_version); - fio_printf(out, "\n#Result backup info\n"); - fio_printf(out, "timelineid = %d\n", backup->tli); + fio_fprintf(out, "\n#Result backup info\n"); + fio_fprintf(out, "timelineid = %d\n", backup->tli); /* LSN returned by pg_start_backup */ - fio_printf(out, "start-lsn = %X/%X\n", + fio_fprintf(out, "start-lsn = %X/%X\n", (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); /* LSN returned by pg_stop_backup */ - fio_printf(out, "stop-lsn = %X/%X\n", + fio_fprintf(out, "stop-lsn = %X/%X\n", (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); time2iso(timestamp, lengthof(timestamp), backup->start_time); - fio_printf(out, "start-time = '%s'\n", timestamp); + fio_fprintf(out, "start-time = '%s'\n", timestamp); if (backup->end_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->end_time); - fio_printf(out, "end-time = '%s'\n", timestamp); + fio_fprintf(out, "end-time = '%s'\n", timestamp); } - fio_printf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); + fio_fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - fio_printf(out, "recovery-time = '%s'\n", timestamp); + fio_fprintf(out, "recovery-time = '%s'\n", timestamp); } /* @@ -481,20 +481,20 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) * WAL segments in archive 'wal' directory. */ if (backup->data_bytes != BYTES_INVALID) - fio_printf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); + fio_fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); if (backup->wal_bytes != BYTES_INVALID) - fio_printf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + fio_fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); - fio_printf(out, "status = %s\n", status2str(backup->status)); + fio_fprintf(out, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ if (backup->parent_backup != 0) - fio_printf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); + fio_fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); /* print connection info except password */ if (backup->primary_conninfo) - fio_printf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); + fio_fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); } /* @@ -507,14 +507,14 @@ write_backup(pgBackup *backup) char conf_path[MAXPGPATH]; pgBackupGetPath(backup, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); - fp = fio_open(conf_path, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(conf_path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, strerror(errno)); pgBackupWriteControl(fp, backup); - fio_close(fp); + fio_fclose(fp); } /* @@ -528,15 +528,15 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - fp = fio_open(path, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "cannot open file list \"%s\": %s", path, strerror(errno)); print_file_list(fp, files, root); - if (fio_flush(fp) != 0 || - fio_close(fp)) + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); } @@ -587,7 +587,7 @@ readBackupControlFile(const char *path) }; pgBackupInit(backup); - if (access(path, F_OK) != 0) + if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", path); pgBackupFree(backup); diff --git a/src/data.c b/src/data.c index 05644a88b..87db7047f 100644 --- a/src/data.c +++ b/src/data.c @@ -419,12 +419,12 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size); /* write data page */ - if (fio_write(out, write_buffer, write_buffer_size) != write_buffer_size) + if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) { int errno_tmp = errno; fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "File: %s, cannot write backup at block %u : %s", file->path, blknum, strerror(errno_tmp)); } @@ -512,7 +512,7 @@ backup_data_file(backup_files_arg* arguments, nblocks = file->size/BLCKSZ; /* open backup file for write */ - out = fio_open(to_path, PG_BINARY_W, FIO_BACKUP_HOST); + out = fio_fopen(to_path, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) { int errno_tmp = errno; @@ -575,13 +575,13 @@ backup_data_file(backup_files_arg* arguments, { int errno_tmp = errno; fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot change mode of \"%s\": %s", file->path, strerror(errno_tmp)); } - if (fio_flush(out) != 0 || - fio_close(out)) + if (fio_fflush(out) != 0 || + fio_fclose(out)) elog(ERROR, "cannot write backup file \"%s\": %s", to_path, strerror(errno)); fclose(in); @@ -638,7 +638,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ - out = fio_open(to_path, PG_BINARY_W "+", FIO_DB_HOST); + out = fio_fopen(to_path, PG_BINARY_W "+", FIO_DB_HOST); if (out == NULL) { int errno_tmp = errno; @@ -734,27 +734,27 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, /* * Seek and write the restored page. */ - if (fio_seek(out, write_pos) < 0) + if (fio_fseek(out, write_pos) < 0) elog(ERROR, "cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); if (write_header) { - if (fio_write(out, &header, sizeof(header)) != sizeof(header)) + if (fio_fwrite(out, &header, sizeof(header)) != sizeof(header)) elog(ERROR, "cannot write header of block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } if (header.compressed_size < BLCKSZ) { - if (fio_write(out, page.data, BLCKSZ) != BLCKSZ) + if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } else { /* if page wasn't compressed, we've read full block */ - if (fio_write(out, compressed_page.data, BLCKSZ) != BLCKSZ) + if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } @@ -772,7 +772,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, size_t file_size = 0; /* get file current size */ - fio_seek(out, 0); + fio_fseek(out, 0); file_size = ftell(out); if (file_size > file->n_blocks * BLCKSZ) @@ -792,7 +792,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, /* * Truncate file to this length. */ - if (fio_truncate(out, write_pos) != 0) + if (fio_ftruncate(out, write_pos) != 0) elog(ERROR, "cannot truncate \"%s\": %s", file->path, strerror(errno)); elog(VERBOSE, "Delta truncate file %s to block %u", @@ -806,13 +806,13 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, if (in) fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); } - if (fio_flush(out) != 0 || - fio_close(out)) + if (fio_fflush(out) != 0 || + fio_fclose(out)) elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); if (in) fclose(in); @@ -858,7 +858,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fio_open(to_path, PG_BINARY_W, FIO_BACKUP_HOST); + out = fio_fopen(to_path, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) { int errno_tmp = errno; @@ -871,7 +871,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if (fstat(fileno(in), &st) == -1) { fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot stat \"%s\": %s", file->path, strerror(errno)); } @@ -884,12 +884,12 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) break; - if (fio_write(out, buf, read_len) != read_len) + if (fio_fwrite(out, buf, read_len) != read_len) { errno_tmp = errno; /* oops */ fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -903,7 +903,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) if (!feof(in)) { fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot read backup mode file \"%s\": %s", file->path, strerror(errno_tmp)); } @@ -911,12 +911,12 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* copy odd part. */ if (read_len > 0) { - if (fio_write(out, buf, read_len) != read_len) + if (fio_fwrite(out, buf, read_len) != read_len) { errno_tmp = errno; /* oops */ fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -936,13 +936,13 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) { errno_tmp = errno; fclose(in); - fio_close(out); + fio_fclose(out); elog(ERROR, "cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); } - if (fio_flush(out) != 0 || - fio_close(out)) + if (fio_fflush(out) != 0 || + fio_fclose(out)) elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); fclose(in); @@ -1059,7 +1059,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fio_open(to_path_temp, PG_BINARY_W, FIO_BACKUP_HOST); + out = fio_fopen(to_path_temp, PG_BINARY_W, FIO_BACKUP_HOST); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1097,7 +1097,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fio_write(out, buf, read_len) != read_len) + if (fio_fwrite(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1125,8 +1125,8 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fio_flush(out) != 0 || - fio_close(out)) + if (fio_fflush(out) != 0 || + fio_fclose(out)) { errno_temp = errno; unlink(to_path_temp); @@ -1217,7 +1217,7 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fio_open(to_path_temp, PG_BINARY_W, FIO_DB_HOST); + out = fio_fopen(to_path_temp, PG_BINARY_W, FIO_DB_HOST); if (out == NULL) elog(ERROR, "Cannot open destination WAL file \"%s\": %s", to_path_temp, strerror(errno)); @@ -1254,7 +1254,7 @@ get_wal_file(const char *from_path, const char *to_path) if (read_len > 0) { - if (fio_write(out, buf, read_len) != read_len) + if (fio_fwrite(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1278,8 +1278,8 @@ get_wal_file(const char *from_path, const char *to_path) } } - if (fio_flush(out) != 0 || - fio_close(out)) + if (fio_fflush(out) != 0 || + fio_fclose(out)) { errno_temp = errno; unlink(to_path_temp); diff --git a/src/dir.c b/src/dir.c index 9509a2f50..ac4d096ee 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1206,7 +1206,7 @@ print_file_list(FILE *out, const parray *files, const char *root) if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - fio_printf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + fio_fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\"", @@ -1215,19 +1215,19 @@ print_file_list(FILE *out, const parray *files, const char *root) deparse_compress_alg(file->compress_alg)); if (file->is_datafile) - fio_printf(out, ",\"segno\":\"%d\"", file->segno); + fio_fprintf(out, ",\"segno\":\"%d\"", file->segno); #ifndef WIN32 if (S_ISLNK(file->mode)) #else if (pgwin32_is_junction(file->path)) #endif - fio_printf(out, ",\"linked\":\"%s\"", file->linked); + fio_fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) - fio_printf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + fio_fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); - fio_printf(out, "}\n"); + fio_fprintf(out, "}\n"); } } diff --git a/src/fetch.c b/src/fetch.c index 17e77025a..56194611d 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -34,7 +34,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) int len; snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); - if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) + if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, FIO_DB_HOST)) == -1) { if (safe) return NULL; @@ -43,7 +43,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) fullpath, strerror(errno)); } - if (fstat(fd, &statbuf) < 0) + if (fio_stat(fd, &statbuf) < 0) { if (safe) return NULL; @@ -53,10 +53,9 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) } len = statbuf.st_size; - buffer = pg_malloc(len + 1); - if (read(fd, buffer, len) != len) + if (fio_read(fd, buffer, len) != len) { if (safe) return NULL; @@ -65,7 +64,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) fullpath, strerror(errno)); } - close(fd); + fio_close(fd); /* Zero-terminate the buffer. */ buffer[len] = '\0'; diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f7365f5c..29b98d866 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -860,8 +860,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private_data->xlogpath); private_data->xlogexists = true; - private_data->xlogfile = open(private_data->xlogpath, - O_RDONLY | PG_BINARY, 0); + private_data->xlogfile = fio_open(private_data->xlogpath, + O_RDONLY | PG_BINARY, FIO_DB_HOST); if (private_data->xlogfile < 0) { @@ -910,14 +910,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* Read the requested page */ if (private_data->xlogfile != -1) { - if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + if (fio_seek(private_data->xlogfile, (off_t) targetPageOff) < 0) { elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", private_data->thread_num, private_data->xlogpath, strerror(errno)); return -1; } - if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (fio_read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", private_data->thread_num, private_data->xlogpath, strerror(errno)); @@ -993,7 +993,7 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) private_data = (XLogPageReadPrivate *) xlogreader->private_data; if (private_data->xlogfile >= 0) { - close(private_data->xlogfile); + fio_close(private_data->xlogfile); private_data->xlogfile = -1; } #ifdef HAVE_LIBZ diff --git a/src/utils/file.c b/src/utils/file.c index 7febd8261..b562fc13d 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -6,32 +6,41 @@ #include "pg_probackup.h" #include "file.h" -#define PRINTF_BUF_SIZE 1024 +#define PRINTF_BUF_SIZE 1024 +#define FILE_PERMISSIONS 0600 -static pthread_mutex_t fio_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t fio_read_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t fio_write_mutex = PTHREAD_MUTEX_INITIALIZER; static unsigned long fio_fdset = 0; static void* fio_stdin_buffer; static int fio_stdout = 0; static int fio_stdin = 0; +#define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) + void fio_redirect(int in, int out) { fio_stdin = in; fio_stdout = out; } -static bool fio_is_remote_file(FILE* fd) +static bool fio_is_remote_file(FILE* file) +{ + return (size_t)file <= FIO_FDMAX; +} + +static bool fio_is_remote_fd(int fd) { - return (size_t)fd <= FIO_FDMAX; + return (fd & FIO_PIPE_MARKER) != 0; } static bool fio_is_remote(fio_location location) { return (location == FIO_BACKUP_HOST && is_remote_agent) - || (location == FIO_DB_HOST && ssh_host != NULL); + || (location == FIO_DB_HOST && !is_remote_agent && ssh_host != NULL); } -static ssize_t fio_read(int fd, void* buf, size_t size) +static ssize_t fio_read_all(int fd, void* buf, size_t size) { size_t offs = 0; while (offs < size) @@ -42,10 +51,29 @@ static ssize_t fio_read(int fd, void* buf, size_t size) continue; } return rc; + } else if (rc == 0) { + break; } offs += rc; } - return size; + return offs; +} + +static ssize_t fio_write_all(int fd, void const* buf, size_t size) +{ + size_t offs = 0; + while (offs < size) + { + ssize_t rc = write(fd, (char*)buf + offs, size - offs); + if (rc <= 0) { + if (errno == EINTR) { + continue; + } + return rc; + } + offs += rc; + } + return offs; } FILE* fio_open_stream(char const* path, fio_location location) @@ -54,18 +82,19 @@ FILE* fio_open_stream(char const* path, fio_location location) if (fio_is_remote(location)) { fio_header hdr; - hdr.cop = FIO_READ; + hdr.cop = FIO_LOAD; hdr.size = strlen(path) + 1; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, path, hdr.size), hdr.size); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - IO_CHECK(fio_read(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SEND); if (hdr.size > 0) { Assert(fio_stdin_buffer == NULL); fio_stdin_buffer = malloc(hdr.size); - IO_CHECK(fio_read(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); + IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); f = fmemopen(fio_stdin_buffer, hdr.size, "r"); } else @@ -90,32 +119,50 @@ int fio_close_stream(FILE* f) return fclose(f); } -FILE* fio_open(char const* path, char const* mode, fio_location location) +int fio_open(char const* path, int mode, fio_location location) { - FILE* f; + int fd; if (fio_is_remote(location)) { int i; fio_header hdr; unsigned long mask; - SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) { - return NULL; + return -1; } - hdr.cop = strchr(mode,'+') ? FIO_OPEN_EXISTED : FIO_OPEN_NEW; + hdr.cop = FIO_OPEN; hdr.handle = i; hdr.size = strlen(path) + 1; + hdr.arg = mode; fio_fdset |= 1 << i; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, path, hdr.size), hdr.size); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - f = (FILE*)(size_t)(i + 1); + fd = i | FIO_PIPE_MARKER; + } + else + { + fd = open(path, mode, FILE_PERMISSIONS); + } + return fd; +} + +FILE* fio_fopen(char const* path, char const* mode, fio_location location) +{ + FILE* f; + if (fio_is_remote(location)) + { + int flags = O_RDWR|O_CREAT|PG_BINARY|(strchr(mode, '+') ? 0 : O_TRUNC); + int fd = fio_open(path, flags, location); + f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); } else { @@ -124,12 +171,12 @@ FILE* fio_open(char const* path, char const* mode, fio_location location) return f; } -int fio_printf(FILE* f, char const* format, ...) +int fio_fprintf(FILE* f, char const* format, ...) { int rc; va_list args; va_start (args, format); - if (fio_stdout) + if (fio_is_remote_file(f)) { char buf[PRINTF_BUF_SIZE]; #ifdef HAS_VSNPRINTF @@ -138,7 +185,7 @@ int fio_printf(FILE* f, char const* format, ...) rc = vsprintf(buf, format, args); #endif if (rc > 0) { - fio_write(f, buf, rc); + fio_fwrite(f, buf, rc); } } else @@ -149,7 +196,7 @@ int fio_printf(FILE* f, char const* format, ...) return rc; } -int fio_flush(FILE* f) +int fio_fflush(FILE* f) { int rc = 0; if (!fio_is_remote_file(f)) @@ -162,99 +209,234 @@ int fio_flush(FILE* f) return rc; } -int fio_close(FILE* f) +int fio_flush(int fd) { - if (fio_is_remote_file(f)) + return fio_is_remote_fd(fd) ? 0 : fsync(fd); +} + +int fio_fclose(FILE* f) +{ + return fio_is_remote_file(f) + ? fio_close(fio_fileno(f)) + : fclose(f); +} + +int fio_close(int fd) +{ + if (fio_is_remote_fd(fd)) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); hdr.cop = FIO_CLOSE; - hdr.handle = (size_t)f - 1; + hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; fio_fdset &= ~(1 << hdr.handle); - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else { - return fclose(f); + return close(fd); } } -int fio_truncate(FILE* f, off_t size) +int fio_ftruncate(FILE* f, off_t size) { - if (fio_is_remote_file(f)) + return fio_is_remote_file(f) + ? fio_truncate(fio_fileno(f), size) + : ftruncate(fileno(f), size); +} + +int fio_truncate(int fd, off_t size) +{ + if (fio_is_remote_fd(fd)) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); hdr.cop = FIO_TRUNCATE; - hdr.handle = (size_t)f - 1; + hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; hdr.arg = size; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else { - return ftruncate(fileno(f), size); + return ftruncate(fd, size); } } -int fio_seek(FILE* f, off_t offs) +int fio_fseek(FILE* f, off_t offs) { - if (fio_is_remote_file(f)) + return fio_is_remote_file(f) + ? fio_seek(fio_fileno(f), offs) + : fseek(f, offs, SEEK_SET); +} + +int fio_seek(int fd, off_t offs) +{ + if (fio_is_remote_fd(fd)) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); hdr.cop = FIO_SEEK; - hdr.handle = (size_t)f - 1; + hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; hdr.arg = offs; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else { - return fseek(f, offs, SEEK_SET); + return lseek(fd, offs, SEEK_SET); } } -size_t fio_write(FILE* f, void const* buf, size_t size) +size_t fio_fwrite(FILE* f, void const* buf, size_t size) { - if (fio_is_remote_file(f)) + return fio_is_remote_file(f) + ? fio_write(fio_fileno(f), buf, size) + : fwrite(buf, 1, size, f); +} + +ssize_t fio_write(int fd, void const* buf, size_t size) +{ + if (fio_is_remote_fd(fd)) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); hdr.cop = FIO_WRITE; - hdr.handle = (size_t)f - 1; + hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = size; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, buf, size), size); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return size; } else { - return fwrite(buf, 1, size, f); + return write(fd, buf, size); + } +} + +size_t fio_fread(FILE* f, void* buf, size_t size) +{ + return fio_is_remote_file(f) + ? fio_read(fio_fileno(f), buf, size) + : fread(buf, 1, size, f); +} + +ssize_t fio_read(int fd, void* buf, size_t size) +{ + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_READ; + hdr.handle = fd & ~FIO_PIPE_MARKER; + hdr.size = 0; + hdr.arg = size; + + SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SEND); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); + + return hdr.size; + } + else + { + return read(fd, buf, size); + } +} + +int fio_stat(int fd, struct stat* st) +{ + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_STAT; + hdr.handle = fd & ~FIO_PIPE_MARKER; + hdr.size = 0; + + SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_STAT); + IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); + + SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); + + return hdr.arg; + } + else + { + return fstat(fd, st); + } +} + +int fio_access(char const* path, int mode, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_ACCESS; + hdr.handle = -1; + hdr.size = path_len; + hdr.arg = mode; + + SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_ACCESS); + + SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); + + return hdr.arg; + } + else + { + return access(path, mode); } } @@ -269,11 +451,13 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location hdr.handle = -1; hdr.size = old_path_len + new_path_len; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, old_path, old_path_len), old_path_len); - IO_CHECK(write(fio_stdout, new_path, new_path_len), new_path_len); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); + IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -292,10 +476,12 @@ int fio_unlink(char const* path, fio_location location) hdr.handle = -1; hdr.size = path_len; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, path, path_len), path_len); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -315,10 +501,12 @@ int fio_mkdir(char const* path, int mode, fio_location location) hdr.size = path_len; hdr.arg = mode; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, path, path_len), path_len); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -338,10 +526,12 @@ int fio_chmod(char const* path, int mode, fio_location location) hdr.size = path_len; hdr.arg = mode; - IO_CHECK(write(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(write(fio_stdout, path, path_len), path_len); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - SYS_CHECK(pthread_mutex_unlock(&fio_mutex)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -356,22 +546,22 @@ static void fio_send_file(int out, char const* path) fio_header hdr; void* buf = NULL; + hdr.cop = FIO_SEND; hdr.size = 0; - hdr.cop = FIO_READ; if (fd >= 0) { off_t size = lseek(fd, 0, SEEK_END); buf = malloc(size); lseek(fd, 0, SEEK_SET); - IO_CHECK(fio_read(fd, buf, size), size); + IO_CHECK(fio_read_all(fd, buf, size), size); hdr.size = size; SYS_CHECK(close(fd)); } - IO_CHECK(write(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (buf) { - IO_CHECK(write(out, buf, hdr.size), hdr.size); + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); free(buf); } } @@ -379,30 +569,56 @@ static void fio_send_file(int out, char const* path) void fio_communicate(int in, int out) { int fd[FIO_FDMAX]; - char buf[BLCKSZ*2]; /* need more space for page header */ + size_t buf_size = 128*1024; + char* buf = (char*)malloc(buf_size); fio_header hdr; int rc; - while ((rc = read(in, &hdr, sizeof hdr)) == sizeof(hdr)) { + while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { if (hdr.size != 0) { - Assert(hdr.size < sizeof(buf)); - IO_CHECK(fio_read(in, buf, hdr.size), hdr.size); + if (hdr.size > buf_size) { + buf_size = hdr.size; + buf = (char*)realloc(buf, buf_size); + } + IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); } switch (hdr.cop) { - case FIO_READ: + case FIO_LOAD: fio_send_file(out, buf); break; - case FIO_OPEN_NEW: - SYS_CHECK(fd[hdr.handle] = open(buf, O_RDWR|O_CREAT|O_TRUNC, 0777)); - break; - case FIO_OPEN_EXISTED: - SYS_CHECK(fd[hdr.handle] = open(buf, O_RDWR|O_CREAT, 0777)); + case FIO_OPEN: + SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); break; case FIO_CLOSE: SYS_CHECK(close(fd[hdr.handle])); break; case FIO_WRITE: - IO_CHECK(write(fd[hdr.handle], buf, hdr.size), hdr.size); + IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); + break; + case FIO_READ: + if ((size_t)hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); + } + rc = read(fd[hdr.handle], buf, hdr.arg); + hdr.cop = FIO_SEND; + hdr.size = rc > 0 ? rc : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, rc), rc); + break; + case FIO_STAT: + { + struct stat st; + hdr.size = sizeof(st); + hdr.arg = fstat(fd[hdr.handle], &st); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); + break; + } + case FIO_ACCESS: + hdr.size = 0; + hdr.arg = access(buf, hdr.arg); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_RENAME: SYS_CHECK(rename(buf, buf + strlen(buf))); @@ -426,7 +642,7 @@ void fio_communicate(int in, int out) Assert(false); } } - + free(buf); if (rc != 0) { perror("read"); exit(EXIT_FAILURE); diff --git a/src/utils/file.h b/src/utils/file.h index c5ec6118d..3b7cc1121 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -5,8 +5,7 @@ typedef enum { - FIO_OPEN_NEW, - FIO_OPEN_EXISTED, + FIO_OPEN, FIO_CLOSE, FIO_WRITE, FIO_RENAME, @@ -15,7 +14,11 @@ typedef enum FIO_CHMOD, FIO_SEEK, FIO_TRUNCATE, - FIO_READ + FIO_READ, + FIO_LOAD, + FIO_STAT, + FIO_SEND, + FIO_ACCESS } fio_operations; typedef enum @@ -25,9 +28,10 @@ typedef enum } fio_location; #define FIO_FDMAX 64 +#define FIO_PIPE_MARKER 0x40000000 #define SYS_CHECK(cmd) do if ((cmd) < 0) { perror(#cmd); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "Receive %d bytes instead of %d\n", _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d\n", __FILE__, __LINE__, _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) typedef struct { unsigned cop : 4; @@ -36,20 +40,32 @@ typedef struct { unsigned arg; } fio_header; -extern void fio_redirect(int in, int out); -extern void fio_communicate(int in, int out); - -extern FILE* fio_open(char const* name, char const* mode, fio_location location); -extern int fio_close(FILE* f); -extern size_t fio_write(FILE* f, void const* buf, size_t size); -extern int fio_printf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); -extern int fio_flush(FILE* f); -extern int fio_seek(FILE* f, off_t offs); -extern int fio_truncate(FILE* f, off_t size); -extern int fio_rename(char const* old_path, char const* new_path, fio_location location); -extern int fio_unlink(char const* path, fio_location location); -extern int fio_mkdir(char const* path, int mode, fio_location location); -extern int fio_chmod(char const* path, int mode, fio_location location); +extern void fio_redirect(int in, int out); +extern void fio_communicate(int in, int out); + +extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); +extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); +extern size_t fio_fread(FILE* f, void* buf, size_t size); +extern int fio_fprintf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); +extern int fio_fflush(FILE* f); +extern int fio_fseek(FILE* f, off_t offs); +extern int fio_ftruncate(FILE* f, off_t size); +extern int fio_fclose(FILE* f); + +extern int fio_open(char const* name, int mode, fio_location location); +extern ssize_t fio_write(int fd, void const* buf, size_t size); +extern ssize_t fio_read(int fd, void* buf, size_t size); +extern int fio_flush(int fd); +extern int fio_seek(int fd, off_t offs); +extern int fio_stat(int fd, struct stat* st); +extern int fio_truncate(int fd, off_t size); +extern int fio_close(int fd); + +extern int fio_rename(char const* old_path, char const* new_path, fio_location location); +extern int fio_unlink(char const* path, fio_location location); +extern int fio_mkdir(char const* path, int mode, fio_location location); +extern int fio_chmod(char const* path, int mode, fio_location location); +extern int fio_access(char const* path, int mode, fio_location location); extern FILE* fio_open_stream(char const* name, fio_location location); extern int fio_close_stream(FILE* f); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ba4b077e6..0c58ae26f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -2164,23 +2164,6 @@ pgut_strdup(const char *str) return ret; } -FILE * -pgut_fopen(const char *path, const char *mode, bool missing_ok) -{ - FILE *fp; - - if ((fp = fopen(path, mode)) == NULL) - { - if (missing_ok && errno == ENOENT) - return NULL; - - elog(ERROR, "could not open file \"%s\": %s", - path, strerror(errno)); - } - - return fp; -} - #ifdef WIN32 static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); #define select select_win32 diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 9aac75cac..8150a27b4 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -132,11 +132,6 @@ extern char *pgut_strdup(const char *str); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) -/* - * file operations - */ -extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); - /* * Assert */ diff --git a/src/walmethods.c b/src/walmethods.c new file mode 100644 index 000000000..d384b99f6 --- /dev/null +++ b/src/walmethods.c @@ -0,0 +1,384 @@ +/*------------------------------------------------------------------------- + * + * walmethods.c - implementations of different ways to write received wal + * + * NOTE! The caller must ensure that only one method is instantiated in + * any given program, and that it's only instantiated once! + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.c + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "pgtar.h" +#include "common/file_perm.h" +#include "common/file_utils.h" + +#include "receivelog.h" +#include "streamutil.h" +#include "pg_probackup.h" + +/* Size of zlib buffer for .tar.gz */ +#define ZLIB_OUT_SIZE 4096 + +/*------------------------------------------------------------------------- + * WalDirectoryMethod - write wal to a directory looking like pg_wal + *------------------------------------------------------------------------- + */ + +/* + * Global static data for this method + */ +typedef struct DirectoryMethodData +{ + char *basedir; + int compression; + bool sync; +} DirectoryMethodData; +static DirectoryMethodData *dir_data = NULL; + +/* + * Local file handle + */ +typedef struct DirectoryMethodFile +{ + int fd; + off_t currpos; + char *pathname; + char *fullpath; + char *temp_suffix; +#ifdef HAVE_LIBZ + gzFile gzfp; +#endif +} DirectoryMethodFile; + +static const char * +dir_getlasterror(void) +{ + /* Directory method always sets errno, so just use strerror */ + return strerror(errno); +} + +static Walfile +dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) +{ + static char tmppath[MAXPGPATH]; + int fd; + DirectoryMethodFile *f; +#ifdef HAVE_LIBZ + gzFile gzfp = NULL; +#endif + + snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", + dir_data->basedir, pathname, + dir_data->compression > 0 ? ".gz" : "", + temp_suffix ? temp_suffix : ""); + + /* + * Open a file for non-compressed as well as compressed files. Tracking + * the file descriptor is important for dir_sync() method as gzflush() + * does not do any system calls to fsync() to make changes permanent on + * disk. + */ + fd = fio_open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, FIO_BACKUP_HOST); + if (fd < 0) + return NULL; + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + gzfp = gzdopen(fd, "wb"); + if (gzfp == NULL) + { + close(fd); + return NULL; + } + + if (gzsetparams(gzfp, dir_data->compression, + Z_DEFAULT_STRATEGY) != Z_OK) + { + gzclose(gzfp); + return NULL; + } + } +#endif + + /* Do pre-padding on non-compressed files */ + if (pad_to_size && dir_data->compression == 0) + { + PGAlignedXLogBlock zerobuf; + int bytes; + + memset(zerobuf.data, 0, XLOG_BLCKSZ); + for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ) + { + errno = 0; + if (fio_write(fd, zerobuf.data, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + int save_errno = errno; + + fio_close(fd); + + /* + * If write didn't set errno, assume problem is no disk space. + */ + errno = save_errno ? save_errno : ENOSPC; + return NULL; + } + } + + if (fio_seek(fd, 0) != 0) + { + int save_errno = errno; + + fio_close(fd); + errno = save_errno; + return NULL; + } + } + + /* + * fsync WAL file and containing directory, to ensure the file is + * persistently created and zeroed (if padded). That's particularly + * important when using synchronous mode, where the file is modified and + * fsynced in-place, without a directory fsync. + */ + if (!is_remote_agent && dir_data->sync) + { + if (fsync_fname(tmppath, false, progname) != 0 || + fsync_parent_path(tmppath, progname) != 0) + { +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + gzclose(gzfp); + else +#endif + close(fd); + return NULL; + } + } + + f = pg_malloc0(sizeof(DirectoryMethodFile)); +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + f->gzfp = gzfp; +#endif + f->fd = fd; + f->currpos = 0; + f->pathname = pg_strdup(pathname); + f->fullpath = pg_strdup(tmppath); + if (temp_suffix) + f->temp_suffix = pg_strdup(temp_suffix); + + return f; +} + +static ssize_t +dir_write(Walfile f, const void *buf, size_t count) +{ + ssize_t r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + + Assert(f != NULL); + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + r = (ssize_t) gzwrite(df->gzfp, buf, count); + else +#endif + r = fio_write(df->fd, buf, count); + if (r > 0) + df->currpos += r; + return r; +} + +static off_t +dir_get_current_pos(Walfile f) +{ + Assert(f != NULL); + + /* Use a cached value to prevent lots of reseeks */ + return ((DirectoryMethodFile *) f)->currpos; +} + +static int +dir_close(Walfile f, WalCloseMethod method) +{ + int r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + static char tmppath[MAXPGPATH]; + static char tmppath2[MAXPGPATH]; + + Assert(f != NULL); + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + r = gzclose(df->gzfp); + else +#endif + r = fio_close(df->fd); + + if (r == 0) + { + /* Build path to the current version of the file */ + if (method == CLOSE_NORMAL && df->temp_suffix) + { + /* + * If we have a temp prefix, normal operation is to rename the + * file. + */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", + dir_data->basedir, df->pathname, + dir_data->compression > 0 ? ".gz" : "", + df->temp_suffix); + snprintf(tmppath2, sizeof(tmppath2), "%s/%s%s", + dir_data->basedir, df->pathname, + dir_data->compression > 0 ? ".gz" : ""); + r = durable_rename(tmppath, tmppath2, progname); + } + else if (method == CLOSE_UNLINK) + { + /* Unlink the file once it's closed */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", + dir_data->basedir, df->pathname, + dir_data->compression > 0 ? ".gz" : "", + df->temp_suffix ? df->temp_suffix : ""); + r = unlink(tmppath); + } + else + { + /* + * Else either CLOSE_NORMAL and no temp suffix, or + * CLOSE_NO_RENAME. In this case, fsync the file and containing + * directory if sync mode is requested. + */ + if (dir_data->sync && !is_remote_agent) + { + r = fsync_fname(df->fullpath, false, progname); + if (r == 0) + r = fsync_parent_path(df->fullpath, progname); + } + } + } + + pg_free(df->pathname); + pg_free(df->fullpath); + if (df->temp_suffix) + pg_free(df->temp_suffix); + pg_free(df); + + return r; +} + +static int +dir_sync(Walfile f) +{ + Assert(f != NULL); + + if (!dir_data->sync) + return 0; + +#ifdef HAVE_LIBZ + if (dir_data->compression > 0) + { + if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK) + return -1; + } +#endif + + return fio_flush(((DirectoryMethodFile *) f)->fd); +} + +static ssize_t +dir_get_file_size(const char *pathname) +{ + struct stat statbuf; + static char tmppath[MAXPGPATH]; + int fd; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + fd = fio_open(tmppath, O_RDONLY|PG_BINARY, FIO_BACKUP_HOST); + if (fd >= 0) + { + if (fio_stat(fd, &statbuf) != 0) + { + fio_close(fd); + return -1; + } + fio_close(fd); + return statbuf.st_size; + } + return -1; +} + +static bool +dir_existsfile(const char *pathname) +{ + static char tmppath[MAXPGPATH]; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + return fio_access(tmppath, F_OK, FIO_BACKUP_HOST) == 0; +} + +static bool +dir_finish(void) +{ + if (dir_data->sync && !is_remote_agent) + { + /* + * Files are fsynced when they are closed, but we need to fsync the + * directory entry here as well. + */ + if (fsync_fname(dir_data->basedir, true, progname) != 0) + return false; + } + return true; +} + + +WalWriteMethod * +CreateWalDirectoryMethod(const char *basedir, int compression, bool sync) +{ + WalWriteMethod *method; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = dir_open_for_write; + method->write = dir_write; + method->get_current_pos = dir_get_current_pos; + method->get_file_size = dir_get_file_size; + method->close = dir_close; + method->sync = dir_sync; + method->existsfile = dir_existsfile; + method->finish = dir_finish; + method->getlasterror = dir_getlasterror; + + dir_data = pg_malloc0(sizeof(DirectoryMethodData)); + dir_data->compression = compression; + dir_data->basedir = pg_strdup(basedir); + dir_data->sync = sync; + + return method; +} + +void +FreeWalDirectoryMethod(void) +{ + pg_free(dir_data->basedir); + pg_free(dir_data); +} + diff --git a/src/walmethods.h b/src/walmethods.h new file mode 100644 index 000000000..76eb249b9 --- /dev/null +++ b/src/walmethods.h @@ -0,0 +1,92 @@ +/*------------------------------------------------------------------------- + * + * walmethods.h + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.h + *------------------------------------------------------------------------- + */ + + +typedef void *Walfile; + +typedef enum +{ + CLOSE_NORMAL, + CLOSE_UNLINK, + CLOSE_NO_RENAME +} WalCloseMethod; + +/* + * A WalWriteMethod structure represents the different methods used + * to write the streaming WAL as it's received. + * + * All methods that have a failure return indicator will set state + * allowing the getlasterror() method to return a suitable message. + * Commonly, errno is this state (or part of it); so callers must take + * care not to clobber errno between a failed method call and use of + * getlasterror() to retrieve the message. + */ +typedef struct WalWriteMethod WalWriteMethod; +struct WalWriteMethod +{ + /* + * Open a target file. Returns Walfile, or NULL if open failed. If a temp + * suffix is specified, a file with that name will be opened, and then + * automatically renamed in close(). If pad_to_size is specified, the file + * will be padded with NUL up to that size, if supported by the Walmethod. + */ + Walfile (*open_for_write) (const char *pathname, const char *temp_suffix, size_t pad_to_size); + + /* + * Close an open Walfile, using one or more methods for handling automatic + * unlinking etc. Returns 0 on success, other values for error. + */ + int (*close) (Walfile f, WalCloseMethod method); + + /* Check if a file exist */ + bool (*existsfile) (const char *pathname); + + /* Return the size of a file, or -1 on failure. */ + ssize_t (*get_file_size) (const char *pathname); + + /* + * Write count number of bytes to the file, and return the number of bytes + * actually written or -1 for error. + */ + ssize_t (*write) (Walfile f, const void *buf, size_t count); + + /* Return the current position in a file or -1 on error */ + off_t (*get_current_pos) (Walfile f); + + /* + * fsync the contents of the specified file. Returns 0 on success. + */ + int (*sync) (Walfile f); + + /* + * Clean up the Walmethod, closing any shared resources. For methods like + * tar, this includes writing updated headers. Returns true if the + * close/write/sync of shared resources succeeded, otherwise returns false + * (but the resources are still closed). + */ + bool (*finish) (void); + + /* Return a text for the last error in this Walfile */ + const char *(*getlasterror) (void); +}; + +/* + * Available WAL methods: + * - WalDirectoryMethod - write WAL to regular files in a standard pg_wal + * - TarDirectoryMethod - write WAL to a tarfile corresponding to pg_wal + * (only implements the methods required for pg_basebackup, + * not all those required for pg_receivewal) + */ +WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, + int compression, bool sync); + +/* Cleanup routines for previously-created methods */ +void FreeWalDirectoryMethod(void); From be6a4e9bcbc83822e3c726507c4e3e50744e6474 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 4 Nov 2018 23:20:07 +0300 Subject: [PATCH 0026/2107] Remote backup works --- .gitignore | 2 +- src/backup.c | 19 ++++++---- src/data.c | 42 ++++++++-------------- src/dir.c | 14 ++++---- src/fetch.c | 2 +- src/merge.c | 2 +- src/parsexlog.c | 37 +++++++++---------- src/pg_probackup.c | 17 +++++++-- src/pg_probackup.h | 7 ++-- src/restore.c | 8 ++--- src/utils/file.c | 89 ++++++++++++++++++++++++++++++++++++++++------ src/utils/file.h | 24 ++++++++++--- src/utils/remote.c | 6 ++-- src/walmethods.c | 17 +++------ 14 files changed, 184 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index da388019c..1898a58de 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ /src/datapagemap.h /src/logging.h /src/receivelog.c -/src/receivelog.h +/src/receivelog.hщщ /src/streamutil.c /src/streamutil.h /src/xlogreader.c diff --git a/src/backup.c b/src/backup.c index 9ef93fb9f..9593e6ab6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -943,6 +943,16 @@ do_backup(time_t start_time) //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", // current.data_bytes); + if (is_remote_agent) + fio_transfer(¤t.start_time,current.start_time); + else + complete_backup(); + + return 0; +} + +void complete_backup(void) +{ pgBackupValidate(¤t); elog(INFO, "Backup %s completed", base36enc(current.start_time)); @@ -953,8 +963,6 @@ do_backup(time_t start_time) */ if (delete_expired || delete_wal) do_retention_purge(); - - return 0; } /* @@ -1529,13 +1537,13 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { if (!file_exists) { - file_exists = fileExists(wal_segment_path); + file_exists = fileExists(wal_segment_path, is_start_lsn ? FIO_DB_HOST : FIO_BACKUP_HOST); /* Try to find compressed WAL file */ if (!file_exists) { #ifdef HAVE_LIBZ - file_exists = fileExists(gz_wal_segment_path); + file_exists = fileExists(gz_wal_segment_path, is_start_lsn ? FIO_DB_HOST : FIO_BACKUP_HOST); if (file_exists) elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); #endif @@ -1822,7 +1830,6 @@ pg_stop_backup(pgBackup *backup) PQerrorMessage(conn), stop_backup_query); } elog(INFO, "pg_stop backup() successfully executed"); - sleep(20); } backup_in_progress = false; @@ -2155,7 +2162,7 @@ backup_files(void *arg) skip = true; /* ...skip copying file. */ } if (skip || - !copy_file(arguments->from_root, arguments->to_root, file)) + !copy_file(arguments->from_root, arguments->to_root, file, FIO_BACKUP_HOST)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", diff --git a/src/data.c b/src/data.c index 87db7047f..eb2230b68 100644 --- a/src/data.c +++ b/src/data.c @@ -769,13 +769,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, */ if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) { - size_t file_size = 0; - - /* get file current size */ - fio_fseek(out, 0); - file_size = ftell(out); - - if (file_size > file->n_blocks * BLCKSZ) + struct stat st; + if (fio_ffstat(out, &st) == 0 && st.st_size > file->n_blocks * BLCKSZ) { truncate_from = file->n_blocks; need_truncate = true; @@ -824,7 +819,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * it is either small control file or already compressed cfs file. */ bool -copy_file(const char *from_root, const char *to_root, pgFile *file) +copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location location) { char to_path[MAXPGPATH]; FILE *in; @@ -858,7 +853,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fio_fopen(to_path, PG_BINARY_W, FIO_BACKUP_HOST); + out = fio_fopen(to_path, PG_BINARY_W, location); if (out == NULL) { int errno_tmp = errno; @@ -932,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) file->crc = crc; /* update file permission */ - if (fio_chmod(to_path, st.st_mode, FIO_BACKUP_HOST) == -1) + if (fio_chmod(to_path, st.st_mode, location) == -1) { errno_tmp = errno; fclose(in); @@ -1039,7 +1034,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); - if (!overwrite && fileExists(gz_to_path)) + if (!overwrite && fileExists(gz_to_path, FIO_BACKUP_HOST)) elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); @@ -1054,7 +1049,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (!overwrite && fileExists(to_path)) + if (!overwrite && fileExists(to_path, FIO_BACKUP_HOST)) elog(ERROR, "WAL segment \"%s\" already exists.", to_path); snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); @@ -1335,9 +1330,8 @@ get_wal_file(const char *from_path, const char *to_path) bool calc_file_checksum(pgFile *file) { - FILE *in; + int in; size_t read_len = 0; - int errno_tmp; char buf[BLCKSZ]; struct stat st; pg_crc32 crc; @@ -1350,8 +1344,8 @@ calc_file_checksum(pgFile *file) file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); - if (in == NULL) + in = fio_open(file->path, O_RDONLY|PG_BINARY, FIO_BACKUP_HOST); + if (in < 0) { FIN_TRADITIONAL_CRC32(crc); file->crc = crc; @@ -1365,16 +1359,16 @@ calc_file_checksum(pgFile *file) } /* stat source file to change mode of destination file */ - if (fstat(fileno(in), &st) == -1) + if (fio_fstat(in, &st) == -1) { - fclose(in); + fio_close(in); elog(ERROR, "cannot stat \"%s\": %s", file->path, strerror(errno)); } for (;;) { - read_len = fread(buf, 1, sizeof(buf), in); + read_len = fio_read(in, buf, sizeof(buf)); if(read_len == 0) break; @@ -1386,19 +1380,11 @@ calc_file_checksum(pgFile *file) file->read_size += read_len; } - errno_tmp = errno; - if (!feof(in)) - { - fclose(in); - elog(ERROR, "cannot read backup mode file \"%s\": %s", - file->path, strerror(errno_tmp)); - } - /* finish CRC calculation and store into pgFile */ FIN_TRADITIONAL_CRC32(crc); file->crc = crc; - fclose(in); + fio_close(in); return true; } diff --git a/src/dir.c b/src/dir.c index ac4d096ee..bf5ec27f8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -162,8 +162,8 @@ pgFileNew(const char *path, bool omit_symlink) struct stat st; pgFile *file; - /* stat the file */ - if ((omit_symlink ? stat(path, &st) : lstat(path, &st)) == -1) + /* stat the file */ + if (fio_stat(path, &st, omit_symlink, FIO_BACKUP_HOST) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -403,7 +403,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ - if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path, FIO_LOCAL_HOST)) { FILE *black_list_file = NULL; char buf[MAXPGPATH * 2]; @@ -935,7 +935,7 @@ create_data_directories(const char *data_dir, const char *backup_dir, size_t i; char backup_database_dir[MAXPGPATH], to_path[MAXPGPATH]; - + sleep(30); dirs = parray_new(); if (extract_tablespaces) { @@ -1088,7 +1088,7 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path)) + if (!fileExists(map_path, FIO_LOCAL_HOST)) { elog(LOG, "there is no file tablespace_map"); return; @@ -1490,11 +1490,11 @@ dir_is_empty(const char *path) * Return true if the path is a existing regular file. */ bool -fileExists(const char *path) +fileExists(const char *path, fio_location location) { struct stat buf; - if (stat(path, &buf) == -1 && errno == ENOENT) + if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT) return false; else if (!S_ISREG(buf.st_mode)) return false; diff --git a/src/fetch.c b/src/fetch.c index 56194611d..72ab2cbc6 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -43,7 +43,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) fullpath, strerror(errno)); } - if (fio_stat(fd, &statbuf) < 0) + if (fio_fstat(fd, &statbuf) < 0) { if (safe) return NULL; diff --git a/src/merge.c b/src/merge.c index 2873eb5c4..8ea5481c1 100644 --- a/src/merge.c +++ b/src/merge.c @@ -448,7 +448,7 @@ merge_files(void *arg) /* * We need to decompress target file only if it exists. */ - if (fileExists(to_path_tmp)) + if (fileExists(to_path_tmp, FIO_LOCAL_HOST)) { /* * file->path points to the file in from_root directory. But we diff --git a/src/parsexlog.c b/src/parsexlog.c index 29b98d866..0eb4d1fb0 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -86,22 +86,22 @@ static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime typedef struct XLogPageReadPrivate { - int thread_num; - const char *archivedir; - TimeLineID tli; - uint32 xlog_seg_size; - - bool manual_switch; - bool need_switch; - - int xlogfile; - XLogSegNo xlogsegno; - char xlogpath[MAXPGPATH]; - bool xlogexists; - + int thread_num; + const char *archivedir; + TimeLineID tli; + uint32 xlog_seg_size; + + bool manual_switch; + bool need_switch; + + int xlogfile; + XLogSegNo xlogsegno; + char xlogpath[MAXPGPATH]; + bool xlogexists; + fio_location location; #ifdef HAVE_LIBZ - gzFile gz_xlogfile; - char gz_xlogpath[MAXPGPATH]; + gzFile gz_xlogfile; + char gz_xlogpath[MAXPGPATH]; #endif } XLogPageReadPrivate; @@ -853,7 +853,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", private_data->archivedir, xlogfname); - if (fileExists(private_data->xlogpath)) + if (fileExists(private_data->xlogpath, private_data->location)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", private_data->thread_num, @@ -861,7 +861,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private_data->xlogexists = true; private_data->xlogfile = fio_open(private_data->xlogpath, - O_RDONLY | PG_BINARY, FIO_DB_HOST); + O_RDONLY | PG_BINARY, private_data->location); if (private_data->xlogfile < 0) { @@ -879,7 +879,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, snprintf(private_data->gz_xlogpath, sizeof(private_data->gz_xlogpath), "%s.gz", private_data->xlogpath); - if (fileExists(private_data->gz_xlogpath)) + if (fileExists(private_data->gz_xlogpath, private_data->location)) { elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", private_data->thread_num, private_data->gz_xlogpath); @@ -965,6 +965,7 @@ InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, private_data->tli = tli; private_data->xlog_seg_size = xlog_seg_size; private_data->xlogfile = -1; + private_data->location = FIO_BACKUP_HOST; if (allocate_reader) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 124625c72..e0e1955b8 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -389,7 +389,11 @@ main(int argc, char *argv[]) fio_redirect(STDIN_FILENO, STDOUT_FILENO); } else { /* Execute remote probackup */ - remote_execute(argc, argv, backup_subcmd == BACKUP_CMD); + int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD); + if (status != 0) + { + return status; + } } } @@ -537,13 +541,22 @@ main(int argc, char *argv[]) case INIT_CMD: return do_init(); case BACKUP_CMD: + current.stream = stream_wal; + if (ssh_host && !is_remote_agent) + { + current.status = BACKUP_STATUS_DONE; + StrNCpy(current.program_version, PROGRAM_VERSION, + sizeof(current.program_version)); + complete_backup(); + return 0; + } + else { const char *backup_mode; time_t start_time; start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); - current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a3e76b95b..b4bd8edf5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -378,6 +378,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); +extern void complete_backup(void); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, @@ -475,7 +476,7 @@ extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); -extern void remote_execute(int argc, char *argv[], bool do_backup); +extern int remote_execute(int argc, char *argv[], bool do_backup); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS #define COMPRESS_LEVEL_DEFAULT 1 @@ -501,7 +502,7 @@ extern parray *dir_read_file_list(const char *root, const char *file_txt); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); -extern bool fileExists(const char *path); +extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, bool omit_symlink); @@ -523,7 +524,7 @@ extern bool backup_data_file(backup_files_arg* arguments, extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header); -extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); +extern bool copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location location); extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); diff --git a/src/restore.c b/src/restore.c index f798675b5..2cf291d8e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -587,14 +587,14 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + elog(INFO, "The file didn`t change. Skip restore: %s", file->path); continue; } /* Directories were created before */ if (S_ISDIR(file->mode)) { - elog(VERBOSE, "directory, skip"); + elog(INFO, "directory, skip"); continue; } @@ -611,7 +611,7 @@ restore_files(void *arg) * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + elog(INFO, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) { @@ -624,7 +624,7 @@ restore_files(void *arg) false); } else - copy_file(from_root, pgdata, file); + copy_file(from_root, pgdata, file, FIO_DB_HOST); /* print size of restored file */ if (file->write_size != BYTES_INVALID) diff --git a/src/utils/file.c b/src/utils/file.c index b562fc13d..cbbd0aa38 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -377,13 +377,20 @@ ssize_t fio_read(int fd, void* buf, size_t size) } } -int fio_stat(int fd, struct stat* st) +int fio_ffstat(FILE* f, struct stat* st) +{ + return fio_is_remote_file(f) + ? fio_fstat(fio_fileno(f), st) + : fio_fstat(fileno(f), st); +} + +int fio_fstat(int fd, struct stat* st) { if (fio_is_remote_fd(fd)) { fio_header hdr; - hdr.cop = FIO_STAT; + hdr.cop = FIO_FSTAT; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; @@ -395,7 +402,7 @@ int fio_stat(int fd, struct stat* st) SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_STAT); + Assert(hdr.cop == FIO_FSTAT); IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); @@ -408,6 +415,40 @@ int fio_stat(int fd, struct stat* st) } } +int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + + hdr.cop = FIO_STAT; + hdr.handle = -1; + hdr.arg = follow_symlinks; + hdr.size = path_len; + + SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_STAT); + IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); + + SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); + + return hdr.arg; + } + else + { + return follow_symlinks ? stat(path, st) : lstat(path, st); + } +} + int fio_access(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) @@ -566,12 +607,32 @@ static void fio_send_file(int out, char const* path) } } +void fio_transfer(void* addr, size_t value) +{ + struct { + fio_header hdr; + fio_binding bind; + } msg; + + msg.hdr.cop = FIO_TRANSFER; + msg.hdr.size = sizeof(fio_binding); + msg.bind.address = (size_t*)addr; + msg.bind.value = value; + + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + IO_CHECK(fio_write_all(fio_stdout, &msg, sizeof(msg)), sizeof(msg)); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); +} + void fio_communicate(int in, int out) { int fd[FIO_FDMAX]; size_t buf_size = 128*1024; char* buf = (char*)malloc(buf_size); fio_header hdr; + struct stat st; int rc; while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { @@ -606,15 +667,18 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, rc), rc); break; + case FIO_FSTAT: + hdr.size = sizeof(st); + hdr.arg = fstat(fd[hdr.handle], &st); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); + break; case FIO_STAT: - { - struct stat st; - hdr.size = sizeof(st); - hdr.arg = fstat(fd[hdr.handle], &st); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); - break; - } + hdr.size = sizeof(st); + hdr.arg = hdr.arg ? stat(buf, &st) : lstat(buf, &st); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); + break; case FIO_ACCESS: hdr.size = 0; hdr.arg = access(buf, hdr.arg); @@ -638,6 +702,9 @@ void fio_communicate(int in, int out) case FIO_TRUNCATE: SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; + case FIO_TRANSFER: + *((fio_binding*)buf)->address = ((fio_binding*)buf)->value; + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 3b7cc1121..123167a74 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -17,12 +17,15 @@ typedef enum FIO_READ, FIO_LOAD, FIO_STAT, + FIO_FSTAT, FIO_SEND, - FIO_ACCESS + FIO_ACCESS, + FIO_TRANSFER } fio_operations; typedef enum { + FIO_LOCAL_HOST, FIO_DB_HOST, FIO_BACKUP_HOST } fio_location; @@ -33,13 +36,20 @@ typedef enum #define SYS_CHECK(cmd) do if ((cmd) < 0) { perror(#cmd); exit(EXIT_FAILURE); } while (0) #define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d\n", __FILE__, __LINE__, _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) -typedef struct { +typedef struct +{ unsigned cop : 4; unsigned handle : 8; unsigned size : 20; unsigned arg; } fio_header; +typedef struct +{ + size_t* address; + size_t value; +} fio_binding; + extern void fio_redirect(int in, int out); extern void fio_communicate(int in, int out); @@ -51,13 +61,14 @@ extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); extern int fio_ftruncate(FILE* f, off_t size); extern int fio_fclose(FILE* f); +extern int fio_ffstat(FILE* f, struct stat* st); extern int fio_open(char const* name, int mode, fio_location location); extern ssize_t fio_write(int fd, void const* buf, size_t size); extern ssize_t fio_read(int fd, void* buf, size_t size); extern int fio_flush(int fd); extern int fio_seek(int fd, off_t offs); -extern int fio_stat(int fd, struct stat* st); +extern int fio_fstat(int fd, struct stat* st); extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); @@ -66,9 +77,12 @@ extern int fio_unlink(char const* path, fio_location location); extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); extern int fio_access(char const* path, int mode, fio_location location); +extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); + +extern FILE* fio_open_stream(char const* name, fio_location location); +extern int fio_close_stream(FILE* f); -extern FILE* fio_open_stream(char const* name, fio_location location); -extern int fio_close_stream(FILE* f); +extern void fio_transfer(void* addr, size_t value); #endif diff --git a/src/utils/remote.c b/src/utils/remote.c index f7b3f4c5b..2dd6cfe17 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -21,7 +21,7 @@ static int append_option(char* buf, size_t buf_size, size_t dst, char const* src return dst + len + 1; } -void remote_execute(int argc, char* argv[], bool listen) +int remote_execute(int argc, char* argv[], bool listen) { char cmd[MAX_CMDLINE_LENGTH]; size_t dst = 0; @@ -66,6 +66,7 @@ void remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(close(outfd[1])); SYS_CHECK(execvp(ssh_argv[0], ssh_argv)); + return -1; } else { SYS_CHECK(close(outfd[0])); /* These are being used by the child */ SYS_CHECK(close(infd[1])); @@ -74,9 +75,10 @@ void remote_execute(int argc, char* argv[], bool listen) int status; fio_communicate(infd[0], outfd[1]); SYS_CHECK(wait(&status)); - exit(status); + return status; } else { fio_redirect(infd[0], outfd[1]); /* write to stdout */ + return 0; } } } diff --git a/src/walmethods.c b/src/walmethods.c index d384b99f6..a179945fc 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -305,23 +305,14 @@ dir_get_file_size(const char *pathname) { struct stat statbuf; static char tmppath[MAXPGPATH]; - int fd; snprintf(tmppath, sizeof(tmppath), "%s/%s", dir_data->basedir, pathname); - fd = fio_open(tmppath, O_RDONLY|PG_BINARY, FIO_BACKUP_HOST); - if (fd >= 0) - { - if (fio_stat(fd, &statbuf) != 0) - { - fio_close(fd); - return -1; - } - fio_close(fd); - return statbuf.st_size; - } - return -1; + if (fio_stat(tmppath, &statbuf, true, FIO_BACKUP_HOST) != 0) + return -1; + + return statbuf.st_size; } static bool From 9a4780700cd86a60f02037c69cc4dd66bf277090 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 5 Nov 2018 15:23:46 +0300 Subject: [PATCH 0027/2107] First successful restore --- src/backup.c | 33 +++++++++++++++++++-------------- src/data.c | 20 ++++++++++++-------- src/dir.c | 11 +++++------ src/pg_probackup.h | 2 +- src/restore.c | 6 +++--- src/walmethods.c | 14 +++++++++++++- src/walmethods.h | 4 ++-- 7 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9593e6ab6..2dc6af1ee 100644 --- a/src/backup.c +++ b/src/backup.c @@ -66,10 +66,11 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; + parray *files_list; } StreamThreadArg; static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1}; +static StreamThreadArg stream_thread_arg = {"", NULL, 1, NULL}; static int is_ptrack_enable = false; bool is_ptrack_support = false; @@ -475,6 +476,7 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + parray *xlog_files_list = NULL; elog(LOG, "Database backup start"); @@ -604,8 +606,12 @@ do_backup_instance(void) elog(ERROR, "Cannot continue backup because stream connect has failed."); } - /* By default there are some error */ + if (is_remote_agent) + xlog_files_list = parray_new(); + + /* By default there are some error */ stream_thread_arg.ret = 1; + stream_thread_arg.files_list = xlog_files_list; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } @@ -761,18 +767,18 @@ do_backup_instance(void) /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { - parray *xlog_files_list; - char pg_xlog_path[MAXPGPATH]; - - /* Scan backup PG_XLOG_DIR */ - xlog_files_list = parray_new(); - join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + if (xlog_files_list == NULL) + { + char pg_xlog_path[MAXPGPATH]; + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + } for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - if (S_ISREG(file->mode)) calc_file_checksum(file); /* Remove file path root prefix*/ @@ -784,7 +790,6 @@ do_backup_instance(void) free(ptr); } } - /* Add xlog files into the list of backed up files */ parray_concat(backup_files_list, xlog_files_list); parray_free(xlog_files_list); @@ -1874,7 +1879,7 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true); + file = pgFileNew(backup_label, true, FIO_BACKUP_HOST); calc_file_checksum(file); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -1917,7 +1922,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true); + file = pgFileNew(tablespace_map, true, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) calc_file_checksum(file); free(file->path); @@ -2608,7 +2613,7 @@ StreamLog(void *arg) ctl.sysidentifier = NULL; #if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true, stream_arg->files_list); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; #else diff --git a/src/data.c b/src/data.c index eb2230b68..e3725395a 100644 --- a/src/data.c +++ b/src/data.c @@ -1331,7 +1331,8 @@ bool calc_file_checksum(pgFile *file) { int in; - size_t read_len = 0; + ssize_t read_len = 0; + int errno_tmp; char buf[BLCKSZ]; struct stat st; pg_crc32 crc; @@ -1366,13 +1367,8 @@ calc_file_checksum(pgFile *file) strerror(errno)); } - for (;;) + while ((read_len = fio_read(in, buf, sizeof(buf))) > 0) { - read_len = fio_read(in, buf, sizeof(buf)); - - if(read_len == 0) - break; - /* update CRC */ COMP_TRADITIONAL_CRC32(crc, buf, read_len); @@ -1380,7 +1376,15 @@ calc_file_checksum(pgFile *file) file->read_size += read_len; } - /* finish CRC calculation and store into pgFile */ + errno_tmp = errno; + if (read_len < 0) + { + fio_close(in); + elog(ERROR, "cannot read backup mode file \"%s\": %s", + file->path, strerror(errno_tmp)); + } + + /* finish CRC calculation and store into pgFile */ FIN_TRADITIONAL_CRC32(crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index bf5ec27f8..bf5e50084 100644 --- a/src/dir.c +++ b/src/dir.c @@ -157,13 +157,13 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink) +pgFileNew(const char *path, bool omit_symlink, fio_location location) { struct stat st; pgFile *file; /* stat the file */ - if (fio_stat(path, &st, omit_symlink, FIO_BACKUP_HOST) < 0) + if (fio_stat(path, &st, omit_symlink, location) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -432,7 +432,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false); + file = pgFileNew(root, false, FIO_LOCAL_HOST); if (file == NULL) return; @@ -661,7 +661,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink); + file = pgFileNew(child, omit_symlink, FIO_LOCAL_HOST); if (file == NULL) continue; @@ -795,7 +795,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false); + dir = pgFileNew(path, false, FIO_LOCAL_HOST); parray_append(files, dir); } @@ -935,7 +935,6 @@ create_data_directories(const char *data_dir, const char *backup_dir, size_t i; char backup_database_dir[MAXPGPATH], to_path[MAXPGPATH]; - sleep(30); dirs = parray_new(); if (extract_tablespaces) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b4bd8edf5..20399734e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -505,7 +505,7 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, fio_location location); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); diff --git a/src/restore.c b/src/restore.c index 2cf291d8e..51d9afc3f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -587,14 +587,14 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(INFO, "The file didn`t change. Skip restore: %s", file->path); + elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); continue; } /* Directories were created before */ if (S_ISDIR(file->mode)) { - elog(INFO, "directory, skip"); + elog(VERBOSE, "directory, skip"); continue; } @@ -611,7 +611,7 @@ restore_files(void *arg) * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(INFO, "Restoring file %s, is_datafile %i, is_cfs %i", + elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); if (file->is_datafile && !file->is_cfs) { diff --git a/src/walmethods.c b/src/walmethods.c index a179945fc..39f4f990e 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -45,6 +45,7 @@ typedef struct DirectoryMethodData char *basedir; int compression; bool sync; + parray *files_list; } DirectoryMethodData; static DirectoryMethodData *dir_data = NULL; @@ -231,6 +232,8 @@ dir_close(Walfile f, WalCloseMethod method) if (r == 0) { + char const* file_path = NULL; + /* Build path to the current version of the file */ if (method == CLOSE_NORMAL && df->temp_suffix) { @@ -246,6 +249,7 @@ dir_close(Walfile f, WalCloseMethod method) dir_data->basedir, df->pathname, dir_data->compression > 0 ? ".gz" : ""); r = durable_rename(tmppath, tmppath2, progname); + file_path = tmppath2; } else if (method == CLOSE_UNLINK) { @@ -263,6 +267,7 @@ dir_close(Walfile f, WalCloseMethod method) * CLOSE_NO_RENAME. In this case, fsync the file and containing * directory if sync mode is requested. */ + file_path = df->fullpath; if (dir_data->sync && !is_remote_agent) { r = fsync_fname(df->fullpath, false, progname); @@ -270,6 +275,12 @@ dir_close(Walfile f, WalCloseMethod method) r = fsync_parent_path(df->fullpath, progname); } } + if (file_path && dir_data->files_list) + { + pgFile* file = pgFileNew(file_path, false, FIO_BACKUP_HOST); + Assert(file != NULL); + parray_append(dir_data->files_list, file); + } } pg_free(df->pathname); @@ -343,7 +354,7 @@ dir_finish(void) WalWriteMethod * -CreateWalDirectoryMethod(const char *basedir, int compression, bool sync) +CreateWalDirectoryMethod(const char *basedir, int compression, bool sync, parray* files_list) { WalWriteMethod *method; @@ -362,6 +373,7 @@ CreateWalDirectoryMethod(const char *basedir, int compression, bool sync) dir_data->compression = compression; dir_data->basedir = pg_strdup(basedir); dir_data->sync = sync; + dir_data->files_list = files_list; return method; } diff --git a/src/walmethods.h b/src/walmethods.h index 76eb249b9..b4dd40c53 100644 --- a/src/walmethods.h +++ b/src/walmethods.h @@ -9,7 +9,7 @@ *------------------------------------------------------------------------- */ - +#include "utils/parray.h" typedef void *Walfile; typedef enum @@ -86,7 +86,7 @@ struct WalWriteMethod * not all those required for pg_receivewal) */ WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, - int compression, bool sync); + int compression, bool sync, parray* file_list); /* Cleanup routines for previously-created methods */ void FreeWalDirectoryMethod(void); From 6b029ebfdc5b72983c495a87bf61ba58799ddadf Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 5 Nov 2018 23:18:44 +0300 Subject: [PATCH 0028/2107] Support compression --- src/data.c | 37 +++++++++++++++--------------- src/parsexlog.c | 7 +++--- src/utils/file.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++- src/utils/file.h | 12 +++++++++- src/walmethods.c | 39 ++++++++++++++++++-------------- 5 files changed, 113 insertions(+), 40 deletions(-) diff --git a/src/data.c b/src/data.c index e3725395a..f5cfeefbd 100644 --- a/src/data.c +++ b/src/data.c @@ -989,7 +989,7 @@ copy_meta(const char *from_path, const char *to_path, bool unlink_on_error, fio_ if (stat(from_path, &st) == -1) { if (unlink_on_error) - unlink(to_path); + fio_unlink(to_path, location); elog(ERROR, "Cannot stat file \"%s\": %s", from_path, strerror(errno)); } @@ -997,7 +997,7 @@ copy_meta(const char *from_path, const char *to_path, bool unlink_on_error, fio_ if (fio_chmod(to_path, st.st_mode, location) == -1) { if (unlink_on_error) - unlink(to_path); + fio_unlink(to_path, location); elog(ERROR, "Cannot change mode of file \"%s\": %s", to_path, strerror(errno)); } @@ -1020,6 +1020,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ char gz_to_path[MAXPGPATH]; gzFile gz_out = NULL; + int gz_tmp = -1; #endif /* open file for read */ @@ -1039,7 +1040,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = gzopen(to_path_temp, PG_BINARY_W); + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, &gz_tmp, FIO_BACKUP_HOST); if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", compress_level, to_path_temp, get_gz_error(gz_out, errno)); @@ -1070,7 +1071,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (ferror(in)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot read source WAL file \"%s\": %s", from_path, strerror(errno_temp)); @@ -1084,7 +1085,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (gzwrite(gz_out, buf, read_len) != read_len) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", to_path_temp, get_gz_error(gz_out, errno_temp)); } @@ -1095,7 +1096,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (fio_fwrite(out, buf, read_len) != read_len) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, strerror(errno_temp)); } @@ -1109,10 +1110,10 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ if (is_compress) { - if (gzclose(gz_out) != 0) + if (fio_gzclose(gz_out, to_path_temp, gz_tmp) != 0) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", to_path_temp, get_gz_error(gz_out, errno_temp)); } @@ -1124,7 +1125,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, fio_fclose(out)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot write WAL file \"%s\": %s", to_path_temp, strerror(errno_temp)); } @@ -1133,7 +1134,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (fclose(in)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot close source WAL file \"%s\": %s", from_path, strerror(errno_temp)); } @@ -1144,7 +1145,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", to_path_temp, to_path_p, strerror(errno_temp)); } @@ -1229,7 +1230,7 @@ get_wal_file(const char *from_path, const char *to_path) if (read_len != sizeof(buf) && !gzeof(gz_in)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", gz_from_path, get_gz_error(gz_in, errno_temp)); } @@ -1241,7 +1242,7 @@ get_wal_file(const char *from_path, const char *to_path) if (ferror(in)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot read source WAL file \"%s\": %s", from_path, strerror(errno_temp)); } @@ -1252,7 +1253,7 @@ get_wal_file(const char *from_path, const char *to_path) if (fio_fwrite(out, buf, read_len) != read_len) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, strerror(errno_temp)); } @@ -1277,7 +1278,7 @@ get_wal_file(const char *from_path, const char *to_path) fio_fclose(out)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot write WAL file \"%s\": %s", to_path_temp, strerror(errno_temp)); } @@ -1288,7 +1289,7 @@ get_wal_file(const char *from_path, const char *to_path) if (gzclose(gz_in) != 0) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", gz_from_path, get_gz_error(gz_in, errno_temp)); } @@ -1299,7 +1300,7 @@ get_wal_file(const char *from_path, const char *to_path) if (fclose(in)) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot close source WAL file \"%s\": %s", from_path, strerror(errno_temp)); } @@ -1311,7 +1312,7 @@ get_wal_file(const char *from_path, const char *to_path) if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) { errno_temp = errno; - unlink(to_path_temp); + fio_unlink(to_path_temp, FIO_DB_HOST); elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", to_path_temp, to_path, strerror(errno_temp)); } diff --git a/src/parsexlog.c b/src/parsexlog.c index 0eb4d1fb0..3c34d5061 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -102,6 +102,7 @@ typedef struct XLogPageReadPrivate #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; + int gz_tmp; #endif } XLogPageReadPrivate; @@ -885,8 +886,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private_data->thread_num, private_data->gz_xlogpath); private_data->xlogexists = true; - private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, - "rb"); + private_data->gz_xlogfile = fio_gzopen(private_data->gz_xlogpath, + "rb", &private_data->gz_tmp, private_data->location); if (private_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", @@ -1000,7 +1001,7 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) #ifdef HAVE_LIBZ else if (private_data->gz_xlogfile != NULL) { - gzclose(private_data->gz_xlogfile); + fio_gzclose(private_data->gz_xlogfile, private_data->gz_xlogpath, private_data->gz_tmp); private_data->gz_xlogfile = NULL; } #endif diff --git a/src/utils/file.c b/src/utils/file.c index cbbd0aa38..ad69a60cd 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -36,7 +36,8 @@ static bool fio_is_remote_fd(int fd) static bool fio_is_remote(fio_location location) { - return (location == FIO_BACKUP_HOST && is_remote_agent) + return location == FIO_REMOTE_HOST + || (location == FIO_BACKUP_HOST && is_remote_agent) || (location == FIO_DB_HOST && !is_remote_agent && ssh_host != NULL); } @@ -581,6 +582,61 @@ int fio_chmod(char const* path, int mode, fio_location location) } } +#ifdef HAVE_LIBZ +gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location location) +{ + gzFile file; + if (fio_is_remote(location)) + { + int fd = mkstemp("gz.XXXXXX"); + if (fd < 0) + return NULL; + *tmp_fd = fd; + file = gzdopen(fd, mode); + } + else + { + *tmp_fd = -1; + file = gzopen(path, mode); + } + return file; +} + +int fio_gzclose(gzFile file, char const* path, int tmp_fd) +{ + if (tmp_fd >= 0) + { + off_t size; + void* buf; + int fd; + + SYS_CHECK(gzflush(file, Z_FINISH)); + + size = lseek(tmp_fd, 0, SEEK_END); + buf = malloc(size); + + lseek(tmp_fd, 0, SEEK_SET); + IO_CHECK(read(tmp_fd, buf, size), size); + + SYS_CHECK(gzclose(file)); /* should close tmp_fd */ + + fd = fio_open(path, O_RDWR|O_CREAT|O_TRUNC, FILE_PERMISSIONS); + if (fd < 0) { + free(buf); + return -1; + } + IO_CHECK(fio_write(fd, buf, size), size); + free(buf); + return fio_close(fd); + } + else + { + return gzclose(file); + } +} +#endif + + static void fio_send_file(int out, char const* path) { int fd = open(path, O_RDONLY); diff --git a/src/utils/file.h b/src/utils/file.h index 123167a74..875f0c321 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -3,6 +3,10 @@ #include +#ifdef HAVE_LIBZ +#include +#endif + typedef enum { FIO_OPEN, @@ -27,7 +31,8 @@ typedef enum { FIO_LOCAL_HOST, FIO_DB_HOST, - FIO_BACKUP_HOST + FIO_BACKUP_HOST, + FIO_REMOTE_HOST } fio_location; #define FIO_FDMAX 64 @@ -82,6 +87,11 @@ extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, extern FILE* fio_open_stream(char const* name, fio_location location); extern int fio_close_stream(FILE* f); +#ifdef HAVE_LIBZ +extern gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location location); +extern int fio_gzclose(gzFile file, char const* path, int tmp_fd); +#endif + extern void fio_transfer(void* addr, size_t value); #endif diff --git a/src/walmethods.c b/src/walmethods.c index 39f4f990e..802a8e9c6 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -61,6 +61,7 @@ typedef struct DirectoryMethodFile char *temp_suffix; #ifdef HAVE_LIBZ gzFile gzfp; + int gz_tmp; #endif } DirectoryMethodFile; @@ -74,11 +75,12 @@ dir_getlasterror(void) static Walfile dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) { - static char tmppath[MAXPGPATH]; - int fd; + char tmppath[MAXPGPATH]; + int fd = -1; DirectoryMethodFile *f; #ifdef HAVE_LIBZ gzFile gzfp = NULL; + int gz_tmp = -1; #endif snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", @@ -92,17 +94,12 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ * does not do any system calls to fsync() to make changes permanent on * disk. */ - fd = fio_open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, FIO_BACKUP_HOST); - if (fd < 0) - return NULL; - #ifdef HAVE_LIBZ if (dir_data->compression > 0) { - gzfp = gzdopen(fd, "wb"); + gzfp = fio_gzopen(tmppath, "wb", &gz_tmp, FIO_BACKUP_HOST); if (gzfp == NULL) { - close(fd); return NULL; } @@ -113,8 +110,13 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ return NULL; } } + else #endif - + { + fd = fio_open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, FIO_BACKUP_HOST); + if (fd < 0) + return NULL; + } /* Do pre-padding on non-compressed files */ if (pad_to_size && dir_data->compression == 0) { @@ -173,7 +175,10 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ f = pg_malloc0(sizeof(DirectoryMethodFile)); #ifdef HAVE_LIBZ if (dir_data->compression > 0) + { f->gzfp = gzfp; + f->gz_tmp = gz_tmp; + } #endif f->fd = fd; f->currpos = 0; @@ -218,14 +223,14 @@ dir_close(Walfile f, WalCloseMethod method) { int r; DirectoryMethodFile *df = (DirectoryMethodFile *) f; - static char tmppath[MAXPGPATH]; - static char tmppath2[MAXPGPATH]; + char tmppath[MAXPGPATH]; + char tmppath2[MAXPGPATH]; Assert(f != NULL); #ifdef HAVE_LIBZ if (dir_data->compression > 0) - r = gzclose(df->gzfp); + r = fio_gzclose(df->gzfp, df->fullpath, df->gz_tmp); else #endif r = fio_close(df->fd); @@ -248,7 +253,7 @@ dir_close(Walfile f, WalCloseMethod method) snprintf(tmppath2, sizeof(tmppath2), "%s/%s%s", dir_data->basedir, df->pathname, dir_data->compression > 0 ? ".gz" : ""); - r = durable_rename(tmppath, tmppath2, progname); + r = fio_rename(tmppath, tmppath2, FIO_BACKUP_HOST); file_path = tmppath2; } else if (method == CLOSE_UNLINK) @@ -258,7 +263,7 @@ dir_close(Walfile f, WalCloseMethod method) dir_data->basedir, df->pathname, dir_data->compression > 0 ? ".gz" : "", df->temp_suffix ? df->temp_suffix : ""); - r = unlink(tmppath); + r = fio_unlink(tmppath, FIO_BACKUP_HOST); } else { @@ -267,7 +272,7 @@ dir_close(Walfile f, WalCloseMethod method) * CLOSE_NO_RENAME. In this case, fsync the file and containing * directory if sync mode is requested. */ - file_path = df->fullpath; + file_path = df->fullpath; if (dir_data->sync && !is_remote_agent) { r = fsync_fname(df->fullpath, false, progname); @@ -315,7 +320,7 @@ static ssize_t dir_get_file_size(const char *pathname) { struct stat statbuf; - static char tmppath[MAXPGPATH]; + char tmppath[MAXPGPATH]; snprintf(tmppath, sizeof(tmppath), "%s/%s", dir_data->basedir, pathname); @@ -329,7 +334,7 @@ dir_get_file_size(const char *pathname) static bool dir_existsfile(const char *pathname) { - static char tmppath[MAXPGPATH]; + char tmppath[MAXPGPATH]; snprintf(tmppath, sizeof(tmppath), "%s/%s", dir_data->basedir, pathname); From 6c9dfcfe8290df7745356ac9183d82dca61ade98 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 7 Nov 2018 04:21:56 +0300 Subject: [PATCH 0029/2107] PGPRO-2095: use latest replayed lsn instead of STOP LSN --- src/backup.c | 80 +++++++++++++++++++++++++++------------------- src/pg_probackup.h | 4 +++ src/util.c | 30 ++++++++--------- 3 files changed, 67 insertions(+), 47 deletions(-) diff --git a/src/backup.c b/src/backup.c index 602ab823e..e3e1d60ad 100644 --- a/src/backup.c +++ b/src/backup.c @@ -756,28 +756,25 @@ do_backup_instance(void) parray_free(prev_backup_filelist); } - /* Copy pg_control in case of backup from replica >= 9.6 */ + /* In case of backup from replica >= 9.6 we must fix minRecPoint, + * First we must find pg_control in backup_files_list. + */ if (current.from_replica && !exclusive_backup) { + char pg_control_path[MAXPGPATH]; + + snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", pgdata, "global/pg_control"); + for (i = 0; i < parray_num(backup_files_list); i++) { pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); - if (strcmp(tmp_file->name, "pg_control") == 0) + if (strcmp(tmp_file->path, pg_control_path) == 0) { pg_control = tmp_file; break; } } - - if (!pg_control) - elog(ERROR, "Failed to locate pg_control in copied files"); - - if (is_remote_backup) - remote_copy_file(NULL, pg_control); - else - if (!copy_file(pgdata, database_path, pg_control)) - elog(ERROR, "Failed to copy pg_control"); } @@ -1160,9 +1157,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) */ pg_switch_wal(conn); - //elog(INFO, "START LSN: %X/%X", - // (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn)); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ wait_wal_lsn(backup->start_lsn, true, false); @@ -1175,8 +1169,10 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) /* ...for others wait for previous segment */ wait_wal_lsn(backup->start_lsn, true, true); - /* Wait for start_lsn to be replayed by replica */ - if (backup->from_replica) + /* In case of backup from replica for PostgreSQL 9.5 + * wait for start_lsn to be replayed by replica + */ + if (backup->from_replica && exclusive_backup) wait_replica_wal_lsn(backup->start_lsn, true); } @@ -1526,7 +1522,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); /* - * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is + * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory if it is * stream and non-page backup. Page backup needs archived WAL files, so we * wait for 'lsn' in archive 'wal' directory for page backups. * @@ -1547,7 +1543,12 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { join_path_components(wal_segment_path, arclog_path, wal_segment); wal_segment_dir = arclog_path; - timeout = archive_timeout; + + if (archive_timeout > 0) + timeout = archive_timeout; + else + timeout = ARCHIVE_TIMEOUT_DEFAULT; + } if (wait_prev_segment) @@ -1780,14 +1781,29 @@ pg_stop_backup(pgBackup *backup) * Stop the non-exclusive backup. Besides stop_lsn it returns from * pg_stop_backup(false) copy of the backup label and tablespace map * so they can be written to disk by the caller. + * In case of backup from replica >= 9.6 we do not trust minRecPoint + * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. */ - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)"; + if (current.from_replica) + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," +#if PG_VERSION_NUM >= 100000 + " pg_catalog.pg_last_wal_replay_lsn()," +#else + " pg_catalog.pg_last_xlog_replay_location()," +#endif + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + else + stop_backup_query = "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; } else @@ -1873,14 +1889,14 @@ pg_stop_backup(pgBackup *backup) /* Calculate LSN */ stop_backup_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - //if (!XRecOffIsValid(stop_backup_lsn)) - //{ - // stop_backup_lsn = restore_lsn; - //} - if (!XRecOffIsValid(stop_backup_lsn)) - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + { + if (XRecOffIsNull(stop_backup_lsn)) + stop_backup_lsn = stop_backup_lsn + SizeOfXLogLongPHD; + else + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + } /* Write backup_label and tablespace_map */ if (!exclusive_backup) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b75bb5817..f5d6bb5cb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -57,6 +57,10 @@ #define XID_FMT "%u" #endif +/* Check if an XLogRecPtr value is pointed to 0 offset */ +#define XRecOffIsNull(xlrp) \ + ((xlrp) % XLOG_BLCKSZ == 0) + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, diff --git a/src/util.c b/src/util.c index 5f059c374..e20cda175 100644 --- a/src/util.c +++ b/src/util.c @@ -119,7 +119,7 @@ writeControlFile(ControlFileData *ControlFile, char *path) /* copy controlFileSize */ buffer = pg_malloc(ControlFileSize); - memcpy(buffer, &ControlFile, sizeof(ControlFileData)); + memcpy(buffer, ControlFile, sizeof(ControlFileData)); /* Write pg_control */ unlink(path); @@ -136,8 +136,8 @@ writeControlFile(ControlFileData *ControlFile, char *path) if (fsync(fd) != 0) elog(ERROR, "Failed to fsync file: %s", path); - pg_free(buffer); close(fd); + pg_free(buffer); } /* @@ -290,9 +290,7 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } -/* MinRecoveryPoint 'as-is' is not to be trusted - * Use STOP LSN instead - */ +/* MinRecoveryPoint 'as-is' is not to be trusted */ void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) { @@ -301,20 +299,21 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba size_t size; char fullpath[MAXPGPATH]; - elog(LOG, "Setting minRecPoint to STOP LSN: %X/%X", - (uint32) (stop_backup_lsn >> 32), - (uint32) stop_backup_lsn); - - /* Path to pg_control in backup */ - snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); - - /* First fetch file... */ - buffer = slurpFile(backup_path, XLOG_CONTROL_FILE, &size, false); + /* First fetch file content */ + buffer = slurpFile(pgdata, XLOG_CONTROL_FILE, &size, false); if (buffer == NULL) elog(ERROR, "ERROR"); digestControlFile(&ControlFile, buffer, size); + elog(LOG, "Current minRecPoint %X/%X", + (uint32) (ControlFile.minRecoveryPoint >> 32), + (uint32) ControlFile.minRecoveryPoint); + + elog(LOG, "Setting minRecPoint to %X/%X", + (uint32) (stop_backup_lsn >> 32), + (uint32) stop_backup_lsn); + ControlFile.minRecoveryPoint = stop_backup_lsn; /* Update checksum in pg_control header */ @@ -327,7 +326,8 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba /* paranoia */ checkControlFile(&ControlFile); - /* update pg_control */ + /* overwrite pg_control */ + snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); writeControlFile(&ControlFile, fullpath); /* Update pg_control checksum in backup_list */ From 16b9b369815f8527ce5d046df9351bf7f358f660 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 7 Nov 2018 04:29:54 +0300 Subject: [PATCH 0030/2107] bugfix: change units for archive-timeout & archive-timeout to seconds, log-rotation-age to minutes --- src/configure.c | 6 +++--- src/pg_probackup.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/configure.c b/src/configure.c index 30845607d..1714d9f51 100644 --- a/src/configure.c +++ b/src/configure.c @@ -245,7 +245,7 @@ readBackupCatalogConfigFile(void) { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, { 'U', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, + { 'U', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, /* connection options */ { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, @@ -257,14 +257,14 @@ readBackupCatalogConfigFile(void) { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, - { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, + { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, /* other options */ { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, #if PG_VERSION_NUM >= 110000 {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, #endif /* archive options */ - { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, + { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, {0} }; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index cf51fd107..b1b51ceb3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -199,7 +199,7 @@ static pgut_option options[] = { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, { 'U', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, + { 'U', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, /* connection options */ { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, From 1f951e4e71d1d70e4ed8931cb8841d03b1f8db95 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 7 Nov 2018 04:57:06 +0300 Subject: [PATCH 0031/2107] revert log-rotation-age unit change --- src/configure.c | 2 +- src/pg_probackup.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/configure.c b/src/configure.c index 1714d9f51..6d3cf9cb6 100644 --- a/src/configure.c +++ b/src/configure.c @@ -245,7 +245,7 @@ readBackupCatalogConfigFile(void) { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, { 'U', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + { 'U', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, /* connection options */ { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b1b51ceb3..cf51fd107 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -199,7 +199,7 @@ static pgut_option options[] = { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, { 'U', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MIN }, + { 'U', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, /* connection options */ { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, From 62d0f5fc9740ec095095e3e812a88be7aa716ab8 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 7 Nov 2018 22:06:50 +0300 Subject: [PATCH 0032/2107] Fix bug with page compression --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index a8a9e4de7..cc2edac58 100644 --- a/src/data.c +++ b/src/data.c @@ -368,7 +368,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, BackupPageHeader header; size_t write_buffer_size = sizeof(header); char write_buffer[BLCKSZ+sizeof(header)]; - char compressed_page[BLCKSZ]; + char compressed_page[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ if(page_state == SkipCurrentPage) return; @@ -395,7 +395,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, Assert (header.compressed_size <= BLCKSZ); /* The page was successfully compressed. */ - if (header.compressed_size > 0) + if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) { memcpy(write_buffer, &header, sizeof(header)); memcpy(write_buffer + sizeof(header), From dec2e62429ac540f5dde6ff922d579fca41fc5fe Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 12:23:04 +0300 Subject: [PATCH 0033/2107] Provide backward compatibility for decompression bug --- src/data.c | 63 +++++++++++++++++++++++++++++++++++++++++++--- src/pg_probackup.c | 2 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index f5cfeefbd..d7908bc74 100644 --- a/src/data.c +++ b/src/data.c @@ -99,6 +99,55 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; } + +#define ZLIB_MAGIC 0x78 + +/* + * Before version 2.0.23 there was a bug in pro_backup that pages which compressed + * size is exactly the same as original size are not treated as compressed. + * This check tries to detect and decompress such pages. + * There is no 100% criteria to determine whether page is compressed or not. + * But at least we will do this check only for pages which will no pass validation step. + */ +static bool +page_may_be_compressed(Page page, CompressAlg alg) +{ + PageHeader phdr; + + phdr = (PageHeader) page; + + /* First check if page header is valid (it seems to be fast enough check) */ + if (!(PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special))) + { + /* ... end only if it is invalid, then do more checks */ + int major, middle, minor; + if ( parse_program_version(current.program_version) >= 20023) + { + /* Versions 2.0.23 and higher don't have such bug */ + return false; + } +#ifdef HAVE_LIBZ + /* For zlib we can check page magic: + * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like + */ + if (alg == ZLIB_COMPRESS && *(char*)page != ZLIB_MAGIC) + { + return false; + } +#endif + /* otherwize let's try to decompress the page */ + return true; + } + return false; +} + /* * When copying datafiles to backup we validate and compress them block * by block. Thus special header is required for each data block. @@ -714,7 +763,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); - if (header.compressed_size != BLCKSZ) + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg)) { int32 uncompressed_size = 0; @@ -1579,7 +1629,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); - if (header.compressed_size != BLCKSZ) + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg)) { int32 uncompressed_size = 0; @@ -1589,9 +1640,15 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) file->compress_alg); if (uncompressed_size != BLCKSZ) + { + if (header.compressed_size == BLCKSZ) + { + is_valid = false; + continue; + } elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); - + } if (validate_one_page(page.data, file, blknum, stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) is_valid = false; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e0e1955b8..6aed05478 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -18,7 +18,7 @@ #include "utils/thread.h" -const char *PROGRAM_VERSION = "2.0.22"; +const char *PROGRAM_VERSION = "2.0.23"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; From 05e3211e2765d0b6d20e0c13799084076783a17d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 12:27:42 +0300 Subject: [PATCH 0034/2107] Provide backward compatibility for decompression bug --- src/data.c | 63 +++++++++++++++++++++++++++++++++++++++++++--- src/pg_probackup.c | 2 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index cc2edac58..876e7f1ed 100644 --- a/src/data.c +++ b/src/data.c @@ -99,6 +99,55 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; } + +#define ZLIB_MAGIC 0x78 + +/* + * Before version 2.0.23 there was a bug in pro_backup that pages which compressed + * size is exactly the same as original size are not treated as compressed. + * This check tries to detect and decompress such pages. + * There is no 100% criteria to determine whether page is compressed or not. + * But at least we will do this check only for pages which will no pass validation step. + */ +static bool +page_may_be_compressed(Page page, CompressAlg alg) +{ + PageHeader phdr; + + phdr = (PageHeader) page; + + /* First check if page header is valid (it seems to be fast enough check) */ + if (!(PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special))) + { + /* ... end only if it is invalid, then do more checks */ + int major, middle, minor; + if ( parse_program_version(current.program_version) >= 20023) + { + /* Versions 2.0.23 and higher don't have such bug */ + return false; + } +#ifdef HAVE_LIBZ + /* For zlib we can check page magic: + * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like + */ + if (alg == ZLIB_COMPRESS && *(char*)page != ZLIB_MAGIC) + { + return false; + } +#endif + /* otherwize let's try to decompress the page */ + return true; + } + return false; +} + /* * When copying datafiles to backup we validate and compress them block * by block. Thus special header is required for each data block. @@ -717,7 +766,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); - if (header.compressed_size != BLCKSZ) + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg)) { int32 uncompressed_size = 0; @@ -1595,7 +1645,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); - if (header.compressed_size != BLCKSZ) + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg)) { int32 uncompressed_size = 0; @@ -1605,9 +1656,15 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) file->compress_alg); if (uncompressed_size != BLCKSZ) + { + if (header.compressed_size == BLCKSZ) + { + is_valid = false; + continue; + } elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); - + } if (validate_one_page(page.data, file, blknum, stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) is_valid = false; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index cf51fd107..82107097c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,7 +17,7 @@ #include "utils/thread.h" -const char *PROGRAM_VERSION = "2.0.22"; +const char *PROGRAM_VERSION = "2.0.23"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; From 5e12fec6ab34b0b031b9f380dd63ab884d8657a2 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 12:43:19 +0300 Subject: [PATCH 0035/2107] Eliminate unsued varaibles --- src/data.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 876e7f1ed..a8378f098 100644 --- a/src/data.c +++ b/src/data.c @@ -127,8 +127,7 @@ page_may_be_compressed(Page page, CompressAlg alg) phdr->pd_special == MAXALIGN(phdr->pd_special))) { /* ... end only if it is invalid, then do more checks */ - int major, middle, minor; - if ( parse_program_version(current.program_version) >= 20023) + if (parse_program_version(current.program_version) >= 20023) { /* Versions 2.0.23 and higher don't have such bug */ return false; From ea3ed9fa171f5807bdb4790f8b64a0211cb729a7 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:04:55 +0300 Subject: [PATCH 0036/2107] Instead of current global backup variable use function local argument, which is sent by callers --- src/data.c | 59 +++++++++++++++++++++++-------- src/merge.c | 9 +++-- src/pg_probackup.h | 7 ++-- src/restore.c | 3 +- src/validate.c | 11 +++--- tests/expected/option_version.out | 2 +- 6 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/data.c b/src/data.c index a8378f098..5edeefa48 100644 --- a/src/data.c +++ b/src/data.c @@ -57,7 +57,7 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) */ static int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, int level) + CompressAlg alg, int level, const char **errormsg) { switch (alg) { @@ -66,7 +66,13 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_compress(dst, dst_size, src, src_size, level); + { + int32 ret; + ret = zlib_compress(dst, dst_size, src, src_size, level); + if (ret != Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -81,7 +87,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, */ static int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg) + CompressAlg alg, const char **errormsg) { switch (alg) { @@ -90,7 +96,13 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - return zlib_decompress(dst, dst_size, src, src_size); + { + int32 ret; + ret = zlib_decompress(dst, dst_size, src, src_size); + if (ret != Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_decompress(src, src_size, dst, dst_size); @@ -110,7 +122,7 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, * But at least we will do this check only for pages which will no pass validation step. */ static bool -page_may_be_compressed(Page page, CompressAlg alg) +page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) { PageHeader phdr; @@ -127,7 +139,7 @@ page_may_be_compressed(Page page, CompressAlg alg) phdr->pd_special == MAXALIGN(phdr->pd_special))) { /* ... end only if it is invalid, then do more checks */ - if (parse_program_version(current.program_version) >= 20023) + if (backup_version >= 20023) { /* Versions 2.0.23 and higher don't have such bug */ return false; @@ -434,9 +446,16 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, } else { + const char *errormsg = NULL; + /* The page was not truncated, so we need to compress it */ header.compressed_size = do_compress(compressed_page, BLCKSZ, - page, BLCKSZ, calg, clevel); + page, BLCKSZ, calg, clevel, + &errormsg); + /* Something went wrong and errormsg was assigned, throw a warning */ + if (header.compressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); file->compress_alg = calg; file->read_size += BLCKSZ; @@ -473,7 +492,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, fclose(in); fclose(out); - elog(ERROR, "File: %s, cannot write backup at block %u : %s", + elog(ERROR, "File: %s, cannot write backup at block %u: %s", file->path, blknum, strerror(errno_tmp)); } @@ -661,7 +680,7 @@ backup_data_file(backup_files_arg* arguments, */ void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, - bool write_header) + bool write_header, uint32 backup_version) { FILE *in = NULL; FILE *out = NULL; @@ -766,14 +785,19 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, read_len, header.compressed_size); if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg)) + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) { int32 uncompressed_size = 0; + const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, header.compressed_size, - file->compress_alg); + file->compress_alg, &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); if (uncompressed_size != BLCKSZ) elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", @@ -1578,7 +1602,8 @@ validate_one_page(Page page, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) +check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, + uint32 backup_version) { size_t read_len = 0; bool is_valid = true; @@ -1645,14 +1670,20 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version) blknum, file->path, read_len, header.compressed_size); if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg)) + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) { int32 uncompressed_size = 0; + const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, compressed_page.data, header.compressed_size, - file->compress_alg); + file->compress_alg, + &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); if (uncompressed_size != BLCKSZ) { diff --git a/src/merge.c b/src/merge.c index 2464199f2..137f1acda 100644 --- a/src/merge.c +++ b/src/merge.c @@ -457,14 +457,16 @@ merge_files(void *arg) file->path = to_path_tmp; /* Decompress first/target file */ - restore_data_file(tmp_file_path, file, false, false); + restore_data_file(tmp_file_path, file, false, false, + parse_program_version(to_backup->program_version)); file->path = prev_path; } /* Merge second/source file with first/target file */ restore_data_file(tmp_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false); + false, + parse_program_version(from_backup->program_version)); elog(VERBOSE, "Compress file and save it to the directory \"%s\"", argument->to_root); @@ -496,7 +498,8 @@ merge_files(void *arg) /* We can merge in-place here */ restore_data_file(to_path_tmp, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - true); + true, + parse_program_version(from_backup->program_version)); /* * We need to calculate write_size, restore_data_file() doesn't diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7bd87e56c..182a647bf 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -517,7 +517,8 @@ extern bool backup_data_file(backup_files_arg* arguments, CompressAlg calg, int clevel); extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, - bool write_header); + bool write_header, + uint32 backup_version); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, @@ -526,8 +527,8 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern bool calc_file_checksum(pgFile *file); -extern bool check_file_pages(pgFile* file, - XLogRecPtr stop_lsn, uint32 checksum_version); +extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern void extractPageMap(const char *archivedir, diff --git a/src/restore.c b/src/restore.c index c08e647cc..439f3c4e1 100644 --- a/src/restore.c +++ b/src/restore.c @@ -621,7 +621,8 @@ restore_files(void *arg) file->path + strlen(from_root) + 1); restore_data_file(to_path, file, arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false); + false, + parse_program_version(arguments->backup->program_version)); } else copy_file(from_root, pgdata, file); diff --git a/src/validate.c b/src/validate.c index 4fa7d78b6..5cbb9b261 100644 --- a/src/validate.c +++ b/src/validate.c @@ -28,6 +28,7 @@ typedef struct bool corrupted; XLogRecPtr stop_lsn; uint32 checksum_version; + uint32 backup_version; /* * Return value from the thread. @@ -92,11 +93,6 @@ pgBackupValidate(pgBackup *backup) pg_atomic_clear_flag(&file->lock); } - /* - * We use program version to calculate checksum in pgBackupValidateFiles() - */ - validate_backup_version = parse_program_version(backup->program_version); - /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (validate_files_arg *) @@ -111,6 +107,7 @@ pgBackupValidate(pgBackup *backup) arg->corrupted = false; arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; + arg->backup_version = parse_program_version(backup->program_version); /* By default there are some error */ threads_args[i].ret = 1; @@ -233,7 +230,9 @@ pgBackupValidateFiles(void *arg) /* validate relation blocks */ if (file->is_datafile) { - if (!check_file_pages(file, arguments->stop_lsn, arguments->checksum_version)) + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) arguments->corrupted = true; } } diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index cb0a30d4e..6a0391c2e 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.22 \ No newline at end of file +pg_probackup 2.0.23 \ No newline at end of file From 728e3d5bdc174c8c604c7ee8a57350b4c1847a5e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 8 Nov 2018 15:25:28 +0300 Subject: [PATCH 0037/2107] Avoid compression warnings for already compressed pages --- src/data.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 5edeefa48..d3e48e9a4 100644 --- a/src/data.c +++ b/src/data.c @@ -449,7 +449,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, const char *errormsg = NULL; /* The page was not truncated, so we need to compress it */ - header.compressed_size = do_compress(compressed_page, BLCKSZ, + header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), page, BLCKSZ, calg, clevel, &errormsg); /* Something went wrong and errormsg was assigned, throw a warning */ @@ -459,7 +459,6 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->compress_alg = calg; file->read_size += BLCKSZ; - Assert (header.compressed_size <= BLCKSZ); /* The page was successfully compressed. */ if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) From 53c1a05b9b5406af0bcad81b8a8f87cf11794f75 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:50:24 +0300 Subject: [PATCH 0038/2107] Remove unnecessary validate_backup_version --- src/validate.c | 4 +- src/validate.c.autosave | 532 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 533 insertions(+), 3 deletions(-) create mode 100644 src/validate.c.autosave diff --git a/src/validate.c b/src/validate.c index 5cbb9b261..2f75cd4b2 100644 --- a/src/validate.c +++ b/src/validate.c @@ -19,8 +19,6 @@ static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; -/* Program version of a current backup */ -static uint32 validate_backup_version = 0; typedef struct { @@ -220,7 +218,7 @@ pgBackupValidateFiles(void *arg) * To avoid this problem we need to use different algorithm, CRC-32 in * this case. */ - crc = pgFileGetCRC(file->path, validate_backup_version <= 20021); + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", diff --git a/src/validate.c.autosave b/src/validate.c.autosave new file mode 100644 index 000000000..1c07e0aeb --- /dev/null +++ b/src/validate.c.autosave @@ -0,0 +1,532 @@ +/*------------------------------------------------------------------------- + * + * validate.c: validate backup files. + * + * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include + +#include "utils/thread.h" + +static void *pgBackupValidateFiles(void *arg); +static void do_validate_instance(void); + +static bool corrupted_backup_found = false; + +typedef struct +{ + parray *files; + bool corrupted; + XLogRecPtr stop_lsn; + uint32 checksum_version; + uint32 backup_version; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + +if (is_absolute_path(host)) + if (hostaddr && *hostaddr) + printf(_("You are connected to database \"%s\" as user \"%s\" on address \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), hostaddr, PQport(pset.db)); + else + printf(_("You are connected to database \"%s\" as user \"%s\" via socket in \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), host, PQport(pset.db)); +else + if (hostaddr && *hostaddr && strcmp(host, hostaddr) != 0) + printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" (address \"%s\") at port \"%s\".\n"), + db, PQuser(pset.db), host, hostaddr, PQport(pset.db)); + else + printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), + db, PQuser(pset.db), host, PQport(pset.db)); + +/* + * Validate backup files. + */ +void +pgBackupValidate(pgBackup *backup) +{ + char base_path[MAXPGPATH]; + char path[MAXPGPATH]; + parray *files; + bool corrupted = false; + bool validation_isok = true; + /* arrays with meta info for multi threaded validate */ + pthread_t *threads; + validate_files_arg *threads_args; + int i; + + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && + backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_CORRUPT) + { + elog(WARNING, "Backup %s has status %s. Skip validation.", + base36enc(backup->start_time), status2str(backup->status)); + corrupted_backup_found = true; + return; + } + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + /* backups in MERGING status must have an option of revalidation without losing MERGING status + else if (backup->status == BACKUP_STATUS_MERGING) + { + some message here + } + */ + else + elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + + if (backup->backup_mode != BACKUP_MODE_FULL && + backup->backup_mode != BACKUP_MODE_DIFF_PAGE && + backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && + backup->backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + + pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + files = dir_read_file_list(base_path, path); + + /* setup threads */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + pg_atomic_clear_flag(&file->lock); + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (validate_files_arg *) + palloc(sizeof(validate_files_arg) * num_threads); + + /* Validate files */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + arg->files = files; + arg->corrupted = false; + arg->stop_lsn = backup->stop_lsn; + arg->checksum_version = backup->checksum_version; + arg->backup_version = parse_program_version(backup->program_version); + /* By default there are some error */ + threads_args[i].ret = 1; + + pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + validate_files_arg *arg = &(threads_args[i]); + + pthread_join(threads[i], NULL); + if (arg->corrupted) + corrupted = true; + if (arg->ret == 1) + validation_isok = false; + } + if (!validation_isok) + elog(ERROR, "Data files validation failed"); + + pfree(threads); + pfree(threads_args); + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); + + /* Update backup status */ + backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; + write_backup_status(backup); + + if (corrupted) + elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + else + elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); +} + +/* + * Validate files in the backup. + * NOTE: If file is not valid, do not use ERROR log message, + * rather throw a WARNING and set arguments->corrupted = true. + * This is necessary to update backup status. + */ +static void * +pgBackupValidateFiles(void *arg) +{ + int i; + validate_files_arg *arguments = (validate_files_arg *)arg; + pg_crc32 crc; + + for (i = 0; i < parray_num(arguments->files); i++) + { + struct stat st; + pgFile *file = (pgFile *) parray_get(arguments->files, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + if (interrupted) + elog(ERROR, "Interrupted during validate"); + + /* Validate only regular files */ + if (!S_ISREG(file->mode)) + continue; + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + continue; + + /* + * Currently we don't compute checksums for + * cfs_compressed data files, so skip them. + */ + if (file->is_cfs) + continue; + + /* print progress */ + elog(VERBOSE, "Validate files: (%d/%lu) %s", + i + 1, (unsigned long) parray_num(arguments->files), file->path); + + if (stat(file->path, &st) == -1) + { + if (errno == ENOENT) + elog(WARNING, "Backup file \"%s\" is not found", file->path); + else + elog(WARNING, "Cannot stat backup file \"%s\": %s", + file->path, strerror(errno)); + arguments->corrupted = true; + break; + } + + if (file->write_size != st.st_size) + { + elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", + file->path, file->write_size, (unsigned long) st.st_size); + arguments->corrupted = true; + break; + } + + /* + * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we + * use CRC-32. + * + * pg_control stores its content and checksum of the content, calculated + * using CRC-32C. If we calculate checksum of the whole pg_control using + * CRC-32C we get same checksum constantly. It might be because of the + * CRC-32C algorithm. + * To avoid this problem we need to use different algorithm, CRC-32 in + * this case. + */ + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; + + /* validate relation blocks */ + if (file->is_datafile) + { + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) + arguments->corrupted = true; + } + } + } + + /* Data files validation is successful */ + arguments->ret = 0; + + return NULL; +} + +/* + * Validate all backups in the backup catalog. + * If --instance option was provided, validate only backups of this instance. + */ +int +do_validate_all(void) +{ + if (instance_name == NULL) + { + /* Show list of instances */ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + + errno = 0; + while ((dent = readdir(dir))) + { + char child[MAXPGPATH]; + struct stat st; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance_name = dent->d_name; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + xlog_seg_size = get_config_xlog_seg_size(); + + do_validate_instance(); + } + } + else + { + do_validate_instance(); + } + + if (corrupted_backup_found) + { + elog(WARNING, "Some backups are not valid"); + return 1; + } + else + elog(INFO, "All backups are valid"); + + return 0; +} + +/* + * Validate all backups in the given instance of the backup catalog. + */ +static void +do_validate_instance(void) +{ + char *current_backup_id; + int i; + int j; + parray *backups; + pgBackup *current_backup = NULL; + + elog(INFO, "Validate backups of the instance '%s'", instance_name); + + /* Get exclusive lock of backup catalog */ + catalog_lock(); + + /* Get list of all backups sorted in order of descending start time */ + backups = catalog_get_backup_list(INVALID_BACKUP_ID); + + /* Examine backups one by one and validate them */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *base_full_backup; + char *parent_backup_id; + + current_backup = (pgBackup *) parray_get(backups, i); + + /* Find ancestor for incremental backup */ + if (current_backup->backup_mode != BACKUP_MODE_FULL) + { + pgBackup *tmp_backup = NULL; + int result; + + result = scan_parent_chain(current_backup, &tmp_backup); + + /* chain is broken */ + if (result == 0) + { + /* determine missing backup ID */ + + parent_backup_id = base36enc_dup(tmp_backup->parent_backup); + corrupted_backup_found = true; + + /* orphanize current_backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + current_backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(current_backup); + elog(WARNING, "Backup %s is orphaned because his parent %s is missing", + base36enc(current_backup->start_time), + parent_backup_id); + } + else + { + elog(WARNING, "Backup %s has missing parent %s", + base36enc(current_backup->start_time), parent_backup_id); + } + continue; + } + /* chain is whole, but at least one parent is invalid */ + else if (result == 1) + { + /* determine corrupt backup ID */ + parent_backup_id = base36enc_dup(tmp_backup->start_time); + + /* Oldest corrupt backup has a chance for revalidation */ + if (current_backup->start_time != tmp_backup->start_time) + { + /* orphanize current_backup */ + if (current_backup->status == BACKUP_STATUS_OK) + { + current_backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(current_backup); + elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", + base36enc(current_backup->start_time), parent_backup_id, + status2str(tmp_backup->status)); + } + else + { + elog(WARNING, "Backup %s has parent %s with status: %s", + base36enc(current_backup->start_time),parent_backup_id, + status2str(tmp_backup->status)); + } + continue; + } + base_full_backup = find_parent_full_backup(current_backup); + } + /* chain is whole, all parents are valid at first glance, + * current backup validation can proceed + */ + else + base_full_backup = tmp_backup; + } + else + base_full_backup = current_backup; + + /* Valiate backup files*/ + pgBackupValidate(current_backup); + + /* Validate corresponding WAL files */ + if (current_backup->status == BACKUP_STATUS_OK) + validate_wal(current_backup, arclog_path, 0, + 0, 0, base_full_backup->tli, xlog_seg_size); + + /* + * Mark every descendant of corrupted backup as orphan + */ + if (current_backup->status == BACKUP_STATUS_CORRUPT) + { + /* This is ridiculous but legal. + * PAGE1_2b <- OK + * PAGE1_2a <- OK + * PAGE1_1b <- ORPHAN + * PAGE1_1a <- CORRUPT + * FULL1 <- OK + */ + + corrupted_backup_found = true; + current_backup_id = base36enc_dup(current_backup->start_time); + + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (is_parent(current_backup->start_time, backup, false)) + { + if (backup->status == BACKUP_STATUS_OK) + { + backup->status = BACKUP_STATUS_ORPHAN; + write_backup_status(backup); + + elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", + base36enc(backup->start_time), + current_backup_id, + status2str(current_backup->status)); + } + } + } + free(current_backup_id); + } + + /* For every OK backup we try to revalidate all his ORPHAN descendants. */ + if (current_backup->status == BACKUP_STATUS_OK) + { + /* revalidate all ORPHAN descendats + * be very careful not to miss a missing backup + * for every backup we must check that he is descendant of current_backup + */ + for (j = i - 1; j >= 0; j--) + { + pgBackup *backup = (pgBackup *) parray_get(backups, j); + pgBackup *tmp_backup = NULL; + int result; + + //PAGE3b ORPHAN + //PAGE2b ORPHAN ----- + //PAGE6a ORPHAN | + //PAGE5a CORRUPT | + //PAGE4a missing | + //PAGE3a missing | + //PAGE2a ORPHAN | + //PAGE1a OK <- we are here <-| + //FULL OK + + if (is_parent(current_backup->start_time, backup, false)) + { + /* Revalidation make sense only if parent chain is whole. + * is_parent() do not guarantee that. + */ + result = scan_parent_chain(backup, &tmp_backup); + + if (result == 1) + { + /* revalidation make sense only if oldest invalid backup is current_backup + */ + + if (tmp_backup->start_time != backup->start_time) + continue; + + if (backup->status == BACKUP_STATUS_ORPHAN) + { + /* Revaliate backup files*/ + pgBackupValidate(backup); + + if (backup->status == BACKUP_STATUS_OK) + { + //tmp_backup = find_parent_full_backup(dest_backup); + /* Revalidation successful, validate corresponding WAL files */ + validate_wal(backup, arclog_path, 0, + 0, 0, current_backup->tli, + xlog_seg_size); + } + } + + if (backup->status != BACKUP_STATUS_OK) + { + corrupted_backup_found = true; + continue; + } + } + } + } + } + } + + /* cleanup */ + parray_walk(backups, pgBackupFree); + parray_free(backups); +} From cf6e7445e5a39876da304fdab38c13a682f11d5c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 15:51:58 +0300 Subject: [PATCH 0039/2107] Remove .autosave --- src/validate.c.autosave | 532 ---------------------------------------- 1 file changed, 532 deletions(-) delete mode 100644 src/validate.c.autosave diff --git a/src/validate.c.autosave b/src/validate.c.autosave deleted file mode 100644 index 1c07e0aeb..000000000 --- a/src/validate.c.autosave +++ /dev/null @@ -1,532 +0,0 @@ -/*------------------------------------------------------------------------- - * - * validate.c: validate backup files. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -#include "utils/thread.h" - -static void *pgBackupValidateFiles(void *arg); -static void do_validate_instance(void); - -static bool corrupted_backup_found = false; - -typedef struct -{ - parray *files; - bool corrupted; - XLogRecPtr stop_lsn; - uint32 checksum_version; - uint32 backup_version; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} validate_files_arg; - -if (is_absolute_path(host)) - if (hostaddr && *hostaddr) - printf(_("You are connected to database \"%s\" as user \"%s\" on address \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), hostaddr, PQport(pset.db)); - else - printf(_("You are connected to database \"%s\" as user \"%s\" via socket in \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), host, PQport(pset.db)); -else - if (hostaddr && *hostaddr && strcmp(host, hostaddr) != 0) - printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" (address \"%s\") at port \"%s\".\n"), - db, PQuser(pset.db), host, hostaddr, PQport(pset.db)); - else - printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), - db, PQuser(pset.db), host, PQport(pset.db)); - -/* - * Validate backup files. - */ -void -pgBackupValidate(pgBackup *backup) -{ - char base_path[MAXPGPATH]; - char path[MAXPGPATH]; - parray *files; - bool corrupted = false; - bool validation_isok = true; - /* arrays with meta info for multi threaded validate */ - pthread_t *threads; - validate_files_arg *threads_args; - int i; - - /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE && - backup->status != BACKUP_STATUS_ORPHAN && - backup->status != BACKUP_STATUS_CORRUPT) - { - elog(WARNING, "Backup %s has status %s. Skip validation.", - base36enc(backup->start_time), status2str(backup->status)); - corrupted_backup_found = true; - return; - } - - if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) - elog(INFO, "Validating backup %s", base36enc(backup->start_time)); - /* backups in MERGING status must have an option of revalidation without losing MERGING status - else if (backup->status == BACKUP_STATUS_MERGING) - { - some message here - } - */ - else - elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); - - if (backup->backup_mode != BACKUP_MODE_FULL && - backup->backup_mode != BACKUP_MODE_DIFF_PAGE && - backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && - backup->backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); - - pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, path); - - /* setup threads */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - pg_atomic_clear_flag(&file->lock); - } - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (validate_files_arg *) - palloc(sizeof(validate_files_arg) * num_threads); - - /* Validate files */ - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - arg->files = files; - arg->corrupted = false; - arg->stop_lsn = backup->stop_lsn; - arg->checksum_version = backup->checksum_version; - arg->backup_version = parse_program_version(backup->program_version); - /* By default there are some error */ - threads_args[i].ret = 1; - - pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); - } - - /* Wait theads */ - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - pthread_join(threads[i], NULL); - if (arg->corrupted) - corrupted = true; - if (arg->ret == 1) - validation_isok = false; - } - if (!validation_isok) - elog(ERROR, "Data files validation failed"); - - pfree(threads); - pfree(threads_args); - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); - - /* Update backup status */ - backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; - write_backup_status(backup); - - if (corrupted) - elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); - else - elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); -} - -/* - * Validate files in the backup. - * NOTE: If file is not valid, do not use ERROR log message, - * rather throw a WARNING and set arguments->corrupted = true. - * This is necessary to update backup status. - */ -static void * -pgBackupValidateFiles(void *arg) -{ - int i; - validate_files_arg *arguments = (validate_files_arg *)arg; - pg_crc32 crc; - - for (i = 0; i < parray_num(arguments->files); i++) - { - struct stat st; - pgFile *file = (pgFile *) parray_get(arguments->files, i); - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - if (interrupted) - elog(ERROR, "Interrupted during validate"); - - /* Validate only regular files */ - if (!S_ISREG(file->mode)) - continue; - /* - * Skip files which has no data, because they - * haven't changed between backups. - */ - if (file->write_size == BYTES_INVALID) - continue; - - /* - * Currently we don't compute checksums for - * cfs_compressed data files, so skip them. - */ - if (file->is_cfs) - continue; - - /* print progress */ - elog(VERBOSE, "Validate files: (%d/%lu) %s", - i + 1, (unsigned long) parray_num(arguments->files), file->path); - - if (stat(file->path, &st) == -1) - { - if (errno == ENOENT) - elog(WARNING, "Backup file \"%s\" is not found", file->path); - else - elog(WARNING, "Cannot stat backup file \"%s\": %s", - file->path, strerror(errno)); - arguments->corrupted = true; - break; - } - - if (file->write_size != st.st_size) - { - elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file->path, file->write_size, (unsigned long) st.st_size); - arguments->corrupted = true; - break; - } - - /* - * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we - * use CRC-32. - * - * pg_control stores its content and checksum of the content, calculated - * using CRC-32C. If we calculate checksum of the whole pg_control using - * CRC-32C we get same checksum constantly. It might be because of the - * CRC-32C algorithm. - * To avoid this problem we need to use different algorithm, CRC-32 in - * this case. - */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); - if (crc != file->crc) - { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, file->crc, crc); - arguments->corrupted = true; - - /* validate relation blocks */ - if (file->is_datafile) - { - if (!check_file_pages(file, arguments->stop_lsn, - arguments->checksum_version, - arguments->backup_version)) - arguments->corrupted = true; - } - } - } - - /* Data files validation is successful */ - arguments->ret = 0; - - return NULL; -} - -/* - * Validate all backups in the backup catalog. - * If --instance option was provided, validate only backups of this instance. - */ -int -do_validate_all(void) -{ - if (instance_name == NULL) - { - /* Show list of instances */ - char path[MAXPGPATH]; - DIR *dir; - struct dirent *dent; - - /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir = opendir(path); - if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); - - errno = 0; - while ((dent = readdir(dir))) - { - char child[MAXPGPATH]; - struct stat st; - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - instance_name = dent->d_name; - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); - xlog_seg_size = get_config_xlog_seg_size(); - - do_validate_instance(); - } - } - else - { - do_validate_instance(); - } - - if (corrupted_backup_found) - { - elog(WARNING, "Some backups are not valid"); - return 1; - } - else - elog(INFO, "All backups are valid"); - - return 0; -} - -/* - * Validate all backups in the given instance of the backup catalog. - */ -static void -do_validate_instance(void) -{ - char *current_backup_id; - int i; - int j; - parray *backups; - pgBackup *current_backup = NULL; - - elog(INFO, "Validate backups of the instance '%s'", instance_name); - - /* Get exclusive lock of backup catalog */ - catalog_lock(); - - /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(INVALID_BACKUP_ID); - - /* Examine backups one by one and validate them */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *base_full_backup; - char *parent_backup_id; - - current_backup = (pgBackup *) parray_get(backups, i); - - /* Find ancestor for incremental backup */ - if (current_backup->backup_mode != BACKUP_MODE_FULL) - { - pgBackup *tmp_backup = NULL; - int result; - - result = scan_parent_chain(current_backup, &tmp_backup); - - /* chain is broken */ - if (result == 0) - { - /* determine missing backup ID */ - - parent_backup_id = base36enc_dup(tmp_backup->parent_backup); - corrupted_backup_found = true; - - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) - { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); - elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(current_backup->start_time), - parent_backup_id); - } - else - { - elog(WARNING, "Backup %s has missing parent %s", - base36enc(current_backup->start_time), parent_backup_id); - } - continue; - } - /* chain is whole, but at least one parent is invalid */ - else if (result == 1) - { - /* determine corrupt backup ID */ - parent_backup_id = base36enc_dup(tmp_backup->start_time); - - /* Oldest corrupt backup has a chance for revalidation */ - if (current_backup->start_time != tmp_backup->start_time) - { - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) - { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(current_backup->start_time), parent_backup_id, - status2str(tmp_backup->status)); - } - else - { - elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(current_backup->start_time),parent_backup_id, - status2str(tmp_backup->status)); - } - continue; - } - base_full_backup = find_parent_full_backup(current_backup); - } - /* chain is whole, all parents are valid at first glance, - * current backup validation can proceed - */ - else - base_full_backup = tmp_backup; - } - else - base_full_backup = current_backup; - - /* Valiate backup files*/ - pgBackupValidate(current_backup); - - /* Validate corresponding WAL files */ - if (current_backup->status == BACKUP_STATUS_OK) - validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli, xlog_seg_size); - - /* - * Mark every descendant of corrupted backup as orphan - */ - if (current_backup->status == BACKUP_STATUS_CORRUPT) - { - /* This is ridiculous but legal. - * PAGE1_2b <- OK - * PAGE1_2a <- OK - * PAGE1_1b <- ORPHAN - * PAGE1_1a <- CORRUPT - * FULL1 <- OK - */ - - corrupted_backup_found = true; - current_backup_id = base36enc_dup(current_backup->start_time); - - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(current_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK) - { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); - - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - current_backup_id, - status2str(current_backup->status)); - } - } - } - free(current_backup_id); - } - - /* For every OK backup we try to revalidate all his ORPHAN descendants. */ - if (current_backup->status == BACKUP_STATUS_OK) - { - /* revalidate all ORPHAN descendats - * be very careful not to miss a missing backup - * for every backup we must check that he is descendant of current_backup - */ - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - pgBackup *tmp_backup = NULL; - int result; - - //PAGE3b ORPHAN - //PAGE2b ORPHAN ----- - //PAGE6a ORPHAN | - //PAGE5a CORRUPT | - //PAGE4a missing | - //PAGE3a missing | - //PAGE2a ORPHAN | - //PAGE1a OK <- we are here <-| - //FULL OK - - if (is_parent(current_backup->start_time, backup, false)) - { - /* Revalidation make sense only if parent chain is whole. - * is_parent() do not guarantee that. - */ - result = scan_parent_chain(backup, &tmp_backup); - - if (result == 1) - { - /* revalidation make sense only if oldest invalid backup is current_backup - */ - - if (tmp_backup->start_time != backup->start_time) - continue; - - if (backup->status == BACKUP_STATUS_ORPHAN) - { - /* Revaliate backup files*/ - pgBackupValidate(backup); - - if (backup->status == BACKUP_STATUS_OK) - { - //tmp_backup = find_parent_full_backup(dest_backup); - /* Revalidation successful, validate corresponding WAL files */ - validate_wal(backup, arclog_path, 0, - 0, 0, current_backup->tli, - xlog_seg_size); - } - } - - if (backup->status != BACKUP_STATUS_OK) - { - corrupted_backup_found = true; - continue; - } - } - } - } - } - } - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); -} From a0ec849b4a0fdf6a746a8e923fe566d1abca0a02 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 8 Nov 2018 18:13:42 +0300 Subject: [PATCH 0040/2107] We should check only negative return values --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index d3e48e9a4..eeb2952dc 100644 --- a/src/data.c +++ b/src/data.c @@ -69,7 +69,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, { int32 ret; ret = zlib_compress(dst, dst_size, src, src_size, level); - if (ret != Z_OK && errormsg) + if (ret < Z_OK && errormsg) *errormsg = zError(ret); return ret; } @@ -99,7 +99,7 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, { int32 ret; ret = zlib_decompress(dst, dst_size, src, src_size); - if (ret != Z_OK && errormsg) + if (ret < Z_OK && errormsg) *errormsg = zError(ret); return ret; } From c530d28869d9865fd98d13c0b723db5f3d88779c Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Nov 2018 19:38:22 +0300 Subject: [PATCH 0041/2107] change datafile validation algorithm: - validate file block by block by default, not only in case of file-level checksum corruption; - add an option: --skip-block-validation to disable this behaviour; - calculate file checksum at the same time as validate blocks; --- src/data.c | 22 ++++++++++++++++++-- src/dir.c | 30 ++++---------------------- src/help.c | 11 +++++++++- src/pg_probackup.c | 3 +++ src/pg_probackup.h | 27 ++++++++++++++++++++++-- src/validate.c | 52 +++++++++++++++++++++++++++------------------- 6 files changed, 93 insertions(+), 52 deletions(-) diff --git a/src/data.c b/src/data.c index eeb2952dc..4fc53783b 100644 --- a/src/data.c +++ b/src/data.c @@ -1601,12 +1601,14 @@ validate_one_page(Page page, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, - uint32 backup_version) +check_file_pages(pgFile *file, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version) { size_t read_len = 0; bool is_valid = true; FILE *in; + pg_crc32 crc; + bool use_crc32c = (backup_version <= 20021); elog(VERBOSE, "validate relation blocks for file %s", file->name); @@ -1623,6 +1625,9 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, file->path, strerror(errno)); } + /* calc CRC of backup file */ + INIT_FILE_CRC32(use_crc32c, crc); + /* read and validate pages one by one */ while (true) { @@ -1647,6 +1652,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, blknum, file->path, strerror(errno_tmp)); } + COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); + if (header.block < blknum) elog(ERROR, "backup is broken at file->path %s block %u", file->path, blknum); @@ -1668,6 +1675,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); + COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + if (header.compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) @@ -1706,5 +1715,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, } } + FIN_FILE_CRC32(use_crc32c, crc); + + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + is_valid = false; + } + return is_valid; } diff --git a/src/dir.c b/src/dir.c index 9e55c8217..5c6f60d82 100644 --- a/src/dir.c +++ b/src/dir.c @@ -267,28 +267,6 @@ pgFileGetCRC(const char *file_path, bool use_crc32c) size_t len; int errno_tmp; -#define INIT_FILE_CRC32(crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - /* open file in binary read mode */ fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) @@ -296,20 +274,20 @@ do { \ file_path, strerror(errno)); /* calc CRC of backup file */ - INIT_FILE_CRC32(crc); + INIT_FILE_CRC32(use_crc32c, crc); while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); - COMP_FILE_CRC32(crc, buf, len); + COMP_FILE_CRC32(use_crc32c, crc, buf, len); } errno_tmp = errno; if (!feof(fp)) elog(WARNING, "cannot read \"%s\": %s", file_path, strerror(errno_tmp)); if (len > 0) - COMP_FILE_CRC32(crc, buf, len); - FIN_FILE_CRC32(crc); + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + FIN_FILE_CRC32(use_crc32c, crc); fclose(fp); diff --git a/src/help.c b/src/help.c index 1cf6d4041..409c8f825 100644 --- a/src/help.c +++ b/src/help.c @@ -118,6 +118,7 @@ help_pg_probackup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); @@ -127,12 +128,14 @@ help_pg_probackup(void) printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); @@ -203,7 +206,8 @@ help_backup(void) printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -215,6 +219,7 @@ help_backup(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -279,6 +284,7 @@ help_restore(void) printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -305,6 +311,7 @@ help_restore(void) printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); printf(_(" --no-validate disable backup validation during restore\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -335,6 +342,7 @@ help_validate(void) printf(_(" [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline]\n\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -348,6 +356,7 @@ help_validate(void) printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 82107097c..ea99672dc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -89,6 +89,8 @@ static pgRecoveryTarget *recovery_target_options = NULL; bool restore_as_replica = false; bool restore_no_validate = false; +bool skip_block_validation = false; + /* delete options */ bool delete_wal = false; bool delete_expired = false; @@ -179,6 +181,7 @@ static pgut_option options[] = { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, + { 'b', 29, "skip-block-validation", &skip_block_validation, SOURCE_CMDLINE }, /* delete options */ { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 182a647bf..c6ff63435 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -65,6 +65,28 @@ typedef enum CompressAlg ZLIB_COMPRESS, } CompressAlg; +#define INIT_FILE_CRC32(use_crc32c, crc) \ +do { \ + if (use_crc32c) \ + INIT_CRC32C(crc); \ + else \ + INIT_TRADITIONAL_CRC32(crc); \ +} while (0) +#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ +do { \ + if (use_crc32c) \ + COMP_CRC32C((crc), (data), (len)); \ + else \ + COMP_TRADITIONAL_CRC32(crc, data, len); \ +} while (0) +#define FIN_FILE_CRC32(use_crc32c, crc) \ +do { \ + if (use_crc32c) \ + FIN_CRC32C(crc); \ + else \ + FIN_TRADITIONAL_CRC32(crc); \ +} while (0) + /* Information about single file (or dir) in backup */ typedef struct pgFile { @@ -339,6 +361,7 @@ extern bool exclusive_backup; /* restore options */ extern bool restore_as_replica; +extern bool skip_block_validation; /* delete options */ extern bool delete_wal; @@ -527,9 +550,9 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern bool calc_file_checksum(pgFile *file); -extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn, +extern bool check_file_pages(pgFile* file, + XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); - /* parsexlog.c */ extern void extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, diff --git a/src/validate.c b/src/validate.c index 2f75cd4b2..55ea5ebe4 100644 --- a/src/validate.c +++ b/src/validate.c @@ -208,32 +208,42 @@ pgBackupValidateFiles(void *arg) } /* - * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we - * use CRC-32. - * - * pg_control stores its content and checksum of the content, calculated - * using CRC-32C. If we calculate checksum of the whole pg_control using - * CRC-32C we get same checksum constantly. It might be because of the - * CRC-32C algorithm. - * To avoid this problem we need to use different algorithm, CRC-32 in - * this case. + * If option skip-block-validation is set, compute only file-level CRC for + * datafiles, otherwise check them block by block. */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); - if (crc != file->crc) + if (!file->is_datafile || skip_block_validation) { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, file->crc, crc); - arguments->corrupted = true; - - /* validate relation blocks */ - if (file->is_datafile) + /* + * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we + * use CRC-32. + * + * pg_control stores its content and checksum of the content, calculated + * using CRC-32C. If we calculate checksum of the whole pg_control using + * CRC-32C we get same checksum constantly. It might be because of the + * CRC-32C algorithm. + * To avoid this problem we need to use different algorithm, CRC-32 in + * this case. + */ + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); + if (crc != file->crc) { - if (!check_file_pages(file, arguments->stop_lsn, - arguments->checksum_version, - arguments->backup_version)) - arguments->corrupted = true; + elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + file->path, file->crc, crc); + arguments->corrupted = true; } } + else + { + /* + * validate relation block by block + * check page headers, checksums (if enabled) + * and compute checksum of the file + */ + if (!check_file_pages(file, arguments->stop_lsn, + arguments->checksum_version, + arguments->backup_version)) + arguments->corrupted = true; + } } /* Data files validation is successful */ From 141ba33000d21fc3b5f78e9678552b6acf28f40d Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Nov 2018 20:49:30 +0300 Subject: [PATCH 0042/2107] fix: close file after validation --- src/data.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data.c b/src/data.c index 4fc53783b..1d1a2ee9e 100644 --- a/src/data.c +++ b/src/data.c @@ -1716,6 +1716,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, } FIN_FILE_CRC32(use_crc32c, crc); + fclose(in); if (crc != file->crc) { From 4870fa7f68728cb51bc81f3d4f74cd53b1dd76f6 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 9 Nov 2018 13:59:56 +0300 Subject: [PATCH 0043/2107] more informative error message for failed WAL record read --- src/parsexlog.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f7365f5c..ee7b5076f 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -237,10 +237,11 @@ doExtractPageMap(void *arg) */ if (XLogRecPtrIsInvalid(found)) { - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X. %s", private_data->thread_num, (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); + (uint32) (extract_arg->startpoint), + (xlogreader->errormsg_buf[0] != '\0')?xlogreader->errormsg_buf:""); PrintXLogCorruptionMsg(private_data, ERROR); } extract_arg->startpoint = found; From 77c59158beab1648e0884aa1a6f4aadedac040c3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 1 Nov 2018 19:10:20 +0300 Subject: [PATCH 0044/2107] PGPRO-2071: Refactor pg_probackup configure set/get, use InstanceConfig --- Makefile | 4 +- gen_probackup_project.pl | 3 +- msvs/template.pg_probackup.vcxproj | 2 + msvs/template.pg_probackup96.vcxproj | 2 + msvs/template.pg_probackup_2.vcxproj | 2 + src/archive.c | 13 +- src/backup.c | 111 +- src/catalog.c | 6 +- src/configure.c | 652 +++++------ src/data.c | 5 +- src/delete.c | 35 +- src/dir.c | 12 +- src/init.c | 28 +- src/merge.c | 6 +- src/parsexlog.c | 4 +- src/pg_probackup.c | 222 ++-- src/pg_probackup.h | 60 +- src/restore.c | 30 +- src/show.c | 26 +- src/util.c | 36 +- src/utils/configuration.c | 1499 ++++++++++++++++++++++++++ src/utils/configuration.h | 106 ++ src/utils/json.c | 20 +- src/utils/json.h | 5 +- src/utils/logger.c | 91 +- src/utils/logger.h | 39 +- src/utils/pgut.c | 1376 +---------------------- src/utils/pgut.h | 91 +- src/validate.c | 16 +- 29 files changed, 2291 insertions(+), 2211 deletions(-) create mode 100644 src/utils/configuration.c create mode 100644 src/utils/configuration.h diff --git a/Makefile b/Makefile index 55d4428c0..482fa2e96 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ PROGRAM = pg_probackup # utils -OBJS = src/utils/json.o src/utils/logger.o src/utils/parray.o \ - src/utils/pgut.o src/utils/thread.o +OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ + src/utils/parray.o src/utils/pgut.o src/utils/thread.o OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index e14f1d4be..7c21b8d43 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -149,7 +149,8 @@ sub build_pgprobackup 'validate.c' ); $probackup->AddFiles( - 'contrib/pg_probackup/src/utils', + 'contrib/pg_probackup/src/utils', + 'configuration.c', 'json.c', 'logger.c', 'parray.c', diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj index a0a3c7a8d..6b0cc1e33 100644 --- a/msvs/template.pg_probackup.vcxproj +++ b/msvs/template.pg_probackup.vcxproj @@ -187,6 +187,7 @@ + @@ -201,6 +202,7 @@ + diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj index 3c62734ea..3606e404e 100644 --- a/msvs/template.pg_probackup96.vcxproj +++ b/msvs/template.pg_probackup96.vcxproj @@ -186,6 +186,7 @@ + @@ -199,6 +200,7 @@ + diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj index 1f103ac8d..cd8ce31ec 100644 --- a/msvs/template.pg_probackup_2.vcxproj +++ b/msvs/template.pg_probackup_2.vcxproj @@ -182,6 +182,7 @@ + @@ -194,6 +195,7 @@ + diff --git a/src/archive.c b/src/archive.c index 2953b89e8..3333f0961 100644 --- a/src/archive.c +++ b/src/archive.c @@ -28,7 +28,6 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; int64 system_id; - pgBackupConfig *config; bool is_compress = false; if (wal_file_name == NULL && wal_file_path == NULL) @@ -44,16 +43,16 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) elog(ERROR, "getcwd() error"); /* verify that archive-push --instance parameter is valid */ - config = readBackupCatalogConfigFile(); system_id = get_system_identifier(current_dir); - if (config->pgdata == NULL) + if (instance_config.pgdata == NULL) elog(ERROR, "cannot read pg_probackup.conf for this instance"); - if(system_id != config->system_identifier) + if(system_id != instance_config.system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", - wal_file_name, instance_name, config->system_identifier, system_id); + wal_file_name, instance_name, instance_config.system_identifier, + system_id); /* Create 'archlog_path' directory. Do nothing if it already exists. */ dir_create_dir(arclog_path, DIR_PERMISSION); @@ -63,11 +62,11 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); - if (compress_alg == PGLZ_COMPRESS) + if (instance_config.compress_alg == PGLZ_COMPRESS) elog(ERROR, "pglz compression is not supported"); #ifdef HAVE_LIBZ - if (compress_alg == ZLIB_COMPRESS) + if (instance_config.compress_alg == ZLIB_COMPRESS) is_compress = IsXLogFileName(wal_file_name); #endif diff --git a/src/backup.c b/src/backup.c index b2aed3186..d4e1bbf0f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -150,7 +150,10 @@ get_remote_pgdata_filelist(parray *files) int resultStatus; int i; - backup_conn_replication = pgut_connect_replication(pgut_dbname); + backup_conn_replication = pgut_connect_replication(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) elog(ERROR,"%s: could not send replication command \"%s\": %s", @@ -397,7 +400,10 @@ remote_backup_files(void *arg) if (!pg_atomic_test_set_flag(&file->lock)) continue; - file_backup_conn = pgut_connect_replication(pgut_dbname); + file_backup_conn = pgut_connect_replication(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); /* check for interrupt */ if (interrupted) @@ -487,16 +493,19 @@ do_backup_instance(void) TimeLineID starttli; XLogRecPtr startpos; - backup_conn_replication = pgut_connect_replication(pgut_dbname); + backup_conn_replication = pgut_connect_replication(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); /* Check replication prorocol connection */ if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) elog(ERROR, "Failed to send command for remote backup"); // TODO implement the check -// if (&sysidentifier != system_identifier) +// if (&sysidentifier != instance_config.system_identifier) // elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", -// system_identifier, sysidentifier); +// instance_config.system_identifier, sysidentifier); current.tli = starttli; @@ -579,7 +588,10 @@ do_backup_instance(void) /* * Connect in replication mode to the server. */ - stream_thread_arg.conn = pgut_connect_replication(pgut_dbname); + stream_thread_arg.conn = pgut_connect_replication(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) { @@ -616,7 +628,8 @@ do_backup_instance(void) if (is_remote_backup) get_remote_pgdata_filelist(backup_files_list); else - dir_list_file(backup_files_list, pgdata, true, true, false); + dir_list_file(backup_files_list, instance_config.pgdata, + true, true, false); /* * Sort pathname ascending. It is necessary to create intermediate @@ -632,7 +645,7 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileComparePath); /* Extract information about files in backup_list parsing their names:*/ - parse_backup_filelist_filenames(backup_files_list, pgdata); + parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -653,7 +666,7 @@ do_backup_instance(void) * reading WAL segments present in archives up to the point * where this backup has started. */ - extractPageMap(arclog_path, current.tli, xlog_seg_size, + extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, prev_backup->start_lsn, current.start_lsn, backup_files_list); } @@ -680,7 +693,7 @@ do_backup_instance(void) char database_path[MAXPGPATH]; if (!is_remote_backup) - dir_name = GetRelativePath(file->path, pgdata); + dir_name = GetRelativePath(file->path, instance_config.pgdata); else dir_name = file->path; @@ -710,7 +723,7 @@ do_backup_instance(void) { backup_files_arg *arg = &(threads_args[i]); - arg->from_root = pgdata; + arg->from_root = instance_config.pgdata; arg->to_root = database_path; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; @@ -790,7 +803,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - pgBackupWriteFileList(¤t, backup_files_list, pgdata); + pgBackupWriteFileList(¤t, backup_files_list, instance_config.pgdata); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -816,9 +829,8 @@ do_backup_instance(void) int do_backup(time_t start_time) { - /* PGDATA and BACKUP_MODE are always required */ - if (pgdata == NULL) + if (instance_config.pgdata == NULL) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); if (current.backup_mode == BACKUP_MODE_INVALID) @@ -826,7 +838,9 @@ do_backup(time_t start_time) "(-b, --backup-mode)"); /* Create connection for PostgreSQL */ - backup_conn = pgut_connect(pgut_dbname); + backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); pgut_atexit_push(backup_disconnect, NULL); current.primary_conninfo = pgut_get_conninfo_string(backup_conn); @@ -836,8 +850,8 @@ do_backup(time_t start_time) elog(ERROR, "Failed to retreive wal_segment_size"); #endif - current.compress_alg = compress_alg; - current.compress_level = compress_level; + current.compress_alg = instance_config.compress_alg; + current.compress_level = instance_config.compress_level; /* Confirm data block size and xlog block size are compatible */ confirm_block_size("block_size", BLCKSZ); @@ -886,11 +900,14 @@ do_backup(time_t start_time) if (current.from_replica) { /* Check master connection options */ - if (master_host == NULL) + if (instance_config.master_host == NULL) elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); /* Create connection to master server */ - master_conn = pgut_connect_extended(master_host, master_port, master_db, master_user); + master_conn = pgut_connect(instance_config.master_host, + instance_config.master_port, + instance_config.master_db, + instance_config.master_user); } /* Get exclusive lock of backup catalog */ @@ -929,9 +946,9 @@ do_backup(time_t start_time) /* compute size of wal files of this backup stored in the archive */ if (!current.stream) { - current.wal_bytes = xlog_seg_size * - (current.stop_lsn / xlog_seg_size - - current.start_lsn / xlog_seg_size + 1); + current.wal_bytes = instance_config.xlog_seg_size * + (current.stop_lsn / instance_config.xlog_seg_size - + current.start_lsn / instance_config.xlog_seg_size + 1); } /* Backup is done. Update backup status */ @@ -1038,17 +1055,17 @@ check_system_identifiers(void) uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata); + system_id_pgdata = get_system_identifier(instance_config.pgdata); system_id_conn = get_remote_system_identifier(backup_conn); - if (system_id_conn != system_identifier) + if (system_id_conn != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but connected instance system id is " UINT64_FORMAT, - system_identifier, system_id_conn); - if (system_id_pgdata != system_identifier) + instance_config.system_identifier, system_id_conn); + if (system_id_pgdata != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but target backup directory system id is " UINT64_FORMAT, - system_identifier, system_id_pgdata); + instance_config.system_identifier, system_id_pgdata); } /* @@ -1283,7 +1300,9 @@ pg_ptrack_clear(void) dbOid = atoi(PQgetvalue(res_db, i, 1)); tblspcOid = atoi(PQgetvalue(res_db, i, 2)); - tmp_conn = pgut_connect(dbname); + tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); PQclear(res); @@ -1399,7 +1418,9 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, return NULL; } - tmp_conn = pgut_connect(dbname); + tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); sprintf(params[0], "%i", tablespace_oid); sprintf(params[1], "%i", rel_filenode); res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", @@ -1482,10 +1503,11 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) tli = get_current_timeline(false); /* Compute the name of the WAL file containig requested LSN */ - GetXLogSegNo(lsn, targetSegNo, xlog_seg_size); + GetXLogSegNo(lsn, targetSegNo, instance_config.xlog_seg_size); if (wait_prev_segment) targetSegNo--; - GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size); + GetXLogFileName(wal_segment, tli, targetSegNo, + instance_config.xlog_seg_size); /* * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is @@ -1509,7 +1531,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { join_path_components(wal_segment_path, arclog_path, wal_segment); wal_segment_dir = arclog_path; - timeout = archive_timeout; + timeout = instance_config.archive_timeout; } if (wait_prev_segment) @@ -1552,7 +1574,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) /* * A WAL segment found. Check LSN on it. */ - if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) + if (wal_contains_lsn(wal_segment_dir, lsn, tli, + instance_config.xlog_seg_size)) /* Target LSN was found */ { elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); @@ -1650,11 +1673,12 @@ wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) elog(INFO, "Wait for target LSN %X/%X to be received by replica", (uint32) (lsn >> 32), (uint32) lsn); - if (replica_timeout > 0 && try_count > replica_timeout) + if (instance_config.replica_timeout > 0 && + try_count > instance_config.replica_timeout) elog(ERROR, "Target LSN %X/%X could not be recevied by replica " "in %d seconds", (uint32) (lsn >> 32), (uint32) lsn, - replica_timeout); + instance_config.replica_timeout); } } @@ -1962,7 +1986,8 @@ pg_stop_backup(pgBackup *backup) elog(LOG, "Getting the Recovery Time from WAL"); - if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, + if (!read_recovery_info(xlog_path, backup->tli, + instance_config.xlog_seg_size, backup->start_lsn, backup->stop_lsn, &backup->recovery_time, &backup->recovery_xid)) { @@ -2134,7 +2159,8 @@ backup_files(void *arg) if (!backup_data_file(arguments, to_path, file, arguments->prev_start_lsn, current.backup_mode, - compress_alg, compress_level)) + instance_config.compress_alg, + instance_config.compress_level)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); @@ -2336,9 +2362,9 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) rel_path = relpathperm(rnode, forknum); if (segno > 0) - path = psprintf("%s/%s.%u", pgdata, rel_path, segno); + path = psprintf("%s/%s.%u", instance_config.pgdata, rel_path, segno); else - path = psprintf("%s/%s", pgdata, rel_path); + path = psprintf("%s/%s", instance_config.pgdata, rel_path); pg_free(rel_path); @@ -2578,7 +2604,7 @@ StreamLog(void *arg) /* * Always start streaming at the beginning of a segment */ - startpos -= startpos % xlog_seg_size; + startpos -= startpos % instance_config.xlog_seg_size; /* Initialize timeout */ stream_stop_timeout = 0; @@ -2692,7 +2718,10 @@ pg_ptrack_get_block(backup_files_arg *arguments, if (arguments->backup_conn == NULL) { - arguments->backup_conn = pgut_connect(pgut_dbname); + arguments->backup_conn = pgut_connect(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); } if (arguments->cancel_conn == NULL) diff --git a/src/catalog.c b/src/catalog.c index 41676f4c9..307e4751f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -15,6 +15,8 @@ #include #include +#include "utils/configuration.h" + static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); @@ -559,7 +561,7 @@ readBackupControlFile(const char *path) char *compress_alg = NULL; int parsed_options; - pgut_option options[] = + ConfigOption options[] = { {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, @@ -594,7 +596,7 @@ readBackupControlFile(const char *path) return NULL; } - parsed_options = pgut_readopt(path, options, WARNING, true); + parsed_options = config_read_opt(path, options, WARNING, true); if (parsed_options == 0) { diff --git a/src/configure.c b/src/configure.c index 6d3cf9cb6..7637761d8 100644 --- a/src/configure.c +++ b/src/configure.c @@ -9,209 +9,213 @@ #include "pg_probackup.h" +#include "utils/configuration.h" #include "utils/json.h" -static void opt_log_level_console(pgut_option *opt, const char *arg); -static void opt_log_level_file(pgut_option *opt, const char *arg); -static void opt_compress_alg(pgut_option *opt, const char *arg); +static void assign_log_level_console(ConfigOption *opt, const char *arg); +static void assign_log_level_file(ConfigOption *opt, const char *arg); +static void assign_compress_alg(ConfigOption *opt, const char *arg); + +static char *get_log_level_console(ConfigOption *opt); +static char *get_log_level_file(ConfigOption *opt); +static char *get_compress_alg(ConfigOption *opt); static void show_configure_start(void); static void show_configure_end(void); -static void show_configure(pgBackupConfig *config); -static void show_configure_json(pgBackupConfig *config); +static void show_configure_plain(ConfigOption *opt); +static void show_configure_json(ConfigOption *opt); -static pgBackupConfig *cur_config = NULL; +#define ARCHIVE_TIMEOUT_DEFAULT 300 +#define REPLICA_TIMEOUT_DEFAULT 300 -static PQExpBufferData show_buf; -static int32 json_level = 0; +#define RETENTION_REDUNDANCY_DEFAULT 0 +#define RETENTION_WINDOW_DEFAULT 0 + +#define OPTION_INSTANCE_GROUP "Backup instance information" +#define OPTION_CONN_GROUP "Connection parameters" +#define OPTION_REPLICA_GROUP "Replica parameters" +#define OPTION_ARCHIVE_GROUP "Archive parameters" +#define OPTION_LOG_GROUP "Logging parameters" +#define OPTION_RETENTION_GROUP "Retention parameters" +#define OPTION_COMPRESS_GROUP "Compression parameters" /* - * All this code needs refactoring. + * Short name should be non-printable ASCII character. */ - -/* Set configure options */ -int -do_configure(bool show_only) -{ - pgBackupConfig *config = readBackupCatalogConfigFile(); - if (pgdata) - config->pgdata = pgdata; - if (pgut_dbname) - config->pgdatabase = pgut_dbname; - if (host) - config->pghost = host; - if (port) - config->pgport = port; - if (username) - config->pguser = username; - - if (master_host) - config->master_host = master_host; - if (master_port) - config->master_port = master_port; - if (master_db) - config->master_db = master_db; - if (master_user) - config->master_user = master_user; - - if (replica_timeout) - config->replica_timeout = replica_timeout; - - if (archive_timeout) - config->archive_timeout = archive_timeout; - - if (log_level_console) - config->log_level_console = log_level_console; - if (log_level_file) - config->log_level_file = log_level_file; - if (log_filename) - config->log_filename = log_filename; - if (error_log_filename) - config->error_log_filename = error_log_filename; - if (log_directory) - config->log_directory = log_directory; - if (log_rotation_size) - config->log_rotation_size = log_rotation_size; - if (log_rotation_age) - config->log_rotation_age = log_rotation_age; - - if (retention_redundancy) - config->retention_redundancy = retention_redundancy; - if (retention_window) - config->retention_window = retention_window; - - if (compress_alg) - config->compress_alg = compress_alg; - if (compress_level) - config->compress_level = compress_level; - - if (show_only) - show_configure(config); - else - writeBackupCatalogConfigFile(config); - - return 0; -} - -void -pgBackupConfigInit(pgBackupConfig *config) +ConfigOption instance_options[] = { - config->system_identifier = 0; - + /* Instance options */ + { + 's', 'D', "pgdata", + &instance_config.pgdata, SOURCE_CMD, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + { + 'U', 200, "system-identifier", + &instance_config.system_identifier, SOURCE_FILE_STRICT, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, #if PG_VERSION_NUM >= 110000 - config->xlog_seg_size = 0; -#else - config->xlog_seg_size = XLOG_SEG_SIZE; + { + 'u', 201, "xlog-seg-size", + &instance_config.xlog_seg_size, SOURCE_FILE_STRICT, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, #endif + /* Connection options */ + { + 's', 'd', "pgdatabase", + &instance_config.pgdatabase, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'h', "pghost", + &instance_config.pghost, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'p', "pgport", + &instance_config.pgport, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'U', "pguser", + &instance_config.pguser, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + /* Replica options */ + { + 's', 202, "master-db", + &instance_config.master_db, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 203, "master-host", + &instance_config.master_host, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 204, "master-port", + &instance_config.master_port, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 205, "master-user", + &instance_config.master_user, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 'u', 206, "replica-timeout", + &instance_config.replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value + }, + /* Archive options */ + { + 'u', 207, "archive-timeout", + &instance_config.archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value + }, + /* Logging options */ + { + 'f', 208, "log-level-console", + assign_log_level_console, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, get_log_level_console + }, + { + 'f', 209, "log-level-file", + assign_log_level_file, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, get_log_level_file + }, + { + 's', 210, "log-filename", + &instance_config.logger.log_filename, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 211, "error-log-filename", + &instance_config.logger.error_log_filename, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 212, "log-directory", + &instance_config.logger.log_directory, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 'U', 213, "log-rotation-size", + &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value + }, + { + 'U', 214, "log-rotation-age", + &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value + }, + /* Retention options */ + { + 'u', 215, "retention-redundancy", + &instance_config.retention_redundancy, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, + { + 'u', 216, "retention-window", + &instance_config.retention_window, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, + /* Compression options */ + { + 'f', 217, "compress-algorithm", + assign_compress_alg, SOURCE_CMD, 0, + OPTION_COMPRESS_GROUP, 0, get_compress_alg + }, + { + 'u', 218, "compress-level", + &instance_config.compress_level, SOURCE_CMD, 0, + OPTION_COMPRESS_GROUP, 0, option_get_value + }, + { 0 } +}; - config->pgdata = NULL; - config->pgdatabase = NULL; - config->pghost = NULL; - config->pgport = NULL; - config->pguser = NULL; - - config->master_host = NULL; - config->master_port = NULL; - config->master_db = NULL; - config->master_user = NULL; - config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - - config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; - - config->log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; - config->log_level_file = LOG_LEVEL_FILE_DEFAULT; - config->log_filename = LOG_FILENAME_DEFAULT; - config->error_log_filename = NULL; - config->log_directory = LOG_DIRECTORY_DEFAULT; - config->log_rotation_size = LOG_ROTATION_SIZE_DEFAULT; - config->log_rotation_age = LOG_ROTATION_AGE_DEFAULT; - - config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; - config->retention_window = RETENTION_WINDOW_DEFAULT; +/* An instance configuration with default options */ +InstanceConfig instance_config; - config->compress_alg = COMPRESS_ALG_DEFAULT; - config->compress_level = COMPRESS_LEVEL_DEFAULT; -} +static PQExpBufferData show_buf; +static int32 json_level = 0; +static const char *current_group = NULL; +/* + * Show configure options including default values. + */ void -writeBackupCatalogConfig(FILE *out, pgBackupConfig *config) +do_show_config(void) { - uint64 res; - const char *unit; + int i; - fprintf(out, "#Backup instance info\n"); - fprintf(out, "PGDATA = %s\n", config->pgdata); - fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier); -#if PG_VERSION_NUM >= 110000 - fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size); -#endif - - fprintf(out, "#Connection parameters:\n"); - if (config->pgdatabase) - fprintf(out, "PGDATABASE = %s\n", config->pgdatabase); - if (config->pghost) - fprintf(out, "PGHOST = %s\n", config->pghost); - if (config->pgport) - fprintf(out, "PGPORT = %s\n", config->pgport); - if (config->pguser) - fprintf(out, "PGUSER = %s\n", config->pguser); - - fprintf(out, "#Replica parameters:\n"); - if (config->master_host) - fprintf(out, "master-host = %s\n", config->master_host); - if (config->master_port) - fprintf(out, "master-port = %s\n", config->master_port); - if (config->master_db) - fprintf(out, "master-db = %s\n", config->master_db); - if (config->master_user) - fprintf(out, "master-user = %s\n", config->master_user); - - convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_MS, - &res, &unit); - fprintf(out, "replica-timeout = " UINT64_FORMAT "%s\n", res, unit); - - fprintf(out, "#Archive parameters:\n"); - convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_MS, - &res, &unit); - fprintf(out, "archive-timeout = " UINT64_FORMAT "%s\n", res, unit); - - fprintf(out, "#Logging parameters:\n"); - fprintf(out, "log-level-console = %s\n", deparse_log_level(config->log_level_console)); - fprintf(out, "log-level-file = %s\n", deparse_log_level(config->log_level_file)); - fprintf(out, "log-filename = %s\n", config->log_filename); - if (config->error_log_filename) - fprintf(out, "error-log-filename = %s\n", config->error_log_filename); - - if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) - fprintf(out, "log-directory = %s/%s\n", backup_path, config->log_directory); - else - fprintf(out, "log-directory = %s\n", config->log_directory); - /* Convert values from base unit */ - convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, - &res, &unit); - fprintf(out, "log-rotation-size = " UINT64_FORMAT "%s\n", res, (res)?unit:"KB"); - - convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_MS, - &res, &unit); - fprintf(out, "log-rotation-age = " UINT64_FORMAT "%s\n", res, (res)?unit:"min"); - - fprintf(out, "#Retention parameters:\n"); - fprintf(out, "retention-redundancy = %u\n", config->retention_redundancy); - fprintf(out, "retention-window = %u\n", config->retention_window); + show_configure_start(); - fprintf(out, "#Compression parameters:\n"); + for (i = 0; instance_options[i].type; i++) + { + if (show_format == SHOW_PLAIN) + show_configure_plain(&instance_options[i]); + else + show_configure_json(&instance_options[i]); + } - fprintf(out, "compress-algorithm = %s\n", deparse_compress_alg(config->compress_alg)); - fprintf(out, "compress-level = %d\n", config->compress_level); + show_configure_end(); } +/* + * Save configure options into BACKUP_CATALOG_CONF_FILE. Do not save default + * values into the file. + */ void -writeBackupCatalogConfigFile(pgBackupConfig *config) +do_set_config(void) { char path[MAXPGPATH]; FILE *fp; + int i; join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); fp = fopen(path, "wt"); @@ -219,114 +223,99 @@ writeBackupCatalogConfigFile(pgBackupConfig *config) elog(ERROR, "cannot create %s: %s", BACKUP_CATALOG_CONF_FILE, strerror(errno)); - writeBackupCatalogConfig(fp, config); + current_group = NULL; + + for (i = 0; instance_options[i].type; i++) + { + ConfigOption *opt = &instance_options[i]; + char *value; + + /* Save only options from command line */ + if (opt->source != SOURCE_CMD && + /* ...or options from the previous configure file */ + opt->source != SOURCE_FILE && opt->source != SOURCE_FILE_STRICT) + continue; + + value = opt->get_value(opt); + if (value == NULL) + continue; + + if (current_group == NULL || strcmp(opt->group, current_group) != 0) + { + current_group = opt->group; + fprintf(fp, "# %s\n", current_group); + } + + fprintf(fp, "%s = %s\n", opt->lname, value); + pfree(value); + } + fclose(fp); } - -pgBackupConfig* -readBackupCatalogConfigFile(void) +void +init_config(InstanceConfig *config) { - pgBackupConfig *config = pgut_new(pgBackupConfig); - char path[MAXPGPATH]; + MemSet(config, 0, sizeof(InstanceConfig)); - pgut_option options[] = - { - /* retention options */ - { 'u', 0, "retention-redundancy", &(config->retention_redundancy),SOURCE_FILE_STRICT }, - { 'u', 0, "retention-window", &(config->retention_window), SOURCE_FILE_STRICT }, - /* compression options */ - { 'f', 0, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, - { 'u', 0, "compress-level", &(config->compress_level), SOURCE_CMDLINE }, - /* logging options */ - { 'f', 0, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, - { 'f', 0, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, - { 's', 0, "log-filename", &(config->log_filename), SOURCE_CMDLINE }, - { 's', 0, "error-log-filename", &(config->error_log_filename), SOURCE_CMDLINE }, - { 's', 0, "log-directory", &(config->log_directory), SOURCE_CMDLINE }, - { 'U', 0, "log-rotation-size", &(config->log_rotation_size), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 0, "log-rotation-age", &(config->log_rotation_age), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, - /* connection options */ - { 's', 0, "pgdata", &(config->pgdata), SOURCE_FILE_STRICT }, - { 's', 0, "pgdatabase", &(config->pgdatabase), SOURCE_FILE_STRICT }, - { 's', 0, "pghost", &(config->pghost), SOURCE_FILE_STRICT }, - { 's', 0, "pgport", &(config->pgport), SOURCE_FILE_STRICT }, - { 's', 0, "pguser", &(config->pguser), SOURCE_FILE_STRICT }, - /* replica options */ - { 's', 0, "master-host", &(config->master_host), SOURCE_FILE_STRICT }, - { 's', 0, "master-port", &(config->master_port), SOURCE_FILE_STRICT }, - { 's', 0, "master-db", &(config->master_db), SOURCE_FILE_STRICT }, - { 's', 0, "master-user", &(config->master_user), SOURCE_FILE_STRICT }, - { 'u', 0, "replica-timeout", &(config->replica_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, - /* other options */ - { 'U', 0, "system-identifier", &(config->system_identifier), SOURCE_FILE_STRICT }, + /* + * Starting from PostgreSQL 11 WAL segment size may vary. Prior to + * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. + */ #if PG_VERSION_NUM >= 110000 - {'u', 0, "xlog-seg-size", &config->xlog_seg_size, SOURCE_FILE_STRICT}, + config->xlog_seg_size = 0; +#else + config->xlog_seg_size = XLOG_SEG_SIZE; #endif - /* archive options */ - { 'u', 0, "archive-timeout", &(config->archive_timeout), SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, - {0} - }; - cur_config = config; + config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; - pgBackupConfigInit(config); - pgut_readopt(path, options, ERROR, true); + /* Copy logger defaults */ + config->logger = logger_config; -#if PG_VERSION_NUM >= 110000 - if (!IsValidWalSegSize(config->xlog_seg_size)) - elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size); -#endif + config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; + config->retention_window = RETENTION_WINDOW_DEFAULT; - return config; + config->compress_alg = COMPRESS_ALG_DEFAULT; + config->compress_level = COMPRESS_LEVEL_DEFAULT; } -/* - * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE. - */ -uint32 -get_config_xlog_seg_size(void) +static void +assign_log_level_console(ConfigOption *opt, const char *arg) { -#if PG_VERSION_NUM >= 110000 - char path[MAXPGPATH]; - uint32 seg_size; - pgut_option options[] = - { - {'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT}, - {0} - }; - - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - pgut_readopt(path, options, ERROR, false); - - if (!IsValidWalSegSize(seg_size)) - elog(ERROR, "Invalid WAL segment size %u", seg_size); - - return seg_size; - -#else - return (uint32) XLOG_SEG_SIZE; -#endif + instance_config.logger.log_level_console = parse_log_level(arg); } static void -opt_log_level_console(pgut_option *opt, const char *arg) +assign_log_level_file(ConfigOption *opt, const char *arg) { - cur_config->log_level_console = parse_log_level(arg); + instance_config.logger.log_level_file = parse_log_level(arg); } static void -opt_log_level_file(pgut_option *opt, const char *arg) +assign_compress_alg(ConfigOption *opt, const char *arg) { - cur_config->log_level_file = parse_log_level(arg); + instance_config.compress_alg = parse_compress_alg(arg); } -static void -opt_compress_alg(pgut_option *opt, const char *arg) +static char * +get_log_level_console(ConfigOption *opt) { - cur_config->compress_alg = parse_compress_alg(arg); + return pstrdup(deparse_log_level(instance_config.logger.log_level_console)); +} + +static char * +get_log_level_file(ConfigOption *opt) +{ + return pstrdup(deparse_log_level(instance_config.logger.log_level_file)); +} + +static char * +get_compress_alg(ConfigOption *opt) +{ + return pstrdup(deparse_compress_alg(instance_config.compress_alg)); } /* @@ -335,12 +324,15 @@ opt_compress_alg(pgut_option *opt, const char *arg) static void show_configure_start(void) { - if (show_format == SHOW_PLAIN) - return; - - /* For now we need buffer only for JSON format */ - json_level = 0; initPQExpBuffer(&show_buf); + + if (show_format == SHOW_PLAIN) + current_group = NULL; + else + { + json_level = 0; + json_add(&show_buf, JT_BEGIN_OBJECT, &json_level); + } } /* @@ -350,28 +342,38 @@ static void show_configure_end(void) { if (show_format == SHOW_PLAIN) - return; + current_group = NULL; else + { + json_add(&show_buf, JT_END_OBJECT, &json_level); appendPQExpBufferChar(&show_buf, '\n'); + } fputs(show_buf.data, stdout); termPQExpBuffer(&show_buf); } /* - * Show configure information of pg_probackup. + * Plain output. */ + static void -show_configure(pgBackupConfig *config) +show_configure_plain(ConfigOption *opt) { - show_configure_start(); + char *value; - if (show_format == SHOW_PLAIN) - writeBackupCatalogConfig(stdout, config); - else - show_configure_json(config); + value = opt->get_value(opt); + if (value == NULL) + return; - show_configure_end(); + if (current_group == NULL || strcmp(opt->group, current_group) != 0) + { + current_group = opt->group; + appendPQExpBuffer(&show_buf, "# %s\n", current_group); + } + + appendPQExpBuffer(&show_buf, "%s = %s\n", opt->lname, value); + pfree(value); } /* @@ -379,109 +381,15 @@ show_configure(pgBackupConfig *config) */ static void -show_configure_json(pgBackupConfig *config) +show_configure_json(ConfigOption *opt) { - PQExpBuffer buf = &show_buf; - uint64 res; - const char *unit; + char *value; - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - json_add_value(buf, "pgdata", config->pgdata, json_level, false); - - json_add_key(buf, "system-identifier", json_level, true); - appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier); - -#if PG_VERSION_NUM >= 110000 - json_add_key(buf, "xlog-seg-size", json_level, true); - appendPQExpBuffer(buf, "%u", config->xlog_seg_size); -#endif - - /* Connection parameters */ - if (config->pgdatabase) - json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true); - if (config->pghost) - json_add_value(buf, "pghost", config->pghost, json_level, true); - if (config->pgport) - json_add_value(buf, "pgport", config->pgport, json_level, true); - if (config->pguser) - json_add_value(buf, "pguser", config->pguser, json_level, true); - - /* Replica parameters */ - if (config->master_host) - json_add_value(buf, "master-host", config->master_host, json_level, - true); - if (config->master_port) - json_add_value(buf, "master-port", config->master_port, json_level, - true); - if (config->master_db) - json_add_value(buf, "master-db", config->master_db, json_level, true); - if (config->master_user) - json_add_value(buf, "master-user", config->master_user, json_level, - true); - - json_add_key(buf, "replica-timeout", json_level, true); - convert_from_base_unit_u(config->replica_timeout, OPTION_UNIT_MS, - &res, &unit); - appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); - - /* Archive parameters */ - json_add_key(buf, "archive-timeout", json_level, true); - convert_from_base_unit_u(config->archive_timeout, OPTION_UNIT_MS, - &res, &unit); - appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, unit); - - /* Logging parameters */ - json_add_value(buf, "log-level-console", - deparse_log_level(config->log_level_console), json_level, - true); - json_add_value(buf, "log-level-file", - deparse_log_level(config->log_level_file), json_level, - true); - json_add_value(buf, "log-filename", config->log_filename, json_level, - true); - if (config->error_log_filename) - json_add_value(buf, "error-log-filename", config->error_log_filename, - json_level, true); - - if (strcmp(config->log_directory, LOG_DIRECTORY_DEFAULT) == 0) - { - char log_directory_fullpath[MAXPGPATH]; - - sprintf(log_directory_fullpath, "%s/%s", - backup_path, config->log_directory); - - json_add_value(buf, "log-directory", log_directory_fullpath, - json_level, true); - } - else - json_add_value(buf, "log-directory", config->log_directory, - json_level, true); - - json_add_key(buf, "log-rotation-size", json_level, true); - convert_from_base_unit_u(config->log_rotation_size, OPTION_UNIT_KB, - &res, &unit); - appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"KB"); - - json_add_key(buf, "log-rotation-age", json_level, true); - convert_from_base_unit_u(config->log_rotation_age, OPTION_UNIT_MS, - &res, &unit); - appendPQExpBuffer(buf, UINT64_FORMAT "%s", res, (res)?unit:"min"); - - /* Retention parameters */ - json_add_key(buf, "retention-redundancy", json_level, true); - appendPQExpBuffer(buf, "%u", config->retention_redundancy); - - json_add_key(buf, "retention-window", json_level, true); - appendPQExpBuffer(buf, "%u", config->retention_window); - - /* Compression parameters */ - json_add_value(buf, "compress-algorithm", - deparse_compress_alg(config->compress_alg), json_level, - true); - - json_add_key(buf, "compress-level", json_level, true); - appendPQExpBuffer(buf, "%d", config->compress_level); + value = opt->get_value(opt); + if (value == NULL) + return; - json_add(buf, JT_END_OBJECT, &json_level); + json_add_value(&show_buf, opt->lname, value, json_level, + opt->type == 's' || opt->flags & OPTION_UNIT); + pfree(value); } diff --git a/src/data.c b/src/data.c index 1d1a2ee9e..8d8da69eb 100644 --- a/src/data.c +++ b/src/data.c @@ -1122,9 +1122,10 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); gz_out = gzopen(to_path_temp, PG_BINARY_W); - if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) + if (gzsetparams(gz_out, instance_config.compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", - compress_level, to_path_temp, get_gz_error(gz_out, errno)); + instance_config.compress_level, to_path_temp, + get_gz_error(gz_out, errno)); to_path_p = gz_to_path; } diff --git a/src/delete.c b/src/delete.c index c5f16af7d..f32503580 100644 --- a/src/delete.c +++ b/src/delete.c @@ -109,7 +109,7 @@ do_delete(time_t backup_id) } } - delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); } /* cleanup */ @@ -125,9 +125,7 @@ int do_retention_purge(void) { parray *backup_list; - uint32 backup_num; size_t i; - time_t days_threshold = time(NULL) - (retention_window * 60 * 60 * 24); XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; bool keep_next_backup = true; /* Do not delete first full backup */ @@ -135,13 +133,13 @@ do_retention_purge(void) if (delete_expired) { - if (retention_redundancy > 0) - elog(LOG, "REDUNDANCY=%u", retention_redundancy); - if (retention_window > 0) - elog(LOG, "WINDOW=%u", retention_window); + if (instance_config.retention_redundancy > 0) + elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy); + if (instance_config.retention_window > 0) + elog(LOG, "WINDOW=%u", instance_config.retention_window); - if (retention_redundancy == 0 - && retention_window == 0) + if (instance_config.retention_redundancy == 0 + && instance_config.retention_window == 0) { elog(WARNING, "Retention policy is not set"); if (!delete_wal) @@ -162,9 +160,15 @@ do_retention_purge(void) /* Find target backups to be deleted */ if (delete_expired && - (retention_redundancy > 0 || retention_window > 0)) + (instance_config.retention_redundancy > 0 || + instance_config.retention_window > 0)) { - backup_num = 0; + time_t days_threshold; + uint32 backup_num = 0; + + days_threshold = time(NULL) - + (instance_config.retention_window * 60 * 60 * 24); + for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); @@ -182,8 +186,9 @@ do_retention_purge(void) /* Evaluate retention_redundancy if this backup is eligible for removal */ if (keep_next_backup || - retention_redundancy >= backup_num_evaluate + 1 || - (retention_window > 0 && backup->recovery_time >= days_threshold)) + instance_config.retention_redundancy >= backup_num_evaluate + 1 || + (instance_config.retention_window > 0 && + backup->recovery_time >= days_threshold)) { /* Save LSN and Timeline to remove unnecessary WAL segments */ oldest_lsn = backup->start_lsn; @@ -226,7 +231,7 @@ do_retention_purge(void) /* Purge WAL files */ if (delete_wal) { - delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size); + delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); } /* Cleanup */ @@ -439,7 +444,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size); + delete_walfiles(InvalidXLogRecPtr, 0, instance_config.xlog_seg_size); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); diff --git a/src/dir.c b/src/dir.c index 5c6f60d82..241ddb5f1 100644 --- a/src/dir.c +++ b/src/dir.c @@ -19,6 +19,8 @@ #include #include +#include "utils/configuration.h" + /* * The contents of these directories are removed or recreated during server * start so they are not included in backups. The directories themselves are @@ -379,7 +381,8 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ - if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) + if (root && instance_config.pgdata && + strcmp(root, instance_config.pgdata) == 0 && fileExists(path)) { FILE *black_list_file = NULL; char buf[MAXPGPATH * 2]; @@ -393,7 +396,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, while (fgets(buf, lengthof(buf), black_list_file) != NULL) { - join_path_components(black_item, pgdata, buf); + join_path_components(black_item, instance_config.pgdata, buf); if (black_item[strlen(black_item) - 1] == '\n') black_item[strlen(black_item) - 1] = '\0'; @@ -844,7 +847,7 @@ get_tablespace_created(const char *link) * Copy of function tablespace_list_append() from pg_basebackup.c. */ void -opt_tablespace_map(pgut_option *opt, const char *arg) +opt_tablespace_map(ConfigOption *opt, const char *arg) { TablespaceListCell *cell = pgut_new(TablespaceListCell); char *dst; @@ -1121,7 +1124,8 @@ check_tablespace_mapping(pgBackup *backup) /* Sort links by the path of a linked file*/ parray_qsort(links, pgFileCompareLinked); - if (log_level_console <= LOG || log_level_file <= LOG) + if (logger_config.log_level_console <= LOG || + logger_config.log_level_file <= LOG) elog(LOG, "check tablespace directories of backup %s", base36enc(backup->start_time)); diff --git a/src/init.c b/src/init.c index 75b50e4f0..fb9b7bbb2 100644 --- a/src/init.c +++ b/src/init.c @@ -21,7 +21,7 @@ do_init(void) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; - int results; + int results; results = pg_check_dir(backup_path); if (results == 4) /* exists and not empty*/ @@ -54,17 +54,16 @@ do_add_instance(void) char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; struct stat st; - pgBackupConfig *config = pgut_new(pgBackupConfig); /* PGDATA is always required */ - if (pgdata == NULL) + if (instance_config.pgdata == NULL) elog(ERROR, "Required parameter not specified: PGDATA " "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - system_identifier = get_system_identifier(pgdata); + instance_config.system_identifier = get_system_identifier(instance_config.pgdata); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ - xlog_seg_size = get_xlog_seg_size(pgdata); + instance_config.xlog_seg_size = get_xlog_seg_size(instance_config.pgdata); /* Ensure that all root directories already exist */ if (access(backup_path, F_OK) != 0) @@ -93,14 +92,19 @@ do_add_instance(void) dir_create_dir(arclog_path, DIR_PERMISSION); /* - * Wite initial config. system-identifier and pgdata are set in - * init subcommand and will never be updated. + * Write initial configuration file. + * system-identifier, xlog-seg-size and pgdata are set in init subcommand + * and will never be updated. + * + * We need to manually set options source to save them to the configuration + * file. */ - pgBackupConfigInit(config); - config->system_identifier = system_identifier; - config->xlog_seg_size = xlog_seg_size; - config->pgdata = pgdata; - writeBackupCatalogConfigFile(config); + config_set_opt(instance_options, &instance_config.system_identifier, + SOURCE_FILE); + config_set_opt(instance_options, &instance_config.xlog_seg_size, + SOURCE_FILE); + /* pgdata was set through command line */ + do_set_config(); elog(INFO, "Instance '%s' successfully inited", instance_name); return 0; diff --git a/src/merge.c b/src/merge.c index 137f1acda..08150dc5a 100644 --- a/src/merge.c +++ b/src/merge.c @@ -319,9 +319,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_backup->wal_bytes = xlog_seg_size * - (to_backup->stop_lsn / xlog_seg_size - - to_backup->start_lsn / xlog_seg_size + 1); + to_backup->wal_bytes = instance_config.xlog_seg_size * + (to_backup->stop_lsn / instance_config.xlog_seg_size - + to_backup->start_lsn / instance_config.xlog_seg_size + 1); else to_backup->wal_bytes = BYTES_INVALID; diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f7365f5c..9b33537cd 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -227,7 +227,7 @@ doExtractPageMap(void *arg) #endif if (xlogreader == NULL) elog(ERROR, "Thread [%d]: out of memory", private_data->thread_num); - xlogreader->system_identifier = system_identifier; + xlogreader->system_identifier = instance_config.system_identifier; found = XLogFindNextRecord(xlogreader, extract_arg->startpoint); @@ -976,7 +976,7 @@ InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, #endif if (xlogreader == NULL) elog(ERROR, "out of memory"); - xlogreader->system_identifier = system_identifier; + xlogreader->system_identifier = instance_config.system_identifier; } return xlogreader; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ea99672dc..f3ff76c78 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -15,6 +15,7 @@ #include +#include "utils/configuration.h" #include "utils/thread.h" const char *PROGRAM_VERSION = "2.0.23"; @@ -41,7 +42,6 @@ typedef enum ProbackupSubcmd /* directory options */ char *backup_path = NULL; -char *pgdata = NULL; /* * path or to the data files in the backup catalog * $BACKUP_PATH/backups/instance_name @@ -66,13 +66,6 @@ char *replication_slot = NULL; bool backup_logs = false; bool smooth_checkpoint; bool is_remote_backup = false; -/* Wait timeout for WAL segment archiving */ -uint32 archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; -const char *master_db = NULL; -const char *master_host = NULL; -const char *master_port= NULL; -const char *master_user = NULL; -uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; /* restore options */ static char *target_time; @@ -94,32 +87,13 @@ bool skip_block_validation = false; /* delete options */ bool delete_wal = false; bool delete_expired = false; -bool apply_to_all = false; bool force_delete = false; -/* retention options */ -uint32 retention_redundancy = 0; -uint32 retention_window = 0; - /* compression options */ -CompressAlg compress_alg = COMPRESS_ALG_DEFAULT; -int compress_level = COMPRESS_LEVEL_DEFAULT; bool compress_shortcut = false; - /* other options */ char *instance_name; -uint64 system_identifier = 0; - -/* - * Starting from PostgreSQL 11 WAL segment size may vary. Prior to - * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. - */ -#if PG_VERSION_NUM >= 110000 -uint32 xlog_seg_size = 0; -#else -uint32 xlog_seg_size = XLOG_SEG_SIZE; -#endif /* archive push options */ static char *wal_file_path; @@ -135,93 +109,64 @@ static ProbackupSubcmd backup_subcmd = NO_CMD; static bool help_opt = false; -static void opt_backup_mode(pgut_option *opt, const char *arg); -static void opt_log_level_console(pgut_option *opt, const char *arg); -static void opt_log_level_file(pgut_option *opt, const char *arg); -static void opt_compress_alg(pgut_option *opt, const char *arg); -static void opt_show_format(pgut_option *opt, const char *arg); +static void opt_backup_mode(ConfigOption *opt, const char *arg); +static void opt_show_format(ConfigOption *opt, const char *arg); static void compress_init(void); -static pgut_option options[] = +/* + * Short name should be non-printable ASCII character. + */ +static ConfigOption cmd_options[] = { /* directory options */ - { 'b', 1, "help", &help_opt, SOURCE_CMDLINE }, - { 's', 'D', "pgdata", &pgdata, SOURCE_CMDLINE }, - { 's', 'B', "backup-path", &backup_path, SOURCE_CMDLINE }, + { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, + { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, /* common options */ - { 'u', 'j', "threads", &num_threads, SOURCE_CMDLINE }, - { 'b', 2, "stream", &stream_wal, SOURCE_CMDLINE }, - { 'b', 3, "progress", &progress, SOURCE_CMDLINE }, - { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMDLINE }, + { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, + { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, + { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, + { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, /* backup options */ - { 'b', 10, "backup-pg-log", &backup_logs, SOURCE_CMDLINE }, - { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMDLINE }, - { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMDLINE }, - { 's', 'S', "slot", &replication_slot, SOURCE_CMDLINE }, - { 'u', 11, "archive-timeout", &archive_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, - { 'b', 12, "delete-wal", &delete_wal, SOURCE_CMDLINE }, - { 'b', 13, "delete-expired", &delete_expired, SOURCE_CMDLINE }, - { 's', 14, "master-db", &master_db, SOURCE_CMDLINE, }, - { 's', 15, "master-host", &master_host, SOURCE_CMDLINE, }, - { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, - { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, - { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, + { 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, + { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, + { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, + { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, + { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, + { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, /* TODO not completed feature. Make it unavailiable from user level - { 'b', 18, "remote", &is_remote_backup, SOURCE_CMDLINE, }, */ + { 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */ /* restore options */ - { 's', 20, "time", &target_time, SOURCE_CMDLINE }, - { 's', 21, "xid", &target_xid, SOURCE_CMDLINE }, - { 's', 22, "inclusive", &target_inclusive, SOURCE_CMDLINE }, - { 'u', 23, "timeline", &target_tli, SOURCE_CMDLINE }, - { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMDLINE }, - { 'b', 24, "immediate", &target_immediate, SOURCE_CMDLINE }, - { 's', 25, "recovery-target-name", &target_name, SOURCE_CMDLINE }, - { 's', 26, "recovery-target-action", &target_action, SOURCE_CMDLINE }, - { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE }, - { 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE }, - { 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE }, - { 'b', 29, "skip-block-validation", &skip_block_validation, SOURCE_CMDLINE }, + { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, + { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, + { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, + { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, + { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, + { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, + { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, + { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, + { 'b', 143, "no-validate", &restore_no_validate, SOURCE_CMD_STRICT }, + { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, + { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, /* delete options */ - { 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE }, - { 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE }, - { 'b', 132, "all", &apply_to_all, SOURCE_CMDLINE }, + { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, + { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, /* TODO not implemented yet */ - { 'b', 133, "force", &force_delete, SOURCE_CMDLINE }, - /* retention options */ - { 'u', 134, "retention-redundancy", &retention_redundancy, SOURCE_CMDLINE }, - { 'u', 135, "retention-window", &retention_window, SOURCE_CMDLINE }, + { 'b', 147, "force", &force_delete, SOURCE_CMD_STRICT }, /* compression options */ - { 'f', 136, "compress-algorithm", opt_compress_alg, SOURCE_CMDLINE }, - { 'u', 137, "compress-level", &compress_level, SOURCE_CMDLINE }, - { 'b', 138, "compress", &compress_shortcut, SOURCE_CMDLINE }, - /* logging options */ - { 'f', 140, "log-level-console", opt_log_level_console, SOURCE_CMDLINE }, - { 'f', 141, "log-level-file", opt_log_level_file, SOURCE_CMDLINE }, - { 's', 142, "log-filename", &log_filename, SOURCE_CMDLINE }, - { 's', 143, "error-log-filename", &error_log_filename, SOURCE_CMDLINE }, - { 's', 144, "log-directory", &log_directory, SOURCE_CMDLINE }, - { 'U', 145, "log-rotation-size", &log_rotation_size, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_KB }, - { 'U', 146, "log-rotation-age", &log_rotation_age, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_MS }, + { 'b', 148, "compress", &compress_shortcut, SOURCE_CMD_STRICT }, /* connection options */ - { 's', 'd', "pgdatabase", &pgut_dbname, SOURCE_CMDLINE }, - { 's', 'h', "pghost", &host, SOURCE_CMDLINE }, - { 's', 'p', "pgport", &port, SOURCE_CMDLINE }, - { 's', 'U', "pguser", &username, SOURCE_CMDLINE }, - { 'B', 'w', "no-password", &prompt_password, SOURCE_CMDLINE }, - { 'b', 'W', "password", &force_password, SOURCE_CMDLINE }, + { 'B', 'w', "no-password", &prompt_password, SOURCE_CMD_STRICT }, + { 'b', 'W', "password", &force_password, SOURCE_CMD_STRICT }, /* other options */ - { 'U', 150, "system-identifier", &system_identifier, SOURCE_FILE_STRICT }, - { 's', 151, "instance", &instance_name, SOURCE_CMDLINE }, -#if PG_VERSION_NUM >= 110000 - { 'u', 152, "xlog-seg-size", &xlog_seg_size, SOURCE_FILE_STRICT}, -#endif + { 's', 149, "instance", &instance_name, SOURCE_CMD_STRICT }, /* archive-push options */ - { 's', 160, "wal-file-path", &wal_file_path, SOURCE_CMDLINE }, - { 's', 161, "wal-file-name", &wal_file_name, SOURCE_CMDLINE }, - { 'b', 162, "overwrite", &file_overwrite, SOURCE_CMDLINE }, + { 's', 150, "wal-file-path", &wal_file_path, SOURCE_CMD_STRICT }, + { 's', 151, "wal-file-name", &wal_file_name, SOURCE_CMD_STRICT }, + { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ - { 'f', 170, "format", opt_show_format, SOURCE_CMDLINE }, + { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, { 0 } }; @@ -237,9 +182,12 @@ main(int argc, char *argv[]) struct stat stat_buf; int rc; - /* initialize configuration */ + /* Initialize current backup */ pgBackupInit(¤t); + /* Initialize current instance configuration */ + init_config(&instance_config); + PROGRAM_NAME = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "pgscripts"); @@ -352,8 +300,10 @@ main(int argc, char *argv[]) } optind += 1; - /* Parse command line arguments */ - pgut_getopt(argc, argv, options); + /* Parse command line only arguments */ + config_get_opt(argc, argv, cmd_options, instance_options); + + pgut_init(); if (help_opt) help_command(command_name); @@ -424,34 +374,36 @@ main(int argc, char *argv[]) * Read options from env variables or from config file, * unless we're going to set them via set-config. */ - if (instance_name && backup_subcmd != SET_CONFIG_CMD) + if (instance_name) { char path[MAXPGPATH]; /* Read environment variables */ - pgut_getopt_env(options); + config_get_opt_env(instance_options); /* Read options from configuration file */ join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - pgut_readopt(path, options, ERROR, true); + config_read_opt(path, instance_options, ERROR, true); } /* Initialize logger */ - init_logger(backup_path); + init_logger(backup_path, &instance_config.logger); /* * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. */ - if (pgdata != NULL && !is_absolute_path(pgdata)) + if (instance_config.pgdata != NULL && + !is_absolute_path(instance_config.pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); #if PG_VERSION_NUM >= 110000 /* Check xlog-seg-size option */ if (instance_name && backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != ADD_INSTANCE_CMD && !IsValidWalSegSize(xlog_seg_size)) - elog(ERROR, "Invalid WAL segment size %u", xlog_seg_size); + backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != SET_CONFIG_CMD && + !IsValidWalSegSize(instance_config.xlog_seg_size)) + elog(ERROR, "Invalid WAL segment size %u", instance_config.xlog_seg_size); #endif /* Sanity check of --backup-id option */ @@ -471,12 +423,12 @@ main(int argc, char *argv[]) } /* Setup stream options. They are used in streamutil.c. */ - if (host != NULL) - dbhost = pstrdup(host); - if (port != NULL) - dbport = pstrdup(port); - if (username != NULL) - dbuser = pstrdup(username); + if (instance_config.pghost != NULL) + dbhost = pstrdup(instance_config.pghost); + if (instance_config.pgport != NULL) + dbport = pstrdup(instance_config.pgport); + if (instance_config.pguser != NULL) + dbuser = pstrdup(instance_config.pguser); /* setup exclusion list for file search */ if (!backup_logs) @@ -559,9 +511,11 @@ main(int argc, char *argv[]) do_merge(current.backup_id); break; case SHOW_CONFIG_CMD: - return do_configure(true); + do_show_config(); + break; case SET_CONFIG_CMD: - return do_configure(false); + do_set_config(); + break; case NO_CMD: /* Should not happen */ elog(ERROR, "Unknown subcommand"); @@ -571,25 +525,13 @@ main(int argc, char *argv[]) } static void -opt_backup_mode(pgut_option *opt, const char *arg) +opt_backup_mode(ConfigOption *opt, const char *arg) { current.backup_mode = parse_backup_mode(arg); } static void -opt_log_level_console(pgut_option *opt, const char *arg) -{ - log_level_console = parse_log_level(arg); -} - -static void -opt_log_level_file(pgut_option *opt, const char *arg) -{ - log_level_file = parse_log_level(arg); -} - -static void -opt_show_format(pgut_option *opt, const char *arg) +opt_show_format(ConfigOption *opt, const char *arg) { const char *v = arg; size_t len; @@ -612,12 +554,6 @@ opt_show_format(pgut_option *opt, const char *arg) elog(ERROR, "Invalid show format \"%s\"", arg); } -static void -opt_compress_alg(pgut_option *opt, const char *arg) -{ - compress_alg = parse_compress_alg(arg); -} - /* * Initialize compress and sanity checks for compress. */ @@ -626,20 +562,20 @@ compress_init(void) { /* Default algorithm is zlib */ if (compress_shortcut) - compress_alg = ZLIB_COMPRESS; + instance_config.compress_alg = ZLIB_COMPRESS; if (backup_subcmd != SET_CONFIG_CMD) { - if (compress_level != COMPRESS_LEVEL_DEFAULT - && compress_alg == NOT_DEFINED_COMPRESS) + if (instance_config.compress_level != COMPRESS_LEVEL_DEFAULT + && instance_config.compress_alg == NOT_DEFINED_COMPRESS) elog(ERROR, "Cannot specify compress-level option without compress-alg option"); } - if (compress_level < 0 || compress_level > 9) + if (instance_config.compress_level < 0 || instance_config.compress_level > 9) elog(ERROR, "--compress-level value must be in the range from 0 to 9"); - if (compress_level == 0) - compress_alg = NOT_DEFINED_COMPRESS; + if (instance_config.compress_level == 0) + instance_config.compress_alg = NOT_DEFINED_COMPRESS; if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { @@ -648,7 +584,7 @@ compress_init(void) elog(ERROR, "This build does not support zlib compression"); else #endif - if (compress_alg == PGLZ_COMPRESS && num_threads > 1) + if (instance_config.compress_alg == PGLZ_COMPRESS && num_threads > 1) elog(ERROR, "Multithread backup does not support pglz compression"); } } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c6ff63435..314b734d7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -24,6 +24,7 @@ #include "port/atomics.h" #endif +#include "utils/configuration.h" #include "utils/logger.h" #include "utils/parray.h" #include "utils/pgut.h" @@ -158,7 +159,11 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) #define BLOCKNUM_INVALID (-1) -typedef struct pgBackupConfig +/* + * An instance configuration. It can be stored in a configuration file or passed + * from command line. + */ +typedef struct InstanceConfig { uint64 system_identifier; uint32 xlog_seg_size; @@ -173,24 +178,24 @@ typedef struct pgBackupConfig const char *master_port; const char *master_db; const char *master_user; - int replica_timeout; + uint32 replica_timeout; - int archive_timeout; + /* Wait timeout for WAL segment archiving */ + uint32 archive_timeout; - int log_level_console; - int log_level_file; - char *log_filename; - char *error_log_filename; - char *log_directory; - uint64 log_rotation_size; - uint64 log_rotation_age; + /* Logger parameters */ + LoggerConfig logger; + /* Retention options. 0 disables the option. */ uint32 retention_redundancy; uint32 retention_window; CompressAlg compress_alg; int compress_level; -} pgBackupConfig; +} InstanceConfig; + +extern ConfigOption instance_options[]; +extern InstanceConfig instance_config; typedef struct pgBackup pgBackup; @@ -331,7 +336,6 @@ typedef struct /* directory options */ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; -extern char *pgdata; extern char arclog_path[MAXPGPATH]; /* common options */ @@ -345,15 +349,7 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; -#define ARCHIVE_TIMEOUT_DEFAULT 300 -extern uint32 archive_timeout; extern bool is_remote_backup; -extern const char *master_db; -extern const char *master_host; -extern const char *master_port; -extern const char *master_user; -#define REPLICA_TIMEOUT_DEFAULT 300 -extern uint32 replica_timeout; extern bool is_ptrack_support; extern bool is_checksum_enabled; @@ -366,25 +362,13 @@ extern bool skip_block_validation; /* delete options */ extern bool delete_wal; extern bool delete_expired; -extern bool apply_to_all; extern bool force_delete; -/* retention options. 0 disables the option */ -#define RETENTION_REDUNDANCY_DEFAULT 0 -#define RETENTION_WINDOW_DEFAULT 0 - -extern uint32 retention_redundancy; -extern uint32 retention_window; - /* compression options */ -extern CompressAlg compress_alg; -extern int compress_level; extern bool compress_shortcut; /* other options */ extern char *instance_name; -extern uint64 system_identifier; -extern uint32 xlog_seg_size; /* show options */ extern ShowFormat show_format; @@ -435,13 +419,9 @@ extern int do_archive_get(char *wal_file_path, char *wal_file_name); /* in configure.c */ -extern int do_configure(bool show_only); -extern void pgBackupConfigInit(pgBackupConfig *config); -extern void writeBackupCatalogConfig(FILE *out, pgBackupConfig *config); -extern void writeBackupCatalogConfigFile(pgBackupConfig *config); -extern pgBackupConfig* readBackupCatalogConfigFile(void); - -extern uint32 get_config_xlog_seg_size(void); +extern void do_show_config(void); +extern void do_set_config(void); +extern void init_config(InstanceConfig *config); /* in show.c */ extern int do_show(time_t requested_backup_id); @@ -510,7 +490,7 @@ extern void create_data_directories(const char *data_dir, bool extract_tablespaces); extern void read_tablespace_map(parray *files, const char *backup_dir); -extern void opt_tablespace_map(pgut_option *opt, const char *arg); +extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); diff --git a/src/restore.c b/src/restore.c index 439f3c4e1..e04c60814 100644 --- a/src/restore.c +++ b/src/restore.c @@ -59,12 +59,13 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore) { - if (pgdata == NULL) + if (instance_config.pgdata == NULL) elog(ERROR, "required parameter not specified: PGDATA (-D, --pgdata)"); /* Check if restore destination empty */ - if (!dir_is_empty(pgdata)) - elog(ERROR, "restore destination is not empty: \"%s\"", pgdata); + if (!dir_is_empty(instance_config.pgdata)) + elog(ERROR, "restore destination is not empty: \"%s\"", + instance_config.pgdata); } if (instance_name == NULL) @@ -320,7 +321,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ validate_wal(dest_backup, arclog_path, rt->recovery_target_time, rt->recovery_target_xid, rt->recovery_target_lsn, - base_full_backup->tli, xlog_seg_size); + base_full_backup->tli, instance_config.xlog_seg_size); } /* Orphinize every OK descendant of corrupted backup */ else @@ -445,7 +446,7 @@ restore_backup(pgBackup *backup) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - create_data_directories(pgdata, this_backup_path, true); + create_data_directories(instance_config.pgdata, this_backup_path, true); /* * Get list of files which need to be restored. @@ -497,7 +498,8 @@ restore_backup(pgBackup *backup) parray_walk(files, pgFileFree); parray_free(files); - if (log_level_console <= LOG || log_level_file <= LOG) + if (logger_config.log_level_console <= LOG || + logger_config.log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); } @@ -517,12 +519,12 @@ remove_deleted_files(pgBackup *backup) pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(pgdata, filelist_path); + files = dir_read_file_list(instance_config.pgdata, filelist_path); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, pgdata, true, true, false); + dir_list_file(files_restored, instance_config.pgdata, true, true, false); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); @@ -534,8 +536,10 @@ remove_deleted_files(pgBackup *backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - if (log_level_console <= LOG || log_level_file <= LOG) - elog(LOG, "deleted %s", GetRelativePath(file->path, pgdata)); + if (logger_config.log_level_console <= LOG || + logger_config.log_level_file <= LOG) + elog(LOG, "deleted %s", GetRelativePath(file->path, + instance_config.pgdata)); } } @@ -617,7 +621,7 @@ restore_files(void *arg) { char to_path[MAXPGPATH]; - join_path_components(to_path, pgdata, + join_path_components(to_path, instance_config.pgdata, file->path + strlen(from_root) + 1); restore_data_file(to_path, file, arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, @@ -625,7 +629,7 @@ restore_files(void *arg) parse_program_version(arguments->backup->program_version)); } else - copy_file(from_root, pgdata, file); + copy_file(from_root, instance_config.pgdata, file); /* print size of restored file */ if (file->write_size != BYTES_INVALID) @@ -660,7 +664,7 @@ create_recovery_conf(time_t backup_id, elog(LOG, "----------------------------------------"); elog(LOG, "creating recovery.conf"); - snprintf(path, lengthof(path), "%s/recovery.conf", pgdata); + snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); fp = fopen(path, "wt"); if (fp == NULL) elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, diff --git a/src/show.c b/src/show.c index 389428952..115edf205 100644 --- a/src/show.c +++ b/src/show.c @@ -521,8 +521,8 @@ show_instance_json(parray *backup_list) /* Begin of instance object */ json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "instance", instance_name, json_level, false); - json_add_key(buf, "backups", json_level, true); + json_add_value(buf, "instance", instance_name, json_level, true); + json_add_key(buf, "backups", json_level); /* * List backups. @@ -542,7 +542,7 @@ show_instance_json(parray *backup_list) json_add(buf, JT_BEGIN_OBJECT, &json_level); json_add_value(buf, "id", base36enc(backup->start_time), json_level, - false); + true); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", @@ -558,20 +558,20 @@ show_instance_json(parray *backup_list) deparse_compress_alg(backup->compress_alg), json_level, true); - json_add_key(buf, "compress-level", json_level, true); + json_add_key(buf, "compress-level", json_level); appendPQExpBuffer(buf, "%d", backup->compress_level); json_add_value(buf, "from-replica", backup->from_replica ? "true" : "false", json_level, - true); + false); - json_add_key(buf, "block-size", json_level, true); + json_add_key(buf, "block-size", json_level); appendPQExpBuffer(buf, "%u", backup->block_size); - json_add_key(buf, "xlog-block-size", json_level, true); + json_add_key(buf, "xlog-block-size", json_level); appendPQExpBuffer(buf, "%u", backup->wal_block_size); - json_add_key(buf, "checksum-version", json_level, true); + json_add_key(buf, "checksum-version", json_level); appendPQExpBuffer(buf, "%u", backup->checksum_version); json_add_value(buf, "program-version", backup->program_version, @@ -579,10 +579,10 @@ show_instance_json(parray *backup_list) json_add_value(buf, "server-version", backup->server_version, json_level, true); - json_add_key(buf, "current-tli", json_level, true); + json_add_key(buf, "current-tli", json_level); appendPQExpBuffer(buf, "%d", backup->tli); - json_add_key(buf, "parent-tli", json_level, true); + json_add_key(buf, "parent-tli", json_level); parent_tli = get_parent_tli(backup->tli); appendPQExpBuffer(buf, "%u", parent_tli); @@ -603,7 +603,7 @@ show_instance_json(parray *backup_list) json_add_value(buf, "end-time", timestamp, json_level, true); } - json_add_key(buf, "recovery-xid", json_level, true); + json_add_key(buf, "recovery-xid", json_level); appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); if (backup->recovery_time > 0) @@ -614,13 +614,13 @@ show_instance_json(parray *backup_list) if (backup->data_bytes != BYTES_INVALID) { - json_add_key(buf, "data-bytes", json_level, true); + json_add_key(buf, "data-bytes", json_level); appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); } if (backup->wal_bytes != BYTES_INVALID) { - json_add_key(buf, "wal-bytes", json_level, true); + json_add_key(buf, "wal-bytes", json_level); appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); } diff --git a/src/util.c b/src/util.c index 8d4974ca9..8c3ea725c 100644 --- a/src/util.c +++ b/src/util.c @@ -112,7 +112,8 @@ get_current_timeline(bool safe) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, + safe); if (safe && buffer == NULL) return 0; @@ -241,7 +242,8 @@ get_data_checksum_version(bool safe) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata, "global/pg_control", &size, safe); + buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, + safe); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -250,36 +252,6 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } - -/* - * Convert time_t value to ISO-8601 format string. Always set timezone offset. - */ -void -time2iso(char *buf, size_t len, time_t time) -{ - struct tm *ptm = gmtime(&time); - time_t gmt = mktime(ptm); - time_t offset; - char *ptr = buf; - - ptm = localtime(&time); - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), "%c%02d", - (offset >= 0) ? '+' : '-', - abs((int) offset) / SECS_PER_HOUR); - - if (abs((int) offset) % SECS_PER_HOUR != 0) - { - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), ":%02d", - abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); - } -} - /* * Parse string representation of the server version. */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c new file mode 100644 index 000000000..9b88d1594 --- /dev/null +++ b/src/utils/configuration.c @@ -0,0 +1,1499 @@ +/*------------------------------------------------------------------------- + * + * configuration.c: - function implementations to work with pg_probackup + * configurations. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "configuration.h" +#include "logger.h" +#include "pgut.h" + +#include "datatype/timestamp.h" + +#include "getopt_long.h" + +#include + +#define MAXPG_LSNCOMPONENT 8 + +/* + * Unit conversion tables. + * + * Copied from guc.c. + */ +#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ + +typedef struct +{ + char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or + * "min" */ + int base_unit; /* OPTION_UNIT_XXX */ + int multiplier; /* If positive, multiply the value with this + * for unit -> base_unit conversion. If + * negative, divide (with the absolute value) */ +} unit_conversion; + +static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; + +static const unit_conversion memory_unit_conversion_table[] = +{ + {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, + {"GB", OPTION_UNIT_KB, 1024 * 1024}, + {"MB", OPTION_UNIT_KB, 1024}, + {"KB", OPTION_UNIT_KB, 1}, + {"kB", OPTION_UNIT_KB, 1}, + + {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, + + {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, + {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, + {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, + + {""} /* end of table marker */ +}; + +static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; + +static const unit_conversion time_unit_conversion_table[] = +{ + {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, + {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, + {"min", OPTION_UNIT_MS, 1000 * 60}, + {"s", OPTION_UNIT_MS, 1000}, + {"ms", OPTION_UNIT_MS, 1}, + + {"d", OPTION_UNIT_S, 60 * 60 * 24}, + {"h", OPTION_UNIT_S, 60 * 60}, + {"min", OPTION_UNIT_S, 60}, + {"s", OPTION_UNIT_S, 1}, + {"ms", OPTION_UNIT_S, -1000}, + + {"d", OPTION_UNIT_MIN, 60 * 24}, + {"h", OPTION_UNIT_MIN, 60}, + {"min", OPTION_UNIT_MIN, 1}, + {"s", OPTION_UNIT_MIN, -60}, + {"ms", OPTION_UNIT_MIN, -1000 * 60}, + + {""} /* end of table marker */ +}; + +/* + * Reading functions. + */ + +static uint32 +option_length(const ConfigOption opts[]) +{ + uint32 len; + + for (len = 0; opts && opts[len].type; len++) { } + + return len; +} + +static int +option_has_arg(char type) +{ + switch (type) + { + case 'b': + case 'B': + return no_argument; + default: + return required_argument; + } +} + +static void +option_copy(struct option dst[], const ConfigOption opts[], size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + { + dst[i].name = opts[i].lname; + dst[i].has_arg = option_has_arg(opts[i].type); + dst[i].flag = NULL; + dst[i].val = opts[i].sname; + } +} + +static ConfigOption * +option_find(int c, ConfigOption opts1[]) +{ + size_t i; + + for (i = 0; opts1 && opts1[i].type; i++) + if (opts1[i].sname == c) + return &opts1[i]; + + return NULL; /* not found */ +} + +static char * +longopts_to_optstring(const struct option opts[], const size_t len) +{ + size_t i; + char *result; + char *s; + + result = pgut_malloc(len * 2 + 1); + + s = result; + for (i = 0; i < len; i++) + { + if (!isprint(opts[i].val)) + continue; + *s++ = opts[i].val; + if (opts[i].has_arg != no_argument) + *s++ = ':'; + } + *s = '\0'; + + return result; +} + +/* + * Compare two strings ignore cases and ignore. + */ +static bool +key_equals(const char *lhs, const char *rhs) +{ + for (; *lhs && *rhs; lhs++, rhs++) + { + if (strchr("-_ ", *lhs)) + { + if (!strchr("-_ ", *rhs)) + return false; + } + else if (ToLower(*lhs) != ToLower(*rhs)) + return false; + } + + return *lhs == '\0' && *rhs == '\0'; +} + +static void +assign_option(ConfigOption *opt, const char *optarg, OptionSource src) +{ + const char *message; + + if (opt == NULL) + { + fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); + exit_or_abort(ERROR); + } + + if (opt->source > src) + { + /* high prior value has been set already. */ + return; + } + /* Allow duplicate entries for function option */ + else if (src >= SOURCE_CMD && opt->source >= src && opt->type != 'f') + { + message = "specified only once"; + } + else + { + OptionSource orig_source = opt->source; + + /* can be overwritten if non-command line source */ + opt->source = src; + + switch (opt->type) + { + case 'b': + case 'B': + if (optarg == NULL) + { + *((bool *) opt->var) = (opt->type == 'b'); + return; + } + else if (parse_bool(optarg, (bool *) opt->var)) + { + return; + } + message = "a boolean"; + break; + case 'f': + ((option_assign_fn) opt->var)(opt, optarg); + return; + case 'i': + if (parse_int32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit signed integer"; + break; + case 'u': + if (parse_uint32(optarg, opt->var, opt->flags)) + return; + message = "a 32bit unsigned integer"; + break; + case 'I': + if (parse_int64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit signed integer"; + break; + case 'U': + if (parse_uint64(optarg, opt->var, opt->flags)) + return; + message = "a 64bit unsigned integer"; + break; + case 's': + if (orig_source != SOURCE_DEFAULT) + free(*(char **) opt->var); + *(char **) opt->var = pgut_strdup(optarg); + if (strcmp(optarg,"") != 0) + return; + message = "a valid string"; + break; + case 't': + if (parse_time(optarg, opt->var, + opt->source == SOURCE_FILE)) + return; + message = "a time"; + break; + default: + elog(ERROR, "Invalid option type: %c", opt->type); + return; /* keep compiler quiet */ + } + } + + if (isprint(opt->sname)) + elog(ERROR, "Option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "Option --%s should be %s: '%s'", + opt->lname, message, optarg); +} + +static const char * +skip_space(const char *str, const char *line) +{ + while (IsSpace(*str)) { str++; } + return str; +} + +static const char * +get_next_token(const char *src, char *dst, const char *line) +{ + const char *s; + int i; + int j; + + if ((s = skip_space(src, line)) == NULL) + return NULL; + + /* parse quoted string */ + if (*s == '\'') + { + s++; + for (i = 0, j = 0; s[i] != '\0'; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + dst[j] = '\b'; + break; + case 'f': + dst[j] = '\f'; + break; + case 'n': + dst[j] = '\n'; + break; + case 'r': + dst[j] = '\r'; + break; + case 't': + dst[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + dst[j] = ((char) octVal); + } + break; + default: + dst[j] = s[i]; + break; + } + } + else if (s[i] == '\'') + { + i++; + /* doubled quote becomes just one quote */ + if (s[i] == '\'') + dst[j] = s[i]; + else + break; + } + else + dst[j] = s[i]; + j++; + } + } + else + { + i = j = strcspn(s, "#\n\r\t\v"); + memcpy(dst, s, j); + } + + dst[j] = '\0'; + return s + i; +} + +static bool +parse_pair(const char buffer[], char key[], char value[]) +{ + const char *start; + const char *end; + + key[0] = value[0] = '\0'; + + /* + * parse key + */ + start = buffer; + if ((start = skip_space(start, buffer)) == NULL) + return false; + + end = start + strcspn(start, "=# \n\r\t\v"); + + /* skip blank buffer */ + if (end - start <= 0) + { + if (*start == '=') + elog(ERROR, "Syntax error in \"%s\"", buffer); + return false; + } + + /* key found */ + strncpy(key, start, end - start); + key[end - start] = '\0'; + + /* find key and value split char */ + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '=') + { + elog(ERROR, "Syntax error in \"%s\"", buffer); + return false; + } + + start++; + + /* + * parse value + */ + if ((end = get_next_token(start, value, buffer)) == NULL) + return false; + + if ((start = skip_space(end, buffer)) == NULL) + return false; + + if (*start != '\0' && *start != '#') + { + elog(ERROR, "Syntax error in \"%s\"", buffer); + return false; + } + + return true; +} + +/* + * Returns the current user name. + */ +static const char * +get_username(void) +{ + const char *ret; + +#ifndef WIN32 + struct passwd *pw; + + pw = getpwuid(geteuid()); + ret = (pw ? pw->pw_name : NULL); +#else + static char username[128]; /* remains after function exit */ + DWORD len = sizeof(username) - 1; + + if (GetUserName(username, &len)) + ret = username; + else + { + _dosmaperr(GetLastError()); + ret = NULL; + } +#endif + + if (ret == NULL) + elog(ERROR, "Could not get current user name: %s", strerror(errno)); + return ret; +} + +/* + * Process options passed from command line. + */ +int +config_get_opt(int argc, char **argv, ConfigOption cmd_options[], + ConfigOption options[]) +{ + int c; + int optindex = 0; + char *optstring; + struct option *longopts; + uint32 cmd_len, + len; + + cmd_len = option_length(cmd_options); + len = option_length(options); + + longopts = pgut_newarray(struct option, + cmd_len + len + 1 /* zero/end option */); + + /* Concatenate two options */ + option_copy(longopts, cmd_options, cmd_len); + option_copy(longopts + cmd_len, options, len + 1); + + optstring = longopts_to_optstring(longopts, cmd_len + len); + + /* Assign named options */ + while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) + { + ConfigOption *opt; + + opt = option_find(c, cmd_options); + if (opt == NULL) + opt = option_find(c, options); + + if (opt && + opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) + elog(ERROR, "Option %s cannot be specified in command line", + opt->lname); + /* Check 'opt == NULL' is performed in assign_option() */ + assign_option(opt, optarg, SOURCE_CMD); + } + + return optind; +} + +/* + * Get configuration from configuration file. + * Return number of parsed options. + */ +int +config_read_opt(const char *path, ConfigOption options[], int elevel, + bool strict) +{ + FILE *fp; + char buf[1024]; + char key[1024]; + char value[1024]; + int parsed_options = 0; + + if (!options) + return parsed_options; + + if ((fp = pgut_fopen(path, "rt", true)) == NULL) + return parsed_options; + + while (fgets(buf, lengthof(buf), fp)) + { + size_t i; + + for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) + buf[i - 1] = '\0'; + + if (parse_pair(buf, key, value)) + { + for (i = 0; options[i].type; i++) + { + ConfigOption *opt = &options[i]; + + if (key_equals(key, opt->lname)) + { + if (opt->allowed < SOURCE_FILE && + opt->allowed != SOURCE_FILE_STRICT) + elog(elevel, "Option %s cannot be specified in file", + opt->lname); + else if (opt->source <= SOURCE_FILE) + { + assign_option(opt, value, SOURCE_FILE); + parsed_options++; + } + break; + } + } + if (strict && !options[i].type) + elog(elevel, "Invalid option \"%s\" in file \"%s\"", key, path); + } + } + + fclose(fp); + + return parsed_options; +} + +/* + * Process options passed as environment variables. + */ +void +config_get_opt_env(ConfigOption options[]) +{ + size_t i; + + for (i = 0; options && options[i].type; i++) + { + ConfigOption *opt = &options[i]; + const char *value = NULL; + + /* If option was already set do not check env */ + if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) + continue; + + if (strcmp(opt->lname, "pgdata") == 0) + value = getenv("PGDATA"); + if (strcmp(opt->lname, "port") == 0) + value = getenv("PGPORT"); + if (strcmp(opt->lname, "host") == 0) + value = getenv("PGHOST"); + if (strcmp(opt->lname, "username") == 0) + value = getenv("PGUSER"); + if (strcmp(opt->lname, "pgdatabase") == 0) + { + value = getenv("PGDATABASE"); + if (value == NULL) + value = getenv("PGUSER"); + if (value == NULL) + value = get_username(); + } + + if (value) + assign_option(opt, value, SOURCE_ENV); + } +} + +/* + * Manually set source of the option. Find it by the pointer var. + */ +void +config_set_opt(ConfigOption options[], void *var, OptionSource source) +{ + int i; + + for (i = 0; options[i].type; i++) + { + ConfigOption *opt = &options[i]; + + if (opt->var == var) + { + if ((opt->allowed == SOURCE_FILE_STRICT && source != SOURCE_FILE) || + (opt->allowed == SOURCE_CMD_STRICT && source != SOURCE_CMD) || + (opt->allowed < source && opt->allowed >= SOURCE_ENV)) + elog(ERROR, "Invalid option source %d for %s", + source, opt->lname); + + opt->source = source; + break; + } + } +} + +/* + * Return value of the function in the string representation. Result is + * allocated string. + */ +char * +option_get_value(ConfigOption *opt) +{ + int64 value = 0; + uint64 value_u = 0; + const char *unit = NULL; + + /* + * If it is defined a unit for the option get readable value from base with + * unit name. + */ + if (opt->flags & OPTION_UNIT) + { + if (opt->type == 'i') + convert_from_base_unit(*((int32 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'i') + convert_from_base_unit(*((int64 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'u') + convert_from_base_unit_u(*((uint32 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + else if (opt->type == 'U') + convert_from_base_unit_u(*((uint64 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + } + + /* Get string representation itself */ + switch (opt->type) + { + case 'b': + case 'B': + return psprintf("%s", *((bool *) opt->var) ? "true" : "false"); + case 'i': + if (opt->flags & OPTION_UNIT) + return psprintf(INT64_FORMAT "%s", value, unit); + else + return psprintf("%d", *((int32 *) opt->var)); + case 'u': + if (opt->flags & OPTION_UNIT) + return psprintf(UINT64_FORMAT "%s", value_u, unit); + else + return psprintf("%u", *((uint32 *) opt->var)); + case 'I': + if (opt->flags & OPTION_UNIT) + return psprintf(INT64_FORMAT "%s", value, unit); + else + return psprintf(INT64_FORMAT, *((int64 *) opt->var)); + case 'U': + if (opt->flags & OPTION_UNIT) + return psprintf(UINT64_FORMAT "%s", value_u, unit); + else + return psprintf(UINT64_FORMAT, *((uint64 *) opt->var)); + case 's': + if (*((char **) opt->var) == NULL) + return NULL; + return pstrdup(*((char **) opt->var)); + case 't': + { + char *timestamp; + time_t t = *((time_t *) opt->var); + + if (t > 0) + { + timestamp = palloc(100); + time2iso(timestamp, 100, t); + } + else + timestamp = palloc0(1 /* just null termination */); + return timestamp; + } + default: + elog(ERROR, "Invalid option type: %c", opt->type); + return NULL; /* keep compiler quiet */ + } +} + +/* + * Parsing functions + */ + +/* + * Convert a value from one of the human-friendly units ("kB", "min" etc.) + * to the given base unit. 'value' and 'unit' are the input value and unit + * to convert from. The converted value is stored in *base_value. + * + * Returns true on success, false if the input unit is not recognized. + */ +static bool +convert_to_base_unit(int64 value, const char *unit, + int base_unit, int64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + { + /* Check for integer overflow first */ + if (value > PG_INT64_MAX / table[i].multiplier) + return false; + + *base_value = value * table[i].multiplier; + } + return true; + } + } + return false; +} + +/* + * Unsigned variant of convert_to_base_unit() + */ +static bool +convert_to_base_unit_u(uint64 value, const char *unit, + int base_unit, uint64 *base_value) +{ + const unit_conversion *table; + int i; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit && + strcmp(unit, table[i].unit) == 0) + { + if (table[i].multiplier < 0) + *base_value = value / (-table[i].multiplier); + else + { + /* Check for integer overflow first */ + if (value > PG_UINT64_MAX / table[i].multiplier) + return false; + + *base_value = value * table[i].multiplier; + } + return true; + } + } + return false; +} + +static bool +parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Unsigned variant of parse_unit() + */ +static bool +parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) +{ + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + /* Handle possible unit */ + if (*unit_str != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(unit_str++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *unit_str)) + unit_str++; + + if (*unit_str == '\0') + converted = convert_to_base_unit_u(value, unit, + (flags & OPTION_UNIT), + base_value); + if (!converted) + return false; + } + + return true; +} + +/* + * Try to interpret value as boolean value. Valid values are: true, + * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + */ +bool +parse_bool(const char *value, bool *result) +{ + return parse_bool_with_len(value, strlen(value), result); +} + +bool +parse_bool_with_len(const char *value, size_t len, bool *result) +{ + switch (*value) + { + case 't': + case 'T': + if (pg_strncasecmp(value, "true", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'f': + case 'F': + if (pg_strncasecmp(value, "false", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'y': + case 'Y': + if (pg_strncasecmp(value, "yes", len) == 0) + { + if (result) + *result = true; + return true; + } + break; + case 'n': + case 'N': + if (pg_strncasecmp(value, "no", len) == 0) + { + if (result) + *result = false; + return true; + } + break; + case 'o': + case 'O': + /* 'o' is not unique enough */ + if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = true; + return true; + } + else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) + { + if (result) + *result = false; + return true; + } + break; + case '1': + if (len == 1) + { + if (result) + *result = true; + return true; + } + break; + case '0': + if (len == 1) + { + if (result) + *result = false; + return true; + } + break; + default: + break; + } + + if (result) + *result = false; /* suppress compiler warning */ + return false; +} + +/* + * Parse string as 32bit signed int. + * valid range: -2147483648 ~ 2147483647 + */ +bool +parse_int32(const char *value, int32 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = PG_INT32_MAX; + return true; + } + + errno = 0; + val = strtol(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + /* Check for integer overflow */ + if (errno == ERANGE || val != (int64) ((int32) val)) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + /* Check for integer overflow again */ + if (val != (int64) ((int32) val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as 32bit unsigned int. + * valid range: 0 ~ 4294967295 (2^32-1) + */ +bool +parse_uint32(const char *value, uint32 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = PG_UINT32_MAX; + return true; + } + + errno = 0; + val = strtoul(value, &endptr, 0); + if (endptr == value || (*endptr && flags == 0)) + return false; + + /* Check for integer overflow */ + if (errno == ERANGE || val != (uint64) ((uint32) val)) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + /* Check for integer overflow again */ + if (val != (uint64) ((uint32) val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as int64 + * valid range: -9223372036854775808 ~ 9223372036854775807 + */ +bool +parse_int64(const char *value, int64 *result, int flags) +{ + int64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = PG_INT64_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtol(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoll(value, &endptr, 0); +#else + val = strtol(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Parse string as uint64 + * valid range: 0 ~ (2^64-1) + */ +bool +parse_uint64(const char *value, uint64 *result, int flags) +{ + uint64 val; + char *endptr; + + if (strcmp(value, INFINITE_STR) == 0) + { + *result = PG_UINT64_MAX; + return true; + } + + errno = 0; +#if defined(HAVE_LONG_INT_64) + val = strtoul(value, &endptr, 0); +#elif defined(HAVE_LONG_LONG_INT_64) + val = strtoull(value, &endptr, 0); +#else + val = strtoul(value, &endptr, 0); +#endif + if (endptr == value || (*endptr && flags == 0)) + return false; + + if (errno == ERANGE) + return false; + + if (!parse_unit_u(endptr, flags, val, &val)) + return false; + + *result = val; + + return true; +} + +/* + * Convert ISO-8601 format string to time_t value. + * + * If utc_default is true, then if timezone offset isn't specified tz will be + * +00:00. + */ +bool +parse_time(const char *value, time_t *result, bool utc_default) +{ + size_t len; + int fields_num, + tz = 0, + i; + bool tz_set = false; + char *tmp; + struct tm tm; + char junk[2]; + + /* tmp = replace( value, !isalnum, ' ' ) */ + tmp = pgut_malloc(strlen(value) + + 1); + len = 0; + fields_num = 1; + + while (*value) + { + if (IsAlnum(*value)) + { + tmp[len++] = *value; + value++; + } + else if (fields_num < 6) + { + fields_num++; + tmp[len++] = ' '; + value++; + } + /* timezone field is 7th */ + else if ((*value == '-' || *value == '+') && fields_num == 6) + { + int hr, + min, + sec = 0; + char *cp; + + errno = 0; + hr = strtol(value + 1, &cp, 10); + if ((value + 1) == cp || errno == ERANGE) + return false; + + /* explicit delimiter? */ + if (*cp == ':') + { + errno = 0; + min = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + if (*cp == ':') + { + errno = 0; + sec = strtol(cp + 1, &cp, 10); + if (errno == ERANGE) + return false; + } + } + /* otherwise, might have run things together... */ + else if (*cp == '\0' && strlen(value) > 3) + { + min = hr % 100; + hr = hr / 100; + /* we could, but don't, support a run-together hhmmss format */ + } + else + min = 0; + + /* Range-check the values; see notes in datatype/timestamp.h */ + if (hr < 0 || hr > MAX_TZDISP_HOUR) + return false; + if (min < 0 || min >= MINS_PER_HOUR) + return false; + if (sec < 0 || sec >= SECS_PER_MINUTE) + return false; + + tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; + if (*value == '-') + tz = -tz; + + tz_set = true; + + fields_num++; + value = cp; + } + /* wrong format */ + else if (!IsSpace(*value)) + return false; + } + tmp[len] = '\0'; + + /* parse for "YYYY-MM-DD HH:MI:SS" */ + memset(&tm, 0, sizeof(tm)); + tm.tm_year = 0; /* tm_year is year - 1900 */ + tm.tm_mon = 0; /* tm_mon is 0 - 11 */ + tm.tm_mday = 1; /* tm_mday is 1 - 31 */ + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", + &tm.tm_year, &tm.tm_mon, &tm.tm_mday, + &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); + free(tmp); + + if (i < 1 || 6 < i) + return false; + + /* adjust year */ + if (tm.tm_year < 100) + tm.tm_year += 2000 - 1900; + else if (tm.tm_year >= 1900) + tm.tm_year -= 1900; + + /* adjust month */ + if (i > 1) + tm.tm_mon -= 1; + + /* determine whether Daylight Saving Time is in effect */ + tm.tm_isdst = -1; + + *result = mktime(&tm); + + /* adjust time zone */ + if (tz_set || utc_default) + { + time_t ltime = time(NULL); + struct tm *ptm = gmtime(<ime); + time_t gmt = mktime(ptm); + time_t offset; + + /* UTC time */ + *result -= tz; + + /* Get local time */ + ptm = localtime(<ime); + offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); + + *result += offset; + } + + return true; +} + +/* + * Try to parse value as an integer. The accepted formats are the + * usual decimal, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. + * + * If the string parses okay, return true, else false. + * If okay and result is not NULL, return the value in *result. + * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable + * HINT message, or NULL if no hint provided. + */ +bool +parse_int(const char *value, int *result, int flags, const char **hintmsg) +{ + int64 val; + char *endptr; + + /* To suppress compiler warnings, always set output params */ + if (result) + *result = 0; + if (hintmsg) + *hintmsg = NULL; + + /* We assume here that int64 is at least as wide as long */ + errno = 0; + val = strtol(value, &endptr, 0); + + if (endptr == value) + return false; /* no HINT for integer syntax error */ + + if (errno == ERANGE || val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + + /* allow whitespace between integer and unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + /* Handle possible unit */ + if (*endptr != '\0') + { + char unit[MAX_UNIT_LEN + 1]; + int unitlen; + bool converted = false; + + if ((flags & OPTION_UNIT) == 0) + return false; /* this setting does not accept a unit */ + + unitlen = 0; + while (*endptr != '\0' && !isspace((unsigned char) *endptr) && + unitlen < MAX_UNIT_LEN) + unit[unitlen++] = *(endptr++); + unit[unitlen] = '\0'; + /* allow whitespace after unit */ + while (isspace((unsigned char) *endptr)) + endptr++; + + if (*endptr == '\0') + converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), + &val); + if (!converted) + { + /* invalid unit, or garbage after the unit; set hint and fail. */ + if (hintmsg) + { + if (flags & OPTION_UNIT_MEMORY) + *hintmsg = memory_units_hint; + else + *hintmsg = time_units_hint; + } + return false; + } + + /* Check for overflow due to units conversion */ + if (val != (int64) ((int32) val)) + { + if (hintmsg) + *hintmsg = "Value exceeds integer range."; + return false; + } + } + + if (result) + *result = (int) val; + return true; +} + +bool +parse_lsn(const char *value, XLogRecPtr *result) +{ + uint32 xlogid; + uint32 xrecoff; + int len1; + int len2; + + len1 = strspn(value, "0123456789abcdefABCDEF"); + if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') + elog(ERROR, "invalid LSN \"%s\"", value); + len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); + if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') + elog(ERROR, "invalid LSN \"%s\"", value); + + if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) + *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; + else + { + elog(ERROR, "invalid LSN \"%s\"", value); + return false; + } + + return true; +} + +/* + * Convert a value in some base unit to a human-friendly unit. The output + * unit is chosen so that it's the greatest unit that can represent the value + * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is + * converted to 1 MB, but 1025 is represented as 1025 kB. + */ +void +convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + /* Check for integer overflow first */ + if (base_value > PG_INT64_MAX / (-table[i].multiplier)) + continue; + + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Unsigned variant of convert_from_base_unit() + */ +void +convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit) +{ + const unit_conversion *table; + int i; + + *unit = NULL; + + if (base_unit & OPTION_UNIT_MEMORY) + table = memory_unit_conversion_table; + else + table = time_unit_conversion_table; + + for (i = 0; *table[i].unit; i++) + { + if (base_unit == table[i].base_unit) + { + /* + * Accept the first conversion that divides the value evenly. We + * assume that the conversions for each base unit are ordered from + * greatest unit to the smallest! + */ + if (table[i].multiplier < 0) + { + /* Check for integer overflow first */ + if (base_value > PG_UINT64_MAX / (-table[i].multiplier)) + continue; + + *value = base_value * (-table[i].multiplier); + *unit = table[i].unit; + break; + } + else if (base_value % table[i].multiplier == 0) + { + *value = base_value / table[i].multiplier; + *unit = table[i].unit; + break; + } + } + } + + Assert(*unit != NULL); +} + +/* + * Convert time_t value to ISO-8601 format string. Always set timezone offset. + */ +void +time2iso(char *buf, size_t len, time_t time) +{ + struct tm *ptm = gmtime(&time); + time_t gmt = mktime(ptm); + time_t offset; + char *ptr = buf; + + ptm = localtime(&time); + offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); + + strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); + + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), "%c%02d", + (offset >= 0) ? '+' : '-', + abs((int) offset) / SECS_PER_HOUR); + + if (abs((int) offset) % SECS_PER_HOUR != 0) + { + ptr += strlen(ptr); + snprintf(ptr, len - (ptr - buf), ":%02d", + abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); + } +} diff --git a/src/utils/configuration.h b/src/utils/configuration.h new file mode 100644 index 000000000..9602f1d66 --- /dev/null +++ b/src/utils/configuration.h @@ -0,0 +1,106 @@ +/*------------------------------------------------------------------------- + * + * configuration.h: - prototypes of functions and structures for + * configuration. + * + * Copyright (c) 2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef CONFIGURATION_H +#define CONFIGURATION_H + +#include "postgres_fe.h" +#include "access/xlogdefs.h" + +#define INFINITE_STR "INFINITE" + +typedef enum OptionSource +{ + SOURCE_DEFAULT, + SOURCE_FILE_STRICT, + SOURCE_CMD_STRICT, + SOURCE_ENV, + SOURCE_FILE, + SOURCE_CMD, + SOURCE_CONST +} OptionSource; + +typedef struct ConfigOption ConfigOption; + +typedef void (*option_assign_fn) (ConfigOption *opt, const char *arg); +/* Returns allocated string value */ +typedef char *(*option_get_fn) (ConfigOption *opt); + +/* + * type: + * b: bool (true) + * B: bool (false) + * f: option_fn + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string + * t: time_t + */ +struct ConfigOption +{ + char type; + uint8 sname; /* short name */ + const char *lname; /* long name */ + void *var; /* pointer to variable */ + OptionSource allowed; /* allowed source */ + OptionSource source; /* actual source */ + const char *group; /* option group name */ + int flags; /* option unit */ + option_get_fn get_value; /* function to get the value as a string, + should return allocated string*/ +}; + +/* + * bit values in "flags" of an option + */ +#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ +#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ +#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ +#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ +#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ + +#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ +#define OPTION_UNIT_S 0x20000 /* value is in seconds */ +#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ +#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ + +#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) + +extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], + ConfigOption options[]); +extern int config_read_opt(const char *path, ConfigOption options[], int elevel, + bool strict); +extern void config_get_opt_env(ConfigOption options[]); +extern void config_set_opt(ConfigOption options[], void *var, + OptionSource source); + +extern char *option_get_value(ConfigOption *opt); + +extern bool parse_bool(const char *value, bool *result); +extern bool parse_bool_with_len(const char *value, size_t len, bool *result); +extern bool parse_int32(const char *value, int32 *result, int flags); +extern bool parse_uint32(const char *value, uint32 *result, int flags); +extern bool parse_int64(const char *value, int64 *result, int flags); +extern bool parse_uint64(const char *value, uint64 *result, int flags); +extern bool parse_time(const char *value, time_t *result, bool utc_default); +extern bool parse_int(const char *value, int *result, int flags, + const char **hintmsg); +extern bool parse_lsn(const char *value, XLogRecPtr *result); + +extern void time2iso(char *buf, size_t len, time_t time); + +extern void convert_from_base_unit(int64 base_value, int base_unit, + int64 *value, const char **unit); +extern void convert_from_base_unit_u(uint64 base_value, int base_unit, + uint64 *value, const char **unit); + +#endif /* CONFIGURATION_H */ diff --git a/src/utils/json.c b/src/utils/json.c index 3afbe9e70..a2608304b 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -12,6 +12,8 @@ static void json_add_indent(PQExpBuffer buf, int32 level); static void json_add_escaped(PQExpBuffer buf, const char *str); +static bool add_comma = false; + /* * Start or end json token. Currently it is a json object or array. * @@ -25,6 +27,7 @@ json_add(PQExpBuffer buf, JsonToken type, int32 *level) case JT_BEGIN_ARRAY: appendPQExpBufferChar(buf, '['); *level += 1; + add_comma = false; break; case JT_END_ARRAY: *level -= 1; @@ -33,11 +36,13 @@ json_add(PQExpBuffer buf, JsonToken type, int32 *level) else json_add_indent(buf, *level); appendPQExpBufferChar(buf, ']'); + add_comma = true; break; case JT_BEGIN_OBJECT: json_add_indent(buf, *level); appendPQExpBufferChar(buf, '{'); *level += 1; + add_comma = false; break; case JT_END_OBJECT: *level -= 1; @@ -46,6 +51,7 @@ json_add(PQExpBuffer buf, JsonToken type, int32 *level) else json_add_indent(buf, *level); appendPQExpBufferChar(buf, '}'); + add_comma = true; break; default: break; @@ -56,7 +62,7 @@ json_add(PQExpBuffer buf, JsonToken type, int32 *level) * Add json object's key. If it isn't first key we need to add a comma. */ void -json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) +json_add_key(PQExpBuffer buf, const char *name, int32 level) { if (add_comma) appendPQExpBufferChar(buf, ','); @@ -64,6 +70,8 @@ json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) json_add_escaped(buf, name); appendPQExpBufferStr(buf, ": "); + + add_comma = true; } /* @@ -72,10 +80,14 @@ json_add_key(PQExpBuffer buf, const char *name, int32 level, bool add_comma) */ void json_add_value(PQExpBuffer buf, const char *name, const char *value, - int32 level, bool add_comma) + int32 level, bool escaped) { - json_add_key(buf, name, level, add_comma); - json_add_escaped(buf, value); + json_add_key(buf, name, level); + + if (escaped) + json_add_escaped(buf, value); + else + appendPQExpBufferStr(buf, value); } static void diff --git a/src/utils/json.h b/src/utils/json.h index cf5a70648..0344a5d75 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -25,9 +25,8 @@ typedef enum } JsonToken; extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); -extern void json_add_key(PQExpBuffer buf, const char *name, int32 level, - bool add_comma); +extern void json_add_key(PQExpBuffer buf, const char *name, int32 level); extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, - int32 level, bool add_comma); + int32 level, bool escaped); #endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 4cdbf7211..446bdb2ae 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -15,24 +15,17 @@ #include "pgut.h" #include "thread.h" -/* Logger parameters */ - -int log_level_console = LOG_LEVEL_CONSOLE_DEFAULT; -int log_level_file = LOG_LEVEL_FILE_DEFAULT; - -char *log_filename = NULL; -char *error_log_filename = NULL; -char *log_directory = NULL; -/* - * If log_path is empty logging is not initialized. - * We will log only into stderr - */ -char log_path[MAXPGPATH] = ""; +#include "utils/configuration.h" -/* Maximum size of an individual log file in kilobytes */ -uint64 log_rotation_size = 0; -/* Maximum lifetime of an individual log file in minutes */ -uint64 log_rotation_age = 0; +/* Logger parameters */ +LoggerConfig logger_config = { + LOG_LEVEL_CONSOLE_DEFAULT, + LOG_LEVEL_FILE_DEFAULT, + LOG_FILENAME_DEFAULT, + NULL, + LOG_ROTATION_SIZE_DEFAULT, + LOG_ROTATION_AGE_DEFAULT +}; /* Implementation for logging.h */ @@ -68,17 +61,24 @@ static bool loggin_in_progress = false; static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; +/* + * Initialize logger. + * + * If log_directory wasn't set by a user we use full path: + * backup_directory/log + */ void -init_logger(const char *root_path) +init_logger(const char *root_path, LoggerConfig *config) { /* Set log path */ - if (log_level_file != LOG_OFF || error_log_filename) + if (config->log_directory == NULL) { - if (log_directory) - strcpy(log_path, log_directory); - else - join_path_components(log_path, root_path, LOG_DIRECTORY_DEFAULT); + config->log_directory = palloc(MAXPGPATH); + join_path_components(config->log_directory, + root_path, LOG_DIRECTORY_DEFAULT); } + + logger_config = *config; } static void @@ -157,10 +157,11 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) time_t log_time = (time_t) time(NULL); char strfbuf[128]; - write_to_file = elevel >= log_level_file && log_path[0] != '\0'; - write_to_error_log = elevel >= ERROR && error_log_filename && - log_path[0] != '\0'; - write_to_stderr = elevel >= log_level_console && !file_only; + write_to_file = elevel >= logger_config.log_level_file && + logger_config.log_directory && logger_config.log_directory[0] != '\0'; + write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && + logger_config.log_directory && logger_config.log_directory[0] != '\0'; + write_to_stderr = elevel >= logger_config.log_level_console && !file_only; pthread_lock(&log_file_mutex); #ifdef WIN32 @@ -192,10 +193,10 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) { if (log_file == NULL) { - if (log_filename == NULL) + if (logger_config.log_filename == NULL) open_logfile(&log_file, LOG_FILENAME_DEFAULT); else - open_logfile(&log_file, log_filename); + open_logfile(&log_file, logger_config.log_filename); } fprintf(log_file, "%s: ", strfbuf); @@ -214,7 +215,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) if (write_to_error_log) { if (error_log_file == NULL) - open_logfile(&error_log_file, error_log_filename); + open_logfile(&error_log_file, logger_config.error_log_filename); fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); @@ -263,7 +264,7 @@ elog_stderr(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < log_level_console && elevel < ERROR) + if (elevel < logger_config.log_level_console && elevel < ERROR) return; va_start(args, fmt); @@ -290,7 +291,8 @@ elog(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + if (elevel < logger_config.log_level_console && + elevel < logger_config.log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -310,7 +312,7 @@ elog_file(int elevel, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < log_level_file && elevel < ERROR) + if (elevel < logger_config.log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -351,7 +353,8 @@ pg_log(eLogType type, const char *fmt, ...) * Do not log message if severity level is less than log_level. * It is the little optimisation to put it here not in elog_internal(). */ - if (elevel < log_level_console && elevel < log_level_file && elevel < ERROR) + if (elevel < logger_config.log_level_console && + elevel < logger_config.log_level_file && elevel < ERROR) return; va_start(args, fmt); @@ -437,12 +440,13 @@ logfile_getname(const char *format, time_t timestamp) size_t len; struct tm *tm = localtime(×tamp); - if (log_path[0] == '\0') + if (logger_config.log_directory == NULL || + logger_config.log_directory[0] == '\0') elog_stderr(ERROR, "logging path is not set"); filename = (char *) palloc(MAXPGPATH); - snprintf(filename, MAXPGPATH, "%s/", log_path); + snprintf(filename, MAXPGPATH, "%s/", logger_config.log_directory); len = strlen(filename); @@ -464,7 +468,7 @@ logfile_open(const char *filename, const char *mode) /* * Create log directory if not present; ignore errors */ - mkdir(log_path, S_IRWXU); + mkdir(logger_config.log_directory, S_IRWXU); fh = fopen(filename, mode); @@ -498,7 +502,7 @@ open_logfile(FILE **file, const char *filename_format) filename = logfile_getname(filename_format, cur_time); - /* "log_path" was checked in logfile_getname() */ + /* "log_directory" was checked in logfile_getname() */ snprintf(control, MAXPGPATH, "%s.rotation", filename); if (stat(filename, &st) == -1) @@ -516,10 +520,11 @@ open_logfile(FILE **file, const char *filename_format) logfile_exists = true; /* First check for rotation */ - if (log_rotation_size > 0 || log_rotation_age > 0) + if (logger_config.log_rotation_size > 0 || + logger_config.log_rotation_age > 0) { /* Check for rotation by age */ - if (log_rotation_age > 0) + if (logger_config.log_rotation_age > 0) { struct stat control_st; @@ -550,7 +555,7 @@ open_logfile(FILE **file, const char *filename_format) rotation_requested = (cur_time - creation_time) > /* convert to seconds from milliseconds */ - log_rotation_age / 1000; + logger_config.log_rotation_age / 1000; } else elog_stderr(ERROR, "cannot read creation timestamp from " @@ -561,10 +566,10 @@ open_logfile(FILE **file, const char *filename_format) } /* Check for rotation by size */ - if (!rotation_requested && log_rotation_size > 0) + if (!rotation_requested && logger_config.log_rotation_size > 0) rotation_requested = st.st_size >= /* convert to bytes */ - log_rotation_size * 1024L; + logger_config.log_rotation_size * 1024L; } logfile_open: diff --git a/src/utils/logger.h b/src/utils/logger.h index 15ec38f1b..dba4489e9 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -21,33 +21,36 @@ #define ERROR 1 #define LOG_OFF 10 -/* Logger parameters */ - -extern int log_to_file; -extern int log_level_console; -extern int log_level_file; +typedef struct LoggerConfig +{ + int log_level_console; + int log_level_file; + char *log_filename; + char *error_log_filename; + char *log_directory; + /* Maximum size of an individual log file in kilobytes */ + uint64 log_rotation_size; + /* Maximum lifetime of an individual log file in minutes */ + uint64 log_rotation_age; +} LoggerConfig; -extern char *log_filename; -extern char *error_log_filename; -extern char *log_directory; -extern char log_path[MAXPGPATH]; +/* Logger parameters */ +extern LoggerConfig logger_config; -#define LOG_ROTATION_SIZE_DEFAULT 0 -#define LOG_ROTATION_AGE_DEFAULT 0 -extern uint64 log_rotation_size; -extern uint64 log_rotation_age; +#define LOG_ROTATION_SIZE_DEFAULT 0 +#define LOG_ROTATION_AGE_DEFAULT 0 -#define LOG_LEVEL_CONSOLE_DEFAULT INFO -#define LOG_LEVEL_FILE_DEFAULT LOG_OFF +#define LOG_LEVEL_CONSOLE_DEFAULT INFO +#define LOG_LEVEL_FILE_DEFAULT LOG_OFF -#define LOG_FILENAME_DEFAULT "pg_probackup.log" -#define LOG_DIRECTORY_DEFAULT "log" +#define LOG_FILENAME_DEFAULT "pg_probackup.log" +#define LOG_DIRECTORY_DEFAULT "log" #undef elog extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); -extern void init_logger(const char *root_path); +extern void init_logger(const char *root_path, LoggerConfig *config); extern int parse_log_level(const char *level); extern const char *deparse_log_level(int level); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index b789a326e..ccf832b97 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -20,17 +20,9 @@ #include "pgut.h" #include "logger.h" -#define MAX_TZDISP_HOUR 15 /* maximum allowed hour part */ -#define SECS_PER_MINUTE 60 -#define MINS_PER_HOUR 60 -#define MAXPG_LSNCOMPONENT 8 const char *PROGRAM_NAME = NULL; -const char *pgut_dbname = NULL; -const char *host = NULL; -const char *port = NULL; -const char *username = NULL; static char *password = NULL; bool prompt_password = true; bool force_password = false; @@ -43,1298 +35,19 @@ bool interrupted = false; bool in_cleanup = false; bool in_password = false; -static bool parse_pair(const char buffer[], char key[], char value[]); - /* Connection routines */ static void init_cancel_handler(void); static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); static void on_after_exec(PGcancel *thread_cancel_conn); static void on_interrupt(void); static void on_cleanup(void); -static void exit_or_abort(int exitcode); -static const char *get_username(void); static pqsigfunc oldhandler = NULL; -/* - * Unit conversion tables. - * - * Copied from guc.c. - */ -#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ - -typedef struct -{ - char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or - * "min" */ - int base_unit; /* OPTION_UNIT_XXX */ - int multiplier; /* If positive, multiply the value with this - * for unit -> base_unit conversion. If - * negative, divide (with the absolute value) */ -} unit_conversion; - -static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; - -static const unit_conversion memory_unit_conversion_table[] = -{ - {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, - {"GB", OPTION_UNIT_KB, 1024 * 1024}, - {"MB", OPTION_UNIT_KB, 1024}, - {"KB", OPTION_UNIT_KB, 1}, - {"kB", OPTION_UNIT_KB, 1}, - - {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, - {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, - {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, - {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, - - {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, - {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, - {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, - {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, - - {""} /* end of table marker */ -}; - -static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; - -static const unit_conversion time_unit_conversion_table[] = -{ - {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, - {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, - {"min", OPTION_UNIT_MS, 1000 * 60}, - {"s", OPTION_UNIT_MS, 1000}, - {"ms", OPTION_UNIT_MS, 1}, - - {"d", OPTION_UNIT_S, 60 * 60 * 24}, - {"h", OPTION_UNIT_S, 60 * 60}, - {"min", OPTION_UNIT_S, 60}, - {"s", OPTION_UNIT_S, 1}, - {"ms", OPTION_UNIT_S, -1000}, - - {"d", OPTION_UNIT_MIN, 60 * 24}, - {"h", OPTION_UNIT_MIN, 60}, - {"min", OPTION_UNIT_MIN, 1}, - {"s", OPTION_UNIT_MIN, -60}, - {"ms", OPTION_UNIT_MIN, -1000 * 60}, - - {""} /* end of table marker */ -}; - -static size_t -option_length(const pgut_option opts[]) -{ - size_t len; - - for (len = 0; opts && opts[len].type; len++) { } - - return len; -} - -static int -option_has_arg(char type) -{ - switch (type) - { - case 'b': - case 'B': - return no_argument; - default: - return required_argument; - } -} - -static void -option_copy(struct option dst[], const pgut_option opts[], size_t len) -{ - size_t i; - - for (i = 0; i < len; i++) - { - dst[i].name = opts[i].lname; - dst[i].has_arg = option_has_arg(opts[i].type); - dst[i].flag = NULL; - dst[i].val = opts[i].sname; - } -} - -static pgut_option * -option_find(int c, pgut_option opts1[]) -{ - size_t i; - - for (i = 0; opts1 && opts1[i].type; i++) - if (opts1[i].sname == c) - return &opts1[i]; - - return NULL; /* not found */ -} - -static void -assign_option(pgut_option *opt, const char *optarg, pgut_optsrc src) -{ - const char *message; - - if (opt == NULL) - { - fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); - exit_or_abort(ERROR); - } - - if (opt->source > src) - { - /* high prior value has been set already. */ - return; - } - /* Allow duplicate entries for function option */ - else if (src >= SOURCE_CMDLINE && opt->source >= src && opt->type != 'f') - { - message = "specified only once"; - } - else - { - pgut_optsrc orig_source = opt->source; - - /* can be overwritten if non-command line source */ - opt->source = src; - - switch (opt->type) - { - case 'b': - case 'B': - if (optarg == NULL) - { - *((bool *) opt->var) = (opt->type == 'b'); - return; - } - else if (parse_bool(optarg, (bool *) opt->var)) - { - return; - } - message = "a boolean"; - break; - case 'f': - ((pgut_optfn) opt->var)(opt, optarg); - return; - case 'i': - if (parse_int32(optarg, opt->var, opt->flags)) - return; - message = "a 32bit signed integer"; - break; - case 'u': - if (parse_uint32(optarg, opt->var, opt->flags)) - return; - message = "a 32bit unsigned integer"; - break; - case 'I': - if (parse_int64(optarg, opt->var, opt->flags)) - return; - message = "a 64bit signed integer"; - break; - case 'U': - if (parse_uint64(optarg, opt->var, opt->flags)) - return; - message = "a 64bit unsigned integer"; - break; - case 's': - if (orig_source != SOURCE_DEFAULT) - free(*(char **) opt->var); - *(char **) opt->var = pgut_strdup(optarg); - if (strcmp(optarg,"") != 0) - return; - message = "a valid string"; - break; - case 't': - if (parse_time(optarg, opt->var, - opt->source == SOURCE_FILE)) - return; - message = "a time"; - break; - default: - elog(ERROR, "invalid option type: %c", opt->type); - return; /* keep compiler quiet */ - } - } - - if (isprint(opt->sname)) - elog(ERROR, "option -%c, --%s should be %s: '%s'", - opt->sname, opt->lname, message, optarg); - else - elog(ERROR, "option --%s should be %s: '%s'", - opt->lname, message, optarg); -} - -/* - * Convert a value from one of the human-friendly units ("kB", "min" etc.) - * to the given base unit. 'value' and 'unit' are the input value and unit - * to convert from. The converted value is stored in *base_value. - * - * Returns true on success, false if the input unit is not recognized. - */ -static bool -convert_to_base_unit(int64 value, const char *unit, - int base_unit, int64 *base_value) -{ - const unit_conversion *table; - int i; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit && - strcmp(unit, table[i].unit) == 0) - { - if (table[i].multiplier < 0) - *base_value = value / (-table[i].multiplier); - else - { - /* Check for integer overflow first */ - if (value > PG_INT64_MAX / table[i].multiplier) - return false; - - *base_value = value * table[i].multiplier; - } - return true; - } - } - return false; -} - -/* - * Unsigned variant of convert_to_base_unit() - */ -static bool -convert_to_base_unit_u(uint64 value, const char *unit, - int base_unit, uint64 *base_value) -{ - const unit_conversion *table; - int i; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit && - strcmp(unit, table[i].unit) == 0) - { - if (table[i].multiplier < 0) - *base_value = value / (-table[i].multiplier); - else - { - /* Check for integer overflow first */ - if (value > PG_UINT64_MAX / table[i].multiplier) - return false; - - *base_value = value * table[i].multiplier; - } - return true; - } - } - return false; -} - -/* - * Convert a value in some base unit to a human-friendly unit. The output - * unit is chosen so that it's the greatest unit that can represent the value - * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is - * converted to 1 MB, but 1025 is represented as 1025 kB. - */ -void -convert_from_base_unit(int64 base_value, int base_unit, - int64 *value, const char **unit) -{ - const unit_conversion *table; - int i; - - *unit = NULL; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit) - { - /* - * Accept the first conversion that divides the value evenly. We - * assume that the conversions for each base unit are ordered from - * greatest unit to the smallest! - */ - if (table[i].multiplier < 0) - { - /* Check for integer overflow first */ - if (base_value > PG_INT64_MAX / (-table[i].multiplier)) - continue; - - *value = base_value * (-table[i].multiplier); - *unit = table[i].unit; - break; - } - else if (base_value % table[i].multiplier == 0) - { - *value = base_value / table[i].multiplier; - *unit = table[i].unit; - break; - } - } - } - - Assert(*unit != NULL); -} - -/* - * Unsigned variant of convert_from_base_unit() - */ void -convert_from_base_unit_u(uint64 base_value, int base_unit, - uint64 *value, const char **unit) -{ - const unit_conversion *table; - int i; - - *unit = NULL; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit) - { - /* - * Accept the first conversion that divides the value evenly. We - * assume that the conversions for each base unit are ordered from - * greatest unit to the smallest! - */ - if (table[i].multiplier < 0) - { - /* Check for integer overflow first */ - if (base_value > PG_UINT64_MAX / (-table[i].multiplier)) - continue; - - *value = base_value * (-table[i].multiplier); - *unit = table[i].unit; - break; - } - else if (base_value % table[i].multiplier == 0) - { - *value = base_value / table[i].multiplier; - *unit = table[i].unit; - break; - } - } - } - - Assert(*unit != NULL); -} - -static bool -parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) -{ - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - /* Handle possible unit */ - if (*unit_str != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(unit_str++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - if (*unit_str == '\0') - converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), - base_value); - if (!converted) - return false; - } - - return true; -} - -/* - * Unsigned variant of parse_unit() - */ -static bool -parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) -{ - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - /* Handle possible unit */ - if (*unit_str != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(unit_str++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - if (*unit_str == '\0') - converted = convert_to_base_unit_u(value, unit, (flags & OPTION_UNIT), - base_value); - if (!converted) - return false; - } - - return true; -} - -/* - * Try to interpret value as boolean value. Valid values are: true, - * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. - * If the string parses okay, return true, else false. - * If okay and result is not NULL, return the value in *result. - */ -bool -parse_bool(const char *value, bool *result) -{ - return parse_bool_with_len(value, strlen(value), result); -} - -bool -parse_bool_with_len(const char *value, size_t len, bool *result) -{ - switch (*value) - { - case 't': - case 'T': - if (pg_strncasecmp(value, "true", len) == 0) - { - if (result) - *result = true; - return true; - } - break; - case 'f': - case 'F': - if (pg_strncasecmp(value, "false", len) == 0) - { - if (result) - *result = false; - return true; - } - break; - case 'y': - case 'Y': - if (pg_strncasecmp(value, "yes", len) == 0) - { - if (result) - *result = true; - return true; - } - break; - case 'n': - case 'N': - if (pg_strncasecmp(value, "no", len) == 0) - { - if (result) - *result = false; - return true; - } - break; - case 'o': - case 'O': - /* 'o' is not unique enough */ - if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) - { - if (result) - *result = true; - return true; - } - else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) - { - if (result) - *result = false; - return true; - } - break; - case '1': - if (len == 1) - { - if (result) - *result = true; - return true; - } - break; - case '0': - if (len == 1) - { - if (result) - *result = false; - return true; - } - break; - default: - break; - } - - if (result) - *result = false; /* suppress compiler warning */ - return false; -} - -/* - * Parse string as 32bit signed int. - * valid range: -2147483648 ~ 2147483647 - */ -bool -parse_int32(const char *value, int32 *result, int flags) -{ - int64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_INT32_MAX; - return true; - } - - errno = 0; - val = strtol(value, &endptr, 0); - if (endptr == value || (*endptr && flags == 0)) - return false; - - /* Check for integer overflow */ - if (errno == ERANGE || val != (int64) ((int32) val)) - return false; - - if (!parse_unit(endptr, flags, val, &val)) - return false; - - /* Check for integer overflow again */ - if (val != (int64) ((int32) val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as 32bit unsigned int. - * valid range: 0 ~ 4294967295 (2^32-1) - */ -bool -parse_uint32(const char *value, uint32 *result, int flags) +pgut_init(void) { - uint64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_UINT32_MAX; - return true; - } - - errno = 0; - val = strtoul(value, &endptr, 0); - if (endptr == value || (*endptr && flags == 0)) - return false; - - /* Check for integer overflow */ - if (errno == ERANGE || val != (uint64) ((uint32) val)) - return false; - - if (!parse_unit_u(endptr, flags, val, &val)) - return false; - - /* Check for integer overflow again */ - if (val != (uint64) ((uint32) val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as int64 - * valid range: -9223372036854775808 ~ 9223372036854775807 - */ -bool -parse_int64(const char *value, int64 *result, int flags) -{ - int64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_INT64_MAX; - return true; - } - - errno = 0; -#if defined(HAVE_LONG_INT_64) - val = strtol(value, &endptr, 0); -#elif defined(HAVE_LONG_LONG_INT_64) - val = strtoll(value, &endptr, 0); -#else - val = strtol(value, &endptr, 0); -#endif - if (endptr == value || (*endptr && flags == 0)) - return false; - - if (errno == ERANGE) - return false; - - if (!parse_unit(endptr, flags, val, &val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as uint64 - * valid range: 0 ~ (2^64-1) - */ -bool -parse_uint64(const char *value, uint64 *result, int flags) -{ - uint64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_UINT64_MAX; - return true; - } - - errno = 0; -#if defined(HAVE_LONG_INT_64) - val = strtoul(value, &endptr, 0); -#elif defined(HAVE_LONG_LONG_INT_64) - val = strtoull(value, &endptr, 0); -#else - val = strtoul(value, &endptr, 0); -#endif - if (endptr == value || (*endptr && flags == 0)) - return false; - - if (errno == ERANGE) - return false; - - if (!parse_unit_u(endptr, flags, val, &val)) - return false; - - *result = val; - - return true; -} - -/* - * Convert ISO-8601 format string to time_t value. - * - * If utc_default is true, then if timezone offset isn't specified tz will be - * +00:00. - */ -bool -parse_time(const char *value, time_t *result, bool utc_default) -{ - size_t len; - int fields_num, - tz = 0, - i; - bool tz_set = false; - char *tmp; - struct tm tm; - char junk[2]; - - /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + + 1); - len = 0; - fields_num = 1; - - while (*value) - { - if (IsAlnum(*value)) - { - tmp[len++] = *value; - value++; - } - else if (fields_num < 6) - { - fields_num++; - tmp[len++] = ' '; - value++; - } - /* timezone field is 7th */ - else if ((*value == '-' || *value == '+') && fields_num == 6) - { - int hr, - min, - sec = 0; - char *cp; - - errno = 0; - hr = strtol(value + 1, &cp, 10); - if ((value + 1) == cp || errno == ERANGE) - return false; - - /* explicit delimiter? */ - if (*cp == ':') - { - errno = 0; - min = strtol(cp + 1, &cp, 10); - if (errno == ERANGE) - return false; - if (*cp == ':') - { - errno = 0; - sec = strtol(cp + 1, &cp, 10); - if (errno == ERANGE) - return false; - } - } - /* otherwise, might have run things together... */ - else if (*cp == '\0' && strlen(value) > 3) - { - min = hr % 100; - hr = hr / 100; - /* we could, but don't, support a run-together hhmmss format */ - } - else - min = 0; - - /* Range-check the values; see notes in datatype/timestamp.h */ - if (hr < 0 || hr > MAX_TZDISP_HOUR) - return false; - if (min < 0 || min >= MINS_PER_HOUR) - return false; - if (sec < 0 || sec >= SECS_PER_MINUTE) - return false; - - tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; - if (*value == '-') - tz = -tz; - - tz_set = true; - - fields_num++; - value = cp; - } - /* wrong format */ - else if (!IsSpace(*value)) - return false; - } - tmp[len] = '\0'; - - /* parse for "YYYY-MM-DD HH:MI:SS" */ - memset(&tm, 0, sizeof(tm)); - tm.tm_year = 0; /* tm_year is year - 1900 */ - tm.tm_mon = 0; /* tm_mon is 0 - 11 */ - tm.tm_mday = 1; /* tm_mday is 1 - 31 */ - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", - &tm.tm_year, &tm.tm_mon, &tm.tm_mday, - &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); - free(tmp); - - if (i < 1 || 6 < i) - return false; - - /* adjust year */ - if (tm.tm_year < 100) - tm.tm_year += 2000 - 1900; - else if (tm.tm_year >= 1900) - tm.tm_year -= 1900; - - /* adjust month */ - if (i > 1) - tm.tm_mon -= 1; - - /* determine whether Daylight Saving Time is in effect */ - tm.tm_isdst = -1; - - *result = mktime(&tm); - - /* adjust time zone */ - if (tz_set || utc_default) - { - time_t ltime = time(NULL); - struct tm *ptm = gmtime(<ime); - time_t gmt = mktime(ptm); - time_t offset; - - /* UTC time */ - *result -= tz; - - /* Get local time */ - ptm = localtime(<ime); - offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); - - *result += offset; - } - - return true; -} - -/* - * Try to parse value as an integer. The accepted formats are the - * usual decimal, octal, or hexadecimal formats, optionally followed by - * a unit name if "flags" indicates a unit is allowed. - * - * If the string parses okay, return true, else false. - * If okay and result is not NULL, return the value in *result. - * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable - * HINT message, or NULL if no hint provided. - */ -bool -parse_int(const char *value, int *result, int flags, const char **hintmsg) -{ - int64 val; - char *endptr; - - /* To suppress compiler warnings, always set output params */ - if (result) - *result = 0; - if (hintmsg) - *hintmsg = NULL; - - /* We assume here that int64 is at least as wide as long */ - errno = 0; - val = strtol(value, &endptr, 0); - - if (endptr == value) - return false; /* no HINT for integer syntax error */ - - if (errno == ERANGE || val != (int64) ((int32) val)) - { - if (hintmsg) - *hintmsg = "Value exceeds integer range."; - return false; - } - - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *endptr)) - endptr++; - - /* Handle possible unit */ - if (*endptr != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*endptr != '\0' && !isspace((unsigned char) *endptr) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(endptr++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *endptr)) - endptr++; - - if (*endptr == '\0') - converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), - &val); - if (!converted) - { - /* invalid unit, or garbage after the unit; set hint and fail. */ - if (hintmsg) - { - if (flags & OPTION_UNIT_MEMORY) - *hintmsg = memory_units_hint; - else - *hintmsg = time_units_hint; - } - return false; - } - - /* Check for overflow due to units conversion */ - if (val != (int64) ((int32) val)) - { - if (hintmsg) - *hintmsg = "Value exceeds integer range."; - return false; - } - } - - if (result) - *result = (int) val; - return true; -} - -bool -parse_lsn(const char *value, XLogRecPtr *result) -{ - uint32 xlogid; - uint32 xrecoff; - int len1; - int len2; - - len1 = strspn(value, "0123456789abcdefABCDEF"); - if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') - elog(ERROR, "invalid LSN \"%s\"", value); - len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); - if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') - elog(ERROR, "invalid LSN \"%s\"", value); - - if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) - *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; - else - { - elog(ERROR, "invalid LSN \"%s\"", value); - return false; - } - - return true; -} - -static char * -longopts_to_optstring(const struct option opts[], const size_t len) -{ - size_t i; - char *result; - char *s; - - result = pgut_malloc(len * 2 + 1); - - s = result; - for (i = 0; i < len; i++) - { - if (!isprint(opts[i].val)) - continue; - *s++ = opts[i].val; - if (opts[i].has_arg != no_argument) - *s++ = ':'; - } - *s = '\0'; - - return result; -} - -void -pgut_getopt_env(pgut_option options[]) -{ - size_t i; - - for (i = 0; options && options[i].type; i++) - { - pgut_option *opt = &options[i]; - const char *value = NULL; - - /* If option was already set do not check env */ - if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) - continue; - - if (strcmp(opt->lname, "pgdata") == 0) - value = getenv("PGDATA"); - if (strcmp(opt->lname, "port") == 0) - value = getenv("PGPORT"); - if (strcmp(opt->lname, "host") == 0) - value = getenv("PGHOST"); - if (strcmp(opt->lname, "username") == 0) - value = getenv("PGUSER"); - if (strcmp(opt->lname, "pgdatabase") == 0) - { - value = getenv("PGDATABASE"); - if (value == NULL) - value = getenv("PGUSER"); - if (value == NULL) - value = get_username(); - } - - if (value) - assign_option(opt, value, SOURCE_ENV); - } -} - -int -pgut_getopt(int argc, char **argv, pgut_option options[]) -{ - int c; - int optindex = 0; - char *optstring; - pgut_option *opt; - struct option *longopts; - size_t len; - - len = option_length(options); - longopts = pgut_newarray(struct option, len + 1 /* zero/end option */); - option_copy(longopts, options, len); - - optstring = longopts_to_optstring(longopts, len); - - /* Assign named options */ - while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) - { - opt = option_find(c, options); - if (opt && opt->allowed < SOURCE_CMDLINE) - elog(ERROR, "option %s cannot be specified in command line", - opt->lname); - /* Check 'opt == NULL' is performed in assign_option() */ - assign_option(opt, optarg, SOURCE_CMDLINE); - } - init_cancel_handler(); atexit(on_cleanup); - - return optind; -} - -/* compare two strings ignore cases and ignore -_ */ -static bool -key_equals(const char *lhs, const char *rhs) -{ - for (; *lhs && *rhs; lhs++, rhs++) - { - if (strchr("-_ ", *lhs)) - { - if (!strchr("-_ ", *rhs)) - return false; - } - else if (ToLower(*lhs) != ToLower(*rhs)) - return false; - } - - return *lhs == '\0' && *rhs == '\0'; -} - -/* - * Get configuration from configuration file. - * Return number of parsed options - */ -int -pgut_readopt(const char *path, pgut_option options[], int elevel, bool strict) -{ - FILE *fp; - char buf[1024]; - char key[1024]; - char value[1024]; - int parsed_options = 0; - - if (!options) - return parsed_options; - - if ((fp = pgut_fopen(path, "rt", true)) == NULL) - return parsed_options; - - while (fgets(buf, lengthof(buf), fp)) - { - size_t i; - - for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) - buf[i - 1] = '\0'; - - if (parse_pair(buf, key, value)) - { - for (i = 0; options[i].type; i++) - { - pgut_option *opt = &options[i]; - - if (key_equals(key, opt->lname)) - { - if (opt->allowed < SOURCE_FILE && - opt->allowed != SOURCE_FILE_STRICT) - elog(elevel, "option %s cannot be specified in file", opt->lname); - else if (opt->source <= SOURCE_FILE) - { - assign_option(opt, value, SOURCE_FILE); - parsed_options++; - } - break; - } - } - if (strict && !options[i].type) - elog(elevel, "invalid option \"%s\" in file \"%s\"", key, path); - } - } - - fclose(fp); - - return parsed_options; -} - -static const char * -skip_space(const char *str, const char *line) -{ - while (IsSpace(*str)) { str++; } - return str; -} - -static const char * -get_next_token(const char *src, char *dst, const char *line) -{ - const char *s; - int i; - int j; - - if ((s = skip_space(src, line)) == NULL) - return NULL; - - /* parse quoted string */ - if (*s == '\'') - { - s++; - for (i = 0, j = 0; s[i] != '\0'; i++) - { - if (s[i] == '\\') - { - i++; - switch (s[i]) - { - case 'b': - dst[j] = '\b'; - break; - case 'f': - dst[j] = '\f'; - break; - case 'n': - dst[j] = '\n'; - break; - case 'r': - dst[j] = '\r'; - break; - case 't': - dst[j] = '\t'; - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - { - int k; - long octVal = 0; - - for (k = 0; - s[i + k] >= '0' && s[i + k] <= '7' && k < 3; - k++) - octVal = (octVal << 3) + (s[i + k] - '0'); - i += k - 1; - dst[j] = ((char) octVal); - } - break; - default: - dst[j] = s[i]; - break; - } - } - else if (s[i] == '\'') - { - i++; - /* doubled quote becomes just one quote */ - if (s[i] == '\'') - dst[j] = s[i]; - else - break; - } - else - dst[j] = s[i]; - j++; - } - } - else - { - i = j = strcspn(s, "#\n\r\t\v"); - memcpy(dst, s, j); - } - - dst[j] = '\0'; - return s + i; -} - -static bool -parse_pair(const char buffer[], char key[], char value[]) -{ - const char *start; - const char *end; - - key[0] = value[0] = '\0'; - - /* - * parse key - */ - start = buffer; - if ((start = skip_space(start, buffer)) == NULL) - return false; - - end = start + strcspn(start, "=# \n\r\t\v"); - - /* skip blank buffer */ - if (end - start <= 0) - { - if (*start == '=') - elog(ERROR, "syntax error in \"%s\"", buffer); - return false; - } - - /* key found */ - strncpy(key, start, end - start); - key[end - start] = '\0'; - - /* find key and value split char */ - if ((start = skip_space(end, buffer)) == NULL) - return false; - - if (*start != '=') - { - elog(ERROR, "syntax error in \"%s\"", buffer); - return false; - } - - start++; - - /* - * parse value - */ - if ((end = get_next_token(start, value, buffer)) == NULL) - return false; - - if ((start = skip_space(end, buffer)) == NULL) - return false; - - if (*start != '\0' && *start != '#') - { - elog(ERROR, "syntax error in \"%s\"", buffer); - return false; - } - - return true; } /* @@ -1492,14 +205,8 @@ pgut_get_conninfo_string(PGconn *conn) } PGconn * -pgut_connect(const char *dbname) -{ - return pgut_connect_extended(host, port, dbname, username); -} - -PGconn * -pgut_connect_extended(const char *pghost, const char *pgport, - const char *dbname, const char *login) +pgut_connect(const char *host, const char *port, + const char *dbname, const char *username) { PGconn *conn; @@ -1510,13 +217,13 @@ pgut_connect_extended(const char *pghost, const char *pgport, elog(ERROR, "You cannot specify --password and --no-password options together"); if (!password && force_password) - prompt_for_password(login); + prompt_for_password(username); /* Start the connection. Loop until we have a password if requested by backend. */ for (;;) { - conn = PQsetdbLogin(pghost, pgport, NULL, NULL, - dbname, login, password); + conn = PQsetdbLogin(host, port, NULL, NULL, + dbname, username, password); if (PQstatus(conn) == CONNECTION_OK) return conn; @@ -1524,7 +231,7 @@ pgut_connect_extended(const char *pghost, const char *pgport, if (conn && PQconnectionNeedsPassword(conn) && prompt_password) { PQfinish(conn); - prompt_for_password(login); + prompt_for_password(username); if (interrupted) elog(ERROR, "interrupted"); @@ -1543,14 +250,8 @@ pgut_connect_extended(const char *pghost, const char *pgport, } PGconn * -pgut_connect_replication(const char *dbname) -{ - return pgut_connect_replication_extended(host, port, dbname, username); -} - -PGconn * -pgut_connect_replication_extended(const char *pghost, const char *pgport, - const char *dbname, const char *pguser) +pgut_connect_replication(const char *host, const char *port, + const char *dbname, const char *username) { PGconn *tmpconn; int argcount = 7; /* dbname, replication, fallback_app_name, @@ -1566,7 +267,7 @@ pgut_connect_replication_extended(const char *pghost, const char *pgport, elog(ERROR, "You cannot specify --password and --no-password options together"); if (!password && force_password) - prompt_for_password(pguser); + prompt_for_password(username); i = 0; @@ -1584,22 +285,22 @@ pgut_connect_replication_extended(const char *pghost, const char *pgport, values[i] = PROGRAM_NAME; i++; - if (pghost) + if (host) { keywords[i] = "host"; - values[i] = pghost; + values[i] = host; i++; } - if (pguser) + if (username) { keywords[i] = "user"; - values[i] = pguser; + values[i] = username; i++; } - if (pgport) + if (port) { keywords[i] = "port"; - values[i] = pgport; + values[i] = port; i++; } @@ -1630,7 +331,7 @@ pgut_connect_replication_extended(const char *pghost, const char *pgport, if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) { PQfinish(tmpconn); - prompt_for_password(pguser); + prompt_for_password(username); keywords[i] = "password"; values[i] = password; continue; @@ -1666,7 +367,8 @@ pgut_execute_parallel(PGconn* conn, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + if (logger_config.log_level_console <= VERBOSE || + logger_config.log_level_file <= VERBOSE) { int i; @@ -1728,7 +430,8 @@ pgut_execute_extended(PGconn* conn, const char *query, int nParams, elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + if (logger_config.log_level_console <= VERBOSE || + logger_config.log_level_file <= VERBOSE) { int i; @@ -1786,7 +489,8 @@ pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elog(ERROR, "interrupted"); /* write query to elog if verbose */ - if (log_level_console <= VERBOSE || log_level_file <= VERBOSE) + if (logger_config.log_level_console <= VERBOSE || + logger_config.log_level_file <= VERBOSE) { int i; @@ -2079,7 +783,7 @@ on_cleanup(void) call_atexit_callbacks(false); } -static void +void exit_or_abort(int exitcode) { if (in_cleanup) @@ -2095,38 +799,6 @@ exit_or_abort(int exitcode) } } -/* - * Returns the current user name. - */ -static const char * -get_username(void) -{ - const char *ret; - -#ifndef WIN32 - struct passwd *pw; - - pw = getpwuid(geteuid()); - ret = (pw ? pw->pw_name : NULL); -#else - static char username[128]; /* remains after function exit */ - DWORD len = sizeof(username) - 1; - - if (GetUserName(username, &len)) - ret = username; - else - { - _dosmaperr(GetLastError()); - ret = NULL; - } -#endif - - if (ret == NULL) - elog(ERROR, "%s: could not get current user name: %s", - PROGRAM_NAME, strerror(errno)); - return ret; -} - void * pgut_malloc(size_t size) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 6e9a50fda..bde5dee1c 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -12,63 +12,10 @@ #define PGUT_H #include "postgres_fe.h" -#include "access/xlogdefs.h" #include "libpq-fe.h" -#define INFINITE_STR "INFINITE" - -typedef enum pgut_optsrc -{ - SOURCE_DEFAULT, - SOURCE_FILE_STRICT, - SOURCE_ENV, - SOURCE_FILE, - SOURCE_CMDLINE, - SOURCE_CONST -} pgut_optsrc; - -/* - * type: - * b: bool (true) - * B: bool (false) - * f: pgut_optfn - * i: 32bit signed integer - * u: 32bit unsigned integer - * I: 64bit signed integer - * U: 64bit unsigned integer - * s: string - * t: time_t - */ -typedef struct pgut_option -{ - char type; - uint8 sname; /* short name */ - const char *lname; /* long name */ - void *var; /* pointer to variable */ - pgut_optsrc allowed; /* allowed source */ - pgut_optsrc source; /* actual source */ - int flags; /* option unit */ -} pgut_option; - -typedef void (*pgut_optfn) (pgut_option *opt, const char *arg); typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); -/* - * bit values in "flags" of an option - */ -#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ -#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ -#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ -#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ -#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ - -#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ -#define OPTION_UNIT_S 0x20000 /* value is in seconds */ -#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ -#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ - -#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) - /* * pgut client variables and functions */ @@ -82,10 +29,6 @@ extern void pgut_help(bool details); /* * pgut framework variables and functions */ -extern const char *pgut_dbname; -extern const char *host; -extern const char *port; -extern const char *username; extern bool prompt_password; extern bool force_password; @@ -93,23 +36,21 @@ extern bool interrupted; extern bool in_cleanup; extern bool in_password; /* User prompts password */ -extern int pgut_getopt(int argc, char **argv, pgut_option options[]); -extern int pgut_readopt(const char *path, pgut_option options[], int elevel, - bool strict); -extern void pgut_getopt_env(pgut_option options[]); extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); +extern void pgut_init(void); +extern void exit_or_abort(int exitcode); + /* * Database connections */ extern char *pgut_get_conninfo_string(PGconn *conn); -extern PGconn *pgut_connect(const char *dbname); -extern PGconn *pgut_connect_extended(const char *pghost, const char *pgport, - const char *dbname, const char *login); -extern PGconn *pgut_connect_replication(const char *dbname); -extern PGconn *pgut_connect_replication_extended(const char *pghost, const char *pgport, - const char *dbname, const char *pguser); +extern PGconn *pgut_connect(const char *host, const char *port, + const char *dbname, const char *username); +extern PGconn *pgut_connect_replication(const char *host, const char *port, + const char *dbname, + const char *username); extern void pgut_disconnect(PGconn *conn); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params); @@ -154,22 +95,6 @@ extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); #define AssertMacro(x) ((void) 0) #endif -extern bool parse_bool(const char *value, bool *result); -extern bool parse_bool_with_len(const char *value, size_t len, bool *result); -extern bool parse_int32(const char *value, int32 *result, int flags); -extern bool parse_uint32(const char *value, uint32 *result, int flags); -extern bool parse_int64(const char *value, int64 *result, int flags); -extern bool parse_uint64(const char *value, uint64 *result, int flags); -extern bool parse_time(const char *value, time_t *result, bool utc_default); -extern bool parse_int(const char *value, int *result, int flags, - const char **hintmsg); -extern bool parse_lsn(const char *value, XLogRecPtr *result); - -extern void convert_from_base_unit(int64 base_value, int base_unit, - int64 *value, const char **unit); -extern void convert_from_base_unit_u(uint64 base_value, int base_unit, - uint64 *value, const char **unit); - #define IsSpace(c) (isspace((unsigned char)(c))) #define IsAlpha(c) (isalpha((unsigned char)(c))) #define IsAlnum(c) (isalnum((unsigned char)(c))) diff --git a/src/validate.c b/src/validate.c index 55ea5ebe4..69f62cbab 100644 --- a/src/validate.c +++ b/src/validate.c @@ -275,6 +275,7 @@ do_validate_all(void) errno = 0; while ((dent = readdir(dir))) { + char conf_path[MAXPGPATH]; char child[MAXPGPATH]; struct stat st; @@ -291,10 +292,16 @@ do_validate_all(void) if (!S_ISDIR(st.st_mode)) continue; + /* + * Initialize instance configuration. + */ instance_name = dent->d_name; - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); - xlog_seg_size = get_config_xlog_seg_size(); + join_path_components(conf_path, backup_instance_path, + BACKUP_CATALOG_CONF_FILE); + config_read_opt(conf_path, instance_options, ERROR, false); do_validate_instance(); } @@ -418,7 +425,8 @@ do_validate_instance(void) /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli, xlog_seg_size); + 0, 0, base_full_backup->tli, + instance_config.xlog_seg_size); /* * Mark every descendant of corrupted backup as orphan @@ -506,7 +514,7 @@ do_validate_instance(void) /* Revalidation successful, validate corresponding WAL files */ validate_wal(backup, arclog_path, 0, 0, 0, current_backup->tli, - xlog_seg_size); + instance_config.xlog_seg_size); } } From 12a6dcdd93ec5dddc2d24df326005a91c41b36d5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Nov 2018 18:45:29 +0300 Subject: [PATCH 0045/2107] PGPRO-1905: check message about system id mismatch --- tests/page.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/page.py b/tests/page.py index 3d19a81d9..1d8238e17 100644 --- a/tests/page.py +++ b/tests/page.py @@ -936,6 +936,8 @@ def test_page_backup_with_alien_wal_segment(self): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'could not read WAL record at' in e.message and + 'WAL file is from different database system: WAL file database system identifier is' in e.message and + 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file_destination) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -961,6 +963,8 @@ def test_page_backup_with_alien_wal_segment(self): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'could not read WAL record at' in e.message and + 'WAL file is from different database system: WAL file database system identifier is' in e.message and + 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file_destination) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From 4995652ef5b1a225e6efb90b9512704654078009 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Sat, 10 Nov 2018 15:49:36 +0300 Subject: [PATCH 0046/2107] fix decompression of BLCKSZ pages --- src/data.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index 1d1a2ee9e..d9fd8837c 100644 --- a/src/data.c +++ b/src/data.c @@ -722,6 +722,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, size_t read_len; DataPage compressed_page; /* used as read buffer */ DataPage page; + int32 uncompressed_size = 0; /* File didn`t changed. Nothig to copy */ if (file->write_size == BYTES_INVALID) @@ -777,17 +778,23 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, Assert(header.compressed_size <= BLCKSZ); + /* read a page from file */ read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", blknum, file->path, read_len, header.compressed_size); + /* + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. + */ if (header.compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) { - int32 uncompressed_size = 0; const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, @@ -820,7 +827,11 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno)); } - if (header.compressed_size < BLCKSZ) + /* if we uncompressed the page - write page.data, + * if page wasn't compressed - + * write what we've read - compressed_page.data + */ + if (uncompressed_size == BLCKSZ) { if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", @@ -828,7 +839,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } else { - /* if page wasn't compressed, we've read full block */ + /* */ if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) elog(ERROR, "cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); From 88354a8dde65b0b60f1f64450b84596303b12b23 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 16 Oct 2018 18:13:27 +0300 Subject: [PATCH 0047/2107] PGPRO-1892: Continue failed merge command --- src/backup.c | 2 +- src/catalog.c | 14 +++++--- src/data.c | 16 --------- src/delete.c | 27 +++++++------- src/merge.c | 88 ++++++++++++++++++++-------------------------- src/pg_probackup.h | 7 ++-- tests/merge.py | 9 ++++- 7 files changed, 75 insertions(+), 88 deletions(-) diff --git a/src/backup.c b/src/backup.c index b2aed3186..036f3e912 100644 --- a/src/backup.c +++ b/src/backup.c @@ -790,7 +790,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - pgBackupWriteFileList(¤t, backup_files_list, pgdata); + write_backup_filelist(¤t, backup_files_list, pgdata); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) diff --git a/src/catalog.c b/src/catalog.c index 41676f4c9..788c33f76 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -509,18 +509,22 @@ write_backup(pgBackup *backup) fp = fopen(conf_path, "wt"); if (fp == NULL) elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, - strerror(errno)); + strerror(errno)); pgBackupWriteControl(fp, backup); - fclose(fp); + if (fflush(fp) != 0 || + fsync(fileno(fp)) != 0 || + fclose(fp)) + elog(ERROR, "Cannot write configuration file \"%s\": %s", + conf_path, strerror(errno)); } /* * Output the list of files to backup catalog DATABASE_FILE_LIST */ void -pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) +write_backup_filelist(pgBackup *backup, parray *files, const char *root) { FILE *fp; char path[MAXPGPATH]; @@ -529,7 +533,7 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) fp = fopen(path, "wt"); if (fp == NULL) - elog(ERROR, "cannot open file list \"%s\": %s", path, + elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); print_file_list(fp, files, root); @@ -537,7 +541,7 @@ pgBackupWriteFileList(pgBackup *backup, parray *files, const char *root) if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) - elog(ERROR, "cannot write file list \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot write file list \"%s\": %s", path, strerror(errno)); } /* diff --git a/src/data.c b/src/data.c index d9fd8837c..9b6cbd237 100644 --- a/src/data.c +++ b/src/data.c @@ -1037,22 +1037,6 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) return true; } -/* - * Move file from one backup to another. - * We do not apply compression to these files, because - * it is either small control file or already compressed cfs file. - */ -void -move_file(const char *from_root, const char *to_root, pgFile *file) -{ - char to_path[MAXPGPATH]; - - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - if (rename(file->path, to_path) == -1) - elog(ERROR, "Cannot move file \"%s\" to path \"%s\": %s", - file->path, to_path, strerror(errno)); -} - #ifdef HAVE_LIBZ /* * Show error during work with compressed file diff --git a/src/delete.c b/src/delete.c index c5f16af7d..599fd2fdd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,7 +14,6 @@ #include #include -static int delete_backup_files(pgBackup *backup); static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, uint32 xlog_seg_size); @@ -245,7 +244,7 @@ do_retention_purge(void) * Delete backup files of the backup and update the status of the backup to * BACKUP_STATUS_DELETED. */ -static int +void delete_backup_files(pgBackup *backup) { size_t i; @@ -257,11 +256,15 @@ delete_backup_files(pgBackup *backup) * If the backup was deleted already, there is nothing to do. */ if (backup->status == BACKUP_STATUS_DELETED) - return 0; + { + elog(WARNING, "Backup %s already deleted", + base36enc(backup->start_time)); + return; + } time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - elog(INFO, "delete: %s %s", + elog(INFO, "Delete: %s %s", base36enc(backup->start_time), timestamp); /* @@ -283,17 +286,17 @@ delete_backup_files(pgBackup *backup) pgFile *file = (pgFile *) parray_get(files, i); /* print progress */ - elog(VERBOSE, "delete file(%zd/%lu) \"%s\"", i + 1, + elog(VERBOSE, "Delete file(%zd/%lu) \"%s\"", i + 1, (unsigned long) parray_num(files), file->path); if (remove(file->path)) { - elog(WARNING, "can't remove \"%s\": %s", file->path, - strerror(errno)); - parray_walk(files, pgFileFree); - parray_free(files); - - return 1; + if (errno == ENOENT) + elog(VERBOSE, "File \"%s\" is absent", file->path); + else + elog(ERROR, "Cannot remove \"%s\": %s", file->path, + strerror(errno)); + return; } } @@ -301,7 +304,7 @@ delete_backup_files(pgBackup *backup) parray_free(files); backup->status = BACKUP_STATUS_DELETED; - return 0; + return; } /* diff --git a/src/merge.c b/src/merge.c index 137f1acda..3f32fff2d 100644 --- a/src/merge.c +++ b/src/merge.c @@ -77,7 +77,8 @@ do_merge(time_t backup_id) { if (backup->status != BACKUP_STATUS_OK && /* It is possible that previous merging was interrupted */ - backup->status != BACKUP_STATUS_MERGING) + backup->status != BACKUP_STATUS_MERGING && + backup->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", base36enc(backup->start_time), status2str(backup->status)); @@ -164,7 +165,14 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) int i; bool merge_isok = true; - elog(LOG, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + + /* + * Previous merging was interrupted during deleting source backup. It is + * safe just to delete it again. + */ + if (from_backup->status == BACKUP_STATUS_DELETING) + goto delete_source_backup; to_backup->status = BACKUP_STATUS_MERGING; write_backup_status(to_backup); @@ -244,37 +252,38 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) elog(ERROR, "Data files merging failed"); /* - * Files were copied into to_backup and deleted from from_backup. Remove - * remaining directories from from_backup. + * Update to_backup metadata. */ - parray_qsort(files, pgFileComparePathDesc); + to_backup->status = BACKUP_STATUS_OK; + /* Compute summary of size of regular files in the backup */ + to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - if (!S_ISDIR(file->mode)) - continue; - - if (rmdir(file->path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - file->path, strerror(errno)); + if (S_ISDIR(file->mode)) + to_backup->data_bytes += 4096; + /* Count the amount of the data actually copied */ + else if (S_ISREG(file->mode)) + to_backup->data_bytes += file->write_size; } - if (rmdir(from_database_path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - from_database_path, strerror(errno)); - if (unlink(control_file)) - elog(ERROR, "Could not remove file \"%s\": %s", - control_file, strerror(errno)); + /* compute size of wal files of this backup stored in the archive */ + if (!to_backup->stream) + to_backup->wal_bytes = xlog_seg_size * + (to_backup->stop_lsn / xlog_seg_size - + to_backup->start_lsn / xlog_seg_size + 1); + else + to_backup->wal_bytes = BYTES_INVALID; - pgBackupGetPath(from_backup, control_file, lengthof(control_file), - BACKUP_CONTROL_FILE); - if (unlink(control_file)) - elog(ERROR, "Could not remove file \"%s\": %s", - control_file, strerror(errno)); + write_backup_filelist(to_backup, files, from_database_path); + write_backup(to_backup); - if (rmdir(from_backup_path)) - elog(ERROR, "Could not remove directory \"%s\": %s", - from_backup_path, strerror(errno)); +delete_source_backup: + /* + * Files were copied into to_backup. It is time to remove source backup + * entirely. + */ + delete_backup_files(from_backup); /* * Delete files which are not in from_backup file list. @@ -286,7 +295,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); - elog(LOG, "Deleted \"%s\"", file->path); + elog(VERBOSE, "Deleted \"%s\"", file->path); } } @@ -298,34 +307,14 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup_path, from_backup_path, strerror(errno)); /* - * Update to_backup metadata. + * Merging finished, now we can safely update ID of the destination backup. */ pgBackupCopy(to_backup, from_backup); + elog(INFO, "to_backup: %s", base36enc(to_backup->start_time)); /* Correct metadata */ to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; to_backup->parent_backup = INVALID_BACKUP_ID; - /* Compute summary of size of regular files in the backup */ - to_backup->data_bytes = 0; - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - if (S_ISDIR(file->mode)) - to_backup->data_bytes += 4096; - /* Count the amount of the data actually copied */ - else if (S_ISREG(file->mode)) - to_backup->data_bytes += file->write_size; - } - /* compute size of wal files of this backup stored in the archive */ - if (!to_backup->stream) - to_backup->wal_bytes = xlog_seg_size * - (to_backup->stop_lsn / xlog_seg_size - - to_backup->start_lsn / xlog_seg_size + 1); - else - to_backup->wal_bytes = BYTES_INVALID; - - pgBackupWriteFileList(to_backup, files, from_database_path); write_backup(to_backup); /* Cleanup */ @@ -508,10 +497,9 @@ merge_files(void *arg) file->write_size = pgFileSize(to_path_tmp); file->crc = pgFileGetCRC(to_path_tmp, false); } - pgFileDelete(file); } else - move_file(argument->from_root, argument->to_root, file); + copy_file(argument->from_root, argument->to_root, file); if (file->write_size != BYTES_INVALID) elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c6ff63435..cac1d747b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -448,6 +448,7 @@ extern int do_show(time_t requested_backup_id); /* in delete.c */ extern void do_delete(time_t backup_id); +extern void delete_backup_files(pgBackup *backup); extern int do_retention_purge(void); extern int do_delete_instance(void); @@ -478,10 +479,11 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli); extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); -extern void pgBackupWriteFileList(pgBackup *backup, parray *files, +extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root); -extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); +extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, + const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); @@ -543,7 +545,6 @@ extern void restore_data_file(const char *to_path, bool write_header, uint32 backup_version); extern bool copy_file(const char *from_root, const char *to_root, pgFile *file); -extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); diff --git a/tests/merge.py b/tests/merge.py index 0169b2757..db0dc2c37 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -602,7 +602,7 @@ def test_continue_failed_merge(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('move_file') + gdb.set_breakpoint('copy_file') gdb.run_until_break() if gdb.continue_execution_until_break(20) != 'breakpoint-hit': @@ -615,3 +615,10 @@ def test_continue_failed_merge(self): # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 03c9fba7bd7157fad5f44d87a9ebe497ad9cb0f5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 24 Oct 2018 18:53:42 +0300 Subject: [PATCH 0048/2107] PGPRO-1892: Remove obsolete INFO message --- src/merge.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 3f32fff2d..5d22002d5 100644 --- a/src/merge.c +++ b/src/merge.c @@ -310,7 +310,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Merging finished, now we can safely update ID of the destination backup. */ pgBackupCopy(to_backup, from_backup); - elog(INFO, "to_backup: %s", base36enc(to_backup->start_time)); /* Correct metadata */ to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; From fcc4ed74392e727a8c4536a8b410e07a0e3202f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Oct 2018 12:17:22 +0300 Subject: [PATCH 0049/2107] PGPRO-1892: added test_continue_failed_merge_with_corrupted_delta_backup --- tests/merge.py | 128 ++++++++++++++++++++++++++++++++++++++++- tests/validate_test.py | 11 ++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index db0dc2c37..abd645657 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2,7 +2,7 @@ import unittest import os -from .helpers.ptrack_helpers import ProbackupTest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException module_name = "merge" @@ -622,3 +622,129 @@ def test_continue_failed_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_continue_failed_merge_with_corrupted_delta_backup(self): + """ + Fail merge via gdb, corrupt DELTA backup, try to continue merge + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i" + ) + + old_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500" + ) + + node.safe_psql( + "postgres", + "vacuum full t_heap" + ) + + new_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + # DELTA BACKUP + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + backup_id = self.show_pb(backup_dir, "node")[1]["id"] + + # Failed MERGE + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(2) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + # CORRUPT incremental backup + # read block from future + # block_size + backup_header = 8200 + file = os.path.join( + backup_dir, 'backups/node', backup_id_2, 'database', new_path) + with open(file, 'rb') as f: + f.seek(8200) + block_1 = f.read(8200) + f.close + + # write block from future + file = os.path.join( + backup_dir, 'backups/node', backup_id, 'database', old_path) + with open(file, 'r+b') as f: + f.seek(8200) + f.write(block_1) + f.close + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of incremental backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'INSERT ERROR MESSAGE HERE\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# 1. always use parent link when merging (intermediates may be from different chain) +# 2. page backup we are merging with may disappear after failed merge, +# it should not be possible to continue merge after that +# PAGE_A MERGING (disappear) +# FULL MERGING + +# FULL MERGING + +# PAGE_B OK (new backup) +# FULL MERGING + +# 3. Need new test with corrupted FULL backup diff --git a/tests/validate_test.py b/tests/validate_test.py index b3590de3d..ed3c7f101 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -3136,3 +3136,14 @@ def test_corrupt_pg_control_via_resetxlog(self): self.del_test_dir(module_name, fname) # validate empty backup list +# page from future during validate +# page from future during backup + +# corrupt block, so file become unaligned: +# 712 Assert(header.compressed_size <= BLCKSZ); +# 713 +# 714 read_len = fread(compressed_page.data, 1, +# 715 MAXALIGN(header.compressed_size), in); +# 716 if (read_len != MAXALIGN(header.compressed_size)) +# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", +# 718 blknum, file->path, read_len, header.compressed_size); From ee6bab40a98cae698eb452f5ae050d6b91b2e9b0 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 9 Nov 2018 18:32:37 +0300 Subject: [PATCH 0050/2107] PGPRO-1892: Add validation of merged backups --- src/merge.c | 35 +++++++++++++++++++++++++++++++---- tests/merge.py | 14 +++++--------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/merge.c b/src/merge.c index 5d22002d5..e3d6b9f83 100644 --- a/src/merge.c +++ b/src/merge.c @@ -59,7 +59,7 @@ do_merge(time_t backup_id) if (instance_name == NULL) elog(ERROR, "required parameter is not specified: --instance"); - elog(LOG, "Merge started"); + elog(INFO, "Merge started"); catalog_lock(); @@ -129,17 +129,21 @@ do_merge(time_t backup_id) */ for (i = full_backup_idx; i > dest_backup_idx; i--) { - pgBackup *to_backup = (pgBackup *) parray_get(backups, i); pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); - merge_backups(to_backup, from_backup); + full_backup = (pgBackup *) parray_get(backups, i); + merge_backups(full_backup, from_backup); } + pgBackupValidate(full_backup); + if (full_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); + /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); - elog(LOG, "Merge completed"); + elog(INFO, "Merge of backup %s completed", base36enc(backup_id)); } /* @@ -167,6 +171,28 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + /* + * Validate to_backup only if it is BACKUP_STATUS_OK. If it has + * BACKUP_STATUS_MERGING status then it isn't valid backup until merging + * finished. + */ + if (to_backup->status == BACKUP_STATUS_OK) + { + pgBackupValidate(to_backup); + if (to_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Interrupt merging"); + } + + /* + * It is OK to validate from_backup if it has BACKUP_STATUS_OK or + * BACKUP_STATUS_MERGING status. + */ + Assert(from_backup->status == BACKUP_STATUS_OK || + from_backup->status == BACKUP_STATUS_MERGING); + pgBackupValidate(from_backup); + if (from_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Interrupt merging"); + /* * Previous merging was interrupted during deleting source backup. It is * safe just to delete it again. @@ -302,6 +328,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Rename FULL backup directory. */ + elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); if (rename(to_backup_path, from_backup_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", to_backup_path, from_backup_path, strerror(errno)); diff --git a/tests/merge.py b/tests/merge.py index abd645657..826d19f19 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -694,8 +694,6 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): gdb._execute('signal SIGKILL') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - # CORRUPT incremental backup # read block from future # block_size + backup_header = 8200 @@ -723,16 +721,14 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( - e.message, - 'INSERT ERROR MESSAGE HERE\n', + self.assertTrue( + "WARNING: Backup {0} data files are corrupted".format( + backup_id) in e.message and + "ERROR: Merging of backup {0} failed".format( + backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - # Clean after yourself self.del_test_dir(module_name, fname) From 8e716791ba1f6942d93e7ddfd6e983663fe4bd3b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 11 Nov 2018 21:53:00 +0300 Subject: [PATCH 0051/2107] tests: minor fixes --- tests/compatibility.py | 159 ++++++++++++++++++++++++++ tests/compression.py | 67 +++++++++++ tests/replica.py | 248 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 464 insertions(+), 10 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 3d67bf3e0..7c3e137f9 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -311,3 +311,162 @@ def test_backward_compatibility_ptrack(self): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_compression(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # restore OLD FULL with new binary + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname)) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with OLD binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with old binary + self.delete_pb(backup_dir, 'node', backup_id) + + self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress'], + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/compression.py b/tests/compression.py index aa2753821..54bc299c3 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -494,3 +494,70 @@ def test_compression_wrong_algorithm(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_uncompressable_pages(self): + """ + make archive node, create table with uncompressable toast pages, + take backup with compression, make sure that page was not compressed, + restore backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + +# node.safe_psql( +# "postgres", +# "create table t_heap as select i, " +# "repeat('1234567890abcdefghiyklmn', 1)::bytea, " +# "point(0,0) from generate_series(0,1) i") + + node.safe_psql( + "postgres", + "create table t as select i, " + "repeat(md5(i::text),5006056) as fat_attr " + "from generate_series(0,10) i;") + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', + options=[ + '--compress', + '--log-level-file=verbose']) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', + options=[ + '--compress', + '--log-level-file=verbose']) + + # Clean after yourself + # self.del_test_dir(module_name, fname) + +# create table t as select i, repeat(md5('1234567890'), 1)::bytea, point(0,0) from generate_series(0,1) i; + + +# create table t_bytea_1(file oid); +# INSERT INTO t_bytea_1 (file) +# VALUES (lo_import('/home/gsmol/git/postgres/contrib/pg_probackup/tests/expected/sample.random', 24593)); +# insert into t_bytea select string_agg(data,'') from pg_largeobject where pageno > 0; +# \ No newline at end of file diff --git a/tests/replica.py b/tests/replica.py index d74c375c2..f8c16607c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -5,6 +5,7 @@ import subprocess from sys import exit import time +from shutil import copyfile module_name = 'replica' @@ -64,6 +65,7 @@ def test_replica_stream_ptrack_backup(self): "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -80,9 +82,11 @@ def test_replica_stream_ptrack_backup(self): base_dir="{0}/{1}/node".format(module_name, fname)) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -95,7 +99,9 @@ def test_replica_stream_ptrack_backup(self): "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', options=[ @@ -111,9 +117,11 @@ def test_replica_stream_ptrack_backup(self): node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -136,13 +144,12 @@ def test_replica_archive_page_backup(self): pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} + 'checkpoint_timeout': '30s', + 'archive_timeout': '10s'} ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - # force more frequent wal switch - master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') master.slow_start() replica = self.make_simple_node( @@ -180,8 +187,14 @@ def test_replica_archive_page_backup(self): "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000003'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -201,9 +214,11 @@ def test_replica_archive_page_backup(self): node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + node.cleanup() # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal @@ -212,30 +227,41 @@ def test_replica_archive_page_backup(self): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") + "from generate_series(512,22680) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( - backup_dir, 'replica', replica, backup_type='page', + backup_dir, 'replica', + replica, backup_type='page', options=[ '--archive-timeout=300', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) # RESTORE PAGE BACKUP TAKEN FROM replica - node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.append_conf( + 'postgresql.auto.conf', 'archive_mode = off') node.slow_start() + # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + self.add_instance(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -279,15 +305,217 @@ def test_make_replica_via_restore(self): backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') self.restore_node( - backup_dir, 'master', replica, - options=['-R', '--recovery-target-action=promote']) + backup_dir, 'master', replica, options=['-R']) # Settings for Replica - # self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.start() + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.slow_start(replica=True) + + self.add_instance(backup_dir, 'replica', replica) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000003'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + + self.backup_node(backup_dir, 'replica', replica) # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_take_backup_from_delayed_replica(self): + """ + make archive master, take full backups from master, + restore full backup as delayed replica, launch pgbench, + take FULL, PAGE and DELTA backups from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + # stupid hack + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000001'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) + + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.append_conf( + 'recovery.conf', "recovery_min_apply_delay = '300s'") + + replica.slow_start(replica=True) + + master.pgbench_init(scale=10) + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', replica) + + self.backup_node( + backup_dir, 'replica', replica, + data_dir=replica.data_dir, backup_type='page') + + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta') + + pgbench.wait() + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', replica, + options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + pgbench.wait() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_make_block_from_future(self): + """ + make archive master, take full backups from master, + restore full backup as replica, launch pgbench, + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir="{0}/{1}/master".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', 'max_wal_senders': '2', + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + # force more frequent wal switch + #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') + master.slow_start() + + replica = self.make_simple_node( + base_dir="{0}/{1}/replica".format(module_name, fname)) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf( + 'postgresql.auto.conf', 'hot_standby = on') + + replica.slow_start(replica=True) + + self.add_instance(backup_dir, 'replica', replica) + + replica.safe_psql( + 'postgres', + 'checkpoint') + + master.pgbench_init(scale=10) + + self.wait_until_replica_catch_with_master(master, replica) + + +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_last_xlog_receive_location()')) +# +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_last_xlog_replay_location()')) +# +# print(replica.safe_psql( +# 'postgres', +# 'select * from pg_catalog.pg_control_checkpoint()')) +# +# replica.safe_psql( +# 'postgres', +# 'checkpoint') + + pgbench = master.pgbench(options=['-T', '30', '-c', '2', '--no-vacuum']) + + time.sleep(5) + + #self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + exit(1) + self.backup_node(backup_dir, 'replica', replica, options=["--log-level-file=verbose"]) + pgbench.wait() + + # pgbench + master.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256000) i") + + + master.safe_psql( + 'postgres', + 'checkpoint') + + replica.safe_psql( + 'postgres', + 'checkpoint') + + replica.safe_psql( + 'postgres', + 'select * from pg_') + + self.backup_node(backup_dir, 'replica', replica) + exit(1) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From d2271554a2173270857e57d715d1d64a07e6ebaf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 11:51:58 +0300 Subject: [PATCH 0052/2107] tests: minor fixes --- tests/archive.py | 70 +++++++++++++++++++++++++++-------------- tests/backup_test.py | 2 +- tests/compatibility.py | 9 ++---- tests/compression.py | 10 ++---- tests/delta.py | 34 +++++++------------- tests/exclude.py | 2 +- tests/false_positive.py | 4 +-- tests/merge.py | 4 +-- tests/page.py | 11 +++---- tests/ptrack.py | 52 +++++++++++++----------------- tests/ptrack_clean.py | 2 +- tests/ptrack_empty.py | 2 +- tests/replica.py | 26 ++++++++++----- tests/validate_test.py | 24 ++++++-------- 14 files changed, 125 insertions(+), 127 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 8b8eb71aa..4ed783d62 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -5,6 +5,7 @@ import subprocess from sys import exit from time import sleep +from shutil import copyfile module_name = 'archive' @@ -39,8 +40,7 @@ def test_pgpro434_1(self): result = node.safe_psql("postgres", "SELECT * FROM t_heap") self.backup_node( - backup_dir, 'node', node, - options=["--log-level-file=verbose"]) + backup_dir, 'node', node) node.cleanup() self.restore_node( @@ -53,8 +53,7 @@ def test_pgpro434_1(self): # Make backup self.backup_node( - backup_dir, 'node', node, - options=["--log-level-file=verbose"]) + backup_dir, 'node', node) node.cleanup() # Restore Database @@ -253,7 +252,6 @@ def test_pgpro434_3(self): backup_dir, 'node', node, options=[ "--archive-timeout=60", - "--log-level-file=verbose", "--stream"] ) # we should die here because exception is what we expect to happen @@ -402,7 +400,7 @@ def test_arhive_push_file_exists_overwrite(self): self.del_test_dir(module_name, fname) # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_replica_archive(self): """ make node without archiving, take stream backup and @@ -417,7 +415,7 @@ def test_replica_archive(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_senders': '2', - 'checkpoint_timeout': '30s', + 'archive_timeout': '10s', 'max_wal_size': '1GB'} ) self.init_pb(backup_dir) @@ -433,7 +431,7 @@ def test_replica_archive(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") + "from generate_series(0,2560) i") self.backup_node(backup_dir, 'master', master, options=['--stream']) before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -459,9 +457,6 @@ def test_replica_archive(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") - # ADD INSTANCE 'REPLICA' - - sleep(1) backup_id = self.backup_node( backup_dir, 'replica', replica, @@ -469,7 +464,9 @@ def test_replica_archive(self): '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -493,16 +490,28 @@ def test_replica_archive(self): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") + "from generate_series(512,20680) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + master.safe_psql( + "postgres", + "CHECKPOINT") + +# copyfile( +# os.path.join(backup_dir, 'wal/master/000000010000000000000002'), +# os.path.join(backup_dir, 'wal/replica/000000010000000000000002')) + backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=30', '--log-level-file=verbose', - '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)] - ) + '--archive-timeout=30', + '--master-db=postgres', + '--master-host=localhost', + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -511,8 +520,10 @@ def test_replica_archive(self): node.cleanup() self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") @@ -537,7 +548,7 @@ def test_master_and_replica_parallel_archiving(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} + 'archive_timeout': '10s'} ) replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) @@ -568,7 +579,7 @@ def test_master_and_replica_parallel_archiving(self): pgdata_replica = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata_master, pgdata_replica) - self.set_replica(master, replica, synchronous=True) + self.set_replica(master, replica) # ADD INSTANCE REPLICA self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA @@ -579,16 +590,26 @@ def test_master_and_replica_parallel_archiving(self): after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0, 60000) i") + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000001'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ - '--archive-timeout=20', - '--log-level-file=verbose', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)] - ) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -618,7 +639,8 @@ def test_master_and_replica_concurrent_archiving(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} + 'checkpoint_timeout': '30s', + 'archive_timeout': '10s'} ) replica = self.make_simple_node( base_dir="{0}/{1}/replica".format(module_name, fname)) diff --git a/tests/backup_test.py b/tests/backup_test.py index b21aab9b4..9c8a09558 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -328,7 +328,7 @@ def test_page_corruption_heal_via_ptrack_1(self): self.backup_node( backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", '--log-level-file=verbose']) + options=["-j", "4", "--stream", "--log-level-file=verbose"]) # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: diff --git a/tests/compatibility.py b/tests/compatibility.py index 7c3e137f9..39070b3ff 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -94,8 +94,7 @@ def test_backward_compatibility_page(self): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -195,8 +194,7 @@ def test_backward_compatibility_delta(self): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -296,8 +294,7 @@ def test_backward_compatibility_ptrack(self): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/compression.py b/tests/compression.py index 54bc299c3..2e712a152 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -55,9 +55,7 @@ def test_compression_stream_zlib(self): page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=[ - '--stream', '--compress-algorithm=zlib', - '--log-level-console=verbose', - '--log-level-file=verbose']) + '--stream', '--compress-algorithm=zlib']) # PTRACK BACKUP node.safe_psql( @@ -535,8 +533,7 @@ def test_uncompressable_pages(self): backup_dir, 'node', node, backup_type='full', options=[ - '--compress', - '--log-level-file=verbose']) + '--compress']) node.cleanup() @@ -547,8 +544,7 @@ def test_uncompressable_pages(self): backup_dir, 'node', node, backup_type='full', options=[ - '--compress', - '--log-level-file=verbose']) + '--compress']) # Clean after yourself # self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index bdbfac910..55cc03bec 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -80,13 +80,7 @@ def test_delta_vacuum_truncate_1(self): pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, - 'node', - node_restored, - options=[ - "-j", "1", - "--log-level-file=verbose" - ] + backup_dir, 'node', node_restored ) # Physical comparison @@ -176,8 +170,6 @@ def test_delta_vacuum_truncate_2(self): 'node', node_restored, options=[ - "-j", "1", - "--log-level-file=verbose", "-T", "{0}={1}".format( old_tablespace, new_tablespace)] ) @@ -251,13 +243,7 @@ def test_delta_vacuum_truncate_3(self): pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, - 'node', - node_restored, - options=[ - "-j", "1", - "--log-level-file=verbose" - ] + backup_dir, 'node', node_restored ) # Physical comparison @@ -683,7 +669,7 @@ def test_create_db(self): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"]) @@ -717,7 +703,7 @@ def test_create_db(self): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"] ) @@ -815,7 +801,7 @@ def test_exists_in_previous_backup(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) # if self.paranoia: # pgdata_delta = self.pgdata_content( @@ -844,7 +830,7 @@ def test_exists_in_previous_backup(self): node_restored, backup_id=backup_id, options=[ - "-j", "4", "--log-level-file=verbose", + "-j", "4", "--immediate", "--recovery-target-action=promote"]) @@ -1135,7 +1121,7 @@ def test_delta_delete(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_1(self): + def test_delta_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -1174,8 +1160,10 @@ def test_page_corruption_heal_via_ptrack_1(self): f.close self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream", "--log-level-file=verbose"]) + backup_dir, 'node', node, + backup_type="delta", + options=["-j", "4", "--stream", '--log-level-file=verbose']) + # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: diff --git a/tests/exclude.py b/tests/exclude.py index 48b7889c7..3fd3341f2 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -143,7 +143,7 @@ def test_exclude_unlogged_tables_1(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/false_positive.py b/tests/false_positive.py index 04062b791..df7b13343 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -143,7 +143,7 @@ def test_ptrack_concurrent_get_and_clear_1(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) @@ -227,7 +227,7 @@ def test_ptrack_concurrent_get_and_clear_2(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) diff --git a/tests/merge.py b/tests/merge.py index 826d19f19..5f7ae7da9 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -407,17 +407,17 @@ def test_merge_ptrack_truncate(self): node.safe_psql( "postgres", "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( "postgres", "vacuum t_heap") - self.backup_node( + page_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - page_id = self.show_pb(backup_dir, "node")[1]["id"] self.merge_backup(backup_dir, "node", page_id) self.validate_pb(backup_dir) diff --git a/tests/page.py b/tests/page.py index 1d8238e17..d31b8f60f 100644 --- a/tests/page.py +++ b/tests/page.py @@ -62,8 +62,7 @@ def test_page_vacuum_truncate(self): "vacuum t_heap") self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -333,8 +332,7 @@ def test_page_multiple_segments(self): result = node.safe_psql("postgres", "select * from pgbench_accounts") # PAGE BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=["--log-level-file=verbose"]) + backup_dir, 'node', node, backup_type='page') # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -727,7 +725,7 @@ def test_page_backup_with_lost_wal_segment(self): self.backup_node( backup_dir, 'node', node, backup_type='page', - options=["-j", "4", '--log-level-file=verbose']) + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " @@ -797,8 +795,7 @@ def test_page_backup_with_corrupted_wal_segment(self): # Single-thread PAGE backup try: self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=verbose']) + backup_dir, 'node', node, backup_type='page') self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " diff --git a/tests/ptrack.py b/tests/ptrack.py index 721593186..5d01d8823 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -157,13 +157,13 @@ def test_ptrack_uncommited_xact(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) self.restore_node( @@ -246,14 +246,11 @@ def test_ptrack_vacuum_full(self): exit(1) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') + if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -336,14 +333,10 @@ def test_ptrack_vacuum_truncate(self): ) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--log-level-file=verbose'] - ) + backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -409,7 +402,7 @@ def test_ptrack_simple(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) node.safe_psql( @@ -479,7 +472,7 @@ def test_ptrack_get_block(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'], + options=['--stream'], gdb=True ) @@ -566,7 +559,7 @@ def test_ptrack_stream(self): ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--log-level-file=verbose'] + options=['--stream'] ) if self.paranoia: @@ -989,7 +982,7 @@ def test_create_db(self): node.safe_psql("postgres", "SELECT * FROM t_heap") self.backup_node( backup_dir, 'node', node, - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) # CREATE DATABASE DB1 node.safe_psql("postgres", "create database db1") @@ -1002,7 +995,7 @@ def test_create_db(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) if self.paranoia: @@ -1133,7 +1126,8 @@ def test_create_db_on_replica(self): '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(node.port) + '--master-port={0}'.format(node.port), + '--stream' ] ) @@ -1229,7 +1223,7 @@ def test_alter_table_set_tablespace_ptrack(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"] + options=["--stream"] ) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1315,7 +1309,7 @@ def test_alter_database_set_tablespace_ptrack(self): # PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", '--log-level-file=verbose']) + options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1476,7 +1470,7 @@ def test_ptrack_alter_tablespace(self): # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) # GET PHYSICAL CONTENT FROM NODE if self.paranoia: @@ -1517,7 +1511,7 @@ def test_ptrack_alter_tablespace(self): # SECOND PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "--log-level-file=verbose"]) + options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1612,9 +1606,8 @@ def test_ptrack_multiple_segments(self): #result = node.safe_psql("postgres", "select * from pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--log-level-file=verbose"] - ) + backup_dir, 'node', node, backup_type='ptrack') + # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -1683,9 +1676,8 @@ def test_atexit_fail(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=[ - "--stream", "-j 30", - "--log-level-file=verbose"] - ) + "--stream", "-j 30"]) + # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index ae16c6627..076291a69 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -76,7 +76,7 @@ def test_ptrack_clean(self): # Take PTRACK backup to clean every ptrack backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['-j10', '--log-level-file=verbose']) + options=['-j10']) node.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 750a73361..8656f9413 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -67,7 +67,7 @@ def test_ptrack_clean(self): # Take PTRACK backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['-j10', '--log-level-file=verbose']) + options=['-j10']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/replica.py b/tests/replica.py index f8c16607c..1ab8515ec 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -162,7 +162,7 @@ def test_replica_archive_page_backup(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") + "from generate_series(0,2560) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -173,6 +173,7 @@ def test_replica_archive_page_backup(self): # Settings for Replica self.set_replica(master, replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) # Check data correctness on replica @@ -186,7 +187,7 @@ def test_replica_archive_page_backup(self): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") + "from generate_series(256,5120) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") self.add_instance(backup_dir, 'replica', replica) @@ -195,13 +196,23 @@ def test_replica_archive_page_backup(self): os.path.join(backup_dir, 'wal/master/000000010000000000000003'), os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000004'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000004')) + + copyfile( + os.path.join(backup_dir, 'wal/master/000000010000000000000005'), + os.path.join(backup_dir, 'wal/replica/000000010000000000000005')) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ - '--archive-timeout=300', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -235,10 +246,11 @@ def test_replica_archive_page_backup(self): backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=300', + '--archive-timeout=30', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -491,7 +503,7 @@ def test_make_block_from_future(self): #self.backup_node(backup_dir, 'replica', replica, options=['--stream']) exit(1) - self.backup_node(backup_dir, 'replica', replica, options=["--log-level-file=verbose"]) + self.backup_node(backup_dir, 'replica', replica) pgbench.wait() # pgbench diff --git a/tests/validate_test.py b/tests/validate_test.py index ed3c7f101..c0fd49439 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -50,7 +50,7 @@ def test_validate_nullified_heap_page_backup(self): f.close self.backup_node( - backup_dir, 'node', node, options=["--log-level-file=verbose"]) + backup_dir, 'node', node, options=['--log-level-file=verbose']) log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") @@ -259,8 +259,7 @@ def test_validate_corrupted_intermediate_backup(self): # Simple validate try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, - options=['--log-level-file=verbose']) + backup_dir, 'node', backup_id=backup_id_2) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -364,8 +363,7 @@ def test_validate_corrupted_intermediate_backups(self): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, - options=['--log-level-file=verbose']) + backup_dir, 'node', backup_id=backup_id_2) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -520,8 +518,7 @@ def test_validate_corrupted_intermediate_backups_1(self): try: self.validate_pb( backup_dir, 'node', - backup_id=backup_id_4, - options=['--log-level-file=verbose']) + backup_id=backup_id_4) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n" @@ -721,7 +718,6 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.validate_pb( backup_dir, 'node', options=[ - '--log-level-file=verbose', '-i', backup_id_4, '--xid={0}'.format(target_xid)]) self.assertEqual( 1, 0, @@ -866,7 +862,7 @@ def test_validate_instance_with_corrupted_page(self): # Validate Instance try: self.validate_pb( - backup_dir, 'node', options=['--log-level-file=verbose']) + backup_dir, 'node') self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1006,7 +1002,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1092,7 +1088,7 @@ def test_validate_instance_with_corrupted_full(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node', options=['--log-level-file=verbose']) + self.validate_pb(backup_dir, 'node') self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1219,7 +1215,6 @@ def test_validate_corrupt_wal_2(self): 'node', backup_id, options=[ - "--log-level-console=verbose", "--xid={0}".format(target_xid)]) self.assertEqual( 1, 0, @@ -1388,7 +1383,6 @@ def test_validate_corrupt_wal_between_backups(self): 'node', backup_id, options=[ - "--log-level-console=verbose", "--xid={0}".format(target_xid)]) self.assertEqual( 1, 0, @@ -1671,7 +1665,7 @@ def test_validate_corrupted_full(self): os.rename(file_new, file) try: - self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + self.validate_pb(backup_dir) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -1776,7 +1770,7 @@ def test_validate_corrupted_full_1(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir, options=['--log-level-file=verbose']) + self.validate_pb(backup_dir) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( From 6eefeeba8dfe7280252d3abb1e6f424e0f2929ac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 12:15:54 +0300 Subject: [PATCH 0053/2107] tests: expected help fixed --- tests/expected/option_help.out | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 228598ed1..ecc59a893 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -50,6 +50,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-db=db_name] [--master-host=host_name] [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] + [--skip-block-validation] pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [--progress] @@ -59,12 +60,14 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] + [--skip-block-validation] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--recovery-target-name=target-name] [--timeline=timeline] + [--skip-block-validation] pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] From de9c3b64d72693283b6514f0d0d5773613b3c971 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Nov 2018 12:16:34 +0300 Subject: [PATCH 0054/2107] Version 2.0.24 - Major bugfix: incorrect handling of badly compressed blocks, previously there was a risk to restore block in uncompressed state, if compressed size was equal or larger than BLCKSZ - Impromevent: backup from replica >= 9.6 no longer need connection to master - Workaround: wrong minRecPoint in PostgreSQL thanks to commit 8d68ee6(block from future), overwrite minRecPoint with latest applied LSN - Impromevent: merge is now considered stable feature - Impromevent: validation now use more conservative and paranoid approach to file validation, during validation pg_probackup also check block checksumm, make sanity check based on block header information and try to detect blocks from future - New validate/restore options: '--skip-block-validation' - disable aforementioned approach to file validation - Multiple minor fixes --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ea99672dc..00b0fc428 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -17,7 +17,7 @@ #include "utils/thread.h" -const char *PROGRAM_VERSION = "2.0.23"; +const char *PROGRAM_VERSION = "2.0.24"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 6a0391c2e..5280a6f95 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.23 \ No newline at end of file +pg_probackup 2.0.24 \ No newline at end of file From 1e9615f56740d2e31ae693bd11234988b7db1627 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 12 Nov 2018 15:44:22 +0300 Subject: [PATCH 0055/2107] Use InvalidXLogRecPtr to mark infinite end, a couple code cleanup --- src/backup.c | 10 ++++++++-- src/pg_probackup.h | 1 - src/restore.c | 14 +++++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2317bee96..06796e153 100644 --- a/src/backup.c +++ b/src/backup.c @@ -474,6 +474,7 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; + parray *backup_list = NULL; pgFile *pg_control = NULL; @@ -515,7 +516,6 @@ do_backup_instance(void) current.backup_mode == BACKUP_MODE_DIFF_PTRACK || current.backup_mode == BACKUP_MODE_DIFF_DELTA) { - parray *backup_list; char prev_backup_filelist_path[MAXPGPATH]; /* get list of backups already taken */ @@ -525,7 +525,6 @@ do_backup_instance(void) if (prev_backup == NULL) elog(ERROR, "Valid backup on current timeline is not found. " "Create new FULL backup before an incremental one."); - parray_free(backup_list); pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); @@ -832,6 +831,13 @@ do_backup_instance(void) current.data_bytes += file->write_size; } + /* Cleanup */ + if (backup_list) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } + parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); backup_files_list = NULL; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ca45d6042..0fad6d70d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -418,7 +418,6 @@ extern int do_restore_or_validate(time_t target_backup_id, extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); -extern parray * readTimeLineHistory_probackup(TimeLineID targetTLI); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, diff --git a/src/restore.c b/src/restore.c index 439f3c4e1..9cf33515e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -33,6 +33,7 @@ static void restore_backup(pgBackup *backup); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); +static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); @@ -138,7 +139,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); /* Read timeline history files from archives */ - timelines = readTimeLineHistory_probackup(rt->recovery_target_tli); + timelines = read_timeline_history(rt->recovery_target_tli); if (!satisfy_timeline(timelines, current_backup)) { @@ -149,6 +150,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Try to find another backup that satisfies target timeline */ continue; } + + parray_walk(timelines, pfree); + parray_free(timelines); } if (!satisfy_recovery_target(current_backup, rt)) @@ -731,7 +735,7 @@ create_recovery_conf(time_t backup_id, * based on readTimeLineHistory() in timeline.c */ parray * -readTimeLineHistory_probackup(TimeLineID targetTLI) +read_timeline_history(TimeLineID targetTLI) { parray *result; char path[MAXPGPATH]; @@ -820,8 +824,7 @@ readTimeLineHistory_probackup(TimeLineID targetTLI) entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; /* LSN in target timeline is valid */ - /* TODO ensure that -1UL --> -1L fix is correct */ - entry->end = (uint32) (-1L << 32) | -1L; + entry->end = InvalidXLogRecPtr; parray_insert(result, 0, entry); return result; @@ -853,7 +856,8 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && - backup->stop_lsn < timeline->end) + (XLogRecPtrIsInvalid(timeline->end) || + backup->stop_lsn < timeline->end)) return true; } return false; From 0a510f7211a45d6258039ca0a64d82615ba67f1c Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Nov 2018 13:11:26 +0300 Subject: [PATCH 0056/2107] Some fixes for windows build --- src/backup.c | 1 + src/parsexlog.c | 1 + src/pg_probackup.c | 1 + src/utils/logger.c | 1 + src/utils/thread.c | 5 ++++- src/utils/thread.h | 6 ++++++ 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 06796e153..380dc1c0c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -22,6 +22,7 @@ #include #include "utils/thread.h" +#include #define PG_STOP_BACKUP_TIMEOUT 300 diff --git a/src/parsexlog.c b/src/parsexlog.c index ee7b5076f..ed73f70c5 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -22,6 +22,7 @@ #endif #include "utils/thread.h" +#include /* * RmgrNames is an array of resource manager names, to make error messages diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 00b0fc428..e8240e19d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -16,6 +16,7 @@ #include #include "utils/thread.h" +#include const char *PROGRAM_VERSION = "2.0.24"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; diff --git a/src/utils/logger.c b/src/utils/logger.c index 4cdbf7211..05f6fc5bf 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -14,6 +14,7 @@ #include "logger.h" #include "pgut.h" #include "thread.h" +#include /* Logger parameters */ diff --git a/src/utils/thread.c b/src/utils/thread.c index 82c237641..0999a0d5b 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -9,8 +9,11 @@ #include "thread.h" +#ifdef WIN32 +DWORD main_tid = 0; +#else pthread_t main_tid = 0; - +#endif #ifdef WIN32 #include diff --git a/src/utils/thread.h b/src/utils/thread.h index 064605331..6b8349bf5 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -28,7 +28,13 @@ extern int pthread_join(pthread_t th, void **thread_return); #include #endif +#ifdef WIN32 +extern DWORD main_tid; +#else extern pthread_t main_tid; +#endif + + extern int pthread_lock(pthread_mutex_t *mp); From 5179f0219f9b79c835fd45f6548cd6224c8f3352 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 13 Nov 2018 13:46:29 +0300 Subject: [PATCH 0057/2107] Changed the script for creating the Windows project --- gen_probackup_project.pl | 75 ++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index e14f1d4be..d7f88c4db 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -1,10 +1,15 @@ # -*-perl-*- hey - emacs - this is a perl file -BEGIN{ +# my $currpath = cwd(); + +our $pgsrc; +our $currpath; + +BEGIN { +# path to the pg_pprobackup dir +$currpath = File::Basename::dirname(Cwd::abs_path($0)); use Cwd; use File::Basename; - -my $pgsrc=""; -if (@ARGV==1) +if (($#ARGV+1)==1) { $pgsrc = shift @ARGV; if($pgsrc eq "--help"){ @@ -23,14 +28,13 @@ BEGIN chdir($path); chdir("../.."); $pgsrc = cwd(); + $currpath = "contrib/pg_probackup"; } - chdir("$pgsrc/src/tools/msvc"); push(@INC, "$pgsrc/src/tools/msvc"); chdir("../../..") if (-d "../msvc" && -d "../../../src"); } - use Win32; use Carp; use strict; @@ -84,22 +88,27 @@ BEGIN my $bconf = $ENV{CONFIG} || "Release"; my $msbflags = $ENV{MSBFLAGS} || ""; my $buildwhat = $ARGV[1] || ""; -if (uc($ARGV[0]) eq 'DEBUG') -{ - $bconf = "Debug"; -} -elsif (uc($ARGV[0]) ne "RELEASE") -{ - $buildwhat = $ARGV[0] || ""; -} +# if (uc($ARGV[0]) eq 'DEBUG') +# { +# $bconf = "Debug"; +# } +# elsif (uc($ARGV[0]) ne "RELEASE") +# { +# $buildwhat = $ARGV[0] || ""; +# } + +# printf "currpath=$currpath"; + +# exit(0); # ... and do it system("msbuild pg_probackup.vcxproj /verbosity:normal $msbflags /p:Configuration=$bconf" ); - # report status my $status = $? >> 8; +printf("Status: $status\n"); +printf("Output file built in the folder $pgsrc/$bconf/pg_probackup\n"); exit $status; @@ -126,10 +135,10 @@ sub build_pgprobackup #vvs test my $probackup = - $solution->AddProject('pg_probackup', 'exe', 'pg_probackup'); #, 'contrib/pg_probackup' + $solution->AddProject("pg_probackup", 'exe', "pg_probackup"); #, 'contrib/pg_probackup' $probackup->AddDefine('FRONTEND'); $probackup->AddFiles( - 'contrib/pg_probackup/src', + "$currpath/src", 'archive.c', 'backup.c', 'catalog.c', @@ -149,39 +158,39 @@ sub build_pgprobackup 'validate.c' ); $probackup->AddFiles( - 'contrib/pg_probackup/src/utils', + "$currpath/src/utils", 'json.c', 'logger.c', 'parray.c', 'pgut.c', 'thread.c' ); - $probackup->AddFile('src/backend/access/transam/xlogreader.c'); - $probackup->AddFile('src/backend/utils/hash/pg_crc.c'); + $probackup->AddFile("$pgsrc/src/backend/access/transam/xlogreader.c"); + $probackup->AddFile("$pgsrc/src/backend/utils/hash/pg_crc.c"); $probackup->AddFiles( - 'src/bin/pg_basebackup', + "$pgsrc/src/bin/pg_basebackup", 'receivelog.c', 'streamutil.c' ); - if (-e 'src/bin/pg_basebackup/walmethods.c') + if (-e "$pgsrc/src/bin/pg_basebackup/walmethods.c") { - $probackup->AddFile('src/bin/pg_basebackup/walmethods.c'); + $probackup->AddFile("$pgsrc/src/bin/pg_basebackup/walmethods.c"); } - $probackup->AddFile('src/bin/pg_rewind/datapagemap.c'); + $probackup->AddFile("$pgsrc/src/bin/pg_rewind/datapagemap.c"); - $probackup->AddFile('src/interfaces/libpq/pthread-win32.c'); + $probackup->AddFile("$pgsrc/src/interfaces/libpq/pthread-win32.c"); - $probackup->AddIncludeDir('src/bin/pg_basebackup'); - $probackup->AddIncludeDir('src/bin/pg_rewind'); - $probackup->AddIncludeDir('src/interfaces/libpq'); - $probackup->AddIncludeDir('src'); - $probackup->AddIncludeDir('src/port'); + $probackup->AddIncludeDir("$pgsrc/src/bin/pg_basebackup"); + $probackup->AddIncludeDir("$pgsrc/src/bin/pg_rewind"); + $probackup->AddIncludeDir("$pgsrc/src/interfaces/libpq"); + $probackup->AddIncludeDir("$pgsrc/src"); + $probackup->AddIncludeDir("$pgsrc/src/port"); - $probackup->AddIncludeDir('contrib/pg_probackup'); - $probackup->AddIncludeDir('contrib/pg_probackup/src'); - $probackup->AddIncludeDir('contrib/pg_probackup/src/utils'); + $probackup->AddIncludeDir("$currpath"); + $probackup->AddIncludeDir("$currpath/src"); + $probackup->AddIncludeDir("$currpath/src/utils"); $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); $probackup->AddLibrary('ws2_32.lib'); From a769f47217f742bf0356d5602a25bd18c021a3ed Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Tue, 13 Nov 2018 15:03:26 +0300 Subject: [PATCH 0058/2107] Fix compilation under FreeBSD and Solaris --- src/data.c | 2 ++ src/parsexlog.c | 1 + src/util.c | 2 ++ 3 files changed, 5 insertions(+) diff --git a/src/data.c b/src/data.c index 9b6cbd237..b1acfecc4 100644 --- a/src/data.c +++ b/src/data.c @@ -14,6 +14,8 @@ #include "storage/checksum_impl.h" #include +#include + #include #ifdef HAVE_LIBZ diff --git a/src/parsexlog.c b/src/parsexlog.c index ed73f70c5..86827a852 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -22,6 +22,7 @@ #endif #include "utils/thread.h" +#include #include /* diff --git a/src/util.c b/src/util.c index 94a18a9a4..f28206501 100644 --- a/src/util.c +++ b/src/util.c @@ -16,6 +16,8 @@ #include +#include + const char * base36enc(long unsigned int value) { From 4a1ca601afec4939f2e7e5395ff5137c4d1bcda5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 13 Nov 2018 15:49:09 +0300 Subject: [PATCH 0059/2107] PGPRO-2160: to_files may be uninitialized if from_backup has BACKUP_STATUS_DELETING --- src/merge.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/merge.c b/src/merge.c index e3d6b9f83..13263c648 100644 --- a/src/merge.c +++ b/src/merge.c @@ -164,7 +164,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) control_file[MAXPGPATH]; parray *files, *to_files; - pthread_t *threads; + pthread_t *threads = NULL; merge_files_arg *threads_args; int i; bool merge_isok = true; @@ -193,19 +193,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (from_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Interrupt merging"); - /* - * Previous merging was interrupted during deleting source backup. It is - * safe just to delete it again. - */ - if (from_backup->status == BACKUP_STATUS_DELETING) - goto delete_source_backup; - - to_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(to_backup); - - from_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(from_backup); - /* * Make backup paths. */ @@ -216,8 +203,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), DATABASE_DIR); - create_data_directories(to_database_path, from_backup_path, false); - /* * Get list of files which will be modified or removed. */ @@ -238,6 +223,21 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); + /* + * Previous merging was interrupted during deleting source backup. It is + * safe just to delete it again. + */ + if (from_backup->status == BACKUP_STATUS_DELETING) + goto delete_source_backup; + + to_backup->status = BACKUP_STATUS_MERGING; + write_backup_status(to_backup); + + from_backup->status = BACKUP_STATUS_MERGING; + write_backup_status(from_backup); + + create_data_directories(to_database_path, from_backup_path, false); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); @@ -344,8 +344,11 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) write_backup(to_backup); /* Cleanup */ - pfree(threads_args); - pfree(threads); + if (threads) + { + pfree(threads_args); + pfree(threads); + } parray_walk(to_files, pgFileFree); parray_free(to_files); From 8505e78c9139fbe6fa82e8bdaa1d1043436247c3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 13 Nov 2018 18:15:19 +0300 Subject: [PATCH 0060/2107] Bug fix: do not add root slash for pg_wal path --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 86827a852..5b2e32afd 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -536,8 +536,8 @@ validate_wal(pgBackup *backup, const char *archivedir, */ if (backup->stream) { - snprintf(backup_xlog_path, sizeof(backup_xlog_path), "/%s/%s/%s/%s", - backup_instance_path, backup_id, DATABASE_DIR, PG_XLOG_DIR); + pgBackupGetPath2(backup, backup_xlog_path, lengthof(backup_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, seg_size); From 0e445a99e8749719f3c68fad074ea650c7e5a8e0 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 14 Nov 2018 18:10:57 +0300 Subject: [PATCH 0061/2107] Removed obsolete scripts and project templates for Windows --- doit.cmd | 4 +- msvs/pg_probackup.sln | 28 ---- msvs/template.pg_probackup.vcxproj | 213 ------------------------ msvs/template.pg_probackup96.vcxproj | 211 ----------------------- msvs/template.pg_probackup_2.vcxproj | 204 ----------------------- win32build.pl | 240 --------------------------- win32build96.pl | 240 --------------------------- win32build_2.pl | 219 ------------------------ 8 files changed, 3 insertions(+), 1356 deletions(-) delete mode 100644 msvs/pg_probackup.sln delete mode 100644 msvs/template.pg_probackup.vcxproj delete mode 100644 msvs/template.pg_probackup96.vcxproj delete mode 100644 msvs/template.pg_probackup_2.vcxproj delete mode 100644 win32build.pl delete mode 100644 win32build96.pl delete mode 100644 win32build_2.pl diff --git a/doit.cmd b/doit.cmd index b46e3b36d..7830a7ead 100644 --- a/doit.cmd +++ b/doit.cmd @@ -1 +1,3 @@ -perl win32build.pl "C:\PgProject\pgwininstall-ee\builddir\distr_X64_10.4.1\postgresql" "C:\PgProject\pgwininstall-ee\builddir\postgresql\postgrespro-enterprise-10.4.1\src" \ No newline at end of file +CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +SET PERL5LIB=. +perl gen_probackup_project.pl C:\Shared\Postgresql\myPostgres\11\postgrespro \ No newline at end of file diff --git a/msvs/pg_probackup.sln b/msvs/pg_probackup.sln deleted file mode 100644 index 2df4b4042..000000000 --- a/msvs/pg_probackup.sln +++ /dev/null @@ -1,28 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2013 for Windows Desktop -VisualStudioVersion = 12.0.31101.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pg_probackup", "pg_probackup.vcxproj", "{4886B21A-D8CA-4A03-BADF-743B24C88327}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.ActiveCfg = Debug|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|Win32.Build.0 = Debug|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.ActiveCfg = Debug|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Debug|x64.Build.0 = Debug|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.ActiveCfg = Release|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|Win32.Build.0 = Release|Win32 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.ActiveCfg = Release|x64 - {4886B21A-D8CA-4A03-BADF-743B24C88327}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/msvs/template.pg_probackup.vcxproj b/msvs/template.pg_probackup.vcxproj deleted file mode 100644 index a0a3c7a8d..000000000 --- a/msvs/template.pg_probackup.vcxproj +++ /dev/null @@ -1,213 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/msvs/template.pg_probackup96.vcxproj b/msvs/template.pg_probackup96.vcxproj deleted file mode 100644 index 3c62734ea..000000000 --- a/msvs/template.pg_probackup96.vcxproj +++ /dev/null @@ -1,211 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;@PGSRC@;$(IncludePath) - @PGROOT@\lib;$(LibraryPath) - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS32@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - %(AdditionalLibraryDirectories) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/msvs/template.pg_probackup_2.vcxproj b/msvs/template.pg_probackup_2.vcxproj deleted file mode 100644 index 1f103ac8d..000000000 --- a/msvs/template.pg_probackup_2.vcxproj +++ /dev/null @@ -1,204 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {4886B21A-D8CA-4A03-BADF-743B24C88327} - Win32Proj - pg_probackup - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - true - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - false - ../;@PGSRC@\include;@PGSRC@\bin\pg_basebackup;@PGSRC@\bin\pg_rewind;@PGSRC@\include\port\win32_msvc;@PGSRC@\interfaces\libpq;@PGSRC@\include\port\win32;@PGSRC@\port;@ADDINCLUDE@;$(IncludePath) - @PGROOT@\lib;@$(LibraryPath) - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - - - - Level3 - Disabled - _CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - Level3 - - - MaxSpeed - true - true - _CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) - true - - - Console - true - true - true - @ADDLIBS@;libpgfeutils.lib;libpgcommon.lib;libpgport.lib;libpq.lib;ws2_32.lib;%(AdditionalDependencies) - libc;%(IgnoreSpecificDefaultLibraries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/win32build.pl b/win32build.pl deleted file mode 100644 index 148641812..000000000 --- a/win32build.pl +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -our $libpath32=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@ADDLIBS32\@/$libpath32/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary32 -{ - $inc = shift; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary64 -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} - -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - AddLibrary32($config->{icu} . '\lib\icuin.lib'); - AddLibrary32($config->{icu} . '\lib\icuuc.lib'); - AddLibrary32($config->{icu} . '\lib\icudt.lib'); - AddLibrary64($config->{icu} . '\lib64\icuin.lib'); - AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary64($config->{icu} . '\lib64\icudt.lib'); - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - # AddLibrary($config->{libedit} . "\\" . - # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); - AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); - - - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); - AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); - AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; - } - # return $proj; -} - - - - diff --git a/win32build96.pl b/win32build96.pl deleted file mode 100644 index c869e485b..000000000 --- a/win32build96.pl +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -our $libpath32=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup96.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@ADDLIBS32\@/$libpath32/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary32 -{ - $inc = shift; - if ($libpath32 ne '') - { - $libpath32 .= ';'; - } - $libpath32 .= $inc; - -} -sub AddLibrary64 -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} - -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - AddLibrary32($config->{icu} . '\lib\icuin.lib'); - AddLibrary32($config->{icu} . '\lib\icuuc.lib'); - AddLibrary32($config->{icu} . '\lib\icudt.lib'); - AddLibrary64($config->{icu} . '\lib64\icuin.lib'); - AddLibrary64($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary64($config->{icu} . '\lib64\icudt.lib'); - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - # AddLibrary($config->{libedit} . "\\" . - # ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - AddLibrary32($config->{libedit} . '\\lib32\edit.lib'); - AddLibrary64($config->{libedit} . '\\lib64\edit.lib'); - - - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - # AddLibrary($config->{zstd}. "\\".($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib")); - AddLibrary32($config->{zstd}. "\\zstdlib_x86.lib"); - AddLibrary64($config->{zstd}. "\\zstdlib_x64.lib") ; - } - # return $proj; -} - - - - diff --git a/win32build_2.pl b/win32build_2.pl deleted file mode 100644 index a4f75553c..000000000 --- a/win32build_2.pl +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/perl -use JSON; -our $repack_version; -our $pgdir; -our $pgsrc; -if (@ARGV!=2) { - print STDERR "Usage $0 postgress-instalation-root pg-source-dir \n"; - exit 1; -} - - -our $liblist=""; - - -$pgdir = shift @ARGV; -$pgsrc = shift @ARGV if @ARGV; - - -our $arch = $ENV{'ARCH'} || "x64"; -$arch='Win32' if ($arch eq 'x86' || $arch eq 'X86'); -$arch='x64' if $arch eq 'X64'; - -$conffile = $pgsrc."/tools/msvc/config.pl"; - - -die 'Could not find config.pl' - unless (-f $conffile); - -our $config; -do $conffile; - - -if (! -d "$pgdir/bin" || !-d "$pgdir/include" || !-d "$pgdir/lib") { - print STDERR "Directory $pgdir doesn't look like root of postgresql installation\n"; - exit 1; -} -our $includepath=""; -our $libpath=""; -AddProject(); - -print "\n\n"; -print $libpath."\n"; -print $includepath."\n"; - -# open F,"<","META.json" or die "Cannot open META.json: $!\n"; -# { -# local $/ = undef; -# $decoded = decode_json(); -# $repack_version= $decoded->{'version'}; -# } - -# substitute new path in the project files - - - -preprocess_project("./msvs/template.pg_probackup_2.vcxproj","./msvs/pg_probackup.vcxproj"); - -exit 0; - - -sub preprocess_project { - my $in = shift; - my $out = shift; - our $pgdir; - our $adddir; - my $libs; - if (defined $adddir) { - $libs ="$adddir;"; - } else{ - $libs =""; - } - open IN,"<",$in or die "Cannot open $in: $!\n"; - open OUT,">",$out or die "Cannot open $out: $!\n"; - -# $includepath .= ";"; -# $libpath .= ";"; - - while () { - s/\@PGROOT\@/$pgdir/g; - s/\@ADDLIBS\@/$libpath/g; - s/\@PGSRC\@/$pgsrc/g; - s/\@ADDINCLUDE\@/$includepath/g; - - - print OUT $_; - } - close IN; - close OUT; - -} - - - -# my sub -sub AddLibrary -{ - $inc = shift; - if ($libpath ne '') - { - $libpath .= ';'; - } - $libpath .= $inc; - -} -sub AddIncludeDir -{ - # my ($self, $inc) = @_; - $inc = shift; - if ($includepath ne '') - { - $includepath .= ';'; - } - $includepath .= $inc; - -} - -sub AddProject -{ - # my ($self, $name, $type, $folder, $initialdir) = @_; - - if ($config->{zlib}) - { - AddIncludeDir($config->{zlib} . '\include'); - AddLibrary($config->{zlib} . '\lib\zdll.lib'); - } - if ($config->{openssl}) - { - AddIncludeDir($config->{openssl} . '\include'); - if (-e "$config->{openssl}/lib/VC/ssleay32MD.lib") - { - AddLibrary( - $config->{openssl} . '\lib\VC\ssleay32.lib', 1); - AddLibrary( - $config->{openssl} . '\lib\VC\libeay32.lib', 1); - } - else - { - # We don't expect the config-specific library to be here, - # so don't ask for it in last parameter - AddLibrary( - $config->{openssl} . '\lib\ssleay32.lib', 0); - AddLibrary( - $config->{openssl} . '\lib\libeay32.lib', 0); - } - } - if ($config->{nls}) - { - AddIncludeDir($config->{nls} . '\include'); - AddLibrary($config->{nls} . '\lib\libintl.lib'); - } - if ($config->{gss}) - { - AddIncludeDir($config->{gss} . '\inc\krb5'); - AddLibrary($config->{gss} . '\lib\i386\krb5_32.lib'); - AddLibrary($config->{gss} . '\lib\i386\comerr32.lib'); - AddLibrary($config->{gss} . '\lib\i386\gssapi32.lib'); - } - if ($config->{iconv}) - { - AddIncludeDir($config->{iconv} . '\include'); - AddLibrary($config->{iconv} . '\lib\iconv.lib'); - } - if ($config->{icu}) - { - AddIncludeDir($config->{icu} . '\include'); - if ($arch eq 'Win32') - { - AddLibrary($config->{icu} . '\lib\icuin.lib'); - AddLibrary($config->{icu} . '\lib\icuuc.lib'); - AddLibrary($config->{icu} . '\lib\icudt.lib'); - } - else - { - AddLibrary($config->{icu} . '\lib64\icuin.lib'); - AddLibrary($config->{icu} . '\lib64\icuuc.lib'); - AddLibrary($config->{icu} . '\lib64\icudt.lib'); - } - } - if ($config->{xml}) - { - AddIncludeDir($config->{xml} . '\include'); - AddIncludeDir($config->{xml} . '\include\libxml2'); - AddLibrary($config->{xml} . '\lib\libxml2.lib'); - } - if ($config->{xslt}) - { - AddIncludeDir($config->{xslt} . '\include'); - AddLibrary($config->{xslt} . '\lib\libxslt.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - AddLibrary($config->{libedit} . "\\" . - ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - } - if ($config->{uuid}) - { - AddIncludeDir($config->{uuid} . '\include'); - AddLibrary($config->{uuid} . '\lib\uuid.lib'); - } - if ($config->{libedit}) - { - AddIncludeDir($config->{libedit} . '\include'); - AddLibrary($config->{libedit} . "\\" . - ($arch eq 'x64'? 'lib64': 'lib32').'\edit.lib'); - } - if ($config->{zstd}) - { - AddIncludeDir($config->{zstd}); - AddLibrary($config->{zstd}. "\\". - ($arch eq 'x64'? "zstdlib_x64.lib" : "zstdlib_x86.lib") - ); - } - # return $proj; -} - - - - From 4befefada2eeda3eade56f532f28e62e78a638c9 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Wed, 17 Oct 2018 14:44:21 +0300 Subject: [PATCH 0062/2107] Skip copy WAL file to archive if already exists --- src/data.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index b1acfecc4..9d5267a84 100644 --- a/src/data.c +++ b/src/data.c @@ -29,6 +29,9 @@ typedef union DataPage char data[BLCKSZ]; } DataPage; +static bool +fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed); + #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ static int32 @@ -1092,14 +1095,21 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, FILE *in = NULL; FILE *out=NULL; char buf[XLOG_BLCKSZ]; - const char *to_path_p = to_path; + const char *to_path_p; char to_path_temp[MAXPGPATH]; int errno_temp; #ifdef HAVE_LIBZ char gz_to_path[MAXPGPATH]; gzFile gz_out = NULL; + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + to_path_p = gz_to_path; + } + else #endif + to_path_p = to_path; /* open file for read */ in = fopen(from_path, PG_BINARY_R); @@ -1107,30 +1117,30 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); + /* Check if possible to skip copying */ + if (fileExists(to_path_p)) + { + if (fileEqualCRC(from_path, to_path_p, is_compress)) + return; + /* Do not copy and do not rise error. Just quit as normal. */ + else if (!overwrite) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + } + /* open backup file for write */ #ifdef HAVE_LIBZ if (is_compress) { - snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); - - if (!overwrite && fileExists(gz_to_path)) - elog(ERROR, "WAL segment \"%s\" already exists.", gz_to_path); - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); gz_out = gzopen(to_path_temp, PG_BINARY_W); if (gzsetparams(gz_out, compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", compress_level, to_path_temp, get_gz_error(gz_out, errno)); - - to_path_p = gz_to_path; } else #endif { - if (!overwrite && fileExists(to_path)) - elog(ERROR, "WAL segment \"%s\" already exists.", to_path); - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); out = fopen(to_path_temp, PG_BINARY_W); @@ -1724,3 +1734,61 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, return is_valid; } + +static bool +fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) +{ + pg_crc32 crc_from; + pg_crc32 crc_to; + + /* Get checksum of backup file */ +#ifdef HAVE_LIBZ + if (path2_is_compressed) + { + char buf [1024]; + gzFile gz_in = NULL; + + INIT_CRC32C(crc_to); + gz_in = gzopen(path2, PG_BINARY_R); + if (gz_in == NULL) + { + /* There is no such file or it cannot be read */ + elog(LOG, + "Cannot compare WAL file \"%s\" with compressed \"%s\"", + path1, path2); + return false; + } + + for (;;) + { + size_t read_len = 0; + read_len = gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !gzeof(gz_in)) + { + /* An error occurred while reading the file */ + elog(LOG, + "Cannot compare WAL file \"%s\" with compressed \"%s\"", + path1, path2); + return false; + } + COMP_CRC32C(crc_to, buf, read_len); + if (gzeof(gz_in) || read_len == 0) + break; + } + FIN_CRC32C(crc_to); + + if (gzclose(gz_in) != 0) + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + path2, get_gz_error(gz_in, errno)); + } + else +#endif + { + crc_to = pgFileGetCRC(path2); + } + + /* Get checksum of original file */ + crc_from = pgFileGetCRC(path1); + + return EQ_CRC32C(crc_from, crc_to); +} From 6deb3bbe2bda3f03c89cc64867a3f647d8bacdf5 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Wed, 17 Oct 2018 15:40:08 +0300 Subject: [PATCH 0063/2107] fileEqualCRC() refactoring --- src/data.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/data.c b/src/data.c index 9d5267a84..b9b25b119 100644 --- a/src/data.c +++ b/src/data.c @@ -1738,8 +1738,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, static bool fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) { - pg_crc32 crc_from; - pg_crc32 crc_to; + pg_crc32 crc1; + pg_crc32 crc2; /* Get checksum of backup file */ #ifdef HAVE_LIBZ @@ -1748,7 +1748,7 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) char buf [1024]; gzFile gz_in = NULL; - INIT_CRC32C(crc_to); + INIT_CRC32C(crc2); gz_in = gzopen(path2, PG_BINARY_R); if (gz_in == NULL) { @@ -1771,11 +1771,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) path1, path2); return false; } - COMP_CRC32C(crc_to, buf, read_len); + COMP_CRC32C(crc2, buf, read_len); if (gzeof(gz_in) || read_len == 0) break; } - FIN_CRC32C(crc_to); + FIN_CRC32C(crc2); if (gzclose(gz_in) != 0) elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", @@ -1784,11 +1784,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) else #endif { - crc_to = pgFileGetCRC(path2); + crc2 = pgFileGetCRC(path2); } /* Get checksum of original file */ - crc_from = pgFileGetCRC(path1); + crc1 = pgFileGetCRC(path1); - return EQ_CRC32C(crc_from, crc_to); + return EQ_CRC32C(crc1, crc2); } From 20d4dcc291c817e0fd535fbf1b84b641335a4b51 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 18 Oct 2018 12:06:54 +0300 Subject: [PATCH 0064/2107] Modifyed test test_arhive_push_file_exists. Fixed fileEqualCRC() error handling --- src/data.c | 15 +++++---------- tests/archive.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index b9b25b119..1ed5bfee7 100644 --- a/src/data.c +++ b/src/data.c @@ -1751,32 +1751,27 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) INIT_CRC32C(crc2); gz_in = gzopen(path2, PG_BINARY_R); if (gz_in == NULL) - { - /* There is no such file or it cannot be read */ - elog(LOG, + /* File cannot be read */ + elog(ERROR, "Cannot compare WAL file \"%s\" with compressed \"%s\"", path1, path2); - return false; - } for (;;) { size_t read_len = 0; read_len = gzread(gz_in, buf, sizeof(buf)); if (read_len != sizeof(buf) && !gzeof(gz_in)) - { /* An error occurred while reading the file */ - elog(LOG, + elog(ERROR, "Cannot compare WAL file \"%s\" with compressed \"%s\"", path1, path2); - return false; - } + COMP_CRC32C(crc2, buf, read_len); if (gzeof(gz_in) || read_len == 0) break; } FIN_CRC32C(crc2); - + if (gzclose(gz_in) != 0) elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", path2, get_gz_error(gz_in, errno)); diff --git a/tests/archive.py b/tests/archive.py index 4ed783d62..45b235cd8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1,4 +1,6 @@ import os +import shutil +import zlib import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script from datetime import datetime, timedelta @@ -325,7 +327,15 @@ def test_arhive_push_file_exists(self): ) self.assertFalse('pg_probackup archive-push completed successfully' in log_content) - os.remove(file) + wal_src = os.path.join(node.data_dir, 'pg_wal', '000000010000000000000001') + if self.archive_compress: + with open(wal_src, 'rb') as f_in, open(file, 'wb') as f_out: + original_wal = f_in.read() + compressed_wal = zlib.compress(original_wal, 1) + f_out.write(compressed_wal) + else: + shutil.copyfile(wal_src, file) + self.switch_wal_segment(node) sleep(5) From 6d0cbfa2328b2903e2aaa9d216b221524b4bdccb Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 18 Oct 2018 15:05:51 +0300 Subject: [PATCH 0065/2107] Fix test test_arhive_push_file_exists --- src/data.c | 2 +- tests/archive.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/data.c b/src/data.c index 1ed5bfee7..4c75e1c94 100644 --- a/src/data.c +++ b/src/data.c @@ -1124,7 +1124,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, return; /* Do not copy and do not rise error. Just quit as normal. */ else if (!overwrite) - elog(ERROR, "WAL segment \"%s\" already exists.", to_path); + elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p); } /* open backup file for write */ diff --git a/tests/archive.py b/tests/archive.py index 45b235cd8..a311ac442 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1,6 +1,6 @@ import os import shutil -import zlib +import gzip import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script from datetime import datetime, timedelta @@ -329,13 +329,11 @@ def test_arhive_push_file_exists(self): wal_src = os.path.join(node.data_dir, 'pg_wal', '000000010000000000000001') if self.archive_compress: - with open(wal_src, 'rb') as f_in, open(file, 'wb') as f_out: - original_wal = f_in.read() - compressed_wal = zlib.compress(original_wal, 1) - f_out.write(compressed_wal) + with open(wal_src, 'rb') as f_in, gzip.open(file, 'wb', compresslevel=1) as f_out: + shutil.copyfileobj(f_in, f_out) else: shutil.copyfile(wal_src, file) - + self.switch_wal_segment(node) sleep(5) From d04d314ebab7ddafaed9ca8bc1c73d3dadba5c6c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 15 Nov 2018 11:46:42 +0300 Subject: [PATCH 0066/2107] Keep compiler quite --- src/backup.c | 4 ++-- src/merge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 380dc1c0c..dbafd7e1b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2173,7 +2173,7 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { - pgFile **prev_file; + pgFile **prev_file = NULL; /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2214,7 +2214,7 @@ backup_files(void *arg) bool skip = false; /* If non-data file has not changed since last backup... */ - if (file->exists_in_prev && + if (prev_file && file->exists_in_prev && buf.st_mtime < current.parent_backup) { calc_file_checksum(file); diff --git a/src/merge.c b/src/merge.c index 13263c648..c4d3a22f5 100644 --- a/src/merge.c +++ b/src/merge.c @@ -165,7 +165,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) parray *files, *to_files; pthread_t *threads = NULL; - merge_files_arg *threads_args; + merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; From f53395529b95ed923c87271001531b6834853c54 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 15 Nov 2018 18:22:43 +0300 Subject: [PATCH 0067/2107] Refactored calc_file_checksum() --- src/data.c | 72 ++++------------------------------------------ src/dir.c | 37 ++++++++++++++++++------ src/merge.c | 2 +- src/pg_probackup.h | 5 ++-- src/util.c | 2 +- src/validate.c | 3 +- 6 files changed, 40 insertions(+), 81 deletions(-) diff --git a/src/data.c b/src/data.c index 4c75e1c94..924e0fb7d 100644 --- a/src/data.c +++ b/src/data.c @@ -1418,75 +1418,13 @@ get_wal_file(const char *from_path, const char *to_path) * but created in process of backup, such as stream XLOG files, * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. */ -bool +void calc_file_checksum(pgFile *file) { - FILE *in; - size_t read_len = 0; - int errno_tmp; - char buf[BLCKSZ]; - struct stat st; - pg_crc32 crc; - Assert(S_ISREG(file->mode)); - INIT_TRADITIONAL_CRC32(crc); - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - - /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); - if (in == NULL) - { - FIN_TRADITIONAL_CRC32(crc); - file->crc = crc; - - /* maybe deleted, it's not error */ - if (errno == ENOENT) - return false; - - elog(ERROR, "cannot open source file \"%s\": %s", file->path, - strerror(errno)); - } - - /* stat source file to change mode of destination file */ - if (fstat(fileno(in), &st) == -1) - { - fclose(in); - elog(ERROR, "cannot stat \"%s\": %s", file->path, - strerror(errno)); - } - - for (;;) - { - read_len = fread(buf, 1, sizeof(buf), in); - - if(read_len == 0) - break; - - /* update CRC */ - COMP_TRADITIONAL_CRC32(crc, buf, read_len); - - file->write_size += read_len; - file->read_size += read_len; - } - - errno_tmp = errno; - if (!feof(in)) - { - fclose(in); - elog(ERROR, "cannot read backup mode file \"%s\": %s", - file->path, strerror(errno_tmp)); - } - - /* finish CRC calculation and store into pgFile */ - FIN_TRADITIONAL_CRC32(crc); - file->crc = crc; - - fclose(in); - return true; + file->crc = pgFileGetCRC(file->path, false, false, &file->read_size); + file->write_size = file->read_size; } /* @@ -1779,11 +1717,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) else #endif { - crc2 = pgFileGetCRC(path2); + crc2 = pgFileGetCRC(path2, false, true, NULL); } /* Get checksum of original file */ - crc1 = pgFileGetCRC(path1); + crc1 = pgFileGetCRC(path1, false, true, NULL); return EQ_CRC32C(crc1, crc2); } diff --git a/src/dir.c b/src/dir.c index 5c6f60d82..11712c03c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -259,36 +259,55 @@ pgFileDelete(pgFile *file) } pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c) +pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, + size_t *bytes_read) { FILE *fp; pg_crc32 crc = 0; char buf[1024]; size_t len; + size_t total = 0; int errno_tmp; + INIT_FILE_CRC32(use_crc32c, crc); + /* open file in binary read mode */ fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", - file_path, strerror(errno)); + { + if (!raise_on_deleted && errno == ENOENT) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + else + elog(ERROR, "cannot open file \"%s\": %s", + file_path, strerror(errno)); + } - /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); - while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf)) + /* calc CRC of file */ + for (;;) { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, sizeof(buf), fp); + if(len == 0) + break; + /* update CRC */ COMP_FILE_CRC32(use_crc32c, crc, buf, len); + total += len; } + + if (bytes_read) + *bytes_read = total; + errno_tmp = errno; if (!feof(fp)) elog(WARNING, "cannot read \"%s\": %s", file_path, strerror(errno_tmp)); - if (len > 0) - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - FIN_FILE_CRC32(use_crc32c, crc); + FIN_FILE_CRC32(use_crc32c, crc); fclose(fp); return crc; diff --git a/src/merge.c b/src/merge.c index 13263c648..956825613 100644 --- a/src/merge.c +++ b/src/merge.c @@ -524,7 +524,7 @@ merge_files(void *arg) * do that. */ file->write_size = pgFileSize(to_path_tmp); - file->crc = pgFileGetCRC(to_path_tmp, false); + file->crc = pgFileGetCRC(to_path_tmp, false, true, NULL); } } else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0fad6d70d..c838c9239 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -531,7 +531,8 @@ extern pgFile *pgFileNew(const char *path, bool omit_symlink); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c); +extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, + bool raise_on_deleted, size_t *bytes_read); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); @@ -552,7 +553,7 @@ extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); -extern bool calc_file_checksum(pgFile *file); +extern void calc_file_checksum(pgFile *file); extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn, diff --git a/src/util.c b/src/util.c index f28206501..946d5e491 100644 --- a/src/util.c +++ b/src/util.c @@ -334,7 +334,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba writeControlFile(&ControlFile, fullpath); /* Update pg_control checksum in backup_list */ - file->crc = pgFileGetCRC(fullpath, false); + file->crc = pgFileGetCRC(fullpath, false, true, NULL); pg_free(buffer); } diff --git a/src/validate.c b/src/validate.c index 55ea5ebe4..0e6fcb9a3 100644 --- a/src/validate.c +++ b/src/validate.c @@ -224,7 +224,8 @@ pgBackupValidateFiles(void *arg) * To avoid this problem we need to use different algorithm, CRC-32 in * this case. */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021); + crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021, + true, NULL); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", From fe00564707f385870631be429eeb73c062a7edb6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 16 Nov 2018 09:35:41 +0300 Subject: [PATCH 0068/2107] tests: minor fixes --- tests/archive.py | 21 +++-- tests/helpers/ptrack_helpers.py | 4 +- tests/replica.py | 141 ++++++++++++++++++-------------- 3 files changed, 93 insertions(+), 73 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index a311ac442..355c4b070 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -7,7 +7,6 @@ import subprocess from sys import exit from time import sleep -from shutil import copyfile module_name = 'archive' @@ -248,7 +247,9 @@ def test_pgpro434_3(self): node.append_conf( 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( archive_script_path)) + node.slow_start() + try: self.backup_node( backup_dir, 'node', node, @@ -262,6 +263,7 @@ def test_pgpro434_3(self): "Expecting Error because pg_stop_backup failed to answer.\n " "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( "ERROR: pg_stop_backup doesn't answer" in e.message and @@ -327,9 +329,11 @@ def test_arhive_push_file_exists(self): ) self.assertFalse('pg_probackup archive-push completed successfully' in log_content) - wal_src = os.path.join(node.data_dir, 'pg_wal', '000000010000000000000001') + wal_src = os.path.join( + node.data_dir, 'pg_wal', '000000010000000000000001') if self.archive_compress: - with open(wal_src, 'rb') as f_in, gzip.open(file, 'wb', compresslevel=1) as f_out: + with open(wal_src, 'rb') as f_in, gzip.open( + file, 'wb', compresslevel=1) as f_out: shutil.copyfileobj(f_in, f_out) else: shutil.copyfile(wal_src, file) @@ -506,10 +510,6 @@ def test_replica_archive(self): "postgres", "CHECKPOINT") -# copyfile( -# os.path.join(backup_dir, 'wal/master/000000010000000000000002'), -# os.path.join(backup_dir, 'wal/replica/000000010000000000000002')) - backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', @@ -604,10 +604,9 @@ def test_master_and_replica_parallel_archiving(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0, 60000) i") - # TAKE FULL ARCHIVE BACKUP FROM REPLICA - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000001'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) + master.psql( + "postgres", + "CHECKPOINT") backup_id = self.backup_node( backup_dir, 'replica', replica, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 7b4b410bd..b8a093433 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -874,8 +874,8 @@ def get_recovery_conf(self, node): return out_dict def set_archiving( - self, backup_dir, instance, node, replica=False, overwrite=False, compress=False, - old_binary=False): + self, backup_dir, instance, node, replica=False, + overwrite=False, compress=False, old_binary=False): if replica: archive_mode = 'always' diff --git a/tests/replica.py b/tests/replica.py index 1ab8515ec..f1591d816 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -5,7 +5,6 @@ import subprocess from sys import exit import time -from shutil import copyfile module_name = 'replica' @@ -27,8 +26,9 @@ def test_replica_stream_ptrack_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'ptrack_enable': 'on'} ) master.start() self.init_pb(backup_dir) @@ -144,7 +144,6 @@ def test_replica_archive_page_backup(self): pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s', 'archive_timeout': '10s'} ) self.init_pb(backup_dir) @@ -171,7 +170,8 @@ def test_replica_archive_page_backup(self): self.restore_node(backup_dir, 'master', replica) # Settings for Replica - self.set_replica(master, replica) + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) @@ -187,31 +187,23 @@ def test_replica_archive_page_backup(self): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,5120) i") + "from generate_series(256,25120) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") - self.add_instance(backup_dir, 'replica', replica) - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000003'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) - - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000004'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000004')) + master.psql( + "postgres", + "CHECKPOINT") - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000005'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000005')) + self.wait_until_replica_catch_with_master(master, replica) backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ - '--archive-timeout=30', + '--archive-timeout=60', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + '--master-port={0}'.format(master.port)]) self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -222,8 +214,13 @@ def test_replica_archive_page_backup(self): base_dir="{0}/{1}/node".format(module_name, fname)) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + + node.append_conf( + 'postgresql.auto.conf', 'archive_mode = off'.format(node.port)) + node.slow_start() # CHECK DATA CORRECTNESS @@ -234,23 +231,31 @@ def test_replica_archive_page_backup(self): # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal # to original data - master.psql( - "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,22680) i") + master.pgbench_init(scale=5) - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + +# master.psql( +# "postgres", +# "insert into t_heap as select i as id, md5(i::text) as text, " +# "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(512,25120) i") backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=30', + '--archive-timeout=60', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + '--master-port={0}'.format(master.port)]) + + pgbench.wait() + + self.switch_wal_segment(master) + + before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts") self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -258,17 +263,21 @@ def test_replica_archive_page_backup(self): # RESTORE PAGE BACKUP TAKEN FROM replica self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + backup_dir, 'replica', data_dir=node.data_dir, + backup_id=backup_id) node.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + node.append_conf( 'postgresql.auto.conf', 'archive_mode = off') + node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) + after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + self.assertEqual( + before, after, 'Restored data is not equal to original') self.add_instance(backup_dir, 'node', node) self.backup_node( @@ -290,8 +299,9 @@ def test_make_replica_via_restore(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'archive_timeout': '10s'} ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -310,7 +320,7 @@ def test_make_replica_via_restore(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") + "from generate_series(0,8192) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -320,6 +330,7 @@ def test_make_replica_via_restore(self): backup_dir, 'master', replica, options=['-R']) # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) @@ -328,13 +339,9 @@ def test_make_replica_via_restore(self): replica.slow_start(replica=True) - self.add_instance(backup_dir, 'replica', replica) - - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000003'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000003')) - - self.backup_node(backup_dir, 'replica', replica) + self.backup_node( + backup_dir, 'replica', replica, + options=['--archive-timeout=30s', '--stream']) # Clean after yourself self.del_test_dir(module_name, fname) @@ -353,14 +360,13 @@ def test_take_backup_from_delayed_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'archive_timeout': '10s'} ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - # force more frequent wal switch - #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') master.slow_start() replica = self.make_simple_node( @@ -369,6 +375,22 @@ def test_take_backup_from_delayed_replica(self): self.backup_node(backup_dir, 'master', master) + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + master.psql( + "postgres", + "CHECKPOINT") + + master.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + self.restore_node( backup_dir, 'master', replica, options=['-R']) @@ -376,36 +398,35 @@ def test_take_backup_from_delayed_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - # stupid hack - copyfile( - os.path.join(backup_dir, 'wal/master/000000010000000000000001'), - os.path.join(backup_dir, 'wal/replica/000000010000000000000001')) - replica.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf( - 'postgresql.auto.conf', 'hot_standby = on') + replica.slow_start(replica=True) + + self.wait_until_replica_catch_with_master(master, replica) replica.append_conf( 'recovery.conf', "recovery_min_apply_delay = '300s'") - replica.slow_start(replica=True) + replica.restart() master.pgbench_init(scale=10) pgbench = master.pgbench( - options=['-T', '30', '-c', '2', '--no-vacuum']) + options=['-T', '60', '-c', '2', '--no-vacuum']) self.backup_node( - backup_dir, 'replica', replica) + backup_dir, 'replica', + replica, options=['--archive-timeout=60s']) self.backup_node( backup_dir, 'replica', replica, - data_dir=replica.data_dir, backup_type='page') + data_dir=replica.data_dir, + backup_type='page', options=['--archive-timeout=60s']) self.backup_node( - backup_dir, 'replica', replica, backup_type='delta') + backup_dir, 'replica', replica, + backup_type='delta', options=['--archive-timeout=60s']) pgbench.wait() @@ -442,8 +463,8 @@ def test_make_block_from_future(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} + 'wal_level': 'replica', + 'max_wal_senders': '2'} ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) From d84d79668b0c1394aee17564d484c098a6d56eae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 16 Nov 2018 09:39:32 +0300 Subject: [PATCH 0069/2107] disable wait for archive in pg_stop_backup --- src/backup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backup.c b/src/backup.c index dbafd7e1b..a5317d8f7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1802,7 +1802,11 @@ pg_stop_backup(pgBackup *backup) #endif " labelfile," " spcmapfile" +#if PG_VERSION_NUM >= 100000 + " FROM pg_catalog.pg_stop_backup(false, false)"; +#else " FROM pg_catalog.pg_stop_backup(false)"; +#endif else stop_backup_query = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1810,7 +1814,11 @@ pg_stop_backup(pgBackup *backup) " lsn," " labelfile," " spcmapfile" +#if PG_VERSION_NUM >= 100000 + " FROM pg_catalog.pg_stop_backup(false, false)"; +#else " FROM pg_catalog.pg_stop_backup(false)"; +#endif } else From f9a0ae971c92e300edf8d1af8171421731cd8920 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 18 Nov 2018 14:37:34 +0300 Subject: [PATCH 0070/2107] Remove unused include --- src/walmethods.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/walmethods.c b/src/walmethods.c index 802a8e9c6..798402238 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -22,7 +22,6 @@ #endif #include "pgtar.h" -#include "common/file_perm.h" #include "common/file_utils.h" #include "receivelog.h" From 5f7aed02b2c06d07e9580644bb96057d545e072b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 18 Nov 2018 14:40:25 +0300 Subject: [PATCH 0071/2107] Provide compatibility with older Postgres versions --- src/walmethods.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/walmethods.c b/src/walmethods.c index 798402238..f627198fb 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -119,14 +119,14 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ /* Do pre-padding on non-compressed files */ if (pad_to_size && dir_data->compression == 0) { - PGAlignedXLogBlock zerobuf; - int bytes; + char zerobuf[XLOG_BLCKSZ]; + int bytes; - memset(zerobuf.data, 0, XLOG_BLCKSZ); + memset(zerobuf, 0, XLOG_BLCKSZ); for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ) { errno = 0; - if (fio_write(fd, zerobuf.data, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (fio_write(fd, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { int save_errno = errno; From bbac29a29c925ded50c4f84d4addb18be3be8db7 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 19 Nov 2018 17:08:56 +0300 Subject: [PATCH 0072/2107] PGPRO-2071: Use local dbname to connect to a server to get ptrack pagemap --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index c62ca9e99..fc47d22d6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1343,7 +1343,7 @@ pg_ptrack_clear(void) tblspcOid = atoi(PQgetvalue(res_db, i, 2)); tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - instance_config.pgdatabase, + dbname, instance_config.pguser); res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); @@ -1461,7 +1461,7 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, } tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - instance_config.pgdatabase, + dbname, instance_config.pguser); sprintf(params[0], "%i", tablespace_oid); sprintf(params[1], "%i", rel_filenode); From ddf2b9b4b302e319c621038873ca54c59f35c044 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 20 Nov 2018 16:10:18 +0300 Subject: [PATCH 0073/2107] Rename ssh options --- src/pg_probackup.c | 24 ++++++++++++++---------- src/pg_probackup.h | 8 ++++++-- src/utils/file.c | 2 +- src/utils/remote.c | 14 +++++++++----- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1ad811199..ba2fb5e98 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -74,8 +74,10 @@ const char *master_host = NULL; const char *master_port= NULL; const char *master_user = NULL; uint32 replica_timeout = REPLICA_TIMEOUT_DEFAULT; -char *ssh_host; -char *ssh_port; +char *remote_host; +char *remote_port; +char *remote_proto = (char*)"ssh"; +char *ssh_config; bool is_remote_agent; bool is_remote_backup; @@ -172,10 +174,12 @@ static pgut_option options[] = { 's', 16, "master-port", &master_port, SOURCE_CMDLINE, }, { 's', 17, "master-user", &master_user, SOURCE_CMDLINE, }, { 'u', 18, "replica-timeout", &replica_timeout, SOURCE_CMDLINE, SOURCE_DEFAULT, OPTION_UNIT_S }, - { 's', 19, "ssh-host", &ssh_host, SOURCE_CMDLINE, }, - { 's', 20, "ssh-port", &ssh_port, SOURCE_CMDLINE, }, - { 'b', 21, "agent", &is_remote_agent, SOURCE_CMDLINE, }, - { 'b', 22, "remote", &is_remote_backup, SOURCE_CMDLINE, }, + { 's', 19, "remote-host", &remote_host, SOURCE_CMDLINE, }, + { 's', 20, "remote-port", &remote_port, SOURCE_CMDLINE, }, + { 's', 21, "remote-proto", &remote_proto, SOURCE_CMDLINE, }, + { 's', 22, "ssh-config", &ssh_config, SOURCE_CMDLINE, }, + { 'b', 23, "agent", &is_remote_agent, SOURCE_CMDLINE, }, + { 'b', 24, "remote", &is_remote_backup, SOURCE_CMDLINE, }, /* restore options */ { 's', 30, "time", &target_time, SOURCE_CMDLINE }, { 's', 31, "xid", &target_xid, SOURCE_CMDLINE }, @@ -382,8 +386,8 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); - if (ssh_host != NULL && - (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD)) + if (IsSshConnection() + && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD)) { if (is_remote_agent) { if (backup_subcmd != BACKUP_CMD) { @@ -546,7 +550,7 @@ main(int argc, char *argv[]) return do_init(); case BACKUP_CMD: current.stream = stream_wal; - if (ssh_host && !is_remote_agent) + if (IsSshConnection() && !is_remote_agent) { current.status = BACKUP_STATUS_DONE; StrNCpy(current.program_version, PROGRAM_VERSION, @@ -564,7 +568,7 @@ main(int argc, char *argv[]) elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", ssh_host ? "true" : "false"); + stream_wal ? "true" : "false", remote_host ? "true" : "false"); return do_backup(start_time); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 42e446479..59f6f6cd4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -332,6 +332,8 @@ typedef struct XLByteInSeg(xlrp, logSegNo) #endif +#define IsSshConnection() (remote_host != NULL && strcmp(remote_proto, "ssh") == 0) + /* directory options */ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; @@ -351,8 +353,10 @@ extern char *replication_slot; extern bool smooth_checkpoint; #define ARCHIVE_TIMEOUT_DEFAULT 300 extern uint32 archive_timeout; -extern char *ssh_port; -extern char *ssh_host; +extern char *remote_port; +extern char *remote_host; +extern char *remote_proto; +extern char *ssh_config; extern const char *master_db; extern const char *master_host; extern const char *master_port; diff --git a/src/utils/file.c b/src/utils/file.c index ad69a60cd..f007ebba3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -38,7 +38,7 @@ static bool fio_is_remote(fio_location location) { return location == FIO_REMOTE_HOST || (location == FIO_BACKUP_HOST && is_remote_agent) - || (location == FIO_DB_HOST && !is_remote_agent && ssh_host != NULL); + || (location == FIO_DB_HOST && !is_remote_agent && IsSshConnection()); } static ssize_t fio_read_all(int fd, void* buf, size_t size) diff --git a/src/utils/remote.c b/src/utils/remote.c index 2dd6cfe17..b66332fbd 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -25,7 +25,7 @@ int remote_execute(int argc, char* argv[], bool listen) { char cmd[MAX_CMDLINE_LENGTH]; size_t dst = 0; - char* ssh_argv[6]; + char* ssh_argv[8]; int ssh_argc; int i; int outfd[2]; @@ -33,12 +33,16 @@ int remote_execute(int argc, char* argv[], bool listen) pid_t pid; ssh_argc = 0; - ssh_argv[ssh_argc++] = (char*)"ssh"; - if (ssh_port != 0) { + ssh_argv[ssh_argc++] = remote_proto; + if (remote_port != 0) { ssh_argv[ssh_argc++] = (char*)"-p"; - ssh_argv[ssh_argc++] = ssh_port; + ssh_argv[ssh_argc++] = remote_port; } - ssh_argv[ssh_argc++] = ssh_host; + if (ssh_config != 0) { + ssh_argv[ssh_argc++] = (char*)"-F"; + ssh_argv[ssh_argc++] = ssh_config; + } + ssh_argv[ssh_argc++] = remote_host; ssh_argv[ssh_argc++] = cmd+1; ssh_argv[ssh_argc] = NULL; From 6ea7c61c33c71b905ff6ac8f3316051afa94446b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 20 Nov 2018 16:33:36 +0300 Subject: [PATCH 0074/2107] PGPRO-2180: Retreive stop_lsn from previous WAL segment --- src/backup.c | 23 ++++++++- src/parsexlog.c | 125 ++++++++++++++++++++++++++++++++++++++++++--- src/pg_probackup.h | 3 ++ 3 files changed, 142 insertions(+), 9 deletions(-) diff --git a/src/backup.c b/src/backup.c index a5317d8f7..2e36f3f72 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1907,7 +1907,28 @@ pg_stop_backup(pgBackup *backup) if (!XRecOffIsValid(stop_backup_lsn)) { if (XRecOffIsNull(stop_backup_lsn)) - stop_backup_lsn = stop_backup_lsn + SizeOfXLogLongPHD; + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + XLogSegNo segno; + + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; + + GetXLogSegNo(stop_backup_lsn, segno, xlog_seg_size); + /* Retreive stop_lsn from previous segment */ + segno = segno - 1; + stop_backup_lsn = get_last_wal_lsn(xlog_path, backup->start_lsn, + segno, backup->tli, + xlog_seg_size); + } else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); diff --git a/src/parsexlog.c b/src/parsexlog.c index 5b2e32afd..65ae97eb8 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -88,7 +88,7 @@ static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime typedef struct XLogPageReadPrivate { - int thread_num; + int thread_num; const char *archivedir; TimeLineID tli; uint32 xlog_seg_size; @@ -132,8 +132,7 @@ static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader); static void CleanupXLogPageRead(XLogReaderState *xlogreader); -static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, - int elevel); +static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel); static XLogSegNo nextSegNoToRead = 0; static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -239,11 +238,17 @@ doExtractPageMap(void *arg) */ if (XLogRecPtrIsInvalid(found)) { - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X. %s", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint), - (xlogreader->errormsg_buf[0] != '\0')?xlogreader->errormsg_buf:""); + if (xlogreader->errormsg_buf[0] != '\0') + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + private_data->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint), + xlogreader->errormsg_buf); + else + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + private_data->thread_num, + (uint32) (extract_arg->startpoint >> 32), + (uint32) (extract_arg->startpoint)); PrintXLogCorruptionMsg(private_data, ERROR); } extract_arg->startpoint = found; @@ -766,6 +771,104 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, return res; } +/* + * Get last valid LSN within the WAL segment with number 'segno'. If 'start_lsn' + * is in the segment with number 'segno' then start from 'start_lsn', otherwise + * start from offset 0 within the segment. + */ +XLogRecPtr +get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, + XLogSegNo segno, TimeLineID tli, uint32 seg_size) +{ + XLogReaderState *xlogreader; + XLogPageReadPrivate private; + XLogRecPtr startpoint; + XLogSegNo start_segno; + XLogRecPtr res = InvalidXLogRecPtr; + + if (segno == 0) + elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); + + elog(LOG, "Retreiving last LSN of the segment with number " UINT64_FORMAT, + segno); + + xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + + /* + * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. + */ + GetXLogSegNo(start_lsn, start_segno, seg_size); + if (start_segno == segno) + startpoint = start_lsn; + else + { + XLogRecPtr found; + + GetXLogRecPtr(segno, 0, seg_size, startpoint); + found = XLogFindNextRecord(xlogreader, startpoint); + + if (XLogRecPtrIsInvalid(found)) + { + if (xlogreader->errormsg_buf[0] != '\0') + elog(WARNING, "Could not read WAL record at %X/%X: %s", + (uint32) (startpoint >> 32), (uint32) (startpoint), + xlogreader->errormsg_buf); + else + elog(WARNING, "Could not read WAL record at %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + PrintXLogCorruptionMsg(&private, ERROR); + } + startpoint = found; + } + + elog(VERBOSE, "Starting LSN is %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + while (true) + { + XLogRecord *record; + char *errormsg; + XLogSegNo next_segno = 0; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = XLogRecPtrIsInvalid(startpoint) ? xlogreader->EndRecPtr : + startpoint; + + if (errormsg) + elog(WARNING, "Could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "Could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + PrintXLogCorruptionMsg(&private, ERROR); + } + + res = xlogreader->ReadRecPtr; + + /* continue reading at next record */ + startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, next_segno, seg_size); + if (next_segno > segno) + break; + } + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + elog(VERBOSE, "Last LSN is %X/%X", (uint32) (res >> 32), (uint32) (res)); + + return res; +} + #ifdef HAVE_LIBZ /* * Show error during work with compressed file @@ -1035,6 +1138,12 @@ PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) private_data->gz_xlogpath); #endif } + else + { + /* Cannot tell what happened specifically */ + elog(elevel, "Thread [%d]: An error occured during WAL reading", + private_data->thread_num); + } } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c838c9239..a437396a3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -576,6 +576,9 @@ extern bool read_recovery_info(const char *archivedir, TimeLineID tli, TransactionId *recovery_xid); extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, TimeLineID target_tli, uint32 seg_size); +extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, + XLogSegNo segno, TimeLineID tli, + uint32 seg_size); /* in util.c */ extern TimeLineID get_current_timeline(bool safe); From b4672e3ac85f20ed164c740fedfc65f34dc6594b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 21 Nov 2018 18:30:03 +0300 Subject: [PATCH 0075/2107] PGPRO-2180: In pg_stop_backup for replica wait for LSN of prior record --- src/backup.c | 67 +++++++++++++++++++++++++++++++--------------- src/parsexlog.c | 36 ++++++++++++++++--------- src/pg_probackup.h | 4 +-- tests/replica.py | 2 +- 4 files changed, 72 insertions(+), 37 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2e36f3f72..a0fa0023a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -109,7 +109,7 @@ static int checkpoint_timeout(void); //static void backup_list_file(parray *files, const char *root, ) static void parse_backup_filelist_filenames(parray *files, const char *root); static void wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, - bool wait_prev_segment); + bool wait_prev_lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); static void *StreamLog(void *arg); @@ -1166,7 +1166,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ - wait_wal_lsn(backup->start_lsn, true, false); + wait_wal_lsn(backup->start_lsn, true, false, false); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream @@ -1174,7 +1174,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) */ else if (!stream_wal) /* ...for others wait for previous segment */ - wait_wal_lsn(backup->start_lsn, true, true); + wait_wal_lsn(backup->start_lsn, true, false, true); /* In case of backup from replica for PostgreSQL 9.5 * wait for start_lsn to be replayed by replica @@ -1504,7 +1504,8 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, * If 'wait_prev_segment' wait for previous segment. */ static void -wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) +wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, + bool wait_prev_segment) { TimeLineID tli; XLogSegNo targetSegNo; @@ -1515,6 +1516,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) bool file_exists = false; uint32 try_count = 0, timeout; + char *prior_to = (wait_prev_lsn) ? " prior to " : ""; #ifdef HAVE_LIBZ char gz_wal_segment_path[MAXPGPATH]; @@ -1555,14 +1557,13 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) timeout = archive_timeout; else timeout = ARCHIVE_TIMEOUT_DEFAULT; - } if (wait_prev_segment) elog(LOG, "Looking for segment: %s", wal_segment); else - elog(LOG, "Looking for LSN: %X/%X in segment: %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + elog(LOG, "Looking for LSN %s%X/%X in segment: %s", + prior_to, (uint32) (lsn >> 32), (uint32) lsn, wal_segment); #ifdef HAVE_LIBZ snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", @@ -1598,11 +1599,27 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) /* * A WAL segment found. Check LSN on it. */ - if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) - /* Target LSN was found */ + if (!wait_prev_lsn) { - elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); - return; + if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) + /* Target LSN was found */ + { + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return; + } + } + else + { + XLogRecPtr res; + + res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, + lsn, tli, false, xlog_seg_size); + if (!XLogRecPtrIsInvalid(res)) + { + /* LSN of the prior record was found */ + elog(LOG, "Found LSN: %X/%X", (uint32) (res >> 32), (uint32) res); + return; + } } } @@ -1618,16 +1635,18 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) elog(INFO, "Wait for WAL segment %s to be archived", wal_segment_path); else - elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + elog(INFO, "Wait for LSN %s%X/%X in archived WAL segment %s", + prior_to, (uint32) (lsn >> 32), (uint32) lsn, + wal_segment_path); } if (timeout > 0 && try_count > timeout) { if (file_exists) elog(ERROR, "WAL segment %s was archived, " - "but target LSN %X/%X could not be archived in %d seconds", - wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + "but target LSN %s%X/%X could not be archived in %d seconds", + wal_segment, prior_to, (uint32) (lsn >> 32), (uint32) lsn, + timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(ERROR, @@ -1724,6 +1743,7 @@ pg_stop_backup(pgBackup *backup) size_t len; char *val = NULL; char *stop_backup_query = NULL; + bool stop_lsn_exists = false; /* * We will use this values if there are no transactions between start_lsn @@ -1910,7 +1930,6 @@ pg_stop_backup(pgBackup *backup) { char *xlog_path, stream_xlog_path[MAXPGPATH]; - XLogSegNo segno; if (stream_wal) { @@ -1922,12 +1941,14 @@ pg_stop_backup(pgBackup *backup) else xlog_path = arclog_path; - GetXLogSegNo(stop_backup_lsn, segno, xlog_seg_size); - /* Retreive stop_lsn from previous segment */ - segno = segno - 1; stop_backup_lsn = get_last_wal_lsn(xlog_path, backup->start_lsn, - segno, backup->tli, - xlog_seg_size); + stop_backup_lsn, backup->tli, + true, xlog_seg_size); + /* + * Do not check existance of LSN again below using + * wait_wal_lsn(). + */ + stop_lsn_exists = true; } else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", @@ -2040,7 +2061,9 @@ pg_stop_backup(pgBackup *backup) * Wait for stop_lsn to be archived or streamed. * We wait for stop_lsn in stream mode just in case. */ - wait_wal_lsn(stop_backup_lsn, false, false); + if (!stop_lsn_exists) + wait_wal_lsn(stop_backup_lsn, false, + !exclusive_backup && current.from_replica, false); if (stream_wal) { diff --git a/src/parsexlog.c b/src/parsexlog.c index 65ae97eb8..1fc9ac05f 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -772,25 +772,33 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, } /* - * Get last valid LSN within the WAL segment with number 'segno'. If 'start_lsn' + * Get LSN of last or prior record within the WAL segment with number 'segno'. + * If 'start_lsn' * is in the segment with number 'segno' then start from 'start_lsn', otherwise * start from offset 0 within the segment. + * + * Returns LSN which points to end+1 of the last WAL record if seek_prev_segment + * is true. Otherwise returns LSN of the record prior to stop_lsn. */ XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogSegNo segno, TimeLineID tli, uint32 seg_size) + XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, + uint32 seg_size) { XLogReaderState *xlogreader; XLogPageReadPrivate private; XLogRecPtr startpoint; XLogSegNo start_segno; + XLogSegNo segno; XLogRecPtr res = InvalidXLogRecPtr; - if (segno == 0) + GetXLogSegNo(stop_lsn, segno, seg_size); + + if (segno <= 1) elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - elog(LOG, "Retreiving last LSN of the segment with number " UINT64_FORMAT, - segno); + if (seek_prev_segment) + segno = segno - 1; xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); @@ -821,9 +829,6 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, startpoint = found; } - elog(VERBOSE, "Starting LSN is %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - while (true) { XLogRecord *record; @@ -851,21 +856,28 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, PrintXLogCorruptionMsg(&private, ERROR); } - res = xlogreader->ReadRecPtr; - /* continue reading at next record */ startpoint = InvalidXLogRecPtr; GetXLogSegNo(xlogreader->EndRecPtr, next_segno, seg_size); if (next_segno > segno) break; + + if (seek_prev_segment) + { + /* end+1 of last record read */ + res = xlogreader->EndRecPtr; + } + else + res = xlogreader->ReadRecPtr; + + if (xlogreader->EndRecPtr >= stop_lsn) + break; } CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); - elog(VERBOSE, "Last LSN is %X/%X", (uint32) (res >> 32), (uint32) (res)); - return res; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a437396a3..6acdbe4e8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -577,8 +577,8 @@ extern bool read_recovery_info(const char *archivedir, TimeLineID tli, extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, TimeLineID target_tli, uint32 seg_size); extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogSegNo segno, TimeLineID tli, - uint32 seg_size); + XLogRecPtr stop_lsn, TimeLineID tli, + bool seek_prev_segment, uint32 seg_size); /* in util.c */ extern TimeLineID get_current_timeline(bool safe); diff --git a/tests/replica.py b/tests/replica.py index f1591d816..ce976397e 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -551,4 +551,4 @@ def test_make_block_from_future(self): exit(1) # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) From aab0ce3615b0b97f7622bb876ecce16e795bebe8 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 22 Nov 2018 14:44:57 +0300 Subject: [PATCH 0076/2107] PGPRO-2180: Fix stop_streaming(), read recovery info using valid stop LSN --- src/backup.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index a0fa0023a..04478d0ae 100644 --- a/src/backup.c +++ b/src/backup.c @@ -108,8 +108,8 @@ static int checkpoint_timeout(void); //static void backup_list_file(parray *files, const char *root, ) static void parse_backup_filelist_filenames(parray *files, const char *root); -static void wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, - bool wait_prev_lsn, bool wait_prev_segment); +static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, + bool wait_prev_lsn, bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); static void *StreamLog(void *arg); @@ -1502,8 +1502,11 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, * be archived in archive 'wal' directory regardless stream mode. * * If 'wait_prev_segment' wait for previous segment. + * + * Returns LSN of last valid record if wait_prev_segment is not true, otherwise + * returns InvalidXLogRecPtr. */ -static void +static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, bool wait_prev_segment) { @@ -1594,7 +1597,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, { /* Do not check LSN for previous WAL segment */ if (wait_prev_segment) - return; + return InvalidXLogRecPtr; /* * A WAL segment found. Check LSN on it. @@ -1605,7 +1608,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, /* Target LSN was found */ { elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); - return; + return lsn; } } else @@ -1618,7 +1621,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, { /* LSN of the prior record was found */ elog(LOG, "Found LSN: %X/%X", (uint32) (res >> 32), (uint32) res); - return; + return res; } } } @@ -2053,17 +2056,20 @@ pg_stop_backup(pgBackup *backup) { char *xlog_path, stream_xlog_path[MAXPGPATH]; + XLogRecPtr stop_valid_lsn = InvalidXLogRecPtr; /* Wait for stop_lsn to be received by replica */ - if (current.from_replica) - wait_replica_wal_lsn(stop_backup_lsn, false); + /* XXX Do we need this? */ +// if (current.from_replica) +// wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. * We wait for stop_lsn in stream mode just in case. */ if (!stop_lsn_exists) - wait_wal_lsn(stop_backup_lsn, false, - !exclusive_backup && current.from_replica, false); + stop_valid_lsn = wait_wal_lsn(stop_backup_lsn, false, + !exclusive_backup && current.from_replica, + false); if (stream_wal) { @@ -2082,7 +2088,7 @@ pg_stop_backup(pgBackup *backup) /* iterate over WAL from stop_backup lsn to start_backup lsn */ if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, - backup->start_lsn, backup->stop_lsn, + backup->start_lsn, stop_valid_lsn, &backup->recovery_time, &backup->recovery_xid)) { elog(LOG, "Failed to find Recovery Time in WAL. Forced to trust current_timestamp"); @@ -2649,7 +2655,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) if (!XLogRecPtrIsInvalid(stop_backup_lsn)) { - if (xlogpos > stop_backup_lsn) + if (xlogpos >= stop_backup_lsn) { stop_stream_lsn = xlogpos; return true; From 06b1dbe6f1dcfcbaf663256c28ff81d744c20655 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 23 Nov 2018 12:03:29 +0300 Subject: [PATCH 0077/2107] PGPRO-2180: Rewrite stop_lsn during pg_stop_backup --- src/backup.c | 53 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/backup.c b/src/backup.c index 04478d0ae..231398411 100644 --- a/src/backup.c +++ b/src/backup.c @@ -109,7 +109,7 @@ static int checkpoint_timeout(void); //static void backup_list_file(parray *files, const char *root, ) static void parse_backup_filelist_filenames(parray *files, const char *root); static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, - bool wait_prev_lsn, bool wait_prev_segment); + bool wait_prev_segment); static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); static void *StreamLog(void *arg); @@ -1166,7 +1166,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ - wait_wal_lsn(backup->start_lsn, true, false, false); + wait_wal_lsn(backup->start_lsn, true, false); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream @@ -1174,7 +1174,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) */ else if (!stream_wal) /* ...for others wait for previous segment */ - wait_wal_lsn(backup->start_lsn, true, false, true); + wait_wal_lsn(backup->start_lsn, true, true); /* In case of backup from replica for PostgreSQL 9.5 * wait for start_lsn to be replayed by replica @@ -1507,8 +1507,7 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, * returns InvalidXLogRecPtr. */ static XLogRecPtr -wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, - bool wait_prev_segment) +wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { TimeLineID tli; XLogSegNo targetSegNo; @@ -1519,7 +1518,6 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, bool file_exists = false; uint32 try_count = 0, timeout; - char *prior_to = (wait_prev_lsn) ? " prior to " : ""; #ifdef HAVE_LIBZ char gz_wal_segment_path[MAXPGPATH]; @@ -1565,8 +1563,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, if (wait_prev_segment) elog(LOG, "Looking for segment: %s", wal_segment); else - elog(LOG, "Looking for LSN %s%X/%X in segment: %s", - prior_to, (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + elog(LOG, "Looking for LSN %X/%X in segment: %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment); #ifdef HAVE_LIBZ snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", @@ -1602,16 +1600,19 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, /* * A WAL segment found. Check LSN on it. */ - if (!wait_prev_lsn) + if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) + /* Target LSN was found */ { - if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size)) - /* Target LSN was found */ - { - elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); - return lsn; - } + elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); + return lsn; } - else + + /* + * If we failed to get LSN of valid record in a reasonable time, try + * to get LSN of last valid record prior to the target LSN. But only + * in case of a backup from a replica. + */ + if (!exclusive_backup && current.from_replica) { XLogRecPtr res; @@ -1620,7 +1621,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, if (!XLogRecPtrIsInvalid(res)) { /* LSN of the prior record was found */ - elog(LOG, "Found LSN: %X/%X", (uint32) (res >> 32), (uint32) res); + elog(LOG, "Found prior LSN: %X/%X, it is used as stop LSN", + (uint32) (res >> 32), (uint32) res); return res; } } @@ -1638,18 +1640,16 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_lsn, elog(INFO, "Wait for WAL segment %s to be archived", wal_segment_path); else - elog(INFO, "Wait for LSN %s%X/%X in archived WAL segment %s", - prior_to, (uint32) (lsn >> 32), (uint32) lsn, - wal_segment_path); + elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); } if (timeout > 0 && try_count > timeout) { if (file_exists) elog(ERROR, "WAL segment %s was archived, " - "but target LSN %s%X/%X could not be archived in %d seconds", - wal_segment, prior_to, (uint32) (lsn >> 32), (uint32) lsn, - timeout); + "but target LSN %X/%X could not be archived in %d seconds", + wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(ERROR, @@ -2056,7 +2056,6 @@ pg_stop_backup(pgBackup *backup) { char *xlog_path, stream_xlog_path[MAXPGPATH]; - XLogRecPtr stop_valid_lsn = InvalidXLogRecPtr; /* Wait for stop_lsn to be received by replica */ /* XXX Do we need this? */ @@ -2067,9 +2066,7 @@ pg_stop_backup(pgBackup *backup) * We wait for stop_lsn in stream mode just in case. */ if (!stop_lsn_exists) - stop_valid_lsn = wait_wal_lsn(stop_backup_lsn, false, - !exclusive_backup && current.from_replica, - false); + stop_backup_lsn = wait_wal_lsn(stop_backup_lsn, false, false); if (stream_wal) { @@ -2088,7 +2085,7 @@ pg_stop_backup(pgBackup *backup) /* iterate over WAL from stop_backup lsn to start_backup lsn */ if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size, - backup->start_lsn, stop_valid_lsn, + backup->start_lsn, stop_backup_lsn, &backup->recovery_time, &backup->recovery_xid)) { elog(LOG, "Failed to find Recovery Time in WAL. Forced to trust current_timestamp"); From bdf2539887ce6dba0bd20df60108e1639d695641 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 23 Nov 2018 12:41:42 +0300 Subject: [PATCH 0078/2107] Improve log message within SimpleXLogPageRead --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 1fc9ac05f..bb0eb9392 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -922,14 +922,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, private_data->xlog_seg_size)) { - elog(VERBOSE, "Thread [%d]: Need to switch to segno next to %X/%X, current LSN %X/%X", + elog(VERBOSE, "Thread [%d]: Need to switch to the next WAL segment, page LSN %X/%X, record being read LSN %X/%X", private_data->thread_num, (uint32) (targetPagePtr >> 32), (uint32) (targetPagePtr), (uint32) (xlogreader->currRecPtr >> 32), (uint32) (xlogreader->currRecPtr )); /* - * if the last record on the page is not complete, + * If the last record on the page is not complete, * we must continue reading pages in the same thread */ if (!XLogRecPtrIsInvalid(xlogreader->currRecPtr) && From 7532cb36d1e8791611446ea1d8746dfaed5c9816 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 23 Nov 2018 13:20:16 +0300 Subject: [PATCH 0079/2107] PGPRO-2180: Reasonable time --- src/backup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 231398411..0e76dde7d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1612,7 +1612,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) * to get LSN of last valid record prior to the target LSN. But only * in case of a backup from a replica. */ - if (!exclusive_backup && current.from_replica) + if (!exclusive_backup && current.from_replica && + (try_count > timeout / 4)) { XLogRecPtr res; From bb3c0645aaa421dd44e46092923dcb7deca27709 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 23 Nov 2018 15:08:04 +0300 Subject: [PATCH 0080/2107] PGPRO-2180: Use archive_timeout only, do not use checkpoint_timeout --- src/backup.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0e76dde7d..c9fcea3f8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1545,21 +1545,18 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) DATABASE_DIR, PG_XLOG_DIR); join_path_components(wal_segment_path, pg_wal_dir, wal_segment); wal_segment_dir = pg_wal_dir; - - timeout = (uint32) checkpoint_timeout(); - timeout = timeout + timeout * 0.1; } else { join_path_components(wal_segment_path, arclog_path, wal_segment); wal_segment_dir = arclog_path; - - if (archive_timeout > 0) - timeout = archive_timeout; - else - timeout = ARCHIVE_TIMEOUT_DEFAULT; } + if (archive_timeout > 0) + timeout = archive_timeout; + else + timeout = ARCHIVE_TIMEOUT_DEFAULT; + if (wait_prev_segment) elog(LOG, "Looking for segment: %s", wal_segment); else From 7e2fccd0412e81b5949ae02a6360f9e992bbf284 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 23 Nov 2018 18:16:49 +0300 Subject: [PATCH 0081/2107] bugfix: use CRC32 for crc comparison during archive-push if target WAL segment is compressed --- src/data.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index 924e0fb7d..9e54b2cc3 100644 --- a/src/data.c +++ b/src/data.c @@ -1686,7 +1686,7 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) char buf [1024]; gzFile gz_in = NULL; - INIT_CRC32C(crc2); + INIT_FILE_CRC32(false, crc2); gz_in = gzopen(path2, PG_BINARY_R); if (gz_in == NULL) /* File cannot be read */ @@ -1704,11 +1704,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) "Cannot compare WAL file \"%s\" with compressed \"%s\"", path1, path2); - COMP_CRC32C(crc2, buf, read_len); + COMP_FILE_CRC32(false, crc2, buf, read_len); if (gzeof(gz_in) || read_len == 0) break; } - FIN_CRC32C(crc2); + FIN_FILE_CRC32(false, crc2); if (gzclose(gz_in) != 0) elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", From 57ca17ad6c8ad15b23ea940f145efbd1a7f37d36 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 23 Nov 2018 18:40:21 +0300 Subject: [PATCH 0082/2107] bugfix: use CRC32C for crc comparison during archive-push if target WAL segment is compressed --- src/data.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/data.c b/src/data.c index 9e54b2cc3..24c38110e 100644 --- a/src/data.c +++ b/src/data.c @@ -1686,7 +1686,7 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) char buf [1024]; gzFile gz_in = NULL; - INIT_FILE_CRC32(false, crc2); + INIT_FILE_CRC32(true, crc2); gz_in = gzopen(path2, PG_BINARY_R); if (gz_in == NULL) /* File cannot be read */ @@ -1704,11 +1704,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) "Cannot compare WAL file \"%s\" with compressed \"%s\"", path1, path2); - COMP_FILE_CRC32(false, crc2, buf, read_len); + COMP_FILE_CRC32(true, crc2, buf, read_len); if (gzeof(gz_in) || read_len == 0) break; } - FIN_FILE_CRC32(false, crc2); + FIN_FILE_CRC32(true, crc2); if (gzclose(gz_in) != 0) elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", @@ -1717,11 +1717,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) else #endif { - crc2 = pgFileGetCRC(path2, false, true, NULL); + crc2 = pgFileGetCRC(path2, true, true, NULL); } /* Get checksum of original file */ - crc1 = pgFileGetCRC(path1, false, true, NULL); + crc1 = pgFileGetCRC(path1, true, true, NULL); return EQ_CRC32C(crc1, crc2); } From b52ee32440e642c7c575625b37e12e00bb86e85b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 24 Nov 2018 22:57:27 +0300 Subject: [PATCH 0083/2107] tests: minor fixes --- tests/archive.py | 66 ++++++++++--------- tests/helpers/ptrack_helpers.py | 13 ---- tests/page.py | 31 +++++++-- tests/replica.py | 109 -------------------------------- 4 files changed, 58 insertions(+), 161 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 355c4b070..180b7ad6b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2,7 +2,7 @@ import shutil import gzip import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException from datetime import datetime, timedelta import subprocess from sys import exit @@ -221,7 +221,10 @@ def test_pgpro434_2(self): # @unittest.skip("skip") def test_pgpro434_3(self): - """Check pg_stop_backup_timeout, needed backup_timeout""" + """ + Check pg_stop_backup_timeout, needed backup_timeout + Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -236,40 +239,32 @@ def test_pgpro434_3(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - archive_script_path = os.path.join(backup_dir, 'archive_script.sh') - with open(archive_script_path, 'w+') as f: - f.write( - archive_script.format( - backup_dir=backup_dir, node_name='node', count_limit=2)) - - st = os.stat(archive_script_path) - os.chmod(archive_script_path, st.st_mode | 0o111) - node.append_conf( - 'postgresql.auto.conf', "archive_command = '{0} %p %f'".format( - archive_script_path)) - node.slow_start() - try: - self.backup_node( + gdb = self.backup_node( backup_dir, 'node', node, options=[ "--archive-timeout=60", - "--stream"] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pg_stop_backup failed to answer.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - - except ProbackupException as e: - self.assertTrue( - "ERROR: pg_stop_backup doesn't answer" in e.message and - "cancel it" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + "--stream", + "--log-level-file=info"], + gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + node.append_conf( + 'postgresql.auto.conf', "archive_command = 'exit 1'") + node.reload() + + gdb.continue_execution_until_exit() + + log_file = os.path.join(backup_dir, 'log/pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + "ERROR: pg_stop_backup doesn't answer", + log_content, + "pg_stop_backup timeouted") log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: @@ -331,6 +326,7 @@ def test_arhive_push_file_exists(self): wal_src = os.path.join( node.data_dir, 'pg_wal', '000000010000000000000001') + if self.archive_compress: with open(wal_src, 'rb') as f_in, gzip.open( file, 'wb', compresslevel=1) as f_out: @@ -412,7 +408,7 @@ def test_arhive_push_file_exists_overwrite(self): self.del_test_dir(module_name, fname) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_replica_archive(self): """ make node without archiving, take stream backup and @@ -502,7 +498,7 @@ def test_replica_archive(self): "postgres", "insert into t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,20680) i") + "from generate_series(512,80680) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -510,11 +506,13 @@ def test_replica_archive(self): "postgres", "CHECKPOINT") + self.wait_until_replica_catch_with_master(master, replica) + backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', options=[ - '--archive-timeout=30', + '--archive-timeout=60', '--master-db=postgres', '--master-host=localhost', '--master-port={0}'.format(master.port), diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b8a093433..d81d72888 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -60,19 +60,6 @@ } } -archive_script = """ -#!/bin/bash -count=$(ls {backup_dir}/test00* | wc -l) -if [ $count -ge {count_limit} ] -then - exit 1 -else - cp $1 {backup_dir}/wal/{node_name}/$2 - count=$((count+1)) - touch {backup_dir}/test00$count - exit 0 -fi -""" warning = """ Wrong splint in show_pb Original Header: diff --git a/tests/page.py b/tests/page.py index d31b8f60f..f06470825 100644 --- a/tests/page.py +++ b/tests/page.py @@ -3,6 +3,8 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess +import gzip +import shutil module_name = 'page' @@ -781,7 +783,22 @@ def test_page_backup_with_corrupted_wal_segment(self): wals_dir, f)) and not f.endswith('.backup')] wals = map(str, wals) # file = os.path.join(wals_dir, max(wals)) - file = os.path.join(wals_dir, '000000010000000000000004') + + if self.archive_compress: + original_file = os.path.join(wals_dir, '000000010000000000000004.gz') + tmp_file = os.path.join(backup_dir, '000000010000000000000004') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + # drop healthy file + os.remove(original_file) + file = tmp_file + + else: + file = os.path.join(wals_dir, '000000010000000000000004') + + # corrupt file print(file) with open(file, "rb+", 0) as f: f.seek(42) @@ -790,7 +807,14 @@ def test_page_backup_with_corrupted_wal_segment(self): f.close if self.archive_compress: - file = file[:-3] + # compress corrupted file and replace with it old file + with open(file, 'rb') as f_in, gzip.open(original_file, 'wb', compresslevel=1) as f_out: + shutil.copyfileobj(f_in, f_out) + + file = os.path.join(wals_dir, '000000010000000000000004.gz') + + #if self.archive_compress: + # file = file[:-3] # Single-thread PAGE backup try: @@ -915,9 +939,6 @@ def test_page_backup_with_alien_wal_segment(self): print(file_destination) os.rename(file, file_destination) - if self.archive_compress: - file_destination = file_destination[:-3] - # Single-thread PAGE backup try: self.backup_node( diff --git a/tests/replica.py b/tests/replica.py index ce976397e..4fbd08e69 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -236,12 +236,6 @@ def test_replica_archive_page_backup(self): pgbench = master.pgbench( options=['-T', '30', '-c', '2', '--no-vacuum']) -# master.psql( -# "postgres", -# "insert into t_heap as select i as id, md5(i::text) as text, " -# "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(512,25120) i") - backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='page', @@ -449,106 +443,3 @@ def test_take_backup_from_delayed_replica(self): # Clean after yourself self.del_test_dir(module_name, fname) - - @unittest.skip("skip") - def test_make_block_from_future(self): - """ - make archive master, take full backups from master, - restore full backup as replica, launch pgbench, - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2'} - ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - # force more frequent wal switch - #master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') - master.slow_start() - - replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) - replica.cleanup() - - self.backup_node(backup_dir, 'master', master) - - self.restore_node( - backup_dir, 'master', replica, options=['-R']) - - # Settings for Replica - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf( - 'postgresql.auto.conf', 'hot_standby = on') - - replica.slow_start(replica=True) - - self.add_instance(backup_dir, 'replica', replica) - - replica.safe_psql( - 'postgres', - 'checkpoint') - - master.pgbench_init(scale=10) - - self.wait_until_replica_catch_with_master(master, replica) - - -# print(replica.safe_psql( -# 'postgres', -# 'select * from pg_catalog.pg_last_xlog_receive_location()')) -# -# print(replica.safe_psql( -# 'postgres', -# 'select * from pg_catalog.pg_last_xlog_replay_location()')) -# -# print(replica.safe_psql( -# 'postgres', -# 'select * from pg_catalog.pg_control_checkpoint()')) -# -# replica.safe_psql( -# 'postgres', -# 'checkpoint') - - pgbench = master.pgbench(options=['-T', '30', '-c', '2', '--no-vacuum']) - - time.sleep(5) - - #self.backup_node(backup_dir, 'replica', replica, options=['--stream']) - exit(1) - self.backup_node(backup_dir, 'replica', replica) - pgbench.wait() - - # pgbench - master.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256000) i") - - - master.safe_psql( - 'postgres', - 'checkpoint') - - replica.safe_psql( - 'postgres', - 'checkpoint') - - replica.safe_psql( - 'postgres', - 'select * from pg_') - - self.backup_node(backup_dir, 'replica', replica) - exit(1) - - # Clean after yourself - self.del_test_dir(module_name, fname) From 640d1c9df40a08fc99c0d9f3c489b45581e8c28d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sun, 25 Nov 2018 00:47:00 +0300 Subject: [PATCH 0084/2107] Make compiler happy --- src/show.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/show.c b/src/show.c index 389428952..990998ba6 100644 --- a/src/show.c +++ b/src/show.c @@ -363,7 +363,7 @@ show_instance_plain(parray *backup_list, bool show_name) time2iso(row->recovery_time, lengthof(row->recovery_time), backup->recovery_time); else - StrNCpy(row->recovery_time, "----", 4); + StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time)); widths[cur] = Max(widths[cur], strlen(row->recovery_time)); cur++; @@ -388,7 +388,7 @@ show_instance_plain(parray *backup_list, bool show_name) snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); else - StrNCpy(row->duration, "----", 4); + StrNCpy(row->duration, "----", sizeof(row->duration)); widths[cur] = Max(widths[cur], strlen(row->duration)); cur++; From 990d67a49e65a37375e46620832c20dac566acb6 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sun, 25 Nov 2018 01:09:26 +0300 Subject: [PATCH 0085/2107] Use *_FILE_CRC32 macroses instead of *_TRADITIONAL_CRC32 --- src/backup.c | 8 ++++---- src/data.c | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index c9fcea3f8..1ea1c01fa 100644 --- a/src/backup.c +++ b/src/backup.c @@ -307,7 +307,7 @@ remote_copy_file(PGconn *conn, pgFile* file) to_path, strerror(errno_tmp)); } - INIT_TRADITIONAL_CRC32(file->crc); + INIT_FILE_CRC32(false, file->crc); /* read from stream and write to backup file */ while (1) @@ -333,14 +333,14 @@ remote_copy_file(PGconn *conn, pgFile* file) { write_buffer_size = Min(row_length, sizeof(buf)); memcpy(buf, copybuf, write_buffer_size); - COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size); + COMP_FILE_CRC32(false, file->crc, buf, write_buffer_size); /* TODO calc checksum*/ if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) { errno_tmp = errno; /* oops */ - FIN_TRADITIONAL_CRC32(file->crc); + FIN_FILE_CRC32(false, file->crc); fclose(out); PQfinish(conn); elog(ERROR, "cannot write to \"%s\": %s", to_path, @@ -364,7 +364,7 @@ remote_copy_file(PGconn *conn, pgFile* file) } file->write_size = (int64) file->read_size; - FIN_TRADITIONAL_CRC32(file->crc); + FIN_FILE_CRC32(false, file->crc); fclose(out); } diff --git a/src/data.c b/src/data.c index 24c38110e..7dc3814d5 100644 --- a/src/data.c +++ b/src/data.c @@ -487,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, blknum, header.compressed_size, write_buffer_size); */ /* Update CRC */ - COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size); + COMP_FILE_CRC32(false, *crc, write_buffer, write_buffer_size); /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) @@ -547,13 +547,13 @@ backup_data_file(backup_files_arg* arguments, /* reset size summary */ file->read_size = 0; file->write_size = 0; - INIT_TRADITIONAL_CRC32(file->crc); + INIT_FILE_CRC32(false, file->crc); /* open backup mode file for read */ in = fopen(file->path, PG_BINARY_R); if (in == NULL) { - FIN_TRADITIONAL_CRC32(file->crc); + FIN_FILE_CRC32(false, file->crc); /* * If file is not found, this is not en error. @@ -658,7 +658,7 @@ backup_data_file(backup_files_arg* arguments, to_path, strerror(errno)); fclose(in); - FIN_TRADITIONAL_CRC32(file->crc); + FIN_FILE_CRC32(false, file->crc); /* * If we have pagemap then file in the backup can't be a zero size. @@ -927,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) struct stat st; pg_crc32 crc; - INIT_TRADITIONAL_CRC32(crc); + INIT_FILE_CRC32(false, crc); /* reset size summary */ file->read_size = 0; @@ -937,7 +937,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) in = fopen(file->path, PG_BINARY_R); if (in == NULL) { - FIN_TRADITIONAL_CRC32(crc); + FIN_FILE_CRC32(false, crc); file->crc = crc; /* maybe deleted, it's not error */ @@ -986,7 +986,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } /* update CRC */ - COMP_TRADITIONAL_CRC32(crc, buf, read_len); + COMP_FILE_CRC32(false, crc, buf, read_len); file->read_size += read_len; } @@ -1013,14 +1013,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } /* update CRC */ - COMP_TRADITIONAL_CRC32(crc, buf, read_len); + COMP_FILE_CRC32(false, crc, buf, read_len); file->read_size += read_len; } file->write_size = (int64) file->read_size; /* finish CRC calculation and store into pgFile */ - FIN_TRADITIONAL_CRC32(crc); + FIN_FILE_CRC32(false, crc); file->crc = crc; /* update file permission */ From e199c4aee8100dcb119f89a82c2192651feccdd3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 26 Nov 2018 14:17:46 +0300 Subject: [PATCH 0086/2107] Use CRC-32C again. Calculate crc for pg_control differently using ControlFileData struct. Bump version to 2.0.25. --- src/backup.c | 16 +++++---- src/data.c | 26 +++++++------- src/merge.c | 4 ++- src/pg_probackup.c | 2 +- src/pg_probackup.h | 12 ++++--- src/restore.c | 2 ++ src/util.c | 56 ++++++++++++++++++++++++++----- src/validate.c | 14 ++++++-- tests/expected/option_version.out | 2 +- 9 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1ea1c01fa..fc10f0cd3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -307,7 +307,7 @@ remote_copy_file(PGconn *conn, pgFile* file) to_path, strerror(errno_tmp)); } - INIT_FILE_CRC32(false, file->crc); + INIT_FILE_CRC32(true, file->crc); /* read from stream and write to backup file */ while (1) @@ -333,14 +333,14 @@ remote_copy_file(PGconn *conn, pgFile* file) { write_buffer_size = Min(row_length, sizeof(buf)); memcpy(buf, copybuf, write_buffer_size); - COMP_FILE_CRC32(false, file->crc, buf, write_buffer_size); + COMP_FILE_CRC32(true, file->crc, buf, write_buffer_size); /* TODO calc checksum*/ if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) { errno_tmp = errno; /* oops */ - FIN_FILE_CRC32(false, file->crc); + FIN_FILE_CRC32(true, file->crc); fclose(out); PQfinish(conn); elog(ERROR, "cannot write to \"%s\": %s", to_path, @@ -364,7 +364,7 @@ remote_copy_file(PGconn *conn, pgFile* file) } file->write_size = (int64) file->read_size; - FIN_FILE_CRC32(false, file->crc); + FIN_FILE_CRC32(true, file->crc); fclose(out); } @@ -763,7 +763,8 @@ do_backup_instance(void) { char pg_control_path[MAXPGPATH]; - snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", pgdata, "global/pg_control"); + snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", + pgdata,"global/pg_control"); for (i = 0; i < parray_num(backup_files_list); i++) { @@ -2262,9 +2263,12 @@ backup_files(void *arg) continue; } } + else if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(arguments->from_root, arguments->to_root, + file); else { - bool skip = false; + bool skip = false; /* If non-data file has not changed since last backup... */ if (prev_file && file->exists_in_prev && diff --git a/src/data.c b/src/data.c index 7dc3814d5..e32312a82 100644 --- a/src/data.c +++ b/src/data.c @@ -487,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, blknum, header.compressed_size, write_buffer_size); */ /* Update CRC */ - COMP_FILE_CRC32(false, *crc, write_buffer, write_buffer_size); + COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); /* write data page */ if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size) @@ -547,13 +547,13 @@ backup_data_file(backup_files_arg* arguments, /* reset size summary */ file->read_size = 0; file->write_size = 0; - INIT_FILE_CRC32(false, file->crc); + INIT_FILE_CRC32(true, file->crc); /* open backup mode file for read */ in = fopen(file->path, PG_BINARY_R); if (in == NULL) { - FIN_FILE_CRC32(false, file->crc); + FIN_FILE_CRC32(true, file->crc); /* * If file is not found, this is not en error. @@ -658,7 +658,7 @@ backup_data_file(backup_files_arg* arguments, to_path, strerror(errno)); fclose(in); - FIN_FILE_CRC32(false, file->crc); + FIN_FILE_CRC32(true, file->crc); /* * If we have pagemap then file in the backup can't be a zero size. @@ -927,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) struct stat st; pg_crc32 crc; - INIT_FILE_CRC32(false, crc); + INIT_FILE_CRC32(true, crc); /* reset size summary */ file->read_size = 0; @@ -937,7 +937,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) in = fopen(file->path, PG_BINARY_R); if (in == NULL) { - FIN_FILE_CRC32(false, crc); + FIN_FILE_CRC32(true, crc); file->crc = crc; /* maybe deleted, it's not error */ @@ -986,7 +986,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } /* update CRC */ - COMP_FILE_CRC32(false, crc, buf, read_len); + COMP_FILE_CRC32(true, crc, buf, read_len); file->read_size += read_len; } @@ -1013,14 +1013,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) strerror(errno_tmp)); } /* update CRC */ - COMP_FILE_CRC32(false, crc, buf, read_len); + COMP_FILE_CRC32(true, crc, buf, read_len); file->read_size += read_len; } file->write_size = (int64) file->read_size; /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(false, crc); + FIN_FILE_CRC32(true, crc); file->crc = crc; /* update file permission */ @@ -1423,7 +1423,7 @@ calc_file_checksum(pgFile *file) { Assert(S_ISREG(file->mode)); - file->crc = pgFileGetCRC(file->path, false, false, &file->read_size); + file->crc = pgFileGetCRC(file->path, true, false, &file->read_size); file->write_size = file->read_size; } @@ -1546,14 +1546,14 @@ validate_one_page(Page page, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version) +check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, + uint32 backup_version) { size_t read_len = 0; bool is_valid = true; FILE *in; pg_crc32 crc; - bool use_crc32c = (backup_version <= 20021); + bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; elog(VERBOSE, "validate relation blocks for file %s", file->name); diff --git a/src/merge.c b/src/merge.c index caebe161e..b1e2aaa7b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -524,9 +524,11 @@ merge_files(void *arg) * do that. */ file->write_size = pgFileSize(to_path_tmp); - file->crc = pgFileGetCRC(to_path_tmp, false, true, NULL); + file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL); } } + else if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(argument->from_root, argument->to_root, file); else copy_file(argument->from_root, argument->to_root, file); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e8240e19d..bab455332 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -18,7 +18,7 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.24"; +const char *PROGRAM_VERSION = "2.0.25"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6acdbe4e8..32f99d1e5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -91,6 +91,7 @@ do { \ FIN_TRADITIONAL_CRC32(crc); \ } while (0) + /* Information about single file (or dir) in backup */ typedef struct pgFile { @@ -555,8 +556,7 @@ extern void get_wal_file(const char *from_path, const char *to_path); extern void calc_file_checksum(pgFile *file); -extern bool check_file_pages(pgFile* file, - XLogRecPtr stop_lsn, +extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern void extractPageMap(const char *archivedir, @@ -583,11 +583,15 @@ extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(char *pgdata); +extern uint64 get_system_identifier(const char *pgdata_path); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); +extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); extern uint32 get_xlog_seg_size(char *pgdata_path); -extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); +extern void set_min_recovery_point(pgFile *file, const char *backup_path, + XLogRecPtr stop_backup_lsn); +extern void copy_pgcontrol_file(const char *from_root, const char *to_root, + pgFile *file); extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); diff --git a/src/restore.c b/src/restore.c index 9cf33515e..e6e4a82f8 100644 --- a/src/restore.c +++ b/src/restore.c @@ -628,6 +628,8 @@ restore_files(void *arg) false, parse_program_version(arguments->backup->program_version)); } + else if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(from_root, pgdata, file); else copy_file(from_root, pgdata, file); diff --git a/src/util.c b/src/util.c index 946d5e491..4c222045a 100644 --- a/src/util.c +++ b/src/util.c @@ -206,7 +206,7 @@ get_checkpoint_location(PGconn *conn) } uint64 -get_system_identifier(char *pgdata_path) +get_system_identifier(const char *pgdata_path) { ControlFileData ControlFile; char *buffer; @@ -293,7 +293,27 @@ get_data_checksum_version(bool safe) return ControlFile.data_checksum_version; } -/* MinRecoveryPoint 'as-is' is not to be trusted */ +pg_crc32c +get_pgcontrol_checksum(const char *pgdata_path) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.crc; +} + +/* + * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint + * 'as-is' is not to be trusted. + */ void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) { @@ -321,24 +341,44 @@ set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_ba /* Update checksum in pg_control header */ INIT_CRC32C(ControlFile.crc); - COMP_CRC32C(ControlFile.crc, - (char *) &ControlFile, + COMP_CRC32C(ControlFile.crc, (char *) &ControlFile, offsetof(ControlFileData, crc)); FIN_CRC32C(ControlFile.crc); - /* paranoia */ - checkControlFile(&ControlFile); - /* overwrite pg_control */ snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); writeControlFile(&ControlFile, fullpath); /* Update pg_control checksum in backup_list */ - file->crc = pgFileGetCRC(fullpath, false, true, NULL); + file->crc = ControlFile.crc; pg_free(buffer); } +/* + * Copy pg_control file to backup. We do not apply compression to this file. + */ +void +copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + char to_path[MAXPGPATH]; + + buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false); + + digestControlFile(&ControlFile, buffer, size); + + file->crc = ControlFile.crc; + file->read_size = size; + file->write_size = size; + + join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + writeControlFile(&ControlFile, to_path); + + pg_free(buffer); +} /* * Convert time_t value to ISO-8601 format string. Always set timezone offset. diff --git a/src/validate.c b/src/validate.c index 0e6fcb9a3..92afa2942 100644 --- a/src/validate.c +++ b/src/validate.c @@ -22,6 +22,7 @@ static bool corrupted_backup_found = false; typedef struct { + const char *base_path; parray *files; bool corrupted; XLogRecPtr stop_lsn; @@ -101,6 +102,7 @@ pgBackupValidate(pgBackup *backup) { validate_files_arg *arg = &(threads_args[i]); + arg->base_path = base_path; arg->files = files; arg->corrupted = false; arg->stop_lsn = backup->stop_lsn; @@ -223,9 +225,17 @@ pgBackupValidateFiles(void *arg) * CRC-32C algorithm. * To avoid this problem we need to use different algorithm, CRC-32 in * this case. + * + * Starting from 2.0.25 we calculate crc of pg_control differently. */ - crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021, - true, NULL); + if (arguments->backup_version >= 20025 && + strcmp(file->name, "pg_control") == 0) + crc = get_pgcontrol_checksum(arguments->base_path); + else + crc = pgFileGetCRC(file->path, + arguments->backup_version <= 20021 || + arguments->backup_version >= 20025, + true, NULL); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 5280a6f95..1f2b8a4ed 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.24 \ No newline at end of file +pg_probackup 2.0.25 \ No newline at end of file From c64fd5a43e694d9851381226380b29bc5e9918e7 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 27 Nov 2018 11:19:49 +0300 Subject: [PATCH 0087/2107] Add -ssh-options option --- src/pg_probackup.c | 6 ++++-- src/pg_probackup.h | 1 + src/utils/remote.c | 51 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ba2fb5e98..40573d2a4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,6 +78,7 @@ char *remote_host; char *remote_port; char *remote_proto = (char*)"ssh"; char *ssh_config; +char *ssh_options; bool is_remote_agent; bool is_remote_backup; @@ -178,8 +179,9 @@ static pgut_option options[] = { 's', 20, "remote-port", &remote_port, SOURCE_CMDLINE, }, { 's', 21, "remote-proto", &remote_proto, SOURCE_CMDLINE, }, { 's', 22, "ssh-config", &ssh_config, SOURCE_CMDLINE, }, - { 'b', 23, "agent", &is_remote_agent, SOURCE_CMDLINE, }, - { 'b', 24, "remote", &is_remote_backup, SOURCE_CMDLINE, }, + { 's', 23, "ssh-options", &ssh_options, SOURCE_CMDLINE, }, + { 'b', 24, "agent", &is_remote_agent, SOURCE_CMDLINE, }, + { 'b', 25, "remote", &is_remote_backup, SOURCE_CMDLINE, }, /* restore options */ { 's', 30, "time", &target_time, SOURCE_CMDLINE }, { 's', 31, "xid", &target_xid, SOURCE_CMDLINE }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 59f6f6cd4..9714785fa 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -357,6 +357,7 @@ extern char *remote_port; extern char *remote_host; extern char *remote_proto; extern char *ssh_config; +extern char *ssh_options; extern const char *master_db; extern const char *master_host; extern const char *master_port; diff --git a/src/utils/remote.c b/src/utils/remote.c index b66332fbd..e32727143 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -7,7 +7,8 @@ #include "pg_probackup.h" #include "file.h" -#define MAX_CMDLINE_LENGTH 4096 +#define MAX_CMDLINE_LENGTH 4096 +#define MAX_CMDLINE_OPTIONS 256 static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) { @@ -21,11 +22,50 @@ static int append_option(char* buf, size_t buf_size, size_t dst, char const* src return dst + len + 1; } +static int split_options(int argc, char* argv[], int max_options, char* options) +{ + char* opt = options; + char in_quote = '\0'; + while (true) { + switch (*opt) { + case '\'': + case '\"': + if (!in_quote) { + in_quote = *opt++; + continue; + } + if (*opt == in_quote && *++opt != in_quote) { + in_quote = '\0'; + continue; + } + break; + case '\0': + if (opt != options) { + argv[argc++] = options; + if (argc >= max_options) + elog(ERROR, "Too much options"); + } + return argc; + case ' ': + argv[argc++] = options; + if (argc >= max_options) + elog(ERROR, "Too much options"); + *opt++ = '\0'; + options = opt; + continue; + default: + break; + } + opt += 1; + } + return argc; +} + int remote_execute(int argc, char* argv[], bool listen) { char cmd[MAX_CMDLINE_LENGTH]; size_t dst = 0; - char* ssh_argv[8]; + char* ssh_argv[MAX_CMDLINE_OPTIONS]; int ssh_argc; int i; int outfd[2]; @@ -34,14 +74,17 @@ int remote_execute(int argc, char* argv[], bool listen) ssh_argc = 0; ssh_argv[ssh_argc++] = remote_proto; - if (remote_port != 0) { + if (remote_port != NULL) { ssh_argv[ssh_argc++] = (char*)"-p"; ssh_argv[ssh_argc++] = remote_port; } - if (ssh_config != 0) { + if (ssh_config != NULL) { ssh_argv[ssh_argc++] = (char*)"-F"; ssh_argv[ssh_argc++] = ssh_config; } + if (ssh_options != NULL) { + ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, ssh_options); + } ssh_argv[ssh_argc++] = remote_host; ssh_argv[ssh_argc++] = cmd+1; ssh_argv[ssh_argc] = NULL; From 863486be7eab7f20e3ba14b58aa93c1d0cdb2b76 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 27 Nov 2018 16:28:30 +0300 Subject: [PATCH 0088/2107] Remove the limit on the number of extra directories. Save extra directories separately to eliminate name collisions. --- src/backup.c | 45 +++++++++++++++++++++++++++------------------ src/catalog.c | 2 +- src/dir.c | 38 ++++++++++++++++++++++++-------------- src/pg_probackup.h | 5 +++-- 4 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9545d7130..9f2f1b6f1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -463,8 +463,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; - char *extradirs[] = { NULL, NULL, NULL, NULL }; + char extra_path[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -478,23 +477,22 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; parray *backup_list = NULL; + parray *extra_dirs = NULL; pgFile *pg_control = NULL; elog(LOG, "Database backup start"); - i = 0; + extra_dirs = parray_new(); + /* TODO: Add path validation */ if(extradir) { p = strtok(extradir,":"); while(p!=NULL) { - extradirs[i] = (char *)palloc(strlen(p) + 1); - strcpy(extradirs[i],p); - elog(WARNING,"%s",extradirs[i]); - i++; + char * dir = (char *)palloc(strlen(p) + 1); + strcpy(dir,p); + parray_append(extra_dirs, dir); p=strtok(NULL,":"); - if (i==3) - break; } } @@ -654,11 +652,10 @@ do_backup_instance(void) parse_backup_filelist_filenames(backup_files_list, pgdata); /* Append to backup list all files dirictories from extra dirictory option */ - for (i = 0; extradirs[i]; i++) - { - elog(WARNING,"%s",extradirs[i]); - dir_list_file(backup_files_list, extradirs[i], true, true, true, true); - } + for (i = 0; i < parray_num(extra_dirs); i++) + /* Extra dirs numeration starts with 1. 0 value is not extra dir */ + dir_list_file(backup_files_list, (char *) parray_get(extra_dirs, i), + true, true, true, i+1); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -714,8 +711,12 @@ do_backup_instance(void) elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - if (file->is_extra) - join_path_components(dirpath, extra_path, dir_name); + if (file->extra_dir_num) + { + char temp[MAXPGPATH]; + sprintf(temp, "%s%d", extra_path, file->extra_dir_num); + join_path_components(dirpath, temp, dir_name); + } else join_path_components(dirpath, database_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); @@ -783,6 +784,12 @@ do_backup_instance(void) parray_walk(prev_backup_filelist, pgFileFree); parray_free(prev_backup_filelist); } + /* clean extra directories list */ + if (extra_dirs) + { + parray_walk(extra_dirs, pfree); + parray_free(extra_dirs); + } /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -2237,10 +2244,12 @@ backup_files(void *arg) continue; } } - else if (file->is_extra) + else if (file->extra_dir_num) { + char temp[MAXPGPATH]; + sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); if (!copy_file(file->extradir, - arguments->extra, + temp, file)) { file->write_size = BYTES_INVALID; diff --git a/src/catalog.c b/src/catalog.c index fd75ef09b..788c33f76 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -406,7 +406,7 @@ pgBackupCreateDir(pgBackup *backup) { int i; char path[MAXPGPATH]; - char *subdirs[] = { DATABASE_DIR, EXTRA_DIR, NULL }; + char *subdirs[] = { DATABASE_DIR, NULL }; pgBackupGetPath(backup, path, lengthof(path), NULL); diff --git a/src/dir.c b/src/dir.c index 7c341a28a..8b1c193f7 100644 --- a/src/dir.c +++ b/src/dir.c @@ -120,7 +120,8 @@ static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, bool is_extra); + bool omit_symlink, parray *black_list, + int extra_dir_num); static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); @@ -156,7 +157,7 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink, bool is_extra) +pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) { struct stat st; pgFile *file; @@ -174,7 +175,8 @@ pgFileNew(const char *path, bool omit_symlink, bool is_extra) file = pgFileInit(path); file->size = st.st_size; file->mode = st.st_mode; - file->is_extra = is_extra; + file->is_extra = extra_dir_num > 0; + file->extra_dir_num = extra_dir_num; file->extradir = NULL; return file; @@ -374,7 +376,7 @@ BlackListCompare(const void *str1, const void *str2) */ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, - bool add_root, bool is_extra) + bool add_root, int extra_dir_num) { pgFile *file; parray *black_list = NULL; @@ -411,7 +413,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false, is_extra); + file = pgFileNew(root, false, extra_dir_num); if (file == NULL) return; @@ -427,7 +429,8 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); } - dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, is_extra); + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, + extra_dir_num); } /* @@ -615,7 +618,7 @@ dir_check_file(const char *root, pgFile *file) */ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list, bool is_extra) + bool exclude, bool omit_symlink, parray *black_list, int extra_dir_num) { DIR *dir; struct dirent *dent; @@ -644,7 +647,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink, is_extra); + file = pgFileNew(child, omit_symlink, extra_dir_num); if (file == NULL) continue; @@ -677,7 +680,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, } /* If it is extra dir, remember it */ - if (is_extra) + if (extra_dir_num) { file->extradir = pgut_strdup(root); dirname(file->extradir); @@ -705,7 +708,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, root, file, exclude, omit_symlink, - black_list, is_extra); + black_list, extra_dir_num); } if (errno && errno != ENOENT) @@ -1201,11 +1204,12 @@ print_file_list(FILE *out, const parray *files, const char *root) fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"", + "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"" + ", \"extra_dir_num\":\"%u\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), - file->is_extra ? 1 : 0); + file->is_extra ? 1 : 0, file->extra_dir_num); if (file->extradir) fprintf(out, ",\"extradir\":\"%s\"", file->extradir); @@ -1399,6 +1403,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx is_datafile, is_cfs, is_extra, + extra_dir_num, crc, segno, n_blocks; @@ -1412,10 +1417,15 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx get_control_value(buf, "crc", NULL, &crc, true); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); get_control_value(buf, "is_extra", NULL, &is_extra, false); + get_control_value(buf, "extra_dir_num", NULL, &extra_dir_num, false); if (root) - if (is_extra) - join_path_components(filepath, extra_path, path); + if (extra_dir_num) + { + char temp[MAXPGPATH]; + sprintf(temp, "%s%ld", extra_path, extra_dir_num); + join_path_components(filepath, temp, path); + } else join_path_components(filepath, root, path); else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dae77e722..83cbbb2c8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -117,6 +117,7 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; bool is_extra; + bool extra_dir_num; /* Number of extra directory. 0 if not extra */ char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -516,7 +517,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root, bool is_extra); + bool omit_symlink, bool add_root, int extra_dir_num); extern void create_data_directories(const char *data_dir, const char *backup_dir, bool extract_tablespaces); @@ -534,7 +535,7 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink, bool is_extra); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, int extra_dir_num); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); From 59da24d59c016801ac5b53e97aa7a3203d4a8017 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 27 Nov 2018 17:31:35 +0300 Subject: [PATCH 0089/2107] Replace PG_STOP_BACKUP_TIMEOUT by archive-timeout --- src/backup.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index fc10f0cd3..290c59ea5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -24,8 +24,6 @@ #include "utils/thread.h" #include -#define PG_STOP_BACKUP_TIMEOUT 300 - /* * Macro needed to parse ptrack. * NOTE Keep those values syncronised with definitions in ptrack.h @@ -1858,8 +1856,8 @@ pg_stop_backup(pgBackup *backup) } /* - * Wait for the result of pg_stop_backup(), - * but no longer than PG_STOP_BACKUP_TIMEOUT seconds + * Wait for the result of pg_stop_backup(), but no longer than + * archive_timeout seconds */ if (pg_stop_backup_is_sent && !in_cleanup) { @@ -1882,14 +1880,14 @@ pg_stop_backup(pgBackup *backup) elog(INFO, "wait for pg_stop_backup()"); /* - * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds, + * If postgres haven't answered in archive_timeout seconds, * send an interrupt. */ - if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT) + if (pg_stop_backup_timeout > archive_timeout) { pgut_cancel(conn); elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", - PG_STOP_BACKUP_TIMEOUT); + archive_timeout); } } else From 4bb9b775bf49132fb5206a9c39f727c1774ff0ce Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 27 Nov 2018 17:57:43 +0300 Subject: [PATCH 0090/2107] Remove flag is_extra, use extra_dir_num instead --- src/backup.c | 2 +- src/dir.c | 15 +++++---------- src/pg_probackup.h | 1 - src/restore.c | 17 +++++++++-------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9f2f1b6f1..18e7d6351 100644 --- a/src/backup.c +++ b/src/backup.c @@ -702,7 +702,7 @@ do_backup_instance(void) char *dir_name; if (!is_remote_backup) - if (file->is_extra) + if (file->extra_dir_num) dir_name = GetRelativePath(file->path, file->extradir); else dir_name = GetRelativePath(file->path, pgdata); diff --git a/src/dir.c b/src/dir.c index 8b1c193f7..890355ddd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -175,7 +175,6 @@ pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) file = pgFileInit(path); file->size = st.st_size; file->mode = st.st_mode; - file->is_extra = extra_dir_num > 0; file->extra_dir_num = extra_dir_num; file->extradir = NULL; @@ -228,6 +227,7 @@ pgFileInit(const char *path) /* Number of blocks readed during backup */ file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; + file->extra_dir_num = 0; return file; } @@ -1198,18 +1198,16 @@ print_file_list(FILE *out, const parray *files, const char *root) /* omit root directory portion */ if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - else if(file->is_extra) + else if(file->extra_dir_num) path = GetRelativePath(path, file->extradir); fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"is_extra\":\"%u\"" - ", \"extra_dir_num\":\"%u\"", + "\"compress_alg\":\"%s\", \"extra_dir_num\":\"%u\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, - deparse_compress_alg(file->compress_alg), - file->is_extra ? 1 : 0, file->extra_dir_num); + deparse_compress_alg(file->compress_alg), file->extra_dir_num); if (file->extradir) fprintf(out, ",\"extradir\":\"%s\"", file->extradir); @@ -1402,7 +1400,6 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, - is_extra, extra_dir_num, crc, segno, @@ -1416,7 +1413,6 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx get_control_value(buf, "is_cfs", NULL, &is_cfs, false); get_control_value(buf, "crc", NULL, &crc, true); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "is_extra", NULL, &is_extra, false); get_control_value(buf, "extra_dir_num", NULL, &extra_dir_num, false); if (root) @@ -1433,8 +1429,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx file = pgFileInit(filepath); - file->is_extra = is_extra ? true : false; - if (is_extra) + if (extra_dir_num) { get_control_value(buf, "extradir", extradir_str, NULL, true); file->extradir = pgut_strdup(extradir_str); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 83cbbb2c8..2ad13b2af 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -116,7 +116,6 @@ typedef struct pgFile int n_blocks; /* size of the file in blocks, readed during DELTA backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; - bool is_extra; bool extra_dir_num; /* Number of extra directory. 0 if not extra */ char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ diff --git a/src/restore.c b/src/restore.c index c4e7c1a67..de1ba4e59 100644 --- a/src/restore.c +++ b/src/restore.c @@ -478,17 +478,18 @@ restore_backup(pgBackup *backup) char dirpath[MAXPGPATH]; char *dir_name; - if (file->is_extra) + if (file->extra_dir_num) + { + join_path_components(dirpath, file->extradir, dir_name); + elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); dir_name = GetRelativePath(file->path, extra_path); + } else + { dir_name = GetRelativePath(file->path, database_path); - - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - - if (file->is_extra) - join_path_components(dirpath, file->extradir, dir_name); - else + elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); join_path_components(dirpath, pgdata, dir_name); + } dir_create_dir(dirpath, DIR_PERMISSION); } @@ -659,7 +660,7 @@ restore_files(void *arg) false, parse_program_version(arguments->backup->program_version)); } - else if (file->is_extra) + else if (file->extra_dir_num) copy_file(from_root, file->extradir, file); else copy_file(from_root, pgdata, file); From f41df91171d78d464910770b02fe51aac6c22f5b Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Wed, 28 Nov 2018 16:15:35 +0300 Subject: [PATCH 0091/2107] Skip backup file if it has not changed since the last backup --- src/backup.c | 39 +++++++++++++++++++++------------------ src/dir.c | 25 +++++++++++++++++++++++++ src/pg_probackup.h | 3 ++- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/backup.c b/src/backup.c index 18e7d6351..5b911689c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -730,7 +730,7 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileCompareSize); /* Sort the array for binary search */ if (prev_backup_filelist) - parray_qsort(prev_backup_filelist, pgFileComparePath); + parray_qsort(prev_backup_filelist, pgFileComparePathWithExtra); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -2216,11 +2216,13 @@ backup_files(void *arg) char *relative; pgFile key; - relative = GetRelativePath(file->path, arguments->from_root); + relative = GetRelativePath(file->path, file->extra_dir_num ? + file->extradir:arguments->from_root); key.path = relative; + key.extra_dir_num = file->extra_dir_num; prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, - &key, pgFileComparePath); + &key, pgFileComparePathWithExtra); if (prev_file) /* File exists in previous backup */ file->exists_in_prev = true; @@ -2244,21 +2246,10 @@ backup_files(void *arg) continue; } } - else if (file->extra_dir_num) - { - char temp[MAXPGPATH]; - sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); - if (!copy_file(file->extradir, - temp, - file)) - { - file->write_size = BYTES_INVALID; - elog(VERBOSE, "File \"%s\" was not copied extra files to backup", file->path); - continue; - } - } else { + const char *src; + const char *dst; bool skip = false; /* If non-data file has not changed since last backup... */ @@ -2270,8 +2261,20 @@ backup_files(void *arg) if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc)) skip = true; /* ...skip copying file. */ } - if (skip || - !copy_file(arguments->from_root, arguments->to_root, file)) + /* Set file paths */ + if (file->extra_dir_num) + { + char temp[MAXPGPATH]; + sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); + src = file->extradir; + dst = temp; + } + else + { + src = arguments->from_root; + dst = arguments->to_root; + } + if (skip || !copy_file(src, dst, file)) { file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", diff --git a/src/dir.c b/src/dir.c index 890355ddd..fe0804ed6 100644 --- a/src/dir.c +++ b/src/dir.c @@ -329,6 +329,30 @@ pgFileComparePath(const void *f1, const void *f2) return strcmp(f1p->path, f2p->path); } +/* + * Compare two pgFile with their path and extra_dir_num + * in ascending order of ASCII code. + */ +int +pgFileComparePathWithExtra(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + int res; + + res = strcmp(f1p->path, f2p->path); + if (!res) + { + if (f1p->extra_dir_num > f2p->extra_dir_num) + return 1; + else if (f1p->extra_dir_num < f2p->extra_dir_num) + return -1; + else + return 0; + } + return res; +} + /* Compare two pgFile with their path in descending order of ASCII code. */ int pgFileComparePathDesc(const void *f1, const void *f2) @@ -1440,6 +1464,7 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx file->is_cfs = is_cfs ? true : false; file->crc = (pg_crc32) crc; file->compress_alg = parse_compress_alg(compress_alg_string); + file->extra_dir_num = extra_dir_num; /* * Optional fields diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2ad13b2af..2cae08b8c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -116,7 +116,7 @@ typedef struct pgFile int n_blocks; /* size of the file in blocks, readed during DELTA backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; - bool extra_dir_num; /* Number of extra directory. 0 if not extra */ + int extra_dir_num; /* Number of extra directory. 0 if not extra */ char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -540,6 +540,7 @@ extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c); extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileComparePathWithExtra(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); From 82101e032ab0245d20adb17dfc9603f6421d9e21 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Nov 2018 21:19:10 +0300 Subject: [PATCH 0092/2107] tests: minor fixes --- tests/option_test.py | 24 ++++++++++++------------ tests/ptrack_clean.py | 7 +++++-- tests/ptrack_empty.py | 8 ++++---- tests/ptrack_vacuum_full.py | 11 +++++++---- tests/ptrack_vacuum_truncate.py | 3 ++- tests/show_test.py | 6 ++++-- tests/validate_test.py | 7 +++++++ 7 files changed, 41 insertions(+), 25 deletions(-) diff --git a/tests/option_test.py b/tests/option_test.py index 8bd473fa9..faed430ae 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -127,10 +127,11 @@ def test_options_5(self): self.init_pb(backup_dir)) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # syntax error in pg_probackup.conf - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + conf_file = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(conf_file, "a") as conf: conf.write(" = INFINITE\n") try: self.backup_node(backup_dir, 'node', node) @@ -139,7 +140,7 @@ def test_options_5(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: syntax error in " = INFINITE"\n', + 'ERROR: Syntax error in " = INFINITE"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -147,7 +148,7 @@ def test_options_5(self): self.add_instance(backup_dir, 'node', node) # invalid value in pg_probackup.conf - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + with open(conf_file, "a") as conf: conf.write("BACKUP_MODE=\n") try: @@ -157,7 +158,7 @@ def test_options_5(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: invalid backup-mode ""\n', + 'ERROR: Invalid option "BACKUP_MODE" in file "{0}"\n'.format(conf_file), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -165,7 +166,7 @@ def test_options_5(self): self.add_instance(backup_dir, 'node', node) # Command line parameters should override file values - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + with open(conf_file, "a") as conf: conf.write("retention-redundancy=1\n") self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') @@ -178,11 +179,11 @@ def test_options_5(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: option system-identifier cannot be specified in command line\n', + 'ERROR: Option system-identifier cannot be specified in command line\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # invalid value in pg_probackup.conf - with open(os.path.join(backup_dir, "backups", "node", "pg_probackup.conf"), "a") as conf: + with open(conf_file, "a") as conf: conf.write("SMOOTH_CHECKPOINT=FOO\n") try: @@ -192,7 +193,7 @@ def test_options_5(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - "ERROR: option -C, --smooth-checkpoint should be a boolean: 'FOO'\n", + 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file "{0}"\n'.format(conf_file), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -200,8 +201,7 @@ def test_options_5(self): self.add_instance(backup_dir, 'node', node) # invalid option in pg_probackup.conf - pbconf_path = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") - with open(pbconf_path, "a") as conf: + with open(conf_file, "a") as conf: conf.write("TIMELINEID=1\n") try: @@ -211,7 +211,7 @@ def test_options_5(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: invalid option "TIMELINEID" in file "{0}"\n'.format(pbconf_path), + 'ERROR: Invalid option "TIMELINEID" in file "{0}"\n'.format(conf_file), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 076291a69..2b122f2f9 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -130,7 +130,9 @@ def test_ptrack_clean_replica(self): pg_options={ 'ptrack_enable': 'on', 'wal_level': 'replica', - 'max_wal_senders': '2'}) + 'max_wal_senders': '2', + 'archive_timeout': '30s'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -237,7 +239,8 @@ def test_ptrack_clean_replica(self): options=[ '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) master.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 8656f9413..f20362b10 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -9,9 +9,9 @@ class SimpleTest(ProbackupTest, unittest.TestCase): - # @unittest.skip("skip") + @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_clean(self): + def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -87,9 +87,9 @@ def test_ptrack_clean(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_clean_replica(self): + def test_ptrack_empty_replica(self): """Take backups of every available types from master and check that PTRACK on replica is clean""" fname = self.id().split('.')[3] master = self.make_simple_node( diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index 8e26ecd09..a20ae164e 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -99,9 +99,11 @@ def test_ptrack_vacuum_full_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'wal_level': 'replica', - 'max_wal_senders': '2', 'autovacuum': 'off', - 'checkpoint_timeout': '30s'} + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off', + 'archive_timeout': '30s'} ) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -150,7 +152,8 @@ def test_ptrack_vacuum_full_replica(self): '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port) + '--master-port={0}'.format(master.port), + '--stream' ] ) # TODO: check that all ptrack are nullified diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 8ba9adb69..406e2b3ab 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -147,7 +147,8 @@ def test_ptrack_vacuum_truncate_replica(self): '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port) + '--master-port={0}'.format(master.port), + '--stream' ] ) diff --git a/tests/show_test.py b/tests/show_test.py index 484efce3e..50ad5be15 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -197,7 +197,9 @@ def test_corrupt_control_file(self): fd.write("statuss = OK") fd.close() - self.assertIn('invalid option "statuss" in file'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn( + 'WARNING: Invalid option "statuss" in file'.format(file), + self.show_pb(backup_dir, 'node', as_text=True)) # Clean after yourself - self.del_test_dir(module_name, fname) + # self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index c0fd49439..a7f8d4373 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1519,10 +1519,17 @@ def test_pgpro561(self): backup_dir, 'node1', node1, backup_type='page', options=["--stream"]) self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + node2.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) + node2.append_conf( + 'postgresql.auto.conf', 'archive_mode = off') node2.slow_start() + node2.append_conf( + 'postgresql.auto.conf', 'archive_mode = on') + node2.restart() + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] self.assertEqual( From c14c5a58c5559ed0ae89c45b925238a4144a140b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Nov 2018 21:32:49 +0300 Subject: [PATCH 0093/2107] Version 2.0.25 - Bugfix: changing log-level-file and log-level-console via set-config now works correctly - Bugfix: WAL files crc comparison done by archive-push is now works correctly, previously it`s was always giving mismatch - Impromevent: PG_STOP_BACKUP_TIMEOUT now governed by archive-timeout option - Impromevent: CRC-32c now used to calculate crc for data and WAL files, previously a slower CRC-32 was used - Impromevent: cold replica can be backuped without false-positive WAL timeouts - Impromevent: for PG >= 10 pg_stop_backup now called without wait_for_archive flag - Impromevent: minor fixes for Windows build --- tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 3593f657f..532e72bff 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,8 +16,8 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup_test)) suite.addTests(loader.loadTestsFromModule(compatibility)) - suite.addTests(loader.loadTestsFromModule(cfs_backup)) - suite.addTests(loader.loadTestsFromModule(cfs_restore)) +# suite.addTests(loader.loadTestsFromModule(cfs_backup)) +# suite.addTests(loader.loadTestsFromModule(cfs_restore)) # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) # suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(compression)) From 4155171b1923bce38e1c83cb6d21a99ce3f95de3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Nov 2018 09:19:32 +0300 Subject: [PATCH 0094/2107] update README.md about PG11 support --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 53945eb88..5c2267d08 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,6 @@ The utility is compatible with: * PostgreSQL 9.5, 9.6, 10, 11; -`PTRACK` backup support provided via following options: -* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* Postgres Pro Standard 9.5, 9.6, 10, 11 -* Postgres Pro Enterprise 9.5, 9.6, 10 - As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Choosing between full and page-level incremental backups to speed up backup and recovery * Implementing a single backup strategy for multi-server PostgreSQL clusters @@ -34,6 +29,11 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. * `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). +`PTRACK` backup support provided via following options: +* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) +* Postgres Pro Standard 9.5, 9.6, 10, 11 +* Postgres Pro Enterprise 9.5, 9.6, 10 + ## Limitations `pg_probackup` currently has the following limitations: @@ -48,27 +48,27 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp #DEB Ubuntu|Debian Packages echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update -apt-get install pg-probackup-{10,9.6,9.5} +apt-get install pg-probackup-{11,10,9.6,9.5} #DEB-SRC Packages echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list -apt-get source pg-probackup-{10,9.6,9.5} +apt-get source pg-probackup-{11,10,9.6,9.5} #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5} #SRPM Packages -yumdownloader --source pg_probackup-{10,9.6,9.5} +yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: From 2a62342e8b3778593734ad984264c233d5c2f9e2 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 30 Nov 2018 18:24:14 +0300 Subject: [PATCH 0095/2107] WIP checkdb command --- src/backup.c | 154 ++++++++++++++++++++++++++++++++++++++++++++- src/data.c | 61 ++++++++++++++++++ src/pg_probackup.c | 56 +++++++++++------ src/pg_probackup.h | 4 ++ 4 files changed, 253 insertions(+), 22 deletions(-) diff --git a/src/backup.c b/src/backup.c index d92389293..ed88051ec 100644 --- a/src/backup.c +++ b/src/backup.c @@ -98,6 +98,7 @@ static void *backup_files(void *arg); static void *remote_backup_files(void *arg); static void do_backup_instance(void); +static void do_check_instance(void); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_switch_wal(PGconn *conn); @@ -459,6 +460,100 @@ remote_backup_files(void *arg) return NULL; } +/* TODO think about merging it with do_backup_instance */ +static void +do_check_instance(void) +{ + int i; + char database_path[MAXPGPATH]; + + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); + + /* setup threads */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = instance_config.pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->prev_filelist = NULL; + arg->prev_start_lsn = InvalidXLogRecPtr; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + arg->checkdb_only = true; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + elog(INFO, "Start checking data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + pthread_create(&threads[i], NULL, backup_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(INFO, "Data files are transfered"); + else + elog(ERROR, "Data files transferring failed"); + + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; +} + /* * Take a backup of a single postgresql instance. * Move files from 'pgdata' to a subdirectory in 'backup_path'. @@ -730,6 +825,7 @@ do_backup_instance(void) arg->prev_start_lsn = prev_backup_start_lsn; arg->backup_conn = NULL; arg->cancel_conn = NULL; + arg->checkdb_only = false; /* By default there are some error */ arg->ret = 1; } @@ -856,6 +952,53 @@ do_backup_instance(void) backup_files_list = NULL; } +/* Entry point of pg_probackup CHECKDB subcommand. */ +/* TODO think about merging it with do_backup */ +int +do_checkdb(void) +{ + /* PGDATA is always required */ + if (instance_config.pgdata == NULL) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* Create connection for PostgreSQL */ + backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); + pgut_atexit_push(backup_disconnect, NULL); + + current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + + /* Confirm data block size and xlog block size are compatible */ + confirm_block_size("block_size", BLCKSZ); + confirm_block_size("wal_block_size", XLOG_BLCKSZ); + + current.from_replica = pg_is_in_recovery(); + + /* Confirm that this server version is supported */ + check_server_version(); + + /* Do we need that? */ + current.checksum_version = get_data_checksum_version(true); + + is_checksum_enabled = pg_checksum_enable(); + + if (is_checksum_enabled) + elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + "Data block corruption will be detected"); + else + elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " + "pg_probackup have no way to detect data block corruption without them. " + "Reinitialize PGDATA with option '--data-checksums'."); + + StrNCpy(current.server_version, server_version_str, + sizeof(current.server_version)); + + do_check_instance(); + return 0; +} + /* * Entry point of pg_probackup BACKUP subcommand. */ @@ -2268,6 +2411,7 @@ backup_files(void *arg) /* File exists in previous backup */ file->exists_in_prev = true; } + /* copy the file into backup */ if (file->is_datafile && !file->is_cfs) { @@ -2276,8 +2420,12 @@ backup_files(void *arg) join_path_components(to_path, arguments->to_root, file->path + strlen(arguments->from_root) + 1); + if (arguments->checkdb_only) + { + check_data_file(arguments, file); + } /* backup block by block if datafile AND not compressed by cfs*/ - if (!backup_data_file(arguments, to_path, file, + else if (!backup_data_file(arguments, to_path, file, arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, @@ -2288,10 +2436,10 @@ backup_files(void *arg) continue; } } - else if (strcmp(file->name, "pg_control") == 0) + else if (strcmp(file->name, "pg_control") == 0 && !arguments->checkdb_only) copy_pgcontrol_file(arguments->from_root, arguments->to_root, file); - else + else if (!arguments->checkdb_only) { bool skip = false; diff --git a/src/data.c b/src/data.c index 19bb742b6..65ff6156e 100644 --- a/src/data.c +++ b/src/data.c @@ -503,6 +503,67 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->write_size += write_buffer_size; } +bool +check_data_file(backup_files_arg* arguments, + pgFile *file) +{ + FILE *in; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int page_state; + char curr_page[BLCKSZ]; + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + FIN_FILE_CRC32(true, file->crc); + + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return false; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, 0, //0 = InvalidXLogRecPtr + blknum, nblocks, in, &n_blocks_skipped, + BACKUP_MODE_FULL, curr_page); + if (page_state == PageIsTruncated) + break; + } + + return true; + fclose(in); +} + /* * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 176768766..bcc6d7d7d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -38,7 +38,8 @@ typedef enum ProbackupSubcmd MERGE_CMD, SHOW_CMD, SET_CONFIG_CMD, - SHOW_CONFIG_CMD + SHOW_CONFIG_CMD, + CHECKDB_CMD } ProbackupSubcmd; /* directory options */ @@ -234,6 +235,8 @@ main(int argc, char *argv[]) backup_subcmd = SET_CONFIG_CMD; else if (strcmp(argv[1], "show-config") == 0) backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "checkdb") == 0) + backup_subcmd = CHECKDB_CMD; else if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0 || strcmp(argv[1], "help") == 0) @@ -268,6 +271,7 @@ main(int argc, char *argv[]) * Make command string before getopt_long() will call. It permutes the * content of argv. */ + /* TODO why do we do that only for some commands? */ command_name = pstrdup(argv[1]); if (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || @@ -309,8 +313,8 @@ main(int argc, char *argv[]) if (help_opt) help_command(command_name); - /* backup_path is required for all pg_probackup commands except help */ - if (backup_path == NULL) + /* backup_path is required for all pg_probackup commands except help and checkdb */ + if (backup_path == NULL && backup_subcmd != CHECKDB_CMD) { /* * If command line argument is not set, try to read BACKUP_PATH @@ -320,17 +324,22 @@ main(int argc, char *argv[]) if (backup_path == NULL) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } - canonicalize_path(backup_path); - /* Ensure that backup_path is an absolute path */ - if (!is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); + if (backup_path != NULL) + { + canonicalize_path(backup_path); + + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); - /* Ensure that backup_path is a path to a directory */ - rc = stat(backup_path, &stat_buf); - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + } + /* TODO What is this block about?*/ /* command was initialized for a few commands */ if (command) { @@ -340,9 +349,10 @@ main(int argc, char *argv[]) command = NULL; } + /* TODO it would be better to list commands that require instance option */ /* Option --instance is required for all commands except init and show */ if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != VALIDATE_CMD) + backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) { if (instance_name == NULL) elog(ERROR, "required parameter not specified: --instance"); @@ -352,7 +362,7 @@ main(int argc, char *argv[]) * If --instance option was passed, construct paths for backup data and * xlog files of this backup instance. */ - if (instance_name) + if ((backup_path != NULL) && instance_name) { sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); @@ -375,20 +385,25 @@ main(int argc, char *argv[]) * Read options from env variables or from config file, * unless we're going to set them via set-config. */ - if (instance_name) + if (((backup_path != NULL) && instance_name) + && backup_subcmd != SET_CONFIG_CMD) { - char path[MAXPGPATH]; - + char config_path[MAXPGPATH]; /* Read environment variables */ config_get_opt_env(instance_options); /* Read options from configuration file */ - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(path, instance_options, ERROR, true); + join_path_components(config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + config_read_opt(config_path, instance_options, ERROR, true); } + /* Just read environment variables */ + if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) + config_get_opt_env(instance_options); + /* Initialize logger */ - init_logger(backup_path, &instance_config.logger); + if (backup_subcmd != CHECKDB_CMD) + init_logger(backup_path, &instance_config.logger); /* * We have read pgdata path from command line or from configuration file. @@ -517,6 +532,9 @@ main(int argc, char *argv[]) case SET_CONFIG_CMD: do_set_config(); break; + case CHECKDB_CMD: + do_checkdb(); + break; case NO_CMD: /* Should not happen */ elog(ERROR, "Unknown subcommand"); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0dc649a29..5de5f10a7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -296,6 +296,7 @@ typedef struct PGconn *backup_conn; PGcancel *cancel_conn; + bool checkdb_only; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. @@ -391,6 +392,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); +extern int do_checkdb(void); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, @@ -524,6 +526,8 @@ extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); /* in data.c */ +extern bool check_data_file(backup_files_arg* arguments, + pgFile *file); extern bool backup_data_file(backup_files_arg* arguments, const char *to_path, pgFile *file, XLogRecPtr prev_backup_start_lsn, From a9fd90a8317bcd0932e9392dd022be4c42b4dcba Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 3 Dec 2018 11:31:10 +0300 Subject: [PATCH 0096/2107] PGPRO-2065 checkdb. Draft version --- src/backup.c | 98 ++++++++++++++++++++++++++++++++++++++++------ src/pg_probackup.h | 1 - 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index ed88051ec..c19721d86 100644 --- a/src/backup.c +++ b/src/backup.c @@ -95,6 +95,7 @@ static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); static void *backup_files(void *arg); +static void *check_files(void *arg); static void *remote_backup_files(void *arg); static void do_backup_instance(void); @@ -521,7 +522,6 @@ do_check_instance(void) arg->prev_start_lsn = InvalidXLogRecPtr; arg->backup_conn = NULL; arg->cancel_conn = NULL; - arg->checkdb_only = true; /* By default there are some error */ arg->ret = 1; } @@ -534,7 +534,7 @@ do_check_instance(void) elog(VERBOSE, "Start thread num: %i", i); - pthread_create(&threads[i], NULL, backup_files, arg); + pthread_create(&threads[i], NULL, check_files, arg); } /* Wait threads */ @@ -545,9 +545,9 @@ do_check_instance(void) backup_isok = false; } if (backup_isok) - elog(INFO, "Data files are transfered"); + elog(INFO, "Data files are checked"); else - elog(ERROR, "Data files transferring failed"); + elog(ERROR, "Data files checking failed"); parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); @@ -825,7 +825,6 @@ do_backup_instance(void) arg->prev_start_lsn = prev_backup_start_lsn; arg->backup_conn = NULL; arg->cancel_conn = NULL; - arg->checkdb_only = false; /* By default there are some error */ arg->ret = 1; } @@ -2332,6 +2331,85 @@ backup_disconnect(bool fatal, void *userdata) pgut_disconnect(master_conn); } +static void * +check_files(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + + /* check a file */ + for (i = 0; i < n_backup_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + elog(VERBOSE, "Checking file: \"%s\" ", file->path); + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during backup"); + + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + file->write_size = BYTES_INVALID; + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to backup \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* No need to check directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* check only uncompressed datafiles */ + if (file->is_datafile && !file->is_cfs) + { + char to_path[MAXPGPATH]; + + join_path_components(to_path, arguments->to_root, + file->path + strlen(arguments->from_root) + 1); + + check_data_file(arguments, file); + } + } + else + elog(WARNING, "unexpected file type %d", buf.st_mode); + } + + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + /* * Take a backup of the PGDATA at a file level. * Copy all directories and files listed in backup_files_list. @@ -2420,12 +2498,8 @@ backup_files(void *arg) join_path_components(to_path, arguments->to_root, file->path + strlen(arguments->from_root) + 1); - if (arguments->checkdb_only) - { - check_data_file(arguments, file); - } /* backup block by block if datafile AND not compressed by cfs*/ - else if (!backup_data_file(arguments, to_path, file, + if (!backup_data_file(arguments, to_path, file, arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, @@ -2436,10 +2510,10 @@ backup_files(void *arg) continue; } } - else if (strcmp(file->name, "pg_control") == 0 && !arguments->checkdb_only) + else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(arguments->from_root, arguments->to_root, file); - else if (!arguments->checkdb_only) + else { bool skip = false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5de5f10a7..6dd7c63f0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -296,7 +296,6 @@ typedef struct PGconn *backup_conn; PGcancel *cancel_conn; - bool checkdb_only; /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. From a039fd14aa641d8427cabd9dc0caae75dd21da19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:38:45 +0300 Subject: [PATCH 0097/2107] add warning if waiting for START LSN during ARCHIVE backup is exceeding 30 seconds --- src/backup.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/backup.c b/src/backup.c index d92389293..f308aa891 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1665,6 +1665,11 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); } + if (!stream_wal && is_start_lsn && try_count == 30) + elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); + if (timeout > 0 && try_count > timeout) { if (file_exists) From 529173287f5937a22aaa8dc8aab628c708186242 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:41:39 +0300 Subject: [PATCH 0098/2107] fix tabulation offset --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index f308aa891..7046d1d06 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1667,8 +1667,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (!stream_wal && is_start_lsn && try_count == 30) elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " - "If continius archiving is not set up, use '--stream' option to make autonomous backup. " - "Otherwise check that continius archiving works correctly."); + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); if (timeout > 0 && try_count > timeout) { From 3843f5893368fd1a9c45f4f1135d9fc5808882d5 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 3 Dec 2018 20:07:59 +0300 Subject: [PATCH 0099/2107] Restore extra directories listed only in last backup --- src/backup.c | 45 ++++++++++++++------------------ src/catalog.c | 35 ++++++++++++++++++++++--- src/dir.c | 64 ++++++++++++++++++++++++++++++++++++++-------- src/pg_probackup.h | 6 +++++ src/restore.c | 55 +++++++++++++++++++++++++++------------ 5 files changed, 148 insertions(+), 57 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5b911689c..91813ebb5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -472,7 +472,6 @@ do_backup_instance(void) pthread_t *threads; backup_files_arg *threads_args; bool backup_isok = true; - char *p; pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; @@ -482,19 +481,8 @@ do_backup_instance(void) pgFile *pg_control = NULL; elog(LOG, "Database backup start"); - extra_dirs = parray_new(); - /* TODO: Add path validation */ - if(extradir) - { - p = strtok(extradir,":"); - while(p!=NULL) - { - char * dir = (char *)palloc(strlen(p) + 1); - strcpy(dir,p); - parray_append(extra_dirs, dir); - p=strtok(NULL,":"); - } - } + if(current.extra_dir_str) + extra_dirs = make_extra_directory_list(current.extra_dir_str); /* Initialize size summary */ current.data_bytes = 0; @@ -633,7 +621,17 @@ do_backup_instance(void) if (is_remote_backup) get_remote_pgdata_filelist(backup_files_list); else - dir_list_file(backup_files_list, pgdata, true, true, false, false); + dir_list_file(backup_files_list, pgdata, true, true, false, 0); + + /* + * Append to backup list all files and directories + * from extra directory option + */ + if (extra_dirs) + for (i = 0; i < parray_num(extra_dirs); i++) + /* Extra dirs numeration starts with 1. 0 value is not extra dir */ + dir_list_file(backup_files_list, parray_get(extra_dirs, i), + true, true, false, i+1); /* * Sort pathname ascending. It is necessary to create intermediate @@ -651,12 +649,6 @@ do_backup_instance(void) /* Extract information about files in backup_list parsing their names:*/ parse_backup_filelist_filenames(backup_files_list, pgdata); - /* Append to backup list all files dirictories from extra dirictory option */ - for (i = 0; i < parray_num(extra_dirs); i++) - /* Extra dirs numeration starts with 1. 0 value is not extra dir */ - dir_list_file(backup_files_list, (char *) parray_get(extra_dirs, i), - true, true, true, i+1); - if (current.backup_mode != BACKUP_MODE_FULL) { elog(LOG, "current_tli:%X", current.tli); @@ -786,10 +778,7 @@ do_backup_instance(void) } /* clean extra directories list */ if (extra_dirs) - { - parray_walk(extra_dirs, pfree); - parray_free(extra_dirs); - } + free_dir_list(extra_dirs); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -981,6 +970,10 @@ do_backup(time_t start_time) StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); + /* Save list of extra directories */ + if(extradir) + current.extra_dir_str = extradir; + /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) elog(ERROR, "cannot create backup directory"); @@ -2246,7 +2239,7 @@ backup_files(void *arg) continue; } } - else + else { const char *src; const char *dst; diff --git a/src/catalog.c b/src/catalog.c index 788c33f76..8de7bb569 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -406,7 +406,26 @@ pgBackupCreateDir(pgBackup *backup) { int i; char path[MAXPGPATH]; - char *subdirs[] = { DATABASE_DIR, NULL }; + parray *subdirs = parray_new(); + char *temp; + + temp = palloc(strlen(DATABASE_DIR) + 1); + parray_append(subdirs, temp); + + /* Add extra dirs containers */ + if (backup->extra_dir_str) + { + parray *extradirs_list = make_extra_directory_list(backup->extra_dir_str); + for (int i = 0; i < parray_num(extradirs_list); i++) + { + /* 20 chars is enough to hold the extradir number in string. */ + temp = palloc(strlen(EXTRA_DIR) + 20); + /* Numeration of extradirs starts with 1 */ + makeExtraDirPathByNum(temp, EXTRA_DIR, i+1); + parray_append(subdirs, temp); + } + free_dir_list(extradirs_list); + } pgBackupGetPath(backup, path, lengthof(path), NULL); @@ -416,12 +435,13 @@ pgBackupCreateDir(pgBackup *backup) dir_create_dir(path, DIR_PERMISSION); /* create directories for actual backup files */ - for (i = 0; subdirs[i]; i++) + for (i = 0; i < parray_num(subdirs); i++) { - pgBackupGetPath(backup, path, lengthof(path), subdirs[i]); + pgBackupGetPath(backup, path, lengthof(path), parray_get(subdirs, i)); dir_create_dir(path, DIR_PERMISSION); } + free_dir_list(subdirs); return 0; } @@ -494,6 +514,10 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* print connection info except password */ if (backup->primary_conninfo) fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); + + /* print extra directories list */ + if (backup->extra_dir_str) + fprintf(out, "extra-directory = '%s'\n", backup->extra_dir_str); } /* @@ -587,6 +611,7 @@ readBackupControlFile(const char *path) {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, + {'s', 0, "extra-directory", &backup->extra_dir_str, SOURCE_FILE_STRICT}, {0} }; @@ -816,6 +841,7 @@ pgBackupInit(pgBackup *backup) backup->primary_conninfo = NULL; backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; + backup->extra_dir_str = NULL; } /* @@ -830,6 +856,8 @@ pgBackupCopy(pgBackup *dst, pgBackup *src) if (src->primary_conninfo) dst->primary_conninfo = pstrdup(src->primary_conninfo); + if (src->extra_dir_str) + dst->extra_dir_str = pstrdup(src->extra_dir_str); } /* free pgBackup object */ @@ -839,6 +867,7 @@ pgBackupFree(void *backup) pgBackup *b = (pgBackup *) backup; pfree(b->primary_conninfo); + pfree(b->extra_dir_str); pfree(backup); } diff --git a/src/dir.c b/src/dir.c index fe0804ed6..98cc518dd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -115,8 +115,6 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; -static int BlackListCompare(const void *str1, const void *str2); - static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, @@ -385,7 +383,8 @@ pgFileCompareSize(const void *f1, const void *f2) return 0; } -static int +/* Compare two strings */ +int BlackListCompare(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); @@ -446,15 +445,19 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, elog(WARNING, "Skip \"%s\": unexpected file format", file->path); return; } - if (add_root) + if (extra_dir_num) { - file->extradir = dirname(pgut_strdup(file->path)); - elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s",file->name, file->path, file->extradir); - parray_append(files, file); + file->extradir = pgut_strdup(file->path); + elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s", + file->name, file->path, file->extradir); } + if (add_root) + parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, extra_dir_num); + + free_dir_list(black_list); } /* @@ -705,10 +708,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, /* If it is extra dir, remember it */ if (extra_dir_num) - { - file->extradir = pgut_strdup(root); - dirname(file->extradir); - } + file->extradir = parent->extradir; /* We add the directory anyway */ if (S_ISDIR(file->mode)) @@ -1550,3 +1550,45 @@ pgFileSize(const char *path) return buf.st_size; } + +/* + * Construct parray containing extra directories paths + * from string like /path1:/path2 + */ +parray * +make_extra_directory_list(const char *colon_separated_dirs) +{ + char *p; + parray *list = parray_new(); + char *tmp = palloc(strlen(colon_separated_dirs) + 1); + + /* TODO: Add path validation */ + strcpy(tmp, colon_separated_dirs); + p = strtok(tmp,":"); + while(p!=NULL) + { + char * dir = (char *)palloc(strlen(p) + 1); + strcpy(dir,p); + parray_append(list, dir); + p=strtok(NULL,":"); + } + pfree(tmp); + parray_qsort(list, BlackListCompare); + return list; +} + +/* Free memory of parray containing strings */ +void +free_dir_list(parray *list) +{ + parray_walk(list, pfree); + parray_free(list); +} + +/* Append to string "pattern_path" int "dir_num" */ +void +makeExtraDirPathByNum(char *ret_path, const char *pattern_path, + const int dir_num) +{ + sprintf(ret_path, "%s%d", pattern_path, dir_num); +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2cae08b8c..d0230f45f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -251,6 +251,7 @@ struct pgBackup pgBackup *parent_backup_link; char *primary_conninfo; /* Connection parameters of the backup * in the format suitable for recovery.conf */ + char *extra_dir_str; /* List of extra directories, separated by ':' */ }; /* Recovery target for restore and validate subcommands */ @@ -527,6 +528,10 @@ extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); +extern parray *make_extra_directory_list(const char *colon_separated_dirs); +extern void free_dir_list(parray *list); +extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, + const int dir_num); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); @@ -544,6 +549,7 @@ extern int pgFileComparePathWithExtra(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); +extern int BlackListCompare(const void *str1, const void *str2); /* in data.c */ extern bool backup_data_file(backup_files_arg* arguments, diff --git a/src/restore.c b/src/restore.c index de1ba4e59..9c176daeb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,6 +21,7 @@ typedef struct { parray *files; pgBackup *backup; + parray *extra_dirs; /* * Return value from the thread. @@ -29,14 +30,14 @@ typedef struct int ret; } restore_files_arg; -static void restore_backup(pgBackup *backup); +static void restore_backup(pgBackup *backup, const char *extra_dir_str); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); - +static bool backup_contains_extra(const char *dir, parray *dirs_list); /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. @@ -387,7 +388,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", base36enc(dest_backup->start_time), dest_backup->server_version); - restore_backup(backup); + restore_backup(backup, dest_backup->extra_dir_str); } /* @@ -414,7 +415,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup) +restore_backup(pgBackup *backup, const char *extra_dir_str) { char timestamp[100]; char this_backup_path[MAXPGPATH]; @@ -422,6 +423,7 @@ restore_backup(pgBackup *backup) char extra_path[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; + parray *extra_dirs; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -453,6 +455,15 @@ restore_backup(pgBackup *backup) pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); create_data_directories(pgdata, this_backup_path, true); + if(extra_dir_str) + { + extra_dirs = make_extra_directory_list(extra_dir_str); + for (int i = 0; i < parray_num(extra_dirs); i++) + { + dir_create_dir(parray_get(extra_dirs, i), DIR_PERMISSION); + } + } + /* * Get list of files which need to be restored. */ @@ -472,25 +483,23 @@ restore_backup(pgBackup *backup) { pgFile *file = (pgFile *) parray_get(files, i); - /* if the entry was a directory, create it in the backup */ - if (S_ISDIR(file->mode)) + /* if the entry was an extra directory, create it in the backup */ + if (file->extra_dir_num && S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; char *dir_name; - if (file->extra_dir_num) + if (backup_contains_extra(file->extradir, extra_dirs)) { + char container_dir[MAXPGPATH]; + makeExtraDirPathByNum(container_dir, extra_path, + file->extra_dir_num); + dir_name = GetRelativePath(file->path, container_dir); + elog(VERBOSE, "Create directory \"%s\" NAME %s", + dir_name, file->name); join_path_components(dirpath, file->extradir, dir_name); - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - dir_name = GetRelativePath(file->path, extra_path); + dir_create_dir(dirpath, DIR_PERMISSION); } - else - { - dir_name = GetRelativePath(file->path, database_path); - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); - join_path_components(dirpath, pgdata, dir_name); - } - dir_create_dir(dirpath, DIR_PERMISSION); } /* setup threads */ @@ -506,6 +515,7 @@ restore_backup(pgBackup *backup) arg->files = files; arg->backup = backup; + arg->extra_dirs = extra_dirs; /* By default there are some error */ threads_args[i].ret = 1; @@ -661,7 +671,10 @@ restore_files(void *arg) parse_program_version(arguments->backup->program_version)); } else if (file->extra_dir_num) - copy_file(from_root, file->extradir, file); + { + if (backup_contains_extra(file->extradir, arguments->extra_dirs)) + copy_file(from_root, file->extradir, file); + } else copy_file(from_root, pgdata, file); @@ -1032,3 +1045,11 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } + +/* Check if "dir" presents in "dirs_list" */ +static bool +backup_contains_extra(const char *dir, parray *dirs_list) +{ + void *temp = parray_bsearch(dirs_list, dir, BlackListCompare); + return temp != NULL; +} From 0eab1f6c136e85b515e82c509223c2ad915588ff Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 3 Dec 2018 22:04:40 +0300 Subject: [PATCH 0100/2107] PGPRO-2065 checkdb --amcheck. Draft version --- src/backup.c | 180 +++++++++++++++++++++++++++++++++++++++++++-- src/pg_probackup.c | 6 +- src/pg_probackup.h | 10 ++- 3 files changed, 189 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index c19721d86..845edbcd6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -47,6 +47,8 @@ const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; +static parray *index_list = NULL; + /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -99,7 +101,10 @@ static void *check_files(void *arg); static void *remote_backup_files(void *arg); static void do_backup_instance(void); -static void do_check_instance(void); +static void do_check_instance(bool do_amcheck); +static parray* get_index_list(void); +static bool amcheck_one_index(backup_files_arg *arguments, + Oid indexrelid); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_switch_wal(PGconn *conn); @@ -463,7 +468,7 @@ remote_backup_files(void *arg) /* TODO think about merging it with do_backup_instance */ static void -do_check_instance(void) +do_check_instance(bool do_amcheck) { int i; char database_path[MAXPGPATH]; @@ -507,6 +512,9 @@ do_check_instance(void) /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); + if (do_amcheck) + index_list = get_index_list(); + /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -522,6 +530,7 @@ do_check_instance(void) arg->prev_start_lsn = InvalidXLogRecPtr; arg->backup_conn = NULL; arg->cancel_conn = NULL; + arg->index_list = index_list; /* By default there are some error */ arg->ret = 1; } @@ -954,7 +963,7 @@ do_backup_instance(void) /* Entry point of pg_probackup CHECKDB subcommand. */ /* TODO think about merging it with do_backup */ int -do_checkdb(void) +do_checkdb(bool do_amcheck) { /* PGDATA is always required */ if (instance_config.pgdata == NULL) @@ -994,7 +1003,7 @@ do_checkdb(void) StrNCpy(current.server_version, server_version_str, sizeof(current.server_version)); - do_check_instance(); + do_check_instance(do_amcheck); return 0; } @@ -2337,6 +2346,11 @@ check_files(void *arg) int i; backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = parray_num(arguments->files_list); + int n_indexes = 0; + + /* We only have index_list in amcheck mode */ + if (arguments->index_list) + n_indexes = parray_num(arguments->index_list); /* check a file */ for (i = 0; i < n_backup_files_list; i++) @@ -2351,7 +2365,7 @@ check_files(void *arg) /* check for interrupt */ if (interrupted) - elog(ERROR, "interrupted during backup"); + elog(ERROR, "interrupted during checkdb"); if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", @@ -2400,11 +2414,29 @@ check_files(void *arg) elog(WARNING, "unexpected file type %d", buf.st_mode); } + + /* check index with amcheck */ + for (i = 0; i < n_indexes; i++) + { + pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); + + elog(VERBOSE, "Checking index: \"%s\" ", ind->name); + if (!pg_atomic_test_set_flag(&ind->lock)) + continue; + + /* check for interrupt */ + if (interrupted) + elog(ERROR, "interrupted during checkdb --amcheck"); + + amcheck_one_index(arguments, ind->indexrelid); + } + /* Close connection */ if (arguments->backup_conn) pgut_disconnect(arguments->backup_conn); /* Data files transferring is successful */ + /* TODO where should we set arguments->ret to 1? */ arguments->ret = 0; return NULL; @@ -3105,3 +3137,141 @@ pg_ptrack_get_block(backup_files_arg *arguments, return result; } + +/* Clear ptrack files in all databases of the instance we connected to */ +static parray* +get_index_list(void) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + const char *indexname; + Oid indexrelid; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + dbname, + instance_config.pguser); + + res = pgut_execute(tmp_conn, "select * from pg_extension where extname='amcheck'", 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(ERROR, "cannot check if amcheck is installed in database %s: %s", + dbname, PQerrorMessage(tmp_conn)); + } + + if (PQntuples(res) < 1) + { + elog(VERBOSE, "extension amcheck is not installed in database %s", dbname); + continue; + } + + PQclear(res); + res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" + " FROM pg_index idx " + " JOIN pg_class cls ON cls.oid=idx.indexrelid " + " JOIN pg_am am ON am.oid=cls.relam " + " WHERE am.amname='btree'; ", 0, NULL); + + /* TODO filter system indexes to add them to list only once */ + /* TODO maybe save tablename for load balancing */ + for(i = 0; i < PQntuples(res); i++) + { + pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); + char *name = NULL; + + ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + name = PQgetvalue(res, i, 1); + ind->name = pgut_malloc(strlen(name) + 1); + strcpy(ind->name, name); /* enough buffer size guaranteed */ + + pg_atomic_clear_flag(&ind->lock); + + if (index_list == NULL) + index_list = parray_new(); + + /* TODO maybe keep structure with index name and some other fileds? */ + parray_append(index_list, ind); + } + + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); + return index_list; +} + +/* check one index. Return true if everything is ok, false otherwise. */ +static bool +amcheck_one_index(backup_files_arg *arguments, + Oid indexrelid) +{ + PGresult *res; + char *params[1]; + char *result; + + params[0] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", indexrelid); + + if (arguments->backup_conn == NULL) + { + arguments->backup_conn = pgut_connect(instance_config.pghost, + instance_config.pgport, + instance_config.pgdatabase, + instance_config.pguser); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "SELECT bt_index_check($1)", + 1, (const char **)params, true); + + /* TODO now this check is hidden inside pgut_execute_parallel + * We need to handle failed check as an error. + */ + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + elog(VERBOSE, "amcheck failed for relation oid %u: %s", + indexrelid, PQresultErrorMessage(res)); + + pfree(params[0]); + PQclear(res); + return false; + } + + pfree(params[0]); + PQclear(res); + return true; +} \ No newline at end of file diff --git a/src/pg_probackup.c b/src/pg_probackup.c index bcc6d7d7d..a59e25ccd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -86,6 +86,8 @@ bool restore_no_validate = false; bool skip_block_validation = false; +bool do_amcheck = false; + /* delete options */ bool delete_wal = false; bool delete_expired = false; @@ -151,6 +153,8 @@ static ConfigOption cmd_options[] = { 'b', 143, "no-validate", &restore_no_validate, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, + /* checkdb options */ + { 'b', 155, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -533,7 +537,7 @@ main(int argc, char *argv[]) do_set_config(); break; case CHECKDB_CMD: - do_checkdb(); + do_checkdb(do_amcheck); break; case NO_CMD: /* Should not happen */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6dd7c63f0..5b45cec85 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -129,6 +129,13 @@ typedef struct pgFile * i.e. datafiles without _ptrack */ } pgFile; +typedef struct pg_indexEntry +{ + Oid indexrelid; + char *name; + volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ +} pg_indexEntry; + /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ @@ -295,6 +302,7 @@ typedef struct PGconn *backup_conn; PGcancel *cancel_conn; + parray *index_list; /* * Return value from the thread. @@ -391,7 +399,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); -extern int do_checkdb(void); +extern int do_checkdb(bool do_amcheck); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, From b4275a6f4f197fab6da2f94ac4f6af57aab942d0 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 4 Dec 2018 17:24:11 +0300 Subject: [PATCH 0101/2107] Remote read of config file --- src/utils/configuration.c | 3 ++- src/utils/pgut.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 9b88d1594..2893fcf92 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -11,6 +11,7 @@ #include "configuration.h" #include "logger.h" #include "pgut.h" +#include "file.h" #include "datatype/timestamp.h" @@ -555,7 +556,7 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, } } - fclose(fp); + fio_close_stream(fp); return parsed_options; } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 70eab188b..e12052b9b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -841,7 +841,7 @@ pgut_fopen(const char *path, const char *mode, bool missing_ok) { FILE *fp; - if ((fp = fopen(path, mode)) == NULL) + if ((fp = fio_open_stream(path, FIO_BACKUP_HOST)) == NULL) { if (missing_ok && errno == ENOENT) return NULL; From 4ca180dfa055136c80dba4e69b5f16f859127982 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 4 Dec 2018 18:04:44 +0300 Subject: [PATCH 0102/2107] Add --remote-path option --- src/pg_probackup.c | 10 ++++++---- src/pg_probackup.h | 1 + src/utils/remote.c | 15 +++++++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fece11201..f17176137 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -69,6 +69,7 @@ bool backup_logs = false; bool smooth_checkpoint; char *remote_host; char *remote_port; +char *remote_path; char *remote_proto = (char*)"ssh"; char *ssh_config; char *ssh_options; @@ -145,10 +146,11 @@ static ConfigOption cmd_options[] = { 's', 19, "remote-host", &remote_host, SOURCE_CMD_STRICT, }, { 's', 20, "remote-port", &remote_port, SOURCE_CMD_STRICT, }, { 's', 21, "remote-proto", &remote_proto, SOURCE_CMD_STRICT, }, - { 's', 22, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, - { 's', 23, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, - { 'b', 24, "agent", &is_remote_agent, SOURCE_CMD_STRICT, }, - { 'b', 25, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, + { 's', 22, "remote-path", &remote_path, SOURCE_CMD_STRICT, }, + { 's', 23, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, + { 's', 24, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, + { 'b', 25, "agent", &is_remote_agent, SOURCE_CMD_STRICT, }, + { 'b', 26, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, /* restore options */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a7200cd67..6077cc8f4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -360,6 +360,7 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; +extern char *remote_path; extern char *remote_port; extern char *remote_host; extern char *remote_proto; diff --git a/src/utils/remote.c b/src/utils/remote.c index e32727143..7f903a790 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -71,6 +71,7 @@ int remote_execute(int argc, char* argv[], bool listen) int outfd[2]; int infd[2]; pid_t pid; + char* pg_probackup = argv[0]; ssh_argc = 0; ssh_argv[ssh_argc++] = remote_proto; @@ -86,10 +87,20 @@ int remote_execute(int argc, char* argv[], bool listen) ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, ssh_options); } ssh_argv[ssh_argc++] = remote_host; - ssh_argv[ssh_argc++] = cmd+1; + ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; - for (i = 0; i < argc; i++) { + if (remote_path) + { + char* sep = strrchr(pg_probackup, '/'); + if (sep != NULL) { + pg_probackup = sep + 1; + } + dst = snprintf(cmd, sizeof(cmd), "%s/%s", remote_path, pg_probackup); + } else { + dst = snprintf(cmd, sizeof(cmd), "%s", pg_probackup); + } + for (i = 1; i < argc; i++) { dst = append_option(cmd, sizeof(cmd), dst, argv[i]); } dst = append_option(cmd, sizeof(cmd), dst, "--agent"); From 747bca7a6d08b78fc4b23ed75eaceb0ef5a484ca Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 5 Dec 2018 19:45:23 +0300 Subject: [PATCH 0103/2107] Clear O_EXCL flag for remote backup --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 1b83ec560..0aec6f6e5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -139,7 +139,7 @@ int fio_open(char const* path, int mode, fio_location location) hdr.cop = FIO_OPEN; hdr.handle = i; hdr.size = strlen(path) + 1; - hdr.arg = mode; + hdr.arg = mode & ~O_EXCL; fio_fdset |= 1 << i; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); From 4ac0fab61b21d80c260078b04453f3b6a1b9e16d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 16:03:29 +0300 Subject: [PATCH 0104/2107] Show progress messages with INFO verbose level --- src/delete.c | 10 ++++++---- src/merge.c | 2 +- src/restore.c | 2 +- src/validate.c | 9 +++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/delete.c b/src/delete.c index e7c9ac242..8e9b965fc 100644 --- a/src/delete.c +++ b/src/delete.c @@ -256,6 +256,7 @@ delete_backup_files(pgBackup *backup) char path[MAXPGPATH]; char timestamp[100]; parray *files; + size_t num_files; /* * If the backup was deleted already, there is nothing to do. @@ -286,13 +287,14 @@ delete_backup_files(pgBackup *backup) /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); - for (i = 0; i < parray_num(files); i++) + num_files = parray_num(files); + for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(files, i); - /* print progress */ - elog(VERBOSE, "Delete file(%zd/%lu) \"%s\"", i + 1, - (unsigned long) parray_num(files), file->path); + if (progress) + elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", + i + 1, num_files, file->path); if (remove(file->path)) { diff --git a/src/merge.c b/src/merge.c index b01a90d00..2a24b39dd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -385,7 +385,7 @@ merge_files(void *arg) elog(ERROR, "Interrupted during merging backups"); if (progress) - elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); /* diff --git a/src/restore.c b/src/restore.c index 6024f8bcf..8770c3586 100644 --- a/src/restore.c +++ b/src/restore.c @@ -582,7 +582,7 @@ restore_files(void *arg) rel_path = GetRelativePath(file->path,from_root); if (progress) - elog(LOG, "Progress: (%d/%lu). Process file %s ", + elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), rel_path); /* diff --git a/src/validate.c b/src/validate.c index 1f758d0f0..edb386ef9 100644 --- a/src/validate.c +++ b/src/validate.c @@ -156,9 +156,10 @@ pgBackupValidateFiles(void *arg) { int i; validate_files_arg *arguments = (validate_files_arg *)arg; + int num_files = parray_num(arguments->files); pg_crc32 crc; - for (i = 0; i < parray_num(arguments->files); i++) + for (i = 0; i < num_files; i++) { struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); @@ -186,9 +187,9 @@ pgBackupValidateFiles(void *arg) if (file->is_cfs) continue; - /* print progress */ - elog(VERBOSE, "Validate files: (%d/%lu) %s", - i + 1, (unsigned long) parray_num(arguments->files), file->path); + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); if (stat(file->path, &st) == -1) { From 1bf9fce684b5ffa83e7306c80e2931d7a3206a9e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 6 Dec 2018 17:05:37 +0300 Subject: [PATCH 0105/2107] Change way of transfering variables between agent and server --- src/backup.c | 2 +- src/utils/file.c | 27 ++++++++++++++++----------- src/utils/file.h | 12 +++++++++--- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index f8def77fc..c5bcba6a0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -999,7 +999,7 @@ do_backup(time_t start_time) // current.data_bytes); if (is_remote_agent) - fio_transfer(¤t.start_time,current.start_time); + fio_transfer(FIO_BACKUP_START_TIME); else complete_backup(); diff --git a/src/utils/file.c b/src/utils/file.c index 0aec6f6e5..9357d5197 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -16,6 +16,11 @@ static void* fio_stdin_buffer; static int fio_stdout = 0; static int fio_stdin = 0; +static fio_binding fio_bindings[] = +{ + {¤t.start_time, sizeof(time_t)} +}; + #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) void fio_redirect(int in, int out) @@ -682,23 +687,22 @@ static void fio_send_file(int out, char const* path) } } -void fio_transfer(void* addr, size_t value) +void fio_transfer(fio_shared_variable var) { - struct { - fio_header hdr; - fio_binding bind; - } msg; + size_t var_size = fio_bindings[var].size; + fio_header* msg = (fio_header*)malloc(sizeof(fio_header) + var_size); - msg.hdr.cop = FIO_TRANSFER; - msg.hdr.size = sizeof(fio_binding); - msg.bind.address = (size_t*)addr; - msg.bind.value = value; + msg->cop = FIO_TRANSFER; + msg->arg = var; + msg->size = var_size; + memcpy(msg+1, fio_bindings[var].address, var_size); SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &msg, sizeof(msg)), sizeof(msg)); + IO_CHECK(fio_write_all(fio_stdout, &msg, sizeof(fio_header) + var_size), sizeof(fio_header) + var_size); SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + free(msg); } void fio_communicate(int in, int out) @@ -778,7 +782,8 @@ void fio_communicate(int in, int out) SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; case FIO_TRANSFER: - *((fio_binding*)buf)->address = ((fio_binding*)buf)->value; + Assert(hdr.size == fio_bindings[hdr.arg].size); + memcpy(fio_bindings[hdr.arg].address, buf, hdr.size); break; default: Assert(false); diff --git a/src/utils/file.h b/src/utils/file.h index 6fea1c193..b2a5ac9be 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -36,6 +36,11 @@ typedef enum FIO_REMOTE_HOST } fio_location; +typedef enum +{ + FIO_BACKUP_START_TIME +} fio_shared_variable; + #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 @@ -50,10 +55,11 @@ typedef struct unsigned arg; } fio_header; + typedef struct { - size_t* address; - size_t value; + void* address; + size_t size; } fio_binding; extern void fio_redirect(int in, int out); @@ -93,7 +99,7 @@ extern gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_l extern int fio_gzclose(gzFile file, char const* path, int tmp_fd); #endif -extern void fio_transfer(void* addr, size_t value); +extern void fio_transfer(fio_shared_variable var); #endif From c493d6f900ca3ec7702744f5be907f992bf42bb3 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 6 Dec 2018 18:31:03 +0300 Subject: [PATCH 0106/2107] Add merge backups with extra directories --- src/backup.c | 2 +- src/catalog.c | 5 +- src/dir.c | 44 ++++++++++++---- src/merge.c | 127 ++++++++++++++++++++++++++++++++++++++++++--- src/pg_probackup.h | 6 ++- src/restore.c | 9 ---- 6 files changed, 162 insertions(+), 31 deletions(-) diff --git a/src/backup.c b/src/backup.c index 91813ebb5..38e73e859 100644 --- a/src/backup.c +++ b/src/backup.c @@ -841,7 +841,7 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - write_backup_filelist(¤t, backup_files_list, pgdata); + write_backup_filelist(¤t, backup_files_list, pgdata, NULL); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) diff --git a/src/catalog.c b/src/catalog.c index 8de7bb569..777f59da6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -548,7 +548,8 @@ write_backup(pgBackup *backup) * Output the list of files to backup catalog DATABASE_FILE_LIST */ void -write_backup_filelist(pgBackup *backup, parray *files, const char *root) +write_backup_filelist(pgBackup *backup, parray *files, const char *root, + const char *extra_prefix) { FILE *fp; char path[MAXPGPATH]; @@ -560,7 +561,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root) elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); - print_file_list(fp, files, root); + print_file_list(fp, files, root, extra_prefix); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || diff --git a/src/dir.c b/src/dir.c index 98cc518dd..844196d65 100644 --- a/src/dir.c +++ b/src/dir.c @@ -174,7 +174,6 @@ pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) file->size = st.st_size; file->mode = st.st_mode; file->extra_dir_num = extra_dir_num; - file->extradir = NULL; return file; } @@ -226,6 +225,7 @@ pgFileInit(const char *path) file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; file->extra_dir_num = 0; + file->extradir = NULL; return file; } @@ -457,7 +457,8 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, extra_dir_num); - free_dir_list(black_list); + if (black_list) + free_dir_list(black_list); } /* @@ -1209,7 +1210,8 @@ check_tablespace_mapping(pgBackup *backup) * Print backup content list. */ void -print_file_list(FILE *out, const parray *files, const char *root) +print_file_list(FILE *out, const parray *files, const char *root, + const char *extra_prefix) { size_t i; @@ -1222,8 +1224,18 @@ print_file_list(FILE *out, const parray *files, const char *root) /* omit root directory portion */ if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - else if(file->extra_dir_num) - path = GetRelativePath(path, file->extradir); + else if (file->extra_dir_num) + { + if (extra_prefix) + { + char extra_root[MAXPGPATH]; + makeExtraDirPathByNum(extra_root, extra_prefix, + file->extra_dir_num); + path = GetRelativePath(path, extra_root); + } + else + path = GetRelativePath(path, file->extradir); + } fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " @@ -1443,7 +1455,9 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx if (extra_dir_num) { char temp[MAXPGPATH]; - sprintf(temp, "%s%ld", extra_path, extra_dir_num); + + Assert(extra_path); + makeExtraDirPathByNum(temp, extra_path, extra_dir_num); join_path_components(filepath, temp, path); } else @@ -1585,10 +1599,22 @@ free_dir_list(parray *list) parray_free(list); } -/* Append to string "pattern_path" int "dir_num" */ +/* Append to string "path_prefix" int "dir_num" */ void -makeExtraDirPathByNum(char *ret_path, const char *pattern_path, +makeExtraDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num) { - sprintf(ret_path, "%s%d", pattern_path, dir_num); + sprintf(ret_path, "%s%d", path_prefix, dir_num); +} + +/* Check if "dir" presents in "dirs_list" */ +bool +backup_contains_extra(const char *dir, parray *dirs_list) +{ + void *search_result; + + if (!dirs_list) /* There is no extra dirs in backup */ + return false; + search_result = parray_bsearch(dirs_list, dir, BlackListCompare); + return search_result != NULL; } diff --git a/src/merge.c b/src/merge.c index 1cd82ed05..0216de1e1 100644 --- a/src/merge.c +++ b/src/merge.c @@ -18,12 +18,14 @@ typedef struct { parray *to_files; parray *files; + parray *from_extra; pgBackup *to_backup; pgBackup *from_backup; - const char *to_root; const char *from_root; + const char *to_extra_prefix; + const char *from_extra_prefix; /* * Return value from the thread. @@ -34,6 +36,10 @@ typedef struct static void merge_backups(pgBackup *backup, pgBackup *next_backup); static void *merge_files(void *arg); +static void +reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra); +static int +get_extra_index(const char *key, const parray *list); /* * Implementation of MERGE command. @@ -159,13 +165,15 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) *from_backup_id = base36enc_dup(from_backup->start_time); char to_backup_path[MAXPGPATH], to_database_path[MAXPGPATH], + to_extra_prefix[MAXPGPATH], from_backup_path[MAXPGPATH], from_database_path[MAXPGPATH], - control_file[MAXPGPATH], - extra_path[MAXPGPATH]; /* TODO: implement merging backups with - * extra files */ + from_extra_prefix[MAXPGPATH], + control_file[MAXPGPATH]; parray *files, - *to_files; + *to_files, + *to_extra, + *from_extra; pthread_t *threads; merge_files_arg *threads_args; int i; @@ -214,9 +222,13 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), DATABASE_DIR); + pgBackupGetPath(to_backup, to_extra_prefix, lengthof(to_database_path), + EXTRA_DIR); pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), DATABASE_DIR); + pgBackupGetPath(from_backup, from_extra_prefix, lengthof(from_database_path), + EXTRA_DIR); create_data_directories(to_database_path, from_backup_path, false); @@ -228,7 +240,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_files = dir_read_file_list(from_database_path, /* Use from_database_path * so root path will be * equal with 'files' */ - extra_path, /* TODO: make use of it */ + from_extra_prefix, control_file); /* To delete from leaf, sort in reversed order */ parray_qsort(to_files, pgFileComparePathDesc); @@ -237,13 +249,26 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(from_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - files = dir_read_file_list(from_database_path, extra_path, control_file); + files = dir_read_file_list(from_database_path, from_extra_prefix, control_file); /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + /* Create extra directories lists */ + if (to_backup->extra_dir_str) + to_extra = make_extra_directory_list(to_backup->extra_dir_str); + if (from_backup->extra_dir_str) + from_extra = make_extra_directory_list(from_backup->extra_dir_str); + + /* + * Rename extra directoties in to_backup (if exists) + * according to numeration of extra dirs in from_backup. + */ + if (to_extra) + reorder_extra_dirs(to_backup, to_extra, from_extra); + /* Setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -262,6 +287,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) arg->from_backup = from_backup; arg->to_root = to_database_path; arg->from_root = from_database_path; + arg->from_extra = from_extra; + arg->to_extra_prefix = to_extra_prefix; + arg->from_extra_prefix = from_extra_prefix; /* By default there are some error */ arg->ret = 1; @@ -304,7 +332,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) else to_backup->wal_bytes = BYTES_INVALID; - write_backup_filelist(to_backup, files, from_database_path); + write_backup_filelist(to_backup, files, from_database_path, + from_extra_prefix); write_backup(to_backup); delete_source_backup: @@ -321,6 +350,11 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) { pgFile *file = (pgFile *) parray_get(to_files, i); + if (file->extra_dir_num && + backup_contains_extra(file->extradir, from_extra)) + /* Dir already removed*/ + continue; + if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { pgFileDelete(file); @@ -527,6 +561,21 @@ merge_files(void *arg) file->crc = pgFileGetCRC(to_path_tmp, false); } } + else if (file->extra_dir_num) + { + char from_root[MAXPGPATH]; + char to_root[MAXPGPATH]; + int new_dir_num; + + Assert(argument->from_extra); + new_dir_num = get_extra_index(file->extradir, argument->from_extra); + makeExtraDirPathByNum(from_root, argument->from_extra_prefix, + file->extra_dir_num); + makeExtraDirPathByNum(to_root, argument->to_extra_prefix, + new_dir_num); + copy_file(from_root, to_root, file); + file->extra_dir_num = new_dir_num; + } else copy_file(argument->from_root, argument->to_root, file); @@ -540,3 +589,65 @@ merge_files(void *arg) return NULL; } + +/* Recursively delete a directory and its contents */ +static void +remove_dir_with_files(const char *path) +{ + parray *files = parray_new(); + dir_list_file(files, path, true, true, true, 0); + parray_qsort(files, pgFileComparePathDesc); + for (int i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + pgFileDelete(file); + elog(VERBOSE, "Deleted \"%s\"", file->path); + } +} + +/* Get index of extra directory */ +static int +get_extra_index(const char *key, const parray *list) +{ + if (!list) /* Nowhere to search */ + return -1; + for (int i = 0; i < parray_num(list); i++) + { + if (strcmp(key, parray_get(list, i)) == 0) + return i + 1; + } + return -1; +} + +/* Rename directories in to_backup according to order in from_extra */ +static void +reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra) +{ + char extradir_template[MAXPGPATH]; + + Assert(to_extra); + pgBackupGetPath(to_backup, extradir_template, + lengthof(extradir_template), EXTRA_DIR); + for (int i = 0; i < parray_num(to_extra); i++) + { + int from_num = get_extra_index(parray_get(to_extra, i), from_extra); + if (from_num == -1) + { + char old_path[MAXPGPATH]; + makeExtraDirPathByNum(old_path, extradir_template, i + 1); + remove_dir_with_files(old_path); + } + else if (from_num != i + 1) + { + char old_path[MAXPGPATH]; + char new_path[MAXPGPATH]; + makeExtraDirPathByNum(old_path, extradir_template, i + 1); + makeExtraDirPathByNum(new_path, extradir_template, from_num); + elog(VERBOSE, "Rename %s to %s", old_path, new_path); + if (rename (old_path, new_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + old_path, new_path, strerror(errno)); + } + } +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d0230f45f..55a513616 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -491,7 +491,7 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root); + const char *root, const char *extra_prefix); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); @@ -526,12 +526,14 @@ extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(pgut_option *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); -extern void print_file_list(FILE *out, const parray *files, const char *root); +extern void print_file_list(FILE *out, const parray *files, const char *root, + const char *extra_prefix); extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); extern parray *make_extra_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); +extern bool backup_contains_extra(const char *dir, parray *dirs_list); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); diff --git a/src/restore.c b/src/restore.c index 9c176daeb..0c037db00 100644 --- a/src/restore.c +++ b/src/restore.c @@ -37,7 +37,6 @@ static void create_recovery_conf(time_t backup_id, static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void remove_deleted_files(pgBackup *backup); -static bool backup_contains_extra(const char *dir, parray *dirs_list); /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. @@ -1045,11 +1044,3 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } - -/* Check if "dir" presents in "dirs_list" */ -static bool -backup_contains_extra(const char *dir, parray *dirs_list) -{ - void *temp = parray_bsearch(dirs_list, dir, BlackListCompare); - return temp != NULL; -} From e753643b8d65ad2656bca485f962de348f279433 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 6 Dec 2018 19:00:08 +0300 Subject: [PATCH 0107/2107] Add error message in case of invalid compression algorithm --- src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/data.c b/src/data.c index 19bb742b6..de6491031 100644 --- a/src/data.c +++ b/src/data.c @@ -98,6 +98,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, { case NONE_COMPRESS: case NOT_DEFINED_COMPRESS: + if (errormsg) + *errormsg = "Invalid compression algorithm"; return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: From 8b1b53bc3bc17d1c4894db1fa618e46e125f5980 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Dec 2018 19:07:56 +0300 Subject: [PATCH 0108/2107] tests: test_merge_compressed_and_uncompressed_backups added --- tests/merge.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 5f7ae7da9..5a314dcf3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,6 +163,87 @@ def test_merge_compressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[1] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "DELTA") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do not compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From 34c05740b05dd1ee89745d9b4678e1220ed6b812 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 19:41:10 +0300 Subject: [PATCH 0109/2107] Restore to_backup sizes after copying metadata from from_backup --- src/merge.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/merge.c b/src/merge.c index 2a24b39dd..6d6d978e2 100644 --- a/src/merge.c +++ b/src/merge.c @@ -168,6 +168,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; + int64 to_data_bytes, + to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -282,26 +284,28 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ to_backup->status = BACKUP_STATUS_OK; /* Compute summary of size of regular files in the backup */ - to_backup->data_bytes = 0; + to_data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_backup->data_bytes += 4096; + to_data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_backup->data_bytes += file->write_size; + to_data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_backup->wal_bytes = instance_config.xlog_seg_size * + to_wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_backup->wal_bytes = BYTES_INVALID; + to_wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -314,6 +318,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Delete files which are not in from_backup file list. */ + parray_qsort(files, pgFileComparePathDesc); for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); @@ -341,6 +346,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; to_backup->parent_backup = INVALID_BACKUP_ID; + /* Restore sizes */ + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); /* Cleanup */ From 59ab21e5c2be7c311b8474a714ff6880c398e790 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 09:43:38 +0300 Subject: [PATCH 0110/2107] tests: test_merge_compressed_backups_1 added --- tests/merge.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 5a314dcf3..cbe84d5c3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,9 +163,9 @@ def test_merge_compressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) - def test_merge_compressed_and_uncompressed_backups(self): + def test_merge_compressed_backups_1(self): """ - Test MERGE command with compressed and uncompressed backups + Test MERGE command with compressed backups """ fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") @@ -203,16 +203,12 @@ def test_merge_compressed_and_uncompressed_backups(self): self.backup_node( backup_dir, "node", node, backup_type="delta", options=['--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[1] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "DELTA") # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) pgbench.wait() - # Do not compressed PAGE backup + # Do compressed PAGE backup self.backup_node( backup_dir, "node", node, backup_type="page", options=['--compress-algorithm=zlib']) @@ -244,6 +240,82 @@ def test_merge_compressed_and_uncompressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From 1b58ec8c5c4ecd9499c787d928272b250bed3828 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 7 Dec 2018 11:07:00 +0300 Subject: [PATCH 0111/2107] Fix variable transfer --- src/utils/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 9357d5197..ea901d47b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -18,7 +18,7 @@ static int fio_stdin = 0; static fio_binding fio_bindings[] = { - {¤t.start_time, sizeof(time_t)} + {¤t.start_time, sizeof(current.start_time)} }; #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -699,7 +699,7 @@ void fio_transfer(fio_shared_variable var) SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &msg, sizeof(fio_header) + var_size), sizeof(fio_header) + var_size); + IO_CHECK(fio_write_all(fio_stdout, msg, sizeof(fio_header) + var_size), sizeof(fio_header) + var_size); SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); free(msg); From f3c62d6714d2670426a9bf341a813e2ba0d5d592 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 7 Dec 2018 11:48:03 +0300 Subject: [PATCH 0112/2107] Fix remote backup validate --- src/backup.c | 3 +++ src/fetch.c | 4 ++-- src/pg_probackup.h | 3 ++- src/util.c | 14 +++++++------- src/utils/file.c | 3 ++- src/utils/file.h | 3 ++- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index c5bcba6a0..69dafbc8e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -999,7 +999,10 @@ do_backup(time_t start_time) // current.data_bytes); if (is_remote_agent) + { fio_transfer(FIO_BACKUP_START_TIME); + fio_transfer(FIO_BACKUP_STOP_LSN); + } else complete_backup(); diff --git a/src/fetch.c b/src/fetch.c index a9661b989..51aea915d 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -25,7 +25,7 @@ * */ char * -slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) +slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fio_location location) { int fd; char *buffer; @@ -34,7 +34,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe) int len; snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); - if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, FIO_DB_HOST)) == -1) + if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) { if (safe) return NULL; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6077cc8f4..0e53e4d96 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -454,7 +454,8 @@ extern int do_delete_instance(void); extern char *slurpFile(const char *datadir, const char *path, size_t *filesize, - bool safe); + bool safe, + fio_location location); extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ diff --git a/src/util.c b/src/util.c index 523d23f93..6ded3a54e 100644 --- a/src/util.c +++ b/src/util.c @@ -154,7 +154,7 @@ get_current_timeline(bool safe) /* First fetch file... */ buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, - safe); + safe, FIO_DB_HOST); if (safe && buffer == NULL) return 0; @@ -212,7 +212,7 @@ get_system_identifier(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -263,7 +263,7 @@ get_xlog_seg_size(char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -284,7 +284,7 @@ get_data_checksum_version(bool safe) /* First fetch file... */ buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, - safe); + safe, FIO_DB_HOST); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -301,7 +301,7 @@ get_pgcontrol_checksum(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false); + buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_BACKUP_HOST); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -324,7 +324,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, char fullpath[MAXPGPATH]; /* First fetch file content */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false); + buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); if (buffer == NULL) elog(ERROR, "ERROR"); @@ -367,7 +367,7 @@ copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file, fi size_t size; char to_path[MAXPGPATH]; - buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false); + buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); digestControlFile(&ControlFile, buffer, size); diff --git a/src/utils/file.c b/src/utils/file.c index ea901d47b..5390da613 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -18,7 +18,8 @@ static int fio_stdin = 0; static fio_binding fio_bindings[] = { - {¤t.start_time, sizeof(current.start_time)} + {¤t.start_time, sizeof(current.start_time)}, + {¤t.stop_lsn, sizeof(current.stop_lsn)} }; #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) diff --git a/src/utils/file.h b/src/utils/file.h index b2a5ac9be..9bdbc5b91 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -38,7 +38,8 @@ typedef enum typedef enum { - FIO_BACKUP_START_TIME + FIO_BACKUP_START_TIME, + FIO_BACKUP_STOP_LSN } fio_shared_variable; #define FIO_FDMAX 64 From 53726536efceb385725ff2f6474098b167b4d643 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Fri, 7 Dec 2018 13:34:57 +0300 Subject: [PATCH 0113/2107] Move extra directories in one folder "extra_directories" --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 55a513616..87a415f8a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -47,7 +47,7 @@ #define PG_BACKUP_LABEL_FILE "backup_label" #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" -#define EXTRA_DIR "extradir" +#define EXTRA_DIR "extra_directories/extradir" /* Direcotry/File permission */ #define DIR_PERMISSION (0700) From 22c760097539d5a8b1bfde1bc14baa9ddc2bd5cf Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 7 Dec 2018 14:27:25 +0300 Subject: [PATCH 0114/2107] Support remote readdir --- src/backup.c | 4 +- src/delete.c | 2 +- src/dir.c | 24 +++++----- src/pg_probackup.h | 2 +- src/restore.c | 2 +- src/utils/file.c | 112 +++++++++++++++++++++++++++++++++++++++++++++ src/utils/file.h | 14 ++++-- 7 files changed, 139 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index 69dafbc8e..4f4a2a5e6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -636,7 +636,7 @@ do_backup_instance(void) get_remote_pgdata_filelist(backup_files_list); else dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false); + true, true, false, FIO_DB_HOST); /* * Sort pathname ascending. It is necessary to create intermediate @@ -813,7 +813,7 @@ do_backup_instance(void) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, FIO_BACKUP_HOST); } for (i = 0; i < parray_num(xlog_files_list); i++) { diff --git a/src/delete.c b/src/delete.c index e7c9ac242..7f8d0779a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -282,7 +282,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); pgBackupGetPath(backup, path, lengthof(path), NULL); - dir_list_file(files, path, false, true, true); + dir_list_file(files, path, false, true, true, FIO_BACKUP_HOST); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); diff --git a/src/dir.c b/src/dir.c index 326ea999a..42e9cb1a1 100644 --- a/src/dir.c +++ b/src/dir.c @@ -123,7 +123,7 @@ static int BlackListCompare(const void *str1, const void *str2); static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list); + bool omit_symlink, parray *black_list, fio_location location); static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); @@ -394,7 +394,7 @@ BlackListCompare(const void *str1, const void *str2) */ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, - bool add_root) + bool add_root, fio_location location) { pgFile *file; parray *black_list = NULL; @@ -403,14 +403,14 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ if (root && instance_config.pgdata && - strcmp(root, instance_config.pgdata) == 0 && fileExists(path, FIO_LOCAL_HOST)) + strcmp(root, instance_config.pgdata) == 0 && fileExists(path, location)) { FILE *black_list_file = NULL; char buf[MAXPGPATH * 2]; char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fopen(path, PG_BINARY_R); + black_list_file = fio_open_stream(path, location); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); @@ -428,7 +428,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(black_list, black_item); } - fclose(black_list_file); + fio_close_stream(black_list_file); parray_qsort(black_list, BlackListCompare); } @@ -444,7 +444,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (add_root) parray_append(files, file); - dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); + dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, location); } /* @@ -632,7 +632,7 @@ dir_check_file(const char *root, pgFile *file) */ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list) + bool exclude, bool omit_symlink, parray *black_list, fio_location location) { DIR *dir; struct dirent *dent; @@ -641,7 +641,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, elog(ERROR, "\"%s\" is not a directory", parent->path); /* Open directory and list contents */ - dir = opendir(parent->path); + dir = fio_opendir(parent->path, location); if (dir == NULL) { if (errno == ENOENT) @@ -654,7 +654,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, } errno = 0; - while ((dent = readdir(dir))) + while ((dent = fio_readdir(dir))) { pgFile *file; char child[MAXPGPATH]; @@ -715,17 +715,17 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, root, file, exclude, omit_symlink, - black_list); + black_list, location); } if (errno && errno != ENOENT) { int errno_tmp = errno; - closedir(dir); + fio_closedir(dir); elog(ERROR, "cannot read directory \"%s\": %s", parent->path, strerror(errno_tmp)); } - closedir(dir); + fio_closedir(dir); } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0e53e4d96..4b3dddf7e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -506,7 +506,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root); + bool omit_symlink, bool add_root, fio_location location); extern void create_data_directories(const char *data_dir, const char *backup_dir, bool extract_tablespaces, diff --git a/src/restore.c b/src/restore.c index 3bea283a8..bb7ce2b33 100644 --- a/src/restore.c +++ b/src/restore.c @@ -528,7 +528,7 @@ remove_deleted_files(pgBackup *backup) /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, instance_config.pgdata, true, true, false); + dir_list_file(files_restored, instance_config.pgdata, true, true, false, FIO_BACKUP_HOST); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); diff --git a/src/utils/file.c b/src/utils/file.c index 5390da613..a5938f173 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -126,6 +126,94 @@ int fio_close_stream(FILE* f) return fclose(f); } +DIR* fio_opendir(char const* path, fio_location location) +{ + DIR* dir; + if (fio_is_remote(location)) + { + int i; + fio_header hdr; + unsigned long mask; + + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + + mask = fio_fdset; + for (i = 0; (mask & 1) != 0; i++, mask >>= 1); + if (i == FIO_FDMAX) { + return NULL; + } + hdr.cop = FIO_OPENDIR; + hdr.handle = i; + hdr.size = strlen(path) + 1; + fio_fdset |= 1 << i; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + dir = (DIR*)(size_t)(i + 1); + } + else + { + dir = opendir(path); + } + return dir; +} + +struct dirent* fio_readdir(DIR *dir) +{ + if (fio_is_remote_file((FILE*)dir)) + { + fio_header hdr; + static struct dirent entry; + + SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); + + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + hdr.cop = FIO_READDIR; + hdr.handle = (size_t)dir - 1; + hdr.size = 0; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SEND); + if (hdr.size) { + Assert(hdr.size == sizeof(entry)); + IO_CHECK(fio_read_all(fio_stdin, &entry, sizeof(entry)), sizeof(entry)); + } + SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); + + return hdr.size ? &entry : NULL; + } + else + { + return readdir(dir); + } +} + +int fio_closedir(DIR *dir) +{ + if (fio_is_remote_file((FILE*)dir)) + { + fio_header hdr; + hdr.cop = FIO_CLOSEDIR; + hdr.handle = (size_t)dir - 1; + hdr.size = 0; + + SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); + return 0; + } + else + { + return closedir(dir); + } +} + + int fio_open(char const* path, int mode, fio_location location) { int fd; @@ -709,6 +797,8 @@ void fio_transfer(fio_shared_variable var) void fio_communicate(int in, int out) { int fd[FIO_FDMAX]; + DIR* dir[FIO_FDMAX]; + struct dirent* entry; size_t buf_size = 128*1024; char* buf = (char*)malloc(buf_size); fio_header hdr; @@ -727,6 +817,28 @@ void fio_communicate(int in, int out) case FIO_LOAD: fio_send_file(out, buf); break; + case FIO_OPENDIR: + dir[hdr.handle] = opendir(buf); + break; + case FIO_READDIR: + entry = readdir(dir[hdr.handle]); + hdr.cop = FIO_SEND; + if (entry != NULL) { + hdr.size = sizeof(*entry); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); + } else { + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + break; + case FIO_CLOSEDIR: + entry = readdir(dir[hdr.handle]); + hdr.cop = FIO_SEND; + hdr.size = 0; + hdr.arg = closedir(dir[hdr.handle]); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; case FIO_OPEN: SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); break; diff --git a/src/utils/file.h b/src/utils/file.h index 9bdbc5b91..978ebc2ad 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -3,6 +3,7 @@ #include #include +#include #ifdef HAVE_LIBZ #include @@ -25,7 +26,10 @@ typedef enum FIO_FSTAT, FIO_SEND, FIO_ACCESS, - FIO_TRANSFER + FIO_TRANSFER, + FIO_OPENDIR, + FIO_READDIR, + FIO_CLOSEDIR } fio_operations; typedef enum @@ -50,8 +54,8 @@ typedef enum typedef struct { - unsigned cop : 4; - unsigned handle : 8; + unsigned cop : 5; + unsigned handle : 7; unsigned size : 20; unsigned arg; } fio_header; @@ -91,7 +95,9 @@ extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); extern int fio_access(char const* path, int mode, fio_location location); extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); - +extern DIR* fio_opendir(char const* path, fio_location location); +extern struct dirent * fio_readdir(DIR *dirp); +extern int fio_closedir(DIR *dirp); extern FILE* fio_open_stream(char const* name, fio_location location); extern int fio_close_stream(FILE* f); From fd8657f1cf47c75fdcf0e81b56acbbcdfd3f1858 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 7 Dec 2018 16:25:13 +0300 Subject: [PATCH 0115/2107] Add agent version check --- src/backup.c | 4 ++-- src/pg_probackup.c | 18 +++++++++++------- src/pg_probackup.h | 6 ++++-- src/utils/file.c | 10 +++------- src/utils/remote.c | 1 + src/walmethods.c | 6 +++--- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4f4a2a5e6..24fd07352 100644 --- a/src/backup.c +++ b/src/backup.c @@ -618,7 +618,7 @@ do_backup_instance(void) elog(ERROR, "Cannot continue backup because stream connect has failed."); } - if (is_remote_agent) + if (remote_agent) xlog_files_list = parray_new(); /* By default there are some error */ @@ -998,7 +998,7 @@ do_backup(time_t start_time) //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", // current.data_bytes); - if (is_remote_agent) + if (remote_agent) { fio_transfer(FIO_BACKUP_START_TIME); fio_transfer(FIO_BACKUP_STOP_LSN); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f17176137..30f7a798e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -20,7 +20,7 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.25"; +const char *PROGRAM_VERSION = "2.0.26"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; @@ -73,7 +73,7 @@ char *remote_path; char *remote_proto = (char*)"ssh"; char *ssh_config; char *ssh_options; -bool is_remote_agent; +char *remote_agent; bool is_remote_backup; /* restore options */ @@ -149,7 +149,7 @@ static ConfigOption cmd_options[] = { 's', 22, "remote-path", &remote_path, SOURCE_CMD_STRICT, }, { 's', 23, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, { 's', 24, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, - { 'b', 25, "agent", &is_remote_agent, SOURCE_CMD_STRICT, }, + { 's', 25, "agent", &remote_agent, SOURCE_CMD_STRICT, }, { 'b', 26, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, /* restore options */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, @@ -318,6 +318,10 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); + if (remote_agent != NULL && strcmp(remote_agent, current.program_version) != 0) + elog(ERROR, "Agent version %s doesn't match master pg_probackup version %s", + remote_agent, current.program_version); + pgut_init(); if (help_opt) @@ -343,7 +347,7 @@ main(int argc, char *argv[]) if (IsSshConnection() && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD)) { - if (is_remote_agent) { + if (remote_agent) { if (backup_subcmd != BACKUP_CMD) { fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; @@ -359,7 +363,7 @@ main(int argc, char *argv[]) } } - if (!is_remote_agent) + if (!remote_agent) { /* Ensure that backup_path is a path to a directory */ rc = stat(backup_path, &stat_buf); @@ -399,7 +403,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && !is_remote_agent) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && !remote_agent) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -506,7 +510,7 @@ main(int argc, char *argv[]) return do_init(); case BACKUP_CMD: current.stream = stream_wal; - if (IsSshConnection() && !is_remote_agent) + if (IsSshConnection() && !remote_agent) { current.status = BACKUP_STATUS_DONE; StrNCpy(current.program_version, PROGRAM_VERSION, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4b3dddf7e..5c4e24a36 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -360,14 +360,16 @@ extern char *replication_slot; /* backup options */ extern bool smooth_checkpoint; + +/* remote probackup options */ extern char *remote_path; extern char *remote_port; extern char *remote_host; extern char *remote_proto; extern char *ssh_config; extern char *ssh_options; -extern bool is_remote_backup; -extern bool is_remote_agent; +extern char* remote_agent; +extern bool is_remote_backup; extern bool is_ptrack_support; extern bool is_checksum_enabled; diff --git a/src/utils/file.c b/src/utils/file.c index a5938f173..7a1daadfa 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -43,8 +43,8 @@ static bool fio_is_remote_fd(int fd) static bool fio_is_remote(fio_location location) { return location == FIO_REMOTE_HOST - || (location == FIO_BACKUP_HOST && is_remote_agent) - || (location == FIO_DB_HOST && !is_remote_agent && IsSshConnection()); + || (location == FIO_BACKUP_HOST && remote_agent) + || (location == FIO_DB_HOST && !remote_agent && IsSshConnection()); } static ssize_t fio_read_all(int fd, void* buf, size_t size) @@ -833,11 +833,7 @@ void fio_communicate(int in, int out) } break; case FIO_CLOSEDIR: - entry = readdir(dir[hdr.handle]); - hdr.cop = FIO_SEND; - hdr.size = 0; - hdr.arg = closedir(dir[hdr.handle]); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + SYS_CHECK(closedir(dir[hdr.handle])); break; case FIO_OPEN: SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); diff --git a/src/utils/remote.c b/src/utils/remote.c index 7f903a790..1b22a8ecd 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -104,6 +104,7 @@ int remote_execute(int argc, char* argv[], bool listen) dst = append_option(cmd, sizeof(cmd), dst, argv[i]); } dst = append_option(cmd, sizeof(cmd), dst, "--agent"); + dst = append_option(cmd, sizeof(cmd), dst, current.program_version); cmd[dst] = '\0'; SYS_CHECK(pipe(infd)); diff --git a/src/walmethods.c b/src/walmethods.c index f627198fb..ceb24f09a 100644 --- a/src/walmethods.c +++ b/src/walmethods.c @@ -156,7 +156,7 @@ dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_ * important when using synchronous mode, where the file is modified and * fsynced in-place, without a directory fsync. */ - if (!is_remote_agent && dir_data->sync) + if (!remote_agent && dir_data->sync) { if (fsync_fname(tmppath, false, progname) != 0 || fsync_parent_path(tmppath, progname) != 0) @@ -272,7 +272,7 @@ dir_close(Walfile f, WalCloseMethod method) * directory if sync mode is requested. */ file_path = df->fullpath; - if (dir_data->sync && !is_remote_agent) + if (dir_data->sync && !remote_agent) { r = fsync_fname(df->fullpath, false, progname); if (r == 0) @@ -344,7 +344,7 @@ dir_existsfile(const char *pathname) static bool dir_finish(void) { - if (dir_data->sync && !is_remote_agent) + if (dir_data->sync && !remote_agent) { /* * Files are fsynced when they are closed, but we need to fsync the From 2837399e6bdb9ccde4a28e68aa3e9757af4dabb5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 7 Dec 2018 17:24:21 +0300 Subject: [PATCH 0116/2107] Fix MERGE of compressed and uncompressed backups --- src/merge.c | 116 ++++++++++++++++++++++--------------------------- tests/merge.py | 2 - 2 files changed, 51 insertions(+), 67 deletions(-) diff --git a/src/merge.c b/src/merge.c index 6d6d978e2..e06695860 100644 --- a/src/merge.c +++ b/src/merge.c @@ -131,7 +131,6 @@ do_merge(time_t backup_id) { pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); - full_backup = (pgBackup *) parray_get(backups, i); merge_backups(full_backup, from_backup); } @@ -168,8 +167,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; - int64 to_data_bytes, - to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -283,29 +280,37 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Update to_backup metadata. */ to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + to_backup->start_lsn = from_backup->start_lsn; + to_backup->stop_lsn = from_backup->stop_lsn; + to_backup->recovery_time = from_backup->recovery_time; + to_backup->recovery_xid = from_backup->recovery_xid; + /* + * If one of the backups isn't "stream" backup then the target backup become + * non-stream backup too. + */ + to_backup->stream = to_backup->stream && from_backup->stream; /* Compute summary of size of regular files in the backup */ - to_data_bytes = 0; + to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_data_bytes += 4096; + to_backup->data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_data_bytes += file->write_size; + to_backup->data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_wal_bytes = instance_config.xlog_seg_size * + to_backup->wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_wal_bytes = BYTES_INVALID; + to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -330,27 +335,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) } } - /* - * Rename FULL backup directory. - */ - elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); - if (rename(to_backup_path, from_backup_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - to_backup_path, from_backup_path, strerror(errno)); - - /* - * Merging finished, now we can safely update ID of the destination backup. - */ - pgBackupCopy(to_backup, from_backup); - /* Correct metadata */ - to_backup->backup_mode = BACKUP_MODE_FULL; - to_backup->status = BACKUP_STATUS_OK; - to_backup->parent_backup = INVALID_BACKUP_ID; - /* Restore sizes */ - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; - write_backup(to_backup); - /* Cleanup */ if (threads) { @@ -384,6 +368,8 @@ merge_files(void *arg) for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); + pgFile *to_file; + pgFile **res_file; if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -392,17 +378,24 @@ merge_files(void *arg) if (interrupted) elog(ERROR, "Interrupted during merging backups"); + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + to_file = (res_file) ? *res_file : NULL; + /* * Skip files which haven't changed since previous backup. But in case * of DELTA backup we should consider n_blocks to truncate the target * backup. */ - if (file->write_size == BYTES_INVALID && - file->n_blocks == -1) + if (file->write_size == BYTES_INVALID && file->n_blocks == -1) { elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", file->path); @@ -411,27 +404,16 @@ merge_files(void *arg) * If the file wasn't changed in PAGE backup, retreive its * write_size from previous FULL backup. */ - if (S_ISREG(file->mode)) + if (to_file) { - pgFile **res_file; - - res_file = parray_bsearch(argument->to_files, file, - pgFileComparePathDesc); - if (res_file && *res_file) - { - file->compress_alg = (*res_file)->compress_alg; - file->write_size = (*res_file)->write_size; - file->crc = (*res_file)->crc; - } + file->compress_alg = to_file->compress_alg; + file->write_size = to_file->write_size; + file->crc = to_file->crc; } continue; } - /* Directories were created before */ - if (S_ISDIR(file->mode)) - continue; - /* * Move the file. We need to decompress it and compress again if * necessary. @@ -447,7 +429,7 @@ merge_files(void *arg) file->path + to_root_len + 1); /* - * We need more complicate algorithm if target file exists and it is + * We need more complicate algorithm if target file should be * compressed. */ if (to_backup->compress_alg == PGLZ_COMPRESS || @@ -462,33 +444,36 @@ merge_files(void *arg) /* * Merge files: - * - decompress first file - * - decompress second file and merge with first decompressed file + * - if target file exists restore and decompress it to the temp + * path + * - decompress source file if necessary and merge it with the + * target decompressed file * - compress result file */ - elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", - tmp_file_path); - - prev_path = file->path; /* - * We need to decompress target file only if it exists. + * We need to decompress target file if it exists. */ - if (fileExists(to_path_tmp)) + if (to_file) { + elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", + tmp_file_path); + /* * file->path points to the file in from_root directory. But we * need the file in directory to_root. */ - file->path = to_path_tmp; - - /* Decompress first/target file */ - restore_data_file(tmp_file_path, file, false, false, + prev_path = to_file->path; + to_file->path = to_path_tmp; + /* Decompress target file into temporary one */ + restore_data_file(tmp_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); - - file->path = prev_path; + to_file->path = prev_path; } - /* Merge second/source file with first/target file */ + else + elog(VERBOSE, "Restore source file into the temporary path \"%s\"", + tmp_file_path); + /* Merge source file with target file */ restore_data_file(tmp_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, false, @@ -497,7 +482,8 @@ merge_files(void *arg) elog(VERBOSE, "Compress file and save it to the directory \"%s\"", argument->to_root); - /* Again we need change path */ + /* Again we need to change path */ + prev_path = file->path; file->path = tmp_file_path; /* backup_data_file() requires file size to calculate nblocks */ file->size = pgFileSize(file->path); diff --git a/tests/merge.py b/tests/merge.py index cbe84d5c3..c3b3169ad 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -875,8 +875,6 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Backup {0} data files are corrupted".format( - backup_id) in e.message and "ERROR: Merging of backup {0} failed".format( backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From b455420a18b1470726c3b44714cbc74800d40df2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 23:39:39 +0300 Subject: [PATCH 0117/2107] tests: more merge tests added --- tests/merge.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index c3b3169ad..e42094412 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -316,6 +316,159 @@ def test_merge_compressed_and_uncompressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups_1(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_and_uncompressed_backups_2(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do uncompressed FULL backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From c1903db22b0812f3b327a8e3a0cba3299413d7d2 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 16:19:51 +0300 Subject: [PATCH 0118/2107] Fix format warnings --- src/archive.c | 4 ++-- src/data.c | 12 ++++++------ src/restore.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/archive.c b/src/archive.c index 3333f0961..774d12c48 100644 --- a/src/archive.c +++ b/src/archive.c @@ -27,7 +27,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; - int64 system_id; + uint64 system_id; bool is_compress = false; if (wal_file_name == NULL && wal_file_path == NULL) @@ -50,7 +50,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if(system_id != instance_config.system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", + "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, wal_file_name, instance_name, instance_config.system_identifier, system_id); diff --git a/src/data.c b/src/data.c index de6491031..0d0c5e5cc 100644 --- a/src/data.c +++ b/src/data.c @@ -232,8 +232,8 @@ read_page_from_file(pgFile *file, BlockNumber blknum, return 0; } else - elog(WARNING, "File: %s, block %u, expected block size %d," - "but read %lu, try again", + elog(WARNING, "File: %s, block %u, expected block size %u," + "but read %zu, try again", file->path, blknum, BLCKSZ, read_len); } @@ -382,7 +382,7 @@ prepare_page(backup_files_arg *arguments, else if (page_size != BLCKSZ) { free(ptrack_page); - elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + elog(ERROR, "File: %s, block %u, expected block size %d, but read %zu", file->path, absolute_blknum, BLCKSZ, page_size); } else @@ -574,7 +574,7 @@ backup_data_file(backup_files_arg* arguments, if (file->size % BLCKSZ != 0) { fclose(in); - elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); } /* @@ -789,7 +789,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); /* @@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); diff --git a/src/restore.c b/src/restore.c index 8770c3586..e3591e067 100644 --- a/src/restore.c +++ b/src/restore.c @@ -480,7 +480,7 @@ restore_backup(pgBackup *backup) /* By default there are some error */ threads_args[i].ret = 1; - elog(LOG, "Start thread for num:%li", parray_num(files)); + elog(LOG, "Start thread for num:%zu", parray_num(files)); pthread_create(&threads[i], NULL, restore_files, arg); } From dd18928e1984927edafe175a2a1bb30f9f64d66e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 18:02:22 +0300 Subject: [PATCH 0119/2107] Make compiler happy --- src/utils/logger.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index fa1c0039b..a4f2d6928 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -232,18 +232,23 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr) + if (write_to_stderr && write_to_file) { write_elevel(stderr, elevel); - if (write_to_file) - vfprintf(stderr, fmt, std_args); - else - vfprintf(stderr, fmt, args); + + vfprintf(stderr, fmt, std_args); fputc('\n', stderr); fflush(stderr); - if (write_to_file) - va_end(std_args); + va_end(std_args); + } + else if (write_to_stderr) + { + write_elevel(stderr, elevel); + + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); } exit_if_necessary(elevel); From 9ac30ada5443e97de9152de7bab5c3b9752b2abb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Dec 2018 18:49:39 +0300 Subject: [PATCH 0120/2107] bugfix: correctly log cmdline to logfile --- src/pg_probackup.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 176768766..04b5d4862 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -331,15 +331,6 @@ main(int argc, char *argv[]) if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) elog(ERROR, "-B, --backup-path must be a path to directory"); - /* command was initialized for a few commands */ - if (command) - { - elog_file(INFO, "command: %s", command); - - pfree(command); - command = NULL; - } - /* Option --instance is required for all commands except init and show */ if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) @@ -390,6 +381,15 @@ main(int argc, char *argv[]) /* Initialize logger */ init_logger(backup_path, &instance_config.logger); + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + /* * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. From eead6b6307b464b67e567516609baac7fce7560c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 19:00:37 +0300 Subject: [PATCH 0121/2107] Free unnecessary file object --- src/dir.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dir.c b/src/dir.c index 710598b7a..ae8cb20bc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -443,6 +443,9 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); + + if (!add_root) + pgFileFree(file); } /* From e97fd446434d49f5d63ed0eb7b4eb8447a7ec90d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 10 Dec 2018 19:40:36 +0300 Subject: [PATCH 0122/2107] Fix passing prgram version to agent --- src/pg_probackup.c | 1 - src/pg_probackup.h | 1 + src/utils/remote.c | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 30f7a798e..b00e3e623 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -20,7 +20,6 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.26"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5c4e24a36..dcb8f7ce3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -167,6 +167,7 @@ typedef enum ShowFormat #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) #define BLOCKNUM_INVALID (-1) +#define PROGRAM_VERSION "2.0.26" /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/src/utils/remote.c b/src/utils/remote.c index 1b22a8ecd..2cab0e3a2 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -104,7 +104,7 @@ int remote_execute(int argc, char* argv[], bool listen) dst = append_option(cmd, sizeof(cmd), dst, argv[i]); } dst = append_option(cmd, sizeof(cmd), dst, "--agent"); - dst = append_option(cmd, sizeof(cmd), dst, current.program_version); + dst = append_option(cmd, sizeof(cmd), dst, PROGRAM_VERSION); cmd[dst] = '\0'; SYS_CHECK(pipe(infd)); From 8a635895646a74fb19df4aac1e00eaf5b5164bee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 11 Dec 2018 12:02:38 +0300 Subject: [PATCH 0123/2107] tests: checkdb module added --- tests/__init__.py | 4 +++- tests/helpers/ptrack_helpers.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 532e72bff..5a2820715 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,8 @@ ptrack_vacuum, ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ - exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test + exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ + checkdb def load_tests(loader, tests, pattern): @@ -16,6 +17,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup_test)) suite.addTests(loader.loadTestsFromModule(compatibility)) + suite.addTests(loader.loadTestsFromModule(checkdb)) # suite.addTests(loader.loadTestsFromModule(cfs_backup)) # suite.addTests(loader.loadTestsFromModule(cfs_restore)) # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d81d72888..055ea1f2b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -667,6 +667,23 @@ def backup_node( return self.run_pb(cmd_list + options, async, gdb, old_binary) + def checkdb_node( + self, instance, backup_dir=False, data_dir=False, + options=[], async=False, gdb=False, old_binary=False + ): + + cmd_list = ["checkdb"] + + if backup_dir: + cmd_list += ["-B", backup_dir] + + if data_dir: + cmd_list += ["-D", data_dir] + + cmd_list += ["--instance={0}".format(instance)] + + return self.run_pb(cmd_list + options, async, gdb, old_binary) + def merge_backup( self, backup_dir, instance, backup_id, async=False, gdb=False, old_binary=False, options=[]): From ccccb1a743f1965c337bb25f5fdf7d8e44c3708f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 11 Dec 2018 12:03:46 +0300 Subject: [PATCH 0124/2107] tests: checkdb module added --- tests/checkdb.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/checkdb.py diff --git a/tests/checkdb.py b/tests/checkdb.py new file mode 100644 index 000000000..ad154e77a --- /dev/null +++ b/tests/checkdb.py @@ -0,0 +1,62 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +module_name = 'checkdb' + + +class CheckdbTest(ProbackupTest, unittest.TestCase): + + def test_checkdb_index_loss(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + } + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "create index on t_heap(id)" + ) + + gdb = self.checkdb_node( + 'node', data_dir=node.data_dir, + gdb=True, options=['-d', 'postgres', '--amcheck', '-p', str(node.port)]) + + gdb.set_breakpoint('amcheck_one_index') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + + gdb.continue_execution_until_exit() + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From 385f979fe7daf91001eea4f4c123807bb6f576ee Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:27:36 +0300 Subject: [PATCH 0125/2107] Make compiler happy again --- src/utils/logger.c | 98 ++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index a4f2d6928..ba054a62c 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -40,10 +40,10 @@ typedef enum void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); -static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) - pg_attribute_printf(3, 0); +static void elog_internal(int elevel, bool file_only, const char *message); static void elog_stderr(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf(1, 0); /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); @@ -74,7 +74,7 @@ init_logger(const char *root_path, LoggerConfig *config) /* Set log path */ if (config->log_directory == NULL) { - config->log_directory = palloc(MAXPGPATH); + config->log_directory = pgut_malloc(MAXPGPATH); join_path_components(config->log_directory, root_path, LOG_DIRECTORY_DEFAULT); } @@ -148,13 +148,11 @@ exit_if_necessary(int elevel) * Actual implementation for elog() and pg_log(). */ static void -elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +elog_internal(int elevel, bool file_only, const char *message) { bool write_to_file, write_to_error_log, write_to_stderr; - va_list error_args, - std_args; time_t log_time = (time_t) time(NULL); char strfbuf[128]; @@ -165,22 +163,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) write_to_stderr = elevel >= logger_config.log_level_console && !file_only; pthread_lock(&log_file_mutex); -#ifdef WIN32 - std_args = NULL; - error_args = NULL; -#endif loggin_in_progress = true; - /* We need copy args only if we need write to error log file */ - if (write_to_error_log) - va_copy(error_args, args); - /* - * We need copy args only if we need write to stderr. But do not copy args - * if we need to log only to stderr. - */ - if (write_to_stderr && write_to_file) - va_copy(std_args, args); - if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); @@ -203,8 +187,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(log_file, "%s: ", strfbuf); write_elevel(log_file, elevel); - vfprintf(log_file, fmt, args); - fputc('\n', log_file); + fprintf(log_file, "%s\n", message); fflush(log_file); } @@ -221,33 +204,19 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - vfprintf(error_log_file, fmt, error_args); - fputc('\n', error_log_file); + fprintf(error_log_file, "%s\n", message); fflush(error_log_file); - - va_end(error_args); } /* * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr && write_to_file) + if (write_to_stderr) { write_elevel(stderr, elevel); - vfprintf(stderr, fmt, std_args); - fputc('\n', stderr); - fflush(stderr); - - va_end(std_args); - } - else if (write_to_stderr) - { - write_elevel(stderr, elevel); - - vfprintf(stderr, fmt, args); - fputc('\n', stderr); + fprintf(stderr, "%s\n", message); fflush(stderr); } @@ -285,12 +254,44 @@ elog_stderr(int elevel, const char *fmt, ...) exit_if_necessary(elevel); } +/* + * Formats text data under the control of fmt and returns it in an allocated + * buffer. + */ +static char * +get_log_message(const char *fmt, va_list args) +{ + size_t len = 256; /* initial assumption about buffer size */ + + for (;;) + { + char *result; + size_t newlen; + va_list copy_args; + + result = (char *) pgut_malloc(len); + + /* Try to format the data */ + va_copy(copy_args, args); + newlen = pvsnprintf(result, len, fmt, copy_args); + va_end(copy_args); + + if (newlen < len) + return result; /* success */ + + /* Release buffer and loop around to try again with larger len. */ + pfree(result); + len = newlen; + } +} + /* * Logs to stderr or to log file and exit if ERROR. */ void elog(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -302,8 +303,11 @@ elog(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -312,6 +316,7 @@ elog(int elevel, const char *fmt, ...) void elog_file(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -322,8 +327,11 @@ elog_file(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, true, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, true, message); + pfree(message); } /* @@ -332,6 +340,7 @@ elog_file(int elevel, const char *fmt, ...) void pg_log(eLogType type, const char *fmt, ...) { + char *message; va_list args; int elevel = INFO; @@ -364,8 +373,11 @@ pg_log(eLogType type, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -450,7 +462,7 @@ logfile_getname(const char *format, time_t timestamp) logger_config.log_directory[0] == '\0') elog_stderr(ERROR, "logging path is not set"); - filename = (char *) palloc(MAXPGPATH); + filename = (char *) pgut_malloc(MAXPGPATH); snprintf(filename, MAXPGPATH, "%s/", logger_config.log_directory); From eed59874069c05026ae76c04355ead8f9795d9d9 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:47:21 +0300 Subject: [PATCH 0126/2107] Remove BOM marker from configuration.c --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 9b88d1594..fe50c4946 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1,4 +1,4 @@ -/*------------------------------------------------------------------------- +/*------------------------------------------------------------------------- * * configuration.c: - function implementations to work with pg_probackup * configurations. From 2247716b6831c6939bb0c1156f34dc9ac1a90ec9 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 11 Dec 2018 14:27:10 +0300 Subject: [PATCH 0127/2107] Fix agent version check --- src/pg_probackup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b00e3e623..91c0c5539 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -317,9 +317,9 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); - if (remote_agent != NULL && strcmp(remote_agent, current.program_version) != 0) + if (remote_agent != NULL && strcmp(remote_agent, PROGRAM_VERSION) != 0) elog(ERROR, "Agent version %s doesn't match master pg_probackup version %s", - remote_agent, current.program_version); + remote_agent, PROGRAM_VERSION); pgut_init(); From 4acced609d9e7fd1a2103b27f0b515b32462aa93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:38:45 +0300 Subject: [PATCH 0128/2107] add warning if waiting for START LSN during ARCHIVE backup is exceeding 30 seconds --- src/backup.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/backup.c b/src/backup.c index 845edbcd6..b787c4f92 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1816,6 +1816,11 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); } + if (!stream_wal && is_start_lsn && try_count == 30) + elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); + if (timeout > 0 && try_count > timeout) { if (file_exists) From 7aa81561b2ef8727b27d5cb59aa09fb6b604ccc4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 3 Dec 2018 13:41:39 +0300 Subject: [PATCH 0129/2107] fix tabulation offset --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index b787c4f92..988f26ec2 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1818,8 +1818,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (!stream_wal && is_start_lsn && try_count == 30) elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " - "If continius archiving is not set up, use '--stream' option to make autonomous backup. " - "Otherwise check that continius archiving works correctly."); + "If continius archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continius archiving works correctly."); if (timeout > 0 && try_count > timeout) { From 901062188d44dfb44aec3632059a4ca570396574 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 16:03:29 +0300 Subject: [PATCH 0130/2107] Show progress messages with INFO verbose level --- src/delete.c | 10 ++++++---- src/merge.c | 2 +- src/restore.c | 2 +- src/validate.c | 9 +++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/delete.c b/src/delete.c index e7c9ac242..8e9b965fc 100644 --- a/src/delete.c +++ b/src/delete.c @@ -256,6 +256,7 @@ delete_backup_files(pgBackup *backup) char path[MAXPGPATH]; char timestamp[100]; parray *files; + size_t num_files; /* * If the backup was deleted already, there is nothing to do. @@ -286,13 +287,14 @@ delete_backup_files(pgBackup *backup) /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); - for (i = 0; i < parray_num(files); i++) + num_files = parray_num(files); + for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(files, i); - /* print progress */ - elog(VERBOSE, "Delete file(%zd/%lu) \"%s\"", i + 1, - (unsigned long) parray_num(files), file->path); + if (progress) + elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", + i + 1, num_files, file->path); if (remove(file->path)) { diff --git a/src/merge.c b/src/merge.c index b01a90d00..2a24b39dd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -385,7 +385,7 @@ merge_files(void *arg) elog(ERROR, "Interrupted during merging backups"); if (progress) - elog(LOG, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); /* diff --git a/src/restore.c b/src/restore.c index 6024f8bcf..8770c3586 100644 --- a/src/restore.c +++ b/src/restore.c @@ -582,7 +582,7 @@ restore_files(void *arg) rel_path = GetRelativePath(file->path,from_root); if (progress) - elog(LOG, "Progress: (%d/%lu). Process file %s ", + elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), rel_path); /* diff --git a/src/validate.c b/src/validate.c index 1f758d0f0..edb386ef9 100644 --- a/src/validate.c +++ b/src/validate.c @@ -156,9 +156,10 @@ pgBackupValidateFiles(void *arg) { int i; validate_files_arg *arguments = (validate_files_arg *)arg; + int num_files = parray_num(arguments->files); pg_crc32 crc; - for (i = 0; i < parray_num(arguments->files); i++) + for (i = 0; i < num_files; i++) { struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); @@ -186,9 +187,9 @@ pgBackupValidateFiles(void *arg) if (file->is_cfs) continue; - /* print progress */ - elog(VERBOSE, "Validate files: (%d/%lu) %s", - i + 1, (unsigned long) parray_num(arguments->files), file->path); + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, num_files, file->path); if (stat(file->path, &st) == -1) { From bda055cf7003635e5950be515955f219722ef866 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 6 Dec 2018 19:00:08 +0300 Subject: [PATCH 0131/2107] Add error message in case of invalid compression algorithm --- src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/data.c b/src/data.c index 65ff6156e..138d64e34 100644 --- a/src/data.c +++ b/src/data.c @@ -98,6 +98,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, { case NONE_COMPRESS: case NOT_DEFINED_COMPRESS: + if (errormsg) + *errormsg = "Invalid compression algorithm"; return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: From ccc0ccf35642a275b29b726fd1d5a160f6e05cf2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Dec 2018 19:07:56 +0300 Subject: [PATCH 0132/2107] tests: test_merge_compressed_and_uncompressed_backups added --- tests/merge.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 5f7ae7da9..5a314dcf3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,6 +163,87 @@ def test_merge_compressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[1] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "DELTA") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do not compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From 737c97d708fe1dc91292bb6ff5cdb05fdbe6c63b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 6 Dec 2018 19:41:10 +0300 Subject: [PATCH 0133/2107] Restore to_backup sizes after copying metadata from from_backup --- src/merge.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/merge.c b/src/merge.c index 2a24b39dd..6d6d978e2 100644 --- a/src/merge.c +++ b/src/merge.c @@ -168,6 +168,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; + int64 to_data_bytes, + to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -282,26 +284,28 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ to_backup->status = BACKUP_STATUS_OK; /* Compute summary of size of regular files in the backup */ - to_backup->data_bytes = 0; + to_data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_backup->data_bytes += 4096; + to_data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_backup->data_bytes += file->write_size; + to_data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_backup->wal_bytes = instance_config.xlog_seg_size * + to_wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_backup->wal_bytes = BYTES_INVALID; + to_wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -314,6 +318,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Delete files which are not in from_backup file list. */ + parray_qsort(files, pgFileComparePathDesc); for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); @@ -341,6 +346,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->backup_mode = BACKUP_MODE_FULL; to_backup->status = BACKUP_STATUS_OK; to_backup->parent_backup = INVALID_BACKUP_ID; + /* Restore sizes */ + to_backup->data_bytes = to_data_bytes; + to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); /* Cleanup */ From b36c9c7180bc611d2e1ced5c1c848a607cd530be Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 09:43:38 +0300 Subject: [PATCH 0134/2107] tests: test_merge_compressed_backups_1 added --- tests/merge.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 5a314dcf3..cbe84d5c3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -163,9 +163,9 @@ def test_merge_compressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) - def test_merge_compressed_and_uncompressed_backups(self): + def test_merge_compressed_backups_1(self): """ - Test MERGE command with compressed and uncompressed backups + Test MERGE command with compressed backups """ fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") @@ -203,16 +203,12 @@ def test_merge_compressed_and_uncompressed_backups(self): self.backup_node( backup_dir, "node", node, backup_type="delta", options=['--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[1] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "DELTA") # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) pgbench.wait() - # Do not compressed PAGE backup + # Do compressed PAGE backup self.backup_node( backup_dir, "node", node, backup_type="page", options=['--compress-algorithm=zlib']) @@ -244,6 +240,82 @@ def test_merge_compressed_and_uncompressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From affc01012fd62682fe9cf2d685e6c50da2b106b0 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 7 Dec 2018 17:24:21 +0300 Subject: [PATCH 0135/2107] Fix MERGE of compressed and uncompressed backups --- src/merge.c | 116 ++++++++++++++++++++++--------------------------- tests/merge.py | 2 - 2 files changed, 51 insertions(+), 67 deletions(-) diff --git a/src/merge.c b/src/merge.c index 6d6d978e2..e06695860 100644 --- a/src/merge.c +++ b/src/merge.c @@ -131,7 +131,6 @@ do_merge(time_t backup_id) { pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); - full_backup = (pgBackup *) parray_get(backups, i); merge_backups(full_backup, from_backup); } @@ -168,8 +167,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_files_arg *threads_args = NULL; int i; bool merge_isok = true; - int64 to_data_bytes, - to_wal_bytes; elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); @@ -283,29 +280,37 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Update to_backup metadata. */ to_backup->status = BACKUP_STATUS_OK; + to_backup->parent_backup = INVALID_BACKUP_ID; + to_backup->start_lsn = from_backup->start_lsn; + to_backup->stop_lsn = from_backup->stop_lsn; + to_backup->recovery_time = from_backup->recovery_time; + to_backup->recovery_xid = from_backup->recovery_xid; + /* + * If one of the backups isn't "stream" backup then the target backup become + * non-stream backup too. + */ + to_backup->stream = to_backup->stream && from_backup->stream; /* Compute summary of size of regular files in the backup */ - to_data_bytes = 0; + to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) - to_data_bytes += 4096; + to_backup->data_bytes += 4096; /* Count the amount of the data actually copied */ else if (S_ISREG(file->mode)) - to_data_bytes += file->write_size; + to_backup->data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ if (!to_backup->stream) - to_wal_bytes = instance_config.xlog_seg_size * + to_backup->wal_bytes = instance_config.xlog_seg_size * (to_backup->stop_lsn / instance_config.xlog_seg_size - to_backup->start_lsn / instance_config.xlog_seg_size + 1); else - to_wal_bytes = BYTES_INVALID; + to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path); - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; write_backup(to_backup); delete_source_backup: @@ -330,27 +335,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) } } - /* - * Rename FULL backup directory. - */ - elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); - if (rename(to_backup_path, from_backup_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - to_backup_path, from_backup_path, strerror(errno)); - - /* - * Merging finished, now we can safely update ID of the destination backup. - */ - pgBackupCopy(to_backup, from_backup); - /* Correct metadata */ - to_backup->backup_mode = BACKUP_MODE_FULL; - to_backup->status = BACKUP_STATUS_OK; - to_backup->parent_backup = INVALID_BACKUP_ID; - /* Restore sizes */ - to_backup->data_bytes = to_data_bytes; - to_backup->wal_bytes = to_wal_bytes; - write_backup(to_backup); - /* Cleanup */ if (threads) { @@ -384,6 +368,8 @@ merge_files(void *arg) for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); + pgFile *to_file; + pgFile **res_file; if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -392,17 +378,24 @@ merge_files(void *arg) if (interrupted) elog(ERROR, "Interrupted during merging backups"); + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); + res_file = parray_bsearch(argument->to_files, file, + pgFileComparePathDesc); + to_file = (res_file) ? *res_file : NULL; + /* * Skip files which haven't changed since previous backup. But in case * of DELTA backup we should consider n_blocks to truncate the target * backup. */ - if (file->write_size == BYTES_INVALID && - file->n_blocks == -1) + if (file->write_size == BYTES_INVALID && file->n_blocks == -1) { elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", file->path); @@ -411,27 +404,16 @@ merge_files(void *arg) * If the file wasn't changed in PAGE backup, retreive its * write_size from previous FULL backup. */ - if (S_ISREG(file->mode)) + if (to_file) { - pgFile **res_file; - - res_file = parray_bsearch(argument->to_files, file, - pgFileComparePathDesc); - if (res_file && *res_file) - { - file->compress_alg = (*res_file)->compress_alg; - file->write_size = (*res_file)->write_size; - file->crc = (*res_file)->crc; - } + file->compress_alg = to_file->compress_alg; + file->write_size = to_file->write_size; + file->crc = to_file->crc; } continue; } - /* Directories were created before */ - if (S_ISDIR(file->mode)) - continue; - /* * Move the file. We need to decompress it and compress again if * necessary. @@ -447,7 +429,7 @@ merge_files(void *arg) file->path + to_root_len + 1); /* - * We need more complicate algorithm if target file exists and it is + * We need more complicate algorithm if target file should be * compressed. */ if (to_backup->compress_alg == PGLZ_COMPRESS || @@ -462,33 +444,36 @@ merge_files(void *arg) /* * Merge files: - * - decompress first file - * - decompress second file and merge with first decompressed file + * - if target file exists restore and decompress it to the temp + * path + * - decompress source file if necessary and merge it with the + * target decompressed file * - compress result file */ - elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"", - tmp_file_path); - - prev_path = file->path; /* - * We need to decompress target file only if it exists. + * We need to decompress target file if it exists. */ - if (fileExists(to_path_tmp)) + if (to_file) { + elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", + tmp_file_path); + /* * file->path points to the file in from_root directory. But we * need the file in directory to_root. */ - file->path = to_path_tmp; - - /* Decompress first/target file */ - restore_data_file(tmp_file_path, file, false, false, + prev_path = to_file->path; + to_file->path = to_path_tmp; + /* Decompress target file into temporary one */ + restore_data_file(tmp_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); - - file->path = prev_path; + to_file->path = prev_path; } - /* Merge second/source file with first/target file */ + else + elog(VERBOSE, "Restore source file into the temporary path \"%s\"", + tmp_file_path); + /* Merge source file with target file */ restore_data_file(tmp_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, false, @@ -497,7 +482,8 @@ merge_files(void *arg) elog(VERBOSE, "Compress file and save it to the directory \"%s\"", argument->to_root); - /* Again we need change path */ + /* Again we need to change path */ + prev_path = file->path; file->path = tmp_file_path; /* backup_data_file() requires file size to calculate nblocks */ file->size = pgFileSize(file->path); diff --git a/tests/merge.py b/tests/merge.py index cbe84d5c3..c3b3169ad 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -875,8 +875,6 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Backup {0} data files are corrupted".format( - backup_id) in e.message and "ERROR: Merging of backup {0} failed".format( backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From 32e0d5b3e5a9f4d76ac78d53e0bcc7dfcc4e304c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Dec 2018 23:39:39 +0300 Subject: [PATCH 0136/2107] tests: more merge tests added --- tests/merge.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index c3b3169ad..e42094412 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -316,6 +316,159 @@ def test_merge_compressed_and_uncompressed_backups(self): node.cleanup() self.del_test_dir(module_name, fname) + def test_merge_compressed_and_uncompressed_backups_1(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + def test_merge_compressed_and_uncompressed_backups_2(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=["--data-checksums"], + pg_options={ + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do uncompressed FULL backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") def test_merge_tablespaces(self): """ From cdb857ca5a4bee0c656a004698e48602e9cee464 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 16:19:51 +0300 Subject: [PATCH 0137/2107] Fix format warnings --- src/archive.c | 4 ++-- src/data.c | 12 ++++++------ src/restore.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/archive.c b/src/archive.c index 3333f0961..774d12c48 100644 --- a/src/archive.c +++ b/src/archive.c @@ -27,7 +27,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; - int64 system_id; + uint64 system_id; bool is_compress = false; if (wal_file_name == NULL && wal_file_path == NULL) @@ -50,7 +50,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if(system_id != instance_config.system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = %ld instead of %ld", + "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, wal_file_name, instance_name, instance_config.system_identifier, system_id); diff --git a/src/data.c b/src/data.c index 138d64e34..8b2a8ac45 100644 --- a/src/data.c +++ b/src/data.c @@ -232,8 +232,8 @@ read_page_from_file(pgFile *file, BlockNumber blknum, return 0; } else - elog(WARNING, "File: %s, block %u, expected block size %d," - "but read %lu, try again", + elog(WARNING, "File: %s, block %u, expected block size %u," + "but read %zu, try again", file->path, blknum, BLCKSZ, read_len); } @@ -382,7 +382,7 @@ prepare_page(backup_files_arg *arguments, else if (page_size != BLCKSZ) { free(ptrack_page); - elog(ERROR, "File: %s, block %u, expected block size %d, but read %lu", + elog(ERROR, "File: %s, block %u, expected block size %d, but read %zu", file->path, absolute_blknum, BLCKSZ, page_size); } else @@ -543,7 +543,7 @@ check_data_file(backup_files_arg* arguments, if (file->size % BLCKSZ != 0) { fclose(in); - elog(ERROR, "File: %s, invalid file size %lu", file->path, file->size); + elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); } /* @@ -850,7 +850,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); /* @@ -1681,7 +1681,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", + elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); diff --git a/src/restore.c b/src/restore.c index 8770c3586..e3591e067 100644 --- a/src/restore.c +++ b/src/restore.c @@ -480,7 +480,7 @@ restore_backup(pgBackup *backup) /* By default there are some error */ threads_args[i].ret = 1; - elog(LOG, "Start thread for num:%li", parray_num(files)); + elog(LOG, "Start thread for num:%zu", parray_num(files)); pthread_create(&threads[i], NULL, restore_files, arg); } From 8a36f53aa7e1dd7efcfd56de5484614599d19a63 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 18:02:22 +0300 Subject: [PATCH 0138/2107] Make compiler happy --- src/utils/logger.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index fa1c0039b..a4f2d6928 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -232,18 +232,23 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr) + if (write_to_stderr && write_to_file) { write_elevel(stderr, elevel); - if (write_to_file) - vfprintf(stderr, fmt, std_args); - else - vfprintf(stderr, fmt, args); + + vfprintf(stderr, fmt, std_args); fputc('\n', stderr); fflush(stderr); - if (write_to_file) - va_end(std_args); + va_end(std_args); + } + else if (write_to_stderr) + { + write_elevel(stderr, elevel); + + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + fflush(stderr); } exit_if_necessary(elevel); From 3d104a19801dfe8c48cea8c312da80218cc3b6ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Dec 2018 18:49:39 +0300 Subject: [PATCH 0139/2107] bugfix: correctly log cmdline to logfile --- src/pg_probackup.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a59e25ccd..177635fcc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -343,16 +343,6 @@ main(int argc, char *argv[]) elog(ERROR, "-B, --backup-path must be a path to directory"); } - /* TODO What is this block about?*/ - /* command was initialized for a few commands */ - if (command) - { - elog_file(INFO, "command: %s", command); - - pfree(command); - command = NULL; - } - /* TODO it would be better to list commands that require instance option */ /* Option --instance is required for all commands except init and show */ if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && @@ -404,11 +394,20 @@ main(int argc, char *argv[]) /* Just read environment variables */ if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) config_get_opt_env(instance_options); - + /* Initialize logger */ if (backup_subcmd != CHECKDB_CMD) init_logger(backup_path, &instance_config.logger); + /* command was initialized for a few commands */ + if (command) + { + elog_file(INFO, "command: %s", command); + + pfree(command); + command = NULL; + } + /* * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. From 5a6dcd9d1342d77f307bbc7e7868244c60b68939 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 10 Dec 2018 19:00:37 +0300 Subject: [PATCH 0140/2107] Free unnecessary file object --- src/dir.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dir.c b/src/dir.c index 710598b7a..ae8cb20bc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -443,6 +443,9 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list); + + if (!add_root) + pgFileFree(file); } /* From af239aa498039071f73490e053df5b7988df4fcf Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:27:36 +0300 Subject: [PATCH 0141/2107] Make compiler happy again --- src/utils/logger.c | 98 ++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index a4f2d6928..ba054a62c 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -40,10 +40,10 @@ typedef enum void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); -static void elog_internal(int elevel, bool file_only, const char *fmt, va_list args) - pg_attribute_printf(3, 0); +static void elog_internal(int elevel, bool file_only, const char *message); static void elog_stderr(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); +static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf(1, 0); /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); @@ -74,7 +74,7 @@ init_logger(const char *root_path, LoggerConfig *config) /* Set log path */ if (config->log_directory == NULL) { - config->log_directory = palloc(MAXPGPATH); + config->log_directory = pgut_malloc(MAXPGPATH); join_path_components(config->log_directory, root_path, LOG_DIRECTORY_DEFAULT); } @@ -148,13 +148,11 @@ exit_if_necessary(int elevel) * Actual implementation for elog() and pg_log(). */ static void -elog_internal(int elevel, bool file_only, const char *fmt, va_list args) +elog_internal(int elevel, bool file_only, const char *message) { bool write_to_file, write_to_error_log, write_to_stderr; - va_list error_args, - std_args; time_t log_time = (time_t) time(NULL); char strfbuf[128]; @@ -165,22 +163,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) write_to_stderr = elevel >= logger_config.log_level_console && !file_only; pthread_lock(&log_file_mutex); -#ifdef WIN32 - std_args = NULL; - error_args = NULL; -#endif loggin_in_progress = true; - /* We need copy args only if we need write to error log file */ - if (write_to_error_log) - va_copy(error_args, args); - /* - * We need copy args only if we need write to stderr. But do not copy args - * if we need to log only to stderr. - */ - if (write_to_stderr && write_to_file) - va_copy(std_args, args); - if (write_to_file || write_to_error_log) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); @@ -203,8 +187,7 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(log_file, "%s: ", strfbuf); write_elevel(log_file, elevel); - vfprintf(log_file, fmt, args); - fputc('\n', log_file); + fprintf(log_file, "%s\n", message); fflush(log_file); } @@ -221,33 +204,19 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args) fprintf(error_log_file, "%s: ", strfbuf); write_elevel(error_log_file, elevel); - vfprintf(error_log_file, fmt, error_args); - fputc('\n', error_log_file); + fprintf(error_log_file, "%s\n", message); fflush(error_log_file); - - va_end(error_args); } /* * Write to stderr if the message was not written to log file. * Write to stderr if the message level is greater than WARNING anyway. */ - if (write_to_stderr && write_to_file) + if (write_to_stderr) { write_elevel(stderr, elevel); - vfprintf(stderr, fmt, std_args); - fputc('\n', stderr); - fflush(stderr); - - va_end(std_args); - } - else if (write_to_stderr) - { - write_elevel(stderr, elevel); - - vfprintf(stderr, fmt, args); - fputc('\n', stderr); + fprintf(stderr, "%s\n", message); fflush(stderr); } @@ -285,12 +254,44 @@ elog_stderr(int elevel, const char *fmt, ...) exit_if_necessary(elevel); } +/* + * Formats text data under the control of fmt and returns it in an allocated + * buffer. + */ +static char * +get_log_message(const char *fmt, va_list args) +{ + size_t len = 256; /* initial assumption about buffer size */ + + for (;;) + { + char *result; + size_t newlen; + va_list copy_args; + + result = (char *) pgut_malloc(len); + + /* Try to format the data */ + va_copy(copy_args, args); + newlen = pvsnprintf(result, len, fmt, copy_args); + va_end(copy_args); + + if (newlen < len) + return result; /* success */ + + /* Release buffer and loop around to try again with larger len. */ + pfree(result); + len = newlen; + } +} + /* * Logs to stderr or to log file and exit if ERROR. */ void elog(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -302,8 +303,11 @@ elog(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -312,6 +316,7 @@ elog(int elevel, const char *fmt, ...) void elog_file(int elevel, const char *fmt, ...) { + char *message; va_list args; /* @@ -322,8 +327,11 @@ elog_file(int elevel, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, true, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, true, message); + pfree(message); } /* @@ -332,6 +340,7 @@ elog_file(int elevel, const char *fmt, ...) void pg_log(eLogType type, const char *fmt, ...) { + char *message; va_list args; int elevel = INFO; @@ -364,8 +373,11 @@ pg_log(eLogType type, const char *fmt, ...) return; va_start(args, fmt); - elog_internal(elevel, false, fmt, args); + message = get_log_message(fmt, args); va_end(args); + + elog_internal(elevel, false, message); + pfree(message); } /* @@ -450,7 +462,7 @@ logfile_getname(const char *format, time_t timestamp) logger_config.log_directory[0] == '\0') elog_stderr(ERROR, "logging path is not set"); - filename = (char *) palloc(MAXPGPATH); + filename = (char *) pgut_malloc(MAXPGPATH); snprintf(filename, MAXPGPATH, "%s/", logger_config.log_directory); From 350422afe8169f5229e8673ab6a8bce355dd100f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 11 Dec 2018 12:47:21 +0300 Subject: [PATCH 0142/2107] Remove BOM marker from configuration.c --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 9b88d1594..fe50c4946 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1,4 +1,4 @@ -/*------------------------------------------------------------------------- +/*------------------------------------------------------------------------- * * configuration.c: - function implementations to work with pg_probackup * configurations. From 5f404b5de472b7e6c3966d24fbd32dd531c84f7f Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Dec 2018 16:55:35 +0300 Subject: [PATCH 0143/2107] pgpro-2065. fix logging for checkdb command --- src/pg_probackup.c | 10 ++++++++-- src/utils/logger.c | 14 +++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 177635fcc..b6a0c4348 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -395,9 +395,15 @@ main(int argc, char *argv[]) if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) config_get_opt_env(instance_options); + if (backup_subcmd == CHECKDB_CMD + && (&instance_config.logger.log_level_file != LOG_OFF) + && (&instance_config.logger.log_directory == NULL)) + elog(ERROR, "Cannot save checkdb logs to a file. " + "You must specify --log-directory option when running checkdb with " + "--log-level-file option enabled."); + /* Initialize logger */ - if (backup_subcmd != CHECKDB_CMD) - init_logger(backup_path, &instance_config.logger); + init_logger(backup_path, &instance_config.logger); /* command was initialized for a few commands */ if (command) diff --git a/src/utils/logger.c b/src/utils/logger.c index ba054a62c..b3db36037 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -71,12 +71,16 @@ static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; void init_logger(const char *root_path, LoggerConfig *config) { - /* Set log path */ - if (config->log_directory == NULL) + /* + * If logging to file is enabled and log_directory wasn't set + * by user, init the path with default value: backup_directory/log/ + * */ + if (config->log_level_file != LOG_OFF + && config->log_directory == NULL) { - config->log_directory = pgut_malloc(MAXPGPATH); - join_path_components(config->log_directory, - root_path, LOG_DIRECTORY_DEFAULT); + config->log_directory = pgut_malloc(MAXPGPATH); + join_path_components(config->log_directory, + root_path, LOG_DIRECTORY_DEFAULT); } logger_config = *config; From 30cf45816d27b2bb877bc54689f71f09483dc004 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 12 Dec 2018 00:29:20 +0300 Subject: [PATCH 0144/2107] check backup version before backup validation, throw an error if backup version > binary version --- src/validate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/validate.c b/src/validate.c index edb386ef9..14d347ca0 100644 --- a/src/validate.c +++ b/src/validate.c @@ -52,6 +52,14 @@ pgBackupValidate(pgBackup *backup) validate_files_arg *threads_args; int i; + /* Check backup version */ + if (backup->program_version && + parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE && From 23d1a26e024c3fb66f4d5c514abd80ed53558cae Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 15:10:25 +0300 Subject: [PATCH 0145/2107] Fix MERGE of uncompressed with compressed backups --- src/catalog.c | 14 -------------- src/data.c | 23 +++++++++++++++++------ src/dir.c | 2 +- src/merge.c | 23 ++++++++++++++++++++++- src/pg_probackup.h | 1 - 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 77ac0f342..b30fcc664 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -820,20 +820,6 @@ pgBackupInit(pgBackup *backup) backup->server_version[0] = '\0'; } -/* - * Copy backup metadata from **src** into **dst**. - */ -void -pgBackupCopy(pgBackup *dst, pgBackup *src) -{ - pfree(dst->primary_conninfo); - - memcpy(dst, src, sizeof(pgBackup)); - - if (src->primary_conninfo) - dst->primary_conninfo = pstrdup(src->primary_conninfo); -} - /* free pgBackup object */ void pgBackupFree(void *backup) diff --git a/src/data.c b/src/data.c index 0d0c5e5cc..ccbe9ca87 100644 --- a/src/data.c +++ b/src/data.c @@ -829,6 +829,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, if (write_header) { + /* We uncompressed the page, so its size is BLCKSZ */ + header.compressed_size = BLCKSZ; if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) elog(ERROR, "cannot write header of block %u of \"%s\": %s", blknum, file->path, strerror(errno)); @@ -1592,19 +1594,23 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (read_len == 0 && feof(in)) break; /* EOF found */ else if (read_len != 0 && feof(in)) - elog(ERROR, + elog(WARNING, "odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(ERROR, "cannot read header of block %u of \"%s\": %s", + elog(WARNING, "cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); + return false; } COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u", + { + elog(WARNING, "backup is broken at file->path %s block %u", file->path, blknum); + return false; + } blknum = header.block; @@ -1620,8 +1626,11 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", + { + elog(WARNING, "cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); + return false; + } COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); @@ -1648,11 +1657,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, is_valid = false; continue; } - elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + elog(WARNING, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); + return false; } + if (validate_one_page(page.data, file, blknum, - stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) + stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) is_valid = false; } else diff --git a/src/dir.c b/src/dir.c index ae8cb20bc..386f49bcf 100644 --- a/src/dir.c +++ b/src/dir.c @@ -211,7 +211,7 @@ pgFileInit(const char *path) strcpy(file->path, path); /* enough buffer size guaranteed */ /* Get file name from the path */ - file_name = strrchr(file->path, '/'); + file_name = last_dir_separator(file->path); if (file_name == NULL) file->name = file->path; else diff --git a/src/merge.c b/src/merge.c index e06695860..b936469bd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -91,7 +91,8 @@ do_merge(time_t backup_id) } else { - Assert(dest_backup); + if (dest_backup == NULL) + elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); if (backup->start_time != prev_parent) continue; @@ -335,6 +336,20 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) } } + /* + * Rename FULL backup directory. + */ + elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); + if (rename(to_backup_path, from_backup_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + to_backup_path, from_backup_path, strerror(errno)); + + /* + * Merging finished, now we can safely update ID of the destination backup. + */ + to_backup->start_time = from_backup->start_time; + write_backup(to_backup); + /* Cleanup */ if (threads) { @@ -526,6 +541,12 @@ merge_files(void *arg) else copy_file(argument->from_root, argument->to_root, file); + /* + * We need to save compression algorithm type of the target backup to be + * able to restore in the future. + */ + file->compress_alg = to_backup->compress_alg; + if (file->write_size != BYTES_INVALID) elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0dc649a29..546c1016c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -476,7 +476,6 @@ extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); extern void pgBackupInit(pgBackup *backup); -extern void pgBackupCopy(pgBackup *dst, pgBackup *src); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); From 26c2e066f669265355523c164db9d26d4f4840e7 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 12 Dec 2018 17:07:18 +0300 Subject: [PATCH 0146/2107] code cleanup. remove obsolete exit_or_abort() function --- src/utils/configuration.c | 6 ++---- src/utils/pgut.c | 16 ---------------- src/utils/pgut.h | 1 - 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index fe50c4946..61dcaf0a5 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -187,10 +187,8 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src) const char *message; if (opt == NULL) - { - fprintf(stderr, "Try \"%s --help\" for more information.\n", PROGRAM_NAME); - exit_or_abort(ERROR); - } + elog(ERROR, "Option is not found. Try \"%s --help\" for more information.\n", + PROGRAM_NAME); if (opt->source > src) { diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ccf832b97..91bfd8922 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -783,22 +783,6 @@ on_cleanup(void) call_atexit_callbacks(false); } -void -exit_or_abort(int exitcode) -{ - if (in_cleanup) - { - /* oops, error in cleanup*/ - call_atexit_callbacks(true); - abort(); - } - else - { - /* normal exit */ - exit(exitcode); - } -} - void * pgut_malloc(size_t size) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index bde5dee1c..ca44fb391 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -40,7 +40,6 @@ extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); extern void pgut_init(void); -extern void exit_or_abort(int exitcode); /* * Database connections From 0b05cf0343a688178fafa33ad916c81875287234 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 17:27:31 +0300 Subject: [PATCH 0147/2107] Improve comment within on_interrupt() --- src/utils/pgut.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ccf832b97..e08b53ea4 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -703,7 +703,10 @@ on_interrupt(void) /* Set interruped flag */ interrupted = true; - /* User promts password, call on_cleanup() byhand */ + /* + * User promts password, call on_cleanup() byhand. If we don't do that we + * will stuck forever until an user enters a password. + */ if (in_password) { on_cleanup(); From 5ad509a40efec5af6ce22c5cd8c08b2dee94147b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 12 Dec 2018 17:52:01 +0300 Subject: [PATCH 0148/2107] Fix comment introduced in 0b05cf0343 --- src/utils/pgut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index e08b53ea4..a7e12e913 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -704,8 +704,8 @@ on_interrupt(void) interrupted = true; /* - * User promts password, call on_cleanup() byhand. If we don't do that we - * will stuck forever until an user enters a password. + * User promts password, call on_cleanup() byhand. Unless we do that we will + * get stuck forever until a user enters a password. */ if (in_password) { From ffd4b2f5cfc8d893f6c7dffff7b37a933a932ce6 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 12 Dec 2018 18:19:04 +0300 Subject: [PATCH 0149/2107] pgpro-2065. WIP. fix amcheck return status, call amcheck function using correct schema name --- src/backup.c | 107 +++++++++++++++++++++++++++++++-------------- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 + 3 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/backup.c b/src/backup.c index 988f26ec2..ebf5819bf 100644 --- a/src/backup.c +++ b/src/backup.c @@ -95,6 +95,7 @@ static bool pg_stop_backup_is_sent = false; */ static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); +static void threads_conn_disconnect(bool fatal, void *userdata); static void *backup_files(void *arg); static void *check_files(void *arg); @@ -104,7 +105,7 @@ static void do_backup_instance(void); static void do_check_instance(bool do_amcheck); static parray* get_index_list(void); static bool amcheck_one_index(backup_files_arg *arguments, - Oid indexrelid); + pg_indexEntry *ind); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); static void pg_switch_wal(PGconn *conn); @@ -535,6 +536,8 @@ do_check_instance(bool do_amcheck) arg->ret = 1; } + pgut_atexit_push(threads_conn_disconnect, NULL); + /* Run threads */ elog(INFO, "Start checking data files"); for (i = 0; i < num_threads; i++) @@ -2345,6 +2348,27 @@ backup_disconnect(bool fatal, void *userdata) pgut_disconnect(master_conn); } +/* + * Disconnect backup connections created in threads during quit pg_probackup. + */ +static void +threads_conn_disconnect(bool fatal, void *userdata) +{ + int i; + + elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); + for (i = 0; i < num_threads; i++) + { +// backup_files_arg *arg = &(threads_args[i]); +// +// if (arg->backup_conn) +// { +// pgut_cancel(arg->backup_conn); +// pgut_disconnect(arg->backup_conn); +// } + } +} + static void * check_files(void *arg) { @@ -2352,6 +2376,7 @@ check_files(void *arg) backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = parray_num(arguments->files_list); int n_indexes = 0; + char dbname[NAMEDATALEN]; /* We only have index_list in amcheck mode */ if (arguments->index_list) @@ -2421,19 +2446,34 @@ check_files(void *arg) /* check index with amcheck */ + /* TODO sort index_list by dbname */ for (i = 0; i < n_indexes; i++) { pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); - elog(VERBOSE, "Checking index: \"%s\" ", ind->name); if (!pg_atomic_test_set_flag(&ind->lock)) continue; /* check for interrupt */ if (interrupted) elog(ERROR, "interrupted during checkdb --amcheck"); - - amcheck_one_index(arguments, ind->indexrelid); + + elog(VERBOSE, "Checking index: \"%s\" ", ind->name); + + /* reconnect, if index to check is in another database */ + if (strcmp((const char *) &dbname, ind->dbname) != 0) + { + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + arguments->backup_conn = pgut_connect(instance_config.pghost, + instance_config.pgport, + ind->dbname, + instance_config.pguser); + arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + } + + amcheck_one_index(arguments, ind); } /* Close connection */ @@ -3150,10 +3190,10 @@ get_index_list(void) PGresult *res_db, *res; const char *dbname; + const char *nspname = NULL; int i; Oid dbOid, tblspcOid; char *params[2]; - const char *indexname; Oid indexrelid; params[0] = palloc(64); @@ -3176,7 +3216,9 @@ get_index_list(void) dbname, instance_config.pguser); - res = pgut_execute(tmp_conn, "select * from pg_extension where extname='amcheck'", 0, NULL); + res = pgut_execute(tmp_conn, "select extname, nspname, extversion from pg_namespace " + "n join pg_extension e on n.oid=e.extnamespace where e.extname='amcheck'", + 0, NULL); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3189,9 +3231,16 @@ get_index_list(void) { elog(VERBOSE, "extension amcheck is not installed in database %s", dbname); continue; - } - + } + + if (nspname) + free(nspname); + + nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); + strcpy(nspname, PQgetvalue(res, 0, 1)); + PQclear(res); + res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" " FROM pg_index idx " " JOIN pg_class cls ON cls.oid=idx.indexrelid " @@ -3210,17 +3259,20 @@ get_index_list(void) ind->name = pgut_malloc(strlen(name) + 1); strcpy(ind->name, name); /* enough buffer size guaranteed */ + ind->dbname = pgut_malloc(strlen(dbname) + 1); + strcpy(ind->dbname, dbname); /* enough buffer size guaranteed */ + + ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); + strcpy(ind->amcheck_nspname, nspname); pg_atomic_clear_flag(&ind->lock); if (index_list == NULL) index_list = parray_new(); - /* TODO maybe keep structure with index name and some other fileds? */ parray_append(index_list, ind); } PQclear(res); - pgut_disconnect(tmp_conn); } @@ -3233,48 +3285,35 @@ get_index_list(void) /* check one index. Return true if everything is ok, false otherwise. */ static bool amcheck_one_index(backup_files_arg *arguments, - Oid indexrelid) + pg_indexEntry *ind) { PGresult *res; char *params[1]; char *result; - + char *query; params[0] = palloc(64); - /* - * Use tmp_conn, since we may work in parallel threads. - * We can connect to any database. - */ - sprintf(params[0], "%i", indexrelid); - - if (arguments->backup_conn == NULL) - { - arguments->backup_conn = pgut_connect(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - } + sprintf(params[0], "%i", ind->indexrelid); - if (arguments->cancel_conn == NULL) - arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); + sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, - "SELECT bt_index_check($1)", - 1, (const char **)params, true); + query, 1, (const char **)params, true); - /* TODO now this check is hidden inside pgut_execute_parallel - * We need to handle failed check as an error. - */ - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (PQresultStatus(res) != PGRES_TUPLES_OK) { elog(VERBOSE, "amcheck failed for relation oid %u: %s", - indexrelid, PQresultErrorMessage(res)); + ind->indexrelid, PQresultErrorMessage(res)); pfree(params[0]); PQclear(res); return false; } + else + elog(VERBOSE, "amcheck succeeded for relation oid %u", + ind->indexrelid); pfree(params[0]); PQclear(res); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b6a0c4348..4e5855d8c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -396,7 +396,7 @@ main(int argc, char *argv[]) config_get_opt_env(instance_options); if (backup_subcmd == CHECKDB_CMD - && (&instance_config.logger.log_level_file != LOG_OFF) + && (instance_config.logger.log_level_file != LOG_OFF) && (&instance_config.logger.log_directory == NULL)) elog(ERROR, "Cannot save checkdb logs to a file. " "You must specify --log-directory option when running checkdb with " diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5b45cec85..ed293f941 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -133,6 +133,8 @@ typedef struct pg_indexEntry { Oid indexrelid; char *name; + char *dbname; + char *amcheck_nspname; /* schema where amcheck extention is located */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ } pg_indexEntry; From 9484b6de27c3768e7105a87f5c94e989611767fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 12 Dec 2018 20:32:42 +0300 Subject: [PATCH 0150/2107] tests: test_page_create_db added --- tests/page.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/page.py b/tests/page.py index f06470825..d0ebbb9b9 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import QueryException from datetime import datetime, timedelta import subprocess import gzip @@ -1030,3 +1031,120 @@ def test_multithread_page_backup_with_toast(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_create_db(self): + """ + Make node, take full backup, create database db1, take page backup, + restore database and check it presense + """ + self.maxDiff = None + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB', + 'max_wal_senders': '2', + 'checkpoint_timeout': '5min', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + self.backup_node( + backup_dir, 'node', node) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1000) i") + + # PAGE BACKUP + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir="{0}/{1}/node_restored".format(module_name, fname) + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + node_restored.safe_psql('db1', 'select 1') + node_restored.cleanup() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) From eef5769ffd3da14e9d4faa649b0b162dbd606af6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 10:20:49 +0300 Subject: [PATCH 0151/2107] tests: pgdata_compare now detect missing or redundant directories --- tests/helpers/ptrack_helpers.py | 48 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d81d72888..0bc27abf7 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1047,7 +1047,7 @@ def del_test_dir(self, module_name, fname): except: pass - def pgdata_content(self, directory, ignore_ptrack=True): + def pgdata_content(self, pgdata, ignore_ptrack=True): """ return dict with directory content. " " TAKE IT AFTER CHECKPOINT or BACKUP""" dirs_to_ignore = [ @@ -1064,9 +1064,10 @@ def pgdata_content(self, directory, ignore_ptrack=True): # '_ptrack' # ) directory_dict = {} - directory_dict['pgdata'] = directory + directory_dict['pgdata'] = pgdata directory_dict['files'] = {} - for root, dirs, files in os.walk(directory, followlinks=True): + directory_dict['dirs'] = [] + for root, dirs, files in os.walk(pgdata, followlinks=True): dirs[:] = [d for d in dirs if d not in dirs_to_ignore] for file in files: if ( @@ -1076,7 +1077,7 @@ def pgdata_content(self, directory, ignore_ptrack=True): continue file_fullpath = os.path.join(root, file) - file_relpath = os.path.relpath(file_fullpath, directory) + file_relpath = os.path.relpath(file_fullpath, pgdata) directory_dict['files'][file_relpath] = {'is_datafile': False} directory_dict['files'][file_relpath]['md5'] = hashlib.md5( open(file_fullpath, 'rb').read()).hexdigest() @@ -1089,12 +1090,51 @@ def pgdata_content(self, directory, ignore_ptrack=True): file_fullpath, size_in_pages ) + for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True): + for directory in dirs: + directory_path = os.path.join(root, directory) + directory_relpath = os.path.relpath(directory_path, pgdata) + + found = False + for d in dirs_to_ignore: + if d in directory_relpath: + found = True + break + + # check if directory already here as part of larger directory + if not found: + for d in directory_dict['dirs']: + # print("OLD dir {0}".format(d)) + if directory_relpath in d: + found = True + break + + if not found: + directory_dict['dirs'].append(directory_relpath) + return directory_dict def compare_pgdata(self, original_pgdata, restored_pgdata): """ return dict with directory content. DO IT BEFORE RECOVERY""" fail = False error_message = 'Restored PGDATA is not equal to original!\n' + + # Compare directories + for directory in restored_pgdata['dirs']: + if directory not in original_pgdata['dirs']: + fail = True + error_message += '\nDirectory is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in original_pgdata['dirs']: + if directory not in restored_pgdata['dirs']: + fail = True + error_message += '\nDirectory dissappeared' + error_message += ' in restored PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for file in restored_pgdata['files']: # File is present in RESTORED PGDATA # but not present in ORIGINAL From a5cef9890829c90b4df15efee6b5e5cda6c9c782 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 11:45:04 +0300 Subject: [PATCH 0152/2107] tests: test_merge_tablespaces and test_merge_tablespaces_1 added --- tests/helpers/ptrack_helpers.py | 2 +- tests/merge.py | 154 ++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0bc27abf7..8ea90c240 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1123,7 +1123,7 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): for directory in restored_pgdata['dirs']: if directory not in original_pgdata['dirs']: fail = True - error_message += '\nDirectory is not present' + error_message += '\nDirectory was not present' error_message += ' in original PGDATA: {0}\n'.format( os.path.join(restored_pgdata['pgdata'], directory)) diff --git a/tests/merge.py b/tests/merge.py index e42094412..8ce1795e6 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -3,6 +3,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import shutil module_name = "merge" @@ -471,9 +472,162 @@ def test_merge_compressed_and_uncompressed_backups_2(self): # @unittest.skip("skip") def test_merge_tablespaces(self): + """ + Create tablespace with table + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Create new tablespace + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + + # Drop old tablespace + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + # this compare should fall because we lost some directories + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_merge_tablespaces_1(self): """ Some test here """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # CREATE NEW TABLESPACE + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) def test_merge_page_truncate(self): """ From 4c249ee1b558deff5b3e1b763ec80c348e4f3880 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 13 Dec 2018 12:50:26 +0300 Subject: [PATCH 0153/2107] Send signal to agent in case of parent process exit --- src/utils/remote.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 2cab0e3a2..5c5677aef 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "pg_probackup.h" #include "file.h" @@ -61,6 +62,13 @@ static int split_options(int argc, char* argv[], int max_options, char* options) return argc; } +static int child_pid; + +static void kill_child(void) +{ + kill(child_pid, SIGTERM); +} + int remote_execute(int argc, char* argv[], bool listen) { char cmd[MAX_CMDLINE_LENGTH]; @@ -70,7 +78,6 @@ int remote_execute(int argc, char* argv[], bool listen) int i; int outfd[2]; int infd[2]; - pid_t pid; char* pg_probackup = argv[0]; ssh_argc = 0; @@ -110,9 +117,9 @@ int remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); - SYS_CHECK(pid = fork()); + SYS_CHECK(child_pid = fork()); - if (pid == 0) { /* child */ + if (child_pid == 0) { /* child */ SYS_CHECK(close(STDIN_FILENO)); SYS_CHECK(close(STDOUT_FILENO)); @@ -129,7 +136,7 @@ int remote_execute(int argc, char* argv[], bool listen) } else { SYS_CHECK(close(outfd[0])); /* These are being used by the child */ SYS_CHECK(close(infd[1])); - + atexit(kill_child); if (listen) { int status; fio_communicate(infd[0], outfd[1]); From e16bdc6660bb5666c485a897e4f444bcdf3d9aab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 13 Dec 2018 13:21:57 +0300 Subject: [PATCH 0154/2107] tests: add some comments --- tests/merge.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 8ce1795e6..2dbe8ee0c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -473,7 +473,10 @@ def test_merge_compressed_and_uncompressed_backups_2(self): # @unittest.skip("skip") def test_merge_tablespaces(self): """ - Create tablespace with table + Create tablespace with table, take FULL backup, + create another tablespace with another table and drop previous + tablespace, take page backup, merge it and restore + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -550,7 +553,10 @@ def test_merge_tablespaces(self): # @unittest.skip("skip") def test_merge_tablespaces_1(self): """ - Some test here + Create tablespace with table, take FULL backup, + create another tablespace with another table, take page backup, + drop first tablespace and take delta backup, + merge it and restore """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1202,3 +1208,4 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # FULL MERGING # 3. Need new test with corrupted FULL backup +# 4. different compression levels From 65e0ff9565afee4b8f03cde3a2da5369a5719e54 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 13 Dec 2018 14:14:59 +0300 Subject: [PATCH 0155/2107] =?UTF-8?q?pgpro-2065.=20refactor=20the=20code,?= =?UTF-8?q?=20add=20option=20--block-validation=20=D0=B0for=20checkdb=20co?= =?UTF-8?q?mmand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backup.c | 279 +++++++++++++++++++++------------------------ src/pg_probackup.c | 4 +- src/pg_probackup.h | 2 +- 3 files changed, 137 insertions(+), 148 deletions(-) diff --git a/src/backup.c b/src/backup.c index ebf5819bf..6c170ffb0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -97,12 +97,13 @@ static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); static void threads_conn_disconnect(bool fatal, void *userdata); +static void pgdata_basic_setup(void); + static void *backup_files(void *arg); static void *check_files(void *arg); static void *remote_backup_files(void *arg); static void do_backup_instance(void); -static void do_check_instance(bool do_amcheck); static parray* get_index_list(void); static bool amcheck_one_index(backup_files_arg *arguments, pg_indexEntry *ind); @@ -467,105 +468,6 @@ remote_backup_files(void *arg) return NULL; } -/* TODO think about merging it with do_backup_instance */ -static void -do_check_instance(bool do_amcheck) -{ - int i; - char database_path[MAXPGPATH]; - - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - backup_files_arg *threads_args; - bool backup_isok = true; - - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - /* initialize backup list */ - backup_files_list = parray_new(); - - /* list files with the logical path. omit $PGDATA */ - dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); - - /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - * - * Sorted array is used at least in parse_backup_filelist_filenames(), - * extractPageMap(), make_pagemap_from_ptrack(). - */ - parray_qsort(backup_files_list, pgFileComparePath); - /* Extract information about files in backup_list parsing their names:*/ - parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); - - /* setup threads */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - pg_atomic_clear_flag(&file->lock); - } - - /* Sort by size for load balancing */ - parray_qsort(backup_files_list, pgFileCompareSize); - - if (do_amcheck) - index_list = get_index_list(); - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); - - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - arg->from_root = instance_config.pgdata; - arg->to_root = database_path; - arg->files_list = backup_files_list; - arg->prev_filelist = NULL; - arg->prev_start_lsn = InvalidXLogRecPtr; - arg->backup_conn = NULL; - arg->cancel_conn = NULL; - arg->index_list = index_list; - /* By default there are some error */ - arg->ret = 1; - } - - pgut_atexit_push(threads_conn_disconnect, NULL); - - /* Run threads */ - elog(INFO, "Start checking data files"); - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - elog(VERBOSE, "Start thread num: %i", i); - - pthread_create(&threads[i], NULL, check_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - backup_isok = false; - } - if (backup_isok) - elog(INFO, "Data files are checked"); - else - elog(ERROR, "Data files checking failed"); - - parray_walk(backup_files_list, pgFileFree); - parray_free(backup_files_list); - backup_files_list = NULL; -} - /* * Take a backup of a single postgresql instance. * Move files from 'pgdata' to a subdirectory in 'backup_path'. @@ -964,9 +866,123 @@ do_backup_instance(void) } /* Entry point of pg_probackup CHECKDB subcommand. */ -/* TODO think about merging it with do_backup */ +/* TODO consider moving some code common with do_backup_instance + * to separate function ot to pgdata_basic_setup */ int -do_checkdb(bool do_amcheck) +do_checkdb(bool do_block_validation, bool do_amcheck) +{ + int i; + char database_path[MAXPGPATH]; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + + pgdata_basic_setup(); + + if (do_block_validation) + { + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + /* initialize backup list */ + backup_files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); + + /* setup threads */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + pg_atomic_clear_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); + } + + if (do_amcheck) + index_list = get_index_list(); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = instance_config.pgdata; + arg->to_root = database_path; + arg->files_list = backup_files_list; + arg->index_list = index_list; + arg->prev_filelist = NULL; + arg->prev_start_lsn = InvalidXLogRecPtr; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + pgut_atexit_push(threads_conn_disconnect, NULL); + + /* Run threads */ + elog(INFO, "Start checking data files"); + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + pthread_create(&threads[i], NULL, check_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + if (backup_isok) + elog(INFO, "Data files are checked"); + else + elog(ERROR, "Data files checking failed"); + + if (backup_files_list) + { + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; + } + + return 0; +} + +/* + * Common code for CHECKDB and BACKUP commands. + * Ensure that we're able to connect to the instance + * check compatibility and fill basic info. + * TODO maybe move it to pg_probackup.c + */ +static void +pgdata_basic_setup(void) { /* PGDATA is always required */ if (instance_config.pgdata == NULL) @@ -990,7 +1006,6 @@ do_checkdb(bool do_amcheck) /* Confirm that this server version is supported */ check_server_version(); - /* Do we need that? */ current.checksum_version = get_data_checksum_version(true); is_checksum_enabled = pg_checksum_enable(); @@ -1005,9 +1020,6 @@ do_checkdb(bool do_amcheck) StrNCpy(current.server_version, server_version_str, sizeof(current.server_version)); - - do_check_instance(do_amcheck); - return 0; } /* @@ -1016,22 +1028,17 @@ do_checkdb(bool do_amcheck) int do_backup(time_t start_time) { - /* PGDATA and BACKUP_MODE are always required */ - if (instance_config.pgdata == NULL) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + /* + * setup backup_conn, do some compatibility checks and + * fill basic info about instance + */ + pgdata_basic_setup(); + + /* below perform checks specific for backup command */ if (current.backup_mode == BACKUP_MODE_INVALID) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - /* Create connection for PostgreSQL */ - backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - pgut_atexit_push(backup_disconnect, NULL); - - current.primary_conninfo = pgut_get_conninfo_string(backup_conn); - #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) elog(ERROR, "Failed to retreive wal_segment_size"); @@ -1040,31 +1047,10 @@ do_backup(time_t start_time) current.compress_alg = instance_config.compress_alg; current.compress_level = instance_config.compress_level; - /* Confirm data block size and xlog block size are compatible */ - confirm_block_size("block_size", BLCKSZ); - confirm_block_size("wal_block_size", XLOG_BLCKSZ); - - current.from_replica = pg_is_in_recovery(); - - /* Confirm that this server version is supported */ - check_server_version(); - /* TODO fix it for remote backup*/ if (!is_remote_backup) current.checksum_version = get_data_checksum_version(true); - is_checksum_enabled = pg_checksum_enable(); - - if (is_checksum_enabled) - elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " - "Data block corruption will be detected"); - else - elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " - "pg_probackup have no way to detect data block corruption without them. " - "Reinitialize PGDATA with option '--data-checksums'."); - - StrNCpy(current.server_version, server_version_str, - sizeof(current.server_version)); current.stream = stream_wal; is_ptrack_support = pg_ptrack_support(); @@ -2374,13 +2360,12 @@ check_files(void *arg) { int i; backup_files_arg *arguments = (backup_files_arg *) arg; - int n_backup_files_list = parray_num(arguments->files_list); + int n_backup_files_list = 0; int n_indexes = 0; char dbname[NAMEDATALEN]; - /* We only have index_list in amcheck mode */ - if (arguments->index_list) - n_indexes = parray_num(arguments->index_list); + if (arguments->files_list) + n_backup_files_list = parray_num(arguments->files_list); /* check a file */ for (i = 0; i < n_backup_files_list; i++) @@ -2444,8 +2429,10 @@ check_files(void *arg) elog(WARNING, "unexpected file type %d", buf.st_mode); } + /* Check indexes with amcheck */ + if (arguments->index_list) + n_indexes = parray_num(arguments->index_list); - /* check index with amcheck */ /* TODO sort index_list by dbname */ for (i = 0; i < n_indexes; i++) { @@ -3190,7 +3177,7 @@ get_index_list(void) PGresult *res_db, *res; const char *dbname; - const char *nspname = NULL; + char *nspname = NULL; int i; Oid dbOid, tblspcOid; char *params[2]; @@ -3229,7 +3216,7 @@ get_index_list(void) if (PQntuples(res) < 1) { - elog(VERBOSE, "extension amcheck is not installed in database %s", dbname); + elog(WARNING, "extension amcheck is not installed in database %s", dbname); continue; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 4e5855d8c..68113f711 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -86,6 +86,7 @@ bool restore_no_validate = false; bool skip_block_validation = false; +bool do_block_validation = false; bool do_amcheck = false; /* delete options */ @@ -155,6 +156,7 @@ static ConfigOption cmd_options[] = { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 155, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, + { 'b', 156, "block-validation", &do_block_validation, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -542,7 +544,7 @@ main(int argc, char *argv[]) do_set_config(); break; case CHECKDB_CMD: - do_checkdb(do_amcheck); + do_checkdb(do_block_validation, do_amcheck); break; case NO_CMD: /* Should not happen */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ed293f941..6f2757676 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -401,7 +401,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); -extern int do_checkdb(bool do_amcheck); +extern int do_checkdb(bool do_block_validation, bool do_amcheck); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, From 5b2cae5e81d4dede2cec495e35f3fe5f5935bcb3 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 13 Dec 2018 16:20:51 +0300 Subject: [PATCH 0156/2107] pgpro-2065. filter global indexes --- src/backup.c | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6c170ffb0..de992fc76 100644 --- a/src/backup.c +++ b/src/backup.c @@ -918,8 +918,15 @@ do_checkdb(bool do_block_validation, bool do_amcheck) } if (do_amcheck) + { index_list = get_index_list(); + /* no need to setup threads. there's nothing to do */ + if ((!index_list) & (!do_block_validation)) + elog(ERROR, "Cannot perform 'checkdb --amcheck', since " + "this backup instance does not contain any databases with amcheck installed"); + } + /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -942,8 +949,10 @@ do_checkdb(bool do_block_validation, bool do_amcheck) pgut_atexit_push(threads_conn_disconnect, NULL); - /* Run threads */ + /* TODO write better info message */ elog(INFO, "Start checking data files"); + + /* Run threads */ for (i = 0; i < num_threads; i++) { backup_files_arg *arg = &(threads_args[i]); @@ -960,6 +969,8 @@ do_checkdb(bool do_block_validation, bool do_amcheck) if (threads_args[i].ret == 1) backup_isok = false; } + + /* TODO write better info message */ if (backup_isok) elog(INFO, "Data files are checked"); else @@ -2335,7 +2346,7 @@ backup_disconnect(bool fatal, void *userdata) } /* - * Disconnect backup connections created in threads during quit pg_probackup. + * Disconnect checkdb connections created in threads during quit pg_probackup. */ static void threads_conn_disconnect(bool fatal, void *userdata) @@ -2343,8 +2354,8 @@ threads_conn_disconnect(bool fatal, void *userdata) int i; elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); - for (i = 0; i < num_threads; i++) - { +// for (i = 0; i < num_threads; i++) +// { // backup_files_arg *arg = &(threads_args[i]); // // if (arg->backup_conn) @@ -2352,7 +2363,7 @@ threads_conn_disconnect(bool fatal, void *userdata) // pgut_cancel(arg->backup_conn); // pgut_disconnect(arg->backup_conn); // } - } +// } } static void * @@ -3182,6 +3193,7 @@ get_index_list(void) Oid dbOid, tblspcOid; char *params[2]; Oid indexrelid; + bool first_db_with_amcheck = true; params[0] = palloc(64); params[1] = palloc(64); @@ -3228,14 +3240,30 @@ get_index_list(void) PQclear(res); - res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" + /* + * In order to avoid duplicates, select global indexes + * (tablespace pg_global with oid 1664) only once + */ + if (first_db_with_amcheck) + { + res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" + " FROM pg_index idx " + " JOIN pg_class cls ON cls.oid=idx.indexrelid " + " JOIN pg_am am ON am.oid=cls.relam " + " WHERE am.amname='btree' AND cls.relpersistence != 't'" + " AND idx.indisready AND idx.indisvalid; ", 0, NULL); + first_db_with_amcheck = false; + } + else + res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" " FROM pg_index idx " " JOIN pg_class cls ON cls.oid=idx.indexrelid " " JOIN pg_am am ON am.oid=cls.relam " - " WHERE am.amname='btree'; ", 0, NULL); + " WHERE am.amname='btree' AND cls.relpersistence != 't'" + " AND idx.indisready AND idx.indisvalid AND cls.reltablespace!=1664; ", 0, NULL); - /* TODO filter system indexes to add them to list only once */ - /* TODO maybe save tablename for load balancing */ + + /* add info needed to check indexes into index_list */ for(i = 0; i < PQntuples(res); i++) { pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); @@ -3266,6 +3294,7 @@ get_index_list(void) pfree(params[0]); pfree(params[1]); PQclear(res_db); + return index_list; } From eec11c074c839bd465780673c417ce7a3f539823 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 13 Dec 2018 17:11:45 +0300 Subject: [PATCH 0157/2107] Fix comment in main() --- src/pg_probackup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 04b5d4862..ada27e3e8 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -363,8 +363,9 @@ main(int argc, char *argv[]) } /* - * Read options from env variables or from config file, - * unless we're going to set them via set-config. + * We read options from command line, now we need to read them from + * configuration file since we got backup path and instance name. + * For some commands an instance option isn't required, see above. */ if (instance_name) { From 5401eeb8573287b6d1d2c8c8f70c0aa5f6dc1671 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 13 Dec 2018 17:13:21 +0300 Subject: [PATCH 0158/2107] pgpro-2065. fix options parsing --- src/pg_probackup.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 54b7e81f1..b46ae8a43 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -377,11 +377,8 @@ main(int argc, char *argv[]) } /* - * Read options from env variables or from config file, - * unless we're going to set them via set-config. - */ - if (((backup_path != NULL) && instance_name) - && backup_subcmd != SET_CONFIG_CMD) + * Read options from env variables or from config file */ + if ((backup_path != NULL) && instance_name) { char config_path[MAXPGPATH]; /* Read environment variables */ From 58373a5e174b8ca0dd80a53f9a41e343a09d3360 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 13 Dec 2018 18:04:44 +0300 Subject: [PATCH 0159/2107] Add comments for FIO functions --- src/pg_probackup.c | 7 +++++-- src/pg_probackup.h | 1 + src/utils/file.c | 45 ++++++++++++++++++++++++++++++++++++++++++--- src/utils/file.h | 8 ++++---- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 91c0c5539..8f1cefe93 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -318,9 +318,12 @@ main(int argc, char *argv[]) config_get_opt(argc, argv, cmd_options, instance_options); if (remote_agent != NULL && strcmp(remote_agent, PROGRAM_VERSION) != 0) - elog(ERROR, "Agent version %s doesn't match master pg_probackup version %s", + { + uint32 agent_version = parse_program_version(remote_agent); + elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, + "Agent version %s doesn't match master pg_probackup version %s", remote_agent, PROGRAM_VERSION); - + } pgut_init(); if (help_opt) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dcb8f7ce3..aa025de97 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -168,6 +168,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) #define BLOCKNUM_INVALID (-1) #define PROGRAM_VERSION "2.0.26" +#define AGENT_PROTOCOL_VERSION 20026 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/src/utils/file.c b/src/utils/file.c index 7a1daadfa..78cfdccad 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -22,31 +22,37 @@ static fio_binding fio_bindings[] = {¤t.stop_lsn, sizeof(current.stop_lsn)} }; +/* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) +/* Use specified file descriptors as stding/stdout for FIO functions */ void fio_redirect(int in, int out) { fio_stdin = in; fio_stdout = out; } +/* Check if FILE handle is local or remote (created by FIO) */ static bool fio_is_remote_file(FILE* file) { return (size_t)file <= FIO_FDMAX; } +/* Check if file descriptor is local or remote (created by FIO) */ static bool fio_is_remote_fd(int fd) { return (fd & FIO_PIPE_MARKER) != 0; } +/* Check if specified location is local for current node */ static bool fio_is_remote(fio_location location) { return location == FIO_REMOTE_HOST - || (location == FIO_BACKUP_HOST && remote_agent) + || (location == FIO_BACKUP_HOST && remote_agent) /* agent is launched at Postgres side */ || (location == FIO_DB_HOST && !remote_agent && IsSshConnection()); } +/* Try to read specified amount of bytes unless error or EOF are encountered */ static ssize_t fio_read_all(int fd, void* buf, size_t size) { size_t offs = 0; @@ -66,6 +72,7 @@ static ssize_t fio_read_all(int fd, void* buf, size_t size) return offs; } +/* Try to write specified amount of bytes unless error is encountered */ static ssize_t fio_write_all(int fd, void const* buf, size_t size) { size_t offs = 0; @@ -83,6 +90,7 @@ static ssize_t fio_write_all(int fd, void const* buf, size_t size) return offs; } +/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ FILE* fio_open_stream(char const* path, fio_location location) { FILE* f; @@ -116,6 +124,7 @@ FILE* fio_open_stream(char const* path, fio_location location) return f; } +/* Close input stream */ int fio_close_stream(FILE* f) { if (fio_stdin_buffer) @@ -126,6 +135,7 @@ int fio_close_stream(FILE* f) return fclose(f); } +/* Open directory */ DIR* fio_opendir(char const* path, fio_location location) { DIR* dir; @@ -161,6 +171,7 @@ DIR* fio_opendir(char const* path, fio_location location) return dir; } +/* Get next directory entry */ struct dirent* fio_readdir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) @@ -193,6 +204,7 @@ struct dirent* fio_readdir(DIR *dir) } } +/* Close directory */ int fio_closedir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) @@ -213,7 +225,7 @@ int fio_closedir(DIR *dir) } } - +/* Open file */ int fio_open(char const* path, int mode, fio_location location) { int fd; @@ -250,6 +262,7 @@ int fio_open(char const* path, int mode, fio_location location) return fd; } +/* Open stdio file */ FILE* fio_fopen(char const* path, char const* mode, fio_location location) { FILE* f; @@ -266,6 +279,7 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) return f; } +/* Format output to file stream */ int fio_fprintf(FILE* f, char const* format, ...) { int rc; @@ -291,6 +305,7 @@ int fio_fprintf(FILE* f, char const* format, ...) return rc; } +/* Flush stream data (does nothing for remote file) */ int fio_fflush(FILE* f) { int rc = 0; @@ -304,11 +319,13 @@ int fio_fflush(FILE* f) return rc; } +/* Sync file to the disk (does nothing for remote file) */ int fio_flush(int fd) { return fio_is_remote_fd(fd) ? 0 : fsync(fd); } +/* Close output stream */ int fio_fclose(FILE* f) { return fio_is_remote_file(f) @@ -316,6 +333,7 @@ int fio_fclose(FILE* f) : fclose(f); } +/* Close file */ int fio_close(int fd) { if (fio_is_remote_fd(fd)) @@ -340,6 +358,7 @@ int fio_close(int fd) } } +/* Truncate stdio file */ int fio_ftruncate(FILE* f, off_t size) { return fio_is_remote_file(f) @@ -347,6 +366,7 @@ int fio_ftruncate(FILE* f, off_t size) : ftruncate(fileno(f), size); } +/* Truncate file */ int fio_truncate(int fd, off_t size) { if (fio_is_remote_fd(fd)) @@ -371,6 +391,7 @@ int fio_truncate(int fd, off_t size) } } +/* Set position in stdio file */ int fio_fseek(FILE* f, off_t offs) { return fio_is_remote_file(f) @@ -378,6 +399,7 @@ int fio_fseek(FILE* f, off_t offs) : fseek(f, offs, SEEK_SET); } +/* Set position in file */ int fio_seek(int fd, off_t offs) { if (fio_is_remote_fd(fd)) @@ -402,6 +424,7 @@ int fio_seek(int fd, off_t offs) } } +/* Write data to stdio file */ size_t fio_fwrite(FILE* f, void const* buf, size_t size) { return fio_is_remote_file(f) @@ -409,6 +432,7 @@ size_t fio_fwrite(FILE* f, void const* buf, size_t size) : fwrite(buf, 1, size, f); } +/* Write data to the file */ ssize_t fio_write(int fd, void const* buf, size_t size) { if (fio_is_remote_fd(fd)) @@ -433,6 +457,7 @@ ssize_t fio_write(int fd, void const* buf, size_t size) } } +/* Read data from stdio file */ size_t fio_fread(FILE* f, void* buf, size_t size) { return fio_is_remote_file(f) @@ -440,6 +465,7 @@ size_t fio_fread(FILE* f, void* buf, size_t size) : fread(buf, 1, size, f); } +/* Read data from file */ ssize_t fio_read(int fd, void* buf, size_t size) { if (fio_is_remote_fd(fd)) @@ -472,6 +498,7 @@ ssize_t fio_read(int fd, void* buf, size_t size) } } +/* Get information about stdio file */ int fio_ffstat(FILE* f, struct stat* st) { return fio_is_remote_file(f) @@ -479,6 +506,7 @@ int fio_ffstat(FILE* f, struct stat* st) : fio_fstat(fileno(f), st); } +/* Get information about file descriptor */ int fio_fstat(int fd, struct stat* st) { if (fio_is_remote_fd(fd)) @@ -510,6 +538,7 @@ int fio_fstat(int fd, struct stat* st) } } +/* Get information about file */ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location) { if (fio_is_remote(location)) @@ -544,6 +573,7 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati } } +/* Check presence of the file */ int fio_access(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) @@ -576,6 +606,7 @@ int fio_access(char const* path, int mode, fio_location location) } } +/* Rename file */ int fio_rename(char const* old_path, char const* new_path, fio_location location) { if (fio_is_remote(location)) @@ -602,6 +633,7 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location } } +/* Remove file */ int fio_unlink(char const* path, fio_location location) { if (fio_is_remote(location)) @@ -626,6 +658,7 @@ int fio_unlink(char const* path, fio_location location) } } +/* Create directory */ int fio_mkdir(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) @@ -651,6 +684,7 @@ int fio_mkdir(char const* path, int mode, fio_location location) } } +/* Checnge file mode */ int fio_chmod(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) @@ -677,6 +711,8 @@ int fio_chmod(char const* path, int mode, fio_location location) } #ifdef HAVE_LIBZ +/* Open compressed file. In case of remove file, it is fetched to local temporary file in read-only mode or is written + * to temoporary file and rtansfered to remote host by fio_gzclose. */ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location location) { gzFile file; @@ -715,6 +751,7 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location return file; } +/* Close compressed file. In case of writing remote file, content of temporary file is trasfered to remote host */ int fio_gzclose(gzFile file, char const* path, int tmp_fd) { if (tmp_fd >= 0) @@ -749,7 +786,7 @@ int fio_gzclose(gzFile file, char const* path, int tmp_fd) } #endif - +/* Send file content */ static void fio_send_file(int out, char const* path) { int fd = open(path, O_RDONLY); @@ -776,6 +813,7 @@ static void fio_send_file(int out, char const* path) } } +/* Send values of variables. Variables are described infio_bindings array and "var" is index in this array. */ void fio_transfer(fio_shared_variable var) { size_t var_size = fio_bindings[var].size; @@ -794,6 +832,7 @@ void fio_transfer(fio_shared_variable var) free(msg); } +/* Execute commands at remote host */ void fio_communicate(int in, int out) { int fd[FIO_FDMAX]; diff --git a/src/utils/file.h b/src/utils/file.h index 978ebc2ad..0d145f0d2 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -34,10 +34,10 @@ typedef enum typedef enum { - FIO_LOCAL_HOST, - FIO_DB_HOST, - FIO_BACKUP_HOST, - FIO_REMOTE_HOST + FIO_LOCAL_HOST, /* data is locate at local host */ + FIO_DB_HOST, /* data is located at Postgres server host */ + FIO_BACKUP_HOST, /* data is located at backup host */ + FIO_REMOTE_HOST /* date is located at remote host */ } fio_location; typedef enum From ae5aa3db7343f57e273f3d2f510cba5ec80467c7 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 13 Dec 2018 18:32:39 +0300 Subject: [PATCH 0160/2107] Implement remove ARCHIVE commands --- src/archive.c | 2 +- src/pg_probackup.c | 7 ++++--- src/pg_probackup.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/archive.c b/src/archive.c index 3333f0961..d0f4893fb 100644 --- a/src/archive.c +++ b/src/archive.c @@ -55,7 +55,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) system_id); /* Create 'archlog_path' directory. Do nothing if it already exists. */ - dir_create_dir(arclog_path, DIR_PERMISSION); + fio_mkdir(arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); join_path_components(absolute_wal_file_path, current_dir, wal_file_path); join_path_components(backup_wal_file_path, arclog_path, wal_file_name); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 8f1cefe93..57d70d4d0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -347,7 +347,8 @@ main(int argc, char *argv[]) elog(ERROR, "-B, --backup-path must be an absolute path"); if (IsSshConnection() - && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD)) + && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || + backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) { if (remote_agent) { if (backup_subcmd != BACKUP_CMD) { @@ -357,8 +358,8 @@ main(int argc, char *argv[]) fio_redirect(STDIN_FILENO, STDOUT_FILENO); } else { /* Execute remote probackup */ - int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD); - if (status != 0) + int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD); + if (status != 0 || backup_subcmd == ARCHIVE_PUSH_CMD) { return status; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index aa025de97..7034dbb7d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -526,7 +526,7 @@ extern parray *dir_read_file_list(const char *root, const char *file_txt); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); -extern bool fileExists(const char *path, fio_location location); +extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, bool omit_symlink, fio_location location); From 2220b49d6dd5ad1a54956f23571f51b05b917331 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 13 Dec 2018 21:02:27 +0300 Subject: [PATCH 0161/2107] Fix merge of extradirectories. Minor improvements. --- src/backup.c | 23 ++++++++++++----------- src/catalog.c | 1 + src/delete.c | 2 +- src/dir.c | 13 ++++--------- src/merge.c | 25 +++++++++++++++++++++---- src/pg_probackup.h | 2 +- src/restore.c | 27 ++++++++++++++------------- src/validate.c | 6 +++--- 8 files changed, 57 insertions(+), 42 deletions(-) diff --git a/src/backup.c b/src/backup.c index d1f3fa1b5..96deb2518 100644 --- a/src/backup.c +++ b/src/backup.c @@ -468,7 +468,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; /* Temp value. Used as template */ + char extra_prefix[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -579,7 +579,7 @@ do_backup_instance(void) pg_start_backup(label, smooth_checkpoint, ¤t); pgBackupGetPath(¤t, database_path, lengthof(database_path),DATABASE_DIR); - pgBackupGetPath(¤t, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(¤t, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); /* start stream replication */ if (stream_wal) @@ -712,12 +712,12 @@ do_backup_instance(void) else dir_name = file->path; - elog(VERBOSE, "Create directory \"%s\" NAME %s", dir_name, file->name); + elog(VERBOSE, "Create directory \"%s\"", dir_name); if (file->extra_dir_num) { char temp[MAXPGPATH]; - snprintf(temp, MAXPGPATH, "%s%d", extra_path, file->extra_dir_num); + snprintf(temp, MAXPGPATH, "%s%d", extra_prefix, file->extra_dir_num); join_path_components(dirpath, temp, dir_name); } else @@ -745,7 +745,7 @@ do_backup_instance(void) arg->from_root = instance_config.pgdata; arg->to_root = database_path; - arg->extra = extra_path; + arg->extra = extra_prefix; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; @@ -829,7 +829,7 @@ do_backup_instance(void) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0); for (i = 0; i < parray_num(xlog_files_list); i++) { @@ -2040,7 +2040,7 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true, false); + file = pgFileNew(backup_label, true, 0); calc_file_checksum(file); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -2084,7 +2084,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true, false); + file = pgFileNew(tablespace_map, true, 0); if (S_ISREG(file->mode)) calc_file_checksum(file); free(file->path); @@ -2331,6 +2331,7 @@ backup_files(void *arg) const char *src; const char *dst; bool skip = false; + char extra_dst[MAXPGPATH]; /* If non-data file has not changed since last backup... */ if (prev_file && file->exists_in_prev && @@ -2344,10 +2345,10 @@ backup_files(void *arg) /* Set file paths */ if (file->extra_dir_num) { - char temp[MAXPGPATH]; - sprintf(temp, "%s%d", arguments->extra, file->extra_dir_num); + makeExtraDirPathByNum(extra_dst, arguments->extra, + file->extra_dir_num); src = file->extradir; - dst = temp; + dst = extra_dst; } else { diff --git a/src/catalog.c b/src/catalog.c index 149c70b7b..ed223557c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -854,6 +854,7 @@ void pgBackupCopy(pgBackup *dst, pgBackup *src) { pfree(dst->primary_conninfo); + pfree(dst->extra_dir_str); memcpy(dst, src, sizeof(pgBackup)); diff --git a/src/delete.c b/src/delete.c index b82f0dba4..e01b613f5 100644 --- a/src/delete.c +++ b/src/delete.c @@ -283,7 +283,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); pgBackupGetPath(backup, path, lengthof(path), NULL); - dir_list_file(files, path, false, true, true, false); + dir_list_file(files, path, false, true, true, 0); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); diff --git a/src/dir.c b/src/dir.c index bc86c563a..f166fa5e8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -15,7 +15,6 @@ #endif #include "catalog/pg_tablespace.h" -#include #include #include #include @@ -468,11 +467,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, return; } if (extra_dir_num) - { file->extradir = pgut_strdup(file->path); - elog(VERBOSE,"Dir_list_file add root Name: %s Path: %s %s", - file->name, file->path, file->extradir); - } if (add_root) parray_append(files, file); @@ -835,7 +830,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false, false); + dir = pgFileNew(path, false, 0); parray_append(files, dir); } @@ -1436,7 +1431,7 @@ get_control_value(const char *str, const char *name, * If root is not NULL, path will be absolute path. */ parray * -dir_read_file_list(const char *root, const char *extra_path, const char *file_txt) +dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt) { FILE *fp; parray *files; @@ -1479,8 +1474,8 @@ dir_read_file_list(const char *root, const char *extra_path, const char *file_tx { char temp[MAXPGPATH]; - Assert(extra_path); - makeExtraDirPathByNum(temp, extra_path, extra_dir_num); + Assert(extra_prefix); + makeExtraDirPathByNum(temp, extra_prefix, extra_dir_num); join_path_components(filepath, temp, path); } else diff --git a/src/merge.c b/src/merge.c index 6ac8602e9..0e9c75c55 100644 --- a/src/merge.c +++ b/src/merge.c @@ -170,9 +170,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) from_extra_prefix[MAXPGPATH], control_file[MAXPGPATH]; parray *files, - *to_files, - *to_extra, - *from_extra; + *to_files; + parray *to_extra = NULL, + *from_extra = NULL; pthread_t *threads = NULL; merge_files_arg *threads_args = NULL; int i; @@ -273,6 +273,23 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) { pgFile *file = (pgFile *) parray_get(files, i); + /* if the entry was an extra directory, create it in the backup */ + if (file->extra_dir_num && S_ISDIR(file->mode)) + { + char dirpath[MAXPGPATH]; + char *dir_name; + char old_container[MAXPGPATH]; + char new_container[MAXPGPATH]; + + makeExtraDirPathByNum(old_container, from_extra_prefix, + file->extra_dir_num); + makeExtraDirPathByNum(new_container, to_extra_prefix, + file->extra_dir_num); + dir_name = GetRelativePath(file->path, old_container); + elog(VERBOSE, "Create directory \"%s\"", dir_name); + join_path_components(dirpath, new_container, dir_name); + dir_create_dir(dirpath, DIR_PERMISSION); + } pg_atomic_init_flag(&file->lock); } @@ -316,6 +333,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + to_backup->extra_dir_str = from_backup->extra_dir_str; /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. @@ -625,7 +643,6 @@ reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra) { char extradir_template[MAXPGPATH]; - Assert(to_extra); pgBackupGetPath(to_backup, extradir_template, lengthof(extradir_template), EXTRA_DIR); for (int i = 0; i < parray_num(to_extra); i++) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3dfed3ac9..1f7ef002a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -513,7 +513,7 @@ extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *extra_prefix); -extern parray *dir_read_file_list(const char *root, const char *extra_path, const char *file_txt); +extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt); extern parray *make_extra_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, diff --git a/src/restore.c b/src/restore.c index 41bf0410b..1a9f76d1b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -22,6 +22,7 @@ typedef struct parray *files; pgBackup *backup; parray *extra_dirs; + char *extra_prefix; /* * Return value from the thread. @@ -420,10 +421,10 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) char timestamp[100]; char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *extra_dirs; + parray *extra_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -468,15 +469,15 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) * Get list of files which need to be restored. */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, extra_path, list_path); + files = dir_read_file_list(database_path, extra_prefix, list_path); /* Restore directories in do_backup_instance way */ parray_qsort(files, pgFileComparePath); /* - * Make directories before backup + * Make extra directories before restore * and setup threads at the same time */ for (i = 0; i < parray_num(files); i++) @@ -492,11 +493,10 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if (backup_contains_extra(file->extradir, extra_dirs)) { char container_dir[MAXPGPATH]; - makeExtraDirPathByNum(container_dir, extra_path, + makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); - elog(VERBOSE, "Create directory \"%s\" NAME %s", - dir_name, file->name); + elog(VERBOSE, "Create directory \"%s\"", dir_name); join_path_components(dirpath, file->extradir, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } @@ -516,6 +516,7 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) arg->files = files; arg->backup = backup; arg->extra_dirs = extra_dirs; + arg->extra_prefix = extra_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -558,18 +559,18 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, extra_path, filelist_path); + files = dir_read_file_list(instance_config.pgdata, extra_prefix, filelist_path); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, instance_config.pgdata, true, true, false, false); + dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); @@ -678,7 +679,7 @@ restore_files(void *arg) else if (file->extra_dir_num) { if (backup_contains_extra(file->extradir, arguments->extra_dirs)) - copy_file(from_root, file->extradir, file); + copy_file(arguments->extra_prefix, file->extradir, file); } else copy_file(from_root, instance_config.pgdata, file); diff --git a/src/validate.c b/src/validate.c index 1b893ec4d..a241d7e52 100644 --- a/src/validate.c +++ b/src/validate.c @@ -43,7 +43,7 @@ void pgBackupValidate(pgBackup *backup) { char base_path[MAXPGPATH]; - char extra_path[MAXPGPATH]; + char extra_prefix[MAXPGPATH]; char path[MAXPGPATH]; parray *files; bool corrupted = false; @@ -83,9 +83,9 @@ pgBackupValidate(pgBackup *backup) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_path, lengthof(extra_path), EXTRA_DIR); + pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, extra_path, path); + files = dir_read_file_list(base_path, extra_prefix, path); /* setup threads */ for (i = 0; i < parray_num(files); i++) From 13c4df812e202bd3c3306c698f3efc493e889a46 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 14 Dec 2018 16:55:32 +0300 Subject: [PATCH 0162/2107] bugfix: undeclared compress_alg --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ada27e3e8..10478970e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -582,7 +582,7 @@ compress_init(void) if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ - if (compress_alg == ZLIB_COMPRESS) + if (instance_config.compress_alg == ZLIB_COMPRESS) elog(ERROR, "This build does not support zlib compression"); else #endif From 53cb7c1dd4f08ecb1ce1b93aeab03aca1803449e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 14 Dec 2018 18:41:23 +0300 Subject: [PATCH 0163/2107] Fix removing files which are not in from_backup file list --- src/merge.c | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/merge.c b/src/merge.c index b936469bd..545a05054 100644 --- a/src/merge.c +++ b/src/merge.c @@ -208,10 +208,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(to_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - to_files = dir_read_file_list(from_database_path, /* Use from_database_path - * so root path will be - * equal with 'files' */ - control_file); + to_files = dir_read_file_list(NULL, control_file); /* To delete from leaf, sort in reversed order */ parray_qsort(to_files, pgFileComparePathDesc); /* @@ -219,7 +216,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(from_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - files = dir_read_file_list(from_database_path, control_file); + files = dir_read_file_list(NULL, control_file); /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); @@ -331,8 +328,18 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { + char to_file_path[MAXPGPATH]; + char *prev_path; + + /* We need full path, file object has relative path */ + join_path_components(to_file_path, to_database_path, file->path); + prev_path = file->path; + file->path = to_file_path; + pgFileDelete(file); elog(VERBOSE, "Deleted \"%s\"", file->path); + + file->path = prev_path; } } @@ -378,13 +385,14 @@ merge_files(void *arg) pgBackup *from_backup = argument->from_backup; int i, num_files = parray_num(argument->files); - int to_root_len = strlen(argument->to_root); for (i = 0; i < num_files; i++) { pgFile *file = (pgFile *) parray_get(argument->files, i); pgFile *to_file; pgFile **res_file; + char from_file_path[MAXPGPATH]; + char *prev_file_path; if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -429,19 +437,23 @@ merge_files(void *arg) continue; } + /* We need to make full path, file object has relative path */ + join_path_components(from_file_path, argument->from_root, file->path); + prev_file_path = file->path; + file->path = from_file_path; + /* * Move the file. We need to decompress it and compress again if * necessary. */ - elog(VERBOSE, "Moving file \"%s\", is_datafile %d, is_cfs %d", + elog(VERBOSE, "Merging file \"%s\", is_datafile %d, is_cfs %d", file->path, file->is_database, file->is_cfs); if (file->is_datafile && !file->is_cfs) { - char to_path_tmp[MAXPGPATH]; /* Path of target file */ + char to_file_path[MAXPGPATH]; /* Path of target file */ - join_path_components(to_path_tmp, argument->to_root, - file->path + to_root_len + 1); + join_path_components(to_file_path, argument->to_root, prev_file_path); /* * We need more complicate algorithm if target file should be @@ -453,7 +465,7 @@ merge_files(void *arg) char tmp_file_path[MAXPGPATH]; char *prev_path; - snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_path_tmp); + snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_file_path); /* Start the magic */ @@ -479,7 +491,7 @@ merge_files(void *arg) * need the file in directory to_root. */ prev_path = to_file->path; - to_file->path = to_path_tmp; + to_file->path = to_file_path; /* Decompress target file into temporary one */ restore_data_file(tmp_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); @@ -494,7 +506,7 @@ merge_files(void *arg) false, parse_program_version(from_backup->program_version)); - elog(VERBOSE, "Compress file and save it to the directory \"%s\"", + elog(VERBOSE, "Compress file and save it into the directory \"%s\"", argument->to_root); /* Again we need to change path */ @@ -504,7 +516,7 @@ merge_files(void *arg) file->size = pgFileSize(file->path); /* Now we can compress the file */ backup_data_file(NULL, /* We shouldn't need 'arguments' here */ - to_path_tmp, file, + to_file_path, file, to_backup->start_lsn, to_backup->backup_mode, to_backup->compress_alg, @@ -523,7 +535,7 @@ merge_files(void *arg) else { /* We can merge in-place here */ - restore_data_file(to_path_tmp, file, + restore_data_file(to_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, true, parse_program_version(from_backup->program_version)); @@ -532,8 +544,8 @@ merge_files(void *arg) * We need to calculate write_size, restore_data_file() doesn't * do that. */ - file->write_size = pgFileSize(to_path_tmp); - file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL); + file->write_size = pgFileSize(to_file_path); + file->crc = pgFileGetCRC(to_file_path, true, true, NULL); } } else if (strcmp(file->name, "pg_control") == 0) @@ -548,8 +560,11 @@ merge_files(void *arg) file->compress_alg = to_backup->compress_alg; if (file->write_size != BYTES_INVALID) - elog(LOG, "Moved file \"%s\": " INT64_FORMAT " bytes", + elog(LOG, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); + + /* Restore relative path */ + file->path = prev_file_path; } /* Data files merging is successful */ From ac2236cc3df09e993cccaed4c42692eb1c46b411 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 14 Dec 2018 20:59:47 +0300 Subject: [PATCH 0164/2107] pgpro-2065. WIP. refactor code, add snapshot to amcheck. TODO fix eternal cycle in do_amchecl --- src/backup.c | 434 ++++++++++++++++++++++++++++++--------------- src/pg_probackup.h | 3 +- 2 files changed, 297 insertions(+), 140 deletions(-) diff --git a/src/backup.c b/src/backup.c index de992fc76..c453e37a3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -100,11 +100,16 @@ static void threads_conn_disconnect(bool fatal, void *userdata); static void pgdata_basic_setup(void); static void *backup_files(void *arg); -static void *check_files(void *arg); static void *remote_backup_files(void *arg); static void do_backup_instance(void); -static parray* get_index_list(void); + +static void do_block_validation(void); +static void do_amcheck(void); +static void *check_files(void *arg); +static void *check_indexes(void *arg); +static parray* get_index_list(PGresult* res_db, int db_number, + bool* first_db_with_amcheck, PGconn* db_conn); static bool amcheck_one_index(backup_files_arg *arguments, pg_indexEntry *ind); @@ -865,11 +870,8 @@ do_backup_instance(void) backup_files_list = NULL; } -/* Entry point of pg_probackup CHECKDB subcommand. */ -/* TODO consider moving some code common with do_backup_instance - * to separate function ot to pgdata_basic_setup */ -int -do_checkdb(bool do_block_validation, bool do_amcheck) +static void +do_block_validation(void) { int i; char database_path[MAXPGPATH]; @@ -878,54 +880,40 @@ do_checkdb(bool do_block_validation, bool do_amcheck) backup_files_arg *threads_args; bool backup_isok = true; - pgdata_basic_setup(); - - if (do_block_validation) - { - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - /* initialize backup list */ - backup_files_list = parray_new(); + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); - /* list files with the logical path. omit $PGDATA */ - dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); + /* initialize backup list */ + backup_files_list = parray_new(); - /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - * - * Sorted array is used at least in parse_backup_filelist_filenames(), - * extractPageMap(), make_pagemap_from_ptrack(). - */ - parray_qsort(backup_files_list, pgFileComparePath); - /* Extract information about files in backup_list parsing their names:*/ - parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); + /* list files with the logical path. omit $PGDATA */ + dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); - /* setup threads */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - pg_atomic_clear_flag(&file->lock); - } + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_backup_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(backup_files_list, pgFileComparePath); + /* Extract information about files in backup_list parsing their names:*/ + parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); - /* Sort by size for load balancing */ - parray_qsort(backup_files_list, pgFileCompareSize); + /* setup threads */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + pg_atomic_clear_flag(&file->lock); } - if (do_amcheck) - { - index_list = get_index_list(); + /* Sort by size for load balancing */ + parray_qsort(backup_files_list, pgFileCompareSize); - /* no need to setup threads. there's nothing to do */ - if ((!index_list) & (!do_block_validation)) - elog(ERROR, "Cannot perform 'checkdb --amcheck', since " - "this backup instance does not contain any databases with amcheck installed"); - } /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -938,7 +926,7 @@ do_checkdb(bool do_block_validation, bool do_amcheck) arg->from_root = instance_config.pgdata; arg->to_root = database_path; arg->files_list = backup_files_list; - arg->index_list = index_list; + arg->index_list = NULL; arg->prev_filelist = NULL; arg->prev_start_lsn = InvalidXLogRecPtr; arg->backup_conn = NULL; @@ -982,6 +970,122 @@ do_checkdb(bool do_block_validation, bool do_amcheck) parray_free(backup_files_list); backup_files_list = NULL; } +} + +static void +do_amcheck(void) +{ + int i; + char database_path[MAXPGPATH]; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + backup_files_arg *threads_args; + bool backup_isok = true; + const char *dbname; + PGresult *res_db; + int n_databases = 0; + bool first_db_with_amcheck = false; + PGconn *db_conn = NULL; + + pgBackupGetPath(¤t, database_path, lengthof(database_path), + DATABASE_DIR); + + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + n_databases = PQntuples(res_db); + elog(WARNING, "number of databases found: %d", n_databases); + + /* TODO Warn user that one connection is used for snaphot */ + if (num_threads > 1) + num_threads--; + + /* For each database check indexes. In parallel. */ + for(i = 0; i < n_databases; i++) + { + if (index_list != NULL) + free(index_list); + + elog(WARNING, "get index list for db %d of %d", i, n_databases); + index_list = get_index_list(res_db, i, + &first_db_with_amcheck, db_conn); + if (index_list == NULL) + { + if (db_conn) + pgut_disconnect(db_conn); + continue; + } + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + arg->from_root = instance_config.pgdata; + arg->to_root = database_path; + arg->files_list = NULL; + arg->index_list = index_list; + arg->prev_filelist = NULL; + arg->prev_start_lsn = InvalidXLogRecPtr; + arg->backup_conn = NULL; + arg->cancel_conn = NULL; + /* By default there are some error */ + arg->ret = 1; + } + + pgut_atexit_push(threads_conn_disconnect, NULL); + + /* TODO write better info message */ + elog(INFO, "Start checking data files"); + + /* Run threads */ + for (i = 0; i < num_threads; i++) + { + backup_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + pthread_create(&threads[i], NULL, check_indexes, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + backup_isok = false; + } + pgut_disconnect(db_conn); + } + + /* TODO write better info message */ + if (backup_isok) + elog(INFO, "Indexes are checked"); + else + elog(ERROR, "Indexs checking failed"); + + if (backup_files_list) + { + parray_walk(backup_files_list, pgFileFree); + parray_free(backup_files_list); + backup_files_list = NULL; + } +} + +/* Entry point of pg_probackup CHECKDB subcommand. */ +/* TODO consider moving some code common with do_backup_instance + * to separate function ot to pgdata_basic_setup */ +int +do_checkdb(bool need_block_validation, bool need_amcheck) +{ + pgdata_basic_setup(); + + if (need_block_validation) + do_block_validation(); + if (need_amcheck) + do_amcheck(); return 0; } @@ -2372,7 +2476,6 @@ check_files(void *arg) int i; backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = 0; - int n_indexes = 0; char dbname[NAMEDATALEN]; if (arguments->files_list) @@ -2440,11 +2543,29 @@ check_files(void *arg) elog(WARNING, "unexpected file type %d", buf.st_mode); } + /* Close connection */ + if (arguments->backup_conn) + pgut_disconnect(arguments->backup_conn); + + /* Data files transferring is successful */ + /* TODO where should we set arguments->ret to 1? */ + arguments->ret = 0; + + return NULL; +} + +static void * +check_indexes(void *arg) +{ + int i; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_indexes = 0; + /* Check indexes with amcheck */ if (arguments->index_list) n_indexes = parray_num(arguments->index_list); - /* TODO sort index_list by dbname */ + elog(WARNING, "n_indexes %d", n_indexes); for (i = 0; i < n_indexes; i++) { pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); @@ -2456,19 +2577,33 @@ check_files(void *arg) if (interrupted) elog(ERROR, "interrupted during checkdb --amcheck"); - elog(VERBOSE, "Checking index: \"%s\" ", ind->name); + elog(VERBOSE, "Checking index number %d of %d : \"%s\" ", i,n_indexes, ind->name); - /* reconnect, if index to check is in another database */ - if (strcmp((const char *) &dbname, ind->dbname) != 0) + if (arguments->backup_conn == NULL) { - if (arguments->backup_conn) - pgut_disconnect(arguments->backup_conn); + PGresult *res; + char *query; arguments->backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, ind->dbname, instance_config.pguser); arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL, false); + PQclear(res); + + query = palloc(strlen("SET TRANSACTION SNAPSHOT '' ") + strlen(ind->snapshot)); + sprintf(query, "SET TRANSACTION SNAPSHOT '%s'", ind->snapshot); + + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + query, 0, NULL, false); + PQclear(res); + + free(query); } amcheck_one_index(arguments, ind); @@ -3183,117 +3318,136 @@ pg_ptrack_get_block(backup_files_arg *arguments, /* Clear ptrack files in all databases of the instance we connected to */ static parray* -get_index_list(void) +get_index_list(PGresult *res_db, int db_number, + bool *first_db_with_amcheck, PGconn *db_conn) { - PGresult *res_db, - *res; - const char *dbname; + PGresult *res; char *nspname = NULL; - int i; + char *snapshot = NULL; Oid dbOid, tblspcOid; - char *params[2]; Oid indexrelid; - bool first_db_with_amcheck = true; + int i; - params[0] = palloc(64); - params[1] = palloc(64); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", - 0, NULL); + dbname = PQgetvalue(res_db, db_number, 0); + if (strcmp(dbname, "template0") == 0) + return NULL; - for(i = 0; i < PQntuples(res_db); i++) - { - PGconn *tmp_conn; + elog(WARNING, "get_index_list db_number %d dbname %s", db_number, dbname); - dbname = PQgetvalue(res_db, i, 0); - if (strcmp(dbname, "template0") == 0) - continue; + dbOid = atoi(PQgetvalue(res_db, db_number, 1)); + tblspcOid = atoi(PQgetvalue(res_db, db_number, 2)); - dbOid = atoi(PQgetvalue(res_db, i, 1)); - tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + db_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + dbname, + instance_config.pguser); - tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - dbname, - instance_config.pguser); + res = pgut_execute(db_conn, "select extname, nspname, extversion from pg_namespace " + "n join pg_extension e on n.oid=e.extnamespace where e.extname='amcheck'", + 0, NULL); - res = pgut_execute(tmp_conn, "select extname, nspname, extversion from pg_namespace " - "n join pg_extension e on n.oid=e.extnamespace where e.extname='amcheck'", - 0, NULL); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(ERROR, "cannot check if amcheck is installed in database %s: %s", + dbname, PQerrorMessage(db_conn)); + } - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - elog(ERROR, "cannot check if amcheck is installed in database %s: %s", - dbname, PQerrorMessage(tmp_conn)); - } + if (PQntuples(res) < 1) + { + elog(WARNING, "extension amcheck is not installed in database %s", dbname); + return NULL; + } + + nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); + strcpy(nspname, PQgetvalue(res, 0, 1)); + elog(WARNING, "index_list for db %s nspname %s", dbname, nspname); + + /* + * In order to avoid duplicates, select global indexes + * (tablespace pg_global with oid 1664) only once + */ + if (*first_db_with_amcheck) + { + elog(WARNING, "FIRST AMCHECK for db %s nspname %s", dbname, nspname); + res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); + PQclear(res); + + res = pgut_execute(db_conn, "SELECT pg_export_snapshot();", 0, NULL); if (PQntuples(res) < 1) - { - elog(WARNING, "extension amcheck is not installed in database %s", dbname); - continue; - } + elog(ERROR, "Failed to export snapshot for amcheck in database %s", dbname); - if (nspname) - free(nspname); + snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); + strcpy(snapshot, PQgetvalue(res, 0, 0)); - nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); - strcpy(nspname, PQgetvalue(res, 0, 1)); + elog(WARNING, "exported snapshot '%s'", snapshot); + PQclear(res); + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" + " FROM pg_index idx " + " JOIN pg_class cls ON cls.oid=idx.indexrelid " + " JOIN pg_am am ON am.oid=cls.relam " + " WHERE am.amname='btree' AND cls.relpersistence != 't'" + " AND idx.indisready AND idx.indisvalid; ", 0, NULL); + *first_db_with_amcheck = false; + } + else + { + elog(WARNING, "NOT FIRST AMCHECK for db %s nspname %s", dbname, nspname); + + res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); PQclear(res); - /* - * In order to avoid duplicates, select global indexes - * (tablespace pg_global with oid 1664) only once - */ - if (first_db_with_amcheck) - { - res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" - " FROM pg_index idx " - " JOIN pg_class cls ON cls.oid=idx.indexrelid " - " JOIN pg_am am ON am.oid=cls.relam " - " WHERE am.amname='btree' AND cls.relpersistence != 't'" - " AND idx.indisready AND idx.indisvalid; ", 0, NULL); - first_db_with_amcheck = false; - } - else - res = pgut_execute(tmp_conn, "SELECT cls.oid, cls.relname" - " FROM pg_index idx " - " JOIN pg_class cls ON cls.oid=idx.indexrelid " - " JOIN pg_am am ON am.oid=cls.relam " - " WHERE am.amname='btree' AND cls.relpersistence != 't'" - " AND idx.indisready AND idx.indisvalid AND cls.reltablespace!=1664; ", 0, NULL); + res = pgut_execute(db_conn, "SELECT pg_export_snapshot();", 0, NULL); + + if (PQntuples(res) < 1) + elog(ERROR, "Failed to export snapshot for amcheck in database %s", dbname); + if (snapshot) + free(snapshot); + snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); + strcpy(snapshot, PQgetvalue(res, 0, 0)); - /* add info needed to check indexes into index_list */ - for(i = 0; i < PQntuples(res); i++) - { - pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); - char *name = NULL; + elog(WARNING, "exported snapshot '%s'", snapshot); + PQclear(res); - ind->indexrelid = atoi(PQgetvalue(res, i, 0)); - name = PQgetvalue(res, i, 1); - ind->name = pgut_malloc(strlen(name) + 1); - strcpy(ind->name, name); /* enough buffer size guaranteed */ + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" + " FROM pg_index idx " + " JOIN pg_class cls ON cls.oid=idx.indexrelid " + " JOIN pg_am am ON am.oid=cls.relam " + " WHERE am.amname='btree' AND cls.relpersistence != 't'" + " AND idx.indisready AND idx.indisvalid AND cls.reltablespace!=1664; ", 0, NULL); + } - ind->dbname = pgut_malloc(strlen(dbname) + 1); - strcpy(ind->dbname, dbname); /* enough buffer size guaranteed */ + /* add info needed to check indexes into index_list */ + for(i = 0; i < PQntuples(res); i++) + { + pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); + char *name = NULL; - ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); - strcpy(ind->amcheck_nspname, nspname); - pg_atomic_clear_flag(&ind->lock); + ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + name = PQgetvalue(res, i, 1); + ind->name = pgut_malloc(strlen(name) + 1); + strcpy(ind->name, name); /* enough buffer size guaranteed */ - if (index_list == NULL) - index_list = parray_new(); + ind->dbname = pgut_malloc(strlen(dbname) + 1); + strcpy(ind->dbname, dbname); - parray_append(index_list, ind); - } + ind->snapshot = pgut_malloc(strlen(snapshot) + 1); + strcpy(ind->snapshot, snapshot); - PQclear(res); - pgut_disconnect(tmp_conn); + ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); + strcpy(ind->amcheck_nspname, nspname); + pg_atomic_clear_flag(&ind->lock); + + if (index_list == NULL) + index_list = parray_new(); + +// elog(WARNING, "add to index_list index '%s' dbname '%s'",ind->name, ind->dbname); + parray_append(index_list, ind); } - pfree(params[0]); - pfree(params[1]); - PQclear(res_db); + PQclear(res); return index_list; } @@ -3311,6 +3465,8 @@ amcheck_one_index(backup_files_arg *arguments, sprintf(params[0], "%i", ind->indexrelid); +// elog(WARNING, "amcheck_one_index %s", ind->name); + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index da146caf0..ca0e7382a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -134,6 +134,7 @@ typedef struct pg_indexEntry Oid indexrelid; char *name; char *dbname; + char *snapshot; /* snapshot for index check */ char *amcheck_nspname; /* schema where amcheck extention is located */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ } pg_indexEntry; @@ -401,7 +402,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); -extern int do_checkdb(bool do_block_validation, bool do_amcheck); +extern int do_checkdb(bool need_block_validation, bool need_amcheck); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, From b99097a71b150b4a15f11c001c22d1839acaedb0 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 17 Dec 2018 16:03:58 +0300 Subject: [PATCH 0165/2107] pgpro-2065. fix cycle in do_amcheck, remove unused variables and debug messages. TODO: code cleanup, fix exit statuses for threads --- src/backup.c | 59 ++++++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/src/backup.c b/src/backup.c index c453e37a3..11a99a959 100644 --- a/src/backup.c +++ b/src/backup.c @@ -981,10 +981,9 @@ do_amcheck(void) pthread_t *threads; backup_files_arg *threads_args; bool backup_isok = true; - const char *dbname; PGresult *res_db; int n_databases = 0; - bool first_db_with_amcheck = false; + bool first_db_with_amcheck = true; PGconn *db_conn = NULL; pgBackupGetPath(¤t, database_path, lengthof(database_path), @@ -993,7 +992,6 @@ do_amcheck(void) res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", 0, NULL); n_databases = PQntuples(res_db); - elog(WARNING, "number of databases found: %d", n_databases); /* TODO Warn user that one connection is used for snaphot */ if (num_threads > 1) @@ -1002,10 +1000,14 @@ do_amcheck(void) /* For each database check indexes. In parallel. */ for(i = 0; i < n_databases; i++) { + int j; + if (index_list != NULL) + { free(index_list); + index_list = NULL; + } - elog(WARNING, "get index list for db %d of %d", i, n_databases); index_list = get_index_list(res_db, i, &first_db_with_amcheck, db_conn); if (index_list == NULL) @@ -1019,9 +1021,9 @@ do_amcheck(void) threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); - for (i = 0; i < num_threads; i++) + for (j = 0; j < num_threads; j++) { - backup_files_arg *arg = &(threads_args[i]); + backup_files_arg *arg = &(threads_args[j]); arg->from_root = instance_config.pgdata; arg->to_root = database_path; @@ -1037,24 +1039,23 @@ do_amcheck(void) pgut_atexit_push(threads_conn_disconnect, NULL); - /* TODO write better info message */ - elog(INFO, "Start checking data files"); + elog(INFO, "Start checking indexes with amcheck"); /* Run threads */ - for (i = 0; i < num_threads; i++) + for (j = 0; j < num_threads; j++) { - backup_files_arg *arg = &(threads_args[i]); + backup_files_arg *arg = &(threads_args[j]); - elog(VERBOSE, "Start thread num: %i", i); + elog(VERBOSE, "Start thread num: %i", j); - pthread_create(&threads[i], NULL, check_indexes, arg); + pthread_create(&threads[j], NULL, check_indexes, arg); } /* Wait threads */ - for (i = 0; i < num_threads; i++) + for (j = 0; j < num_threads; j++) { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) + pthread_join(threads[j], NULL); + if (threads_args[j].ret == 1) backup_isok = false; } pgut_disconnect(db_conn); @@ -1062,7 +1063,7 @@ do_amcheck(void) /* TODO write better info message */ if (backup_isok) - elog(INFO, "Indexes are checked"); + elog(INFO, "Indexes are checked"); else elog(ERROR, "Indexs checking failed"); @@ -2476,7 +2477,6 @@ check_files(void *arg) int i; backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = 0; - char dbname[NAMEDATALEN]; if (arguments->files_list) n_backup_files_list = parray_num(arguments->files_list); @@ -2536,7 +2536,8 @@ check_files(void *arg) join_path_components(to_path, arguments->to_root, file->path + strlen(arguments->from_root) + 1); - check_data_file(arguments, file); + if (!check_data_file(arguments, file)) + arguments->ret = 1; } } else @@ -2565,7 +2566,6 @@ check_indexes(void *arg) if (arguments->index_list) n_indexes = parray_num(arguments->index_list); - elog(WARNING, "n_indexes %d", n_indexes); for (i = 0; i < n_indexes; i++) { pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); @@ -2606,14 +2606,15 @@ check_indexes(void *arg) free(query); } - amcheck_one_index(arguments, ind); + /* remember that we have a failed check */ + if (!amcheck_one_index(arguments, ind)) + arguments->ret = 1; } /* Close connection */ if (arguments->backup_conn) pgut_disconnect(arguments->backup_conn); - /* Data files transferring is successful */ /* TODO where should we set arguments->ret to 1? */ arguments->ret = 0; @@ -3324,19 +3325,12 @@ get_index_list(PGresult *res_db, int db_number, PGresult *res; char *nspname = NULL; char *snapshot = NULL; - Oid dbOid, tblspcOid; - Oid indexrelid; int i; dbname = PQgetvalue(res_db, db_number, 0); if (strcmp(dbname, "template0") == 0) return NULL; - elog(WARNING, "get_index_list db_number %d dbname %s", db_number, dbname); - - dbOid = atoi(PQgetvalue(res_db, db_number, 1)); - tblspcOid = atoi(PQgetvalue(res_db, db_number, 2)); - db_conn = pgut_connect(instance_config.pghost, instance_config.pgport, dbname, instance_config.pguser); @@ -3360,7 +3354,6 @@ get_index_list(PGresult *res_db, int db_number, nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); strcpy(nspname, PQgetvalue(res, 0, 1)); - elog(WARNING, "index_list for db %s nspname %s", dbname, nspname); /* * In order to avoid duplicates, select global indexes @@ -3368,7 +3361,6 @@ get_index_list(PGresult *res_db, int db_number, */ if (*first_db_with_amcheck) { - elog(WARNING, "FIRST AMCHECK for db %s nspname %s", dbname, nspname); res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); PQclear(res); @@ -3380,7 +3372,6 @@ get_index_list(PGresult *res_db, int db_number, snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); strcpy(snapshot, PQgetvalue(res, 0, 0)); - elog(WARNING, "exported snapshot '%s'", snapshot); PQclear(res); res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" @@ -3393,8 +3384,6 @@ get_index_list(PGresult *res_db, int db_number, } else { - elog(WARNING, "NOT FIRST AMCHECK for db %s nspname %s", dbname, nspname); - res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); PQclear(res); @@ -3408,7 +3397,6 @@ get_index_list(PGresult *res_db, int db_number, snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); strcpy(snapshot, PQgetvalue(res, 0, 0)); - elog(WARNING, "exported snapshot '%s'", snapshot); PQclear(res); res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" @@ -3459,13 +3447,12 @@ amcheck_one_index(backup_files_arg *arguments, { PGresult *res; char *params[1]; - char *result; char *query; params[0] = palloc(64); sprintf(params[0], "%i", ind->indexrelid); -// elog(WARNING, "amcheck_one_index %s", ind->name); + elog(VERBOSE, "amcheck index: '%s'", ind->name); query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); From 4650c6f6f80f94e17a5e958ccd868927d7e1653a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 17 Dec 2018 17:44:42 +0300 Subject: [PATCH 0166/2107] tests: changes for windows compatibility(work in progress) --- tests/helpers/ptrack_helpers.py | 295 ++++++++++++++++---------------- 1 file changed, 143 insertions(+), 152 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 8ea90c240..232daa39d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -122,15 +122,15 @@ def slow_start(self, replica=False): while True: try: self.poll_query_until( - "postgres", - "SELECT not pg_is_in_recovery()") + 'postgres', + 'SELECT not pg_is_in_recovery()') break except Exception as e: continue else: self.poll_query_until( - "postgres", - "SELECT pg_is_in_recovery()") + 'postgres', + 'SELECT pg_is_in_recovery()') # while True: # try: @@ -155,18 +155,18 @@ def __init__(self, *args, **kwargs): self.test_env = os.environ.copy() envs_list = [ - "LANGUAGE", - "LC_ALL", - "PGCONNECT_TIMEOUT", - "PGDATA", - "PGDATABASE", - "PGHOSTADDR", - "PGREQUIRESSL", - "PGSERVICE", - "PGSSLMODE", - "PGUSER", - "PGPORT", - "PGHOST" + 'LANGUAGE', + 'LC_ALL', + 'PGCONNECT_TIMEOUT', + 'PGDATA', + 'PGDATABASE', + 'PGHOSTADDR', + 'PGREQUIRESSL', + 'PGSERVICE', + 'PGSSLMODE', + 'PGUSER', + 'PGPORT', + 'PGHOST' ] for e in envs_list: @@ -175,8 +175,8 @@ def __init__(self, *args, **kwargs): except: pass - self.test_env["LC_MESSAGES"] = "C" - self.test_env["LC_TIME"] = "C" + self.test_env['LC_MESSAGES'] = 'C' + self.test_env['LC_TIME'] = 'C' self.paranoia = False if 'PG_PROBACKUP_PARANOIA' in self.test_env: @@ -210,19 +210,19 @@ def __init__(self, *args, **kwargs): self.user = self.get_username() self.probackup_path = None - if "PGPROBACKUPBIN" in self.test_env: + if 'PGPROBACKUPBIN' in self.test_env: if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and - os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) + os.path.isfile(self.test_env['PGPROBACKUPBIN']) and + os.access(self.test_env['PGPROBACKUPBIN'], os.X_OK) ): - self.probackup_path = self.test_env["PGPROBACKUPBIN"] + self.probackup_path = self.test_env['PGPROBACKUPBIN'] else: if self.verbose: print('PGPROBACKUPBIN is not an executable file') if not self.probackup_path: probackup_path_tmp = os.path.join( - testgres.get_pg_config()["BINDIR"], 'pg_probackup') + testgres.get_pg_config()['BINDIR'], 'pg_probackup') if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): @@ -233,7 +233,7 @@ def __init__(self, *args, **kwargs): if not self.probackup_path: probackup_path_tmp = os.path.abspath(os.path.join( - self.dir_path, "../pg_probackup")) + self.dir_path, '../pg_probackup')) if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): @@ -247,16 +247,16 @@ def __init__(self, *args, **kwargs): exit(1) os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ":" + os.environ['PATH'] + self.probackup_path) + ':' + os.environ['PATH'] self.probackup_old_path = None - if "PGPROBACKUPBIN_OLD" in self.test_env: + if 'PGPROBACKUPBIN_OLD' in self.test_env: if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN_OLD"]) and - os.access(self.test_env["PGPROBACKUPBIN_OLD"], os.X_OK) + os.path.isfile(self.test_env['PGPROBACKUPBIN_OLD']) and + os.access(self.test_env['PGPROBACKUPBIN_OLD'], os.X_OK) ): - self.probackup_old_path = self.test_env["PGPROBACKUPBIN_OLD"] + self.probackup_old_path = self.test_env['PGPROBACKUPBIN_OLD'] else: if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') @@ -280,40 +280,37 @@ def make_simple_node( initdb_params=initdb_params, allow_streaming=set_replication) # Sane default parameters - node.append_conf("postgresql.auto.conf", "max_connections = 100") - node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB") - node.append_conf("postgresql.auto.conf", "fsync = on") - node.append_conf("postgresql.auto.conf", "wal_level = logical") - node.append_conf("postgresql.auto.conf", "hot_standby = 'off'") + node.append_conf('postgresql.auto.conf', 'max_connections = 100') + node.append_conf('postgresql.auto.conf', 'shared_buffers = 10MB') + node.append_conf('postgresql.auto.conf', 'fsync = on') + node.append_conf('postgresql.auto.conf', 'wal_level = logical') + node.append_conf('postgresql.auto.conf', 'hot_standby = off') node.append_conf( - "postgresql.auto.conf", "log_line_prefix = '%t [%p]: [%l-1] '") - node.append_conf("postgresql.auto.conf", "log_statement = none") - node.append_conf("postgresql.auto.conf", "log_duration = on") + 'postgresql.auto.conf', "log_line_prefix = '%t [%p]: [%l-1] '") + node.append_conf('postgresql.auto.conf', 'log_statement = none') + node.append_conf('postgresql.auto.conf', 'log_duration = on') node.append_conf( - "postgresql.auto.conf", "log_min_duration_statement = 0") - node.append_conf("postgresql.auto.conf", "log_connections = on") - node.append_conf("postgresql.auto.conf", "log_disconnections = on") + 'postgresql.auto.conf', 'log_min_duration_statement = 0') + node.append_conf('postgresql.auto.conf', 'log_connections = on') + node.append_conf('postgresql.auto.conf', 'log_disconnections = on') # Apply given parameters for key, value in six.iteritems(pg_options): - node.append_conf("postgresql.auto.conf", "%s = %s" % (key, value)) + node.append_conf('postgresql.auto.conf', '%s = %s' % (key, value)) # Allow replication in pg_hba.conf if set_replication: node.append_conf( - "pg_hba.conf", - "local replication all trust\n") - node.append_conf( - "postgresql.auto.conf", - "max_wal_senders = 10") + 'postgresql.auto.conf', + 'max_wal_senders = 10') return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( - "postgres", - "select exists" + 'postgres', + 'select exists' " (select 1 from pg_tablespace where spcname = '{0}')".format( tblspc_name) ) @@ -329,11 +326,11 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( tblspc_name, tblspc_path) if cfs: - cmd += " with (compression=true)" + cmd += ' with (compression=true)' if not os.path.exists(tblspc_path): os.makedirs(tblspc_path) - res = node.safe_psql("postgres", cmd) + res = node.safe_psql('postgres', cmd) # Check that tablespace was successfully created # self.assertEqual( # res[0], 0, @@ -344,13 +341,13 @@ def get_tblspace_path(self, node, tblspc_name): def get_fork_size(self, node, fork_name): return node.execute( - "postgres", + 'postgres', "select pg_relation_size('{0}')/8192".format(fork_name))[0][0] def get_fork_path(self, node, fork_name): return os.path.join( node.base_dir, 'data', node.execute( - "postgres", + 'postgres', "select pg_relation_filepath('{0}')".format( fork_name))[0][0] ) @@ -378,7 +375,7 @@ def get_md5_per_page_for_fork(self, file, size_in_pages): end_page = pages_per_segment[segment_number] else: file_desc = os.open( - file+".{0}".format(segment_number), os.O_RDONLY + file+'.{0}'.format(segment_number), os.O_RDONLY ) start_page = max(md5_per_page)+1 end_page = end_page + pages_per_segment[segment_number] @@ -481,8 +478,8 @@ def check_ptrack_sanity(self, idx_dict): idx_dict['ptrack'][PageNum]) ) print( - " Old checksumm: {0}\n" - " New checksumm: {1}".format( + ' Old checksumm: {0}\n' + ' New checksumm: {1}'.format( idx_dict['old_pages'][PageNum], idx_dict['new_pages'][PageNum]) ) @@ -545,7 +542,7 @@ def check_ptrack_clean(self, idx_dict, size): def run_pb(self, command, async=False, gdb=False, old_binary=False): if not self.probackup_old_path and old_binary: - print("PGPROBACKUPBIN_OLD is not set") + print('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -571,7 +568,7 @@ def run_pb(self, command, async=False, gdb=False, old_binary=False): [binary_path] + command, stderr=subprocess.STDOUT, env=self.test_env - ).decode("utf-8") + ).decode('utf-8') if command[0] == 'backup': # return backup ID for line in self.output.splitlines(): @@ -580,7 +577,7 @@ def run_pb(self, command, async=False, gdb=False, old_binary=False): else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode("utf-8"), self.cmd) + raise ProbackupException(e.output.decode('utf-8'), self.cmd) def run_binary(self, command, async=False): if self.verbose: @@ -599,18 +596,18 @@ def run_binary(self, command, async=False): command, stderr=subprocess.STDOUT, env=self.test_env - ).decode("utf-8") + ).decode('utf-8') return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode("utf-8"), command) + raise ProbackupException(e.output.decode('utf-8'), command) def init_pb(self, backup_dir, old_binary=False): shutil.rmtree(backup_dir, ignore_errors=True) return self.run_pb([ - "init", - "-B", backup_dir + 'init', + '-B', backup_dir ], old_binary=old_binary ) @@ -618,10 +615,10 @@ def init_pb(self, backup_dir, old_binary=False): def add_instance(self, backup_dir, instance, node, old_binary=False): return self.run_pb([ - "add-instance", - "--instance={0}".format(instance), - "-B", backup_dir, - "-D", node.data_dir + 'add-instance', + '--instance={0}'.format(instance), + '-B', backup_dir, + '-D', node.data_dir ], old_binary=old_binary ) @@ -629,9 +626,9 @@ def add_instance(self, backup_dir, instance, node, old_binary=False): def del_instance(self, backup_dir, instance, old_binary=False): return self.run_pb([ - "del-instance", - "--instance={0}".format(instance), - "-B", backup_dir + 'del-instance', + '--instance={0}'.format(instance), + '-B', backup_dir ], old_binary=old_binary ) @@ -641,7 +638,7 @@ def clean_pb(self, backup_dir): def backup_node( self, backup_dir, instance, node, data_dir=False, - backup_type="full", options=[], async=False, gdb=False, + backup_type='full', options=[], async=False, gdb=False, old_binary=False ): if not node and not data_dir: @@ -655,15 +652,15 @@ def backup_node( pgdata = data_dir cmd_list = [ - "backup", - "-B", backup_dir, + 'backup', + '-B', backup_dir, # "-D", pgdata, - "-p", "%i" % node.port, - "-d", "postgres", - "--instance={0}".format(instance) + '-p', '%i' % node.port, + '-d', 'postgres', + '--instance={0}'.format(instance) ] if backup_type: - cmd_list += ["-b", backup_type] + cmd_list += ['-b', backup_type] return self.run_pb(cmd_list + options, async, gdb, old_binary) @@ -671,10 +668,10 @@ def merge_backup( self, backup_dir, instance, backup_id, async=False, gdb=False, old_binary=False, options=[]): cmd_list = [ - "merge", - "-B", backup_dir, - "--instance={0}".format(instance), - "-i", backup_id + 'merge', + '-B', backup_dir, + '--instance={0}'.format(instance), + '-i', backup_id ] return self.run_pb(cmd_list + options, async, gdb, old_binary) @@ -687,13 +684,13 @@ def restore_node( data_dir = node.data_dir cmd_list = [ - "restore", - "-B", backup_dir, - "-D", data_dir, - "--instance={0}".format(instance) + 'restore', + '-B', backup_dir, + '-D', data_dir, + '--instance={0}'.format(instance) ] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) @@ -705,17 +702,17 @@ def show_pb( backup_list = [] specific_record = {} cmd_list = [ - "show", - "-B", backup_dir, + 'show', + '-B', backup_dir, ] if instance: - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] if as_json: - cmd_list += ["--format=json"] + cmd_list += ['--format=json'] if as_text: # You should print it when calling as_text=true @@ -750,7 +747,7 @@ def show_pb( # inverse list so oldest record come first body = body[::-1] # split string in list with string for every header element - header_split = re.split(" +", header) + header_split = re.split(' +', header) # Remove empty items for i in header_split: if i == '': @@ -762,7 +759,7 @@ def show_pb( for backup_record in body: backup_record = backup_record.rstrip() # split list with str for every backup record element - backup_record_split = re.split(" +", backup_record) + backup_record_split = re.split(' +', backup_record) # Remove empty items for i in backup_record_split: if i == '': @@ -787,7 +784,7 @@ def show_pb( ] # print sanitized_show for line in sanitized_show: - name, var = line.partition(" = ")[::2] + name, var = line.partition(' = ')[::2] var = var.strip('"') var = var.strip("'") specific_record[name.strip()] = var @@ -799,13 +796,13 @@ def validate_pb( ): cmd_list = [ - "validate", - "-B", backup_dir + 'validate', + '-B', backup_dir ] if instance: - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) @@ -813,48 +810,48 @@ def delete_pb( self, backup_dir, instance, backup_id=None, options=[], old_binary=False): cmd_list = [ - "delete", - "-B", backup_dir + 'delete', + '-B', backup_dir ] - cmd_list += ["--instance={0}".format(instance)] + cmd_list += ['--instance={0}'.format(instance)] if backup_id: - cmd_list += ["-i", backup_id] + cmd_list += ['-i', backup_id] return self.run_pb(cmd_list + options, old_binary=old_binary) def delete_expired( self, backup_dir, instance, options=[], old_binary=False): cmd_list = [ - "delete", "--expired", "--wal", - "-B", backup_dir, - "--instance={0}".format(instance) + 'delete', '--expired', '--wal', + '-B', backup_dir, + '--instance={0}'.format(instance) ] return self.run_pb(cmd_list + options, old_binary=old_binary) def show_config(self, backup_dir, instance, old_binary=False): out_dict = {} cmd_list = [ - "show-config", - "-B", backup_dir, - "--instance={0}".format(instance) + 'show-config', + '-B', backup_dir, + '--instance={0}'.format(instance) ] res = self.run_pb(cmd_list, old_binary=old_binary).splitlines() for line in res: if not line.startswith('#'): - name, var = line.partition(" = ")[::2] + name, var = line.partition(' = ')[::2] out_dict[name] = var return out_dict def get_recovery_conf(self, node): out_dict = {} with open( - os.path.join(node.data_dir, "recovery.conf"), "r" + os.path.join(node.data_dir, 'recovery.conf'), 'r' ) as recovery_conf: for line in recovery_conf: try: - key, value = line.split("=") + key, value = line.split('=') except: continue out_dict[key.strip()] = value.strip(" '").replace("'\n", "") @@ -870,35 +867,29 @@ def set_archiving( else: archive_mode = 'on' - # node.append_conf( - # "postgresql.auto.conf", - # "wal_level = archive" - # ) node.append_conf( - "postgresql.auto.conf", - "archive_mode = {0}".format(archive_mode) + 'postgresql.auto.conf', + 'archive_mode = {0}'.format(archive_mode) ) - archive_command = "{0} archive-push -B {1} --instance={2} ".format( + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( self.probackup_path, backup_dir, instance) - if os.name == 'posix': - if self.archive_compress or compress: - archive_command = archive_command + "--compress " + if self.archive_compress or compress: + archive_command = archive_command + '--compress ' - if overwrite: - archive_command = archive_command + "--overwrite " + if overwrite: + archive_command = archive_command + '--overwrite ' - archive_command = archive_command + "--wal-file-path %p --wal-file-name %f" + if os.name == 'posix': + archive_command = archive_command + '--wal-file-path %p --wal-file-name %f' + + elif os.name == 'nt': + archive_command = archive_command + '--wal-file-path "%p" --wal-file-name "%f"' node.append_conf( - "postgresql.auto.conf", + 'postgresql.auto.conf', "archive_command = '{0}'".format( archive_command)) - # elif os.name == 'nt': - # node.append_conf( - # "postgresql.auto.conf", - # "archive_command = 'copy %p {0}\\%f'".format(archive_dir) - # ) def set_replica( self, master, replica, @@ -906,18 +897,18 @@ def set_replica( synchronous=False ): replica.append_conf( - "postgresql.auto.conf", "port = {0}".format(replica.port)) + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - replica.append_conf('recovery.conf', "standby_mode = 'on'") + replica.append_conf('recovery.conf', 'standby_mode = on') replica.append_conf( - "recovery.conf", + 'recovery.conf', "primary_conninfo = 'user={0} port={1} application_name={2}" " sslmode=prefer sslcompression=1'".format( self.user, master.port, replica_name) ) if synchronous: master.append_conf( - "postgresql.auto.conf", + 'postgresql.auto.conf', "synchronous_standby_names='{0}'".format(replica_name) ) master.append_conf( @@ -927,7 +918,7 @@ def set_replica( master.reload() def wrong_wal_clean(self, node, wal_size): - wals_dir = os.path.join(self.backup_dir(node), "wal") + wals_dir = os.path.join(self.backup_dir(node), 'wal') wals = [ f for f in os.listdir(wals_dir) if os.path.isfile( os.path.join(wals_dir, f)) @@ -939,25 +930,25 @@ def wrong_wal_clean(self, node, wal_size): def guc_wal_segment_size(self, node): var = node.execute( - "postgres", + 'postgres', "select setting from pg_settings where name = 'wal_segment_size'" ) return int(var[0][0]) * self.guc_wal_block_size(node) def guc_wal_block_size(self, node): var = node.execute( - "postgres", + 'postgres', "select setting from pg_settings where name = 'wal_block_size'" ) return int(var[0][0]) def get_pgpro_edition(self, node): if node.execute( - "postgres", + 'postgres', "select exists (select 1 from" " pg_proc where proname = 'pgpro_edition')" )[0][0]: - var = node.execute("postgres", "select pgpro_edition()") + var = node.execute('postgres', 'select pgpro_edition()') return str(var[0][0]) else: return False @@ -969,9 +960,9 @@ def get_username(self): def version_to_num(self, version): if not version: return 0 - parts = version.split(".") + parts = version.split('.') while len(parts) < 3: - parts.append("0") + parts.append('0') num = 0 for part in parts: num = num * 100 + int(re.sub("[^\d]", "", part)) @@ -986,25 +977,25 @@ def switch_wal_segment(self, node): """ if isinstance(node, testgres.PostgresNode): if self.version_to_num( - node.safe_psql("postgres", "show server_version") + node.safe_psql('postgres', 'show server_version') ) >= self.version_to_num('10.0'): - node.safe_psql("postgres", "select pg_switch_wal()") + node.safe_psql('postgres', 'select pg_switch_wal()') else: - node.safe_psql("postgres", "select pg_switch_xlog()") + node.safe_psql('postgres', 'select pg_switch_xlog()') else: if self.version_to_num( - node.execute("show server_version")[0][0] + node.execute('show server_version')[0][0] ) >= self.version_to_num('10.0'): - node.execute("select pg_switch_wal()") + node.execute('select pg_switch_wal()') else: - node.execute("select pg_switch_xlog()") + node.execute('select pg_switch_xlog()') def wait_until_replica_catch_with_master(self, master, replica): if self.version_to_num( master.safe_psql( - "postgres", - "show server_version")) >= self.version_to_num('10.0'): + 'postgres', + 'show server_version')) >= self.version_to_num('10.0'): master_function = 'pg_catalog.pg_current_wal_lsn()' replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' else: @@ -1022,7 +1013,7 @@ def wait_until_replica_catch_with_master(self, master, replica): def get_version(self, node): return self.version_to_num( - testgres.get_pg_config()["VERSION"].split(" ")[1]) + testgres.get_pg_config()['VERSION'].split(" ")[1]) def get_bin_path(self, binary): return testgres.get_bin_path(binary) @@ -1217,7 +1208,7 @@ def get_async_connect(self, database=None, host=None, port=5432): host = '127.0.0.1' return psycopg2.connect( - database="postgres", + database='postgres', host='127.0.0.1', port=port, async=True @@ -1233,7 +1224,7 @@ def wait(self, connection): elif state == psycopg2.extensions.POLL_READ: select.select([connection.fileno()], [], []) else: - raise psycopg2.OperationalError("poll() returned %s" % state) + raise psycopg2.OperationalError('poll() returned %s' % state) def gdb_attach(self, pid): return GDBobj([str(pid)], self.verbose, attach=True) @@ -1254,7 +1245,7 @@ def __init__(self, cmd, verbose, attach=False): # Check gdb presense try: gdb_version, _ = subprocess.Popen( - ["gdb", "--version"], + ['gdb', '--version'], stdout=subprocess.PIPE ).communicate() except OSError: From c7d7ceb5bbc0042ed055ab9cf2d2a673d4cb6fb7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 17 Dec 2018 17:54:38 +0300 Subject: [PATCH 0167/2107] tests: windows compatibility(WIP) --- tests/helpers/ptrack_helpers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 232daa39d..516017324 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -7,7 +7,7 @@ import testgres import hashlib import re -import pwd +import getpass import select import psycopg2 from time import sleep @@ -89,8 +89,14 @@ def dir_files(base_dir): def is_enterprise(): # pg_config --help + if os.name == 'posix': + cmd = [os.environ['PG_CONFIG'], '--help'] + + elif os.name == 'nt': + cmd = [[os.environ['PG_CONFIG']], ['--help']] + p = subprocess.Popen( - [os.environ['PG_CONFIG'], '--help'], + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -955,7 +961,7 @@ def get_pgpro_edition(self, node): def get_username(self): """ Returns current user name """ - return pwd.getpwuid(os.getuid())[0] + return getpass.getuser() def version_to_num(self, version): if not version: From d557b40f431d3794ca46c157e3642664fec7b45f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 18 Dec 2018 10:49:14 +0300 Subject: [PATCH 0168/2107] PGPRO-1918: Remove instance-level lock, introduce backup-level lock --- src/backup.c | 5 +- src/catalog.c | 124 ++++++++++++++++++++++++++++++--------------- src/delete.c | 12 ++--- src/merge.c | 4 +- src/pg_probackup.h | 6 ++- src/restore.c | 11 ++-- src/validate.c | 9 ++-- 7 files changed, 107 insertions(+), 64 deletions(-) diff --git a/src/backup.c b/src/backup.c index 7046d1d06..b3ff0d002 100644 --- a/src/backup.c +++ b/src/backup.c @@ -943,9 +943,6 @@ do_backup(time_t start_time) instance_config.master_user); } - /* Get exclusive lock of backup catalog */ - catalog_lock(); - /* * Ensure that backup directory was initialized for the same PostgreSQL * instance we opened connection to. And that target backup database PGDATA @@ -955,7 +952,6 @@ do_backup(time_t start_time) if (!is_remote_backup) check_system_identifiers(); - /* Start backup. Update backup status. */ current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; @@ -965,6 +961,7 @@ do_backup(time_t start_time) /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) elog(ERROR, "cannot create backup directory"); + lock_backup(¤t); write_backup(¤t); elog(LOG, "Backup destination is initialized"); diff --git a/src/catalog.c b/src/catalog.c index b30fcc664..c58a8b5b4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -21,23 +21,73 @@ static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); static bool exit_hook_registered = false; -static char lock_file[MAXPGPATH]; +static parray *lock_files = NULL; static void unlink_lock_atexit(void) { - int res; - res = unlink(lock_file); - if (res != 0 && res != ENOENT) - elog(WARNING, "%s: %s", lock_file, strerror(errno)); + int i; + + if (lock_files == NULL) + return; + + for (i = 0; i < parray_num(lock_files); i++) + { + char *lock_file = (char *) parray_get(lock_files, i); + int res; + + res = unlink(lock_file); + if (res != 0 && res != ENOENT) + elog(WARNING, "%s: %s", lock_file, strerror(errno)); + } + + parray_walk(lock_files, pfree); + parray_free(lock_files); + lock_files = NULL; } /* - * Create a lockfile. + * Read backup meta information from BACKUP_CONTROL_FILE. + * If no backup matches, return NULL. + */ +pgBackup * +read_backup(time_t timestamp) +{ + pgBackup tmp; + char conf_path[MAXPGPATH]; + + tmp.start_time = timestamp; + pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + + return readBackupControlFile(conf_path); +} + +/* + * Save the backup status into BACKUP_CONTROL_FILE. + * + * We need to reread the backup using its ID and save it changing only its + * status. */ void -catalog_lock(void) +write_backup_status(pgBackup *backup) { + pgBackup *tmp; + + tmp = read_backup(backup->start_time); + + tmp->status = backup->status; + write_backup(tmp); + + pgBackupFree(tmp); +} + +/* + * Create exclusive lockfile in the backup's directory. + */ +void +lock_backup(pgBackup *backup) +{ + char lock_file[MAXPGPATH]; int fd; char buffer[MAXPGPATH * 2 + 256]; int ntries; @@ -46,7 +96,7 @@ catalog_lock(void) pid_t my_pid, my_p_pid; - join_path_components(lock_file, backup_instance_path, BACKUP_CATALOG_PID); + pgBackupGetPath(backup, lock_file, lengthof(lock_file), BACKUP_CATALOG_PID); /* * If the PID in the lockfile is our own PID or our parent's or @@ -200,41 +250,11 @@ catalog_lock(void) atexit(unlink_lock_atexit); exit_hook_registered = true; } -} -/* - * Read backup meta information from BACKUP_CONTROL_FILE. - * If no backup matches, return NULL. - */ -pgBackup * -read_backup(time_t timestamp) -{ - pgBackup tmp; - char conf_path[MAXPGPATH]; - - tmp.start_time = timestamp; - pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); - - return readBackupControlFile(conf_path); -} - -/* - * Save the backup status into BACKUP_CONTROL_FILE. - * - * We need to reread the backup using its ID and save it changing only its - * status. - */ -void -write_backup_status(pgBackup *backup) -{ - pgBackup *tmp; - - tmp = read_backup(backup->start_time); - - tmp->status = backup->status; - write_backup(tmp); - - pgBackupFree(tmp); + /* Use parray so that the lock files are unlinked in a loop */ + if (lock_files == NULL) + lock_files = parray_new(); + parray_append(lock_files, pgut_strdup(lock_file)); } /* @@ -381,6 +401,26 @@ catalog_get_backup_list(time_t requested_backup_id) return NULL; } +/* + * Lock list of backups. Function goes in backward direction. + */ +void +catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) +{ + int start_idx, + end_idx; + int i; + + if (parray_num(backup_list) == 0) + return; + + start_idx = Max(from_idx, to_idx); + end_idx = Min(from_idx, to_idx); + + for (i = start_idx; i >= end_idx; i--) + lock_backup((pgBackup *) parray_get(backup_list, i)); +} + /* * Find the last completed backup on given timeline */ diff --git a/src/delete.c b/src/delete.c index 8e9b965fc..007240e48 100644 --- a/src/delete.c +++ b/src/delete.c @@ -28,9 +28,6 @@ do_delete(time_t backup_id) XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; - /* Get exclusive lock of backup catalog */ - catalog_lock(); - /* Get complete list of backups */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -76,6 +73,8 @@ do_delete(time_t backup_id) if (parray_num(delete_list) == 0) elog(ERROR, "no backup found, cannot delete"); + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); + /* Delete backups from the end of list */ for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) { @@ -146,9 +145,6 @@ do_retention_purge(void) } } - /* Get exclusive lock of backup catalog */ - catalog_lock(); - /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) @@ -206,6 +202,8 @@ do_retention_purge(void) continue; } + lock_backup(backup); + /* Delete backup and update status to DELETED */ delete_backup_files(backup); backup_deleted = true; @@ -438,6 +436,8 @@ do_delete_instance(void) /* Delete all backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1); + for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); diff --git a/src/merge.c b/src/merge.c index 545a05054..056967e25 100644 --- a/src/merge.c +++ b/src/merge.c @@ -61,8 +61,6 @@ do_merge(time_t backup_id) elog(INFO, "Merge started"); - catalog_lock(); - /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -125,6 +123,8 @@ do_merge(time_t backup_id) Assert(full_backup_idx != dest_backup_idx); + catalog_lock_backup_list(backups, full_backup_idx, dest_backup_idx); + /* * Found target and full backups, merge them and intermediate backups */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 546c1016c..5f1437319 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -43,7 +43,7 @@ #define PG_GLOBAL_DIR "global" #define BACKUP_CONTROL_FILE "backup.control" #define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" -#define BACKUP_CATALOG_PID "pg_probackup.pid" +#define BACKUP_CATALOG_PID "backup.pid" #define DATABASE_FILE_LIST "backup_content.control" #define PG_BACKUP_LABEL_FILE "backup_label" #define PG_BLACK_LIST "black_list" @@ -459,13 +459,15 @@ extern int do_validate_all(void); extern pgBackup *read_backup(time_t timestamp); extern void write_backup(pgBackup *backup); extern void write_backup_status(pgBackup *backup); +extern void lock_backup(pgBackup *backup); extern const char *pgBackupGetBackupMode(pgBackup *backup); extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern void catalog_lock_backup_list(parray *backup_list, int from_idx, + int to_idx); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli); -extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root); diff --git a/src/restore.c b/src/restore.c index e3591e067..3a48cf956 100644 --- a/src/restore.c +++ b/src/restore.c @@ -74,8 +74,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "%s begin.", action); - /* Get exclusive lock of backup catalog */ - catalog_lock(); /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -300,7 +298,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_parent(base_full_backup->start_time, tmp_backup, true)) { - + lock_backup(tmp_backup); pgBackupValidate(tmp_backup); /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ if (tmp_backup->status == BACKUP_STATUS_CORRUPT) @@ -388,6 +386,13 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", base36enc(dest_backup->start_time), dest_backup->server_version); + /* + * Backup was locked during validation if no-validate wasn't + * specified. + */ + if (rt->restore_no_validate) + lock_backup(backup); + restore_backup(backup); } diff --git a/src/validate.c b/src/validate.c index 14d347ca0..3d123e2bc 100644 --- a/src/validate.c +++ b/src/validate.c @@ -53,8 +53,8 @@ pgBackupValidate(pgBackup *backup) int i; /* Check backup version */ - if (backup->program_version && - parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) + if (parse_program_version(backup->program_version) > + parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", @@ -356,9 +356,6 @@ do_validate_instance(void) elog(INFO, "Validate backups of the instance '%s'", instance_name); - /* Get exclusive lock of backup catalog */ - catalog_lock(); - /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -439,6 +436,7 @@ do_validate_instance(void) else base_full_backup = current_backup; + lock_backup(current_backup); /* Valiate backup files*/ pgBackupValidate(current_backup); @@ -525,6 +523,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_ORPHAN) { + lock_backup(backup); /* Revaliate backup files*/ pgBackupValidate(backup); From 04b96de31fe5207c9237a549362f3b109b0def7b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Dec 2018 12:06:04 +0300 Subject: [PATCH 0169/2107] tests: windows compatibility, archive_command for windows --- tests/helpers/ptrack_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 516017324..286ce2e8a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -877,8 +877,15 @@ def set_archiving( 'postgresql.auto.conf', 'archive_mode = {0}'.format(archive_mode) ) - archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) + if os.name == 'posix': + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path, backup_dir, instance) + + elif os.name == 'nt': + archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path.replace("\\","\\\\"), + backup_dir.replace("\\","\\\\"), + instance) if self.archive_compress or compress: archive_command = archive_command + '--compress ' From 98c67194a6feda20116a42076d5d9a3c9359b9f8 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 18 Dec 2018 19:00:33 +0300 Subject: [PATCH 0170/2107] Fixed removing directory for the delete command --- src/delete.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/delete.c b/src/delete.c index 8e9b965fc..870fbc429 100644 --- a/src/delete.c +++ b/src/delete.c @@ -296,15 +296,7 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", i + 1, num_files, file->path); - if (remove(file->path)) - { - if (errno == ENOENT) - elog(VERBOSE, "File \"%s\" is absent", file->path); - else - elog(ERROR, "Cannot remove \"%s\": %s", file->path, - strerror(errno)); - return; - } + pgFileDelete(file); } parray_walk(files, pgFileFree); From da207119875ba2483f3fe3d3ca3b26fa0cbe2ea7 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 20 Dec 2018 17:58:44 +0300 Subject: [PATCH 0171/2107] Remove 'extradir' field from backup_content.control --- src/backup.c | 27 ++++++++++++++++++--------- src/catalog.c | 4 ++-- src/dir.c | 27 ++++++++++----------------- src/merge.c | 22 +++++++++++++--------- src/pg_probackup.h | 9 +++++---- src/restore.c | 34 ++++++++++++++++++++++++---------- 6 files changed, 72 insertions(+), 51 deletions(-) diff --git a/src/backup.c b/src/backup.c index 96deb2518..15ca8bf3a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -706,7 +706,8 @@ do_backup_instance(void) if (!is_remote_backup) if (file->extra_dir_num) - dir_name = GetRelativePath(file->path, file->extradir); + dir_name = GetRelativePath(file->path, + parray_get(extra_dirs, file->extra_dir_num - 1)); else dir_name = GetRelativePath(file->path, instance_config.pgdata); else @@ -745,7 +746,8 @@ do_backup_instance(void) arg->from_root = instance_config.pgdata; arg->to_root = database_path; - arg->extra = extra_prefix; + arg->extra_prefix = extra_prefix; + arg->extra_dirs = extra_dirs; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; @@ -787,9 +789,6 @@ do_backup_instance(void) parray_walk(prev_backup_filelist, pgFileFree); parray_free(prev_backup_filelist); } - /* clean extra directories list */ - if (extra_dirs) - free_dir_list(extra_dirs); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -853,7 +852,12 @@ do_backup_instance(void) } /* Print the list of files to backup catalog */ - write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, NULL); + write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, + NULL, extra_dirs); + + /* clean extra directories list */ + if (extra_dirs) + free_dir_list(extra_dirs); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -2285,6 +2289,11 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { pgFile **prev_file = NULL; + char *extra_path = NULL; + + if (file->extra_dir_num) + extra_path = parray_get(arguments->extra_dirs, + file->extra_dir_num - 1); /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2293,7 +2302,7 @@ backup_files(void *arg) pgFile key; relative = GetRelativePath(file->path, file->extra_dir_num ? - file->extradir:arguments->from_root); + extra_path : arguments->from_root); key.path = relative; key.extra_dir_num = file->extra_dir_num; @@ -2345,9 +2354,9 @@ backup_files(void *arg) /* Set file paths */ if (file->extra_dir_num) { - makeExtraDirPathByNum(extra_dst, arguments->extra, + makeExtraDirPathByNum(extra_dst, arguments->extra_prefix, file->extra_dir_num); - src = file->extradir; + src = extra_path; dst = extra_dst; } else diff --git a/src/catalog.c b/src/catalog.c index a4059c6a1..fd257ea0b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -551,7 +551,7 @@ write_backup(pgBackup *backup) */ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, - const char *extra_prefix) + const char *extra_prefix, parray *extra_list) { FILE *fp; char path[MAXPGPATH]; @@ -563,7 +563,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); - print_file_list(fp, files, root, extra_prefix); + print_file_list(fp, files, root, extra_prefix, extra_list); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || diff --git a/src/dir.c b/src/dir.c index 2a403069e..b636e00f7 100644 --- a/src/dir.c +++ b/src/dir.c @@ -119,7 +119,7 @@ typedef struct TablespaceCreatedList static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, + bool omit_symlink, parray *black_list, int extra_dir_num); static void list_data_directories(parray *files, const char *path, bool is_root, @@ -226,7 +226,6 @@ pgFileInit(const char *path) file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; file->extra_dir_num = 0; - file->extradir = NULL; return file; } @@ -347,7 +346,7 @@ pgFileComparePath(const void *f1, const void *f2) return strcmp(f1p->path, f2p->path); } -/* +/* * Compare two pgFile with their path and extra_dir_num * in ascending order of ASCII code. */ @@ -466,8 +465,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, elog(WARNING, "Skip \"%s\": unexpected file format", file->path); return; } - if (extra_dir_num) - file->extradir = pgut_strdup(file->path); if (add_root) parray_append(files, file); @@ -725,8 +722,6 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, } /* If it is extra dir, remember it */ - if (extra_dir_num) - file->extradir = parent->extradir; /* We add the directory anyway */ if (S_ISDIR(file->mode)) @@ -1229,7 +1224,7 @@ check_tablespace_mapping(pgBackup *backup) */ void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix) + const char *extra_prefix, parray *extra_list) { size_t i; @@ -1243,7 +1238,11 @@ print_file_list(FILE *out, const parray *files, const char *root, if (root && strstr(path, root) == path) path = GetRelativePath(path, root); else if (file->extra_dir_num && !extra_prefix) - path = GetRelativePath(path, file->extradir); + { + Assert(extra_list); + path = GetRelativePath(path, parray_get(extra_list, + file->extra_dir_num - 1)); + } fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " @@ -1253,8 +1252,8 @@ print_file_list(FILE *out, const parray *files, const char *root, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), file->extra_dir_num); - if (file->extradir) - fprintf(out, ",\"extradir\":\"%s\"", file->extradir); + //if (file->extradir) + // fprintf(out, ",\"extradir\":\"%s\"", file->extradir); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -1439,7 +1438,6 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; - char extradir_str[MAXPGPATH]; int64 write_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, @@ -1473,11 +1471,6 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ file = pgFileInit(filepath); - if (extra_dir_num) - { - get_control_value(buf, "extradir", extradir_str, NULL, true); - file->extradir = pgut_strdup(extradir_str); - } file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; diff --git a/src/merge.c b/src/merge.c index 0d3205920..4b74d0409 100644 --- a/src/merge.c +++ b/src/merge.c @@ -324,7 +324,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + + pfree(to_backup->extra_dir_str); to_backup->extra_dir_str = from_backup->extra_dir_str; + from_backup->extra_dir_str = NULL; /* For safe pgBackupFree() */ /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. @@ -351,7 +354,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path, - from_extra_prefix); + from_extra_prefix, NULL); write_backup(to_backup); delete_source_backup: @@ -370,7 +373,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgFile *file = (pgFile *) parray_get(to_files, i); if (file->extra_dir_num && - backup_contains_extra(file->extradir, from_extra)) + backup_contains_extra(parray_get(to_extra, file->extra_dir_num - 1), + from_extra)) /* Dir already removed*/ continue; @@ -403,8 +407,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Merging finished, now we can safely update ID of the destination backup. */ to_backup->start_time = from_backup->start_time; - if (from_backup->extra_dir_str) - to_backup->extra_dir_str = from_backup->extra_dir_str; write_backup(to_backup); /* Cleanup */ @@ -612,18 +614,20 @@ merge_files(void *arg) copy_pgcontrol_file(argument->from_root, argument->to_root, file); else if (file->extra_dir_num) { - char from_root[MAXPGPATH]; - char to_root[MAXPGPATH]; - int new_dir_num; + char from_root[MAXPGPATH]; + char to_root[MAXPGPATH]; + int new_dir_num; + char *file_extra_path = parray_get(argument->from_extra, + file->extra_dir_num - 1); Assert(argument->from_extra); - new_dir_num = get_extra_index(file->extradir, argument->from_extra); + new_dir_num = get_extra_index(file_extra_path, + argument->from_extra); makeExtraDirPathByNum(from_root, argument->from_extra_prefix, file->extra_dir_num); makeExtraDirPathByNum(to_root, argument->to_extra_prefix, new_dir_num); copy_file(from_root, to_root, file); - file->extra_dir_num = new_dir_num; } else copy_file(argument->from_root, argument->to_root, file); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dc72481fc..e472ffac3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -123,7 +123,6 @@ typedef struct pgFile bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; int extra_dir_num; /* Number of extra directory. 0 if not extra */ - char *extradir; /* File from extra directory */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ @@ -292,10 +291,11 @@ typedef struct { const char *from_root; const char *to_root; - const char *extra; + const char *extra_prefix; parray *files_list; parray *prev_filelist; + parray *extra_dirs; XLogRecPtr prev_start_lsn; PGconn *backup_conn; @@ -476,7 +476,8 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, const char *extra_prefix); + const char *root, const char *extra_prefix, + parray *extra_list); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); @@ -511,7 +512,7 @@ extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix); + const char *extra_prefix, parray *extra_list); extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt); extern parray *make_extra_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); diff --git a/src/restore.c b/src/restore.c index e75ef4d9c..8d2b7c0cb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,7 +21,8 @@ typedef struct { parray *files; pgBackup *backup; - parray *extra_dirs; + parray *req_extra_dirs; + parray *cur_extra_dirs; char *extra_prefix; /* @@ -424,7 +425,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) char extra_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *extra_dirs = NULL; + parray *requested_extra_dirs = NULL; + parray *current_extra_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -458,13 +460,16 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if(extra_dir_str) { - extra_dirs = make_extra_directory_list(extra_dir_str); - for (int i = 0; i < parray_num(extra_dirs); i++) + requested_extra_dirs = make_extra_directory_list(extra_dir_str); + for (int i = 0; i < parray_num(requested_extra_dirs); i++) { - dir_create_dir(parray_get(extra_dirs, i), DIR_PERMISSION); + dir_create_dir(parray_get(requested_extra_dirs, i), DIR_PERMISSION); } } + if(backup->extra_dir_str) + current_extra_dirs = make_extra_directory_list(backup->extra_dir_str); + /* * Get list of files which need to be restored. */ @@ -489,15 +494,21 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) { char dirpath[MAXPGPATH]; char *dir_name; + char *extra_path; + + if (!current_extra_dirs || + parray_num(current_extra_dirs) < file->extra_dir_num - 1) + elog(ERROR, "Inconsistent extra directory backup metadata"); - if (backup_contains_extra(file->extradir, extra_dirs)) + extra_path = parray_get(current_extra_dirs, file->extra_dir_num - 1); + if (backup_contains_extra(extra_path, requested_extra_dirs)) { char container_dir[MAXPGPATH]; makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); elog(VERBOSE, "Create directory \"%s\"", dir_name); - join_path_components(dirpath, file->extradir, dir_name); + join_path_components(dirpath, extra_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } } @@ -515,7 +526,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) arg->files = files; arg->backup = backup; - arg->extra_dirs = extra_dirs; + arg->req_extra_dirs = requested_extra_dirs; + arg->cur_extra_dirs = current_extra_dirs; arg->extra_prefix = extra_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -678,8 +690,10 @@ restore_files(void *arg) copy_pgcontrol_file(from_root, instance_config.pgdata, file); else if (file->extra_dir_num) { - if (backup_contains_extra(file->extradir, arguments->extra_dirs)) - copy_file(arguments->extra_prefix, file->extradir, file); + char *extra_path = parray_get(arguments->cur_extra_dirs, + file->extra_dir_num - 1); + if (backup_contains_extra(extra_path, arguments->req_extra_dirs)) + copy_file(arguments->extra_prefix, extra_path, file); } else copy_file(from_root, instance_config.pgdata, file); From 9e8d70d51b35a4744de3132f45f01162cd01862f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 24 Dec 2018 20:02:50 +0300 Subject: [PATCH 0172/2107] Fix handling recursive tablespaces. Earlier it was disabled to have recursive tablespaces. The code was buggy, so fix the code and allow to have such tablespaces. --- src/dir.c | 106 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/src/dir.c b/src/dir.c index 386f49bcf..96debfa85 100644 --- a/src/dir.c +++ b/src/dir.c @@ -118,7 +118,7 @@ typedef struct TablespaceCreatedList static int BlackListCompare(const void *str1, const void *str2); -static bool dir_check_file(const char *root, pgFile *file); +static char dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list); @@ -448,6 +448,10 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, pgFileFree(file); } +#define CHECK_FALSE 0 +#define CHECK_TRUE 1 +#define CHECK_EXCLUDE_FALSE 2 + /* * Check file or directory. * @@ -456,16 +460,21 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, * Skip files: * - skip temp tables files * - skip unlogged tables files + * Skip recursive tablespace content * Set flags for: * - database directories * - datafiles */ -static bool +static char dir_check_file(const char *root, pgFile *file) { const char *rel_path; int i; int sscanf_res; + bool in_tablespace = false; + + rel_path = GetRelativePath(file->path, root); + in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path); /* Check if we need to exclude file by name */ if (S_ISREG(file->mode)) @@ -478,7 +487,7 @@ dir_check_file(const char *root, pgFile *file) { /* Skip */ elog(VERBOSE, "Excluding file: %s", file->name); - return false; + return CHECK_FALSE; } } @@ -487,14 +496,14 @@ dir_check_file(const char *root, pgFile *file) { /* Skip */ elog(VERBOSE, "Excluding file: %s", file->name); - return false; + return CHECK_FALSE; } } /* * If the directory name is in the exclude list, do not list the * contents. */ - else if (S_ISDIR(file->mode)) + else if (S_ISDIR(file->mode) && !in_tablespace) { /* * If the item in the exclude list starts with '/', compare to @@ -510,20 +519,18 @@ dir_check_file(const char *root, pgFile *file) { elog(VERBOSE, "Excluding directory content: %s", file->name); - return false; + return CHECK_EXCLUDE_FALSE; } } else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) { elog(VERBOSE, "Excluding directory content: %s", file->name); - return false; + return CHECK_EXCLUDE_FALSE; } } } - rel_path = GetRelativePath(file->path, root); - /* * Do not copy tablespaces twice. It may happen if the tablespace is located * inside the PGDATA. @@ -539,14 +546,33 @@ dir_check_file(const char *root, pgFile *file) * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY */ if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) - return false; + return CHECK_FALSE; sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", &tblspcOid, tmp_rel_path); if (sscanf_res == 0) - return false; + return CHECK_FALSE; } - if (path_is_prefix_of_path("global", rel_path)) + if (in_tablespace) + { + char tmp_rel_path[MAXPGPATH]; + + sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + /* + * We should skip other files and directories rather than + * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace. + */ + if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) + return CHECK_FALSE; + + if (sscanf_res == 3 && S_ISDIR(file->mode) && + strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) + file->is_database = true; + } + else if (path_is_prefix_of_path("global", rel_path)) { file->tblspcOid = GLOBALTABLESPACE_OID; @@ -562,22 +588,10 @@ dir_check_file(const char *root, pgFile *file) if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) file->is_database = true; } - else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) - { - char tmp_rel_path[MAXPGPATH]; - - sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", - &(file->tblspcOid), tmp_rel_path, - &(file->dbOid)); - - if (sscanf_res == 3 && S_ISDIR(file->mode) && - strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) - file->is_database = true; - } /* Do not backup ptrack_init files */ if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) - return false; + return CHECK_FALSE; /* * Check files located inside database directories including directory @@ -587,10 +601,10 @@ dir_check_file(const char *root, pgFile *file) file->name && file->name[0]) { if (strcmp(file->name, "pg_internal.init") == 0) - return false; + return CHECK_FALSE; /* Do not backup temp files */ else if (file->name[0] == 't' && isdigit(file->name[1])) - return false; + return CHECK_FALSE; else if (isdigit(file->name[0])) { char *fork_name; @@ -605,14 +619,14 @@ dir_check_file(const char *root, pgFile *file) /* Do not backup ptrack files */ if (strcmp(file->forkName, "ptrack") == 0) - return false; + return CHECK_FALSE; } else { len = strlen(file->name); /* reloid.cfm */ if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) - return true; + return CHECK_TRUE; sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), &(file->segno), suffix); @@ -624,7 +638,7 @@ dir_check_file(const char *root, pgFile *file) } } - return true; + return CHECK_TRUE; } /* @@ -659,6 +673,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, { pgFile *file; char child[MAXPGPATH]; + char check_res; join_path_components(child, parent->path, dent->d_name); @@ -694,21 +709,24 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, continue; } - /* We add the directory anyway */ - if (S_ISDIR(file->mode)) - parray_append(files, file); - - if (exclude && !dir_check_file(root, file)) + if (exclude) { - if (S_ISREG(file->mode)) + check_res = dir_check_file(root, file); + if (check_res == CHECK_FALSE) + { + /* Skip */ pgFileFree(file); - /* Skip */ - continue; + continue; + } + else if (check_res == CHECK_EXCLUDE_FALSE) + { + /* We add the directory itself which content was excluded */ + parray_append(files, file); + continue; + } } - /* At least add the file */ - if (S_ISREG(file->mode)) - parray_append(files, file); + parray_append(files, file); /* * If the entry is a directory call dir_list_file_internal() @@ -1219,11 +1237,7 @@ print_file_list(FILE *out, const parray *files, const char *root) if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); -#ifndef WIN32 - if (S_ISLNK(file->mode)) -#else - if (pgwin32_is_junction(file->path)) -#endif + if (file->linked) fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) From 91c4572bbd63ae12d8da11c8525e6fe33018ea00 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 25 Dec 2018 02:42:55 +0300 Subject: [PATCH 0173/2107] Add extra directories remapping on restore --- src/dir.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++ src/pg_probackup.c | 3 +- src/pg_probackup.h | 3 ++ src/restore.c | 11 +++++- 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/dir.c b/src/dir.c index b636e00f7..6bd0c6823 100644 --- a/src/dir.c +++ b/src/dir.c @@ -124,10 +124,13 @@ static void dir_list_file_internal(parray *files, const char *root, static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); +static void free_extra_remap_list(void *cell); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; +/* Extra directories mapping */ +static parray *extra_remap_list = NULL; /* * Create directory, also create parent directories if necessary. @@ -949,6 +952,57 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) tablespace_dirs.tail = cell; } +void +opt_extradir_map(ConfigOption *opt, const char *arg) +{ + TablespaceListCell *cell = pgut_new(TablespaceListCell); + char *dst; + char *dst_ptr; + const char *arg_ptr; + + extra_remap_list = parray_new(); + dst_ptr = dst = cell->old_dir; + for (arg_ptr = arg; *arg_ptr; arg_ptr++) + { + if (dst_ptr - dst >= MAXPGPATH) + elog(ERROR, "directory name too long"); + + if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') + ; /* skip backslash escaping = */ + else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) + { + if (*cell->new_dir) + elog(ERROR, "multiple \"=\" signs in extra directory mapping\n"); + else + dst = dst_ptr = cell->new_dir; + } + else + *dst_ptr++ = *arg_ptr; + } + + if (!*cell->old_dir || !*cell->new_dir) + elog(ERROR, "invalid extra directory mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", arg); + + /* + * This check isn't absolutely necessary. But all tablespaces are created + * with absolute directories, so specifying a non-absolute path here would + * just never match, possibly confusing users. It's also good to be + * consistent with the new_dir check. + */ + if (!is_absolute_path(cell->old_dir)) + elog(ERROR, "old directory is not an absolute path " + "in extra directory mapping: %s\n", + cell->old_dir); + + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "new directory is not an absolute path " + "in extra directory mapping: %s\n", + cell->new_dir); + + parray_append(extra_remap_list, cell); +} + /* * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise * an error if target directories exist. @@ -1219,6 +1273,43 @@ check_tablespace_mapping(pgBackup *backup) parray_free(links); } +char * +check_extra_dir_mapping(char *current_dir) +{ + if (!extra_remap_list) + return current_dir; + + for (int i = 0; i < parray_num(extra_remap_list); i++) + { + TablespaceListCell *cell = parray_get(extra_remap_list, i); + char *old_dir = cell->old_dir; + + if (strcmp(old_dir, current_dir) == 0) + return cell->new_dir; + } + return current_dir; +} + +static void +free_extra_remap_list(void *cell) +{ + TablespaceListCell *cell_ptr; + if (cell == NULL) + return; + cell_ptr = (TablespaceListCell *)cell; + pfree(cell_ptr); +} + +void +clean_extra_dirs_remap_list(void) +{ + if (extra_remap_list) + { + parray_walk(extra_remap_list, free_extra_remap_list); + parray_free(extra_remap_list); + } +} + /* * Print backup content list. */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 7a2c62168..5fea590a0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -137,7 +137,8 @@ static ConfigOption cmd_options[] = { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 's', 155, "extra-directory", &extradir, SOURCE_CMD_STRICT }, + { 's', 155, "extra-directory", &extradir, SOURCE_CMD_STRICT }, + { 'f', 'E', "extra-mapping", opt_extradir_map, SOURCE_CMD_STRICT }, /* TODO not completed feature. Make it unavailiable from user level { 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */ /* restore options */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e472ffac3..670d852d7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -509,7 +509,10 @@ extern void create_data_directories(const char *data_dir, extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); +extern void opt_extradir_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); +extern char* check_extra_dir_mapping(char *current_dir); +extern void clean_extra_dirs_remap_list(void); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *extra_prefix, parray *extra_list); diff --git a/src/restore.c b/src/restore.c index 8d2b7c0cb..43133ff4d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -463,7 +463,9 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) requested_extra_dirs = make_extra_directory_list(extra_dir_str); for (int i = 0; i < parray_num(requested_extra_dirs); i++) { - dir_create_dir(parray_get(requested_extra_dirs, i), DIR_PERMISSION); + char *extra_path = parray_get(requested_extra_dirs, i); + extra_path = check_extra_dir_mapping(extra_path); + dir_create_dir(extra_path, DIR_PERMISSION); } } @@ -504,6 +506,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if (backup_contains_extra(extra_path, requested_extra_dirs)) { char container_dir[MAXPGPATH]; + + extra_path = check_extra_dir_mapping(extra_path); makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); @@ -554,6 +558,8 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) parray_walk(files, pgFileFree); parray_free(files); + clean_extra_dirs_remap_list(); + if (logger_config.log_level_console <= LOG || logger_config.log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); @@ -693,7 +699,10 @@ restore_files(void *arg) char *extra_path = parray_get(arguments->cur_extra_dirs, file->extra_dir_num - 1); if (backup_contains_extra(extra_path, arguments->req_extra_dirs)) + { + extra_path = check_extra_dir_mapping(extra_path); copy_file(arguments->extra_prefix, extra_path, file); + } } else copy_file(from_root, instance_config.pgdata, file); From 7213106cfc57346463181a76074f543bcbef33a0 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 25 Dec 2018 02:43:17 +0300 Subject: [PATCH 0174/2107] Add '--extra-directory' and '-E' options to help.c --- src/help.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/help.c b/src/help.c index 409c8f825..6b2cd9292 100644 --- a/src/help.c +++ b/src/help.c @@ -119,6 +119,7 @@ help_pg_probackup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--extra-directory]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); @@ -129,6 +130,7 @@ help_pg_probackup(void) printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [-E OLDDIR=NEWDIR]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); From 4a6f8c515ee8baa9c7f193ae71b76b82d2696cc6 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 25 Dec 2018 03:11:34 +0300 Subject: [PATCH 0175/2107] Cleanup --- src/dir.c | 11 +---------- src/pg_probackup.c | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/dir.c b/src/dir.c index 6bd0c6823..24e8d5a4c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -984,12 +984,6 @@ opt_extradir_map(ConfigOption *opt, const char *arg) elog(ERROR, "invalid extra directory mapping format \"%s\", " "must be \"OLDDIR=NEWDIR\"", arg); - /* - * This check isn't absolutely necessary. But all tablespaces are created - * with absolute directories, so specifying a non-absolute path here would - * just never match, possibly confusing users. It's also good to be - * consistent with the new_dir check. - */ if (!is_absolute_path(cell->old_dir)) elog(ERROR, "old directory is not an absolute path " "in extra directory mapping: %s\n", @@ -1338,14 +1332,11 @@ print_file_list(FILE *out, const parray *files, const char *root, fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"extra_dir_num\":\"%u\"", + "\"compress_alg\":\"%s\", \"extra_dir_num\":\"%d\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), file->extra_dir_num); - //if (file->extradir) - // fprintf(out, ",\"extradir\":\"%s\"", file->extradir); - if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5fea590a0..fe88840fd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -54,7 +54,7 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; -/* extra directory to backup */ +/* colon separated extra directories list ("/path1:/path2") */ char *extradir = NULL; /* common options */ static char *backup_id_string = NULL; From 31830df36d73b3e68645495ba7310d18eb6404a5 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 25 Dec 2018 14:08:15 +0300 Subject: [PATCH 0176/2107] Added initialization of variable for tablespace renaming --- src/dir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dir.c b/src/dir.c index 96debfa85..3eeb96f38 100644 --- a/src/dir.c +++ b/src/dir.c @@ -894,6 +894,7 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) char *dst_ptr; const char *arg_ptr; + memset(cell, 0, sizeof(TablespaceListCell)); dst_ptr = dst = cell->old_dir; for (arg_ptr = arg; *arg_ptr; arg_ptr++) { From a42895620d5b183969b946af0fb46729b05e99e4 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 25 Dec 2018 14:29:47 +0300 Subject: [PATCH 0177/2107] Fix filling and releasing of black_list --- src/dir.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 3eeb96f38..37f41ddda 100644 --- a/src/dir.c +++ b/src/dir.c @@ -415,6 +415,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, while (fgets(buf, lengthof(buf), black_list_file) != NULL) { + black_item[0] = '\0'; join_path_components(black_item, instance_config.pgdata, buf); if (black_item[strlen(black_item) - 1] == '\n') @@ -423,7 +424,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (black_item[0] == '#' || black_item[0] == '\0') continue; - parray_append(black_list, black_item); + parray_append(black_list, pgut_strdup(black_item)); } fclose(black_list_file); @@ -446,6 +447,12 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (!add_root) pgFileFree(file); + + if (black_list) + { + parray_walk(black_list, pfree); + parray_free(black_list); + } } #define CHECK_FALSE 0 From 02860600d9df5e6686e3a1d63a60b781c2a552ef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 16:44:37 +0300 Subject: [PATCH 0178/2107] bugfix: elog fact of copying file after locking it, not before --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 7046d1d06..845ff03e8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2216,9 +2216,9 @@ backup_files(void *arg) struct stat buf; pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - elog(VERBOSE, "Copying file: \"%s\" ", file->path); if (!pg_atomic_test_set_flag(&file->lock)) continue; + elog(VERBOSE, "Copying file: \"%s\" ", file->path); /* check for interrupt */ if (interrupted) From f8340f5eb8898e598d3cdcc6fde40879b6f9e127 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 17:12:48 +0300 Subject: [PATCH 0179/2107] tests: adapt test_tablespace_in_pgdata_pgpro_1376 to 9e8d70d51b35 --- tests/backup_test.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 9c8a09558..e6d9a336d 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -459,22 +459,9 @@ def test_tablespace_in_pgdata_pgpro_1376(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1000) i") - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of too many levels " - "of symbolic linking\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Too many levels of symbolic links' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + backup_id_1 = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) node.safe_psql( "postgres", @@ -495,7 +482,8 @@ def test_tablespace_in_pgdata_pgpro_1376(self): ).rstrip() list = [] - for root, dirs, files in os.walk(backup_dir): + for root, dirs, files in os.walk(os.path.join( + backup_dir, 'backups/node', backup_id_1)): for file in files: if file == relfilenode: path = os.path.join(root, file) From 7b66795b8a2ed6abf31c4c1c11195db8e0e76613 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 17:48:49 +0300 Subject: [PATCH 0180/2107] tests: windows support --- tests/archive.py | 4 +-- tests/auth_test.py | 35 ++++++++++--------- tests/backup_test.py | 20 +++++------ tests/cfs_backup.py | 10 +++--- tests/cfs_restore.py | 2 +- tests/compression.py | 10 +++--- tests/delete_test.py | 8 ++--- tests/delta.py | 56 +++++++++++++++--------------- tests/exclude.py | 4 +-- tests/false_positive.py | 10 +++--- tests/helpers/ptrack_helpers.py | 6 ++-- tests/merge.py | 22 ++++++------ tests/page.py | 34 +++++++++--------- tests/pgpro560.py | 6 ++-- tests/pgpro589.py | 2 +- tests/ptrack_clean.py | 6 ++-- tests/ptrack_cluster.py | 12 +++---- tests/ptrack_empty.py | 6 ++-- tests/ptrack_move_to_tablespace.py | 2 +- tests/ptrack_recovery.py | 4 +-- tests/replica.py | 2 +- tests/restore_test.py | 44 +++++++++++------------ tests/retention_test.py | 6 ++-- tests/show_test.py | 12 +++---- tests/validate_test.py | 48 ++++++++++++------------- 25 files changed, 186 insertions(+), 185 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 180b7ad6b..45e735e8d 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -732,7 +732,7 @@ def test_archive_pg_receivexlog(self): ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() if self.get_version(node) < 100000: pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: @@ -806,7 +806,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): ) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL 10 for this test') else: diff --git a/tests/auth_test.py b/tests/auth_test.py index fc21a480d..c5f1ae331 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -43,7 +43,7 @@ def test_backup_via_unpriviledged_user(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") @@ -203,25 +203,26 @@ def setUpClass(cls): cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) try: - cls.node.start() + cls.node.slow_start() except StartNodeException: raise unittest.skip("Node hasn't started") - cls.node.safe_psql("postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; \ - GRANT USAGE ON SCHEMA pg_catalog TO backup; \ - GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_current() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; \ - GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; \ - GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; \ - GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') @classmethod diff --git a/tests/backup_test.py b/tests/backup_test.py index e6d9a336d..08abd971e 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -27,7 +27,7 @@ def test_backup_modes_archive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] @@ -97,7 +97,7 @@ def test_smooth_checkpoint(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -121,7 +121,7 @@ def test_incremental_backup_without_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() try: self.backup_node(backup_dir, 'node', node, backup_type="page") @@ -177,7 +177,7 @@ def test_incremental_backup_corrupt_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) file = os.path.join( @@ -241,7 +241,7 @@ def test_ptrack_threads(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -272,7 +272,7 @@ def test_ptrack_threads_stream(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -301,7 +301,7 @@ def test_page_corruption_heal_via_ptrack_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -358,7 +358,7 @@ def test_page_corruption_heal_via_ptrack_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -383,7 +383,7 @@ def test_page_corruption_heal_via_ptrack_2(self): f.write(b"bla") f.flush() f.close - node.start() + node.slow_start() try: self.backup_node( @@ -431,7 +431,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node( node, 'tblspace1', diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 412320327..76d4a0a11 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -34,7 +34,7 @@ def setUp(self): self.add_instance(self.backup_dir, 'node', self.node) self.set_archiving(self.backup_dir, 'node', self.node) - self.node.start() + self.node.slow_start() self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) @@ -745,7 +745,7 @@ def test_multiple_segments(self): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_full, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( full_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -760,7 +760,7 @@ def test_multiple_segments(self): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_page, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( page_result, self.node.safe_psql("postgres", "SELECT * FROM t_heap"), @@ -879,7 +879,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_full, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( full_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), @@ -905,7 +905,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.restore_node( self.backup_dir, 'node', self.node, backup_id=backup_id_page, options=["-j", "4"]) - self.node.start() + self.node.slow_start() self.assertEqual( page_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 1aefef897..a18d0903e 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -43,7 +43,7 @@ def setUp(self): self.add_instance(self.backup_dir, 'node', self.node) self.set_archiving(self.backup_dir, 'node', self.node) - self.node.start() + self.node.slow_start() self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) self.add_data_in_cluster() diff --git a/tests/compression.py b/tests/compression.py index 2e712a152..f2c8614e0 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -30,7 +30,7 @@ def test_compression_stream_zlib(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -142,7 +142,7 @@ def test_compression_archive_zlib(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -250,7 +250,7 @@ def test_compression_stream_pglz(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -360,7 +360,7 @@ def test_compression_archive_pglz(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -471,7 +471,7 @@ def test_compression_wrong_algorithm(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() try: self.backup_node( diff --git a/tests/delete_test.py b/tests/delete_test.py index b36c2c926..f58fc5fda 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -24,7 +24,7 @@ def test_delete_full_backups(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup self.backup_node(backup_dir, 'node', node) @@ -72,7 +72,7 @@ def test_delete_increment_page(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup mode self.backup_node(backup_dir, 'node', node) @@ -112,7 +112,7 @@ def test_delete_increment_ptrack(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # full backup mode self.backup_node(backup_dir, 'node', node) @@ -152,7 +152,7 @@ def test_delete_orphaned_wal_segments(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/delta.py b/tests/delta.py index 55cc03bec..76673ceb1 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -41,7 +41,7 @@ def test_delta_vacuum_truncate_1(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -89,7 +89,7 @@ def test_delta_vacuum_truncate_1(self): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -123,7 +123,7 @@ def test_delta_vacuum_truncate_2(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -180,7 +180,7 @@ def test_delta_vacuum_truncate_2(self): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -214,7 +214,7 @@ def test_delta_vacuum_truncate_3(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -252,7 +252,7 @@ def test_delta_vacuum_truncate_3(self): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -279,7 +279,7 @@ def test_delta_stream(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -318,7 +318,7 @@ def test_delta_stream(self): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -334,7 +334,7 @@ def test_delta_stream(self): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.start() + node.slow_start() delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -365,7 +365,7 @@ def test_delta_archive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) # self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -401,7 +401,7 @@ def test_delta_archive(self): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() full_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -417,7 +417,7 @@ def test_delta_archive(self): "--recovery-target-action=promote"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - node.start() + node.slow_start() delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -451,7 +451,7 @@ def test_delta_multiple_segments(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) # self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -533,7 +533,7 @@ def test_delta_vacuum_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -599,7 +599,7 @@ def test_delta_vacuum_full(self): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -627,7 +627,7 @@ def test_create_db(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -681,7 +681,7 @@ def test_create_db(self): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # DROP DATABASE DB1 node.safe_psql( @@ -716,7 +716,7 @@ def test_create_db(self): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() try: node_restored.safe_psql('db1', 'select 1') @@ -761,7 +761,7 @@ def test_exists_in_previous_backup(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -842,7 +842,7 @@ def test_exists_in_previous_backup(self): # START RESTORED NODE node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -868,7 +868,7 @@ def test_alter_table_set_tablespace_delta(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP self.create_tblspace_in_node(node, 'somedata') @@ -961,7 +961,7 @@ def test_alter_database_set_tablespace_delta(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup @@ -1029,7 +1029,7 @@ def test_alter_database_set_tablespace_delta(self): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1056,7 +1056,7 @@ def test_delta_delete(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -1115,7 +1115,7 @@ def test_delta_delete(self): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1134,7 +1134,7 @@ def test_delta_corruption_heal_via_ptrack_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, @@ -1193,7 +1193,7 @@ def test_page_corruption_heal_via_ptrack_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -1218,7 +1218,7 @@ def test_page_corruption_heal_via_ptrack_2(self): f.write(b"bla") f.flush() f.close - node.start() + node.slow_start() try: self.backup_node( diff --git a/tests/exclude.py b/tests/exclude.py index 3fd3341f2..30e500d61 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -28,7 +28,7 @@ def test_exclude_temp_tables(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() conn = node.connect() with node.connect("postgres") as conn: @@ -122,7 +122,7 @@ def test_exclude_unlogged_tables_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() conn = node.connect() with node.connect("postgres") as conn: diff --git a/tests/false_positive.py b/tests/false_positive.py index df7b13343..24f02276f 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -28,7 +28,7 @@ def test_validate_wal_lost_segment(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -66,7 +66,7 @@ def test_incremental_backup_corrupt_full_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) file = os.path.join( @@ -132,7 +132,7 @@ def test_ptrack_concurrent_get_and_clear_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -216,7 +216,7 @@ def test_ptrack_concurrent_get_and_clear_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -308,7 +308,7 @@ def test_multiple_delete(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 286ce2e8a..43ecaedf3 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -218,10 +218,10 @@ def __init__(self, *args, **kwargs): self.probackup_path = None if 'PGPROBACKUPBIN' in self.test_env: if ( - os.path.isfile(self.test_env['PGPROBACKUPBIN']) and - os.access(self.test_env['PGPROBACKUPBIN'], os.X_OK) + os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and + os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) ): - self.probackup_path = self.test_env['PGPROBACKUPBIN'] + self.probackup_path = self.test_env["PGPROBACKUPBIN"] else: if self.verbose: print('PGPROBACKUPBIN is not an executable file') diff --git a/tests/merge.py b/tests/merge.py index 2dbe8ee0c..6b09c373d 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -26,7 +26,7 @@ def test_merge_full_page(self): self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) self.set_archiving(backup_dir, "node", node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, "node", node) @@ -115,7 +115,7 @@ def test_merge_compressed_backups(self): self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) self.set_archiving(backup_dir, "node", node) - node.start() + node.slow_start() # Do full compressed backup self.backup_node(backup_dir, "node", node, options=[ @@ -493,7 +493,7 @@ def test_merge_tablespaces(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -573,7 +573,7 @@ def test_merge_tablespaces_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -662,7 +662,7 @@ def test_merge_page_truncate(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -757,7 +757,7 @@ def test_merge_delta_truncate(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -853,7 +853,7 @@ def test_merge_ptrack_truncate(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -944,7 +944,7 @@ def test_merge_delta_delete(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -1006,7 +1006,7 @@ def test_merge_delta_delete(self): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1029,7 +1029,7 @@ def test_continue_failed_merge(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup self.backup_node(backup_dir, 'node', node) @@ -1107,7 +1107,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup self.backup_node(backup_dir, 'node', node) diff --git a/tests/page.py b/tests/page.py index d0ebbb9b9..9e23ecaa7 100644 --- a/tests/page.py +++ b/tests/page.py @@ -40,7 +40,7 @@ def test_page_vacuum_truncate(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node_restored.cleanup() - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -128,7 +128,7 @@ def test_page_stream(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -217,7 +217,7 @@ def test_page_archive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL BACKUP node.safe_psql( @@ -317,7 +317,7 @@ def test_page_multiple_segments(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -397,7 +397,7 @@ def test_page_delete(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup @@ -449,7 +449,7 @@ def test_page_delete(self): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -477,7 +477,7 @@ def test_page_delete_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -530,7 +530,7 @@ def test_page_delete_1(self): # START RESTORED NODE node_restored.append_conf( 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) @@ -558,7 +558,7 @@ def test_parallel_pagemap(self): self.add_instance(backup_dir, 'node', node) node_restored.cleanup() self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, 'node', node) @@ -598,7 +598,7 @@ def test_parallel_pagemap(self): node_restored.append_conf( "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - node_restored.start() + node_restored.slow_start() # Check restored node count2 = node_restored.execute("postgres", "select count(*) from test") @@ -627,7 +627,7 @@ def test_parallel_pagemap_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Do full backup self.backup_node(backup_dir, 'node', node) @@ -656,7 +656,7 @@ def test_parallel_pagemap_1(self): # Drop node and restore it node.cleanup() self.restore_node(backup_dir, 'node', node) - node.start() + node.slow_start() # Clean after yourself node.cleanup() @@ -681,7 +681,7 @@ def test_page_backup_with_lost_wal_segment(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -771,7 +771,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -896,11 +896,11 @@ def test_page_backup_with_alien_wal_segment(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.add_instance(backup_dir, 'alien_node', alien_node) self.set_archiving(backup_dir, 'alien_node', alien_node) - alien_node.start() + alien_node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'alien_node', alien_node) @@ -1013,7 +1013,7 @@ def test_multithread_page_backup_with_toast(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index bf3345561..6c2b88634 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -28,7 +28,7 @@ def test_pgpro560_control_file_loss(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() file = os.path.join(node.base_dir,'data', 'global', 'pg_control') os.remove(file) @@ -60,13 +60,13 @@ def test_pgpro560_systemid_mismatch(self): initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node1.start() + node1.slow_start() node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) - node2.start() + node2.slow_start() backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index bd40f16de..d3bb9d908 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -30,7 +30,7 @@ def test_pgpro589(self): # make erroneus archive_command node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") - node.start() + node.slow_start() node.pgbench_init(scale=5) pgbench = node.pgbench( diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 2b122f2f9..605d631a0 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -26,7 +26,7 @@ def test_ptrack_clean(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -136,7 +136,7 @@ def test_ptrack_clean_replica(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -149,7 +149,7 @@ def test_ptrack_clean_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 2fdfe0970..41de27df7 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -26,7 +26,7 @@ def test_ptrack_cluster_on_btree(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -107,7 +107,7 @@ def test_ptrack_cluster_on_gist(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() # Create table and indexes node.safe_psql( @@ -185,7 +185,7 @@ def test_ptrack_cluster_on_btree_replica(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -198,7 +198,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( @@ -285,7 +285,7 @@ def test_ptrack_cluster_on_gist_replica(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -298,7 +298,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index f20362b10..18f289e77 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -27,7 +27,7 @@ def test_ptrack_empty(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -103,7 +103,7 @@ def test_ptrack_empty_replica(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - master.start() + master.slow_start() self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -116,7 +116,7 @@ def test_ptrack_empty_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.start() + replica.slow_start() # Create table master.safe_psql( diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 95a7a5aa1..41b92cf77 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -24,7 +24,7 @@ def test_ptrack_recovery(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 1a6607c9f..5889b06ed 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -25,7 +25,7 @@ def test_ptrack_recovery(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -56,7 +56,7 @@ def test_ptrack_recovery(self): print('Killing postmaster. Losing Ptrack changes') node.stop(['-m', 'immediate', '-D', node.data_dir]) if not node.status(): - node.start() + node.slow_start() else: print("Die! Die! Why won't you die?... Why won't you die?") exit(1) diff --git a/tests/replica.py b/tests/replica.py index 4fbd08e69..052dee2c0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -30,7 +30,7 @@ def test_replica_stream_ptrack_backup(self): 'max_wal_senders': '2', 'ptrack_enable': 'on'} ) - master.start() + master.slow_start() self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) diff --git a/tests/restore_test.py b/tests/restore_test.py index 0afcca8a9..2ada39556 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -27,7 +27,7 @@ def test_restore_full_to_latest(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) pgbench = node.pgbench( @@ -74,7 +74,7 @@ def test_restore_full_page_to_latest(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -122,7 +122,7 @@ def test_restore_to_specific_timeline(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -192,7 +192,7 @@ def test_restore_to_time(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") - node.start() + node.slow_start() node.pgbench_init(scale=2) before = node.execute("postgres", "SELECT * FROM pgbench_branches") @@ -240,7 +240,7 @@ def test_restore_to_xid_inclusive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -304,7 +304,7 @@ def test_restore_to_xid_not_inclusive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -371,7 +371,7 @@ def test_restore_to_lsn_inclusive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -444,7 +444,7 @@ def test_restore_to_lsn_not_inclusive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) with node.connect("postgres") as con: @@ -513,7 +513,7 @@ def test_restore_full_ptrack_archive(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -561,7 +561,7 @@ def test_restore_ptrack(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -620,7 +620,7 @@ def test_restore_full_ptrack_stream(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -675,7 +675,7 @@ def test_restore_full_ptrack_under_load(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=2) @@ -741,7 +741,7 @@ def test_restore_full_under_load_ptrack(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # wal_segment_size = self.guc_wal_segment_size(node) node.pgbench_init(scale=2) @@ -805,7 +805,7 @@ def test_restore_with_tablespace_mapping_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -921,7 +921,7 @@ def test_restore_with_tablespace_mapping_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # Full backup self.backup_node(backup_dir, 'node', node) @@ -1004,7 +1004,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1054,7 +1054,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1103,7 +1103,7 @@ def test_archive_node_backup_stream_pitr(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1151,7 +1151,7 @@ def test_archive_node_backup_archive_pitr_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) if self.paranoia: @@ -1206,7 +1206,7 @@ def test_archive_restore_to_restore_point(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -1255,7 +1255,7 @@ def test_zags_block_corrupt(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) @@ -1336,7 +1336,7 @@ def test_zags_block_corrupt_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) diff --git a/tests/retention_test.py b/tests/retention_test.py index 652f7c39d..91fef215f 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -23,7 +23,7 @@ def test_retention_redundancy_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with open(os.path.join( backup_dir, 'backups', 'node', @@ -80,7 +80,7 @@ def test_retention_window_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with open( os.path.join( @@ -134,7 +134,7 @@ def test_retention_wal(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", diff --git a/tests/show_test.py b/tests/show_test.py index 50ad5be15..655c0b8e3 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -23,7 +23,7 @@ def test_show_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.assertEqual( self.backup_node( @@ -51,7 +51,7 @@ def test_show_json(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.assertEqual( self.backup_node( @@ -79,7 +79,7 @@ def test_corrupt_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -125,7 +125,7 @@ def test_no_control_file(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -154,7 +154,7 @@ def test_empty_control_file(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -185,7 +185,7 @@ def test_corrupt_control_file(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) diff --git a/tests/validate_test.py b/tests/validate_test.py index a7f8d4373..6153a4a4a 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -221,7 +221,7 @@ def test_validate_corrupted_intermediate_backup(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -306,7 +306,7 @@ def test_validate_corrupted_intermediate_backups(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -429,7 +429,7 @@ def test_validate_corrupted_intermediate_backups_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -628,7 +628,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -812,7 +812,7 @@ def test_validate_instance_with_corrupted_page(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -960,7 +960,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -1046,7 +1046,7 @@ def test_validate_instance_with_corrupted_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -1121,7 +1121,7 @@ def test_validate_corrupt_wal_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1183,7 +1183,7 @@ def test_validate_corrupt_wal_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") @@ -1255,7 +1255,7 @@ def test_validate_wal_lost_segment_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.pgbench_init(scale=3) @@ -1333,7 +1333,7 @@ def test_validate_corrupt_wal_between_backups(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -1427,7 +1427,7 @@ def test_pgpro702_688(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1470,7 +1470,7 @@ def test_pgpro688(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) recovery_time = self.show_pb( @@ -1500,7 +1500,7 @@ def test_pgpro561(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) self.set_archiving(backup_dir, 'node1', node1) - node1.start() + node1.slow_start() backup_id = self.backup_node( backup_dir, 'node1', node1, options=["--stream"]) @@ -1594,7 +1594,7 @@ def test_validate_corrupted_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1717,7 +1717,7 @@ def test_validate_corrupted_full_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1829,7 +1829,7 @@ def test_validate_corrupted_full_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -2179,7 +2179,7 @@ def test_validate_corrupted_full_missing(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() self.backup_node(backup_dir, 'node', node) self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -2405,7 +2405,7 @@ def test_file_size_corruption_no_validate(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() node.safe_psql( "postgres", @@ -2481,7 +2481,7 @@ def test_validate_specific_backup_with_missing_backup(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2623,7 +2623,7 @@ def test_validate_specific_backup_with_missing_backup_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2744,7 +2744,7 @@ def test_validate_with_missing_backup_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -2932,7 +2932,7 @@ def test_validate_with_missing_backup_2(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() # CHAIN1 self.backup_node(backup_dir, 'node', node) @@ -3080,7 +3080,7 @@ def test_corrupt_pg_control_via_resetxlog(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.start() + node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) From 9a632f0f568b83d4dd94cffa7d841c09c4f0b37f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 22:04:56 +0300 Subject: [PATCH 0181/2107] tests: windows support, fix test_incremental_backup_corrupt_full() --- tests/backup_test.py | 10 ++++++---- tests/helpers/ptrack_helpers.py | 9 +++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 08abd971e..4c447f08f 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -195,12 +195,14 @@ def test_incremental_backup_corrupt_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "INFO: Validate backups of the instance 'node'\n" in e.message and - "WARNING: Backup file \"{0}\" is not found\n".format( + "INFO: Validate backups of the instance 'node'" in e.message and + "WARNING: Backup file".format( file) in e.message and - "WARNING: Backup {0} data files are corrupted\n".format( + "is not found".format(file) in e.message and + "{0}\" is not found".format(file) in e.message and + "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and - "WARNING: Some backups are not valid\n" in e.message, + "WARNING: Some backups are not valid" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 43ecaedf3..17fc4f236 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -252,8 +252,13 @@ def __init__(self, *args, **kwargs): print('pg_probackup binary is not found') exit(1) - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] + if os.name == 'posix': + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ':' + os.environ['PATH'] + + elif os.name == 'nt': + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ';' + os.environ['PATH'] self.probackup_old_path = None From c1fd7ce031f76f30cdee97e98bb09ff84d583843 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Dec 2018 22:07:54 +0300 Subject: [PATCH 0182/2107] minor fix --- tests/backup_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 4c447f08f..08f397e1e 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -199,7 +199,6 @@ def test_incremental_backup_corrupt_full(self): "WARNING: Backup file".format( file) in e.message and "is not found".format(file) in e.message and - "{0}\" is not found".format(file) in e.message and "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and "WARNING: Some backups are not valid" in e.message, From 800ba045f508f19c69b5a64a37ed4ccce74ee96e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 26 Dec 2018 15:00:05 +0300 Subject: [PATCH 0183/2107] No need to check static array backup->program_version to NULL --- src/validate.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/validate.c b/src/validate.c index 14d347ca0..a283593e6 100644 --- a/src/validate.c +++ b/src/validate.c @@ -53,8 +53,7 @@ pgBackupValidate(pgBackup *backup) int i; /* Check backup version */ - if (backup->program_version && - parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) + if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", From b61df02ce168de07e30124187f07a7c885cad797 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 26 Dec 2018 22:59:13 +0300 Subject: [PATCH 0184/2107] tests: windows compatibility, cross-platform path for base_dir --- tests/archive.py | 28 ++++++------ tests/auth_test.py | 2 +- tests/backup_test.py | 18 ++++---- tests/compatibility.py | 16 +++---- tests/compression.py | 12 ++--- tests/delete_test.py | 27 ++++++----- tests/delta.py | 58 ++++++++++++------------ tests/exclude.py | 8 ++-- tests/false_positive.py | 23 +++++----- tests/init_test.py | 6 +-- tests/merge.py | 36 +++++++-------- tests/option_test.py | 4 +- tests/page.py | 41 +++++++++-------- tests/pgpro560.py | 21 +++++---- tests/pgpro589.py | 2 +- tests/ptrack.py | 62 +++++++++++++------------- tests/ptrack_clean.py | 6 +-- tests/ptrack_cluster.py | 12 ++--- tests/ptrack_empty.py | 10 ++--- tests/ptrack_move_to_tablespace.py | 2 +- tests/ptrack_recovery.py | 2 +- tests/ptrack_truncate.py | 6 +-- tests/ptrack_vacuum.py | 6 +-- tests/ptrack_vacuum_bits_frozen.py | 6 +-- tests/ptrack_vacuum_bits_visibility.py | 2 +- tests/ptrack_vacuum_full.py | 6 +-- tests/ptrack_vacuum_truncate.py | 6 +-- tests/replica.py | 21 ++++----- tests/restore_test.py | 48 ++++++++++---------- tests/retention_test.py | 6 +-- tests/show_test.py | 12 ++--- tests/validate_test.py | 54 +++++++++++----------- 32 files changed, 287 insertions(+), 282 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 45e735e8d..1d0d7af57 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -21,7 +21,7 @@ def test_pgpro434_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -79,7 +79,7 @@ def test_pgpro434_2(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -228,7 +228,7 @@ def test_pgpro434_3(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -283,7 +283,7 @@ def test_arhive_push_file_exists(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -352,7 +352,7 @@ def test_arhive_push_file_exists_overwrite(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -418,7 +418,7 @@ def test_replica_archive(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -432,7 +432,7 @@ def test_replica_archive(self): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() master.psql( @@ -481,7 +481,7 @@ def test_replica_archive(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) node.append_conf( @@ -550,14 +550,14 @@ def test_master_and_replica_parallel_archiving(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'} ) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -640,7 +640,7 @@ def test_master_and_replica_concurrent_archiving(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -648,7 +648,7 @@ def test_master_and_replica_concurrent_archiving(self): 'archive_timeout': '10s'} ) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -723,7 +723,7 @@ def test_archive_pg_receivexlog(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -797,7 +797,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/auth_test.py b/tests/auth_test.py index c5f1ae331..cb13e6076 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -32,7 +32,7 @@ def test_backup_via_unpriviledged_user(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/backup_test.py b/tests/backup_test.py index 08f397e1e..24cd7b36a 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -17,7 +17,7 @@ def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -89,7 +89,7 @@ def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -113,7 +113,7 @@ def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -169,7 +169,7 @@ def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -234,7 +234,7 @@ def test_ptrack_threads(self): """ptrack multi thread backup mode""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -262,7 +262,7 @@ def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -293,7 +293,7 @@ def test_page_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -350,7 +350,7 @@ def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -423,7 +423,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): """PGPRO-1376 """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} diff --git a/tests/compatibility.py b/tests/compatibility.py index 39070b3ff..d8e1137a0 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -16,7 +16,7 @@ def test_backward_compatibility_page(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -47,7 +47,7 @@ def test_backward_compatibility_page(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -116,7 +116,7 @@ def test_backward_compatibility_delta(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -147,7 +147,7 @@ def test_backward_compatibility_delta(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -216,7 +216,7 @@ def test_backward_compatibility_ptrack(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -247,7 +247,7 @@ def test_backward_compatibility_ptrack(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -316,7 +316,7 @@ def test_backward_compatibility_compression(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -342,7 +342,7 @@ def test_backward_compatibility_compression(self): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() diff --git a/tests/compression.py b/tests/compression.py index f2c8614e0..2cb440187 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -18,7 +18,7 @@ def test_compression_stream_zlib(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -131,7 +131,7 @@ def test_compression_archive_zlib(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -238,7 +238,7 @@ def test_compression_stream_pglz(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -348,7 +348,7 @@ def test_compression_archive_pglz(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -458,7 +458,7 @@ def test_compression_wrong_algorithm(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -503,7 +503,7 @@ def test_uncompressable_pages(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/delete_test.py b/tests/delete_test.py index f58fc5fda..f49c01bf2 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -16,10 +16,10 @@ def test_delete_full_backups(self): """delete full backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -64,10 +64,11 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): def test_delete_increment_page(self): """delete increment and all after him""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -104,10 +105,11 @@ def test_delete_increment_page(self): def test_delete_increment_ptrack(self): """delete increment and all after him""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -144,10 +146,11 @@ def test_delete_increment_ptrack(self): def test_delete_orphaned_wal_segments(self): """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/delta.py b/tests/delta.py index 76673ceb1..d51c374cb 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -23,19 +23,17 @@ def test_delta_vacuum_truncate_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) + node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -105,7 +103,7 @@ def test_delta_vacuum_truncate_2(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -116,7 +114,7 @@ def test_delta_vacuum_truncate_2(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -196,7 +194,7 @@ def test_delta_vacuum_truncate_3(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -207,7 +205,7 @@ def test_delta_vacuum_truncate_3(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -266,7 +264,7 @@ def test_delta_stream(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -352,7 +350,7 @@ def test_delta_archive(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -434,7 +432,7 @@ def test_delta_multiple_segments(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -478,7 +476,7 @@ def test_delta_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -517,7 +515,7 @@ def test_delta_vacuum_full(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -527,7 +525,7 @@ def test_delta_vacuum_full(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -613,7 +611,7 @@ def test_create_db(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -659,7 +657,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -746,7 +744,7 @@ def test_exists_in_previous_backup(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -820,7 +818,7 @@ def test_exists_in_previous_backup(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -856,7 +854,7 @@ def test_alter_table_set_tablespace_delta(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -901,7 +899,7 @@ def test_alter_table_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -949,7 +947,7 @@ def test_alter_database_set_tablespace_delta(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1002,7 +1000,7 @@ def test_alter_database_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1043,7 +1041,7 @@ def test_delta_delete(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1092,7 +1090,7 @@ def test_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1125,7 +1123,7 @@ def test_delta_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1184,7 +1182,7 @@ def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1259,7 +1257,7 @@ def test_delta_nullified_heap_page_backup(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1307,7 +1305,7 @@ def test_delta_nullified_heap_page_backup(self): # Restore DELTA backup node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) node_restored.cleanup() diff --git a/tests/exclude.py b/tests/exclude.py index 30e500d61..b290a3b63 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -18,7 +18,7 @@ def test_exclude_temp_tables(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -109,7 +109,7 @@ def test_exclude_unlogged_tables_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -149,8 +149,8 @@ def test_exclude_unlogged_tables_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() self.restore_node( diff --git a/tests/false_positive.py b/tests/false_positive.py index 24f02276f..e67b75988 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -18,12 +18,12 @@ def test_validate_wal_lost_segment(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2'} - ) + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -58,10 +58,10 @@ def test_incremental_backup_corrupt_full_1(self): """page-level backup with corrupted full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -120,7 +120,7 @@ def test_ptrack_concurrent_get_and_clear_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -204,7 +204,7 @@ def test_ptrack_concurrent_get_and_clear_2(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -300,10 +300,11 @@ def test_ptrack_concurrent_get_and_clear_2(self): def test_multiple_delete(self): """delete multiple backups""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/init_test.py b/tests/init_test.py index 0b91dafa7..ff416e244 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -14,7 +14,7 @@ def test_success(self): """Success normal init""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) self.assertEqual( dir_files(backup_dir), @@ -66,7 +66,7 @@ def test_already_exist(self): """Failure with backup catalog already existed""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) try: self.show_pb(backup_dir, 'node') @@ -85,7 +85,7 @@ def test_abs_path(self): """failure with backup catalog should be given as absolute path""" fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname)) + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) try: self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( diff --git a/tests/merge.py b/tests/merge.py index 6b09c373d..dc5e39491 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -19,7 +19,7 @@ def test_merge_full_page(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"] ) @@ -108,7 +108,7 @@ def test_merge_compressed_backups(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"] ) @@ -173,7 +173,7 @@ def test_merge_compressed_backups_1(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -250,7 +250,7 @@ def test_merge_compressed_and_uncompressed_backups(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -326,7 +326,7 @@ def test_merge_compressed_and_uncompressed_backups_1(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -403,7 +403,7 @@ def test_merge_compressed_and_uncompressed_backups_2(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' @@ -481,7 +481,7 @@ def test_merge_tablespaces(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -561,7 +561,7 @@ def test_merge_tablespaces_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -645,7 +645,7 @@ def test_merge_page_truncate(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -656,7 +656,7 @@ def test_merge_page_truncate(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -740,7 +740,7 @@ def test_merge_delta_truncate(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -751,7 +751,7 @@ def test_merge_delta_truncate(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -835,7 +835,7 @@ def test_merge_ptrack_truncate(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -847,7 +847,7 @@ def test_merge_ptrack_truncate(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -931,7 +931,7 @@ def test_merge_delta_delete(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -983,7 +983,7 @@ def test_merge_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1019,7 +1019,7 @@ def test_continue_failed_merge(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica' @@ -1097,7 +1097,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica' diff --git a/tests/option_test.py b/tests/option_test.py index faed430ae..5a64a8b6f 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -52,7 +52,7 @@ def test_options_4(self): fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -118,7 +118,7 @@ def test_options_5(self): fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), pg_options={ 'wal_level': 'logical', 'max_wal_senders': '2'}) diff --git a/tests/page.py b/tests/page.py index 9e23ecaa7..388b0df99 100644 --- a/tests/page.py +++ b/tests/page.py @@ -23,7 +23,7 @@ def test_page_vacuum_truncate(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -34,7 +34,7 @@ def test_page_vacuum_truncate(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -116,7 +116,7 @@ def test_page_stream(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -205,7 +205,7 @@ def test_page_archive(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -300,7 +300,7 @@ def test_page_multiple_segments(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -341,7 +341,7 @@ def test_page_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -384,7 +384,7 @@ def test_page_delete(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -427,7 +427,7 @@ def test_page_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -464,7 +464,7 @@ def test_page_delete_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -508,7 +508,7 @@ def test_page_delete_1(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -544,14 +544,14 @@ def test_parallel_pagemap(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ "hot_standby": "on" } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -619,7 +619,7 @@ def test_parallel_pagemap_1(self): # Initialize instance and backup directory node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={} ) @@ -673,7 +673,7 @@ def test_page_backup_with_lost_wal_segment(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -763,7 +763,7 @@ def test_page_backup_with_corrupted_wal_segment(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -884,13 +884,12 @@ def test_page_backup_with_alien_wal_segment(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) alien_node = self.make_simple_node( - base_dir="{0}/{1}/alien_node".format(module_name, fname) - ) + base_dir=os.path.join(module_name, fname, 'alien_node')) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1004,7 +1003,7 @@ def test_multithread_page_backup_with_toast(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1042,7 +1041,7 @@ def test_page_create_db(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1082,7 +1081,7 @@ def test_page_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 6c2b88634..ed0110741 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -20,11 +20,12 @@ def test_pgpro560_control_file_loss(self): check that backup failed """ fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -55,17 +56,19 @@ def test_pgpro560_systemid_mismatch(self): check that backup failed """ fname = self.id().split('.')[3] - node1 = self.make_simple_node(base_dir="{0}/{1}/node1".format(module_name, fname), + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + node1.slow_start() - node2 = self.make_simple_node(base_dir="{0}/{1}/node2".format(module_name, fname), + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + node2.slow_start() backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/pgpro589.py b/tests/pgpro589.py index d3bb9d908..11809528e 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -19,7 +19,7 @@ def test_pgpro589(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/ptrack.py b/tests/ptrack.py index 5d01d8823..376e36a99 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -21,7 +21,7 @@ def test_ptrack_enable(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -70,7 +70,7 @@ def test_ptrack_disable(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -128,7 +128,7 @@ def test_ptrack_uncommited_xact(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -139,7 +139,7 @@ def test_ptrack_uncommited_xact(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -190,7 +190,7 @@ def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -201,7 +201,7 @@ def test_ptrack_vacuum_full(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -286,7 +286,7 @@ def test_ptrack_vacuum_truncate(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -298,7 +298,7 @@ def test_ptrack_vacuum_truncate(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -372,7 +372,7 @@ def test_ptrack_simple(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -383,7 +383,7 @@ def test_ptrack_simple(self): } ) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -448,7 +448,7 @@ def test_ptrack_get_block(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -520,7 +520,7 @@ def test_ptrack_stream(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -616,7 +616,7 @@ def test_ptrack_archive(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -722,7 +722,7 @@ def test_ptrack_pgpro417(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -806,7 +806,7 @@ def test_page_pgpro417(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -878,7 +878,7 @@ def test_full_pgpro417(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -955,7 +955,7 @@ def test_create_db(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1003,7 +1003,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1082,7 +1082,7 @@ def test_create_db_on_replica(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1106,7 +1106,7 @@ def test_create_db_on_replica(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node( @@ -1159,7 +1159,7 @@ def test_create_db_on_replica(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1183,7 +1183,7 @@ def test_alter_table_set_tablespace_ptrack(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1232,7 +1232,7 @@ def test_alter_table_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname) + base_dir=os.path.join(module_name, fname, 'node_restored') ) node_restored.cleanup() @@ -1280,7 +1280,7 @@ def test_alter_database_set_tablespace_ptrack(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1317,7 +1317,7 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1351,7 +1351,7 @@ def test_drop_tablespace(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1435,7 +1435,7 @@ def test_ptrack_alter_tablespace(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1478,7 +1478,7 @@ def test_ptrack_alter_tablespace(self): # Restore ptrack backup restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') @@ -1548,7 +1548,7 @@ def test_ptrack_multiple_segments(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1613,7 +1613,7 @@ def test_ptrack_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir="{0}/{1}/restored_node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -1655,7 +1655,7 @@ def test_atexit_fail(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 605d631a0..01d364e9e 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -15,7 +15,7 @@ def test_ptrack_clean(self): """Take backups of every available types and check that PTRACK is clean""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -124,7 +124,7 @@ def test_ptrack_clean_replica(self): """Take backups of every available types from master and check that PTRACK on replica is clean""" fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -141,7 +141,7 @@ def test_ptrack_clean_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 41de27df7..270cb1b40 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -15,7 +15,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_cluster_on_btree(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -96,7 +96,7 @@ def test_ptrack_cluster_on_btree(self): def test_ptrack_cluster_on_gist(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -174,7 +174,7 @@ def test_ptrack_cluster_on_gist(self): def test_ptrack_cluster_on_btree_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -190,7 +190,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -274,7 +274,7 @@ def test_ptrack_cluster_on_btree_replica(self): def test_ptrack_cluster_on_gist_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -290,7 +290,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 18f289e77..578c8a73b 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -15,7 +15,7 @@ def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -58,7 +58,7 @@ def test_ptrack_empty(self): node.safe_psql('postgres', 'checkpoint') node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') @@ -93,7 +93,7 @@ def test_ptrack_empty_replica(self): """Take backups of every available types from master and check that PTRACK on replica is clean""" fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -108,7 +108,7 @@ def test_ptrack_empty_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -151,7 +151,7 @@ def test_ptrack_empty_replica(self): self.wait_until_replica_catch_with_master(master, replica) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() # Take PTRACK backup diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py index 41b92cf77..2011d2a2d 100644 --- a/tests/ptrack_move_to_tablespace.py +++ b/tests/ptrack_move_to_tablespace.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py index 5889b06ed..068d7dd09 100644 --- a/tests/ptrack_recovery.py +++ b/tests/ptrack_recovery.py @@ -14,7 +14,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_recovery(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py index 662a93f82..45a431e0e 100644 --- a/tests/ptrack_truncate.py +++ b/tests/ptrack_truncate.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_truncate(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -74,7 +74,7 @@ def test_ptrack_truncate(self): def test_ptrack_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -91,7 +91,7 @@ def test_ptrack_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index 2283c740b..bf3d8d691 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -100,7 +100,7 @@ def test_ptrack_vacuum(self): def test_ptrack_vacuum_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -117,7 +117,7 @@ def test_ptrack_vacuum_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 0da323b9b..637807b52 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_bits_frozen(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -93,7 +93,7 @@ def test_ptrack_vacuum_bits_frozen(self): def test_ptrack_vacuum_bits_frozen_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -109,7 +109,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py index a5fa5206b..53be4401c 100644 --- a/tests/ptrack_vacuum_bits_visibility.py +++ b/tests/ptrack_vacuum_bits_visibility.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_bits_visibility(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py index a20ae164e..826742fba 100644 --- a/tests/ptrack_vacuum_full.py +++ b/tests/ptrack_vacuum_full.py @@ -14,7 +14,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_full(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -95,7 +95,7 @@ def test_ptrack_vacuum_full(self): def test_ptrack_vacuum_full_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -112,7 +112,7 @@ def test_ptrack_vacuum_full_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py index 406e2b3ab..c09913539 100644 --- a/tests/ptrack_vacuum_truncate.py +++ b/tests/ptrack_vacuum_truncate.py @@ -13,7 +13,7 @@ class SimpleTest(ProbackupTest, unittest.TestCase): def test_ptrack_vacuum_truncate(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -94,7 +94,7 @@ def test_ptrack_vacuum_truncate(self): def test_ptrack_vacuum_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -111,7 +111,7 @@ def test_ptrack_vacuum_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) diff --git a/tests/replica.py b/tests/replica.py index 052dee2c0..d9eabcbbe 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -22,7 +22,7 @@ def test_replica_stream_ptrack_backup(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -45,13 +45,14 @@ def test_replica_stream_ptrack_backup(self): # take full backup and restore it self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) # Check data correctness on replica replica.slow_start(replica=True) + exit(1) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) @@ -79,7 +80,7 @@ def test_replica_stream_ptrack_backup(self): # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -138,7 +139,7 @@ def test_replica_archive_page_backup(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -152,7 +153,7 @@ def test_replica_archive_page_backup(self): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -211,7 +212,7 @@ def test_replica_archive_page_backup(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -289,7 +290,7 @@ def test_make_replica_via_restore(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -305,7 +306,7 @@ def test_make_replica_via_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -350,7 +351,7 @@ def test_take_backup_from_delayed_replica(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( - base_dir="{0}/{1}/master".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -364,7 +365,7 @@ def test_take_backup_from_delayed_replica(self): master.slow_start() replica = self.make_simple_node( - base_dir="{0}/{1}/replica".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) diff --git a/tests/restore_test.py b/tests/restore_test.py index 2ada39556..92c85a79c 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -18,7 +18,7 @@ def test_restore_full_to_latest(self): """recovery to latest from full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -66,7 +66,7 @@ def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -114,7 +114,7 @@ def test_restore_to_specific_timeline(self): """recovery to target timeline""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -183,7 +183,7 @@ def test_restore_to_time(self): """recovery to target time""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -232,7 +232,7 @@ def test_restore_to_xid_inclusive(self): """recovery to target xid""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -293,7 +293,7 @@ def test_restore_to_xid_not_inclusive(self): """recovery with target inclusive false""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -358,7 +358,7 @@ def test_restore_to_lsn_inclusive(self): """recovery to target lsn""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -431,7 +431,7 @@ def test_restore_to_lsn_not_inclusive(self): """recovery to target lsn""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -505,7 +505,7 @@ def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -553,7 +553,7 @@ def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} ) @@ -608,7 +608,7 @@ def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -663,7 +663,7 @@ def test_restore_full_ptrack_under_load(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -729,7 +729,7 @@ def test_restore_full_under_load_ptrack(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -794,7 +794,7 @@ def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -913,7 +913,7 @@ def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -995,7 +995,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1045,7 +1045,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1094,7 +1094,7 @@ def test_archive_node_backup_stream_pitr(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1143,7 +1143,7 @@ def test_archive_node_backup_archive_pitr_2(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1198,7 +1198,7 @@ def test_archive_restore_to_restore_point(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1247,7 +1247,7 @@ def test_archive_restore_to_restore_point(self): def test_zags_block_corrupt(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1303,7 +1303,7 @@ def test_zags_block_corrupt(self): node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1325,7 +1325,7 @@ def test_zags_block_corrupt(self): def test_zags_block_corrupt_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica', @@ -1395,7 +1395,7 @@ def test_zags_block_corrupt_1(self): self.switch_wal_segment(node) node_restored = self.make_simple_node( - base_dir="{0}/{1}/node_restored".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node_restored'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/retention_test.py b/tests/retention_test.py index 91fef215f..34c026587 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -15,7 +15,7 @@ def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -72,7 +72,7 @@ def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -126,7 +126,7 @@ def test_retention_wal(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/show_test.py b/tests/show_test.py index 655c0b8e3..eef897180 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -15,7 +15,7 @@ def test_show_1(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -43,7 +43,7 @@ def test_show_json(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -71,7 +71,7 @@ def test_corrupt_2(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -117,7 +117,7 @@ def test_no_control_file(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -146,7 +146,7 @@ def test_empty_control_file(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -177,7 +177,7 @@ def test_corrupt_control_file(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) diff --git a/tests/validate_test.py b/tests/validate_test.py index 6153a4a4a..cdd960e7c 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -21,7 +21,7 @@ def test_validate_nullified_heap_page_backup(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -73,7 +73,7 @@ def test_validate_wal_unreal_values(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -213,7 +213,7 @@ def test_validate_corrupted_intermediate_backup(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -298,7 +298,7 @@ def test_validate_corrupted_intermediate_backups(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -421,7 +421,7 @@ def test_validate_corrupted_intermediate_backups_1(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -620,7 +620,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -804,7 +804,7 @@ def test_validate_instance_with_corrupted_page(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -952,7 +952,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, try to restore backup with --no-validation option""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1038,7 +1038,7 @@ def test_validate_instance_with_corrupted_full(self): corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1113,7 +1113,7 @@ def test_validate_corrupt_wal_1(self): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1175,7 +1175,7 @@ def test_validate_corrupt_wal_1(self): def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1247,7 +1247,7 @@ def test_validate_wal_lost_segment_1(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1325,7 +1325,7 @@ def test_validate_corrupt_wal_between_backups(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -1419,7 +1419,7 @@ def test_pgpro702_688(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1461,7 +1461,7 @@ def test_pgpro688(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1491,7 +1491,7 @@ def test_pgpro561(self): """ fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir="{0}/{1}/node1".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1506,7 +1506,7 @@ def test_pgpro561(self): backup_dir, 'node1', node1, options=["--stream"]) node2 = self.make_simple_node( - base_dir="{0}/{1}/node2".format(module_name, fname)) + base_dir=os.path.join(module_name, fname, 'node2')) node2.cleanup() node1.psql( @@ -1582,7 +1582,7 @@ def test_validate_corrupted_full(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1708,7 +1708,7 @@ def test_validate_corrupted_full_1(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -1820,7 +1820,7 @@ def test_validate_corrupted_full_2(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2170,7 +2170,7 @@ def test_validate_corrupted_full_missing(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2394,7 +2394,7 @@ def test_file_size_corruption_no_validate(self): fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), # initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica'} ) @@ -2472,7 +2472,7 @@ def test_validate_specific_backup_with_missing_backup(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2614,7 +2614,7 @@ def test_validate_specific_backup_with_missing_backup_1(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2735,7 +2735,7 @@ def test_validate_with_missing_backup_1(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -2923,7 +2923,7 @@ def test_validate_with_missing_backup_2(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} @@ -3071,7 +3071,7 @@ def test_corrupt_pg_control_via_resetxlog(self): """ PGPRO-2096 """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} From 361d408c19cebb7d24e826e11cc8a86095dd6a78 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:00:57 +0300 Subject: [PATCH 0185/2107] tests: support new behaviour of node.cleanup() in testgres --- tests/restore_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 92c85a79c..9c2604d18 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -5,6 +5,7 @@ from datetime import datetime import sys import time +import shutil module_name = 'restore' @@ -840,7 +841,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - node.cleanup() + shutil.rmtree(node.data_dir, ignore_errors=True) try: self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen From c7dee40d134c290f9ece90297327f31f2fd0012a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:04:16 +0300 Subject: [PATCH 0186/2107] tests: windows support for test_restore_with_tablespace_mapping_1 --- tests/restore_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 9c2604d18..db69538c4 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -833,10 +833,9 @@ def test_restore_with_tablespace_mapping_1(self): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: restore destination is not empty: "{0}"'.format(node.data_dir), e.message, - 'ERROR: restore destination is not empty: "{0}"\n'.format( - node.data_dir), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 8f9ca4ae9f445ec824139300f116d5c447dd5a2c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:18:19 +0300 Subject: [PATCH 0187/2107] tests: windows support for test_restore_with_tablespace_mapping_1 --- tests/restore_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index db69538c4..35933621b 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -840,7 +840,10 @@ def test_restore_with_tablespace_mapping_1(self): repr(e.message), self.cmd)) # 2 - Try to restore to existing tablespace directory - shutil.rmtree(node.data_dir, ignore_errors=True) + tblspc_path_tmp = os.path.join(node.base_dir, "tblspc_tmp") + os.rename(tblspc_path, tblspc_path_tmp) + node.cleanup() + os.rename(tblspc_path_tmp, tblspc_path) try: self.restore_node(backup_dir, 'node', node) # we should die here because exception is what we expect to happen @@ -850,10 +853,9 @@ def test_restore_with_tablespace_mapping_1(self): "not empty.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: restore tablespace destination is not empty:', e.message, - 'ERROR: restore tablespace destination ' - 'is not empty: "{0}"\n'.format(tblspc_path), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 68ac0b43e26f443abd91c149e5d76fb1a9fd4238 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 13:32:46 +0300 Subject: [PATCH 0188/2107] README: Windows support in beta --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c2267d08..a356b81b5 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * Creating backups from a remote server is currently not supported. * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. -* Microsoft Windows operating system is not supported. +* Microsoft Windows operating system support is in beta stage. * Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. ## Installation and Setup @@ -71,13 +71,28 @@ yum install pg_probackup-{11,10,9.6,9.5} yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). + +## Building from source +### Linux + To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: ```shell make USE_PGXS=1 PG_CONFIG= top_srcdir= ``` +### Windows -Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). +Currently pg_probackup can be build using only MSVC 2013. +Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/10/install-windows-full.html) with MSVC 2013. +If zlib support is needed, src/tools/msvc/config.pl must contain path to directory with compiled zlib. [Example](https://gist.githubusercontent.com/gsmol/80989f976ce9584824ae3b1bfb00bd87/raw/240032950d4ac4801a79625dd00c8f5d4ed1180c/gistfile1.txt) + +```shell +CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +SET PATH=%PATH%;C:\Perl64\bin +SET PATH=%PATH%;C:\msys64\usr\bin +gen_probackup_project.pl C:\path_to_postgresql_source_tree +``` ## Documentation From f6c1966415df3bf81e5dee06b7c3445d9e6ea9fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 17:19:23 +0300 Subject: [PATCH 0189/2107] tests: windows support, eschew using node.restart() --- tests/auth_test.py | 3 ++- tests/ptrack.py | 6 ++++-- tests/replica.py | 3 ++- tests/restore_test.py | 1 - tests/validate_test.py | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index cb13e6076..153044e10 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -126,7 +126,8 @@ def test_backup_via_unpriviledged_user(self): "test1", "create table t1 as select generate_series(0,100)") node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") - node.restart() + node.stop() + node.slow_start() try: self.backup_node( diff --git a/tests/ptrack.py b/tests/ptrack.py index 376e36a99..9f52a4664 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -89,11 +89,13 @@ def test_ptrack_disable(self): # DISABLE PTRACK node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.restart() + node.stop() + node.slow_start() # ENABLE PTRACK node.safe_psql('postgres', "alter system set ptrack_enable to on") - node.restart() + node.stop() + node.slow_start() # PTRACK BACKUP try: diff --git a/tests/replica.py b/tests/replica.py index d9eabcbbe..4eb4b1e20 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -403,7 +403,8 @@ def test_take_backup_from_delayed_replica(self): replica.append_conf( 'recovery.conf', "recovery_min_apply_delay = '300s'") - replica.restart() + replica.stop() + replica.slow_start() master.pgbench_init(scale=10) diff --git a/tests/restore_test.py b/tests/restore_test.py index 35933621b..e130894a3 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -5,7 +5,6 @@ from datetime import datetime import sys import time -import shutil module_name = 'restore' diff --git a/tests/validate_test.py b/tests/validate_test.py index cdd960e7c..1fc0c16ef 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1528,7 +1528,8 @@ def test_pgpro561(self): node2.append_conf( 'postgresql.auto.conf', 'archive_mode = on') - node2.restart() + node2.stop() + node2.slow_start() timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] From 9463b579909d9f3cf74a2edd1fb74d71874c0fba Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 27 Dec 2018 18:47:15 +0300 Subject: [PATCH 0190/2107] Add parameter 'extra-directory' to pg_probackup.conf. Minor improvements. --- src/backup.c | 7 +++++-- src/configure.c | 5 +++++ src/dir.c | 32 ++++++++++++-------------------- src/help.c | 28 ++++++++++++++++++++-------- src/pg_probackup.c | 3 +-- src/pg_probackup.h | 7 +++---- src/restore.c | 7 +++++-- 7 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/backup.c b/src/backup.c index 15ca8bf3a..ff6725467 100644 --- a/src/backup.c +++ b/src/backup.c @@ -991,8 +991,11 @@ do_backup(time_t start_time) sizeof(current.program_version)); /* Save list of extra directories */ - if(extradir) - current.extra_dir_str = extradir; + if (instance_config.extra_dir_str && + strcmp(instance_config.extra_dir_str, "none") != 0) + { + current.extra_dir_str = instance_config.extra_dir_str; + } /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) diff --git a/src/configure.c b/src/configure.c index a9c80922a..b20775884 100644 --- a/src/configure.c +++ b/src/configure.c @@ -61,6 +61,11 @@ ConfigOption instance_options[] = OPTION_INSTANCE_GROUP, 0, option_get_value }, #endif + { + 's', 'E', "extra-directory", + &instance_config.extra_dir_str, SOURCE_CMD, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, /* Connection options */ { 's', 'd', "pgdatabase", diff --git a/src/dir.c b/src/dir.c index 24e8d5a4c..b37fcab63 100644 --- a/src/dir.c +++ b/src/dir.c @@ -116,6 +116,8 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; +static int BlackListCompare(const void *str1, const void *str2); + static bool dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, @@ -124,13 +126,12 @@ static void dir_list_file_internal(parray *files, const char *root, static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); -static void free_extra_remap_list(void *cell); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* Extra directories mapping */ -static parray *extra_remap_list = NULL; +parray *extra_remap_list = NULL; /* * Create directory, also create parent directories if necessary. @@ -405,8 +406,7 @@ pgFileCompareSize(const void *f1, const void *f2) return 0; } -/* Compare two strings */ -int +static int BlackListCompare(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); @@ -724,8 +724,6 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, continue; } - /* If it is extra dir, remember it */ - /* We add the directory anyway */ if (S_ISDIR(file->mode)) parray_append(files, file); @@ -960,7 +958,9 @@ opt_extradir_map(ConfigOption *opt, const char *arg) char *dst_ptr; const char *arg_ptr; - extra_remap_list = parray_new(); + memset(cell, 0, sizeof(TablespaceListCell)); + if (!extra_remap_list) + extra_remap_list = parray_new(); dst_ptr = dst = cell->old_dir; for (arg_ptr = arg; *arg_ptr; arg_ptr++) { @@ -1284,7 +1284,7 @@ check_extra_dir_mapping(char *current_dir) return current_dir; } -static void +void free_extra_remap_list(void *cell) { TablespaceListCell *cell_ptr; @@ -1294,16 +1294,6 @@ free_extra_remap_list(void *cell) pfree(cell_ptr); } -void -clean_extra_dirs_remap_list(void) -{ - if (extra_remap_list) - { - parray_walk(extra_remap_list, free_extra_remap_list); - parray_free(extra_remap_list); - } -} - /* * Print backup content list. */ @@ -1657,14 +1647,16 @@ make_extra_directory_list(const char *colon_separated_dirs) parray *list = parray_new(); char *tmp = palloc(strlen(colon_separated_dirs) + 1); - /* TODO: Add path validation */ strcpy(tmp, colon_separated_dirs); p = strtok(tmp,":"); while(p!=NULL) { char * dir = (char *)palloc(strlen(p) + 1); strcpy(dir,p); - parray_append(list, dir); + if (is_absolute_path(dir)) + parray_append(list, dir); + else + elog(ERROR, "Extra directory \"%s\" is not an absolute path", dir); p=strtok(NULL,":"); } pfree(tmp); diff --git a/src/help.c b/src/help.c index 6b2cd9292..578ef289b 100644 --- a/src/help.c +++ b/src/help.c @@ -119,18 +119,18 @@ help_pg_probackup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [--extra-directory]\n")); + printf(_(" [--extra-directory=extra-directory-path]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [-E OLDDIR=NEWDIR]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); @@ -209,7 +209,8 @@ help_backup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [--skip-block-validation]\n")); + printf(_(" [-E extra-directory=extra-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -221,7 +222,9 @@ help_backup(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); - printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + printf(_(" --skip-block-validation sextra-directoryet to validate only file-level checksum\n")); + printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" backup some directory not from pgdata \n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -283,9 +286,10 @@ help_restore(void) printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica] [--no-validate]\n\n")); + printf(_(" [--restore-as-replica] [--no-validate]\n")); printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -302,6 +306,8 @@ help_restore(void) printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + printf(_(" --extra-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the extra directory from OLDDIR to NEWDIR\n")); printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); printf(_(" --recovery-target-name=target-name\n")); @@ -490,11 +496,14 @@ help_set_config(void) printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n\n")); - printf(_(" [--archive-timeout=timeout]\n\n")); + printf(_(" [--replica-timeout=timeout]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + printf(_(" [-E extra-directory=extra-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" backup some directory not from pgdata \n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -560,11 +569,14 @@ static void help_add_instance(void) { printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n\n")); + printf(_(" --instance=instance_name\n")); + printf(_(" -E extra-directory=extra-directory-path\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --instance=instance_name name of the new instance\n")); + printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" backup some directory not from pgdata \n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fe88840fd..c76bed66e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -137,8 +137,6 @@ static ConfigOption cmd_options[] = { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 's', 155, "extra-directory", &extradir, SOURCE_CMD_STRICT }, - { 'f', 'E', "extra-mapping", opt_extradir_map, SOURCE_CMD_STRICT }, /* TODO not completed feature. Make it unavailiable from user level { 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */ /* restore options */ @@ -147,6 +145,7 @@ static ConfigOption cmd_options[] = { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, + { 'f', 155, "extra-mapping", opt_extradir_map, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 670d852d7..ef988610a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -180,6 +180,7 @@ typedef struct InstanceConfig uint32 xlog_seg_size; char *pgdata; + char *extra_dir_str; const char *pgdatabase; const char *pghost; const char *pgport; @@ -352,9 +353,6 @@ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; -/* extra directory to backup */ -extern char *extradir; - /* common options */ extern int num_threads; extern bool stream_wal; @@ -396,6 +394,7 @@ extern pgBackup current; /* in dir.c */ /* exclude directory list for $PGDATA file listing */ extern const char *pgdata_exclude_dir[]; +extern parray *extra_remap_list; /* in backup.c */ extern int do_backup(time_t start_time); @@ -522,6 +521,7 @@ extern void free_dir_list(parray *list); extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_extra(const char *dir, parray *dirs_list); +extern void free_extra_remap_list(void *cell); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); @@ -540,7 +540,6 @@ extern int pgFileComparePathWithExtra(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); -extern int BlackListCompare(const void *str1, const void *str2); /* in data.c */ extern bool backup_data_file(backup_files_arg* arguments, diff --git a/src/restore.c b/src/restore.c index 43133ff4d..1148f26f9 100644 --- a/src/restore.c +++ b/src/restore.c @@ -407,6 +407,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); + if (extra_remap_list) + { + parray_walk(extra_remap_list, free_extra_remap_list); + parray_free(extra_remap_list); + } elog(INFO, "%s of backup %s completed.", action, base36enc(dest_backup->start_time)); @@ -558,8 +563,6 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) parray_walk(files, pgFileFree); parray_free(files); - clean_extra_dirs_remap_list(); - if (logger_config.log_level_console <= LOG || logger_config.log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); From 7d53afb1c749cdedbff7ac3b086acb24ec57e29b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Dec 2018 22:40:23 +0300 Subject: [PATCH 0191/2107] tests: windows support for validate --- tests/backup_test.py | 2 +- tests/merge.py | 6 ++- tests/validate_test.py | 94 +++++++++++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 24cd7b36a..e3fdac15f 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -484,7 +484,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): list = [] for root, dirs, files in os.walk(os.path.join( - backup_dir, 'backups/node', backup_id_1)): + backup_dir, 'backups', 'node', backup_id_1)): for file in files: if file == relfilenode: path = os.path.join(root, file) diff --git a/tests/merge.py b/tests/merge.py index dc5e39491..8d190811c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1164,7 +1164,8 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # read block from future # block_size + backup_header = 8200 file = os.path.join( - backup_dir, 'backups/node', backup_id_2, 'database', new_path) + backup_dir, 'backups', 'node', + backup_id_2, 'database', new_path) with open(file, 'rb') as f: f.seek(8200) block_1 = f.read(8200) @@ -1172,7 +1173,8 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # write block from future file = os.path.join( - backup_dir, 'backups/node', backup_id, 'database', old_path) + backup_dir, 'backups', 'node', + backup_id, 'database', old_path) with open(file, 'r+b') as f: f.seek(8200) f.write(block_1) diff --git a/tests/validate_test.py b/tests/validate_test.py index 1fc0c16ef..e40fb3890 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -249,7 +249,8 @@ def test_validate_corrupted_intermediate_backup(self): # Corrupt some file file = os.path.join( - backup_dir, 'backups/node', backup_id_2, 'database', file_path) + backup_dir, 'backups', 'node', + backup_id_2, 'database', file_path) with open(file, "r+b", 0) as f: f.seek(42) f.write(b"blah") @@ -342,7 +343,7 @@ def test_validate_corrupted_intermediate_backups(self): # Corrupt some file in FULL backup file_full = os.path.join( - backup_dir, 'backups/node', + backup_dir, 'backups', 'node', backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) @@ -352,7 +353,7 @@ def test_validate_corrupted_intermediate_backups(self): # Corrupt some file in PAGE1 backup file_page1 = os.path.join( - backup_dir, 'backups/node', + backup_dir, 'backups', 'node', backup_id_2, 'database', file_path_t_heap_1) with open(file_page1, "rb+", 0) as f: f.seek(42) @@ -379,8 +380,7 @@ def test_validate_corrupted_intermediate_backups(self): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_1) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_full) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -499,7 +499,7 @@ def test_validate_corrupted_intermediate_backups_1(self): # Corrupt some file in PAGE2 and PAGE5 backups file_page1 = os.path.join( - backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + backup_dir, 'backups', 'node', backup_id_3, 'database', file_page_2) with open(file_page1, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -507,7 +507,7 @@ def test_validate_corrupted_intermediate_backups_1(self): f.close file_page4 = os.path.join( - backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + backup_dir, 'backups', 'node', backup_id_6, 'database', file_page_5) with open(file_page4, "rb+", 0) as f: f.seek(42) f.write(b"blah") @@ -547,8 +547,7 @@ def test_validate_corrupted_intermediate_backups_1(self): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_page1) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_3) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -698,7 +697,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): # Corrupt some file in PAGE2 and PAGE5 backups file_page1 = os.path.join( - backup_dir, 'backups/node', backup_id_3, 'database', file_page_2) + backup_dir, 'backups', 'node', + backup_id_3, 'database', file_page_2) with open(file_page1, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -706,7 +706,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): f.close file_page4 = os.path.join( - backup_dir, 'backups/node', backup_id_6, 'database', file_page_5) + backup_dir, 'backups', 'node', + backup_id_6, 'database', file_page_5) with open(file_page4, "rb+", 0) as f: f.seek(42) f.write(b"blah") @@ -747,8 +748,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_page1) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_3) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -851,7 +851,7 @@ def test_validate_instance_with_corrupted_page(self): # Corrupt some file in FULL backup file_full = os.path.join( - backup_dir, 'backups/node', backup_id_2, + backup_dir, 'backups', 'node', backup_id_2, 'database', file_path_t_heap1) with open(file_full, "rb+", 0) as f: f.seek(84) @@ -906,8 +906,7 @@ def test_validate_instance_with_corrupted_page(self): self.assertTrue( 'INFO: Validating backup {0}'.format( backup_id_2) in e.message and - 'WARNING: Invalid CRC of backup file "{0}"'.format( - file_full) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format( backup_id_2) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -964,7 +963,9 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() @@ -973,14 +974,18 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") # PAGE1 backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') # PAGE2 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') # FULL1 @@ -989,11 +994,15 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): # PAGE3 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -1009,7 +1018,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): self.assertTrue( 'INFO: Validating backup {0}'.format(backup_id_1) in e.message and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -1050,7 +1059,10 @@ def test_validate_instance_with_corrupted_full(self): node.safe_psql( "postgres", - "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() @@ -1059,27 +1071,40 @@ def test_validate_instance_with_corrupted_full(self): node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # PAGE1 - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # PAGE2 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(20000,30000) i") - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.backup_node( + backup_dir, 'node', node) # PAGE3 node.safe_psql( "postgres", - "insert into t_heap select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(30000,40000) i") + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join(backup_dir, 'backups/node', backup_id_1, 'database', file_path_t_heap) + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) with open(file_full, "rb+", 0) as f: f.seek(84) f.write(b"blah") @@ -1089,13 +1114,16 @@ def test_validate_instance_with_corrupted_full(self): # Validate Instance try: self.validate_pb(backup_dir, 'node') - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( 'INFO: Validating backup {0}'.format(backup_id_1) in e.message and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file "{0}"'.format(file_full) in e.message + and 'WARNING: Invalid CRC of backup file' in e.message and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From 723c2b01a26897a4d0f16ff8066be24e75fe2230 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 28 Dec 2018 18:46:12 +0300 Subject: [PATCH 0192/2107] Bug fix: Consider target_lsn during validate WAL --- src/parsexlog.c | 24 ++++++++++++++++++------ src/pg_probackup.c | 8 ++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index a1ca14da5..b645ca703 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -519,6 +519,7 @@ validate_wal(pgBackup *backup, const char *archivedir, TimestampTz last_time = 0; char last_timestamp[100], target_timestamp[100]; + XLogRecPtr last_lsn = InvalidXLogRecPtr; bool all_wal = false; char backup_xlog_path[MAXPGPATH]; @@ -585,6 +586,7 @@ validate_wal(pgBackup *backup, const char *archivedir, /* We can restore at least up to the backup end */ time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); last_xid = backup->recovery_xid; + last_lsn = backup->stop_lsn; if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) || (target_time != 0 && backup->recovery_time >= target_time) @@ -608,6 +610,7 @@ validate_wal(pgBackup *backup, const char *archivedir, timestamp_record = getRecordTimestamp(xlogreader, &last_time); if (XLogRecGetXid(xlogreader) != InvalidTransactionId) last_xid = XLogRecGetXid(xlogreader); + last_lsn = xlogreader->ReadRecPtr; /* Check target xid */ if (TransactionIdIsValid(target_xid) && target_xid == last_xid) @@ -616,12 +619,19 @@ validate_wal(pgBackup *backup, const char *archivedir, break; } /* Check target time */ - else if (target_time != 0 && timestamp_record && timestamptz_to_time_t(last_time) >= target_time) + else if (target_time != 0 && timestamp_record && + timestamptz_to_time_t(last_time) >= target_time) { all_wal = true; break; } - /* If there are no target xid and target time */ + /* Check target lsn */ + else if (XRecOffIsValid(target_xid) && last_lsn >= target_lsn) + { + all_wal = true; + break; + } + /* If there are no target xid, target time and target lsn */ else if (!TransactionIdIsValid(target_xid) && target_time == 0 && xlogreader->ReadRecPtr == backup->stop_lsn) { @@ -638,15 +648,17 @@ validate_wal(pgBackup *backup, const char *archivedir, /* There are all needed WAL records */ if (all_wal) - elog(INFO, "backup validation completed successfully on time %s and xid " XID_FMT, - last_timestamp, last_xid); + elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", + last_timestamp, last_xid, + (uint32) (last_lsn >> 32), (uint32) last_lsn); /* Some needed WAL records are absent */ else { PrintXLogCorruptionMsg(&private, WARNING); - elog(WARNING, "recovery can be done up to time %s and xid " XID_FMT, - last_timestamp, last_xid); + elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", + last_timestamp, last_xid, + (uint32) (last_lsn >> 32), (uint32) last_lsn); if (target_time > 0) time2iso(target_timestamp, lengthof(target_timestamp), diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 10478970e..2cca5a2fe 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -69,10 +69,10 @@ bool smooth_checkpoint; bool is_remote_backup = false; /* restore options */ -static char *target_time; -static char *target_xid; -static char *target_lsn; -static char *target_inclusive; +static char *target_time = NULL; +static char *target_xid = NULL; +static char *target_lsn = NULL; +static char *target_inclusive = NULL; static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; From c69d4ed7f81e9ad0e904ece9a66845829bc27d1b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 28 Dec 2018 19:24:55 +0300 Subject: [PATCH 0193/2107] tests: validate fix (windows support) --- tests/validate_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index e40fb3890..3c1aee59d 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -56,7 +56,7 @@ def test_validate_nullified_heap_page_backup(self): with open(log_file_path) as f: self.assertTrue( - 'LOG: File: {0} blknum 1, empty page'.format(file) in f.read(), + '{0} blknum 1, empty page'.format(file_path) in f.read(), 'Failed to detect nullified block') self.validate_pb(backup_dir) @@ -3134,6 +3134,7 @@ def test_corrupt_pg_control_via_resetxlog(self): self.run_binary( [ pg_resetxlog_path, + '-D', os.path.join(backup_dir, 'backups', 'node', backup_id, 'database'), '-o 42', '-f' From 93cedf1747595b0713b21081243a91dd439be6d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 29 Dec 2018 12:42:59 +0300 Subject: [PATCH 0194/2107] tests: fix test_validate_wal_unreal_values() and test_validate_wal_lost_segment_1() --- tests/validate_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index 3c1aee59d..f2bf52fe1 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -117,9 +117,9 @@ def test_validate_wal_unreal_values(self): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual( + self.assertIn( + 'ERROR: Backup satisfying target options is not found', e.message, - 'ERROR: Backup satisfying target options is not found.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1309,8 +1309,7 @@ def test_validate_wal_lost_segment_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WAL segment \"{0}\" is absent".format( - file) in e.message and + "is absent" in e.message and "WARNING: There are not enough WAL records to consistenly " "restore backup {0}".format(backup_id) in e.message and "WARNING: Backup {0} WAL segments are corrupted".format( From 687b069f091986cf9867343ec5a4bf11db918b08 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Sat, 29 Dec 2018 17:13:34 +0300 Subject: [PATCH 0195/2107] Bug fix: Fix for 723c2b01a --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index b645ca703..44c87ff84 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -626,14 +626,14 @@ validate_wal(pgBackup *backup, const char *archivedir, break; } /* Check target lsn */ - else if (XRecOffIsValid(target_xid) && last_lsn >= target_lsn) + else if (XRecOffIsValid(target_lsn) && last_lsn >= target_lsn) { all_wal = true; break; } /* If there are no target xid, target time and target lsn */ else if (!TransactionIdIsValid(target_xid) && target_time == 0 && - xlogreader->ReadRecPtr == backup->stop_lsn) + !XRecOffIsValid(target_lsn)) { all_wal = true; /* We don't stop here. We want to get last_xid and last_time */ From d8553c06afff82a39ab217c4ca34b6a760934c1d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 15:28:30 +0300 Subject: [PATCH 0196/2107] Version 2.0.26 - Bugfix: fixed undefined behaviour in case of undefined compression algorithm - Bugfix: correctly calculate backup size after MERGE - Bugfix: fixed MERGE of compressed and uncompressed backups - Bugfix: remove unnecessary remaining files after MERGE - Bugfix: consider target LSN (--lsn parameter) during validating WAL - Improvement: check backup program_version during VALIDATE, do not support forward compatibility - Improvement: improve Windows support - Improvement: improve support of tablespaces within PGDATA --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2cca5a2fe..e5057686c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -19,7 +19,7 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.25"; +const char *PROGRAM_VERSION = "2.0.26"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1f2b8a4ed..17afa4fe1 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.25 \ No newline at end of file +pg_probackup 2.0.26 \ No newline at end of file From 1fff8a2d3d238a089c13e6a4343bf2f36601d22e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 16:58:37 +0300 Subject: [PATCH 0197/2107] tests: pass replica=True to replica for slow_start() --- tests/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index 4eb4b1e20..6d6fb33f3 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -404,7 +404,7 @@ def test_take_backup_from_delayed_replica(self): 'recovery.conf', "recovery_min_apply_delay = '300s'") replica.stop() - replica.slow_start() + replica.slow_start(replica=True) master.pgbench_init(scale=10) From dcdd21300b8306c740dfaa5ffe201592bc5c1c8a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 9 Jan 2019 18:25:52 +0300 Subject: [PATCH 0198/2107] tests: do not call exit(1) --- tests/replica.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 6d6fb33f3..9ab49a6ea 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -3,7 +3,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta import subprocess -from sys import exit import time @@ -52,7 +51,6 @@ def test_replica_stream_ptrack_backup(self): # Check data correctness on replica replica.slow_start(replica=True) - exit(1) after = replica.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) From 6cc4ecb9b7c99be24bdedd8546471636a46ad724 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 10 Jan 2019 15:53:29 +0300 Subject: [PATCH 0199/2107] Make extra_remap_list an instance of TablespaceList --- src/dir.c | 100 +++++++++++++-------------------------------- src/help.c | 2 +- src/pg_probackup.h | 3 -- src/restore.c | 5 --- 4 files changed, 30 insertions(+), 80 deletions(-) diff --git a/src/dir.c b/src/dir.c index b37fcab63..2266edc1a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -126,12 +126,14 @@ static void dir_list_file_internal(parray *files, const char *root, static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); +static void opt_path_map(ConfigOption *opt, const char *arg, + TablespaceList *list, const char *type); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* Extra directories mapping */ -parray *extra_remap_list = NULL; +static TablespaceList extra_remap_list = {NULL, NULL}; /* * Create directory, also create parent directories if necessary. @@ -893,13 +895,14 @@ get_tablespace_created(const char *link) } /* - * Split argument into old_dir and new_dir and append to tablespace mapping + * Split argument into old_dir and new_dir and append to mapping * list. * * Copy of function tablespace_list_append() from pg_basebackup.c. */ -void -opt_tablespace_map(ConfigOption *opt, const char *arg) +static void +opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, + const char *type) { TablespaceListCell *cell = pgut_new(TablespaceListCell); char *dst; @@ -917,7 +920,7 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) { if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in tablespace mapping\n"); + elog(ERROR, "multiple \"=\" signs in %s mapping\n", type); else dst = dst_ptr = cell->new_dir; } @@ -926,8 +929,8 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) } if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid tablespace mapping format \"%s\", " - "must be \"OLDDIR=NEWDIR\"", arg); + elog(ERROR, "invalid %s mapping format \"%s\", " + "must be \"OLDDIR=NEWDIR\"", type, arg); /* * This check isn't absolutely necessary. But all tablespaces are created @@ -936,65 +939,32 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) * consistent with the new_dir check. */ if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path in tablespace mapping: %s\n", - cell->old_dir); + elog(ERROR, "old directory is not an absolute path in %s mapping: %s\n", + type, cell->old_dir); if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path in tablespace mapping: %s\n", - cell->new_dir); + elog(ERROR, "new directory is not an absolute path in %s mapping: %s\n", + type, cell->new_dir); - if (tablespace_dirs.tail) - tablespace_dirs.tail->next = cell; + if (list->tail) + list->tail->next = cell; else - tablespace_dirs.head = cell; - tablespace_dirs.tail = cell; + list->head = cell; + list->tail = cell; } +/* Parse tablespace mapping */ void -opt_extradir_map(ConfigOption *opt, const char *arg) +opt_tablespace_map(ConfigOption *opt, const char *arg) { - TablespaceListCell *cell = pgut_new(TablespaceListCell); - char *dst; - char *dst_ptr; - const char *arg_ptr; - - memset(cell, 0, sizeof(TablespaceListCell)); - if (!extra_remap_list) - extra_remap_list = parray_new(); - dst_ptr = dst = cell->old_dir; - for (arg_ptr = arg; *arg_ptr; arg_ptr++) - { - if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "directory name too long"); - - if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') - ; /* skip backslash escaping = */ - else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) - { - if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in extra directory mapping\n"); - else - dst = dst_ptr = cell->new_dir; - } - else - *dst_ptr++ = *arg_ptr; - } - - if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid extra directory mapping format \"%s\", " - "must be \"OLDDIR=NEWDIR\"", arg); - - if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path " - "in extra directory mapping: %s\n", - cell->old_dir); - - if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path " - "in extra directory mapping: %s\n", - cell->new_dir); + opt_path_map(opt, arg, &tablespace_dirs, "tablespace"); +} - parray_append(extra_remap_list, cell); +/* Parse extra directories mapping */ +void +opt_extradir_map(ConfigOption *opt, const char *arg) +{ + opt_path_map(opt, arg, &extra_remap_list, "extra directory"); } /* @@ -1270,12 +1240,10 @@ check_tablespace_mapping(pgBackup *backup) char * check_extra_dir_mapping(char *current_dir) { - if (!extra_remap_list) - return current_dir; + TablespaceListCell *cell; - for (int i = 0; i < parray_num(extra_remap_list); i++) + for (cell = extra_remap_list.head; cell; cell = cell->next) { - TablespaceListCell *cell = parray_get(extra_remap_list, i); char *old_dir = cell->old_dir; if (strcmp(old_dir, current_dir) == 0) @@ -1284,16 +1252,6 @@ check_extra_dir_mapping(char *current_dir) return current_dir; } -void -free_extra_remap_list(void *cell) -{ - TablespaceListCell *cell_ptr; - if (cell == NULL) - return; - cell_ptr = (TablespaceListCell *)cell; - pfree(cell_ptr); -} - /* * Print backup content list. */ diff --git a/src/help.c b/src/help.c index 578ef289b..47a167c44 100644 --- a/src/help.c +++ b/src/help.c @@ -222,7 +222,7 @@ help_backup(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); - printf(_(" --skip-block-validation sextra-directoryet to validate only file-level checksum\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_(" -E --extra-directory=extra-directory-path\n")); printf(_(" backup some directory not from pgdata \n")); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ef988610a..5d91a00ec 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -394,7 +394,6 @@ extern pgBackup current; /* in dir.c */ /* exclude directory list for $PGDATA file listing */ extern const char *pgdata_exclude_dir[]; -extern parray *extra_remap_list; /* in backup.c */ extern int do_backup(time_t start_time); @@ -511,7 +510,6 @@ extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_extradir_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); extern char* check_extra_dir_mapping(char *current_dir); -extern void clean_extra_dirs_remap_list(void); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *extra_prefix, parray *extra_list); @@ -521,7 +519,6 @@ extern void free_dir_list(parray *list); extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_extra(const char *dir, parray *dirs_list); -extern void free_extra_remap_list(void *cell); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); diff --git a/src/restore.c b/src/restore.c index 1148f26f9..2eb6d563b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -407,11 +407,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); - if (extra_remap_list) - { - parray_walk(extra_remap_list, free_extra_remap_list); - parray_free(extra_remap_list); - } elog(INFO, "%s of backup %s completed.", action, base36enc(dest_backup->start_time)); From cf3eaed51aaa9eb1e8e455ffaba4ec6d60d954c3 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Thu, 10 Jan 2019 16:04:31 +0300 Subject: [PATCH 0200/2107] Value 'none' of parameter 'extra-directory' in pg_probackup.conf is case insensitive now --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index ff6725467..255ac69e1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -992,7 +992,7 @@ do_backup(time_t start_time) /* Save list of extra directories */ if (instance_config.extra_dir_str && - strcmp(instance_config.extra_dir_str, "none") != 0) + pg_strcasecmp(instance_config.extra_dir_str, "none") != 0) { current.extra_dir_str = instance_config.extra_dir_str; } From 069b9c10601c31df568c6e9e3aab900212b1f1b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Jan 2019 18:12:00 +0300 Subject: [PATCH 0201/2107] tests: python 3 support --- tests/archive.py | 4 ++-- tests/helpers/ptrack_helpers.py | 26 +++++++++++++------------- tests/ptrack_clean.py | 2 +- tests/validate_test.py | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 1d0d7af57..0881a0505 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -742,7 +742,7 @@ def test_archive_pg_receivexlog(self): [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-D', os.path.join(backup_dir, 'wal', 'node') - ], async=True) + ], asynchronous=True) if pg_receivexlog.returncode: self.assertFalse( @@ -816,7 +816,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') - ], async=True) + ], asynchronous=True) if pg_receivexlog.returncode: self.assertFalse( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 17fc4f236..b19998648 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -124,7 +124,11 @@ def slow_start(self, replica=False): # raise_operational_error=False) self.start() - if not replica: + if replica: + self.poll_query_until( + 'postgres', + 'SELECT pg_is_in_recovery()') + else: while True: try: self.poll_query_until( @@ -133,10 +137,6 @@ def slow_start(self, replica=False): break except Exception as e: continue - else: - self.poll_query_until( - 'postgres', - 'SELECT pg_is_in_recovery()') # while True: # try: @@ -551,7 +551,7 @@ def check_ptrack_clean(self, idx_dict, size): ) ) - def run_pb(self, command, async=False, gdb=False, old_binary=False): + def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False): if not self.probackup_old_path and old_binary: print('PGPROBACKUPBIN_OLD is not set') exit(1) @@ -567,7 +567,7 @@ def run_pb(self, command, async=False, gdb=False, old_binary=False): print(self.cmd) if gdb: return GDBobj([binary_path] + command, self.verbose) - if async: + if asynchronous: return subprocess.Popen( self.cmd, stdout=subprocess.PIPE, @@ -590,11 +590,11 @@ def run_pb(self, command, async=False, gdb=False, old_binary=False): except subprocess.CalledProcessError as e: raise ProbackupException(e.output.decode('utf-8'), self.cmd) - def run_binary(self, command, async=False): + def run_binary(self, command, asynchronous=False): if self.verbose: print([' '.join(map(str, command))]) try: - if async: + if asynchronous: return subprocess.Popen( command, stdin=subprocess.PIPE, @@ -649,7 +649,7 @@ def clean_pb(self, backup_dir): def backup_node( self, backup_dir, instance, node, data_dir=False, - backup_type='full', options=[], async=False, gdb=False, + backup_type='full', options=[], asynchronous=False, gdb=False, old_binary=False ): if not node and not data_dir: @@ -673,10 +673,10 @@ def backup_node( if backup_type: cmd_list += ['-b', backup_type] - return self.run_pb(cmd_list + options, async, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def merge_backup( - self, backup_dir, instance, backup_id, async=False, + self, backup_dir, instance, backup_id, asynchronous=False, gdb=False, old_binary=False, options=[]): cmd_list = [ 'merge', @@ -685,7 +685,7 @@ def merge_backup( '-i', backup_id ] - return self.run_pb(cmd_list + options, async, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def restore_node( self, backup_dir, instance, node=False, diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py index 01d364e9e..3eeba1b76 100644 --- a/tests/ptrack_clean.py +++ b/tests/ptrack_clean.py @@ -149,7 +149,7 @@ def test_ptrack_clean_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( diff --git a/tests/validate_test.py b/tests/validate_test.py index f2bf52fe1..c6d329066 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -2453,7 +2453,7 @@ def test_file_size_corruption_no_validate(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", - options=["-j", "4"], async=False, gdb=False) + options=["-j", "4"], asynchronous=False, gdb=False) node.stop() node.cleanup() @@ -3138,7 +3138,7 @@ def test_corrupt_pg_control_via_resetxlog(self): '-o 42', '-f' ], - async=False) + asynchronous=False) md5_after = hashlib.md5( open(pg_control_path, 'rb').read()).hexdigest() From 1b2ead72df3fe41eae45c0435344a3819389f172 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Jan 2019 18:17:03 +0300 Subject: [PATCH 0202/2107] tests: python3 minor fix --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b19998648..286255c95 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1229,7 +1229,7 @@ def get_async_connect(self, database=None, host=None, port=5432): database='postgres', host='127.0.0.1', port=port, - async=True + async_=True ) def wait(self, connection): From 01770e7e37fdba9bcf491c822d61b7bcabd64fda Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Jan 2019 17:21:06 +0300 Subject: [PATCH 0203/2107] tests: slow_start for replicas --- tests/ptrack_cluster.py | 4 ++-- tests/ptrack_empty.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py index 270cb1b40..7fe312b3b 100644 --- a/tests/ptrack_cluster.py +++ b/tests/ptrack_cluster.py @@ -198,7 +198,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( @@ -298,7 +298,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table and indexes master.safe_psql( diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py index 578c8a73b..35583ff16 100644 --- a/tests/ptrack_empty.py +++ b/tests/ptrack_empty.py @@ -116,7 +116,7 @@ def test_ptrack_empty_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start() + replica.slow_start(replica=True) # Create table master.safe_psql( From 8c7e1aa85dcf2cd325c636ed8e6727cb8b2dbcc5 Mon Sep 17 00:00:00 2001 From: Ilya Skvortsov Date: Fri, 11 Jan 2019 17:44:39 +0300 Subject: [PATCH 0204/2107] try to continue MERGE after fail on backup deleting --- tests/merge.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 8d190811c..27fc2dc68 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1198,6 +1198,93 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_continue_failed_merge_2(self): + """ + Check that failed MERGE on delete can be continued + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + + gdb.set_breakpoint('pgFileDelete') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + print('Failed to hit breakpoint') + exit(1) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Backup {0} has status: DELETING".format( + backup_id_deleted) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 5bcea32f31284233d380990b652f2a73f2aee3cc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Jan 2019 17:54:50 +0300 Subject: [PATCH 0205/2107] tests: comment minor fix --- tests/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 27fc2dc68..4a0cb31cf 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1200,7 +1200,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): def test_continue_failed_merge_2(self): """ - Check that failed MERGE on delete can be continued + Check that failed MERGE on delete can`t be continued """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 94d7c4bb3b8b866ae3112557530a4f075409f36b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Jan 2019 01:42:01 +0300 Subject: [PATCH 0206/2107] tests: crutch-fix json parsing in show_pb() --- tests/helpers/ptrack_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 286255c95..6b6948a36 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -722,8 +722,9 @@ def show_pb( if backup_id: cmd_list += ['-i', backup_id] + # AHTUNG, WARNING will break json parsing if as_json: - cmd_list += ['--format=json'] + cmd_list += ['--format=json', '--log-level-console=error'] if as_text: # You should print it when calling as_text=true From 4d8b6f50c9aeada5ddcc4f828c4493bef092df19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Jan 2019 19:58:15 +0300 Subject: [PATCH 0207/2107] tests: slow_start more robust behaviour --- tests/helpers/ptrack_helpers.py | 44 ++++++++++++++------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6b6948a36..179020da8 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -118,34 +118,28 @@ def __str__(self): def slow_start(self, replica=False): # wait for https://github.com/postgrespro/testgres/pull/50 - # self.poll_query_until( - # "postgres", - # "SELECT not pg_is_in_recovery()", - # raise_operational_error=False) - - self.start() +# self.start() +# self.poll_query_until( +# "postgres", +# "SELECT not pg_is_in_recovery()", +# suppress={testgres.NodeConnection}) if replica: - self.poll_query_until( - 'postgres', - 'SELECT pg_is_in_recovery()') + query = 'SELECT not pg_is_in_recovery()' else: - while True: - try: - self.poll_query_until( - 'postgres', - 'SELECT not pg_is_in_recovery()') - break - except Exception as e: - continue + query = 'SELECT pg_is_in_recovery()' -# while True: -# try: -# self.poll_query_until( -# "postgres", -# "SELECT pg_is_in_recovery()") -# break -# except ProbackupException as e: -# continue + self.start() + while True: + try: + self.safe_psql( + 'postgres', + query) + break + except testgres.QueryException as e: + if 'database system is starting up' in e[0]: + continue + else: + raise e class ProbackupTest(object): From 58cd26f10a0c58368dcda84281b6de0abd85dee7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Jan 2019 20:20:24 +0300 Subject: [PATCH 0208/2107] tests: slow_start() fix --- tests/helpers/ptrack_helpers.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 179020da8..5949eaedd 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -124,17 +124,15 @@ def slow_start(self, replica=False): # "SELECT not pg_is_in_recovery()", # suppress={testgres.NodeConnection}) if replica: - query = 'SELECT not pg_is_in_recovery()' - else: query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' self.start() while True: try: - self.safe_psql( - 'postgres', - query) - break + if self.safe_psql('postgres', query) == 't\n': + break except testgres.QueryException as e: if 'database system is starting up' in e[0]: continue From d6908b353889389b30ba6cc4a5721bcf1201b026 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 15 Jan 2019 01:07:48 +0300 Subject: [PATCH 0209/2107] Fix open file mode --- src/data.c | 2 +- src/utils/file.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index ac7db0438..fd21b1665 100644 --- a/src/data.c +++ b/src/data.c @@ -711,7 +711,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ - out = fio_fopen(to_path, PG_BINARY_W "+", FIO_DB_HOST); + out = fio_fopen(to_path, PG_BINARY_R "+", FIO_DB_HOST); if (out == NULL) { int errno_tmp = errno; diff --git a/src/utils/file.c b/src/utils/file.c index 78cfdccad..d8663ece0 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -275,6 +275,8 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) else { f = fopen(path, mode); + if (f == NULL && strcmp(mode, PG_BINARY_R "+") == 0) + f = fopen(path, PG_BINARY_W); } return f; } From ba53bb5a09c84d10e1adb6c1cb7e9b50025f3fe3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Jan 2019 16:36:14 +0300 Subject: [PATCH 0210/2107] tests: fix test_corrupt_control_file() which was broken by 94d7c4bb3b8b --- tests/show_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/show_test.py b/tests/show_test.py index eef897180..755c76d49 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -199,7 +199,7 @@ def test_corrupt_control_file(self): self.assertIn( 'WARNING: Invalid option "statuss" in file'.format(file), - self.show_pb(backup_dir, 'node', as_text=True)) + self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) # Clean after yourself - # self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname) From 8894d9f9a89d71cb4396eef3068d21e1299fe7d6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Jan 2019 16:40:35 +0300 Subject: [PATCH 0211/2107] tests: fix test_empty_control_file() which was broken by 94d7c4bb3b8b --- tests/show_test.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/show_test.py b/tests/show_test.py index 755c76d49..07f3be844 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -17,8 +17,7 @@ def test_show_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -45,8 +44,7 @@ def test_show_json(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -73,8 +71,7 @@ def test_corrupt_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -119,8 +116,7 @@ def test_no_control_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -135,7 +131,9 @@ def test_no_control_file(self): backup_id, "backup.control") os.remove(file) - self.assertIn('Control file "{0}" doesn\'t exist'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn( + 'Control file "{0}" doesn\'t exist'.format(file), + self.show_pb(backup_dir, 'node', as_text=True)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -148,8 +146,7 @@ def test_empty_control_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -165,7 +162,9 @@ def test_empty_control_file(self): fd = open(file, 'w') fd.close() - self.assertIn('Control file "{0}" is empty'.format(file), self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn( + 'Control file "{0}" is empty'.format(file), + self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -179,8 +178,7 @@ def test_corrupt_control_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 71172fb6de4572c8f570711cb69b384c167d6ea4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Jan 2019 16:46:29 +0300 Subject: [PATCH 0212/2107] tests: fix test_no_control_file() which was broken by 94d7c4bb3b8b --- tests/show_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/show_test.py b/tests/show_test.py index 07f3be844..201839a77 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -133,7 +133,7 @@ def test_no_control_file(self): self.assertIn( 'Control file "{0}" doesn\'t exist'.format(file), - self.show_pb(backup_dir, 'node', as_text=True)) + self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) # Clean after yourself self.del_test_dir(module_name, fname) From 1dbc0c94df92ebaaeb80851d3c8a469df30bc4fa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 16 Jan 2019 16:31:26 +0300 Subject: [PATCH 0213/2107] tests: to pgdata_content() and add_instance() arbitrary cmdline options can now be provided --- tests/helpers/ptrack_helpers.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 5949eaedd..9fafd862a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -615,16 +615,16 @@ def init_pb(self, backup_dir, old_binary=False): old_binary=old_binary ) - def add_instance(self, backup_dir, instance, node, old_binary=False): + def add_instance(self, backup_dir, instance, node, old_binary=False, options=[]): - return self.run_pb([ + cmd = [ 'add-instance', '--instance={0}'.format(instance), '-B', backup_dir, '-D', node.data_dir - ], - old_binary=old_binary - ) + ] + + return self.run_pb(cmd + options, old_binary=old_binary) def del_instance(self, backup_dir, instance, old_binary=False): @@ -1049,7 +1049,7 @@ def del_test_dir(self, module_name, fname): except: pass - def pgdata_content(self, pgdata, ignore_ptrack=True): + def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): """ return dict with directory content. " " TAKE IT AFTER CHECKPOINT or BACKUP""" dirs_to_ignore = [ @@ -1062,6 +1062,9 @@ def pgdata_content(self, pgdata, ignore_ptrack=True): 'backup_label', 'tablespace_map', 'recovery.conf', 'ptrack_control', 'ptrack_init', 'pg_control' ] + + if exclude_dirs: + dirs_to_ignore = dirs_to_ignore + exclude_dirs # suffixes_to_ignore = ( # '_ptrack' # ) From 5cd09774afaa6039cd76bbe6d3c4761f2e8f0f8e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 16 Jan 2019 16:33:27 +0300 Subject: [PATCH 0214/2107] tests for external directories --- tests/__init__.py | 3 +- tests/external.py | 194 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 tests/external.py diff --git a/tests/__init__.py b/tests/__init__.py index 9156b649b..57ccae316 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,7 +8,7 @@ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp + time_stamp, external def load_tests(loader, tests, pattern): @@ -50,6 +50,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) suite.addTests(loader.loadTestsFromModule(time_stamp)) + suite.addTests(loader.loadTestsFromModule(external)) return suite diff --git a/tests/external.py b/tests/external.py new file mode 100644 index 000000000..dd155be68 --- /dev/null +++ b/tests/external.py @@ -0,0 +1,194 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name +import shutil + + +module_name = 'external' + + +class ExternalTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_external_simple(self): + """ + make node, create external directory, take backup + with external directory, restore backup, check that + external directory was successfully copied + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + external_dir = os.path.join(node.base_dir, 'somedirectory') + + # copy postgresql.conf to extra_directory + os.mkdir(external_dir) + shutil.copyfile( + os.path.join(node.data_dir, 'postgresql.conf'), + os.path.join(external_dir, 'postgresql.conf')) + + # create directory in extra_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + '--extra-directory={0}'.format(external_dir)]) + + if self.paranoia: + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.base_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_external_dir_mapping(self): + """ + make node, take full backup, check that restore with + external-dir mapping will end with error, take page backup, + check that restore with external-dir mapping will end with + success + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # copy postgresql.conf to extra_directory + os.mkdir(external_dir1_old) + os.mkdir(external_dir2_old) + shutil.copyfile( + os.path.join(node.data_dir, 'postgresql.conf'), + os.path.join(external_dir1_old, 'postgresql.conf')) + shutil.copyfile( + os.path.join(node.data_dir, 'postgresql.conf'), + os.path.join(external_dir2_old, 'postgresql.conf')) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--extra-mapping={0}={1}".format( + external_dir1_old, external_dir1_new), + "--extra-mapping={0}={1}".format( + external_dir2_old, external_dir2_new)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --tablespace-mapping option' in e.message and + 'have an entry in tablespace_map file' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format(external_dir1_old, external_dir2_old)]) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--extra-mapping={0}={1}".format( + external_dir1_old, external_dir1_new), + "--extra-mapping={0}={1}".format( + external_dir2_old, external_dir2_new)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.base_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.base_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_backup_multiple_extra_dir(self): + """make node, """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # copy some directory from PGDATA to external directories + + + # Clean after yourself + # self.del_test_dir(module_name, fname) + + # extra directory contain symlink to file + # extra directory contain symlink to directory + # latest page backup without extra_dir + # multiple external directories + # --extra-directory=none + # --extra-directory point to a file + # extra directory in config and in command line + # extra directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories + # check that not changed files are not copied by next backup \ No newline at end of file From 93c39d2d53ccc83ae09fa959d43914c94ca0ee47 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 16 Jan 2019 17:47:08 +0300 Subject: [PATCH 0215/2107] tests: test_tablespace_handling() added --- tests/backup_test.py | 115 ++++++++++++++++++++++++++++++++ tests/helpers/ptrack_helpers.py | 11 +++ 2 files changed, 126 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index e3fdac15f..fcc541777 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -512,3 +512,118 @@ def test_tablespace_in_pgdata_pgpro_1376(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_handling(self): + """ + make node, take full backup, check that restore with + tablespace mapping will end with error, take page backup, + check that restore with tablespace mapping will end with + success + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') + + self.create_tblspace_in_node( + node, 'some_lame_tablespace') + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=tblspace1_old_path) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=tblspace2_old_path) + + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace some_lame_tablespace " + "as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + tblspace1_new_path = self.get_tblspace_path(node, 'tblspace1_new') + tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --tablespace-mapping option' in e.message and + 'have an entry in tablespace_map file' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap_lame") + + node.safe_psql( + "postgres", + "drop tablespace some_lame_tablespace") + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"]) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 9fafd862a..237961300 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -626,6 +626,16 @@ def add_instance(self, backup_dir, instance, node, old_binary=False, options=[]) return self.run_pb(cmd + options, old_binary=old_binary) + def set_config(self, backup_dir, instance, old_binary=False, options=[]): + + cmd = [ + 'set-config', + '--instance={0}'.format(instance), + '-B', backup_dir, + ] + + return self.run_pb(cmd + options, old_binary=old_binary) + def del_instance(self, backup_dir, instance, old_binary=False): return self.run_pb([ @@ -1087,6 +1097,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): directory_dict['files'][file_relpath]['md5'] = hashlib.md5( open(file_fullpath, 'rb').read()).hexdigest() + # crappy algorithm if file.isdigit(): directory_dict['files'][file_relpath]['is_datafile'] = True size_in_pages = os.path.getsize(file_fullpath)/8192 From bda5d3b56450d45da4301622f53301b3b9fa0b99 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 16 Jan 2019 17:49:59 +0300 Subject: [PATCH 0216/2107] tests: PGPRO-421 test_backup_multiple_external() added --- tests/external.py | 57 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/external.py b/tests/external.py index dd155be68..19c6b4df0 100644 --- a/tests/external.py +++ b/tests/external.py @@ -56,7 +56,8 @@ def test_external_simple(self): backup_dir, 'node', node, options=["-j", "4"]) if self.paranoia: - pgdata_restored = self.pgdata_content(node.base_dir) + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -147,10 +148,12 @@ def test_external_dir_mapping(self): external_dir2_old, external_dir2_new)]) if self.paranoia: - pgdata = self.pgdata_content(node.base_dir) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.base_dir) + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -158,7 +161,7 @@ def test_external_dir_mapping(self): # @unittest.skip("skip") # @unittest.expectedFailure - def test_backup_multiple_extra_dir(self): + def test_backup_multiple_external(self): """make node, """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -176,11 +179,49 @@ def test_backup_multiple_extra_dir(self): external_dir1_old = self.get_tblspace_path(node, 'external_dir1') external_dir2_old = self.get_tblspace_path(node, 'external_dir2') - # copy some directory from PGDATA to external directories + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=[ + '-E', external_dir1_old]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}".format(external_dir2_old)]) + + if self.paranoia: + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir1']) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - # self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname) # extra directory contain symlink to file # extra directory contain symlink to directory @@ -191,4 +232,6 @@ def test_backup_multiple_extra_dir(self): # extra directory in config and in command line # extra directory contain multuple directories, some of them my be empty # forbid to external-dirs to point to tablespace directories - # check that not changed files are not copied by next backup \ No newline at end of file + # check that not changed files are not copied by next backup + # merge + # complex merge From 577a8d931854fa3f5c74137ab15025291f6e9888 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 17 Jan 2019 19:40:54 +0300 Subject: [PATCH 0217/2107] Fix remote delta backup --- src/catalog.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 5f8d90faa..e6fa0cc2b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -273,7 +273,7 @@ catalog_get_backup_list(time_t requested_backup_id) int i; /* open backup instance backups directory */ - data_dir = opendir(backup_instance_path); + data_dir = fio_opendir(backup_instance_path, FIO_BACKUP_HOST); if (data_dir == NULL) { elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, @@ -283,7 +283,7 @@ catalog_get_backup_list(time_t requested_backup_id) /* scan the directory and list backups */ backups = parray_new(); - for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) + for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0) { char backup_conf_path[MAXPGPATH]; char data_path[MAXPGPATH]; @@ -336,7 +336,7 @@ catalog_get_backup_list(time_t requested_backup_id) goto err_proc; } - closedir(data_dir); + fio_closedir(data_dir); data_dir = NULL; parray_qsort(backups, pgBackupCompareIdDesc); @@ -369,7 +369,7 @@ catalog_get_backup_list(time_t requested_backup_id) err_proc: if (data_dir) - closedir(data_dir); + fio_closedir(data_dir); if (backup) pgBackupFree(backup); if (backups) From 065d981e3609d3d3fdb0ed02ef6597811bbaa0ee Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 00:10:58 +0300 Subject: [PATCH 0218/2107] Fix remote delta backup --- src/catalog.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index e6fa0cc2b..9583edf33 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -247,14 +247,14 @@ pgBackupGetBackupMode(pgBackup *backup) } static bool -IsDir(const char *dirpath, const char *entry) +IsDir(const char *dirpath, const char *entry, fio_location location) { char path[MAXPGPATH]; struct stat st; snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); - return stat(path, &st) == 0 && S_ISDIR(st.st_mode); + return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); } /* @@ -289,7 +289,7 @@ catalog_get_backup_list(time_t requested_backup_id) char data_path[MAXPGPATH]; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, data_ent->d_name) + if (!IsDir(backup_instance_path, data_ent->d_name, FIO_BACKUP_HOST) || data_ent->d_name[0] == '.') continue; From 3b144bbe2ba83ee9fddc3e9d7d2e14e57f50ffb6 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 00:21:42 +0300 Subject: [PATCH 0219/2107] Fix remote delta backup --- src/backup.c | 2 +- src/dir.c | 6 +++--- src/merge.c | 4 ++-- src/pg_probackup.h | 2 +- src/restore.c | 4 ++-- src/validate.c | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6ac1beb51..93cd873d9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -540,7 +540,7 @@ do_backup_instance(void) pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ - prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path); + prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path, FIO_BACKUP_HOST); /* If lsn is not NULL, only pages with higher lsn will be copied. */ prev_backup_start_lsn = prev_backup->start_lsn; diff --git a/src/dir.c b/src/dir.c index dc00e8236..2c21ea3ad 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1404,13 +1404,13 @@ get_control_value(const char *str, const char *name, * If root is not NULL, path will be absolute path. */ parray * -dir_read_file_list(const char *root, const char *file_txt) +dir_read_file_list(const char *root, const char *file_txt, fio_location location) { FILE *fp; parray *files; char buf[MAXPGPATH * 2]; - fp = fopen(file_txt, "rt"); + fp = fio_open_stream(file_txt, location); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", file_txt, strerror(errno)); @@ -1469,7 +1469,7 @@ dir_read_file_list(const char *root, const char *file_txt) parray_append(files, file); } - fclose(fp); + fio_close_stream(fp); return files; } diff --git a/src/merge.c b/src/merge.c index a1109b841..d8343cb01 100644 --- a/src/merge.c +++ b/src/merge.c @@ -208,7 +208,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(to_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - to_files = dir_read_file_list(NULL, control_file); + to_files = dir_read_file_list(NULL, control_file, FIO_BACKUP_HOST); /* To delete from leaf, sort in reversed order */ parray_qsort(to_files, pgFileComparePathDesc); /* @@ -216,7 +216,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) */ pgBackupGetPath(from_backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - files = dir_read_file_list(NULL, control_file); + files = dir_read_file_list(NULL, control_file, FIO_BACKUP_HOST); /* sort by size for load balancing */ parray_qsort(files, pgFileCompareSize); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5de139ff7..5f24c9302 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -520,7 +520,7 @@ extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); extern void print_file_list(FILE *out, const parray *files, const char *root); -extern parray *dir_read_file_list(const char *root, const char *file_txt); +extern parray *dir_read_file_list(const char *root, const char *file_txt, fio_location location); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); diff --git a/src/restore.c b/src/restore.c index de263fbb6..601d1398e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -457,7 +457,7 @@ restore_backup(pgBackup *backup) */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, list_path); + files = dir_read_file_list(database_path, list_path, FIO_DB_HOST); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); @@ -523,7 +523,7 @@ remove_deleted_files(pgBackup *backup) pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, filelist_path); + files = dir_read_file_list(instance_config.pgdata, filelist_path, FIO_DB_HOST); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ diff --git a/src/validate.c b/src/validate.c index a39d6092f..3a9dd7c73 100644 --- a/src/validate.c +++ b/src/validate.c @@ -90,7 +90,7 @@ pgBackupValidate(pgBackup *backup) pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, path); + files = dir_read_file_list(base_path, path, FIO_BACKUP_HOST); /* setup threads */ for (i = 0; i < parray_num(files); i++) From be78ca6ee210c0ef1addf5a63e7ab3b150404f87 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 00:37:18 +0300 Subject: [PATCH 0220/2107] Fix remote delta backup --- src/backup.c | 8 ++++---- src/data.c | 4 ++-- src/pg_probackup.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 93cd873d9..a0c45f1be 100644 --- a/src/backup.c +++ b/src/backup.c @@ -819,7 +819,7 @@ do_backup_instance(void) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); if (S_ISREG(file->mode)) - calc_file_checksum(file); + calc_file_checksum(file, FIO_BACKUP_HOST); /* Remove file path root prefix*/ if (strstr(file->path, database_path) == file->path) { @@ -2029,7 +2029,7 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { file = pgFileNew(backup_label, true, FIO_BACKUP_HOST); - calc_file_checksum(file); + calc_file_checksum(file, FIO_BACKUP_HOST); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); @@ -2073,7 +2073,7 @@ pg_stop_backup(pgBackup *backup) { file = pgFileNew(tablespace_map, true, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) - calc_file_checksum(file); + calc_file_checksum(file, FIO_BACKUP_HOST); free(file->path); file->path = strdup(PG_TABLESPACE_MAP_FILE); parray_append(backup_files_list, file); @@ -2319,7 +2319,7 @@ backup_files(void *arg) if (prev_file && file->exists_in_prev && buf.st_mtime < current.parent_backup) { - calc_file_checksum(file); + calc_file_checksum(file, FIO_DB_HOST); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc)) skip = true; /* ...skip copying file. */ diff --git a/src/data.c b/src/data.c index fd21b1665..a1c3ef859 100644 --- a/src/data.c +++ b/src/data.c @@ -1414,11 +1414,11 @@ get_wal_file(const char *from_path, const char *to_path) * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. */ void -calc_file_checksum(pgFile *file) +calc_file_checksum(pgFile *file, fio_location location) { Assert(S_ISREG(file->mode)); - file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); + file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, location); file->write_size = file->read_size; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5f24c9302..babc643d9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -555,7 +555,7 @@ extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); extern void get_wal_file(const char *from_path, const char *to_path); -extern void calc_file_checksum(pgFile *file); +extern void calc_file_checksum(pgFile *file, fio_location location); extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); From fdfb074703d708d24e291a0d1d92e020a5d3a534 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 00:43:39 +0300 Subject: [PATCH 0221/2107] Fix remote delta backup --- src/data.c | 2 +- src/delete.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index a1c3ef859..d210f0c52 100644 --- a/src/data.c +++ b/src/data.c @@ -667,7 +667,7 @@ backup_data_file(backup_files_arg* arguments, */ if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) { - if (remove(to_path) == -1) + if (fio_unlink(to_path, FIO_BACKUP_HOST) == -1) elog(ERROR, "cannot remove file \"%s\": %s", to_path, strerror(errno)); return false; diff --git a/src/delete.c b/src/delete.c index 053e6aa75..7d81b9728 100644 --- a/src/delete.c +++ b/src/delete.c @@ -296,7 +296,7 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", i + 1, num_files, file->path); - pgFileDelete(file); + fio_unlink(file->path, FIO_BACKUP_HOST); } parray_walk(files, pgFileFree); From 9865e0934df15a2d0ee537030163ca74d7ad2d71 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 18:28:39 +0300 Subject: [PATCH 0222/2107] Fix problem with remote archive-push --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 872526cd6..44ca87cdb 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -351,7 +351,7 @@ main(int argc, char *argv[]) backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) { if (remote_agent) { - if (backup_subcmd != BACKUP_CMD) { + if (backup_subcmd != BACKUP_CMD && backup_subcmd != ARCHIVE_PUSH_CMD) { fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; } From e02b7f7c92190e6f1f0a443125169ec447b964d3 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 22:33:18 +0300 Subject: [PATCH 0223/2107] Support -D options for archive-get andd archive-push commands --- src/archive.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/archive.c b/src/archive.c index add41f640..e715f6fbd 100644 --- a/src/archive.c +++ b/src/archive.c @@ -29,6 +29,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char current_dir[MAXPGPATH]; uint64 system_id; bool is_compress = false; + char* pgdata = instance_config.pgdata; if (wal_file_name == NULL && wal_file_path == NULL) elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); @@ -39,11 +40,14 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - + if (pgdata == NULL) + { + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + pgdata = current_dir; + } /* verify that archive-push --instance parameter is valid */ - system_id = get_system_identifier(current_dir); + system_id = get_system_identifier(pgdata); if (instance_config.pgdata == NULL) elog(ERROR, "cannot read pg_probackup.conf for this instance"); @@ -57,7 +61,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) /* Create 'archlog_path' directory. Do nothing if it already exists. */ fio_mkdir(arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); - join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(absolute_wal_file_path, pgdata, wal_file_path); join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); @@ -87,6 +91,7 @@ do_archive_get(char *wal_file_path, char *wal_file_name) char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; + char* pgdata = instance_config.pgdata; if (wal_file_name == NULL && wal_file_path == NULL) elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); @@ -97,10 +102,13 @@ do_archive_get(char *wal_file_path, char *wal_file_name) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - - join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + if (pgdata == NULL) + { + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + pgdata = current_dir; + } + join_path_components(absolute_wal_file_path, pgdata, wal_file_path); join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-get from %s to %s", From 64510a42616e7cee971c15d3889326d901379426 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 18 Jan 2019 22:45:40 +0300 Subject: [PATCH 0224/2107] Fix remote rename command --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index d8663ece0..d0064c2af 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -914,7 +914,7 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_RENAME: - SYS_CHECK(rename(buf, buf + strlen(buf))); + SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); break; case FIO_UNLINK: SYS_CHECK(unlink(buf)); From fce117de86e971148e8164e848e1758ace6b1b8a Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 11:12:37 +0300 Subject: [PATCH 0225/2107] Grab ssh stderr --- src/utils/remote.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 5c5677aef..4437d9797 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -10,6 +10,7 @@ #define MAX_CMDLINE_LENGTH 4096 #define MAX_CMDLINE_OPTIONS 256 +#define ERR_BUF_SIZE 1024 static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) { @@ -78,6 +79,7 @@ int remote_execute(int argc, char* argv[], bool listen) int i; int outfd[2]; int infd[2]; + int errfd[2]; char* pg_probackup = argv[0]; ssh_argc = 0; @@ -116,31 +118,45 @@ int remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); + SYS_CHECK(pipe(errfd)); SYS_CHECK(child_pid = fork()); if (child_pid == 0) { /* child */ SYS_CHECK(close(STDIN_FILENO)); SYS_CHECK(close(STDOUT_FILENO)); + SYS_CHECK(close(STDERR_FILENO)); SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); - SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + SYS_CHECK(dup2(errfd[1], STDERR_FILENO)); SYS_CHECK(close(infd[0])); SYS_CHECK(close(infd[1])); SYS_CHECK(close(outfd[0])); SYS_CHECK(close(outfd[1])); + SYS_CHECK(close(errfd[0])); + SYS_CHECK(close(errfd[1])); SYS_CHECK(execvp(ssh_argv[0], ssh_argv)); return -1; } else { - SYS_CHECK(close(outfd[0])); /* These are being used by the child */ - SYS_CHECK(close(infd[1])); + SYS_CHECK(close(infd[1])); /* These are being used by the child */ + SYS_CHECK(close(outfd[0])); + SYS_CHECK(close(errfd[1])); atexit(kill_child); if (listen) { int status; fio_communicate(infd[0], outfd[1]); SYS_CHECK(wait(&status)); + if (status != 0) + { + char buf[ERR_BUF_SIZE]; + int offs, rc; + for (offs = 0; (rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0; offs += rc); + buf[offs] = '\0'; + elog(ERROR, buf); + } return status; } else { fio_redirect(infd[0], outfd[1]); /* write to stdout */ From 4ceee57f1d3018d7bc5cd669b0aa69652b498a89 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 14:05:38 +0300 Subject: [PATCH 0226/2107] Some fixes for remote logging --- src/utils/logger.c | 3 ++- src/utils/remote.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index ba054a62c..a94f4f5cf 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -11,6 +11,7 @@ #include +#include "pg_probackup.h" #include "logger.h" #include "pgut.h" #include "thread.h" @@ -160,7 +161,7 @@ elog_internal(int elevel, bool file_only, const char *message) logger_config.log_directory && logger_config.log_directory[0] != '\0'; write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && logger_config.log_directory && logger_config.log_directory[0] != '\0'; - write_to_stderr = elevel >= logger_config.log_level_console && !file_only; + write_to_stderr = elevel >= (remote_agent ? ERROR : logger_config.log_level_console) && !file_only; pthread_lock(&log_file_mutex); loggin_in_progress = true; diff --git a/src/utils/remote.c b/src/utils/remote.c index 4437d9797..ee3f11ecd 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -155,7 +155,7 @@ int remote_execute(int argc, char* argv[], bool listen) int offs, rc; for (offs = 0; (rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0; offs += rc); buf[offs] = '\0'; - elog(ERROR, buf); + elog(ERROR, "%s", strncmp(buf, "ERROR: ", 6) == 0 ? buf + 6 : buf); } return status; } else { From 3e970d1c52ac59b112ca7fe9bb99407d2da29404 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 15:48:06 +0300 Subject: [PATCH 0227/2107] Use elog instead of fprintf(stderr) for remote IO errors --- src/utils/file.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.h b/src/utils/file.h index 0d145f0d2..37f84dff8 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -49,8 +49,8 @@ typedef enum #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 -#define SYS_CHECK(cmd) do if ((cmd) < 0) { perror(#cmd); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d\n", __FILE__, __LINE__, _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) +#define SYS_CHECK(cmd) do if ((cmd) < 0) { elog(ERROR, "%s: %m", #cmd); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { elog(ERROR, "%s:%d: proceeds %d bytes instead of %d\n", __FILE__, __LINE__, _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) typedef struct { From b38aa659bdf1de012ecef95cd287d296275388bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 21 Jan 2019 17:53:50 +0300 Subject: [PATCH 0228/2107] remote backup help update --- src/help.c | 72 ++++++++++++++++++++++++++++++++++++++++++---- src/pg_probackup.c | 17 +++++------ 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/help.c b/src/help.c index 409c8f825..050ff792d 100644 --- a/src/help.c +++ b/src/help.c @@ -119,6 +119,9 @@ help_pg_probackup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); @@ -129,6 +132,9 @@ help_pg_probackup(void) printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); @@ -148,6 +154,9 @@ help_pg_probackup(void) printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n")); printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); @@ -159,10 +168,16 @@ help_pg_probackup(void) printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--overwrite]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n")); printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n")); if ((PROGRAM_URL || PROGRAM_EMAIL)) { @@ -207,7 +222,10 @@ help_backup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [--skip-block-validation]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -272,6 +290,14 @@ help_backup(void) printf(_(" --master-host=host_name database server host of master\n")); printf(_(" --master-port=port database server port of master\n")); printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } static void @@ -284,7 +310,10 @@ help_restore(void) printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--no-validate]\n\n")); - printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [--skip-block-validation]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -333,6 +362,14 @@ help_restore(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } static void @@ -558,11 +595,22 @@ static void help_add_instance(void) { printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n\n")); + printf(_(" --instance=instance_name\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --instance=instance_name name of the new instance\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } static void @@ -583,7 +631,10 @@ help_archive_push(void) printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--overwrite]\n\n")); + printf(_(" [--overwrite]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance to delete\n")); @@ -604,7 +655,10 @@ help_archive_get(void) { printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); - printf(_(" --wal-file-name=wal-file-name\n\n")); + printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance to delete\n")); @@ -612,4 +666,12 @@ help_archive_get(void) printf(_(" relative destination path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the WAL file to retrieve from the archive\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 44ca87cdb..d99b09749 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -142,14 +142,15 @@ static ConfigOption cmd_options[] = { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 's', 19, "remote-host", &remote_host, SOURCE_CMD_STRICT, }, - { 's', 20, "remote-port", &remote_port, SOURCE_CMD_STRICT, }, - { 's', 21, "remote-proto", &remote_proto, SOURCE_CMD_STRICT, }, - { 's', 22, "remote-path", &remote_path, SOURCE_CMD_STRICT, }, - { 's', 23, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, - { 's', 24, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, - { 's', 25, "agent", &remote_agent, SOURCE_CMD_STRICT, }, - { 'b', 26, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, + /* remote options */ + { 's', 19, "remote-proto", &remote_proto, SOURCE_CMD_STRICT, }, + { 'b', 20, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, + { 's', 21, "remote-host", &remote_host, SOURCE_CMD_STRICT, }, + { 's', 22, "remote-port", &remote_port, SOURCE_CMD_STRICT, }, + { 's', 23, "remote-path", &remote_path, SOURCE_CMD_STRICT, }, + { 's', 24, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, + { 's', 25, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, + { 's', 26, "agent", &remote_agent, SOURCE_CMD_STRICT, }, /* restore options */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, From bd241d8253fbb990b18f0b25222d95717edc618a Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 18:49:38 +0300 Subject: [PATCH 0229/2107] Replace %m option with strerror(errno) --- src/utils/file.h | 4 ++-- src/utils/remote.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/file.h b/src/utils/file.h index 37f84dff8..1f460417f 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -49,8 +49,8 @@ typedef enum #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 -#define SYS_CHECK(cmd) do if ((cmd) < 0) { elog(ERROR, "%s: %m", #cmd); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { elog(ERROR, "%s:%d: proceeds %d bytes instead of %d\n", __FILE__, __LINE__, _rc, (int)(size)); exit(EXIT_FAILURE); } } while (0) +#define SYS_CHECK(cmd) do if ((cmd) < 0) { elog(ERROR, "%s: %s", #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { elog(ERROR, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), strerror(errno)); exit(EXIT_FAILURE); } } while (0) typedef struct { diff --git a/src/utils/remote.c b/src/utils/remote.c index ee3f11ecd..525d02b63 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -138,7 +138,8 @@ int remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(close(errfd[0])); SYS_CHECK(close(errfd[1])); - SYS_CHECK(execvp(ssh_argv[0], ssh_argv)); + if (execvp(ssh_argv[0], ssh_argv) < 0) + elog(ERROR, "Failed to spawn %s: %s", ssh_argv[0], strerror(errno)); return -1; } else { SYS_CHECK(close(infd[1])); /* These are being used by the child */ From eea74a8423dcb0b3703268821b723022972a0aba Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 19:21:54 +0300 Subject: [PATCH 0230/2107] More remote options to config --- src/backup.c | 12 ++++++------ src/configure.c | 39 +++++++++++++++++++++++++++++++++++++++ src/pg_probackup.c | 22 ++++------------------ src/pg_probackup.h | 13 +++++-------- src/utils/remote.c | 20 ++++++++++---------- 5 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/backup.c b/src/backup.c index a0c45f1be..de46c7801 100644 --- a/src/backup.c +++ b/src/backup.c @@ -492,7 +492,7 @@ do_backup_instance(void) current.data_bytes = 0; /* Obtain current timeline */ - if (is_remote_backup) + if (instance_config.remote.enabled) { char *sysidentifier; TimeLineID starttli; @@ -632,7 +632,7 @@ do_backup_instance(void) backup_files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - if (is_remote_backup) + if (instance_config.remote.enabled) get_remote_pgdata_filelist(backup_files_list); else dir_list_file(backup_files_list, instance_config.pgdata, @@ -699,7 +699,7 @@ do_backup_instance(void) char *dir_name; char database_path[MAXPGPATH]; - if (!is_remote_backup) + if (!instance_config.remote.enabled) dir_name = GetRelativePath(file->path, instance_config.pgdata); else dir_name = file->path; @@ -749,7 +749,7 @@ do_backup_instance(void) elog(VERBOSE, "Start thread num: %i", i); - if (!is_remote_backup) + if (!instance_config.remote.enabled) pthread_create(&threads[i], NULL, backup_files, arg); else pthread_create(&threads[i], NULL, remote_backup_files, arg); @@ -902,7 +902,7 @@ do_backup(time_t start_time) check_server_version(); /* TODO fix it for remote backup*/ - if (!is_remote_backup) + if (!instance_config.remote.enabled) current.checksum_version = get_data_checksum_version(true); is_checksum_enabled = pg_checksum_enable(); @@ -958,7 +958,7 @@ do_backup(time_t start_time) * belogns to the same instance. */ /* TODO fix it for remote backup */ - if (!is_remote_backup) + if (!instance_config.remote.enabled) check_system_identifiers(); diff --git a/src/configure.c b/src/configure.c index a9c80922a..5856fad57 100644 --- a/src/configure.c +++ b/src/configure.c @@ -37,6 +37,7 @@ static void show_configure_json(ConfigOption *opt); #define OPTION_LOG_GROUP "Logging parameters" #define OPTION_RETENTION_GROUP "Retention parameters" #define OPTION_COMPRESS_GROUP "Compression parameters" +#define OPTION_REMOTE_GROUP "Remote access parameters" /* * Short name should be non-printable ASCII character. @@ -172,6 +173,42 @@ ConfigOption instance_options[] = &instance_config.compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, + /* Remote backup options */ + { + 's', 219, "remote-proto", + &instance_config.remote.proto, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 220, "remote-host", + &instance_config.remote.host, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 221, "remote-port", + &instance_config.remote.port, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 222, "remote-path", + &instance_config.remote.path, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 223, "ssh-options", + &instance_config.remote.ssh_options, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 224, "ssh-config", + &instance_config.remote.ssh_config, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 'b', 225, "remote", + &instance_config.remote.enabled, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, { 0 } }; @@ -277,6 +314,8 @@ init_config(InstanceConfig *config) config->compress_alg = COMPRESS_ALG_DEFAULT; config->compress_level = COMPRESS_LEVEL_DEFAULT; + + config->remote.proto = (char*)"ssh"; } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index d99b09749..76023c79d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -66,14 +66,7 @@ char *replication_slot = NULL; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -char *remote_host; -char *remote_port; -char *remote_path; -char *remote_proto = (char*)"ssh"; -char *ssh_config; -char *ssh_options; char *remote_agent; -bool is_remote_backup; /* restore options */ static char *target_time = NULL; @@ -142,15 +135,6 @@ static ConfigOption cmd_options[] = { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - /* remote options */ - { 's', 19, "remote-proto", &remote_proto, SOURCE_CMD_STRICT, }, - { 'b', 20, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, - { 's', 21, "remote-host", &remote_host, SOURCE_CMD_STRICT, }, - { 's', 22, "remote-port", &remote_port, SOURCE_CMD_STRICT, }, - { 's', 23, "remote-path", &remote_path, SOURCE_CMD_STRICT, }, - { 's', 24, "ssh-config", &ssh_config, SOURCE_CMD_STRICT, }, - { 's', 25, "ssh-options", &ssh_options, SOURCE_CMD_STRICT, }, - { 's', 26, "agent", &remote_agent, SOURCE_CMD_STRICT, }, /* restore options */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, @@ -182,6 +166,8 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, + /* remote options */ + { 's', 155, "agent", &remote_agent, SOURCE_CMD_STRICT, }, { 0 } }; @@ -531,9 +517,9 @@ main(int argc, char *argv[]) start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); - elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote: %s", + elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", remote_host ? "true" : "false"); + stream_wal ? "true" : "false", instance_config.remote.enabled ? "true" : "false"); return do_backup(start_time); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index babc643d9..1afa8f017 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -25,6 +25,7 @@ #include "utils/configuration.h" #include "utils/logger.h" +#include "utils/remote.h" #include "utils/parray.h" #include "utils/pgut.h" #include "utils/file.h" @@ -197,6 +198,9 @@ typedef struct InstanceConfig /* Logger parameters */ LoggerConfig logger; + /* Remote access parameters */ + RemoteConfig remote; + /* Retention options. 0 disables the option. */ uint32 retention_redundancy; uint32 retention_window; @@ -344,7 +348,7 @@ typedef struct XLByteInSeg(xlrp, logSegNo) #endif -#define IsSshConnection() (remote_host != NULL && strcmp(remote_proto, "ssh") == 0) +#define IsSshConnection() (instance_config.remote.enabled && strcmp(instance_config.remote.proto, "ssh") == 0) /* directory options */ extern char *backup_path; @@ -364,14 +368,7 @@ extern char *replication_slot; extern bool smooth_checkpoint; /* remote probackup options */ -extern char *remote_path; -extern char *remote_port; -extern char *remote_host; -extern char *remote_proto; -extern char *ssh_config; -extern char *ssh_options; extern char* remote_agent; -extern bool is_remote_backup; extern bool is_ptrack_support; extern bool is_checksum_enabled; diff --git a/src/utils/remote.c b/src/utils/remote.c index 525d02b63..6e031b66b 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -83,29 +83,29 @@ int remote_execute(int argc, char* argv[], bool listen) char* pg_probackup = argv[0]; ssh_argc = 0; - ssh_argv[ssh_argc++] = remote_proto; - if (remote_port != NULL) { + ssh_argv[ssh_argc++] = instance_config.remote.proto; + if (instance_config.remote.port != NULL) { ssh_argv[ssh_argc++] = (char*)"-p"; - ssh_argv[ssh_argc++] = remote_port; + ssh_argv[ssh_argc++] = instance_config.remote.port; } - if (ssh_config != NULL) { + if (instance_config.remote.ssh_config != NULL) { ssh_argv[ssh_argc++] = (char*)"-F"; - ssh_argv[ssh_argc++] = ssh_config; + ssh_argv[ssh_argc++] = instance_config.remote.ssh_config; } - if (ssh_options != NULL) { - ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, ssh_options); + if (instance_config.remote.ssh_options != NULL) { + ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, instance_config.remote.ssh_options); } - ssh_argv[ssh_argc++] = remote_host; + ssh_argv[ssh_argc++] = instance_config.remote.host; ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; - if (remote_path) + if (instance_config.remote.path) { char* sep = strrchr(pg_probackup, '/'); if (sep != NULL) { pg_probackup = sep + 1; } - dst = snprintf(cmd, sizeof(cmd), "%s/%s", remote_path, pg_probackup); + dst = snprintf(cmd, sizeof(cmd), "%s/%s", instance_config.remote.path, pg_probackup); } else { dst = snprintf(cmd, sizeof(cmd), "%s", pg_probackup); } From 9311590adf38b53c1063d6f87ee71d12f7b82de1 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 21 Jan 2019 19:22:05 +0300 Subject: [PATCH 0231/2107] More remote options to config --- src/utils/remote.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/utils/remote.h diff --git a/src/utils/remote.h b/src/utils/remote.h new file mode 100644 index 000000000..91708f65f --- /dev/null +++ b/src/utils/remote.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * remote.h: - prototypes of remote functions. + * + * Copyright (c) 2017-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#ifndef REMOTE_H +#define REMOTE_H + +typedef struct RemoteConfig +{ + char* proto; + char* host; + char* port; + char* path; + char *ssh_config; + char *ssh_options; + bool enabled; +} RemoteConfig; + +#endif From 0df77d0c11148bb0b7d64cec92a75ba354537dec Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 22 Jan 2019 12:20:58 +0300 Subject: [PATCH 0232/2107] Optional argument for boolean options --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index db3f911b3..efb47ee8d 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -107,7 +107,7 @@ option_has_arg(char type) { case 'b': case 'B': - return no_argument; + return optional_argument; default: return required_argument; } From 03056c7238c4051fda54d91a4f329b5a9a2382fb Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 22 Jan 2019 14:40:49 +0300 Subject: [PATCH 0233/2107] Disable logging to file in agent --- src/utils/file.c | 14 +++++++++++--- src/utils/logger.c | 6 ++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index d0064c2af..1fa995be7 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -268,15 +268,23 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) FILE* f; if (fio_is_remote(location)) { - int flags = O_RDWR|O_CREAT|PG_BINARY|(strchr(mode, '+') ? 0 : O_TRUNC); - int fd = fio_open(path, flags, location); + int flags = O_RDWR|O_CREAT; + int fd; + if (strcmp(mode, PG_BINARY_W) == 0) { + flags |= O_TRUNC|PG_BINARY; + } else if (strncmp(mode, PG_BINARY_R, strlen(PG_BINARY_R)) == 0) { + flags |= PG_BINARY; + } else if (strcmp(mode, "a") == 0) { + flags |= O_APPEND; + } + fd = fio_open(path, flags, location); f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); } else { f = fopen(path, mode); if (f == NULL && strcmp(mode, PG_BINARY_R "+") == 0) - f = fopen(path, PG_BINARY_W); + f = fopen(path, PG_BINARY_W); } return f; } diff --git a/src/utils/logger.c b/src/utils/logger.c index a94f4f5cf..3e0ed16ef 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -157,8 +157,10 @@ elog_internal(int elevel, bool file_only, const char *message) time_t log_time = (time_t) time(NULL); char strfbuf[128]; - write_to_file = elevel >= logger_config.log_level_file && - logger_config.log_directory && logger_config.log_directory[0] != '\0'; + write_to_file = elevel >= logger_config.log_level_file + && logger_config.log_directory + && logger_config.log_directory[0] != '\0' + && !remote_agent; write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && logger_config.log_directory && logger_config.log_directory[0] != '\0'; write_to_stderr = elevel >= (remote_agent ? ERROR : logger_config.log_level_console) && !file_only; From 8f980b6d353393f8b1940194593b3f87a371b031 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 22 Jan 2019 18:18:53 +0300 Subject: [PATCH 0234/2107] Handle string options with spaces --- src/configure.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/configure.c b/src/configure.c index 5856fad57..c6af62082 100644 --- a/src/configure.c +++ b/src/configure.c @@ -280,7 +280,10 @@ do_set_config(void) fprintf(fp, "# %s\n", current_group); } - fprintf(fp, "%s = %s\n", opt->lname, value); + if (strchr(value, ' ')) + fprintf(fp, "%s = '%s'\n", opt->lname, value); + else + fprintf(fp, "%s = %s\n", opt->lname, value); pfree(value); } From f8e9af6930b218e2f5bcfe90d412d834d83c0a81 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 22 Jan 2019 18:45:39 +0300 Subject: [PATCH 0235/2107] Fix remote restore from backup --- src/pg_probackup.c | 2 +- src/restore.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 76023c79d..5b5519a99 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -167,7 +167,7 @@ static ConfigOption cmd_options[] = /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, /* remote options */ - { 's', 155, "agent", &remote_agent, SOURCE_CMD_STRICT, }, + { 's', 155, "agent", &remote_agent, SOURCE_CMD_STRICT }, { 0 } }; diff --git a/src/restore.c b/src/restore.c index 601d1398e..d1c4f740f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -457,7 +457,7 @@ restore_backup(pgBackup *backup) */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, list_path, FIO_DB_HOST); + files = dir_read_file_list(database_path, list_path, FIO_BACKUP_HOST); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); @@ -523,7 +523,7 @@ remove_deleted_files(pgBackup *backup) pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, filelist_path, FIO_DB_HOST); + files = dir_read_file_list(instance_config.pgdata, filelist_path, FIO_BACKUP_HOST); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ From 66ebfcbd045a23377402695dd55f52ecec73d41c Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 22 Jan 2019 20:41:17 +0300 Subject: [PATCH 0236/2107] Remove --remote option --- src/backup.c | 13 +++++++------ src/configure.c | 5 ----- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 3 ++- src/utils/file.c | 2 +- src/utils/remote.h | 1 - 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/backup.c b/src/backup.c index de46c7801..55dacc1a7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -492,7 +492,7 @@ do_backup_instance(void) current.data_bytes = 0; /* Obtain current timeline */ - if (instance_config.remote.enabled) + if (IsReplicationProtocol()) { char *sysidentifier; TimeLineID starttli; @@ -632,7 +632,8 @@ do_backup_instance(void) backup_files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - if (instance_config.remote.enabled) + + if (IsReplicationProtocol()) get_remote_pgdata_filelist(backup_files_list); else dir_list_file(backup_files_list, instance_config.pgdata, @@ -699,7 +700,7 @@ do_backup_instance(void) char *dir_name; char database_path[MAXPGPATH]; - if (!instance_config.remote.enabled) + if (!IsReplicationProtocol()) dir_name = GetRelativePath(file->path, instance_config.pgdata); else dir_name = file->path; @@ -749,7 +750,7 @@ do_backup_instance(void) elog(VERBOSE, "Start thread num: %i", i); - if (!instance_config.remote.enabled) + if (!IsReplicationProtocol()) pthread_create(&threads[i], NULL, backup_files, arg); else pthread_create(&threads[i], NULL, remote_backup_files, arg); @@ -902,7 +903,7 @@ do_backup(time_t start_time) check_server_version(); /* TODO fix it for remote backup*/ - if (!instance_config.remote.enabled) + if (!IsReplicationProtocol()) current.checksum_version = get_data_checksum_version(true); is_checksum_enabled = pg_checksum_enable(); @@ -958,7 +959,7 @@ do_backup(time_t start_time) * belogns to the same instance. */ /* TODO fix it for remote backup */ - if (!instance_config.remote.enabled) + if (!IsReplicationProtocol()) check_system_identifiers(); diff --git a/src/configure.c b/src/configure.c index c6af62082..43b1f3408 100644 --- a/src/configure.c +++ b/src/configure.c @@ -204,11 +204,6 @@ ConfigOption instance_options[] = &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, - { - 'b', 225, "remote", - &instance_config.remote.enabled, SOURCE_CMD, 0, - OPTION_REMOTE_GROUP, 0, option_get_value - }, { 0 } }; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5b5519a99..043e81fee 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -333,7 +333,7 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); - if (IsSshConnection() + if (IsSshProtocol() && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) { @@ -501,7 +501,7 @@ main(int argc, char *argv[]) return do_init(); case BACKUP_CMD: current.stream = stream_wal; - if (IsSshConnection() && !remote_agent) + if (IsSshProtocol() && !remote_agent) { current.status = BACKUP_STATUS_DONE; StrNCpy(current.program_version, PROGRAM_VERSION, @@ -519,7 +519,7 @@ main(int argc, char *argv[]) elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", instance_config.remote.enabled ? "true" : "false"); + stream_wal ? "true" : "false", instance_config.remote.host ? "true" : "false"); return do_backup(start_time); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1afa8f017..7cb7042db 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -348,7 +348,8 @@ typedef struct XLByteInSeg(xlrp, logSegNo) #endif -#define IsSshConnection() (instance_config.remote.enabled && strcmp(instance_config.remote.proto, "ssh") == 0) +#define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) +#define IsReplicationProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "replication") == 0) /* directory options */ extern char *backup_path; diff --git a/src/utils/file.c b/src/utils/file.c index 1fa995be7..c11024968 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -49,7 +49,7 @@ static bool fio_is_remote(fio_location location) { return location == FIO_REMOTE_HOST || (location == FIO_BACKUP_HOST && remote_agent) /* agent is launched at Postgres side */ - || (location == FIO_DB_HOST && !remote_agent && IsSshConnection()); + || (location == FIO_DB_HOST && !remote_agent && IsSshProtocol()); } /* Try to read specified amount of bytes unless error or EOF are encountered */ diff --git a/src/utils/remote.h b/src/utils/remote.h index 91708f65f..6705a7938 100644 --- a/src/utils/remote.h +++ b/src/utils/remote.h @@ -18,7 +18,6 @@ typedef struct RemoteConfig char* path; char *ssh_config; char *ssh_options; - bool enabled; } RemoteConfig; #endif From 8659fc6ab54f39cb55c8c94a94903bfb2717bb6f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 23 Jan 2019 12:18:18 +0300 Subject: [PATCH 0237/2107] Fix passing options with spaces to remote backup --- src/utils/remote.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 6e031b66b..ac4886d11 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -15,13 +15,21 @@ static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) { size_t len = strlen(src); - if (dst + len + 1 >= buf_size) { + if (dst + len + 3 >= buf_size) { fprintf(stderr, "Too long command line\n"); exit(EXIT_FAILURE); } - buf[dst] = ' '; - memcpy(&buf[dst+1], src, len); - return dst + len + 1; + buf[dst++] = ' '; + if (strchr(src, ' ') != NULL) { /* need quotes */ + buf[dst++] = '\''; + memcpy(&buf[dst], src, len); + dst += len; + buf[dst++] = '\''; + } else { + memcpy(&buf[dst], src, len); + dst += len; + } + return dst; } static int split_options(int argc, char* argv[], int max_options, char* options) @@ -93,7 +101,7 @@ int remote_execute(int argc, char* argv[], bool listen) ssh_argv[ssh_argc++] = instance_config.remote.ssh_config; } if (instance_config.remote.ssh_options != NULL) { - ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, instance_config.remote.ssh_options); + ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options)); } ssh_argv[ssh_argc++] = instance_config.remote.host; ssh_argv[ssh_argc++] = cmd; From 6e261cc2adff709e413370afb645b9d8bb56e5a3 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 23 Jan 2019 15:55:00 +0300 Subject: [PATCH 0238/2107] Read configuration before launching remote agent --- src/pg_probackup.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 043e81fee..45345580a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -333,6 +333,23 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); + /* + * We read options from command line, now we need to read them from + * configuration file since we got backup path and instance name. + * For some commands an instance option isn't required, see above. + */ + if (instance_name) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + config_get_opt_env(instance_options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + config_read_opt(path, instance_options, ERROR, true); + } + if (IsSshProtocol() && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) @@ -392,23 +409,6 @@ main(int argc, char *argv[]) } } - /* - * We read options from command line, now we need to read them from - * configuration file since we got backup path and instance name. - * For some commands an instance option isn't required, see above. - */ - if (instance_name) - { - char path[MAXPGPATH]; - - /* Read environment variables */ - config_get_opt_env(instance_options); - - /* Read options from configuration file */ - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(path, instance_options, ERROR, true); - } - /* Initialize logger */ init_logger(backup_path, &instance_config.logger); From 9b6e5da63fc5f5570de0f2576909e1b979b5dd91 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 23 Jan 2019 17:26:32 +0300 Subject: [PATCH 0239/2107] Fix configuration of reomte agent --- src/fetch.c | 9 +++++ src/pg_probackup.c | 83 ++++++++++++++++++++------------------- src/utils/configuration.c | 6 ++- src/utils/remote.c | 24 +++++++++++ 4 files changed, 79 insertions(+), 43 deletions(-) diff --git a/src/fetch.c b/src/fetch.c index 51aea915d..4dfd3a700 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -34,6 +34,15 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi int len; snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); + if (fio_access(fullpath, R_OK, location) != 0) + { + if (safe) + return NULL; + else + elog(ERROR, "could not open file \"%s\" for reading: %s", + fullpath, strerror(errno)); + } + if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) { if (safe) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 45345580a..a769ff9ce 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -329,47 +329,6 @@ main(int argc, char *argv[]) } canonicalize_path(backup_path); - /* Ensure that backup_path is an absolute path */ - if (!is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); - - /* - * We read options from command line, now we need to read them from - * configuration file since we got backup path and instance name. - * For some commands an instance option isn't required, see above. - */ - if (instance_name) - { - char path[MAXPGPATH]; - - /* Read environment variables */ - config_get_opt_env(instance_options); - - /* Read options from configuration file */ - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(path, instance_options, ERROR, true); - } - - if (IsSshProtocol() - && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || - backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) - { - if (remote_agent) { - if (backup_subcmd != BACKUP_CMD && backup_subcmd != ARCHIVE_PUSH_CMD) { - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - } - fio_redirect(STDIN_FILENO, STDOUT_FILENO); - } else { - /* Execute remote probackup */ - int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD); - if (status != 0 || backup_subcmd == ARCHIVE_PUSH_CMD) - { - return status; - } - } - } - if (!remote_agent) { /* Ensure that backup_path is a path to a directory */ @@ -378,6 +337,11 @@ main(int argc, char *argv[]) elog(ERROR, "-B, --backup-path must be a path to directory"); } + /* Ensure that backup_path is an absolute path */ + if (!is_absolute_path(backup_path)) + elog(ERROR, "-B, --backup-path must be an absolute path"); + + /* Option --instance is required for all commands except init and show */ if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD) @@ -409,9 +373,46 @@ main(int argc, char *argv[]) } } + /* + * We read options from command line, now we need to read them from + * configuration file since we got backup path and instance name. + * For some commands an instance option isn't required, see above. + */ + if (instance_name && !remote_agent) + { + char path[MAXPGPATH]; + + /* Read environment variables */ + config_get_opt_env(instance_options); + + /* Read options from configuration file */ + join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + config_read_opt(path, instance_options, ERROR, true); + } + /* Initialize logger */ init_logger(backup_path, &instance_config.logger); + if (IsSshProtocol() + && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || + backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) + { + if (remote_agent) { + if (backup_subcmd != BACKUP_CMD && backup_subcmd != ARCHIVE_PUSH_CMD) { + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; + } + fio_redirect(STDIN_FILENO, STDOUT_FILENO); + } else { + /* Execute remote probackup */ + int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD); + if (status != 0 || backup_subcmd == ARCHIVE_PUSH_CMD) + { + return status; + } + } + } + /* command was initialized for a few commands */ if (command) { diff --git a/src/utils/configuration.c b/src/utils/configuration.c index efb47ee8d..bf477e84c 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -8,6 +8,7 @@ *------------------------------------------------------------------------- */ +#include "pg_probackup.h" #include "configuration.h" #include "logger.h" #include "pgut.h" @@ -493,8 +494,9 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], if (opt == NULL) opt = option_find(c, options); - if (opt && - opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) + if (opt + && !remote_agent + && opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) elog(ERROR, "Option %s cannot be specified in command line", opt->lname); /* Check 'opt == NULL' is performed in assign_option() */ diff --git a/src/utils/remote.c b/src/utils/remote.c index ac4886d11..f466f0333 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -117,11 +117,34 @@ int remote_execute(int argc, char* argv[], bool listen) } else { dst = snprintf(cmd, sizeof(cmd), "%s", pg_probackup); } + for (i = 1; i < argc; i++) { dst = append_option(cmd, sizeof(cmd), dst, argv[i]); } + dst = append_option(cmd, sizeof(cmd), dst, "--agent"); dst = append_option(cmd, sizeof(cmd), dst, PROGRAM_VERSION); + + for (i = 0; instance_options[i].type; i++) { + ConfigOption *opt = &instance_options[i]; + char *value; + char *cmd_opt; + + /* Path only options from file */ + if (opt->source != SOURCE_FILE) + continue; + + value = opt->get_value(opt); + if (value == NULL) + continue; + + cmd_opt = psprintf("--%s=%s", opt->lname, value); + + dst = append_option(cmd, sizeof(cmd), dst, cmd_opt); + pfree(value); + pfree(cmd_opt); + } + cmd[dst] = '\0'; SYS_CHECK(pipe(infd)); @@ -154,6 +177,7 @@ int remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(close(outfd[0])); SYS_CHECK(close(errfd[1])); atexit(kill_child); + if (listen) { int status; fio_communicate(infd[0], outfd[1]); From 516cfa321445f26e8e8647f633e381b57cce8541 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 23 Jan 2019 18:46:51 +0300 Subject: [PATCH 0240/2107] Read agent error messages --- src/utils/logger.c | 4 ++-- src/utils/remote.c | 46 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 3e0ed16ef..4f80b87d6 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -162,8 +162,8 @@ elog_internal(int elevel, bool file_only, const char *message) && logger_config.log_directory[0] != '\0' && !remote_agent; write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && - logger_config.log_directory && logger_config.log_directory[0] != '\0'; - write_to_stderr = elevel >= (remote_agent ? ERROR : logger_config.log_level_console) && !file_only; + logger_config.log_directory && logger_config.log_directory[0] != '\0'&& !remote_agent; + write_to_stderr = (elevel >= (remote_agent ? ERROR : logger_config.log_level_console) && !file_only) || remote_agent; pthread_lock(&log_file_mutex); loggin_in_progress = true; diff --git a/src/utils/remote.c b/src/utils/remote.c index f466f0333..45b8cd62e 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -4,13 +4,14 @@ #include #include #include +#include #include "pg_probackup.h" #include "file.h" #define MAX_CMDLINE_LENGTH 4096 #define MAX_CMDLINE_OPTIONS 256 -#define ERR_BUF_SIZE 1024 +#define ERR_BUF_SIZE 4096 static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) { @@ -78,6 +79,37 @@ static void kill_child(void) kill(child_pid, SIGTERM); } +static void* error_reader_proc(void* arg) +{ + int* errfd = (int*)arg; + char buf[ERR_BUF_SIZE]; + int offs = 0, rc; + + while ((rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0) + { + char* nl; + offs += rc; + buf[offs] = '\0'; + nl = strchr(buf, '\n'); + if (nl != NULL) { + *nl = '\0'; + if (strncmp(buf, "ERROR: ", 7) == 0) { + elog(ERROR, "%s", buf + 7); + } if (strncmp(buf, "WARNING: ", 9) == 0) { + elog(WARNING, "%s", buf + 9); + } else if (strncmp(buf, "LOG: ", 5) == 0) { + elog(LOG, "%s", buf + 5); + } else if (strncmp(buf, "INFO: ", 6) == 0) { + elog(INFO, "%s", buf + 6); + } else { + elog(LOG, "%s", buf); + } + memmove(buf, nl+1, offs -= (nl + 1 - buf)); + } + } + return NULL; +} + int remote_execute(int argc, char* argv[], bool listen) { char cmd[MAX_CMDLINE_LENGTH]; @@ -89,6 +121,7 @@ int remote_execute(int argc, char* argv[], bool listen) int infd[2]; int errfd[2]; char* pg_probackup = argv[0]; + pthread_t error_reader_thread; ssh_argc = 0; ssh_argv[ssh_argc++] = instance_config.remote.proto; @@ -178,18 +211,13 @@ int remote_execute(int argc, char* argv[], bool listen) SYS_CHECK(close(errfd[1])); atexit(kill_child); + pthread_create(&error_reader_thread, NULL, error_reader_proc, errfd); + if (listen) { int status; fio_communicate(infd[0], outfd[1]); + SYS_CHECK(wait(&status)); - if (status != 0) - { - char buf[ERR_BUF_SIZE]; - int offs, rc; - for (offs = 0; (rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0; offs += rc); - buf[offs] = '\0'; - elog(ERROR, "%s", strncmp(buf, "ERROR: ", 6) == 0 ? buf + 6 : buf); - } return status; } else { fio_redirect(infd[0], outfd[1]); /* write to stdout */ From 85b247d76dd877c262e204308c3c6ab8d3ff86eb Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 23 Jan 2019 23:33:22 +0300 Subject: [PATCH 0241/2107] Fix problem with logging filtering --- src/utils/logger.c | 12 ++++++++---- src/utils/remote.c | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 4f80b87d6..3ac7ef672 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -159,12 +159,16 @@ elog_internal(int elevel, bool file_only, const char *message) write_to_file = elevel >= logger_config.log_level_file && logger_config.log_directory - && logger_config.log_directory[0] != '\0' - && !remote_agent; + && logger_config.log_directory[0] != '\0'; write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && - logger_config.log_directory && logger_config.log_directory[0] != '\0'&& !remote_agent; - write_to_stderr = (elevel >= (remote_agent ? ERROR : logger_config.log_level_console) && !file_only) || remote_agent; + logger_config.log_directory && logger_config.log_directory[0] != '\0'; + write_to_stderr = elevel >= logger_config.log_level_console && !file_only; + if (remote_agent) + { + write_to_stderr |= write_to_error_log | write_to_file; + write_to_error_log = write_to_file = false; + } pthread_lock(&log_file_mutex); loggin_in_progress = true; diff --git a/src/utils/remote.c b/src/utils/remote.c index 45b8cd62e..6cda134ca 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -97,6 +97,8 @@ static void* error_reader_proc(void* arg) elog(ERROR, "%s", buf + 7); } if (strncmp(buf, "WARNING: ", 9) == 0) { elog(WARNING, "%s", buf + 9); + } if (strncmp(buf, "VERBOSE: ", 9) == 0) { + elog(VERBOSE, "%s", buf + 9); } else if (strncmp(buf, "LOG: ", 5) == 0) { elog(LOG, "%s", buf + 5); } else if (strncmp(buf, "INFO: ", 6) == 0) { From 4e3e62b1cf1487a3c92085f6a1cfe53aabd3e673 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 24 Jan 2019 00:28:49 +0300 Subject: [PATCH 0242/2107] Fix problem with logging --- src/utils/logger.c | 3 +++ src/utils/remote.c | 29 +++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 3ac7ef672..29efc5922 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -131,6 +131,9 @@ exit_if_necessary(int elevel) pthread_mutex_unlock(&log_file_mutex); } + if (remote_agent) + sleep(1); /* Let parent receive sent messages */ + /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) #ifdef WIN32 diff --git a/src/utils/remote.c b/src/utils/remote.c index 6cda134ca..e5d397a4a 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -79,12 +79,20 @@ static void kill_child(void) kill(child_pid, SIGTERM); } + +static void print_message(char const* buf) +{ + char const* const severity[] = {"VERBOSE", "LOG", "INFO", "NOTICE", "WARNING", "FATAL", "ERROR", ""}; + size_t i; + for (i = 0; strncmp(buf, severity[i], strlen(severity[i])) != 0; i++); + elog(i+VERBOSE, "%s", buf + strlen(severity[i]) + 2); +} + static void* error_reader_proc(void* arg) { int* errfd = (int*)arg; char buf[ERR_BUF_SIZE]; int offs = 0, rc; - while ((rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0) { char* nl; @@ -93,22 +101,15 @@ static void* error_reader_proc(void* arg) nl = strchr(buf, '\n'); if (nl != NULL) { *nl = '\0'; - if (strncmp(buf, "ERROR: ", 7) == 0) { - elog(ERROR, "%s", buf + 7); - } if (strncmp(buf, "WARNING: ", 9) == 0) { - elog(WARNING, "%s", buf + 9); - } if (strncmp(buf, "VERBOSE: ", 9) == 0) { - elog(VERBOSE, "%s", buf + 9); - } else if (strncmp(buf, "LOG: ", 5) == 0) { - elog(LOG, "%s", buf + 5); - } else if (strncmp(buf, "INFO: ", 6) == 0) { - elog(INFO, "%s", buf + 6); - } else { - elog(LOG, "%s", buf); - } + print_message(buf); memmove(buf, nl+1, offs -= (nl + 1 - buf)); } } + if (offs != 0) + { + buf[offs] = '\0'; + print_message(buf); + } return NULL; } From ab94800faba2f80fc53d2dfb29c5aecab0dde656 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 24 Jan 2019 14:17:32 +0300 Subject: [PATCH 0243/2107] Fix log message redirection --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index e5d397a4a..bfc22b6c2 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -85,7 +85,7 @@ static void print_message(char const* buf) char const* const severity[] = {"VERBOSE", "LOG", "INFO", "NOTICE", "WARNING", "FATAL", "ERROR", ""}; size_t i; for (i = 0; strncmp(buf, severity[i], strlen(severity[i])) != 0; i++); - elog(i+VERBOSE, "%s", buf + strlen(severity[i]) + 2); + elog(i+VERBOSE > ERROR ? LOG : i + VERBOSE, "%s", buf + strlen(severity[i]) + 2); } static void* error_reader_proc(void* arg) From df120095b199d005ecb250dfe3b65c7075ac5942 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 24 Jan 2019 14:36:50 +0300 Subject: [PATCH 0244/2107] Fix SIGSEGV in write_backup_status --- src/catalog.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 9583edf33..2dac05f9f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -230,11 +230,13 @@ write_backup_status(pgBackup *backup) pgBackup *tmp; tmp = read_backup(backup->start_time); + if (tmp != NULL) + { + tmp->status = backup->status; + write_backup(tmp); - tmp->status = backup->status; - write_backup(tmp); - - pgBackupFree(tmp); + pgBackupFree(tmp); + } } /* From ab5d6022587b030c612a80381dff4666a4569e1c Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 24 Jan 2019 18:33:53 +0300 Subject: [PATCH 0245/2107] Fix directory remove --- src/dir.c | 2 +- src/utils/file.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index 2c21ea3ad..4e4cffd88 100644 --- a/src/dir.c +++ b/src/dir.c @@ -686,7 +686,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink, FIO_LOCAL_HOST); + file = pgFileNew(child, omit_symlink, location); if (file == NULL) continue; diff --git a/src/utils/file.c b/src/utils/file.c index c11024968..8055ed9d5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -664,7 +664,7 @@ int fio_unlink(char const* path, fio_location location) } else { - return unlink(path); + return remove(path); } } @@ -925,7 +925,7 @@ void fio_communicate(int in, int out) SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); break; case FIO_UNLINK: - SYS_CHECK(unlink(buf)); + SYS_CHECK(remove(buf)); break; case FIO_MKDIR: SYS_CHECK(dir_create_dir(buf, hdr.arg)); From a04361abdddca7f659b0a6d6ec9488385c92a005 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 25 Jan 2019 15:46:31 +0300 Subject: [PATCH 0246/2107] Fix transfer agent error messages --- src/utils/remote.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index bfc22b6c2..f43306026 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -85,7 +85,11 @@ static void print_message(char const* buf) char const* const severity[] = {"VERBOSE", "LOG", "INFO", "NOTICE", "WARNING", "FATAL", "ERROR", ""}; size_t i; for (i = 0; strncmp(buf, severity[i], strlen(severity[i])) != 0; i++); - elog(i+VERBOSE > ERROR ? LOG : i + VERBOSE, "%s", buf + strlen(severity[i]) + 2); + if (i + VERBOSE > ERROR) { + elog(LOG, "%s", buf); + } else { + elog(i+VERBOSE, "%s", buf + strlen(severity[i]) + 2); + } } static void* error_reader_proc(void* arg) From a806c65b5a6cf434eb9b8cf2ad1ea3c66bb72725 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 25 Jan 2019 16:26:48 +0300 Subject: [PATCH 0247/2107] Fix error decoding --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index f43306026..11844b4a4 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -83,7 +83,7 @@ static void kill_child(void) static void print_message(char const* buf) { char const* const severity[] = {"VERBOSE", "LOG", "INFO", "NOTICE", "WARNING", "FATAL", "ERROR", ""}; - size_t i; + int i; for (i = 0; strncmp(buf, severity[i], strlen(severity[i])) != 0; i++); if (i + VERBOSE > ERROR) { elog(LOG, "%s", buf); From 9bf541b85373fc44d0bf4935930ad82831f60923 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Jan 2019 19:37:21 +0300 Subject: [PATCH 0248/2107] tests: snapfs module added --- tests/__init__.py | 3 ++- tests/snapfs.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tests/snapfs.py diff --git a/tests/__init__.py b/tests/__init__.py index 9156b649b..c5e01e298 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,7 +8,7 @@ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp + time_stamp, snapfs def load_tests(loader, tests, pattern): @@ -46,6 +46,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(restore_test)) suite.addTests(loader.loadTestsFromModule(retention_test)) suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(snapfs)) suite.addTests(loader.loadTestsFromModule(validate_test)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) diff --git a/tests/snapfs.py b/tests/snapfs.py new file mode 100644 index 000000000..31de79b5c --- /dev/null +++ b/tests/snapfs.py @@ -0,0 +1,62 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'snapfs' + + +class SnapFSTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'select pg_make_snapshot()') + + node.pgbench_init(scale=100) + + pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + 'postgres', + 'select pg_remove_snashot(1)') + + self.backup_node( + backup_dir, 'node', node) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', + restored_node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 158a3df648e078f9b4aaa5919c4f53a2c90c3f39 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 25 Jan 2019 20:12:46 +0300 Subject: [PATCH 0249/2107] Start implementation of new remote pull approach --- src/backup.c | 18 ++------------ src/pg_probackup.c | 61 +++++++++++++--------------------------------- 2 files changed, 19 insertions(+), 60 deletions(-) diff --git a/src/backup.c b/src/backup.c index 55dacc1a7..1891da3b1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -618,9 +618,6 @@ do_backup_instance(void) elog(ERROR, "Cannot continue backup because stream connect has failed."); } - if (remote_agent) - xlog_files_list = parray_new(); - /* By default there are some error */ stream_thread_arg.ret = 1; stream_thread_arg.files_list = xlog_files_list; @@ -999,19 +996,6 @@ do_backup(time_t start_time) //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", // current.data_bytes); - if (remote_agent) - { - fio_transfer(FIO_BACKUP_START_TIME); - fio_transfer(FIO_BACKUP_STOP_LSN); - } - else - complete_backup(); - - return 0; -} - -void complete_backup(void) -{ pgBackupValidate(¤t); elog(INFO, "Backup %s completed", base36enc(current.start_time)); @@ -1022,6 +1006,8 @@ void complete_backup(void) */ if (delete_expired || delete_wal) do_retention_purge(); + + return 0; } /* diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a769ff9ce..740f462a6 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -304,12 +304,17 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); - if (remote_agent != NULL && strcmp(remote_agent, PROGRAM_VERSION) != 0) + if (remote_agent != NULL) { - uint32 agent_version = parse_program_version(remote_agent); - elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, - "Agent version %s doesn't match master pg_probackup version %s", - remote_agent, PROGRAM_VERSION); + if (strcmp(remote_agent, PROGRAM_VERSION) != 0) + { + uint32 agent_version = parse_program_version(remote_agent); + elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, + "Agent version %s doesn't match master pg_probackup version %s", + remote_agent, PROGRAM_VERSION); + } + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; } pgut_init(); @@ -329,13 +334,10 @@ main(int argc, char *argv[]) } canonicalize_path(backup_path); - if (!remote_agent) - { - /* Ensure that backup_path is a path to a directory */ - rc = stat(backup_path, &stat_buf); - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); - } + /* Ensure that backup_path is a path to a directory */ + rc = stat(backup_path, &stat_buf); + if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) @@ -365,7 +367,7 @@ main(int argc, char *argv[]) * for all commands except init, which doesn't take this parameter * and add-instance which creates new instance. */ - if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && !remote_agent) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { if (access(backup_instance_path, F_OK) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -378,7 +380,7 @@ main(int argc, char *argv[]) * configuration file since we got backup path and instance name. * For some commands an instance option isn't required, see above. */ - if (instance_name && !remote_agent) + if (instance_name) { char path[MAXPGPATH]; @@ -393,26 +395,6 @@ main(int argc, char *argv[]) /* Initialize logger */ init_logger(backup_path, &instance_config.logger); - if (IsSshProtocol() - && (backup_subcmd == BACKUP_CMD || backup_subcmd == ADD_INSTANCE_CMD || backup_subcmd == RESTORE_CMD || - backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD)) - { - if (remote_agent) { - if (backup_subcmd != BACKUP_CMD && backup_subcmd != ARCHIVE_PUSH_CMD) { - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - } - fio_redirect(STDIN_FILENO, STDOUT_FILENO); - } else { - /* Execute remote probackup */ - int status = remote_execute(argc, argv, backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD); - if (status != 0 || backup_subcmd == ARCHIVE_PUSH_CMD) - { - return status; - } - } - } - /* command was initialized for a few commands */ if (command) { @@ -501,22 +483,13 @@ main(int argc, char *argv[]) case INIT_CMD: return do_init(); case BACKUP_CMD: - current.stream = stream_wal; - if (IsSshProtocol() && !remote_agent) - { - current.status = BACKUP_STATUS_DONE; - StrNCpy(current.program_version, PROGRAM_VERSION, - sizeof(current.program_version)); - complete_backup(); - return 0; - } - else { const char *backup_mode; time_t start_time; start_time = time(NULL); backup_mode = deparse_backup_mode(current.backup_mode); + current.stream = stream_wal; elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, From 40ef9e9b1bbc3c9e007bbe90e23e6360991b08bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Jan 2019 21:33:34 +0300 Subject: [PATCH 0250/2107] tests: module logging added --- tests/__init__.py | 3 ++- tests/backup_test.py | 1 - tests/logging.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index c5e01e298..c3e149799 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,7 +8,7 @@ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp, snapfs + time_stamp, snapfs, logging def load_tests(loader, tests, pattern): @@ -27,6 +27,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(exclude)) suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(merge)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(page)) diff --git a/tests/backup_test.py b/tests/backup_test.py index fcc541777..2d4a71bdf 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -2,7 +2,6 @@ import os from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name module_name = 'backup' diff --git a/tests/logging.py b/tests/logging.py index e69de29bb..173a977a1 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -0,0 +1,44 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'logging' + + +class LogTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-2154 + def test_log_rotation(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1s', '--log-rotation-size=1MB']) + + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=verbose']) + + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=verbose'], gdb=True) + + gdb.set_breakpoint('open_logfile') + gdb.run_until_break() + gdb.continue_execution_until_exit() + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From 4450134ac04981b7b18beb796570e5297ceed8b9 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 25 Jan 2019 23:22:52 +0300 Subject: [PATCH 0251/2107] Adopt fio for pull model --- src/data.c | 10 ++-- src/pg_probackup.c | 5 ++ src/pg_probackup.h | 3 +- src/utils/file.c | 113 ++++------------------------------------- src/utils/file.h | 13 +---- src/utils/remote.c | 122 +++------------------------------------------ 6 files changed, 30 insertions(+), 236 deletions(-) diff --git a/src/data.c b/src/data.c index d210f0c52..bde3b27f8 100644 --- a/src/data.c +++ b/src/data.c @@ -220,7 +220,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, elog(ERROR, "File: %s, could not seek to block %u: %s", file->path, blknum, strerror(errno)); - read_len = fread(page, 1, BLCKSZ, in); + read_len = fio_fread(in, page, BLCKSZ); if (read_len != BLCKSZ) { @@ -552,7 +552,7 @@ backup_data_file(backup_files_arg* arguments, INIT_FILE_CRC32(true, file->crc); /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); + in = fio_fopen(file->path, PG_BINARY_R, FIO_DB_HOST); if (in == NULL) { FIN_FILE_CRC32(true, file->crc); @@ -589,7 +589,7 @@ backup_data_file(backup_files_arg* arguments, if (out == NULL) { int errno_tmp = errno; - fclose(in); + fio_fclose(in); elog(ERROR, "cannot open backup file \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -647,7 +647,7 @@ backup_data_file(backup_files_arg* arguments, if (fio_chmod(to_path, FILE_PERMISSION, FIO_BACKUP_HOST) == -1) { int errno_tmp = errno; - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot change mode of \"%s\": %s", file->path, strerror(errno_tmp)); @@ -657,7 +657,7 @@ backup_data_file(backup_files_arg* arguments, fio_fclose(out)) elog(ERROR, "cannot write backup file \"%s\": %s", to_path, strerror(errno)); - fclose(in); + fio_fclose(in); FIN_FILE_CRC32(true, file->crc); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 740f462a6..edd52089f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -41,6 +41,9 @@ typedef enum ProbackupSubcmd SHOW_CONFIG_CMD } ProbackupSubcmd; + +char *pg_probackup; /* Program name (argv[0]) */ + /* directory options */ char *backup_path = NULL; /* @@ -183,6 +186,8 @@ main(int argc, char *argv[]) struct stat stat_buf; int rc; + pg_probackup = argv[0]; + /* Initialize current backup */ pgBackupInit(¤t); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7cb7042db..4d8b49706 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -352,6 +352,7 @@ typedef struct #define IsReplicationProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "replication") == 0) /* directory options */ +extern char *pg_probackup; extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; @@ -497,7 +498,7 @@ extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); -extern int remote_execute(int argc, char *argv[], bool do_backup); +extern bool launch_agent(void); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS #define COMPRESS_LEVEL_DEFAULT 1 diff --git a/src/utils/file.c b/src/utils/file.c index 8055ed9d5..05a497af8 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -9,18 +9,10 @@ #define PRINTF_BUF_SIZE 1024 #define FILE_PERMISSIONS 0600 -static pthread_mutex_t fio_read_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_mutex_t fio_write_mutex = PTHREAD_MUTEX_INITIALIZER; -static unsigned long fio_fdset = 0; -static void* fio_stdin_buffer; -static int fio_stdout = 0; -static int fio_stdin = 0; - -static fio_binding fio_bindings[] = -{ - {¤t.start_time, sizeof(current.start_time)}, - {¤t.stop_lsn, sizeof(current.stop_lsn)} -}; +static __thread unsigned long fio_fdset = 0; +static __thread void* fio_stdin_buffer; +static __thread int fio_stdout = 0; +static __thread int fio_stdin = 0; /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -47,9 +39,11 @@ static bool fio_is_remote_fd(int fd) /* Check if specified location is local for current node */ static bool fio_is_remote(fio_location location) { - return location == FIO_REMOTE_HOST - || (location == FIO_BACKUP_HOST && remote_agent) /* agent is launched at Postgres side */ - || (location == FIO_DB_HOST && !remote_agent && IsSshProtocol()); + bool is_remote = location == FIO_REMOTE_HOST + || (location == FIO_DB_HOST && IsSshProtocol()); + if (is_remote && !fio_stdin && !launch_agent()) + elog(ERROR, "Failed to establish SSH connection: %s", strerror(errno)); + return is_remote; } /* Try to read specified amount of bytes unless error or EOF are encountered */ @@ -145,8 +139,6 @@ DIR* fio_opendir(char const* path, fio_location location) fio_header hdr; unsigned long mask; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) { @@ -160,8 +152,6 @@ DIR* fio_opendir(char const* path, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - dir = (DIR*)(size_t)(i + 1); } else @@ -179,14 +169,10 @@ struct dirent* fio_readdir(DIR *dir) fio_header hdr; static struct dirent entry; - SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); - - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); hdr.cop = FIO_READDIR; hdr.handle = (size_t)dir - 1; hdr.size = 0; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_SEND); @@ -194,7 +180,6 @@ struct dirent* fio_readdir(DIR *dir) Assert(hdr.size == sizeof(entry)); IO_CHECK(fio_read_all(fio_stdin, &entry, sizeof(entry)), sizeof(entry)); } - SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); return hdr.size ? &entry : NULL; } @@ -214,9 +199,7 @@ int fio_closedir(DIR *dir) hdr.handle = (size_t)dir - 1; hdr.size = 0; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -235,8 +218,6 @@ int fio_open(char const* path, int mode, fio_location location) fio_header hdr; unsigned long mask; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) { @@ -251,8 +232,6 @@ int fio_open(char const* path, int mode, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - fd = i | FIO_PIPE_MARKER; } else @@ -350,8 +329,6 @@ int fio_close(int fd) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - hdr.cop = FIO_CLOSE; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; @@ -359,7 +336,6 @@ int fio_close(int fd) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -383,8 +359,6 @@ int fio_truncate(int fd, off_t size) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - hdr.cop = FIO_TRUNCATE; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; @@ -392,7 +366,6 @@ int fio_truncate(int fd, off_t size) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -416,8 +389,6 @@ int fio_seek(int fd, off_t offs) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - hdr.cop = FIO_SEEK; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; @@ -425,7 +396,6 @@ int fio_seek(int fd, off_t offs) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -449,8 +419,6 @@ ssize_t fio_write(int fd, void const* buf, size_t size) { fio_header hdr; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - hdr.cop = FIO_WRITE; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = size; @@ -458,7 +426,6 @@ ssize_t fio_write(int fd, void const* buf, size_t size) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return size; } else @@ -487,19 +454,12 @@ ssize_t fio_read(int fd, void* buf, size_t size) hdr.size = 0; hdr.arg = size; - SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_SEND); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); - return hdr.size; } else @@ -527,19 +487,12 @@ int fio_fstat(int fd, struct stat* st) hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = 0; - SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_FSTAT); IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); - return hdr.arg; } else @@ -561,20 +514,13 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati hdr.arg = follow_symlinks; hdr.size = path_len; - SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_STAT); IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); - return hdr.arg; } else @@ -595,19 +541,12 @@ int fio_access(char const* path, int mode, fio_location location) hdr.size = path_len; hdr.arg = mode; - SYS_CHECK(pthread_mutex_lock(&fio_read_mutex)); - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_ACCESS); - SYS_CHECK(pthread_mutex_unlock(&fio_read_mutex)); - return hdr.arg; } else @@ -628,13 +567,10 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location hdr.handle = -1; hdr.size = old_path_len + new_path_len; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -654,12 +590,9 @@ int fio_unlink(char const* path, fio_location location) hdr.handle = -1; hdr.size = path_len; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -680,12 +613,9 @@ int fio_mkdir(char const* path, int mode, fio_location location) hdr.size = path_len; hdr.arg = mode; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -706,12 +636,9 @@ int fio_chmod(char const* path, int mode, fio_location location) hdr.size = path_len; hdr.arg = mode; - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); return 0; } else @@ -823,24 +750,6 @@ static void fio_send_file(int out, char const* path) } } -/* Send values of variables. Variables are described infio_bindings array and "var" is index in this array. */ -void fio_transfer(fio_shared_variable var) -{ - size_t var_size = fio_bindings[var].size; - fio_header* msg = (fio_header*)malloc(sizeof(fio_header) + var_size); - - msg->cop = FIO_TRANSFER; - msg->arg = var; - msg->size = var_size; - memcpy(msg+1, fio_bindings[var].address, var_size); - - SYS_CHECK(pthread_mutex_lock(&fio_write_mutex)); - - IO_CHECK(fio_write_all(fio_stdout, msg, sizeof(fio_header) + var_size), sizeof(fio_header) + var_size); - - SYS_CHECK(pthread_mutex_unlock(&fio_write_mutex)); - free(msg); -} /* Execute commands at remote host */ void fio_communicate(int in, int out) @@ -939,10 +848,6 @@ void fio_communicate(int in, int out) case FIO_TRUNCATE: SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; - case FIO_TRANSFER: - Assert(hdr.size == fio_bindings[hdr.arg].size); - memcpy(fio_bindings[hdr.arg].address, buf, hdr.size); - break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 1f460417f..58f341d32 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -26,7 +26,6 @@ typedef enum FIO_FSTAT, FIO_SEND, FIO_ACCESS, - FIO_TRANSFER, FIO_OPENDIR, FIO_READDIR, FIO_CLOSEDIR @@ -40,17 +39,11 @@ typedef enum FIO_REMOTE_HOST /* date is located at remote host */ } fio_location; -typedef enum -{ - FIO_BACKUP_START_TIME, - FIO_BACKUP_STOP_LSN -} fio_shared_variable; - #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 -#define SYS_CHECK(cmd) do if ((cmd) < 0) { elog(ERROR, "%s: %s", #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { elog(ERROR, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), strerror(errno)); exit(EXIT_FAILURE); } } while (0) +#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc < 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } } while (0) typedef struct { @@ -106,7 +99,5 @@ extern gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_l extern int fio_gzclose(gzFile file, char const* path, int tmp_fd); #endif -extern void fio_transfer(fio_shared_variable var); - #endif diff --git a/src/utils/remote.c b/src/utils/remote.c index 11844b4a4..0056ddae4 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "pg_probackup.h" #include "file.h" @@ -13,26 +12,6 @@ #define MAX_CMDLINE_OPTIONS 256 #define ERR_BUF_SIZE 4096 -static int append_option(char* buf, size_t buf_size, size_t dst, char const* src) -{ - size_t len = strlen(src); - if (dst + len + 3 >= buf_size) { - fprintf(stderr, "Too long command line\n"); - exit(EXIT_FAILURE); - } - buf[dst++] = ' '; - if (strchr(src, ' ') != NULL) { /* need quotes */ - buf[dst++] = '\''; - memcpy(&buf[dst], src, len); - dst += len; - buf[dst++] = '\''; - } else { - memcpy(&buf[dst], src, len); - dst += len; - } - return dst; -} - static int split_options(int argc, char* argv[], int max_options, char* options) { char* opt = options; @@ -80,55 +59,13 @@ static void kill_child(void) } -static void print_message(char const* buf) -{ - char const* const severity[] = {"VERBOSE", "LOG", "INFO", "NOTICE", "WARNING", "FATAL", "ERROR", ""}; - int i; - for (i = 0; strncmp(buf, severity[i], strlen(severity[i])) != 0; i++); - if (i + VERBOSE > ERROR) { - elog(LOG, "%s", buf); - } else { - elog(i+VERBOSE, "%s", buf + strlen(severity[i]) + 2); - } -} - -static void* error_reader_proc(void* arg) -{ - int* errfd = (int*)arg; - char buf[ERR_BUF_SIZE]; - int offs = 0, rc; - while ((rc = read(errfd[0], &buf[offs], sizeof(buf) - offs)) > 0) - { - char* nl; - offs += rc; - buf[offs] = '\0'; - nl = strchr(buf, '\n'); - if (nl != NULL) { - *nl = '\0'; - print_message(buf); - memmove(buf, nl+1, offs -= (nl + 1 - buf)); - } - } - if (offs != 0) - { - buf[offs] = '\0'; - print_message(buf); - } - return NULL; -} - -int remote_execute(int argc, char* argv[], bool listen) +bool launch_agent(void) { char cmd[MAX_CMDLINE_LENGTH]; - size_t dst = 0; char* ssh_argv[MAX_CMDLINE_OPTIONS]; int ssh_argc; - int i; int outfd[2]; int infd[2]; - int errfd[2]; - char* pg_probackup = argv[0]; - pthread_t error_reader_thread; ssh_argc = 0; ssh_argv[ssh_argc++] = instance_config.remote.proto; @@ -153,83 +90,38 @@ int remote_execute(int argc, char* argv[], bool listen) if (sep != NULL) { pg_probackup = sep + 1; } - dst = snprintf(cmd, sizeof(cmd), "%s/%s", instance_config.remote.path, pg_probackup); + snprintf(cmd, sizeof(cmd), "%s/%s --agent=%s", + instance_config.remote.path, pg_probackup, PROGRAM_VERSION); } else { - dst = snprintf(cmd, sizeof(cmd), "%s", pg_probackup); + snprintf(cmd, sizeof(cmd), "%s --agent=%s", pg_probackup, PROGRAM_VERSION); } - for (i = 1; i < argc; i++) { - dst = append_option(cmd, sizeof(cmd), dst, argv[i]); - } - - dst = append_option(cmd, sizeof(cmd), dst, "--agent"); - dst = append_option(cmd, sizeof(cmd), dst, PROGRAM_VERSION); - - for (i = 0; instance_options[i].type; i++) { - ConfigOption *opt = &instance_options[i]; - char *value; - char *cmd_opt; - - /* Path only options from file */ - if (opt->source != SOURCE_FILE) - continue; - - value = opt->get_value(opt); - if (value == NULL) - continue; - - cmd_opt = psprintf("--%s=%s", opt->lname, value); - - dst = append_option(cmd, sizeof(cmd), dst, cmd_opt); - pfree(value); - pfree(cmd_opt); - } - - cmd[dst] = '\0'; - SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); - SYS_CHECK(pipe(errfd)); SYS_CHECK(child_pid = fork()); if (child_pid == 0) { /* child */ SYS_CHECK(close(STDIN_FILENO)); SYS_CHECK(close(STDOUT_FILENO)); - SYS_CHECK(close(STDERR_FILENO)); SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); - SYS_CHECK(dup2(errfd[1], STDERR_FILENO)); SYS_CHECK(close(infd[0])); SYS_CHECK(close(infd[1])); SYS_CHECK(close(outfd[0])); SYS_CHECK(close(outfd[1])); - SYS_CHECK(close(errfd[0])); - SYS_CHECK(close(errfd[1])); if (execvp(ssh_argv[0], ssh_argv) < 0) - elog(ERROR, "Failed to spawn %s: %s", ssh_argv[0], strerror(errno)); - return -1; + return false; } else { SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); - SYS_CHECK(close(errfd[1])); atexit(kill_child); - pthread_create(&error_reader_thread, NULL, error_reader_proc, errfd); - - if (listen) { - int status; - fio_communicate(infd[0], outfd[1]); - - SYS_CHECK(wait(&status)); - return status; - } else { - fio_redirect(infd[0], outfd[1]); /* write to stdout */ - return 0; - } + fio_redirect(infd[0], outfd[1]); /* write to stdout */ } + return true; } From 3e773e184ccbc774f835c62160533dbe5179954e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 26 Jan 2019 11:43:30 +0300 Subject: [PATCH 0252/2107] Optimize delta backup --- src/data.c | 22 ++++++++++--------- src/utils/file.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++- src/utils/file.h | 9 +++----- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/data.c b/src/data.c index bde3b27f8..e4d1258e2 100644 --- a/src/data.c +++ b/src/data.c @@ -210,19 +210,15 @@ parse_page(Page page, XLogRecPtr *lsn) */ static int read_page_from_file(pgFile *file, BlockNumber blknum, - FILE *in, Page page, XLogRecPtr *page_lsn) + FILE *in, Page page, XLogRecPtr *page_lsn, XLogRecPtr horizon_lsn) { off_t offset = blknum * BLCKSZ; size_t read_len = 0; /* read the block */ - if (fseek(in, offset, SEEK_SET) != 0) - elog(ERROR, "File: %s, could not seek to block %u: %s", - file->path, blknum, strerror(errno)); + read_len = fio_pread(in, page, offset, horizon_lsn); - read_len = fio_fread(in, page, BLCKSZ); - - if (read_len != BLCKSZ) + if (read_len != BLCKSZ && read_len != sizeof(PageHeaderData)) { /* The block could have been truncated. It is fine. */ if (read_len == 0) @@ -232,9 +228,12 @@ read_page_from_file(pgFile *file, BlockNumber blknum, return 0; } else + { elog(WARNING, "File: %s, block %u, expected block size %u," "but read %zu, try again", file->path, blknum, BLCKSZ, read_len); + return -1; + } } /* @@ -267,7 +266,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* Verify checksum */ - if(current.checksum_version) + if (current.checksum_version && read_len == BLCKSZ) { /* * If checksum is wrong, sleep a bit and then try again @@ -330,8 +329,11 @@ prepare_page(backup_files_arg *arguments, { while(!page_is_valid && try_again) { - int result = read_page_from_file(file, blknum, - in, page, &page_lsn); + bool check_lsn = (backup_mode == BACKUP_MODE_DIFF_DELTA + && file->exists_in_prev + && !page_is_truncated); + int result = read_page_from_file(file, blknum, in, page, &page_lsn, + check_lsn ? prev_backup_start_lsn : InvalidXLogRecPtr); try_again--; if (result == 0) diff --git a/src/utils/file.c b/src/utils/file.c index 05a497af8..ea19bacf6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -14,6 +14,12 @@ static __thread void* fio_stdin_buffer; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; +typedef struct +{ + fio_header hdr; + XLogRecPtr lsn; +} fio_pread_request; + /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -374,6 +380,36 @@ int fio_truncate(int fd, off_t size) } } +int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn) +{ + if (fio_is_remote_file(f)) + { + int fd = fio_fileno(f); + fio_pread_request req; + fio_header hdr; + + req.hdr.cop = FIO_PREAD; + req.hdr.handle = fd & ~FIO_PIPE_MARKER; + req.hdr.size = sizeof(XLogRecPtr); + req.hdr.arg = offs; + req.lsn = horizon_lsn; + + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SEND); + if (hdr.size != 0) + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + return hdr.arg <= 0 ? hdr.arg : hdr.arg == BLCKSZ ? hdr.size : 1; + } + else + { + int rc = pread(fileno(f), buf, BLCKSZ, offs); + return rc <= 0 || rc == BLCKSZ ? rc : 1; + } +} + /* Set position in stdio file */ int fio_fseek(FILE* f, off_t offs) { @@ -761,6 +797,7 @@ void fio_communicate(int in, int out) char* buf = (char*)malloc(buf_size); fio_header hdr; struct stat st; + XLogRecPtr horizon_lsn; int rc; while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { @@ -811,7 +848,24 @@ void fio_communicate(int in, int out) hdr.cop = FIO_SEND; hdr.size = rc > 0 ? rc : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, rc), rc); + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + break; + case FIO_PREAD: + if ((size_t)hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); + } + horizon_lsn = *(XLogRecPtr*)buf; + rc = pread(fd[hdr.handle], buf, hdr.arg, BLCKSZ); + hdr.cop = FIO_SEND; + hdr.arg = rc; + hdr.size = (rc == BLCKSZ) + ? PageXLogRecPtrGet(((PageHeader)buf)->pd_lsn) < horizon_lsn + ? sizeof(PageHeaderData) : BLCKSZ + : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_FSTAT: hdr.size = sizeof(st); diff --git a/src/utils/file.h b/src/utils/file.h index 58f341d32..868126ca7 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -1,6 +1,7 @@ #ifndef __FILE__H__ #define __FILE__H__ +#include "storage/bufpage.h" #include #include #include @@ -20,6 +21,7 @@ typedef enum FIO_CHMOD, FIO_SEEK, FIO_TRUNCATE, + FIO_PREAD, FIO_READ, FIO_LOAD, FIO_STAT, @@ -54,18 +56,13 @@ typedef struct } fio_header; -typedef struct -{ - void* address; - size_t size; -} fio_binding; - extern void fio_redirect(int in, int out); extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern size_t fio_fread(FILE* f, void* buf, size_t size); +extern int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn); extern int fio_fprintf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); From 1ba407bca6c21e371cc6042ebaa448bbcb63af69 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 26 Jan 2019 15:47:26 +0300 Subject: [PATCH 0253/2107] Add 'agent' command instead of --agent option --- src/pg_probackup.c | 27 +++++++++++++-------------- src/utils/remote.c | 4 ++-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index edd52089f..847609ab2 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -169,8 +169,6 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, - /* remote options */ - { 's', 155, "agent", &remote_agent, SOURCE_CMD_STRICT }, { 0 } }; @@ -239,6 +237,19 @@ main(int argc, char *argv[]) backup_subcmd = SET_CONFIG_CMD; else if (strcmp(argv[1], "show-config") == 0) backup_subcmd = SHOW_CONFIG_CMD; + else if (strcmp(argv[1], "agent") == 0 && argc > 2) + { + remote_agent = argv[2]; + if (strcmp(remote_agent, PROGRAM_VERSION) != 0) + { + uint32 agent_version = parse_program_version(remote_agent); + elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, + "Agent version %s doesn't match master pg_probackup version %s", + remote_agent, PROGRAM_VERSION); + } + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; + } else if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0 || strcmp(argv[1], "help") == 0) @@ -309,18 +320,6 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); - if (remote_agent != NULL) - { - if (strcmp(remote_agent, PROGRAM_VERSION) != 0) - { - uint32 agent_version = parse_program_version(remote_agent); - elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, - "Agent version %s doesn't match master pg_probackup version %s", - remote_agent, PROGRAM_VERSION); - } - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - } pgut_init(); if (help_opt) diff --git a/src/utils/remote.c b/src/utils/remote.c index 0056ddae4..c32f6ef45 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -90,10 +90,10 @@ bool launch_agent(void) if (sep != NULL) { pg_probackup = sep + 1; } - snprintf(cmd, sizeof(cmd), "%s/%s --agent=%s", + snprintf(cmd, sizeof(cmd), "%s/%s agent %s", instance_config.remote.path, pg_probackup, PROGRAM_VERSION); } else { - snprintf(cmd, sizeof(cmd), "%s --agent=%s", pg_probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } SYS_CHECK(pipe(infd)); From 56472e0f70cb23ef0cd77d3f1b51198db58ecde4 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 26 Jan 2019 16:23:43 +0300 Subject: [PATCH 0254/2107] Fix location for listing backup files --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 1891da3b1..95ac3e081 100644 --- a/src/backup.c +++ b/src/backup.c @@ -634,7 +634,7 @@ do_backup_instance(void) get_remote_pgdata_filelist(backup_files_list); else dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, FIO_DB_HOST); + true, true, false, FIO_BACKUP_HOST); /* * Sort pathname ascending. It is necessary to create intermediate From 7ef0304e767a4318bdb58c8d59cac61c830b5d88 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 26 Jan 2019 17:14:25 +0300 Subject: [PATCH 0255/2107] Fix remote pull backup issues --- src/backup.c | 4 ++-- src/data.c | 14 +++++++------- src/dir.c | 20 ++++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/backup.c b/src/backup.c index 95ac3e081..76add3de6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -634,7 +634,7 @@ do_backup_instance(void) get_remote_pgdata_filelist(backup_files_list); else dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, FIO_BACKUP_HOST); + true, true, false, FIO_DB_HOST); /* * Sort pathname ascending. It is necessary to create intermediate @@ -2231,7 +2231,7 @@ backup_files(void *arg) i + 1, n_backup_files_list, file->path); /* stat file to check its current state */ - ret = stat(file->path, &buf); + ret = fio_stat(file->path, &buf, false, FIO_DB_HOST); if (ret == -1) { if (errno == ENOENT) diff --git a/src/data.c b/src/data.c index e4d1258e2..3112dca87 100644 --- a/src/data.c +++ b/src/data.c @@ -575,7 +575,7 @@ backup_data_file(backup_files_arg* arguments, if (file->size % BLCKSZ != 0) { - fclose(in); + fio_fclose(in); elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); } @@ -1110,7 +1110,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, to_path_p = to_path; /* open file for read */ - in = fopen(from_path, PG_BINARY_R); + in = fio_fopen(from_path, PG_BINARY_R, FIO_DB_HOST); if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, strerror(errno)); @@ -1153,9 +1153,9 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { size_t read_len = 0; - read_len = fread(buf, 1, sizeof(buf), in); + read_len = fio_fread(in, buf, sizeof(buf)); - if (ferror(in)) + if (read_len < 0) { errno_temp = errno; fio_unlink(to_path_temp, FIO_BACKUP_HOST); @@ -1190,7 +1190,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, } } - if (feof(in) || read_len == 0) + if (read_len == 0) break; } @@ -1218,7 +1218,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, } } - if (fclose(in)) + if (fio_fclose(in)) { errno_temp = errno; fio_unlink(to_path_temp, FIO_BACKUP_HOST); @@ -1728,7 +1728,7 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) } /* Get checksum of original file */ - crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_LOCAL_HOST); + crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST); return EQ_CRC32C(crc1, crc2); } diff --git a/src/dir.c b/src/dir.c index 4e4cffd88..118ee8771 100644 --- a/src/dir.c +++ b/src/dir.c @@ -403,14 +403,14 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ if (root && instance_config.pgdata && - strcmp(root, instance_config.pgdata) == 0 && fileExists(path, location)) + strcmp(root, instance_config.pgdata) == 0 && fileExists(path, FIO_BACKUP_HOST)) { FILE *black_list_file = NULL; char buf[MAXPGPATH * 2]; char black_item[MAXPGPATH * 2]; black_list = parray_new(); - black_list_file = fio_open_stream(path, location); + black_list_file = fio_open_stream(path, FIO_BACKUP_HOST); if (black_list_file == NULL) elog(ERROR, "cannot open black_list: %s", strerror(errno)); @@ -433,7 +433,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false, FIO_LOCAL_HOST); + file = pgFileNew(root, false, location); if (file == NULL) return; @@ -773,12 +773,12 @@ list_data_directories(parray *files, const char *path, bool is_root, bool has_child_dirs = false; /* open directory and list contents */ - dir = opendir(path); + dir = fio_opendir(path, FIO_DB_HOST); if (dir == NULL) elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); errno = 0; - while ((dent = readdir(dir))) + while ((dent = fio_readdir(dir))) { char child[MAXPGPATH]; bool skip = false; @@ -791,7 +791,7 @@ list_data_directories(parray *files, const char *path, bool is_root, join_path_components(child, path, dent->d_name); - if (lstat(child, &st) == -1) + if (fio_stat(child, &st, true, FIO_DB_HOST) == -1) elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) @@ -828,7 +828,7 @@ list_data_directories(parray *files, const char *path, bool is_root, } prev_errno = errno; - closedir(dir); + fio_closedir(dir); if (prev_errno && prev_errno != ENOENT) elog(ERROR, "cannot read directory \"%s\": %s", @@ -1116,13 +1116,13 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path, FIO_LOCAL_HOST)) + if (!fileExists(map_path, FIO_DB_HOST)) { elog(LOG, "there is no file tablespace_map"); return; } - fp = fopen(map_path, "rt"); + fp = fio_open_stream(map_path, FIO_DB_HOST); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); @@ -1147,7 +1147,7 @@ read_tablespace_map(parray *files, const char *backup_dir) parray_append(files, file); } - fclose(fp); + fio_close_stream(fp); } /* From f7e2a7a8cb75170fbdc1583b782a32fcdea19302 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 26 Jan 2019 23:56:53 +0300 Subject: [PATCH 0256/2107] Some minor fixes --- src/data.c | 2 +- src/dir.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 3112dca87..9fc9416e1 100644 --- a/src/data.c +++ b/src/data.c @@ -1151,7 +1151,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, /* copy content */ for (;;) { - size_t read_len = 0; + ssize_t read_len = 0; read_len = fio_fread(in, buf, sizeof(buf)); diff --git a/src/dir.c b/src/dir.c index 118ee8771..3cf3490c9 100644 --- a/src/dir.c +++ b/src/dir.c @@ -791,7 +791,7 @@ list_data_directories(parray *files, const char *path, bool is_root, join_path_components(child, path, dent->d_name); - if (fio_stat(child, &st, true, FIO_DB_HOST) == -1) + if (fio_stat(child, &st, false, FIO_DB_HOST) == -1) elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) From 3bedd3b639fca55319dfc7c85c4bf175f6d7e4ea Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 27 Jan 2019 01:11:52 +0300 Subject: [PATCH 0257/2107] Remove walmethods --- Makefile | 16 +- src/backup.c | 24 ++- src/data.c | 16 +- src/utils/file.c | 10 +- src/utils/file.h | 2 +- src/walmethods.c | 391 ----------------------------------------------- src/walmethods.h | 92 ----------- 7 files changed, 39 insertions(+), 512 deletions(-) delete mode 100644 src/walmethods.c delete mode 100644 src/walmethods.h diff --git a/Makefile b/Makefile index bbc8b2e00..98f0e1a02 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,13 @@ OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \ # borrowed files OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ - src/xlogreader.o src/walmethods.o + src/xlogreader.o EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c -INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h src/walmethods.h +INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -39,6 +39,12 @@ else srchome=$(top_srcdir) endif +ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +INCLUDES += src/walmethods.h +endif + PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} @@ -66,6 +72,12 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) +src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ +src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ +endif ifeq ($(PORTNAME), aix) CC=xlc_r diff --git a/src/backup.c b/src/backup.c index 76add3de6..a74d8212c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -66,11 +66,10 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; - parray *files_list; } StreamThreadArg; static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1, NULL}; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; static int is_ptrack_enable = false; bool is_ptrack_support = false; @@ -482,7 +481,6 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; - parray *xlog_files_list = NULL; parray *backup_list = NULL; pgFile *pg_control = NULL; @@ -620,7 +618,6 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; - stream_thread_arg.files_list = xlog_files_list; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } @@ -804,15 +801,14 @@ do_backup_instance(void) /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { - if (xlog_files_list == NULL) - { - char pg_xlog_path[MAXPGPATH]; + parray *xlog_files_list; + char pg_xlog_path[MAXPGPATH]; + + /* Scan backup PG_XLOG_DIR */ + xlog_files_list = parray_new(); + join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, FIO_BACKUP_HOST); - /* Scan backup PG_XLOG_DIR */ - xlog_files_list = parray_new(); - join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, FIO_BACKUP_HOST); - } for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); @@ -2231,7 +2227,7 @@ backup_files(void *arg) i + 1, n_backup_files_list, file->path); /* stat file to check its current state */ - ret = fio_stat(file->path, &buf, false, FIO_DB_HOST); + ret = fio_stat(file->path, &buf, true, FIO_DB_HOST); if (ret == -1) { if (errno == ENOENT) @@ -2758,7 +2754,7 @@ StreamLog(void *arg) ctl.sysidentifier = NULL; #if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true, stream_arg->files_list); + ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; #else diff --git a/src/data.c b/src/data.c index 9fc9416e1..e4b2ecb35 100644 --- a/src/data.c +++ b/src/data.c @@ -213,7 +213,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, FILE *in, Page page, XLogRecPtr *page_lsn, XLogRecPtr horizon_lsn) { off_t offset = blknum * BLCKSZ; - size_t read_len = 0; + ssize_t read_len = 0; /* read the block */ read_len = fio_pread(in, page, offset, horizon_lsn); @@ -1060,22 +1060,22 @@ get_gz_error(gzFile gzf, int errnum) * Copy file attributes */ static void -copy_meta(const char *from_path, const char *to_path, bool unlink_on_error, fio_location location) +copy_meta(const char *from_path, fio_location from_location, const char *to_path, fio_location to_location, bool unlink_on_error) { struct stat st; - if (stat(from_path, &st) == -1) + if (fio_stat(from_path, &st, true, from_location) == -1) { if (unlink_on_error) - fio_unlink(to_path, location); + fio_unlink(to_path, to_location); elog(ERROR, "Cannot stat file \"%s\": %s", from_path, strerror(errno)); } - if (fio_chmod(to_path, st.st_mode, location) == -1) + if (fio_chmod(to_path, st.st_mode, to_location) == -1) { if (unlink_on_error) - fio_unlink(to_path, location); + fio_unlink(to_path, to_location); elog(ERROR, "Cannot change mode of file \"%s\": %s", to_path, strerror(errno)); } @@ -1227,7 +1227,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, } /* update file permission. */ - copy_meta(from_path, to_path_temp, true, FIO_BACKUP_HOST); + copy_meta(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) { @@ -1394,7 +1394,7 @@ get_wal_file(const char *from_path, const char *to_path) } /* update file permission. */ - copy_meta(from_path_p, to_path_temp, true, FIO_DB_HOST); + copy_meta(from_path_p, FIO_BACKUP_HOST, to_path_temp, FIO_DB_HOST, true); if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) { diff --git a/src/utils/file.c b/src/utils/file.c index ea19bacf6..ea99224ee 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -471,11 +471,13 @@ ssize_t fio_write(int fd, void const* buf, size_t size) } /* Read data from stdio file */ -size_t fio_fread(FILE* f, void* buf, size_t size) +ssize_t fio_fread(FILE* f, void* buf, size_t size) { - return fio_is_remote_file(f) - ? fio_read(fio_fileno(f), buf, size) - : fread(buf, 1, size, f); + size_t rc; + if (fio_is_remote_file(f)) + return fio_read(fio_fileno(f), buf, size); + rc = fread(buf, 1, size, f); + return rc == 0 && !feof(f) ? -1 : rc; } /* Read data from file */ diff --git a/src/utils/file.h b/src/utils/file.h index 868126ca7..1acb0aec4 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -61,7 +61,7 @@ extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern size_t fio_fread(FILE* f, void* buf, size_t size); +extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn); extern int fio_fprintf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); extern int fio_fflush(FILE* f); diff --git a/src/walmethods.c b/src/walmethods.c deleted file mode 100644 index ceb24f09a..000000000 --- a/src/walmethods.c +++ /dev/null @@ -1,391 +0,0 @@ -/*------------------------------------------------------------------------- - * - * walmethods.c - implementations of different ways to write received wal - * - * NOTE! The caller must ensure that only one method is instantiated in - * any given program, and that it's only instantiated once! - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * - * IDENTIFICATION - * src/bin/pg_basebackup/walmethods.c - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" - -#include -#include -#include -#ifdef HAVE_LIBZ -#include -#endif - -#include "pgtar.h" -#include "common/file_utils.h" - -#include "receivelog.h" -#include "streamutil.h" -#include "pg_probackup.h" - -/* Size of zlib buffer for .tar.gz */ -#define ZLIB_OUT_SIZE 4096 - -/*------------------------------------------------------------------------- - * WalDirectoryMethod - write wal to a directory looking like pg_wal - *------------------------------------------------------------------------- - */ - -/* - * Global static data for this method - */ -typedef struct DirectoryMethodData -{ - char *basedir; - int compression; - bool sync; - parray *files_list; -} DirectoryMethodData; -static DirectoryMethodData *dir_data = NULL; - -/* - * Local file handle - */ -typedef struct DirectoryMethodFile -{ - int fd; - off_t currpos; - char *pathname; - char *fullpath; - char *temp_suffix; -#ifdef HAVE_LIBZ - gzFile gzfp; - int gz_tmp; -#endif -} DirectoryMethodFile; - -static const char * -dir_getlasterror(void) -{ - /* Directory method always sets errno, so just use strerror */ - return strerror(errno); -} - -static Walfile -dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) -{ - char tmppath[MAXPGPATH]; - int fd = -1; - DirectoryMethodFile *f; -#ifdef HAVE_LIBZ - gzFile gzfp = NULL; - int gz_tmp = -1; -#endif - - snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", - dir_data->basedir, pathname, - dir_data->compression > 0 ? ".gz" : "", - temp_suffix ? temp_suffix : ""); - - /* - * Open a file for non-compressed as well as compressed files. Tracking - * the file descriptor is important for dir_sync() method as gzflush() - * does not do any system calls to fsync() to make changes permanent on - * disk. - */ -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - { - gzfp = fio_gzopen(tmppath, "wb", &gz_tmp, FIO_BACKUP_HOST); - if (gzfp == NULL) - { - return NULL; - } - - if (gzsetparams(gzfp, dir_data->compression, - Z_DEFAULT_STRATEGY) != Z_OK) - { - gzclose(gzfp); - return NULL; - } - } - else -#endif - { - fd = fio_open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, FIO_BACKUP_HOST); - if (fd < 0) - return NULL; - } - /* Do pre-padding on non-compressed files */ - if (pad_to_size && dir_data->compression == 0) - { - char zerobuf[XLOG_BLCKSZ]; - int bytes; - - memset(zerobuf, 0, XLOG_BLCKSZ); - for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ) - { - errno = 0; - if (fio_write(fd, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - int save_errno = errno; - - fio_close(fd); - - /* - * If write didn't set errno, assume problem is no disk space. - */ - errno = save_errno ? save_errno : ENOSPC; - return NULL; - } - } - - if (fio_seek(fd, 0) != 0) - { - int save_errno = errno; - - fio_close(fd); - errno = save_errno; - return NULL; - } - } - - /* - * fsync WAL file and containing directory, to ensure the file is - * persistently created and zeroed (if padded). That's particularly - * important when using synchronous mode, where the file is modified and - * fsynced in-place, without a directory fsync. - */ - if (!remote_agent && dir_data->sync) - { - if (fsync_fname(tmppath, false, progname) != 0 || - fsync_parent_path(tmppath, progname) != 0) - { -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - gzclose(gzfp); - else -#endif - close(fd); - return NULL; - } - } - - f = pg_malloc0(sizeof(DirectoryMethodFile)); -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - { - f->gzfp = gzfp; - f->gz_tmp = gz_tmp; - } -#endif - f->fd = fd; - f->currpos = 0; - f->pathname = pg_strdup(pathname); - f->fullpath = pg_strdup(tmppath); - if (temp_suffix) - f->temp_suffix = pg_strdup(temp_suffix); - - return f; -} - -static ssize_t -dir_write(Walfile f, const void *buf, size_t count) -{ - ssize_t r; - DirectoryMethodFile *df = (DirectoryMethodFile *) f; - - Assert(f != NULL); - -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - r = (ssize_t) gzwrite(df->gzfp, buf, count); - else -#endif - r = fio_write(df->fd, buf, count); - if (r > 0) - df->currpos += r; - return r; -} - -static off_t -dir_get_current_pos(Walfile f) -{ - Assert(f != NULL); - - /* Use a cached value to prevent lots of reseeks */ - return ((DirectoryMethodFile *) f)->currpos; -} - -static int -dir_close(Walfile f, WalCloseMethod method) -{ - int r; - DirectoryMethodFile *df = (DirectoryMethodFile *) f; - char tmppath[MAXPGPATH]; - char tmppath2[MAXPGPATH]; - - Assert(f != NULL); - -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - r = fio_gzclose(df->gzfp, df->fullpath, df->gz_tmp); - else -#endif - r = fio_close(df->fd); - - if (r == 0) - { - char const* file_path = NULL; - - /* Build path to the current version of the file */ - if (method == CLOSE_NORMAL && df->temp_suffix) - { - /* - * If we have a temp prefix, normal operation is to rename the - * file. - */ - snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", - dir_data->basedir, df->pathname, - dir_data->compression > 0 ? ".gz" : "", - df->temp_suffix); - snprintf(tmppath2, sizeof(tmppath2), "%s/%s%s", - dir_data->basedir, df->pathname, - dir_data->compression > 0 ? ".gz" : ""); - r = fio_rename(tmppath, tmppath2, FIO_BACKUP_HOST); - file_path = tmppath2; - } - else if (method == CLOSE_UNLINK) - { - /* Unlink the file once it's closed */ - snprintf(tmppath, sizeof(tmppath), "%s/%s%s%s", - dir_data->basedir, df->pathname, - dir_data->compression > 0 ? ".gz" : "", - df->temp_suffix ? df->temp_suffix : ""); - r = fio_unlink(tmppath, FIO_BACKUP_HOST); - } - else - { - /* - * Else either CLOSE_NORMAL and no temp suffix, or - * CLOSE_NO_RENAME. In this case, fsync the file and containing - * directory if sync mode is requested. - */ - file_path = df->fullpath; - if (dir_data->sync && !remote_agent) - { - r = fsync_fname(df->fullpath, false, progname); - if (r == 0) - r = fsync_parent_path(df->fullpath, progname); - } - } - if (file_path && dir_data->files_list) - { - pgFile* file = pgFileNew(file_path, false, FIO_BACKUP_HOST); - Assert(file != NULL); - parray_append(dir_data->files_list, file); - } - } - - pg_free(df->pathname); - pg_free(df->fullpath); - if (df->temp_suffix) - pg_free(df->temp_suffix); - pg_free(df); - - return r; -} - -static int -dir_sync(Walfile f) -{ - Assert(f != NULL); - - if (!dir_data->sync) - return 0; - -#ifdef HAVE_LIBZ - if (dir_data->compression > 0) - { - if (gzflush(((DirectoryMethodFile *) f)->gzfp, Z_SYNC_FLUSH) != Z_OK) - return -1; - } -#endif - - return fio_flush(((DirectoryMethodFile *) f)->fd); -} - -static ssize_t -dir_get_file_size(const char *pathname) -{ - struct stat statbuf; - char tmppath[MAXPGPATH]; - - snprintf(tmppath, sizeof(tmppath), "%s/%s", - dir_data->basedir, pathname); - - if (fio_stat(tmppath, &statbuf, true, FIO_BACKUP_HOST) != 0) - return -1; - - return statbuf.st_size; -} - -static bool -dir_existsfile(const char *pathname) -{ - char tmppath[MAXPGPATH]; - - snprintf(tmppath, sizeof(tmppath), "%s/%s", - dir_data->basedir, pathname); - - return fio_access(tmppath, F_OK, FIO_BACKUP_HOST) == 0; -} - -static bool -dir_finish(void) -{ - if (dir_data->sync && !remote_agent) - { - /* - * Files are fsynced when they are closed, but we need to fsync the - * directory entry here as well. - */ - if (fsync_fname(dir_data->basedir, true, progname) != 0) - return false; - } - return true; -} - - -WalWriteMethod * -CreateWalDirectoryMethod(const char *basedir, int compression, bool sync, parray* files_list) -{ - WalWriteMethod *method; - - method = pg_malloc0(sizeof(WalWriteMethod)); - method->open_for_write = dir_open_for_write; - method->write = dir_write; - method->get_current_pos = dir_get_current_pos; - method->get_file_size = dir_get_file_size; - method->close = dir_close; - method->sync = dir_sync; - method->existsfile = dir_existsfile; - method->finish = dir_finish; - method->getlasterror = dir_getlasterror; - - dir_data = pg_malloc0(sizeof(DirectoryMethodData)); - dir_data->compression = compression; - dir_data->basedir = pg_strdup(basedir); - dir_data->sync = sync; - dir_data->files_list = files_list; - - return method; -} - -void -FreeWalDirectoryMethod(void) -{ - pg_free(dir_data->basedir); - pg_free(dir_data); -} - diff --git a/src/walmethods.h b/src/walmethods.h deleted file mode 100644 index b4dd40c53..000000000 --- a/src/walmethods.h +++ /dev/null @@ -1,92 +0,0 @@ -/*------------------------------------------------------------------------- - * - * walmethods.h - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * - * IDENTIFICATION - * src/bin/pg_basebackup/walmethods.h - *------------------------------------------------------------------------- - */ - -#include "utils/parray.h" -typedef void *Walfile; - -typedef enum -{ - CLOSE_NORMAL, - CLOSE_UNLINK, - CLOSE_NO_RENAME -} WalCloseMethod; - -/* - * A WalWriteMethod structure represents the different methods used - * to write the streaming WAL as it's received. - * - * All methods that have a failure return indicator will set state - * allowing the getlasterror() method to return a suitable message. - * Commonly, errno is this state (or part of it); so callers must take - * care not to clobber errno between a failed method call and use of - * getlasterror() to retrieve the message. - */ -typedef struct WalWriteMethod WalWriteMethod; -struct WalWriteMethod -{ - /* - * Open a target file. Returns Walfile, or NULL if open failed. If a temp - * suffix is specified, a file with that name will be opened, and then - * automatically renamed in close(). If pad_to_size is specified, the file - * will be padded with NUL up to that size, if supported by the Walmethod. - */ - Walfile (*open_for_write) (const char *pathname, const char *temp_suffix, size_t pad_to_size); - - /* - * Close an open Walfile, using one or more methods for handling automatic - * unlinking etc. Returns 0 on success, other values for error. - */ - int (*close) (Walfile f, WalCloseMethod method); - - /* Check if a file exist */ - bool (*existsfile) (const char *pathname); - - /* Return the size of a file, or -1 on failure. */ - ssize_t (*get_file_size) (const char *pathname); - - /* - * Write count number of bytes to the file, and return the number of bytes - * actually written or -1 for error. - */ - ssize_t (*write) (Walfile f, const void *buf, size_t count); - - /* Return the current position in a file or -1 on error */ - off_t (*get_current_pos) (Walfile f); - - /* - * fsync the contents of the specified file. Returns 0 on success. - */ - int (*sync) (Walfile f); - - /* - * Clean up the Walmethod, closing any shared resources. For methods like - * tar, this includes writing updated headers. Returns true if the - * close/write/sync of shared resources succeeded, otherwise returns false - * (but the resources are still closed). - */ - bool (*finish) (void); - - /* Return a text for the last error in this Walfile */ - const char *(*getlasterror) (void); -}; - -/* - * Available WAL methods: - * - WalDirectoryMethod - write WAL to regular files in a standard pg_wal - * - TarDirectoryMethod - write WAL to a tarfile corresponding to pg_wal - * (only implements the methods required for pg_basebackup, - * not all those required for pg_receivewal) - */ -WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, - int compression, bool sync, parray* file_list); - -/* Cleanup routines for previously-created methods */ -void FreeWalDirectoryMethod(void); From 2e9ea2b2b5f776edbf9e50ec0426f3785217621e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 27 Jan 2019 09:43:22 +0300 Subject: [PATCH 0258/2107] Revert commit e02b7f7c9 --- src/archive.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/archive.c b/src/archive.c index e715f6fbd..add41f640 100644 --- a/src/archive.c +++ b/src/archive.c @@ -29,7 +29,6 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) char current_dir[MAXPGPATH]; uint64 system_id; bool is_compress = false; - char* pgdata = instance_config.pgdata; if (wal_file_name == NULL && wal_file_path == NULL) elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); @@ -40,14 +39,11 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); - if (pgdata == NULL) - { - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - pgdata = current_dir; - } + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + /* verify that archive-push --instance parameter is valid */ - system_id = get_system_identifier(pgdata); + system_id = get_system_identifier(current_dir); if (instance_config.pgdata == NULL) elog(ERROR, "cannot read pg_probackup.conf for this instance"); @@ -61,7 +57,7 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) /* Create 'archlog_path' directory. Do nothing if it already exists. */ fio_mkdir(arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); - join_path_components(absolute_wal_file_path, pgdata, wal_file_path); + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); @@ -91,7 +87,6 @@ do_archive_get(char *wal_file_path, char *wal_file_name) char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; char current_dir[MAXPGPATH]; - char* pgdata = instance_config.pgdata; if (wal_file_name == NULL && wal_file_path == NULL) elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); @@ -102,13 +97,10 @@ do_archive_get(char *wal_file_path, char *wal_file_name) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); - if (pgdata == NULL) - { - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - pgdata = current_dir; - } - join_path_components(absolute_wal_file_path, pgdata, wal_file_path); + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); join_path_components(backup_wal_file_path, arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-get from %s to %s", From 7a90741d75111ee280d1c7c715881fa57fd5e369 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 27 Jan 2019 18:35:57 +0300 Subject: [PATCH 0259/2107] tests: fix merge --- tests/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 4a0cb31cf..056ec035e 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1254,7 +1254,7 @@ def test_continue_failed_merge_2(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_unlink') gdb.run_until_break() if gdb.continue_execution_until_break(20) != 'breakpoint-hit': From 599cce7c2221674853ed1eedfb883bf7c3f1d7ee Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 27 Jan 2019 22:20:22 +0300 Subject: [PATCH 0260/2107] no_argument for short boolean options --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index bf477e84c..60ab6bf11 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -108,7 +108,7 @@ option_has_arg(char type) { case 'b': case 'B': - return optional_argument; + return no_argument;//optional_argument; default: return required_argument; } From 5d1d8bdad78ea7e83f9dab9c4b217b1cdb3d53cb Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 28 Jan 2019 16:05:36 +0300 Subject: [PATCH 0261/2107] [refer #1745] Fix ARCHIVE PUSH command --- src/pg_probackup.c | 5 +++++ src/utils/file.c | 8 ++++++-- src/utils/file.h | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 847609ab2..dcf8278f5 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -356,6 +356,11 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: --instance"); } + MyLocation = IsSshProtocol() + ? backup_subcmd == ARCHIVE_PUSH_CMD + ? FIO_DB_HOST : FIO_BACKUP_HOST + : FIO_LOCAL_HOST; + /* * If --instance option was passed, construct paths for backup data and * xlog files of this backup instance. diff --git a/src/utils/file.c b/src/utils/file.c index ea99224ee..d4aa261d3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -20,6 +20,9 @@ typedef struct XLogRecPtr lsn; } fio_pread_request; +fio_location MyLocation; + + /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -45,8 +48,9 @@ static bool fio_is_remote_fd(int fd) /* Check if specified location is local for current node */ static bool fio_is_remote(fio_location location) { - bool is_remote = location == FIO_REMOTE_HOST - || (location == FIO_DB_HOST && IsSshProtocol()); + bool is_remote = MyLocation != FIO_LOCAL_HOST + && location != FIO_LOCAL_HOST + && location != MyLocation; if (is_remote && !fio_stdin && !launch_agent()) elog(ERROR, "Failed to establish SSH connection: %s", strerror(errno)); return is_remote; diff --git a/src/utils/file.h b/src/utils/file.h index 1acb0aec4..498b63765 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -55,6 +55,7 @@ typedef struct unsigned arg; } fio_header; +extern fio_location MyLocation; extern void fio_redirect(int in, int out); extern void fio_communicate(int in, int out); From 06528021f8cfafde77c320fc7040c85760971d71 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 28 Jan 2019 18:22:35 +0300 Subject: [PATCH 0262/2107] Disable SSH password authentication for multiple SSH sessions --- src/utils/remote.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/remote.c b/src/utils/remote.c index c32f6ef45..1cce69114 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -80,6 +80,11 @@ bool launch_agent(void) if (instance_config.remote.ssh_options != NULL) { ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options)); } + if (num_threads + current.stream > 1) + { + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "PasswordAuthentication=no"; + } ssh_argv[ssh_argc++] = instance_config.remote.host; ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; From 6e46ff10423b3a110d68780ef60787aaff599169 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 30 Jan 2019 19:57:02 +0300 Subject: [PATCH 0263/2107] Disable compression in SSH --- src/utils/remote.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 1cce69114..a3104e78b 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -70,21 +70,28 @@ bool launch_agent(void) ssh_argc = 0; ssh_argv[ssh_argc++] = instance_config.remote.proto; if (instance_config.remote.port != NULL) { - ssh_argv[ssh_argc++] = (char*)"-p"; + ssh_argv[ssh_argc++] = "-p"; ssh_argv[ssh_argc++] = instance_config.remote.port; } if (instance_config.remote.ssh_config != NULL) { - ssh_argv[ssh_argc++] = (char*)"-F"; + ssh_argv[ssh_argc++] = "-F"; ssh_argv[ssh_argc++] = instance_config.remote.ssh_config; } if (instance_config.remote.ssh_options != NULL) { ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options)); } - if (num_threads + current.stream > 1) + if (num_threads > 1) { ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "PasswordAuthentication=no"; } + + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "Compression=no"; + + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "LogLevel=error"; + ssh_argv[ssh_argc++] = instance_config.remote.host; ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; From abf646d0d91f05be27db13338b188e3858aafa98 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 12:03:02 +0300 Subject: [PATCH 0264/2107] tests: module "locking" added --- tests/__init__.py | 3 ++- tests/locking.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/locking.py diff --git a/tests/__init__.py b/tests/__init__.py index c3e149799..ce149f6d5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,7 +8,7 @@ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp, snapfs, logging + time_stamp, snapfs, logging, locking def load_tests(loader, tests, pattern): @@ -27,6 +27,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(exclude)) suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(locking)) suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(merge)) suite.addTests(loader.loadTestsFromModule(option_test)) diff --git a/tests/locking.py b/tests/locking.py new file mode 100644 index 000000000..859a83de6 --- /dev/null +++ b/tests/locking.py @@ -0,0 +1,32 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +module_name = 'locking' + + +class LockingTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_locking_simple(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From 282cf11b30983293478594b8eec7250fd6167c11 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 16:31:11 +0300 Subject: [PATCH 0265/2107] tests: test_locking_simple() added --- tests/locking.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/locking.py b/tests/locking.py index 859a83de6..96810f904 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -17,9 +17,8 @@ def test_locking_simple(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'} - ) + pg_options={'wal_level': 'replica'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -28,5 +27,22 @@ def test_locking_simple(self): self.backup_node(backup_dir, 'node', node) + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + self.AssertTrue(False, 'Failed to hit breakpoint') + + gdb._execute('signal SIGKILL') + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + # Clean after yourself self.del_test_dir(module_name, fname) \ No newline at end of file From 50f5e5013a90f08cb3265cbe45208f40582f3d41 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 18:44:26 +0300 Subject: [PATCH 0266/2107] tests: minor fix in test_locking_simple() --- tests/locking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/locking.py b/tests/locking.py index 96810f904..d92510a5c 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -38,6 +38,8 @@ def test_locking_simple(self): gdb._execute('signal SIGKILL') + self.validate_pb(backup_dir) + self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) From 2c08cb2c2f4be17fb317f87546a94e5f51e1fb9a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 18:57:28 +0300 Subject: [PATCH 0267/2107] tests: test_locking_simple_2() added --- tests/locking.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/locking.py b/tests/locking.py index d92510a5c..ae5431a88 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -11,7 +11,7 @@ class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_locking_simple(self): + def test_locking_simple_1(self): """""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -36,8 +36,6 @@ def test_locking_simple(self): if gdb.continue_execution_until_break(20) != 'breakpoint-hit': self.AssertTrue(False, 'Failed to hit breakpoint') - gdb._execute('signal SIGKILL') - self.validate_pb(backup_dir) self.assertEqual( @@ -46,5 +44,35 @@ def test_locking_simple(self): self.assertEqual( 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_locking_simple_2(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + self.AssertTrue(False, 'Failed to hit breakpoint') + + self.validate_pb(backup_dir) + # Clean after yourself self.del_test_dir(module_name, fname) \ No newline at end of file From 34cb9169c4049377cefe6543d43496449229acd3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 19:47:44 +0300 Subject: [PATCH 0268/2107] tests: module remote added --- tests/__init__.py | 3 ++- tests/remote.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/remote.py diff --git a/tests/__init__.py b/tests/__init__.py index ce149f6d5..181a579d7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,7 +8,7 @@ ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ false_positive, replica, compression, page, ptrack, archive, \ exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp, snapfs, logging, locking + time_stamp, snapfs, logging, locking, remote def load_tests(loader, tests, pattern): @@ -44,6 +44,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) + suite.addTests(loader.loadTestsFromModule(remote)) suite.addTests(loader.loadTestsFromModule(replica)) suite.addTests(loader.loadTestsFromModule(restore_test)) suite.addTests(loader.loadTestsFromModule(retention_test)) diff --git a/tests/remote.py b/tests/remote.py new file mode 100644 index 000000000..245c91adf --- /dev/null +++ b/tests/remote.py @@ -0,0 +1,32 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +module_name = 'remote' + + +class RemoteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_remote_1(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 29d621040870521406abbe1ff88e9577cefadb00 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Jan 2019 20:27:41 +0300 Subject: [PATCH 0269/2107] tests: test_remote_1() added --- tests/remote.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/remote.py b/tests/remote.py index 245c91adf..8230d40d7 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -17,16 +17,31 @@ def test_remote_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) + pg_options={'wal_level': 'replica'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) +# self.set_archiving(backup_dir, 'node', node, remote=True) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--remote-proto=ssh', + '--remote-host=localhost']) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From df4ede98127e9a0f84a041e402dba7469e3a1647 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 31 Jan 2019 23:13:16 +0300 Subject: [PATCH 0270/2107] [refer #PGPRO-1745] Do not send SIGTERM to agent --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index a3104e78b..3b6fc8e24 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -130,7 +130,7 @@ bool launch_agent(void) } else { SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); - atexit(kill_child); + /*atexit(kill_child);*/ fio_redirect(infd[0], outfd[1]); /* write to stdout */ } From f5a7ba34d244ed8d5d0893bf367bef903fc6acfb Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Feb 2019 16:17:12 +0300 Subject: [PATCH 0271/2107] [refer #PGPRO-1745] Fix PREAD agent command --- src/utils/file.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index d4aa261d3..5becb0cb5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -177,7 +177,7 @@ struct dirent* fio_readdir(DIR *dir) if (fio_is_remote_file((FILE*)dir)) { fio_header hdr; - static struct dirent entry; + static __thread struct dirent entry; hdr.cop = FIO_READDIR; hdr.handle = (size_t)dir - 1; @@ -838,6 +838,7 @@ void fio_communicate(int in, int out) break; case FIO_OPEN: SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); + fprintf(stderr, "Open file %s -> %d\n", buf, hdr.handle); break; case FIO_CLOSE: SYS_CHECK(close(fd[hdr.handle])); @@ -857,12 +858,9 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_PREAD: - if ((size_t)hdr.arg > buf_size) { - buf_size = hdr.arg; - buf = (char*)realloc(buf, buf_size); - } horizon_lsn = *(XLogRecPtr*)buf; - rc = pread(fd[hdr.handle], buf, hdr.arg, BLCKSZ); + rc = pread(fd[hdr.handle], buf, BLCKSZ, hdr.arg); + fprintf(stderr, "Read %d bytes from file %d offset %d\n", rc, hdr.handle, hdr.arg); hdr.cop = FIO_SEND; hdr.arg = rc; hdr.size = (rc == BLCKSZ) From 62c2028a0fc03aac4ef0773ba0aec6e2dc0ad31d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Feb 2019 19:32:48 +0300 Subject: [PATCH 0272/2107] tests: support remote tests --- tests/Readme.md | 3 ++ tests/helpers/ptrack_helpers.py | 55 +++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 7a39e2791..315710481 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -19,6 +19,9 @@ Enable compatibility tests: Specify path to pg_probackup binary file. By default tests use /pg_probackup/ export PGPROBACKUPBIN= +Remote backup depends on key authentithication to local machine via ssh as current user. + PGPROBACKUP_SSH_REMOTE=ON + Usage: pip install testgres pip install psycopg2 diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 237961300..b43d86b40 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -264,6 +264,34 @@ def __init__(self, *args, **kwargs): if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') + self.remote = False + self.remote_host = None + self.remote_port = None + self.remote_user = None + + if 'PGPROBACKUP_SSH_REMOTE' in self.test_env: + self.remote = True + +# if 'PGPROBACKUP_SSH_HOST' in self.test_env: +# self.remote_host = self.test_env['PGPROBACKUP_SSH_HOST'] +# else +# print('PGPROBACKUP_SSH_HOST is not set') +# exit(1) +# +# if 'PGPROBACKUP_SSH_PORT' in self.test_env: +# self.remote_port = self.test_env['PGPROBACKUP_SSH_PORT'] +# else +# print('PGPROBACKUP_SSH_PORT is not set') +# exit(1) +# +# if 'PGPROBACKUP_SSH_USER' in self.test_env: +# self.remote_user = self.test_env['PGPROBACKUP_SSH_USER'] +# else +# print('PGPROBACKUP_SSH_USER is not set') +# exit(1) + + + def make_simple_node( self, base_dir=None, @@ -604,14 +632,19 @@ def run_binary(self, command, asynchronous=False): except subprocess.CalledProcessError as e: raise ProbackupException(e.output.decode('utf-8'), command) - def init_pb(self, backup_dir, old_binary=False): + def init_pb(self, backup_dir, options=[], old_binary=False): shutil.rmtree(backup_dir, ignore_errors=True) + if self.remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + return self.run_pb([ 'init', '-B', backup_dir - ], + ] + options, old_binary=old_binary ) @@ -624,6 +657,11 @@ def add_instance(self, backup_dir, instance, node, old_binary=False, options=[]) '-D', node.data_dir ] + if self.remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + return self.run_pb(cmd + options, old_binary=old_binary) def set_config(self, backup_dir, instance, old_binary=False, options=[]): @@ -672,6 +710,11 @@ def backup_node( '-d', 'postgres', '--instance={0}'.format(instance) ] + if self.remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + if backup_type: cmd_list += ['-b', backup_type] @@ -702,6 +745,11 @@ def restore_node( '-D', data_dir, '--instance={0}'.format(instance) ] + if self.remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + if backup_id: cmd_list += ['-i', backup_id] @@ -895,6 +943,9 @@ def set_archiving( backup_dir.replace("\\","\\\\"), instance) + if self.remote: + archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' + if self.archive_compress or compress: archive_command = archive_command + '--compress ' From 9c27c4d928ef74025a953da11be43491f45a73ca Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Feb 2019 20:00:03 +0300 Subject: [PATCH 0273/2107] Add more comments --- src/utils/file.c | 61 +++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 5becb0cb5..b8336883a 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -384,6 +384,12 @@ int fio_truncate(int fd, off_t size) } } + +/* + * Read file from specified location. + * This call is optimized for delat backup, to avoid trasfer of old pages to backup host. + * For delta backup horizon_lsn parameter is assigned value of last backup and for all pages with LSN smaller than horizon_lsn only page header is sent. + */ int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn) { if (fio_is_remote_file(f)) @@ -405,11 +411,16 @@ int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn) if (hdr.size != 0) IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + /* + * We either return <0 for error, either 0 for EOF, either received size (page or page header size) + * for fully read page either 1 for partly read page. 1 is used just to distinguish it with page header size. + */ return hdr.arg <= 0 ? hdr.arg : hdr.arg == BLCKSZ ? hdr.size : 1; } else { int rc = pread(fileno(f), buf, BLCKSZ, offs); + /* See comment above consernign returned value */ return rc <= 0 || rc == BLCKSZ ? rc : 1; } } @@ -796,6 +807,11 @@ static void fio_send_file(int out, char const* path) /* Execute commands at remote host */ void fio_communicate(int in, int out) { + /* + * Map of file and directory descriptors. + * The same mapping is used in agent and master process, so we + * can use the same index at both sides. + */ int fd[FIO_FDMAX]; DIR* dir[FIO_FDMAX]; struct dirent* entry; @@ -806,22 +822,24 @@ void fio_communicate(int in, int out) XLogRecPtr horizon_lsn; int rc; + /* Main loop until command of processing master command */ while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { if (hdr.size != 0) { if (hdr.size > buf_size) { + /* Extend buffer on demand */ buf_size = hdr.size; buf = (char*)realloc(buf, buf_size); } IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); } switch (hdr.cop) { - case FIO_LOAD: + case FIO_LOAD: /* Send file content */ fio_send_file(out, buf); break; - case FIO_OPENDIR: + case FIO_OPENDIR: /* Open directory for traversal */ dir[hdr.handle] = opendir(buf); break; - case FIO_READDIR: + case FIO_READDIR: /* Get next direcrtory entry */ entry = readdir(dir[hdr.handle]); hdr.cop = FIO_SEND; if (entry != NULL) { @@ -833,20 +851,19 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } break; - case FIO_CLOSEDIR: + case FIO_CLOSEDIR: /* Finish directory traversal */ SYS_CHECK(closedir(dir[hdr.handle])); break; - case FIO_OPEN: + case FIO_OPEN: /* Open file */ SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); - fprintf(stderr, "Open file %s -> %d\n", buf, hdr.handle); break; - case FIO_CLOSE: + case FIO_CLOSE: /* Close file */ SYS_CHECK(close(fd[hdr.handle])); break; - case FIO_WRITE: + case FIO_WRITE: /* Write to the current position in file */ IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); break; - case FIO_READ: + case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { buf_size = hdr.arg; buf = (char*)realloc(buf, buf_size); @@ -857,53 +874,53 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; - case FIO_PREAD: + case FIO_PREAD: /* Read from specified position in file, ignoring pages beyond horizon of delta backup */ horizon_lsn = *(XLogRecPtr*)buf; rc = pread(fd[hdr.handle], buf, BLCKSZ, hdr.arg); - fprintf(stderr, "Read %d bytes from file %d offset %d\n", rc, hdr.handle, hdr.arg); hdr.cop = FIO_SEND; hdr.arg = rc; + /* For pages beyond horizon of delta backup transfer only page header */ hdr.size = (rc == BLCKSZ) - ? PageXLogRecPtrGet(((PageHeader)buf)->pd_lsn) < horizon_lsn + ? PageXLogRecPtrGet(((PageHeader)buf)->pd_lsn) < horizon_lsn /* For non-delta backup horizon_lsn == 0, so this condition is always false */ ? sizeof(PageHeaderData) : BLCKSZ : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size != 0) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; - case FIO_FSTAT: + case FIO_FSTAT: /* Get information about opened file */ hdr.size = sizeof(st); hdr.arg = fstat(fd[hdr.handle], &st); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; - case FIO_STAT: + case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); hdr.arg = hdr.arg ? stat(buf, &st) : lstat(buf, &st); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; - case FIO_ACCESS: + case FIO_ACCESS: /* Check presence of file with specified name */ hdr.size = 0; hdr.arg = access(buf, hdr.arg); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; - case FIO_RENAME: + case FIO_RENAME: /* Rename file */ SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); break; - case FIO_UNLINK: + case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ SYS_CHECK(remove(buf)); break; - case FIO_MKDIR: + case FIO_MKDIR: /* Create direcory */ SYS_CHECK(dir_create_dir(buf, hdr.arg)); break; - case FIO_CHMOD: + case FIO_CHMOD: /* Change file mode */ SYS_CHECK(chmod(buf, hdr.arg)); break; - case FIO_SEEK: + case FIO_SEEK: /* Set current position in file */ SYS_CHECK(lseek(fd[hdr.handle], hdr.arg, SEEK_SET)); break; - case FIO_TRUNCATE: + case FIO_TRUNCATE: /* Truncate file */ SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; default: @@ -911,7 +928,7 @@ void fio_communicate(int in, int out) } } free(buf); - if (rc != 0) { + if (rc != 0) { /* Not end of stream: normal pipe close */ perror("read"); exit(EXIT_FAILURE); } From 89c01d20c3bc9761811fa65d980d0d8e39e9eb2b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Feb 2019 23:22:16 +0300 Subject: [PATCH 0274/2107] Check page checksums for all pages of remote delta backup --- src/data.c | 14 +++++++++++--- src/pg_probackup.c | 14 +++++++------- src/utils/file.c | 8 ++++++-- src/utils/remote.c | 4 ++-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/data.c b/src/data.c index e4b2ecb35..47e96d565 100644 --- a/src/data.c +++ b/src/data.c @@ -266,14 +266,22 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* Verify checksum */ - if (current.checksum_version && read_len == BLCKSZ) + if (current.checksum_version) { + BlockNumber blkno = file->segno * RELSEG_SIZE + blknum; + uint16 page_crc = read_len == BLCKSZ + ? pg_checksum_page(page, blkno) + /* + * Recompute Cpage checksum calculated by agent with blkno=0 + * pg_checksum_page is calculating it in this way: + * (((checksum ^ blkno) % 65535) + 1) + */ + : (uint16)(((*(uint16*)((PageHeader)page)->pd_linp - 1) ^ blkno) + 1); /* * If checksum is wrong, sleep a bit and then try again * several times. If it didn't help, throw error */ - if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) - != ((PageHeader) page)->pd_checksum) + if (page_crc != ((PageHeader) page)->pd_checksum) { elog(WARNING, "File: %s blknum %u have wrong checksum, try again", file->path, blknum); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index dcf8278f5..a271adf7a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -338,8 +338,13 @@ main(int argc, char *argv[]) } canonicalize_path(backup_path); + MyLocation = IsSshProtocol() + ? backup_subcmd == ARCHIVE_PUSH_CMD + ? FIO_DB_HOST : FIO_BACKUP_HOST + : FIO_LOCAL_HOST; + /* Ensure that backup_path is a path to a directory */ - rc = stat(backup_path, &stat_buf); + rc = fio_stat(backup_path, &stat_buf, true, FIO_BACKUP_HOST); if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) elog(ERROR, "-B, --backup-path must be a path to directory"); @@ -356,11 +361,6 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: --instance"); } - MyLocation = IsSshProtocol() - ? backup_subcmd == ARCHIVE_PUSH_CMD - ? FIO_DB_HOST : FIO_BACKUP_HOST - : FIO_LOCAL_HOST; - /* * If --instance option was passed, construct paths for backup data and * xlog files of this backup instance. @@ -378,7 +378,7 @@ main(int argc, char *argv[]) */ if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { - if (access(backup_instance_path, F_OK) != 0) + if (fio_access(backup_instance_path, F_OK, FIO_BACKUP_HOST) != 0) elog(ERROR, "Instance '%s' does not exist in this backup catalog", instance_name); } diff --git a/src/utils/file.c b/src/utils/file.c index b8336883a..7b391e7d0 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -4,6 +4,7 @@ #include #include "pg_probackup.h" +#include "storage/checksum.h" #include "file.h" #define PRINTF_BUF_SIZE 1024 @@ -884,9 +885,12 @@ void fio_communicate(int in, int out) ? PageXLogRecPtrGet(((PageHeader)buf)->pd_lsn) < horizon_lsn /* For non-delta backup horizon_lsn == 0, so this condition is always false */ ? sizeof(PageHeaderData) : BLCKSZ : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size == sizeof(PageHeaderData)) + /* calculate checksum without XOR-ing with block number to compare it with page CRC at master */ + *(int16*)((PageHeader)buf)->pd_linp = pg_checksum_page(buf, 0); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_FSTAT: /* Get information about opened file */ hdr.size = sizeof(st); diff --git a/src/utils/remote.c b/src/utils/remote.c index 3b6fc8e24..eced5f87d 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -52,12 +52,12 @@ static int split_options(int argc, char* argv[], int max_options, char* options) } static int child_pid; - +#if 0 static void kill_child(void) { kill(child_pid, SIGTERM); } - +#endif bool launch_agent(void) { From ad8d5b6f12ef03ab8e52d17720ff7844f3a88d3e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 2 Feb 2019 09:46:19 +0300 Subject: [PATCH 0275/2107] More accurately store checksum in page header --- src/data.c | 2 +- src/utils/file.c | 2 +- src/utils/file.h | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 47e96d565..888a57ddd 100644 --- a/src/data.c +++ b/src/data.c @@ -276,7 +276,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * pg_checksum_page is calculating it in this way: * (((checksum ^ blkno) % 65535) + 1) */ - : (uint16)(((*(uint16*)((PageHeader)page)->pd_linp - 1) ^ blkno) + 1); + : (uint16)(((*PAGE_CHECKSUM(page) - 1) ^ blkno) + 1); /* * If checksum is wrong, sleep a bit and then try again * several times. If it didn't help, throw error diff --git a/src/utils/file.c b/src/utils/file.c index 7b391e7d0..efda75748 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -887,7 +887,7 @@ void fio_communicate(int in, int out) : 0; if (hdr.size == sizeof(PageHeaderData)) /* calculate checksum without XOR-ing with block number to compare it with page CRC at master */ - *(int16*)((PageHeader)buf)->pd_linp = pg_checksum_page(buf, 0); + *PAGE_CHECKSUM(buf) = pg_checksum_page(buf, 0); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size != 0) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); diff --git a/src/utils/file.h b/src/utils/file.h index 498b63765..b79eaf3db 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -47,6 +47,14 @@ typedef enum #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) #define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc < 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } } while (0) +/* + * Store one more checksum in page header. + * There is free space at the ned of page header (not used for page verification) + * While delta backup we need to calculate checksum at agent, send it to maini pg_probackup instance + * adjust it according to the real block number and compare with checksum stored in pd_checksum + */ +#define PAGE_CHECKSUM(p) ((uint16*)((p) + sizeof(PageHeaderData)) - 1) + typedef struct { unsigned cop : 5; From 35f2b5b636a0cbbe6f67332c2d1d612df71eeab2 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 3 Feb 2019 09:09:31 +0300 Subject: [PATCH 0276/2107] Undo occasional change in .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1898a58de..da388019c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ /src/datapagemap.h /src/logging.h /src/receivelog.c -/src/receivelog.hщщ +/src/receivelog.h /src/streamutil.c /src/streamutil.h /src/xlogreader.c From ace4ca29aea0a0e82a0b8a4a3ebebd614d974982 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 4 Feb 2019 16:12:07 +0300 Subject: [PATCH 0277/2107] PGPRO-2407: Time for RUNNING backup now calculated on the fly --- src/show.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/show.c b/src/show.c index b50aa80dd..9ff6243fc 100644 --- a/src/show.c +++ b/src/show.c @@ -325,6 +325,7 @@ show_instance_plain(parray *backup_list, bool show_name) uint32 widths[SHOW_FIELDS_COUNT]; uint32 widths_sum = 0; ShowBackendRow *rows; + time_t current_time = time(NULL); for (i = 0; i < SHOW_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -384,7 +385,11 @@ show_instance_plain(parray *backup_list, bool show_name) cur++; /* Time */ - if (backup->end_time != (time_t) 0) + if (backup->status == BACKUP_STATUS_RUNNING) + snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, + difftime(current_time, backup->start_time)); + + else if (backup->end_time != (time_t) 0) snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); else From eae31a18671daf10fd361064f23e392bb3a20554 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 4 Feb 2019 16:48:14 +0300 Subject: [PATCH 0278/2107] write START LSN to backup meta after pg_start_backup execution --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index 845ff03e8..766759942 100644 --- a/src/backup.c +++ b/src/backup.c @@ -574,6 +574,9 @@ do_backup_instance(void) strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); + /* Update running backup meta with START LSN */ + write_backup(¤t); + pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); From 9256d32e8eff3acc0531dcfbfe048542fab979e9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 Feb 2019 20:23:38 +0300 Subject: [PATCH 0279/2107] tests: added test_drop_rel_during_backup_delta(), test_drop_rel_during_backup_page() and test_drop_rel_during_backup_ptrack() --- tests/backup_test.py | 196 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index 2d4a71bdf..926c19d56 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -626,3 +626,199 @@ def test_tablespace_handling(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_rel_during_backup_delta(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DELTA backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + gdb=True, options=['--log-level-file=verbose']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + node.safe_psql( + "postgres", + "DROP TABLE t_heap") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_rel_during_backup_page(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PAGE backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', + gdb=True, options=['--log-level-file=verbose']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_rel_during_backup_ptrack(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'ptrack_enable': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PTRACK backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + gdb=True, options=['--log-level-file=verbose']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From c3cf4d31fa9983940b4278b3d2a9688c4f262037 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 8 Feb 2019 09:34:54 +0300 Subject: [PATCH 0280/2107] tests: disable fsync --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 237961300..86d09032f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -285,7 +285,7 @@ def make_simple_node( # Sane default parameters node.append_conf('postgresql.auto.conf', 'max_connections = 100') node.append_conf('postgresql.auto.conf', 'shared_buffers = 10MB') - node.append_conf('postgresql.auto.conf', 'fsync = on') + node.append_conf('postgresql.auto.conf', 'fsync = off') node.append_conf('postgresql.auto.conf', 'wal_level = logical') node.append_conf('postgresql.auto.conf', 'hot_standby = off') From 2028275c100c57642e8783e1e3ab8383957c98e9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Feb 2019 01:59:43 +0300 Subject: [PATCH 0281/2107] PGPRO-2425: tests test_restore_chain_with_corrupted_backup() and test_restore_chain() added --- tests/restore_test.py | 278 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 277 insertions(+), 1 deletion(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index e130894a3..6665642db 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -4,7 +4,7 @@ import subprocess from datetime import datetime import sys -import time +from time import sleep module_name = 'restore' @@ -1441,3 +1441,279 @@ def test_zags_block_corrupt_1(self): True, 'Failed to start pg_wal_dump: {0}'.format( pg_receivexlog.communicate()[1])) + + # @unittest.skip("skip") + def test_restore_chain(self): + """ + make node, take full backup, take several + ERROR delta backups, take valid delta backup, + restore must be successfull + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[3]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[4]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[5]['status'], + 'Backup STATUS should be "ERROR"') + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_chain_with_corrupted_backup(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + # Take DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + # Take 1 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + # Take 2 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--archive-timeout=0s']) + except ProbackupException as e: + pass + + # Take 3 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Corrupted 4 DELTA + corrupt_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # ORPHAN 5 DELTA + restore_target_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # ORPHAN 6 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # NEXT FULL BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # Next Delta + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # do corrupt 6 DELTA backup + file = os.path.join( + backup_dir, 'backups', 'node', + corrupt_id, 'database', 'global', 'pg_control') + + file_new = os.path.join(backup_dir, 'pg_control') + os.rename(file, file_new) + + # RESTORE BACKUP + node.cleanup() + + print(restore_target_id) + exit(1) + + try: + self.restore_node( + backup_dir, 'node', node, backup_id=restore_target_id) + self.assertEqual( + 1, 0, + "Expecting Error because restore backup is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is orphan'.format(restore_target_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[3]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[4]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[5]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[6]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[7]['status'], + 'Backup STATUS should be "OK"') + + # corruption victim + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node')[8]['status'], + 'Backup STATUS should be "CORRUPT"') + + # orphaned child + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node')[9]['status'], + 'Backup STATUS should be "ORPHAN"') + + # orphaned child + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node')[10]['status'], + 'Backup STATUS should be "ORPHAN"') + + # next FULL + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[11]['status'], + 'Backup STATUS should be "OK"') + + # next DELTA + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[12]['status'], + 'Backup STATUS should be "OK"') + + node.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) From 628cdce7786375f432edc2d05c211b70578a53a2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Feb 2019 02:15:06 +0300 Subject: [PATCH 0282/2107] PGPRO-2425: previously successfull restore was relying on that there are no failed backups between base_full_backup and dest_backup, it could lead to false-positive validation or restore errors despite the fact that parent chain is valid. There even was a small chance of data corruption: if between base_full_backup and dest_backup were located backups from parallel chain. TLDR: restore was rolling backups blindly. See test test_restore_chain_with_corrupted_backup() for an example. --- src/catalog.c | 7 +++++-- src/dir.c | 6 +++--- src/restore.c | 58 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index b30fcc664..54cf0908b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -981,6 +981,9 @@ is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive) if (!child_backup) elog(ERROR, "Target backup cannot be NULL"); + if (inclusive && child_backup->start_time == parent_backup_time) + return true; + while (child_backup->parent_backup_link && child_backup->parent_backup != parent_backup_time) { @@ -990,8 +993,8 @@ is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive) if (child_backup->parent_backup == parent_backup_time) return true; - if (inclusive && child_backup->start_time == parent_backup_time) - return true; + //if (inclusive && child_backup->start_time == parent_backup_time) + // return true; return false; } diff --git a/src/dir.c b/src/dir.c index 37f41ddda..cdb934980 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1047,11 +1047,11 @@ create_data_directories(const char *data_dir, const char *backup_dir, } if (link_sep) - elog(LOG, "create directory \"%s\" and symbolic link \"%.*s\"", + elog(VERBOSE, "create directory \"%s\" and symbolic link \"%.*s\"", linked_path, (int) (link_sep - relative_ptr), relative_ptr); else - elog(LOG, "create directory \"%s\" and symbolic link \"%s\"", + elog(VERBOSE, "create directory \"%s\" and symbolic link \"%s\"", linked_path, relative_ptr); /* Firstly, create linked directory */ @@ -1082,7 +1082,7 @@ create_data_directories(const char *data_dir, const char *backup_dir, } create_directory: - elog(LOG, "create directory \"%s\"", relative_ptr); + elog(VERBOSE, "create directory \"%s\"", relative_ptr); /* This is not symlink, create directory */ join_path_components(to_path, data_dir, relative_ptr); diff --git a/src/restore.c b/src/restore.c index e3591e067..eba0773b2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -57,6 +57,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, int base_full_backup_index = 0; int corrupted_backup_index = 0; char *action = is_restore ? "Restore":"Validate"; + parray *parent_chain = NULL; if (is_restore) { @@ -285,6 +286,27 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore) check_tablespace_mapping(dest_backup); + /* At this point we are sure that parent chain is whole + * so we can build separate array, containing all needed backups, + * to simplify validation and restore + */ + parent_chain = parray_new(); + + /* Take every backup that is a child of base_backup AND parent of dest_backup + * including base_backup and dest_backup + */ + for (i = base_full_backup_index; i >= dest_backup_index; i--) + { + tmp_backup = (pgBackup *) parray_get(backups, i); + + if (is_parent(base_full_backup->start_time, tmp_backup, true) && + is_parent(tmp_backup->start_time, dest_backup, true)) + { + parray_append(parent_chain, tmp_backup); + } + } + + /* for validation or restore with enabled validation */ if (!is_restore || !rt->restore_no_validate) { if (dest_backup->backup_mode != BACKUP_MODE_FULL) @@ -292,27 +314,25 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * Validate backups from base_full_backup to dest_backup. - * At this point we are sure that parent chain is intact. */ - for (i = base_full_backup_index; i >= dest_backup_index; i--) + for (i = 0; i < parray_num(parent_chain); i++) { - tmp_backup = (pgBackup *) parray_get(backups, i); + tmp_backup = (pgBackup *) parray_get(parent_chain, i); - if (is_parent(base_full_backup->start_time, tmp_backup, true)) + pgBackupValidate(tmp_backup); + /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ + if (tmp_backup->status == BACKUP_STATUS_CORRUPT) { - - pgBackupValidate(tmp_backup); - /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ - if (tmp_backup->status == BACKUP_STATUS_CORRUPT) - { - corrupted_backup = tmp_backup; - corrupted_backup_index = i; - break; - } - /* We do not validate WAL files of intermediate backups - * It`s done to speed up restore + corrupted_backup = tmp_backup; + /* we need corrupted backup index from 'backups' not parent_chain + * so we can properly orphanize all its descendants */ + corrupted_backup_index = get_backup_index_number(backups, corrupted_backup); + break; } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ } /* There is no point in wal validation of corrupted backups */ @@ -355,7 +375,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } - // TODO: rewrite restore to use parent_chain /* * If dest backup is corrupted or was orphaned in previous check * produce corresponding error message @@ -376,13 +395,12 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required - * TODO: use parent_link */ if (is_restore) { - for (i = base_full_backup_index; i >= dest_backup_index; i--) + for (i = 0; i < parray_num(parent_chain); i++) { - pgBackup *backup = (pgBackup *) parray_get(backups, i); + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", @@ -405,6 +423,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); + parray_free(parent_chain); elog(INFO, "%s of backup %s completed.", action, base36enc(dest_backup->start_time)); @@ -480,6 +499,7 @@ restore_backup(pgBackup *backup) /* By default there are some error */ threads_args[i].ret = 1; + /* Useless message TODO: rewrite */ elog(LOG, "Start thread for num:%zu", parray_num(files)); pthread_create(&threads[i], NULL, restore_files, arg); From 10b8db397abaef5f5064152d191f4fea716d75b9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Feb 2019 02:30:22 +0300 Subject: [PATCH 0283/2107] tests: minor fix to restore tests --- tests/restore_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 6665642db..169b30809 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1464,12 +1464,15 @@ def test_restore_chain(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + # Take FULL self.backup_node( backup_dir, 'node', node) + # Take DELTA self.backup_node( backup_dir, 'node', node, backup_type='delta') + # Take ERROR DELTA try: self.backup_node( backup_dir, 'node', node, @@ -1477,6 +1480,7 @@ def test_restore_chain(self): except ProbackupException as e: pass + # Take ERROR DELTA try: self.backup_node( backup_dir, 'node', node, @@ -1484,10 +1488,11 @@ def test_restore_chain(self): except ProbackupException as e: pass - + # Take DELTA self.backup_node( backup_dir, 'node', node, backup_type='delta') + # Take ERROR DELTA try: self.backup_node( backup_dir, 'node', node, @@ -1625,9 +1630,6 @@ def test_restore_chain_with_corrupted_backup(self): # RESTORE BACKUP node.cleanup() - print(restore_target_id) - exit(1) - try: self.restore_node( backup_dir, 'node', node, backup_id=restore_target_id) From bed0a451217698cc8af834db7e6759e79c5dd584 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Feb 2019 12:18:07 +0300 Subject: [PATCH 0284/2107] minor bugfix: dissapeared during backup files were marked as 'not changed', now such files are removed from backup_list at the end of the backup --- src/backup.c | 23 ++++++++++++++++++++--- src/data.c | 5 +++++ src/pg_probackup.h | 3 ++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 766759942..45a4737de 100644 --- a/src/backup.c +++ b/src/backup.c @@ -763,6 +763,19 @@ do_backup_instance(void) else elog(ERROR, "Data files transferring failed"); + /* Remove disappeared during backup files from backup_list */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); + + if (tmp_file->write_size == FILE_NOT_FOUND) + { + pg_atomic_clear_flag(&tmp_file->lock); + pgFileFree(tmp_file); + parray_remove(backup_files_list, i); + } + } + /* clean previous backup file list */ if (prev_backup_filelist) { @@ -2241,7 +2254,7 @@ backup_files(void *arg) * If file is not found, this is not en error. * It could have been deleted by concurrent postgres transaction. */ - file->write_size = BYTES_INVALID; + file->write_size = FILE_NOT_FOUND; elog(LOG, "File \"%s\" is not found", file->path); continue; } @@ -2291,7 +2304,9 @@ backup_files(void *arg) instance_config.compress_alg, instance_config.compress_level)) { - file->write_size = BYTES_INVALID; + /* disappeared file not to be confused with 'not changed' */ + if (file->write_size != FILE_NOT_FOUND) + file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; } @@ -2315,7 +2330,9 @@ backup_files(void *arg) if (skip || !copy_file(arguments->from_root, arguments->to_root, file)) { - file->write_size = BYTES_INVALID; + /* disappeared file not to be confused with 'not changed' */ + if (file->write_size != FILE_NOT_FOUND) + file->write_size = BYTES_INVALID; elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); continue; diff --git a/src/data.c b/src/data.c index ccbe9ca87..d2ebdd934 100644 --- a/src/data.c +++ b/src/data.c @@ -564,6 +564,7 @@ backup_data_file(backup_files_arg* arguments, if (errno == ENOENT) { elog(LOG, "File \"%s\" is not found", file->path); + file->write_size = FILE_NOT_FOUND; return false; } @@ -946,7 +947,11 @@ copy_file(const char *from_root, const char *to_root, pgFile *file) /* maybe deleted, it's not error */ if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + file->write_size = FILE_NOT_FOUND; return false; + } elog(ERROR, "cannot open source file \"%s\": %s", file->path, strerror(errno)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 546c1016c..81d66d2c2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -165,7 +165,8 @@ typedef enum ShowFormat /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ -#define BYTES_INVALID (-1) +#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ +#define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) /* From 04a7d456478a52b129dbe7ae984b979e4b716801 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 11 Feb 2019 15:16:59 +0300 Subject: [PATCH 0285/2107] PGPRO-2432: Improve reading of compressed WAL's if Postgrestries to read same record many times --- src/parsexlog.c | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 44c87ff84..5fa2a22f4 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -104,6 +104,9 @@ typedef struct XLogPageReadPrivate #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; + + char gz_buf[XLOG_BLCKSZ]; + uint32 gz_prev_off; #endif } XLogPageReadPrivate; @@ -1057,22 +1060,30 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #ifdef HAVE_LIBZ else { - if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (private_data->gz_prev_off != 0 && + private_data->gz_prev_off == targetPageOff) + memcpy(readBuf, private_data->gz_buf, XLOG_BLCKSZ); + else { - elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); - return -1; - } + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + { + elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", + private_data->thread_num, + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } - if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); - return -1; + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", + private_data->thread_num, + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } + private_data->gz_prev_off = targetPageOff; + memcpy(private_data->gz_buf, readBuf, XLOG_BLCKSZ); } } #endif @@ -1131,6 +1142,7 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) { gzclose(private_data->gz_xlogfile); private_data->gz_xlogfile = NULL; + private_data->gz_prev_off = 0; } #endif private_data->xlogexists = false; From 14f48dcefb93b484373ae4aa5b9c77e48794c073 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 11 Feb 2019 18:07:12 +0300 Subject: [PATCH 0286/2107] Throw an error when during a restore operation the user tries to remap an external directory that is not in the destination backup --- src/dir.c | 41 ++++++++++++++++++++++++++++++++++++++++- src/pg_probackup.h | 3 ++- src/restore.c | 11 +++++++---- tests/external.py | 4 ++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/dir.c b/src/dir.c index f05a2cc21..1f248e402 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1263,8 +1263,47 @@ check_tablespace_mapping(pgBackup *backup) parray_free(links); } +void +check_extra_dir_mapping(pgBackup *backup) +{ + TablespaceListCell *cell; + parray *extra_dirs_to_restore; + bool found; + int i; + + if (!backup->extra_dir_str) + { + if (extra_remap_list.head) + elog(ERROR, "--extra-mapping option's old directory doesn't have " + "an entry in list of extra directories of current " + "backup: \"%s\"", extra_remap_list.head->old_dir); + return; + } + + extra_dirs_to_restore = make_extra_directory_list(backup->extra_dir_str); + for (cell = extra_remap_list.head; cell; cell = cell->next) + { + char *old_dir = cell->old_dir; + + found = false; + for (i = 0; i < parray_num(extra_dirs_to_restore); i++) + { + char *external_dir = parray_get(extra_dirs_to_restore, i); + if (strcmp(old_dir, external_dir) == 0) + { + found = true; + break; + } + } + if (!found) + elog(ERROR, "--extra-mapping option's old directory doesn't have " + "an entry in list of extra directories of current " + "backup: \"%s\"", cell->old_dir); + } +} + char * -check_extra_dir_mapping(char *current_dir) +get_extra_remap(char *current_dir) { TablespaceListCell *cell; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5d91a00ec..e2984755e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -509,7 +509,8 @@ extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_extradir_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); -extern char* check_extra_dir_mapping(char *current_dir); +extern void check_extra_dir_mapping(pgBackup *backup); +extern char *get_extra_remap(char *current_dir); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *extra_prefix, parray *extra_list); diff --git a/src/restore.c b/src/restore.c index 2eb6d563b..f05184cbc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -285,7 +285,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * i.e. empty or not exist. */ if (is_restore) + { check_tablespace_mapping(dest_backup); + check_extra_dir_mapping(dest_backup); + } if (!is_restore || !rt->restore_no_validate) { @@ -461,10 +464,10 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) if(extra_dir_str) { requested_extra_dirs = make_extra_directory_list(extra_dir_str); - for (int i = 0; i < parray_num(requested_extra_dirs); i++) + for (i = 0; i < parray_num(requested_extra_dirs); i++) { char *extra_path = parray_get(requested_extra_dirs, i); - extra_path = check_extra_dir_mapping(extra_path); + extra_path = get_extra_remap(extra_path); dir_create_dir(extra_path, DIR_PERMISSION); } } @@ -507,7 +510,7 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) { char container_dir[MAXPGPATH]; - extra_path = check_extra_dir_mapping(extra_path); + extra_path = get_extra_remap(extra_path); makeExtraDirPathByNum(container_dir, extra_prefix, file->extra_dir_num); dir_name = GetRelativePath(file->path, container_dir); @@ -698,7 +701,7 @@ restore_files(void *arg) file->extra_dir_num - 1); if (backup_contains_extra(extra_path, arguments->req_extra_dirs)) { - extra_path = check_extra_dir_mapping(extra_path); + extra_path = get_extra_remap(extra_path); copy_file(arguments->extra_prefix, extra_path, file); } } diff --git a/tests/external.py b/tests/external.py index 19c6b4df0..582402e69 100644 --- a/tests/external.py +++ b/tests/external.py @@ -127,8 +127,8 @@ def test_external_dir_mapping(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, + 'ERROR: --extra-mapping option' in e.message and + 'have an entry in list of extra directories' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 5c247d0ffa0146cad54e7beb50227af8d337cbd4 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 11 Feb 2019 18:14:31 +0300 Subject: [PATCH 0287/2107] treat snap files as not datafiles. fix a typo in snapfs.py --- src/dir.c | 8 ++++++++ tests/snapfs.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 37f41ddda..b12b3d747 100644 --- a/src/dir.c +++ b/src/dir.c @@ -630,6 +630,14 @@ dir_check_file(const char *root, pgFile *file) } else { + /* + * snapfs files: + * RELFILENODE.BLOCKNO.snapmap.SNAPID + * RELFILENODE.BLOCKNO.snap.SNAPID + */ + if (strstr(file->name, "snap") != NULL) + return true; + len = strlen(file->name); /* reloid.cfm */ if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) diff --git a/tests/snapfs.py b/tests/snapfs.py index 31de79b5c..0bb244206 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -42,7 +42,7 @@ def test_backup_modes_archive(self): node.safe_psql( 'postgres', - 'select pg_remove_snashot(1)') + 'select pg_remove_snapshot(1)') self.backup_node( backup_dir, 'node', node) From c052651b8c8864733bcabbc2660c387b792229d8 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 11 Feb 2019 19:09:21 +0300 Subject: [PATCH 0288/2107] PGPRO-2432: Expand 04a7d456478a to usual files --- src/parsexlog.c | 57 +++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 5fa2a22f4..d8ba3ad76 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -93,6 +93,9 @@ typedef struct XLogPageReadPrivate TimeLineID tli; uint32 xlog_seg_size; + char page_buf[XLOG_BLCKSZ]; + uint32 prev_page_off; + bool manual_switch; bool need_switch; @@ -104,9 +107,6 @@ typedef struct XLogPageReadPrivate #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; - - char gz_buf[XLOG_BLCKSZ]; - uint32 gz_prev_off; #endif } XLogPageReadPrivate; @@ -1040,6 +1040,17 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, */ Assert(private_data->xlogexists); + /* + * Do not read same page read earlier from the file, read it from the buffer + */ + if (private_data->prev_page_off != 0 && + private_data->prev_page_off == targetPageOff) + { + memcpy(readBuf, private_data->page_buf, XLOG_BLCKSZ); + *pageTLI = private_data->tli; + return XLOG_BLCKSZ; + } + /* Read the requested page */ if (private_data->xlogfile != -1) { @@ -1060,34 +1071,28 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #ifdef HAVE_LIBZ else { - if (private_data->gz_prev_off != 0 && - private_data->gz_prev_off == targetPageOff) - memcpy(readBuf, private_data->gz_buf, XLOG_BLCKSZ); - else + if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { - if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) - { - elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); - return -1; - } + elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", + private_data->thread_num, + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; + } - if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); - return -1; - } - private_data->gz_prev_off = targetPageOff; - memcpy(private_data->gz_buf, readBuf, XLOG_BLCKSZ); + if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", + private_data->thread_num, + private_data->gz_xlogpath, + get_gz_error(private_data->gz_xlogfile)); + return -1; } } #endif + memcpy(private_data->page_buf, readBuf, XLOG_BLCKSZ); + private_data->prev_page_off = targetPageOff; *pageTLI = private_data->tli; return XLOG_BLCKSZ; } @@ -1142,9 +1147,9 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) { gzclose(private_data->gz_xlogfile); private_data->gz_xlogfile = NULL; - private_data->gz_prev_off = 0; } #endif + private_data->prev_page_off = 0; private_data->xlogexists = false; } From 8b8337047c63965d4cda31d4d54685066ecff623 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 Feb 2019 20:04:33 +0300 Subject: [PATCH 0289/2107] tests: minor fix --- tests/snapfs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/snapfs.py b/tests/snapfs.py index 0bb244206..ec269cb56 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -11,7 +11,7 @@ class SnapFSTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_backup_modes_archive(self): + def test_snapfs_simple(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -32,7 +32,7 @@ def test_backup_modes_archive(self): 'postgres', 'select pg_make_snapshot()') - node.pgbench_init(scale=100) + node.pgbench_init(scale=10) pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum']) pgbench.wait() @@ -53,7 +53,7 @@ def test_backup_modes_archive(self): self.restore_node( backup_dir, 'node', - restored_node, options=["-j", "4"]) + node, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) From d7f51ac855d8f7bf9e84afd045ffa913d034e6f9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 Feb 2019 20:14:00 +0300 Subject: [PATCH 0290/2107] tests: another minor fix for snapfs --- tests/snapfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snapfs.py b/tests/snapfs.py index ec269cb56..a8acc8354 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -45,7 +45,7 @@ def test_snapfs_simple(self): 'select pg_remove_snapshot(1)') self.backup_node( - backup_dir, 'node', node) + backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) From a12fe744f92e60f6e1cafe1b63dd481bd2956041 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 11 Feb 2019 20:29:52 +0300 Subject: [PATCH 0291/2107] Make possible to backup database dir as exrta dir --- src/backup.c | 5 +++-- src/restore.c | 4 ++-- src/validate.c | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1e92f226c..d853a3b51 100644 --- a/src/backup.c +++ b/src/backup.c @@ -642,7 +642,7 @@ do_backup_instance(void) for (i = 0; i < parray_num(extra_dirs); i++) /* Extra dirs numeration starts with 1. 0 value is not extra dir */ dir_list_file(backup_files_list, parray_get(extra_dirs, i), - true, true, false, i+1); + false, true, false, i+1); /* * Sort pathname ascending. It is necessary to create intermediate @@ -2335,7 +2335,8 @@ backup_files(void *arg) continue; } } - else if (strcmp(file->name, "pg_control") == 0) + else if (!file->extra_dir_num && + strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(arguments->from_root, arguments->to_root, file); else diff --git a/src/restore.c b/src/restore.c index f05184cbc..0978042fe 100644 --- a/src/restore.c +++ b/src/restore.c @@ -693,8 +693,6 @@ restore_files(void *arg) false, parse_program_version(arguments->backup->program_version)); } - else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(from_root, instance_config.pgdata, file); else if (file->extra_dir_num) { char *extra_path = parray_get(arguments->cur_extra_dirs, @@ -705,6 +703,8 @@ restore_files(void *arg) copy_file(arguments->extra_prefix, extra_path, file); } } + else if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(from_root, instance_config.pgdata, file); else copy_file(from_root, instance_config.pgdata, file); diff --git a/src/validate.c b/src/validate.c index ac5edcaf6..709bdd125 100644 --- a/src/validate.c +++ b/src/validate.c @@ -239,7 +239,8 @@ pgBackupValidateFiles(void *arg) * Starting from 2.0.25 we calculate crc of pg_control differently. */ if (arguments->backup_version >= 20025 && - strcmp(file->name, "pg_control") == 0) + strcmp(file->name, "pg_control") == 0 && + !file->extra_dir_num) crc = get_pgcontrol_checksum(arguments->base_path); else crc = pgFileGetCRC(file->path, From dbefa9e183bbe9f6609a5f7378d1df4756cf768f Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 12 Feb 2019 17:19:18 +0300 Subject: [PATCH 0292/2107] Rename feature from "extra directories" to "external directories" --- src/backup.c | 85 ++++++++++++++++-------------- src/catalog.c | 36 +++++++------ src/configure.c | 4 +- src/dir.c | 106 ++++++++++++++++++------------------- src/help.c | 22 ++++---- src/merge.c | 127 +++++++++++++++++++++++---------------------- src/pg_probackup.c | 6 +-- src/pg_probackup.h | 42 ++++++++------- src/restore.c | 93 +++++++++++++++++---------------- src/validate.c | 8 +-- tests/external.py | 34 ++++++------ 11 files changed, 290 insertions(+), 273 deletions(-) diff --git a/src/backup.c b/src/backup.c index d853a3b51..a0c6f867b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -468,7 +468,7 @@ do_backup_instance(void) { int i; char database_path[MAXPGPATH]; - char extra_prefix[MAXPGPATH]; /* Temp value. Used as template */ + char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -481,13 +481,13 @@ do_backup_instance(void) pgBackup *prev_backup = NULL; parray *prev_backup_filelist = NULL; parray *backup_list = NULL; - parray *extra_dirs = NULL; + parray *external_dirs = NULL; pgFile *pg_control = NULL; elog(LOG, "Database backup start"); - if(current.extra_dir_str) - extra_dirs = make_extra_directory_list(current.extra_dir_str); + if(current.external_dir_str) + external_dirs = make_external_directory_list(current.external_dir_str); /* Initialize size summary */ current.data_bytes = 0; @@ -579,7 +579,8 @@ do_backup_instance(void) pg_start_backup(label, smooth_checkpoint, ¤t); pgBackupGetPath(¤t, database_path, lengthof(database_path),DATABASE_DIR); - pgBackupGetPath(¤t, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); + pgBackupGetPath(¤t, external_prefix, lengthof(external_prefix), + EXTERNAL_DIR); /* start stream replication */ if (stream_wal) @@ -636,12 +637,13 @@ do_backup_instance(void) /* * Append to backup list all files and directories - * from extra directory option + * from external directory option */ - if (extra_dirs) - for (i = 0; i < parray_num(extra_dirs); i++) - /* Extra dirs numeration starts with 1. 0 value is not extra dir */ - dir_list_file(backup_files_list, parray_get(extra_dirs, i), + if (external_dirs) + for (i = 0; i < parray_num(external_dirs); i++) + /* External dirs numeration starts with 1. + * 0 value is not external dir */ + dir_list_file(backup_files_list, parray_get(external_dirs, i), false, true, false, i+1); /* @@ -705,9 +707,10 @@ do_backup_instance(void) char *dir_name; if (!is_remote_backup) - if (file->extra_dir_num) + if (file->external_dir_num) dir_name = GetRelativePath(file->path, - parray_get(extra_dirs, file->extra_dir_num - 1)); + parray_get(external_dirs, + file->external_dir_num - 1)); else dir_name = GetRelativePath(file->path, instance_config.pgdata); else @@ -715,10 +718,11 @@ do_backup_instance(void) elog(VERBOSE, "Create directory \"%s\"", dir_name); - if (file->extra_dir_num) + if (file->external_dir_num) { char temp[MAXPGPATH]; - snprintf(temp, MAXPGPATH, "%s%d", extra_prefix, file->extra_dir_num); + snprintf(temp, MAXPGPATH, "%s%d", external_prefix, + file->external_dir_num); join_path_components(dirpath, temp, dir_name); } else @@ -734,7 +738,7 @@ do_backup_instance(void) parray_qsort(backup_files_list, pgFileCompareSize); /* Sort the array for binary search */ if (prev_backup_filelist) - parray_qsort(prev_backup_filelist, pgFileComparePathWithExtra); + parray_qsort(prev_backup_filelist, pgFileComparePathWithExternal); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -746,8 +750,8 @@ do_backup_instance(void) arg->from_root = instance_config.pgdata; arg->to_root = database_path; - arg->extra_prefix = extra_prefix; - arg->extra_dirs = extra_dirs; + arg->external_prefix = external_prefix; + arg->external_dirs = external_dirs; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; @@ -853,11 +857,11 @@ do_backup_instance(void) /* Print the list of files to backup catalog */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, - NULL, extra_dirs); + NULL, external_dirs); - /* clean extra directories list */ - if (extra_dirs) - free_dir_list(extra_dirs); + /* clean external directories list */ + if (external_dirs) + free_dir_list(external_dirs); /* Compute summary of size of regular files in the backup */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -990,11 +994,11 @@ do_backup(time_t start_time) StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); - /* Save list of extra directories */ - if (instance_config.extra_dir_str && - pg_strcasecmp(instance_config.extra_dir_str, "none") != 0) + /* Save list of external directories */ + if (instance_config.external_dir_str && + pg_strcasecmp(instance_config.external_dir_str, "none") != 0) { - current.extra_dir_str = instance_config.extra_dir_str; + current.external_dir_str = instance_config.external_dir_str; } /* Create backup directory and BACKUP_CONTROL_FILE */ @@ -2292,11 +2296,11 @@ backup_files(void *arg) if (S_ISREG(buf.st_mode)) { pgFile **prev_file = NULL; - char *extra_path = NULL; + char *external_path = NULL; - if (file->extra_dir_num) - extra_path = parray_get(arguments->extra_dirs, - file->extra_dir_num - 1); + if (file->external_dir_num) + external_path = parray_get(arguments->external_dirs, + file->external_dir_num - 1); /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2304,13 +2308,13 @@ backup_files(void *arg) char *relative; pgFile key; - relative = GetRelativePath(file->path, file->extra_dir_num ? - extra_path : arguments->from_root); + relative = GetRelativePath(file->path, file->external_dir_num ? + external_path : arguments->from_root); key.path = relative; - key.extra_dir_num = file->extra_dir_num; + key.external_dir_num = file->external_dir_num; prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, - &key, pgFileComparePathWithExtra); + &key, pgFileComparePathWithExternal); if (prev_file) /* File exists in previous backup */ file->exists_in_prev = true; @@ -2335,7 +2339,7 @@ backup_files(void *arg) continue; } } - else if (!file->extra_dir_num && + else if (!file->external_dir_num && strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(arguments->from_root, arguments->to_root, file); @@ -2344,7 +2348,7 @@ backup_files(void *arg) const char *src; const char *dst; bool skip = false; - char extra_dst[MAXPGPATH]; + char external_dst[MAXPGPATH]; /* If non-data file has not changed since last backup... */ if (prev_file && file->exists_in_prev && @@ -2356,12 +2360,13 @@ backup_files(void *arg) skip = true; /* ...skip copying file. */ } /* Set file paths */ - if (file->extra_dir_num) + if (file->external_dir_num) { - makeExtraDirPathByNum(extra_dst, arguments->extra_prefix, - file->extra_dir_num); - src = extra_path; - dst = extra_dst; + makeExternalDirPathByNum(external_dst, + arguments->external_prefix, + file->external_dir_num); + src = external_path; + dst = external_dst; } else { diff --git a/src/catalog.c b/src/catalog.c index fd257ea0b..38c01bb9a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -414,19 +414,21 @@ pgBackupCreateDir(pgBackup *backup) temp = palloc(strlen(DATABASE_DIR) + 1); parray_append(subdirs, temp); - /* Add extra dirs containers */ - if (backup->extra_dir_str) + /* Add external dirs containers */ + if (backup->external_dir_str) { - parray *extradirs_list = make_extra_directory_list(backup->extra_dir_str); - for (int i = 0; i < parray_num(extradirs_list); i++) + parray *external_list; + + external_list = make_external_directory_list(backup->external_dir_str); + for (int i = 0; i < parray_num(external_list); i++) { - /* 20 chars is enough to hold the extradir number in string. */ - temp = palloc(strlen(EXTRA_DIR) + 20); - /* Numeration of extradirs starts with 1 */ - makeExtraDirPathByNum(temp, EXTRA_DIR, i+1); + /* 20 chars is enough to hold the externaldir number in string. */ + temp = palloc(strlen(EXTERNAL_DIR) + 20); + /* Numeration of externaldirs starts with 1 */ + makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); parray_append(subdirs, temp); } - free_dir_list(extradirs_list); + free_dir_list(external_list); } pgBackupGetPath(backup, path, lengthof(path), NULL); @@ -517,9 +519,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->primary_conninfo) fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); - /* print extra directories list */ - if (backup->extra_dir_str) - fprintf(out, "extra-directory = '%s'\n", backup->extra_dir_str); + /* print external directories list */ + if (backup->external_dir_str) + fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); } /* @@ -551,7 +553,7 @@ write_backup(pgBackup *backup) */ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, - const char *extra_prefix, parray *extra_list) + const char *external_prefix, parray *external_list) { FILE *fp; char path[MAXPGPATH]; @@ -563,7 +565,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot open file list \"%s\": %s", path, strerror(errno)); - print_file_list(fp, files, root, extra_prefix, extra_list); + print_file_list(fp, files, root, external_prefix, external_list); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || @@ -614,7 +616,7 @@ readBackupControlFile(const char *path) {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, - {'s', 0, "extra-directory", &backup->extra_dir_str, SOURCE_FILE_STRICT}, + {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, {0} }; @@ -844,7 +846,7 @@ pgBackupInit(pgBackup *backup) backup->primary_conninfo = NULL; backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; - backup->extra_dir_str = NULL; + backup->external_dir_str = NULL; } /* free pgBackup object */ @@ -854,7 +856,7 @@ pgBackupFree(void *backup) pgBackup *b = (pgBackup *) backup; pfree(b->primary_conninfo); - pfree(b->extra_dir_str); + pfree(b->external_dir_str); pfree(backup); } diff --git a/src/configure.c b/src/configure.c index b20775884..aea78244c 100644 --- a/src/configure.c +++ b/src/configure.c @@ -62,8 +62,8 @@ ConfigOption instance_options[] = }, #endif { - 's', 'E', "extra-directory", - &instance_config.extra_dir_str, SOURCE_CMD, 0, + 's', 'E', "external-dirs", + &instance_config.external_dir_str, SOURCE_CMD, 0, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ diff --git a/src/dir.c b/src/dir.c index 1f248e402..6b1867f1f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -122,7 +122,7 @@ static char dir_check_file(const char *root, pgFile *file); static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list, - int extra_dir_num); + int external_dir_num); static void list_data_directories(parray *files, const char *path, bool is_root, bool exclude); @@ -133,7 +133,7 @@ static void opt_path_map(ConfigOption *opt, const char *arg, static TablespaceList tablespace_dirs = {NULL, NULL}; static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* Extra directories mapping */ -static TablespaceList extra_remap_list = {NULL, NULL}; +static TablespaceList external_remap_list = {NULL, NULL}; /* * Create directory, also create parent directories if necessary. @@ -162,7 +162,7 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) +pgFileNew(const char *path, bool omit_symlink, int external_dir_num) { struct stat st; pgFile *file; @@ -180,7 +180,7 @@ pgFileNew(const char *path, bool omit_symlink, int extra_dir_num) file = pgFileInit(path); file->size = st.st_size; file->mode = st.st_mode; - file->extra_dir_num = extra_dir_num; + file->external_dir_num = external_dir_num; return file; } @@ -231,7 +231,7 @@ pgFileInit(const char *path) /* Number of blocks readed during backup */ file->n_blocks = BLOCKNUM_INVALID; file->compress_alg = NOT_DEFINED_COMPRESS; - file->extra_dir_num = 0; + file->external_dir_num = 0; return file; } @@ -353,11 +353,11 @@ pgFileComparePath(const void *f1, const void *f2) } /* - * Compare two pgFile with their path and extra_dir_num + * Compare two pgFile with their path and external_dir_num * in ascending order of ASCII code. */ int -pgFileComparePathWithExtra(const void *f1, const void *f2) +pgFileComparePathWithExternal(const void *f1, const void *f2) { pgFile *f1p = *(pgFile **)f1; pgFile *f2p = *(pgFile **)f2; @@ -366,9 +366,9 @@ pgFileComparePathWithExtra(const void *f1, const void *f2) res = strcmp(f1p->path, f2p->path); if (!res) { - if (f1p->extra_dir_num > f2p->extra_dir_num) + if (f1p->external_dir_num > f2p->external_dir_num) return 1; - else if (f1p->extra_dir_num < f2p->extra_dir_num) + else if (f1p->external_dir_num < f2p->external_dir_num) return -1; else return 0; @@ -423,7 +423,7 @@ BlackListCompare(const void *str1, const void *str2) */ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, - bool add_root, int extra_dir_num) + bool add_root, int external_dir_num) { pgFile *file; parray *black_list = NULL; @@ -462,7 +462,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false, extra_dir_num); + file = pgFileNew(root, false, external_dir_num); if (file == NULL) return; @@ -475,7 +475,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_append(files, file); dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, - extra_dir_num); + external_dir_num); if (!add_root) pgFileFree(file); @@ -686,7 +686,8 @@ dir_check_file(const char *root, pgFile *file) */ static void dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list, int extra_dir_num) + bool exclude, bool omit_symlink, parray *black_list, + int external_dir_num) { DIR *dir; struct dirent *dent; @@ -716,7 +717,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, join_path_components(child, parent->path, dent->d_name); - file = pgFileNew(child, omit_symlink, extra_dir_num); + file = pgFileNew(child, omit_symlink, external_dir_num); if (file == NULL) continue; @@ -773,7 +774,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, root, file, exclude, omit_symlink, - black_list, extra_dir_num); + black_list, external_dir_num); } if (errno && errno != ENOENT) @@ -986,11 +987,11 @@ opt_tablespace_map(ConfigOption *opt, const char *arg) opt_path_map(opt, arg, &tablespace_dirs, "tablespace"); } -/* Parse extra directories mapping */ +/* Parse external directories mapping */ void -opt_extradir_map(ConfigOption *opt, const char *arg) +opt_externaldir_map(ConfigOption *opt, const char *arg) { - opt_path_map(opt, arg, &extra_remap_list, "extra directory"); + opt_path_map(opt, arg, &external_remap_list, "external directory"); } /* @@ -1264,31 +1265,31 @@ check_tablespace_mapping(pgBackup *backup) } void -check_extra_dir_mapping(pgBackup *backup) +check_external_dir_mapping(pgBackup *backup) { TablespaceListCell *cell; - parray *extra_dirs_to_restore; + parray *external_dirs_to_restore; bool found; int i; - if (!backup->extra_dir_str) + if (!backup->external_dir_str) { - if (extra_remap_list.head) - elog(ERROR, "--extra-mapping option's old directory doesn't have " - "an entry in list of extra directories of current " - "backup: \"%s\"", extra_remap_list.head->old_dir); + if (external_remap_list.head) + elog(ERROR, "--external-mapping option's old directory doesn't " + "have an entry in list of external directories of current " + "backup: \"%s\"", external_remap_list.head->old_dir); return; } - extra_dirs_to_restore = make_extra_directory_list(backup->extra_dir_str); - for (cell = extra_remap_list.head; cell; cell = cell->next) + external_dirs_to_restore = make_external_directory_list(backup->external_dir_str); + for (cell = external_remap_list.head; cell; cell = cell->next) { char *old_dir = cell->old_dir; found = false; - for (i = 0; i < parray_num(extra_dirs_to_restore); i++) + for (i = 0; i < parray_num(external_dirs_to_restore); i++) { - char *external_dir = parray_get(extra_dirs_to_restore, i); + char *external_dir = parray_get(external_dirs_to_restore, i); if (strcmp(old_dir, external_dir) == 0) { found = true; @@ -1296,18 +1297,18 @@ check_extra_dir_mapping(pgBackup *backup) } } if (!found) - elog(ERROR, "--extra-mapping option's old directory doesn't have " - "an entry in list of extra directories of current " + elog(ERROR, "--external-mapping option's old directory doesn't " + "have an entry in list of external directories of current " "backup: \"%s\"", cell->old_dir); } } char * -get_extra_remap(char *current_dir) +get_external_remap(char *current_dir) { TablespaceListCell *cell; - for (cell = extra_remap_list.head; cell; cell = cell->next) + for (cell = external_remap_list.head; cell; cell = cell->next) { char *old_dir = cell->old_dir; @@ -1322,7 +1323,7 @@ get_extra_remap(char *current_dir) */ void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix, parray *extra_list) + const char *external_prefix, parray *external_list) { size_t i; @@ -1335,20 +1336,20 @@ print_file_list(FILE *out, const parray *files, const char *root, /* omit root directory portion */ if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - else if (file->extra_dir_num && !extra_prefix) + else if (file->external_dir_num && !external_prefix) { - Assert(extra_list); - path = GetRelativePath(path, parray_get(extra_list, - file->extra_dir_num - 1)); + Assert(external_list); + path = GetRelativePath(path, parray_get(external_list, + file->external_dir_num - 1)); } fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"extra_dir_num\":\"%d\"", + "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, - deparse_compress_alg(file->compress_alg), file->extra_dir_num); + deparse_compress_alg(file->compress_alg), file->external_dir_num); if (file->is_datafile) fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -1511,7 +1512,8 @@ get_control_value(const char *str, const char *name, * If root is not NULL, path will be absolute path. */ parray * -dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt) +dir_read_file_list(const char *root, const char *external_prefix, + const char *file_txt) { FILE *fp; parray *files; @@ -1533,7 +1535,7 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, - extra_dir_num, + external_dir_num, crc, segno, n_blocks; @@ -1546,13 +1548,13 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ get_control_value(buf, "is_cfs", NULL, &is_cfs, false); get_control_value(buf, "crc", NULL, &crc, true); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "extra_dir_num", NULL, &extra_dir_num, false); + get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); - if (extra_dir_num && extra_prefix) + if (external_dir_num && external_prefix) { char temp[MAXPGPATH]; - makeExtraDirPathByNum(temp, extra_prefix, extra_dir_num); + makeExternalDirPathByNum(temp, external_prefix, external_dir_num); join_path_components(filepath, temp, path); } else if (root) @@ -1568,7 +1570,7 @@ dir_read_file_list(const char *root, const char *extra_prefix, const char *file_ file->is_cfs = is_cfs ? true : false; file->crc = (pg_crc32) crc; file->compress_alg = parse_compress_alg(compress_alg_string); - file->extra_dir_num = extra_dir_num; + file->external_dir_num = external_dir_num; /* * Optional fields @@ -1656,11 +1658,11 @@ pgFileSize(const char *path) } /* - * Construct parray containing extra directories paths + * Construct parray containing external directories paths * from string like /path1:/path2 */ parray * -make_extra_directory_list(const char *colon_separated_dirs) +make_external_directory_list(const char *colon_separated_dirs) { char *p; parray *list = parray_new(); @@ -1693,19 +1695,19 @@ free_dir_list(parray *list) /* Append to string "path_prefix" int "dir_num" */ void -makeExtraDirPathByNum(char *ret_path, const char *path_prefix, - const int dir_num) +makeExternalDirPathByNum(char *ret_path, const char *path_prefix, + const int dir_num) { sprintf(ret_path, "%s%d", path_prefix, dir_num); } /* Check if "dir" presents in "dirs_list" */ bool -backup_contains_extra(const char *dir, parray *dirs_list) +backup_contains_external(const char *dir, parray *dirs_list) { void *search_result; - if (!dirs_list) /* There is no extra dirs in backup */ + if (!dirs_list) /* There is no external dirs in backup */ return false; search_result = parray_bsearch(dirs_list, dir, BlackListCompare); return search_result != NULL; diff --git a/src/help.c b/src/help.c index 47a167c44..4aea75eaa 100644 --- a/src/help.c +++ b/src/help.c @@ -119,13 +119,13 @@ help_pg_probackup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [--extra-directory=extra-directory-path]\n")); + printf(_(" [--external-dirs=external-directory-path]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); - printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n")); + printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); @@ -210,7 +210,7 @@ help_backup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [-E extra-directory=extra-directory-path]\n\n")); + printf(_(" [-E external-dirs=external-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -223,7 +223,7 @@ help_backup(void) printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directory not from pgdata \n")); printf(_("\n Logging options:\n")); @@ -286,7 +286,7 @@ help_restore(void) printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); - printf(_(" [--extra-mapping=OLDDIR=NEWDIR]\n")); + printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--no-validate]\n")); @@ -306,8 +306,8 @@ help_restore(void) printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); - printf(_(" --extra-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the extra directory from OLDDIR to NEWDIR\n")); + printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); printf(_(" --recovery-target-name=target-name\n")); @@ -498,11 +498,11 @@ help_set_config(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [-E extra-directory=extra-directory-path]\n\n")); + printf(_(" [-E external-dirs=external-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directory not from pgdata \n")); printf(_("\n Logging options:\n")); @@ -570,12 +570,12 @@ help_add_instance(void) { printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); - printf(_(" -E extra-directory=extra-directory-path\n\n")); + printf(_(" -E external-dirs=external-directory-path\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --instance=instance_name name of the new instance\n")); - printf(_(" -E --extra-directory=extra-directory-path\n")); + printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directory not from pgdata \n")); } diff --git a/src/merge.c b/src/merge.c index 4b74d0409..b0906a5c6 100644 --- a/src/merge.c +++ b/src/merge.c @@ -18,14 +18,14 @@ typedef struct { parray *to_files; parray *files; - parray *from_extra; + parray *from_external; pgBackup *to_backup; pgBackup *from_backup; const char *to_root; const char *from_root; - const char *to_extra_prefix; - const char *from_extra_prefix; + const char *to_external_prefix; + const char *from_external_prefix; /* * Return value from the thread. @@ -37,9 +37,10 @@ typedef struct static void merge_backups(pgBackup *backup, pgBackup *next_backup); static void *merge_files(void *arg); static void -reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra); +reorder_external_dirs(pgBackup *to_backup, parray *to_external, + parray *from_external); static int -get_extra_index(const char *key, const parray *list); +get_external_index(const char *key, const parray *list); /* * Implementation of MERGE command. @@ -165,15 +166,15 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) *from_backup_id = base36enc_dup(from_backup->start_time); char to_backup_path[MAXPGPATH], to_database_path[MAXPGPATH], - to_extra_prefix[MAXPGPATH], + to_external_prefix[MAXPGPATH], from_backup_path[MAXPGPATH], from_database_path[MAXPGPATH], - from_extra_prefix[MAXPGPATH], + from_external_prefix[MAXPGPATH], control_file[MAXPGPATH]; parray *files, *to_files; - parray *to_extra = NULL, - *from_extra = NULL; + parray *to_external = NULL, + *from_external = NULL; pthread_t *threads = NULL; merge_files_arg *threads_args = NULL; int i; @@ -209,13 +210,13 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), DATABASE_DIR); - pgBackupGetPath(to_backup, to_extra_prefix, lengthof(to_database_path), - EXTRA_DIR); + pgBackupGetPath(to_backup, to_external_prefix, lengthof(to_database_path), + EXTERNAL_DIR); pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), DATABASE_DIR); - pgBackupGetPath(from_backup, from_extra_prefix, lengthof(from_database_path), - EXTRA_DIR); + pgBackupGetPath(from_backup, from_external_prefix, lengthof(from_database_path), + EXTERNAL_DIR); /* * Get list of files which will be modified or removed. @@ -252,32 +253,32 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); - /* Create extra directories lists */ - if (to_backup->extra_dir_str) - to_extra = make_extra_directory_list(to_backup->extra_dir_str); - if (from_backup->extra_dir_str) - from_extra = make_extra_directory_list(from_backup->extra_dir_str); + /* Create external directories lists */ + if (to_backup->external_dir_str) + to_external = make_external_directory_list(to_backup->external_dir_str); + if (from_backup->external_dir_str) + from_external = make_external_directory_list(from_backup->external_dir_str); /* - * Rename extra directoties in to_backup (if exists) - * according to numeration of extra dirs in from_backup. + * Rename external directoties in to_backup (if exists) + * according to numeration of external dirs in from_backup. */ - if (to_extra) - reorder_extra_dirs(to_backup, to_extra, from_extra); + if (to_external) + reorder_external_dirs(to_backup, to_external, from_external); /* Setup threads */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - /* if the entry was an extra directory, create it in the backup */ - if (file->extra_dir_num && S_ISDIR(file->mode)) + /* if the entry was an external directory, create it in the backup */ + if (file->external_dir_num && S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; char new_container[MAXPGPATH]; - makeExtraDirPathByNum(new_container, to_extra_prefix, - file->extra_dir_num); + makeExternalDirPathByNum(new_container, to_external_prefix, + file->external_dir_num); join_path_components(dirpath, new_container, file->path); dir_create_dir(dirpath, DIR_PERMISSION); } @@ -294,9 +295,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) arg->from_backup = from_backup; arg->to_root = to_database_path; arg->from_root = from_database_path; - arg->from_extra = from_extra; - arg->to_extra_prefix = to_extra_prefix; - arg->from_extra_prefix = from_extra_prefix; + arg->from_external = from_external; + arg->to_external_prefix = to_external_prefix; + arg->from_external_prefix = from_external_prefix; /* By default there are some error */ arg->ret = 1; @@ -325,9 +326,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; - pfree(to_backup->extra_dir_str); - to_backup->extra_dir_str = from_backup->extra_dir_str; - from_backup->extra_dir_str = NULL; /* For safe pgBackupFree() */ + pfree(to_backup->external_dir_str); + to_backup->external_dir_str = from_backup->external_dir_str; + from_backup->external_dir_str = NULL; /* For safe pgBackupFree() */ /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. @@ -354,7 +355,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->wal_bytes = BYTES_INVALID; write_backup_filelist(to_backup, files, from_database_path, - from_extra_prefix, NULL); + from_external_prefix, NULL); write_backup(to_backup); delete_source_backup: @@ -371,10 +372,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); + char *dir_name = parray_get(to_external, file->external_dir_num - 1); - if (file->extra_dir_num && - backup_contains_extra(parray_get(to_extra, file->extra_dir_num - 1), - from_extra)) + if (file->external_dir_num && + backup_contains_external(dir_name, from_external)) /* Dir already removed*/ continue; @@ -490,11 +491,11 @@ merge_files(void *arg) } /* We need to make full path, file object has relative path */ - if (file->extra_dir_num) + if (file->external_dir_num) { char temp[MAXPGPATH]; - makeExtraDirPathByNum(temp, argument->from_extra_prefix, - file->extra_dir_num); + makeExternalDirPathByNum(temp, argument->from_external_prefix, + file->external_dir_num); join_path_components(from_file_path, temp, file->path); } @@ -612,21 +613,21 @@ merge_files(void *arg) } else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(argument->from_root, argument->to_root, file); - else if (file->extra_dir_num) + else if (file->external_dir_num) { char from_root[MAXPGPATH]; char to_root[MAXPGPATH]; int new_dir_num; - char *file_extra_path = parray_get(argument->from_extra, - file->extra_dir_num - 1); - - Assert(argument->from_extra); - new_dir_num = get_extra_index(file_extra_path, - argument->from_extra); - makeExtraDirPathByNum(from_root, argument->from_extra_prefix, - file->extra_dir_num); - makeExtraDirPathByNum(to_root, argument->to_extra_prefix, - new_dir_num); + char *file_external_path = parray_get(argument->from_external, + file->external_dir_num - 1); + + Assert(argument->from_external); + new_dir_num = get_external_index(file_external_path, + argument->from_external); + makeExternalDirPathByNum(from_root, argument->from_external_prefix, + file->external_dir_num); + makeExternalDirPathByNum(to_root, argument->to_external_prefix, + new_dir_num); copy_file(from_root, to_root, file); } else @@ -668,9 +669,9 @@ remove_dir_with_files(const char *path) } } -/* Get index of extra directory */ +/* Get index of external directory */ static int -get_extra_index(const char *key, const parray *list) +get_external_index(const char *key, const parray *list) { if (!list) /* Nowhere to search */ return -1; @@ -682,29 +683,31 @@ get_extra_index(const char *key, const parray *list) return -1; } -/* Rename directories in to_backup according to order in from_extra */ +/* Rename directories in to_backup according to order in from_external */ static void -reorder_extra_dirs(pgBackup *to_backup, parray *to_extra, parray *from_extra) +reorder_external_dirs(pgBackup *to_backup, parray *to_external, + parray *from_external) { - char extradir_template[MAXPGPATH]; + char externaldir_template[MAXPGPATH]; - pgBackupGetPath(to_backup, extradir_template, - lengthof(extradir_template), EXTRA_DIR); - for (int i = 0; i < parray_num(to_extra); i++) + pgBackupGetPath(to_backup, externaldir_template, + lengthof(externaldir_template), EXTERNAL_DIR); + for (int i = 0; i < parray_num(to_external); i++) { - int from_num = get_extra_index(parray_get(to_extra, i), from_extra); + int from_num = get_external_index(parray_get(to_external, i), + from_external); if (from_num == -1) { char old_path[MAXPGPATH]; - makeExtraDirPathByNum(old_path, extradir_template, i + 1); + makeExternalDirPathByNum(old_path, externaldir_template, i + 1); remove_dir_with_files(old_path); } else if (from_num != i + 1) { char old_path[MAXPGPATH]; char new_path[MAXPGPATH]; - makeExtraDirPathByNum(old_path, extradir_template, i + 1); - makeExtraDirPathByNum(new_path, extradir_template, from_num); + makeExternalDirPathByNum(old_path, externaldir_template, i + 1); + makeExternalDirPathByNum(new_path, externaldir_template, from_num); elog(VERBOSE, "Rename %s to %s", old_path, new_path); if (rename (old_path, new_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index aebbfc438..d5b51df3f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -54,8 +54,8 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; -/* colon separated extra directories list ("/path1:/path2") */ -char *extradir = NULL; +/* colon separated external directories list ("/path1:/path2") */ +char *externaldir = NULL; /* common options */ static char *backup_id_string = NULL; int num_threads = 1; @@ -145,7 +145,7 @@ static ConfigOption cmd_options[] = { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, - { 'f', 155, "extra-mapping", opt_extradir_map, SOURCE_CMD_STRICT }, + { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e2984755e..ac81cd626 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -48,7 +48,7 @@ #define PG_BACKUP_LABEL_FILE "backup_label" #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" -#define EXTRA_DIR "extra_directories/extradir" +#define EXTERNAL_DIR "external_directories/externaldir" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 @@ -122,7 +122,7 @@ typedef struct pgFile int n_blocks; /* size of the file in blocks, readed during DELTA backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; - int extra_dir_num; /* Number of extra directory. 0 if not extra */ + int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ @@ -180,7 +180,7 @@ typedef struct InstanceConfig uint32 xlog_seg_size; char *pgdata; - char *extra_dir_str; + char *external_dir_str; const char *pgdatabase; const char *pghost; const char *pgport; @@ -261,7 +261,8 @@ struct pgBackup pgBackup *parent_backup_link; char *primary_conninfo; /* Connection parameters of the backup * in the format suitable for recovery.conf */ - char *extra_dir_str; /* List of extra directories, separated by ':' */ + char *external_dir_str; /* List of external directories, + * separated by ':' */ }; /* Recovery target for restore and validate subcommands */ @@ -292,11 +293,11 @@ typedef struct { const char *from_root; const char *to_root; - const char *extra_prefix; + const char *external_prefix; parray *files_list; parray *prev_filelist; - parray *extra_dirs; + parray *external_dirs; XLogRecPtr prev_start_lsn; PGconn *backup_conn; @@ -474,8 +475,8 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern void catalog_lock(void); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, const char *extra_prefix, - parray *extra_list); + const char *root, const char *external_prefix, + parray *external_list); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); @@ -500,26 +501,27 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root, int extra_dir_num); + bool omit_symlink, bool add_root, int external_dir_num); extern void create_data_directories(const char *data_dir, const char *backup_dir, bool extract_tablespaces); extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); -extern void opt_extradir_map(ConfigOption *opt, const char *arg); +extern void opt_externaldir_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup); -extern void check_extra_dir_mapping(pgBackup *backup); -extern char *get_extra_remap(char *current_dir); +extern void check_external_dir_mapping(pgBackup *backup); +extern char *get_external_remap(char *current_dir); extern void print_file_list(FILE *out, const parray *files, const char *root, - const char *extra_prefix, parray *extra_list); -extern parray *dir_read_file_list(const char *root, const char *extra_prefix, const char *file_txt); -extern parray *make_extra_directory_list(const char *colon_separated_dirs); + const char *external_prefix, parray *external_list); +extern parray *dir_read_file_list(const char *root, const char *external_prefix, + const char *file_txt); +extern parray *make_external_directory_list(const char *colon_separated_dirs); extern void free_dir_list(parray *list); -extern void makeExtraDirPathByNum(char *ret_path, const char *pattern_path, - const int dir_num); -extern bool backup_contains_extra(const char *dir, parray *dirs_list); +extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, + const int dir_num); +extern bool backup_contains_external(const char *dir, parray *dirs_list); extern int dir_create_dir(const char *path, mode_t mode); extern bool dir_is_empty(const char *path); @@ -527,14 +529,14 @@ extern bool dir_is_empty(const char *path); extern bool fileExists(const char *path); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink, int extra_dir_num); +extern pgFile *pgFileNew(const char *path, bool omit_symlink, int external_dir_num); extern pgFile *pgFileInit(const char *path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read); extern int pgFileComparePath(const void *f1, const void *f2); -extern int pgFileComparePathWithExtra(const void *f1, const void *f2); +extern int pgFileComparePathWithExternal(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); diff --git a/src/restore.c b/src/restore.c index 0978042fe..834ff0b4c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,9 +21,9 @@ typedef struct { parray *files; pgBackup *backup; - parray *req_extra_dirs; - parray *cur_extra_dirs; - char *extra_prefix; + parray *req_external_dirs; + parray *cur_external_dirs; + char *external_prefix; /* * Return value from the thread. @@ -32,7 +32,7 @@ typedef struct int ret; } restore_files_arg; -static void restore_backup(pgBackup *backup, const char *extra_dir_str); +static void restore_backup(pgBackup *backup, const char *external_dir_str); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); @@ -287,7 +287,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore) { check_tablespace_mapping(dest_backup); - check_extra_dir_mapping(dest_backup); + check_external_dir_mapping(dest_backup); } if (!is_restore || !rt->restore_no_validate) @@ -393,7 +393,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", base36enc(dest_backup->start_time), dest_backup->server_version); - restore_backup(backup, dest_backup->extra_dir_str); + restore_backup(backup, dest_backup->external_dir_str); } /* @@ -420,16 +420,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup, const char *extra_dir_str) +restore_backup(pgBackup *backup, const char *external_dir_str) { char timestamp[100]; char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; - char extra_prefix[MAXPGPATH]; + char external_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *requested_extra_dirs = NULL; - parray *current_extra_dirs = NULL; + parray *requested_external_dirs = NULL; + parray *current_external_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -461,61 +461,63 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); create_data_directories(instance_config.pgdata, this_backup_path, true); - if(extra_dir_str) + if(external_dir_str) { - requested_extra_dirs = make_extra_directory_list(extra_dir_str); - for (i = 0; i < parray_num(requested_extra_dirs); i++) + requested_external_dirs = make_external_directory_list(external_dir_str); + for (i = 0; i < parray_num(requested_external_dirs); i++) { - char *extra_path = parray_get(requested_extra_dirs, i); - extra_path = get_extra_remap(extra_path); - dir_create_dir(extra_path, DIR_PERMISSION); + char *external_path = parray_get(requested_external_dirs, i); + external_path = get_external_remap(external_path); + dir_create_dir(external_path, DIR_PERMISSION); } } - if(backup->extra_dir_str) - current_extra_dirs = make_extra_directory_list(backup->extra_dir_str); + if(backup->external_dir_str) + current_external_dirs = make_external_directory_list(backup->external_dir_str); /* * Get list of files which need to be restored. */ pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); + pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), + EXTERNAL_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, extra_prefix, list_path); + files = dir_read_file_list(database_path, external_prefix, list_path); /* Restore directories in do_backup_instance way */ parray_qsort(files, pgFileComparePath); /* - * Make extra directories before restore + * Make external directories before restore * and setup threads at the same time */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); - /* if the entry was an extra directory, create it in the backup */ - if (file->extra_dir_num && S_ISDIR(file->mode)) + /* if the entry was an external directory, create it in the backup */ + if (file->external_dir_num && S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; char *dir_name; - char *extra_path; + char *external_path; - if (!current_extra_dirs || - parray_num(current_extra_dirs) < file->extra_dir_num - 1) - elog(ERROR, "Inconsistent extra directory backup metadata"); + if (!current_external_dirs || + parray_num(current_external_dirs) < file->external_dir_num - 1) + elog(ERROR, "Inconsistent external directory backup metadata"); - extra_path = parray_get(current_extra_dirs, file->extra_dir_num - 1); - if (backup_contains_extra(extra_path, requested_extra_dirs)) + external_path = parray_get(current_external_dirs, + file->external_dir_num - 1); + if (backup_contains_external(external_path, requested_external_dirs)) { char container_dir[MAXPGPATH]; - extra_path = get_extra_remap(extra_path); - makeExtraDirPathByNum(container_dir, extra_prefix, - file->extra_dir_num); + external_path = get_external_remap(external_path); + makeExternalDirPathByNum(container_dir, external_prefix, + file->external_dir_num); dir_name = GetRelativePath(file->path, container_dir); elog(VERBOSE, "Create directory \"%s\"", dir_name); - join_path_components(dirpath, extra_path, dir_name); + join_path_components(dirpath, external_path, dir_name); dir_create_dir(dirpath, DIR_PERMISSION); } } @@ -533,9 +535,9 @@ restore_backup(pgBackup *backup, const char *extra_dir_str) arg->files = files; arg->backup = backup; - arg->req_extra_dirs = requested_extra_dirs; - arg->cur_extra_dirs = current_extra_dirs; - arg->extra_prefix = extra_prefix; + arg->req_external_dirs = requested_external_dirs; + arg->cur_external_dirs = current_external_dirs; + arg->external_prefix = external_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -578,13 +580,13 @@ remove_deleted_files(pgBackup *backup) parray *files; parray *files_restored; char filelist_path[MAXPGPATH]; - char extra_prefix[MAXPGPATH]; + char external_prefix[MAXPGPATH]; int i; pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); - pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); + pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, extra_prefix, filelist_path); + files = dir_read_file_list(instance_config.pgdata, external_prefix, filelist_path); parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ @@ -693,14 +695,15 @@ restore_files(void *arg) false, parse_program_version(arguments->backup->program_version)); } - else if (file->extra_dir_num) + else if (file->external_dir_num) { - char *extra_path = parray_get(arguments->cur_extra_dirs, - file->extra_dir_num - 1); - if (backup_contains_extra(extra_path, arguments->req_extra_dirs)) + char *external_path = parray_get(arguments->cur_external_dirs, + file->external_dir_num - 1); + if (backup_contains_external(external_path, + arguments->req_external_dirs)) { - extra_path = get_extra_remap(extra_path); - copy_file(arguments->extra_prefix, extra_path, file); + external_path = get_external_remap(external_path); + copy_file(arguments->external_prefix, external_path, file); } } else if (strcmp(file->name, "pg_control") == 0) diff --git a/src/validate.c b/src/validate.c index 709bdd125..0ce2d7081 100644 --- a/src/validate.c +++ b/src/validate.c @@ -43,7 +43,7 @@ void pgBackupValidate(pgBackup *backup) { char base_path[MAXPGPATH]; - char extra_prefix[MAXPGPATH]; + char external_prefix[MAXPGPATH]; char path[MAXPGPATH]; parray *files; bool corrupted = false; @@ -90,9 +90,9 @@ pgBackupValidate(pgBackup *backup) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, extra_prefix, lengthof(extra_prefix), EXTRA_DIR); + pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, extra_prefix, path); + files = dir_read_file_list(base_path, external_prefix, path); /* setup threads */ for (i = 0; i < parray_num(files); i++) @@ -240,7 +240,7 @@ pgBackupValidateFiles(void *arg) */ if (arguments->backup_version >= 20025 && strcmp(file->name, "pg_control") == 0 && - !file->extra_dir_num) + !file->external_dir_num) crc = get_pgcontrol_checksum(arguments->base_path); else crc = pgFileGetCRC(file->path, diff --git a/tests/external.py b/tests/external.py index 582402e69..17c89977b 100644 --- a/tests/external.py +++ b/tests/external.py @@ -29,13 +29,13 @@ def test_external_simple(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') external_dir = os.path.join(node.base_dir, 'somedirectory') - # copy postgresql.conf to extra_directory + # copy postgresql.conf to external_directory os.mkdir(external_dir) shutil.copyfile( os.path.join(node.data_dir, 'postgresql.conf'), os.path.join(external_dir, 'postgresql.conf')) - # create directory in extra_directory + # create directory in external_directory self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -44,7 +44,7 @@ def test_external_simple(self): backup_id = self.backup_node( backup_dir, 'node', node, options=[ - '--extra-directory={0}'.format(external_dir)]) + '--external-dirs={0}'.format(external_dir)]) if self.paranoia: pgdata = self.pgdata_content( @@ -93,7 +93,7 @@ def test_external_dir_mapping(self): external_dir1_old = self.get_tblspace_path(node, 'external_dir1') external_dir2_old = self.get_tblspace_path(node, 'external_dir2') - # copy postgresql.conf to extra_directory + # copy postgresql.conf to external_directory os.mkdir(external_dir1_old) os.mkdir(external_dir2_old) shutil.copyfile( @@ -115,9 +115,9 @@ def test_external_dir_mapping(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "--extra-mapping={0}={1}".format( + "--external-mapping={0}={1}".format( external_dir1_old, external_dir1_new), - "--extra-mapping={0}={1}".format( + "--external-mapping={0}={1}".format( external_dir2_old, external_dir2_new)]) # we should die here because exception is what we expect to happen self.assertEqual( @@ -127,8 +127,8 @@ def test_external_dir_mapping(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: --extra-mapping option' in e.message and - 'have an entry in list of extra directories' in e.message, + 'ERROR: --external-mapping option' in e.message and + 'have an entry in list of external directories' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -142,9 +142,9 @@ def test_external_dir_mapping(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "--extra-mapping={0}={1}".format( + "--external-mapping={0}={1}".format( external_dir1_old, external_dir1_new), - "--extra-mapping={0}={1}".format( + "--external-mapping={0}={1}".format( external_dir2_old, external_dir2_new)]) if self.paranoia: @@ -223,14 +223,14 @@ def test_backup_multiple_external(self): # Clean after yourself self.del_test_dir(module_name, fname) - # extra directory contain symlink to file - # extra directory contain symlink to directory - # latest page backup without extra_dir + # external directory contain symlink to file + # external directory contain symlink to directory + # latest page backup without external_dir # multiple external directories - # --extra-directory=none - # --extra-directory point to a file - # extra directory in config and in command line - # extra directory contain multuple directories, some of them my be empty + # --external-dirs=none + # --external-dirs point to a file + # external directory in config and in command line + # external directory contain multuple directories, some of them my be empty # forbid to external-dirs to point to tablespace directories # check that not changed files are not copied by next backup # merge From 841e608fa7d35600621d3a069df1dc28943f6b49 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Feb 2019 19:52:52 +0300 Subject: [PATCH 0293/2107] PGPRO-2068: tests for block from future --- tests/helpers/ptrack_helpers.py | 8 +- tests/pgpro2068.py | 181 ++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 tests/pgpro2068.py diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 86d09032f..48618c24e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1321,12 +1321,16 @@ def __init__(self, cmd, verbose, attach=False): break def set_breakpoint(self, location): + result = self._execute('break ' + location) for line in result: if line.startswith('~"Breakpoint'): return - elif line.startswith('^error') or line.startswith('(gdb)'): + elif line.startswith('=breakpoint-created'): + return + + elif line.startswith('^error'): #or line.startswith('(gdb)'): break elif line.startswith('&"break'): @@ -1430,7 +1434,7 @@ def _execute(self, cmd, running=True): output += [line] if self.verbose: print(repr(line)) - if line == '^done\n' or line.startswith('*stopped'): + if line.startswith('^done') or line.startswith('*stopped'): break if running and line.startswith('*running'): break diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py new file mode 100644 index 000000000..b40a3d435 --- /dev/null +++ b/tests/pgpro2068.py @@ -0,0 +1,181 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from time import sleep +import shutil +import signal + + +module_name = '2068' + + +class BugTest(ProbackupTest, unittest.TestCase): + + def test_minrecpoint_on_replica(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-2068 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '60min', + 'checkpoint_completion_target': '0.9', + 'bgwriter_delay': '10ms', + 'bgwriter_lru_maxpages': '2000', + 'bgwriter_lru_multiplier': '4.0', + 'max_wal_size': '100GB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take full backup and restore it as replica + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'node', replica, options=['-R']) + self.set_replica(node, replica) + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + replica.append_conf( + 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + replica.append_conf( + 'postgresql.auto.conf', 'restart_after_crash = off') + + node.safe_psql( + "postgres", + "CREATE EXTENSION plpythonu") + + node.safe_psql( + "postgres", + "CREATE EXTENSION pageinspect") + + + # pg_last_wal_replay_lsn + replica.slow_start(replica=True) + + + node.pgbench_init(scale=10) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "30"] + ) + + # select pid from pg_stat_activity where backend_type in ('walreceiver', 'checkpointer', 'background writer', 'startup') ; + startup_pid = replica.safe_psql( + 'postgres', + "select pid from pg_stat_activity where backend_type = 'startup'").rstrip() + + checkpointer_pid = replica.safe_psql( + 'postgres', + "select pid from pg_stat_activity where backend_type = 'checkpointer'").rstrip() + + bgwriter_pid = replica.safe_psql( + 'postgres', + "select pid from pg_stat_activity where backend_type = 'background writer'").rstrip() + + sleep(5) + + # startup process + # checkpointer + # writer process + + # block checkpointer on UpdateLastRemovedPtr + gdb_checkpointer = self.gdb_attach(checkpointer_pid) + gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') + gdb_checkpointer.continue_execution_until_break() + + # block recovery in on UpdateMinRecoveryPoint + gdb_recovery = self.gdb_attach(startup_pid) + gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') + gdb_recovery.continue_execution_until_break() + gdb_recovery.set_breakpoint('UpdateControlFile') + gdb_recovery.continue_execution_until_break() + + # stop bgwriter + # gdb_bgwriter = self.gdb_attach(bgwriter_pid) + + pgbench.wait() + pgbench.stdout.close() + + os.kill(int(bgwriter_pid), 9) + gdb_recovery._execute('detach') + gdb_checkpointer._execute('detach') + + try: + replica.stop(['-m', 'immediate', '-D', replica.data_dir]) + except: + pass + + # Promote replica with 'immediate' target action + replica.append_conf( + 'recovery.conf', "recovery_target = 'immediate'") + + replica.append_conf( + 'recovery.conf', "recovery_target_action = 'promote'") + + #os.remove(os.path.join(replica.data_dir, 'postmaster.pid')) + + # sleep(5) + replica.slow_start() + + script = ''' +DO +$$ +relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") +current_xlog_lsn = plpy.execute("select pg_last_wal_replay_lsn() as lsn")[0]['lsn'] +plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) +found_corruption = False +for relation in relations: + pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) + + if pages_from_future.nrows() == 0: + continue + + for page in pages_from_future: + plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) + found_corruption = True +if found_corruption: + plpy.error('Found Corruption') +$$ LANGUAGE plpythonu; +''' + + replica.safe_psql( + 'postgres', + script) + + # error is expected if version < 10.6 + # gdb_backup.continue_execution_until_exit() + + # do basebackup + + # do pg_probackup, expect error + + # Clean after yourself + self.del_test_dir(module_name, fname) From 8bbf1f44c6514e5e01cd7f704abc69e13f0c35b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Feb 2019 20:03:44 +0300 Subject: [PATCH 0294/2107] tests: minor fixes and comments for test_minrecpoint_on_replica() --- tests/pgpro2068.py | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index b40a3d435..82762d33f 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -16,9 +16,6 @@ class BugTest(ProbackupTest, unittest.TestCase): def test_minrecpoint_on_replica(self): """ https://jira.postgrespro.ru/browse/PGPRO-2068 - make node without archive support, make backup which should fail - check that backup status equal to ERROR - check that no files where copied to backup catalogue """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -43,6 +40,7 @@ def test_minrecpoint_on_replica(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) + # start replica replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() @@ -57,6 +55,7 @@ def test_minrecpoint_on_replica(self): replica.append_conf( 'postgresql.auto.conf', 'restart_after_crash = off') + # we need those later node.safe_psql( "postgres", "CREATE EXTENSION plpythonu") @@ -65,29 +64,25 @@ def test_minrecpoint_on_replica(self): "postgres", "CREATE EXTENSION pageinspect") - - # pg_last_wal_replay_lsn replica.slow_start(replica=True) - + # generate some data node.pgbench_init(scale=10) - pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) + options=["-c", "4", "-T", "20"]) pgbench.wait() pgbench.stdout.close() + # generate some more data and leave it in background pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "30"] - ) + options=["-c", "4", "-T", "30"]) - # select pid from pg_stat_activity where backend_type in ('walreceiver', 'checkpointer', 'background writer', 'startup') ; + # get pids of background workers startup_pid = replica.safe_psql( 'postgres', "select pid from pg_stat_activity where backend_type = 'startup'").rstrip() @@ -100,34 +95,31 @@ def test_minrecpoint_on_replica(self): 'postgres', "select pid from pg_stat_activity where backend_type = 'background writer'").rstrip() + # wait for shared_buffer to be filled with dirty data sleep(5) - # startup process - # checkpointer - # writer process - - # block checkpointer on UpdateLastRemovedPtr + # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') gdb_checkpointer.continue_execution_until_break() - # block recovery in on UpdateMinRecoveryPoint + # break recovery on UpdateControlFile gdb_recovery = self.gdb_attach(startup_pid) gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') gdb_recovery.continue_execution_until_break() gdb_recovery.set_breakpoint('UpdateControlFile') gdb_recovery.continue_execution_until_break() - # stop bgwriter - # gdb_bgwriter = self.gdb_attach(bgwriter_pid) - + # stop data generation pgbench.wait() pgbench.stdout.close() + # kill someone, we need a crash os.kill(int(bgwriter_pid), 9) gdb_recovery._execute('detach') gdb_checkpointer._execute('detach') + # just to be sure try: replica.stop(['-m', 'immediate', '-D', replica.data_dir]) except: @@ -136,13 +128,8 @@ def test_minrecpoint_on_replica(self): # Promote replica with 'immediate' target action replica.append_conf( 'recovery.conf', "recovery_target = 'immediate'") - replica.append_conf( 'recovery.conf', "recovery_target_action = 'promote'") - - #os.remove(os.path.join(replica.data_dir, 'postmaster.pid')) - - # sleep(5) replica.slow_start() script = ''' @@ -166,6 +153,7 @@ def test_minrecpoint_on_replica(self): $$ LANGUAGE plpythonu; ''' + # Find blocks from future replica.safe_psql( 'postgres', script) From fb5046343f41e851aafd92bda233fbc4900ef860 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Feb 2019 20:10:04 +0300 Subject: [PATCH 0295/2107] tests: in test_minrecpoint_on_replica() wait a bit longer for shared buffer to fill in with dirty data --- tests/pgpro2068.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 82762d33f..74655209f 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -95,8 +95,8 @@ def test_minrecpoint_on_replica(self): 'postgres', "select pid from pg_stat_activity where backend_type = 'background writer'").rstrip() - # wait for shared_buffer to be filled with dirty data - sleep(5) + # wait for shared buffer on replica to be filled with dirty data + sleep(10) # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) From e225bb830b1be52dc5cc4c0a7d38e22c1ef60a21 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 Feb 2019 11:42:48 +0300 Subject: [PATCH 0296/2107] PGPRO-421 tests: test_external_backward_compatibility() added, minor fixes --- tests/external.py | 107 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 17 deletions(-) diff --git a/tests/external.py b/tests/external.py index 17c89977b..7bd15aa51 100644 --- a/tests/external.py +++ b/tests/external.py @@ -138,6 +138,9 @@ def test_external_dir_mapping(self): "-j", "4", "--stream", "-E", "{0}:{1}".format(external_dir1_old, external_dir2_old)]) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.restore_node( backup_dir, 'node', node_restored, options=[ @@ -147,14 +150,9 @@ def test_external_dir_mapping(self): "--external-mapping={0}={1}".format( external_dir2_old, external_dir2_new)]) - if self.paranoia: - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -205,32 +203,107 @@ def test_backup_multiple_external(self): "-j", "4", "--stream", "-E", "{0}".format(external_dir2_old)]) - if self.paranoia: - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs', 'external_dir1']) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir1']) node.cleanup() + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility(self): + """Description in jira issue PGPRO-434""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + # delta backup with external directories using new binary + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE chain with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + # external directory contain symlink to file # external directory contain symlink to directory # latest page backup without external_dir # multiple external directories # --external-dirs=none # --external-dirs point to a file - # external directory in config and in command line - # external directory contain multuple directories, some of them my be empty + # external directory in config and in command line + + # external directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories # check that not changed files are not copied by next backup # merge From 92788fe4b3fa69dfb288a417669966928d34d887 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Wed, 13 Feb 2019 13:07:24 +0300 Subject: [PATCH 0297/2107] Use palloc0() instead of palloc() when working with external directories --- src/catalog.c | 2 +- src/dir.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 2ed208093..3b1b6c1af 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -423,7 +423,7 @@ pgBackupCreateDir(pgBackup *backup) for (int i = 0; i < parray_num(external_list); i++) { /* 20 chars is enough to hold the externaldir number in string. */ - temp = palloc(strlen(EXTERNAL_DIR) + 20); + temp = palloc0(strlen(EXTERNAL_DIR) + 20); /* Numeration of externaldirs starts with 1 */ makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); parray_append(subdirs, temp); diff --git a/src/dir.c b/src/dir.c index aa6e80b8c..11bfc5441 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1674,13 +1674,13 @@ make_external_directory_list(const char *colon_separated_dirs) { char *p; parray *list = parray_new(); - char *tmp = palloc(strlen(colon_separated_dirs) + 1); + char *tmp = palloc0(strlen(colon_separated_dirs) + 1); strcpy(tmp, colon_separated_dirs); p = strtok(tmp,":"); while(p!=NULL) { - char * dir = (char *)palloc(strlen(p) + 1); + char * dir = (char *)palloc0(strlen(p) + 1); strcpy(dir,p); if (is_absolute_path(dir)) parray_append(list, dir); From 1be24f3e7ca4913892fbfc9518a5f45220613caa Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Wed, 13 Feb 2019 17:39:32 +0300 Subject: [PATCH 0298/2107] Fix creating "database" directory in backup --- src/catalog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/catalog.c b/src/catalog.c index 3b1b6c1af..d45e2da3e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -412,6 +412,7 @@ pgBackupCreateDir(pgBackup *backup) char *temp; temp = palloc(strlen(DATABASE_DIR) + 1); + sprintf(temp, "%s", DATABASE_DIR); parray_append(subdirs, temp); /* Add external dirs containers */ From f911eead63b4637dc40f4ed623d8ff4c708ca142 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 Feb 2019 14:12:55 +0300 Subject: [PATCH 0299/2107] tests: test_external_none() added --- tests/external.py | 90 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/tests/external.py b/tests/external.py index 7bd15aa51..7de4e1abd 100644 --- a/tests/external.py +++ b/tests/external.py @@ -27,13 +27,7 @@ def test_external_simple(self): 'wal_level': 'replica'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - external_dir = os.path.join(node.base_dir, 'somedirectory') - - # copy postgresql.conf to external_directory - os.mkdir(external_dir) - shutil.copyfile( - os.path.join(node.data_dir, 'postgresql.conf'), - os.path.join(external_dir, 'postgresql.conf')) + external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory self.init_pb(backup_dir) @@ -41,7 +35,18 @@ def test_external_simple(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - backup_id = self.backup_node( + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # Fill external directories + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir, options=["-j", "4"]) + + # Full backup with external dir + self.backup_node( backup_dir, 'node', node, options=[ '--external-dirs={0}'.format(external_dir)]) @@ -51,6 +56,7 @@ def test_external_simple(self): node.base_dir, exclude_dirs=['logs']) node.cleanup() + shutil.rmtree(external_dir, ignore_errors=True) self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) @@ -63,6 +69,68 @@ def test_external_simple(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_external_none(self): + """ + make node, create external directory, take backup + with external directory, take delta backup with --external-dirs=none, + restore delta backup, check that + external directory was not copied + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + external_dir = self.get_tblspace_path(node, 'somedirectory') + + # create directory in external_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # Fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir, options=["-j", "4"]) + + # Full backup with external dir + self.backup_node( + backup_dir, 'node', node, + options=[ + '--external-dirs={0}'.format(external_dir)]) + + # Delta backup without external directory + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=['--external-dirs=none']) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'somedirectory']) + + node.cleanup() + shutil.rmtree(external_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + # self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_external_dir_mapping(self): """ @@ -299,12 +367,12 @@ def test_external_backward_compatibility(self): # external directory contain symlink to file # external directory contain symlink to directory # latest page backup without external_dir - # multiple external directories - # --external-dirs=none + # multiple external directories + + # --external-dirs=none + # --external-dirs point to a file # external directory in config and in command line + # external directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories - # check that not changed files are not copied by next backup + # check that not changed files are not copied by next backup + # merge # complex merge From 8e5e62612b58b65166bc0b0ea21dacb341dc0400 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 Feb 2019 18:11:27 +0300 Subject: [PATCH 0300/2107] tests: minor fix for test_page_backup_with_corrupted_wal_segment() --- tests/page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/page.py b/tests/page.py index 388b0df99..67473ed30 100644 --- a/tests/page.py +++ b/tests/page.py @@ -776,7 +776,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.backup_node(backup_dir, 'node', node) # make some wals - node.pgbench_init(scale=3) + node.pgbench_init(scale=4) # delete last wal segment wals_dir = os.path.join(backup_dir, 'wal', 'node') From fb74027c425be4a6c7cf03b51efbc6886a1b1af2 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 15 Feb 2019 14:36:16 +0300 Subject: [PATCH 0301/2107] Rewrite backup method to use push inside segment --- src/data.c | 77 +++++++-------- src/pg_probackup.h | 18 ++++ src/utils/file.c | 231 ++++++++++++++++++++++++++++++++++++--------- src/utils/file.h | 22 +++-- 4 files changed, 248 insertions(+), 100 deletions(-) diff --git a/src/data.c b/src/data.c index 888a57ddd..4747289be 100644 --- a/src/data.c +++ b/src/data.c @@ -60,7 +60,7 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. */ -static int32 +int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg) { @@ -166,22 +166,8 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) return false; } -/* - * When copying datafiles to backup we validate and compress them block - * by block. Thus special header is required for each data block. - */ -typedef struct BackupPageHeader -{ - BlockNumber block; /* block number */ - int32 compressed_size; -} BackupPageHeader; - -/* Special value for compressed_size field */ -#define PageIsTruncated -2 -#define SkipCurrentPage -3 - /* Verify page's header */ -static bool +bool parse_page(Page page, XLogRecPtr *lsn) { PageHeader phdr = (PageHeader) page; @@ -210,15 +196,15 @@ parse_page(Page page, XLogRecPtr *lsn) */ static int read_page_from_file(pgFile *file, BlockNumber blknum, - FILE *in, Page page, XLogRecPtr *page_lsn, XLogRecPtr horizon_lsn) + FILE *in, Page page, XLogRecPtr *page_lsn) { off_t offset = blknum * BLCKSZ; ssize_t read_len = 0; /* read the block */ - read_len = fio_pread(in, page, offset, horizon_lsn); + read_len = fio_pread(in, page, offset); - if (read_len != BLCKSZ && read_len != sizeof(PageHeaderData)) + if (read_len != BLCKSZ) { /* The block could have been truncated. It is fine. */ if (read_len == 0) @@ -269,19 +255,11 @@ read_page_from_file(pgFile *file, BlockNumber blknum, if (current.checksum_version) { BlockNumber blkno = file->segno * RELSEG_SIZE + blknum; - uint16 page_crc = read_len == BLCKSZ - ? pg_checksum_page(page, blkno) - /* - * Recompute Cpage checksum calculated by agent with blkno=0 - * pg_checksum_page is calculating it in this way: - * (((checksum ^ blkno) % 65535) + 1) - */ - : (uint16)(((*PAGE_CHECKSUM(page) - 1) ^ blkno) + 1); /* * If checksum is wrong, sleep a bit and then try again * several times. If it didn't help, throw error */ - if (page_crc != ((PageHeader) page)->pd_checksum) + if (pg_checksum_page(page, blkno) != ((PageHeader) page)->pd_checksum) { elog(WARNING, "File: %s blknum %u have wrong checksum, try again", file->path, blknum); @@ -314,7 +292,7 @@ static int32 prepare_page(backup_files_arg *arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, - FILE *in, int *n_skipped, + FILE *in, BlockNumber *n_skipped, BackupMode backup_mode, Page page) { @@ -337,11 +315,7 @@ prepare_page(backup_files_arg *arguments, { while(!page_is_valid && try_again) { - bool check_lsn = (backup_mode == BACKUP_MODE_DIFF_DELTA - && file->exists_in_prev - && !page_is_truncated); - int result = read_page_from_file(file, blknum, in, page, &page_lsn, - check_lsn ? prev_backup_start_lsn : InvalidXLogRecPtr); + int result = read_page_from_file(file, blknum, in, page, &page_lsn); try_again--; if (result == 0) @@ -533,8 +507,8 @@ backup_data_file(backup_files_arg* arguments, FILE *out; BlockNumber blknum = 0; BlockNumber nblocks = 0; - int n_blocks_skipped = 0; - int n_blocks_read = 0; + BlockNumber n_blocks_skipped = 0; + BlockNumber n_blocks_read = 0; int page_state; char curr_page[BLCKSZ]; @@ -614,16 +588,29 @@ backup_data_file(backup_files_arg* arguments, if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev) { - for (blknum = 0; blknum < nblocks; blknum++) + if (backup_mode != BACKUP_MODE_DIFF_PTRACK && fio_is_remote_file(in)) { - page_state = prepare_page(arguments, file, prev_backup_start_lsn, - blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page); - compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel); - n_blocks_read++; - if (page_state == PageIsTruncated) - break; + int rc = fio_send_pages(in, out, file, + backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + &n_blocks_skipped, calg, clevel); + if (rc < 0) + elog(ERROR, "Failed to read file %s: %s", + file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); + n_blocks_read = rc; + } + else + { + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, prev_backup_start_lsn, + blknum, nblocks, in, &n_blocks_skipped, + backup_mode, curr_page); + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel); + n_blocks_read++; + if (page_state == PageIsTruncated) + break; + } } if (backup_mode == BACKUP_MODE_DIFF_DELTA) file->n_blocks = n_blocks_read; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4d8b49706..3515cb8cb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -309,6 +309,21 @@ typedef struct int ret; } backup_files_arg; +/* + * When copying datafiles to backup we validate and compress them block + * by block. Thus special header is required for each data block. + */ +typedef struct BackupPageHeader +{ + BlockNumber block; /* block number */ + int32 compressed_size; +} BackupPageHeader; + +/* Special value for compressed_size field */ +#define PageIsTruncated -2 +#define SkipCurrentPage -3 + + /* * return pointer that exceeds the length of prefix from character string. * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". @@ -603,6 +618,9 @@ extern char *base36enc_dup(long unsigned int value); extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); +extern bool parse_page(Page page, XLogRecPtr *lsn); +int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level, const char **errormsg); #ifdef WIN32 #ifdef _DEBUG diff --git a/src/utils/file.c b/src/utils/file.c index efda75748..cd1c46667 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -4,24 +4,29 @@ #include #include "pg_probackup.h" -#include "storage/checksum.h" #include "file.h" +#include "storage/checksum.h" #define PRINTF_BUF_SIZE 1024 #define FILE_PERMISSIONS 0600 +#define PAGE_READ_ATTEMPTS 100 static __thread unsigned long fio_fdset = 0; static __thread void* fio_stdin_buffer; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; +fio_location MyLocation; + typedef struct { - fio_header hdr; - XLogRecPtr lsn; -} fio_pread_request; - -fio_location MyLocation; + BlockNumber nblocks; + BlockNumber segBlockNum; + XLogRecPtr horizonLsn; + uint32 checksumVersion; + int calg; + int clevel; +} fio_send_request; /* Convert FIO pseudo handle to index in file descriptor array */ @@ -34,12 +39,6 @@ void fio_redirect(int in, int out) fio_stdout = out; } -/* Check if FILE handle is local or remote (created by FIO) */ -static bool fio_is_remote_file(FILE* file) -{ - return (size_t)file <= FIO_FDMAX; -} - /* Check if file descriptor is local or remote (created by FIO) */ static bool fio_is_remote_fd(int fd) { @@ -388,42 +387,30 @@ int fio_truncate(int fd, off_t size) /* * Read file from specified location. - * This call is optimized for delat backup, to avoid trasfer of old pages to backup host. - * For delta backup horizon_lsn parameter is assigned value of last backup and for all pages with LSN smaller than horizon_lsn only page header is sent. */ -int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn) +int fio_pread(FILE* f, void* buf, off_t offs) { if (fio_is_remote_file(f)) { int fd = fio_fileno(f); - fio_pread_request req; fio_header hdr; - req.hdr.cop = FIO_PREAD; - req.hdr.handle = fd & ~FIO_PIPE_MARKER; - req.hdr.size = sizeof(XLogRecPtr); - req.hdr.arg = offs; - req.lsn = horizon_lsn; + hdr.cop = FIO_PREAD; + hdr.handle = fd & ~FIO_PIPE_MARKER; + hdr.size = sizeof(XLogRecPtr); + hdr.arg = offs; - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_SEND); if (hdr.size != 0) IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - /* - * We either return <0 for error, either 0 for EOF, either received size (page or page header size) - * for fully read page either 1 for partly read page. 1 is used just to distinguish it with page header size. - */ - return hdr.arg <= 0 ? hdr.arg : hdr.arg == BLCKSZ ? hdr.size : 1; + return hdr.arg; } else - { - int rc = pread(fileno(f), buf, BLCKSZ, offs); - /* See comment above consernign returned value */ - return rc <= 0 || rc == BLCKSZ ? rc : 1; - } + return pread(fileno(f), buf, BLCKSZ, offs); } /* Set position in stdio file */ @@ -804,6 +791,164 @@ static void fio_send_file(int out, char const* path) } } +int fio_send_pages(FILE* in, FILE* out, pgFile *file, + XLogRecPtr horizonLsn, BlockNumber* nBlocksSkipped, int calg, int clevel) +{ + struct { + fio_header hdr; + fio_send_request arg; + } req; + BlockNumber n_blocks_read = 0; + BlockNumber blknum = 0; + + Assert(fio_is_remote_file(in)); + + req.hdr.cop = FIO_SEND_PAGES; + req.hdr.size = sizeof(fio_send_request); + req.hdr.handle = fio_fileno(in) & ~FIO_PIPE_MARKER; + + req.arg.nblocks = file->size/BLCKSZ; + req.arg.segBlockNum = file->segno * RELSEG_SIZE; + req.arg.horizonLsn = horizonLsn; + req.arg.checksumVersion = current.checksum_version; + req.arg.calg = calg; + req.arg.clevel = clevel; + + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + while (true) + { + fio_header hdr; + char buf[BLCKSZ + sizeof(BackupPageHeader)]; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_PAGE); + + if (hdr.arg < 0) /* read error */ + return hdr.arg; + + blknum = hdr.arg; + if (hdr.size == 0) /* end of segment */ + break; + + Assert(hdr.size <= sizeof(buf)); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + + if (fio_fwrite(out, buf, hdr.size) != hdr.size) + { + int errno_tmp = errno; + fio_fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u: %s", + file->path, blknum, strerror(errno_tmp)); + } + file->compress_alg = calg; + file->read_size += BLCKSZ; + file->write_size += hdr.size; + n_blocks_read++; + } + *nBlocksSkipped = blknum - n_blocks_read; + return blknum; +} + +static void fio_send_pages_impl(int fd, int out, fio_send_request* req) +{ + BlockNumber blknum; + char read_buffer[BLCKSZ+1]; + fio_header hdr; + + hdr.cop = FIO_PAGE; + read_buffer[BLCKSZ] = 1; /* barrier */ + + for (blknum = 0; blknum < req->nblocks; blknum++) + { + int retry_attempts = PAGE_READ_ATTEMPTS; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + bool is_empty_page = false; + do + { + ssize_t rc = pread(fd, read_buffer, BLCKSZ, blknum*BLCKSZ); + + if (rc <= 0) + { + hdr.size = 0; + if (rc < 0) + { + hdr.arg = -errno; + Assert(hdr.arg < 0); + } + else + { + /* This is the last page */ + hdr.arg = blknum; + } + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return; + } + else if (rc == BLCKSZ) + { + if (!parse_page((Page)read_buffer, &page_lsn)) + { + int i; + for (i = 0; read_buffer[i] == 0; i++); + + /* Page is zeroed. No need to check header and checksum. */ + if (i == BLCKSZ) + { + is_empty_page = true; + break; + } + } + else if (!req->checksumVersion + || pg_checksum_page(read_buffer, req->segBlockNum + blknum) == ((PageHeader)read_buffer)->pd_checksum) + { + break; + } + } + } while (--retry_attempts != 0); + + if (retry_attempts == 0) + { + hdr.size = 0; + hdr.arg = PAGE_CHECKSUM_MISMATCH; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return; + } + /* horizonLsn is not 0 for delta backup. As far as unsigned number are always greater or equal than zero, there is no sense to add more checks */ + if (page_lsn >= req->horizonLsn) + { + char write_buffer[BLCKSZ*2]; + BackupPageHeader* bph = (BackupPageHeader*)write_buffer; + + hdr.arg = bph->block = blknum; + hdr.size = sizeof(BackupPageHeader); + + if (is_empty_page) + { + bph->compressed_size = PageIsTruncated; + } + else + { + const char *errormsg = NULL; + bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), sizeof(write_buffer) - sizeof(BackupPageHeader), + read_buffer, BLCKSZ, req->calg, req->clevel, + &errormsg); + if (bph->compressed_size <= 0 || bph->compressed_size >= BLCKSZ) + { + /* Do not compress page */ + memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); + bph->compressed_size = BLCKSZ; + } + hdr.size += bph->compressed_size; + } + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); + } + } + hdr.size = 0; + hdr.arg = blknum; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} /* Execute commands at remote host */ void fio_communicate(int in, int out) @@ -820,7 +965,6 @@ void fio_communicate(int in, int out) char* buf = (char*)malloc(buf_size); fio_header hdr; struct stat st; - XLogRecPtr horizon_lsn; int rc; /* Main loop until command of processing master command */ @@ -872,25 +1016,18 @@ void fio_communicate(int in, int out) rc = read(fd[hdr.handle], buf, hdr.arg); hdr.cop = FIO_SEND; hdr.size = rc > 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_PREAD: /* Read from specified position in file, ignoring pages beyond horizon of delta backup */ - horizon_lsn = *(XLogRecPtr*)buf; rc = pread(fd[hdr.handle], buf, BLCKSZ, hdr.arg); hdr.cop = FIO_SEND; hdr.arg = rc; - /* For pages beyond horizon of delta backup transfer only page header */ - hdr.size = (rc == BLCKSZ) - ? PageXLogRecPtrGet(((PageHeader)buf)->pd_lsn) < horizon_lsn /* For non-delta backup horizon_lsn == 0, so this condition is always false */ - ? sizeof(PageHeaderData) : BLCKSZ - : 0; - if (hdr.size == sizeof(PageHeaderData)) - /* calculate checksum without XOR-ing with block number to compare it with page CRC at master */ - *PAGE_CHECKSUM(buf) = pg_checksum_page(buf, 0); + hdr.size = rc >= 0 ? rc : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); + IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_FSTAT: /* Get information about opened file */ hdr.size = sizeof(st); @@ -927,6 +1064,10 @@ void fio_communicate(int in, int out) case FIO_TRUNCATE: /* Truncate file */ SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; + case FIO_SEND_PAGES: + Assert(hdr.size == sizeof(fio_send_request)); + fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index b79eaf3db..972cb76e4 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -30,7 +30,9 @@ typedef enum FIO_ACCESS, FIO_OPENDIR, FIO_READDIR, - FIO_CLOSEDIR + FIO_CLOSEDIR, + FIO_SEND_PAGES, + FIO_PAGE } fio_operations; typedef enum @@ -43,18 +45,11 @@ typedef enum #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 +#define PAGE_CHECKSUM_MISMATCH (-256) #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) #define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc < 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } } while (0) -/* - * Store one more checksum in page header. - * There is free space at the ned of page header (not used for page verification) - * While delta backup we need to calculate checksum at agent, send it to maini pg_probackup instance - * adjust it according to the real block number and compare with checksum stored in pd_checksum - */ -#define PAGE_CHECKSUM(p) ((uint16*)((p) + sizeof(PageHeaderData)) - 1) - typedef struct { unsigned cop : 5; @@ -65,13 +60,16 @@ typedef struct extern fio_location MyLocation; +/* Check if FILE handle is local or remote (created by FIO) */ +#define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX) + extern void fio_redirect(int in, int out); extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); -extern int fio_pread(FILE* f, void* buf, off_t offs, XLogRecPtr horizon_lsn); +extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); @@ -79,6 +77,10 @@ extern int fio_ftruncate(FILE* f, off_t size); extern int fio_fclose(FILE* f); extern int fio_ffstat(FILE* f, struct stat* st); +struct pgFile; +extern int fio_send_pages(FILE* in, FILE* out, struct pgFile *file, XLogRecPtr horizonLsn, + BlockNumber* nBlocksSkipped, int calg, int clevel); + extern int fio_open(char const* name, int mode, fio_location location); extern ssize_t fio_write(int fd, void const* buf, size_t size); extern ssize_t fio_read(int fd, void* buf, size_t size); From 7ab3b03060bc76e27eff0ec2ebbe3e563239591c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 Feb 2019 19:35:51 +0300 Subject: [PATCH 0302/2107] tests: fix test_ptrack_vacuum_replica() and test_ptrack_vacuum_bits_frozen_replica() --- tests/ptrack_vacuum.py | 3 ++- tests/ptrack_vacuum_bits_frozen.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py index bf3d8d691..9ecb3a793 100644 --- a/tests/ptrack_vacuum.py +++ b/tests/ptrack_vacuum.py @@ -155,7 +155,8 @@ def test_ptrack_vacuum_replica(self): backup_dir, 'replica', replica, options=[ '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) for i in idx_ptrack: # get fork size and calculate it in pages diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py index 637807b52..e6a72bc64 100644 --- a/tests/ptrack_vacuum_bits_frozen.py +++ b/tests/ptrack_vacuum_bits_frozen.py @@ -148,7 +148,8 @@ def test_ptrack_vacuum_bits_frozen_replica(self): '-j10', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + '--master-port={0}'.format(master.port), + '--stream']) for i in idx_ptrack: # get size of heap and indexes. size calculated in pages From 6b4f698da0695891bf5e457c80add72a9cf4c1fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Feb 2019 15:26:37 +0300 Subject: [PATCH 0303/2107] tests: more tests for backup locking --- tests/helpers/ptrack_helpers.py | 3 +- tests/locking.py | 91 ++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 237961300..6a05787c7 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1432,6 +1432,7 @@ def _execute(self, cmd, running=True): print(repr(line)) if line == '^done\n' or line.startswith('*stopped'): break - if running and line.startswith('*running'): + if running and (line.startswith('*running') or line.startswith('^running')): +# if running and line.startswith('*running'): break return output diff --git a/tests/locking.py b/tests/locking.py index ae5431a88..ebe324968 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -12,7 +12,11 @@ class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_locking_simple_1(self): - """""" + """ + make node, take full backup, stop it in the middle + run validate, expect it to successfully executed, + concurrect RUNNING backup with pid file and active process is legal + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -36,6 +40,12 @@ def test_locking_simple_1(self): if gdb.continue_execution_until_break(20) != 'breakpoint-hit': self.AssertTrue(False, 'Failed to hit breakpoint') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + self.validate_pb(backup_dir) self.assertEqual( @@ -48,7 +58,13 @@ def test_locking_simple_1(self): self.del_test_dir(module_name, fname) def test_locking_simple_2(self): - """""" + """ + make node, take full backup, stop it in the middle, + kill process so no cleanup is done - pid file is in place, + run validate, expect it to not successfully executed, + RUNNING backup with pid file AND without active pid is legal, + but his status must be changed to ERROR and pid file is deleted + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -72,7 +88,78 @@ def test_locking_simple_2(self): if gdb.continue_execution_until_break(20) != 'breakpoint-hit': self.AssertTrue(False, 'Failed to hit breakpoint') + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_running() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + self.validate_pb(backup_dir) + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_locking_simple_3(self): + """ + make node, take full backup, stop it in the middle, + terminate process, delete pid file, + run validate, expect it to not successfully executed, + RUNNING backup without pid file AND without active pid is legal, + his status must be changed to ERROR + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + self.AssertTrue(False, 'Failed to hit breakpoint') + + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_running() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.remove( + os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid')) + + self.validate_pb(backup_dir) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + # Clean after yourself self.del_test_dir(module_name, fname) \ No newline at end of file From 119f118af080a8fe121a0063f97bc7daaabab922 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Feb 2019 15:28:22 +0300 Subject: [PATCH 0304/2107] tests: minor fix for PGPRO-1918 --- tests/locking.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/locking.py b/tests/locking.py index ebe324968..728cbd350 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -97,7 +97,10 @@ def test_locking_simple_2(self): self.assertEqual( 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - self.validate_pb(backup_dir) + try: + self.validate_pb(backup_dir) + except: + pass self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -153,7 +156,10 @@ def test_locking_simple_3(self): os.remove( os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid')) - self.validate_pb(backup_dir) + try: + self.validate_pb(backup_dir) + except: + pass self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) From 3474784895a4f59f831f05e0f983e5b8b284a4ed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Feb 2019 15:29:59 +0300 Subject: [PATCH 0305/2107] tests: rename some locking tests --- tests/locking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/locking.py b/tests/locking.py index 728cbd350..618610488 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -11,7 +11,7 @@ class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_locking_simple_1(self): + def test_locking_running_1(self): """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, @@ -57,7 +57,7 @@ def test_locking_simple_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_locking_simple_2(self): + def test_locking_running_2(self): """ make node, take full backup, stop it in the middle, kill process so no cleanup is done - pid file is in place, @@ -111,7 +111,7 @@ def test_locking_simple_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_locking_simple_3(self): + def test_locking_running_3(self): """ make node, take full backup, stop it in the middle, terminate process, delete pid file, From 00c3bed898d0fcf686ae91aabd6152d62dc5dd94 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 17 Feb 2019 10:27:00 +0300 Subject: [PATCH 0306/2107] Fix incorrect usage of tempname in fio_gzopen --- src/utils/file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index cd1c46667..cd1b1d420 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -696,7 +696,8 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location gzFile file; if (fio_is_remote(location)) { - int fd = mkstemp("gz.XXXXXX"); + char pattern[] = "gz.XXXXXX"; + int fd = mkstemp(pattern); if (fd < 0) return NULL; if (strcmp(mode, PG_BINARY_W) == 0) From 0aa14030f61e24d40fe5c593491e19783a1a1d68 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 17 Feb 2019 15:34:59 +0300 Subject: [PATCH 0307/2107] tests: added test_tablespace_handling_1() and test_tablespace_handling_2() --- tests/backup_test.py | 116 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index 926c19d56..740f08d37 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -627,6 +627,122 @@ def test_tablespace_handling(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_tablespace_handling_1(self): + """ + make node with tablespace A, take full backup, check that restore with + tablespace mapping of tablespace B will end with error + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') + + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=tblspace1_old_path) + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --tablespace-mapping option' in e.message and + 'have an entry in tablespace_map file' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_tablespace_handling_2(self): + """ + make node without tablespaces, take full backup, check that restore with + tablespace mapping will end with error + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --tablespace-mapping option' in e.message and + 'have an entry in tablespace_map file' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" From a552e6a3d3555ab0de71de16cfc8fe32aa53faa4 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 17 Feb 2019 19:40:26 +0300 Subject: [PATCH 0308/2107] Fix fio_closedir --- src/dir.c | 4 ++-- src/utils/file.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index 3cf3490c9..43ae17b41 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1116,13 +1116,13 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path, FIO_DB_HOST)) + if (!fileExists(map_path, FIO_BACKUP_HOST)) { elog(LOG, "there is no file tablespace_map"); return; } - fp = fio_open_stream(map_path, FIO_DB_HOST); + fp = fio_open_stream(map_path, FIO_BACKUP_HOST); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); diff --git a/src/utils/file.c b/src/utils/file.c index cd1b1d420..ec7232526 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -208,6 +208,7 @@ int fio_closedir(DIR *dir) hdr.cop = FIO_CLOSEDIR; hdr.handle = (size_t)dir - 1; hdr.size = 0; + fio_fdset &= ~(1 << hdr.handle); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); return 0; @@ -397,7 +398,7 @@ int fio_pread(FILE* f, void* buf, off_t offs) hdr.cop = FIO_PREAD; hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = sizeof(XLogRecPtr); + hdr.size = 0; hdr.arg = offs; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); From 6f42781d585a722821b572425d2e145aa0552866 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 18 Feb 2019 11:24:01 +0300 Subject: [PATCH 0309/2107] Add fio_symlink --- src/dir.c | 2 +- src/utils/file.c | 27 +++++++++++++++++++++++++++ src/utils/file.h | 2 ++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 43ae17b41..5ee48bb6e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1064,7 +1064,7 @@ create_data_directories(const char *data_dir, const char *backup_dir, /* Secondly, create link */ join_path_components(to_path, to_path, link_name); - if (symlink(linked_path, to_path) < 0) + if (fio_symlink(linked_path, to_path, location) < 0) elog(ERROR, "could not create symbolic link \"%s\": %s", to_path, strerror(errno)); diff --git a/src/utils/file.c b/src/utils/file.c index ec7232526..2a6e346db 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -597,6 +597,30 @@ int fio_access(char const* path, int mode, fio_location location) } } +/* Create symbolink link */ +int fio_symlink(char const* target, char const* link_path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t target_len = strlen(target) + 1; + size_t link_path_len = strlen(link_path) + 1; + hdr.cop = FIO_SYMLINK; + hdr.handle = -1; + hdr.size = target_len + link_path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); + IO_CHECK(fio_write_all(fio_stdout, link_path, link_path_len), link_path_len); + + return 0; + } + else + { + return symlink(target, link_path); + } +} + /* Rename file */ int fio_rename(char const* old_path, char const* new_path, fio_location location) { @@ -1051,6 +1075,9 @@ void fio_communicate(int in, int out) case FIO_RENAME: /* Rename file */ SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); break; + case FIO_SYMLINK: /* Create symbolic link */ + SYS_CHECK(symlink(buf, buf + strlen(buf) + 1)); + break; case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ SYS_CHECK(remove(buf)); break; diff --git a/src/utils/file.h b/src/utils/file.h index 972cb76e4..8f27f480f 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -16,6 +16,7 @@ typedef enum FIO_CLOSE, FIO_WRITE, FIO_RENAME, + FIO_SYMLINK, FIO_UNLINK, FIO_MKDIR, FIO_CHMOD, @@ -91,6 +92,7 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); +extern int fio_symlink(char const* target, char const* link_path, fio_location location); extern int fio_unlink(char const* path, fio_location location); extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); From 7e60de38dbdd0e617d09b536cbe461daf73dc2d8 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 18 Feb 2019 11:44:14 +0300 Subject: [PATCH 0310/2107] Retry remote pull using ptrack --- src/data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/data.c b/src/data.c index 4747289be..f694bb7e6 100644 --- a/src/data.c +++ b/src/data.c @@ -593,6 +593,8 @@ backup_data_file(backup_files_arg* arguments, int rc = fio_send_pages(in, out, file, backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, &n_blocks_skipped, calg, clevel); + if (rc == PAGE_CHECKSUM_MISMATCH && is_ptrack_support) + goto RetryUsingPtrack; if (rc < 0) elog(ERROR, "Failed to read file %s: %s", file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); @@ -600,6 +602,7 @@ backup_data_file(backup_files_arg* arguments, } else { + RetryUsingPtrack: for (blknum = 0; blknum < nblocks; blknum++) { page_state = prepare_page(arguments, file, prev_backup_start_lsn, From 8960f2aae29b163d10b10c8b46b2647886cb790d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 18 Feb 2019 16:55:46 +0300 Subject: [PATCH 0311/2107] PGPRO-1918: Consider lock_backup()'s result --- src/backup.c | 5 +++-- src/catalog.c | 46 +++++++++++++++++++++++++++++++--------------- src/delete.c | 12 ++++++++---- src/merge.c | 7 ++----- src/parsexlog.c | 3 +-- src/pg_probackup.h | 4 ++-- src/restore.c | 18 +++++++++--------- src/validate.c | 30 ++++++++++++++++++++---------- tests/locking.py | 10 +++++----- 9 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/backup.c b/src/backup.c index 41426cb89..1426838f1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -976,8 +976,9 @@ do_backup(time_t start_time) /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) - elog(ERROR, "cannot create backup directory"); - lock_backup(¤t); + elog(ERROR, "Cannot create backup directory"); + if (!lock_backup(¤t)) + elog(ERROR, "Cannot lock backup directory"); write_backup(¤t); elog(LOG, "Backup destination is initialized"); diff --git a/src/catalog.c b/src/catalog.c index e25ff8db7..6e7e70251 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -69,12 +69,13 @@ read_backup(time_t timestamp) * status. */ void -write_backup_status(pgBackup *backup) +write_backup_status(pgBackup *backup, BackupStatus status) { pgBackup *tmp; tmp = read_backup(backup->start_time); + backup->status = status; tmp->status = backup->status; write_backup(tmp); @@ -84,7 +85,7 @@ write_backup_status(pgBackup *backup) /* * Create exclusive lockfile in the backup's directory. */ -void +bool lock_backup(pgBackup *backup) { char lock_file[MAXPGPATH]; @@ -149,7 +150,7 @@ lock_backup(pgBackup *backup) * Couldn't create the pid file. Probably it already exists. */ if ((errno != EEXIST && errno != EACCES) || ntries > 100) - elog(ERROR, "could not create lock file \"%s\": %s", + elog(ERROR, "Could not create lock file \"%s\": %s", lock_file, strerror(errno)); /* @@ -161,22 +162,22 @@ lock_backup(pgBackup *backup) { if (errno == ENOENT) continue; /* race condition; try again */ - elog(ERROR, "could not open lock file \"%s\": %s", + elog(ERROR, "Could not open lock file \"%s\": %s", lock_file, strerror(errno)); } if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) - elog(ERROR, "could not read lock file \"%s\": %s", + elog(ERROR, "Could not read lock file \"%s\": %s", lock_file, strerror(errno)); close(fd); if (len == 0) - elog(ERROR, "lock file \"%s\" is empty", lock_file); + elog(ERROR, "Lock file \"%s\" is empty", lock_file); buffer[len] = '\0'; encoded_pid = atoi(buffer); if (encoded_pid <= 0) - elog(ERROR, "bogus data in lock file \"%s\": \"%s\"", + elog(ERROR, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); /* @@ -190,9 +191,21 @@ lock_backup(pgBackup *backup) */ if (encoded_pid != my_pid && encoded_pid != my_p_pid) { - if (kill(encoded_pid, 0) == 0 || - (errno != ESRCH && errno != EPERM)) - elog(ERROR, "lock file \"%s\" already exists", lock_file); + if (kill(encoded_pid, 0) == 0) + { + elog(WARNING, "Process %d is using backup %s and still is running", + encoded_pid, base36enc(backup->start_time)); + return false; + } + else + { + if (errno == ESRCH) + elog(WARNING, "Process %d which used backup %s no longer exists", + encoded_pid, base36enc(backup->start_time)); + else + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } } /* @@ -201,7 +214,7 @@ lock_backup(pgBackup *backup) * would-be creators. */ if (unlink(lock_file) < 0) - elog(ERROR, "could not remove old lock file \"%s\": %s", + elog(ERROR, "Could not remove old lock file \"%s\": %s", lock_file, strerror(errno)); } @@ -219,7 +232,7 @@ lock_backup(pgBackup *backup) unlink(lock_file); /* if write didn't set errno, assume problem is no disk space */ errno = save_errno ? save_errno : ENOSPC; - elog(ERROR, "could not write lock file \"%s\": %s", + elog(ERROR, "Could not write lock file \"%s\": %s", lock_file, strerror(errno)); } if (fsync(fd) != 0) @@ -229,7 +242,7 @@ lock_backup(pgBackup *backup) close(fd); unlink(lock_file); errno = save_errno; - elog(ERROR, "could not write lock file \"%s\": %s", + elog(ERROR, "Could not write lock file \"%s\": %s", lock_file, strerror(errno)); } if (close(fd) != 0) @@ -238,7 +251,7 @@ lock_backup(pgBackup *backup) unlink(lock_file); errno = save_errno; - elog(ERROR, "could not write lock file \"%s\": %s", + elog(ERROR, "Culd not write lock file \"%s\": %s", lock_file, strerror(errno)); } @@ -255,6 +268,8 @@ lock_backup(pgBackup *backup) if (lock_files == NULL) lock_files = parray_new(); parray_append(lock_files, pgut_strdup(lock_file)); + + return true; } /* @@ -418,7 +433,8 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) end_idx = Min(from_idx, to_idx); for (i = start_idx; i >= end_idx; i--) - lock_backup((pgBackup *) parray_get(backup_list, i)); + if (!lock_backup((pgBackup *) parray_get(backup_list, i))) + elog(ERROR, "Cannot lock backup directory"); } /* diff --git a/src/delete.c b/src/delete.c index 242467891..a448a050b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -202,7 +202,12 @@ do_retention_purge(void) continue; } - lock_backup(backup); + /* + * If the backup still is used do not interrupt go to the next + * backup. + */ + if (!lock_backup(backup)) + continue; /* Delete backup and update status to DELETED */ delete_backup_files(backup); @@ -238,7 +243,7 @@ do_retention_purge(void) if (backup_deleted) elog(INFO, "Purging finished"); else - elog(INFO, "Nothing to delete by retention policy"); + elog(INFO, "There are no backups to delete by retention policy"); return 0; } @@ -275,8 +280,7 @@ delete_backup_files(pgBackup *backup) * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which * the error occurs before deleting all backup files. */ - backup->status = BACKUP_STATUS_DELETING; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_DELETING); /* list files to be deleted */ files = parray_new(); diff --git a/src/merge.c b/src/merge.c index 056967e25..78e29aa6b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -227,11 +227,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (from_backup->status == BACKUP_STATUS_DELETING) goto delete_source_backup; - to_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(to_backup); - - from_backup->status = BACKUP_STATUS_MERGING; - write_backup_status(from_backup); + write_backup_status(to_backup, BACKUP_STATUS_MERGING); + write_backup_status(from_backup, BACKUP_STATUS_MERGING); create_data_directories(to_database_path, from_backup_path, false); diff --git a/src/parsexlog.c b/src/parsexlog.c index d8ba3ad76..5a6258841 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -484,8 +484,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_CORRUPT); elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b958c1d40..921cbd25f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -459,8 +459,8 @@ extern int do_validate_all(void); /* in catalog.c */ extern pgBackup *read_backup(time_t timestamp); extern void write_backup(pgBackup *backup); -extern void write_backup_status(pgBackup *backup); -extern void lock_backup(pgBackup *backup); +extern void write_backup_status(pgBackup *backup, BackupStatus status); +extern bool lock_backup(pgBackup *backup); extern const char *pgBackupGetBackupMode(pgBackup *backup); diff --git a/src/restore.c b/src/restore.c index 5e5e6b8d3..8a67db124 100644 --- a/src/restore.c +++ b/src/restore.c @@ -209,8 +209,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { if (backup->status == BACKUP_STATUS_OK) { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(backup->start_time), missing_backup_id); @@ -242,8 +241,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { if (backup->status == BACKUP_STATUS_OK) { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", @@ -317,7 +315,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { tmp_backup = (pgBackup *) parray_get(parent_chain, i); - lock_backup(tmp_backup); + /* Do not interrupt, validate the next backup */ + if (!lock_backup(tmp_backup)) + continue; + pgBackupValidate(tmp_backup); /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ if (tmp_backup->status == BACKUP_STATUS_CORRUPT) @@ -360,8 +361,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { if (backup->status == BACKUP_STATUS_OK) { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(backup->start_time), @@ -409,8 +409,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Backup was locked during validation if no-validate wasn't * specified. */ - if (rt->restore_no_validate) - lock_backup(backup); + if (rt->restore_no_validate && !lock_backup(backup)) + elog(ERROR, "Cannot lock backup directory"); restore_backup(backup); } diff --git a/src/validate.c b/src/validate.c index 6b704b113..ad3b24178 100644 --- a/src/validate.c +++ b/src/validate.c @@ -59,6 +59,15 @@ pgBackupValidate(pgBackup *backup) "Please upgrade pg_probackup binary.", PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + if (backup->status == BACKUP_STATUS_RUNNING) + { + elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", + base36enc(backup->start_time), status2str(backup->status)); + write_backup_status(backup, BACKUP_STATUS_ERROR); + corrupted_backup_found = true; + return; + } + /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE && @@ -143,8 +152,8 @@ pgBackupValidate(pgBackup *backup) parray_free(files); /* Update backup status */ - backup->status = corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK; - write_backup_status(backup); + write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : + BACKUP_STATUS_OK); if (corrupted) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); @@ -385,8 +394,7 @@ do_validate_instance(void) /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK) { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(current_backup->start_time), parent_backup_id); @@ -410,8 +418,7 @@ do_validate_instance(void) /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK) { - current_backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(current_backup); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(current_backup->start_time), parent_backup_id, status2str(tmp_backup->status)); @@ -435,7 +442,9 @@ do_validate_instance(void) else base_full_backup = current_backup; - lock_backup(current_backup); + /* Do not interrupt, validate the next backup */ + if (!lock_backup(current_backup)) + continue; /* Valiate backup files*/ pgBackupValidate(current_backup); @@ -469,8 +478,7 @@ do_validate_instance(void) { if (backup->status == BACKUP_STATUS_OK) { - backup->status = BACKUP_STATUS_ORPHAN; - write_backup_status(backup); + write_backup_status(backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(backup->start_time), @@ -522,7 +530,9 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_ORPHAN) { - lock_backup(backup); + /* Do not interrupt, validate the next backup */ + if (!lock_backup(backup)) + continue; /* Revaliate backup files*/ pgBackupValidate(backup); diff --git a/tests/locking.py b/tests/locking.py index 618610488..e4ec9dfa4 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -12,7 +12,7 @@ class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_locking_running_1(self): - """ + """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, concurrect RUNNING backup with pid file and active process is legal @@ -46,7 +46,7 @@ def test_locking_running_1(self): self.assertEqual( 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=['--log-level-file=VERBOSE']) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -58,7 +58,7 @@ def test_locking_running_1(self): self.del_test_dir(module_name, fname) def test_locking_running_2(self): - """ + """ make node, take full backup, stop it in the middle, kill process so no cleanup is done - pid file is in place, run validate, expect it to not successfully executed, @@ -112,7 +112,7 @@ def test_locking_running_2(self): self.del_test_dir(module_name, fname) def test_locking_running_3(self): - """ + """ make node, take full backup, stop it in the middle, terminate process, delete pid file, run validate, expect it to not successfully executed, @@ -168,4 +168,4 @@ def test_locking_running_3(self): 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) From 49705d73f472c622a14ce4053f57ee64e95e5f53 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 17:20:13 +0300 Subject: [PATCH 0312/2107] tests: fix test_archive_node_backup_archive_pitr_2() --- tests/restore_test.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/restore_test.py b/tests/restore_test.py index 169b30809..0da40c5f4 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1160,7 +1160,10 @@ def test_archive_node_backup_archive_pitr_2(self): node.safe_psql("postgres", "create table t_heap(a int)") node.stop() - node.cleanup() + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() recovery_time = self.show_pb( backup_dir, 'node', backup_id)['recovery-time'] @@ -1168,7 +1171,7 @@ def test_archive_node_backup_archive_pitr_2(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, + backup_dir, 'node', node_restored, options=[ "-j", "4", '--time={0}'.format(recovery_time), "--recovery-target-action=promote" @@ -1178,12 +1181,12 @@ def test_archive_node_backup_archive_pitr_2(self): repr(self.output), self.cmd)) if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) + pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node.slow_start() + node_restored.slow_start() - result = node.psql("postgres", 'select * from t_heap') + result = node_restored.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) # Clean after yourself From ac3b0175d41adb89d43d9b1e60e042234d7c5d44 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 18:16:53 +0300 Subject: [PATCH 0313/2107] PGPRO-1918 tests: added test_locking_running_validate_2_specific_id() --- tests/helpers/ptrack_helpers.py | 16 +++- tests/locking.py | 129 +++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 12 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 38b12c9bf..9caf68d1f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1383,7 +1383,7 @@ def continue_execution_until_exit(self): if line.startswith('*stopped,reason="breakpoint-hit"'): continue if ( - line.startswith('*stopped,reason="exited-normally"') or + line.startswith('*stopped,reason="exited"') or line == '*stopped\n' ): return @@ -1391,6 +1391,18 @@ def continue_execution_until_exit(self): 'Failed to continue execution until exit.\n' ) + def continue_execution_until_error(self): + result = self._execute('continue', False) + + for line in result: + if line.startswith('^error'): + return + if line.startswith('*stopped,reason="exited'): + return + + raise GdbException( + 'Failed to continue execution until error.\n') + def continue_execution_until_break(self, ignore_count=0): if ignore_count > 0: result = self._execute( @@ -1436,6 +1448,8 @@ def _execute(self, cmd, running=True): print(repr(line)) if line.startswith('^done') or line.startswith('*stopped'): break + if line.startswith('^error'): + break if running and (line.startswith('*running') or line.startswith('^running')): # if running and line.startswith('*running'): break diff --git a/tests/locking.py b/tests/locking.py index e4ec9dfa4..9d885b16c 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -11,7 +11,7 @@ class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_locking_running_1(self): + def test_locking_running_validate_1(self): """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, @@ -46,18 +46,26 @@ def test_locking_running_1(self): self.assertEqual( 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - self.validate_pb(backup_dir, options=['--log-level-file=VERBOSE']) + validate_output = self.validate_pb( + backup_dir, options=['--log-level-console=LOG']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertIn( + "is using backup {0} and still is running".format(backup_id), + validate_output, + '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - self.del_test_dir(module_name, fname) + # self.del_test_dir(module_name, fname) - def test_locking_running_2(self): + def test_locking_running_validate_2(self): """ make node, take full backup, stop it in the middle, kill process so no cleanup is done - pid file is in place, @@ -89,7 +97,7 @@ def test_locking_running_2(self): self.AssertTrue(False, 'Failed to hit breakpoint') gdb._execute('signal SIGKILL') - gdb.continue_execution_until_running() + gdb.continue_execution_until_error() self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -97,10 +105,97 @@ def test_locking_running_2(self): self.assertEqual( 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + try: self.validate_pb(backup_dir) - except: - pass + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "which used backup {0} no longer exists".format( + backup_id) in e.message and + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_locking_running_validate_2_specific_id(self): + """ + make node, take full backup, stop it in the middle, + kill process so no cleanup is done - pid file is in place, + run validate on this specific backup, + expect it to not successfully executed, + RUNNING backup with pid file AND without active pid is legal, + but his status must be changed to ERROR and pid file is deleted + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + if gdb.continue_execution_until_break(20) != 'breakpoint-hit': + self.AssertTrue(False, 'Failed to hit breakpoint') + + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_error() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "which used backup {0} no longer exists".format( + backup_id) in e.message and + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -143,7 +238,7 @@ def test_locking_running_3(self): self.AssertTrue(False, 'Failed to hit breakpoint') gdb._execute('signal SIGKILL') - gdb.continue_execution_until_running() + gdb.continue_execution_until_error() self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -158,8 +253,20 @@ def test_locking_running_3(self): try: self.validate_pb(backup_dir) - except: - pass + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) From e91e9d87a692a786f231c0f588bb073e5e40bf93 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 18 Feb 2019 19:05:45 +0300 Subject: [PATCH 0314/2107] PGPRO-1918: Check RUNNING --- src/restore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 8a67db124..99a163e2c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -124,7 +124,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { if ((current_backup->status == BACKUP_STATUS_DONE || current_backup->status == BACKUP_STATUS_ORPHAN || - current_backup->status == BACKUP_STATUS_CORRUPT) + current_backup->status == BACKUP_STATUS_CORRUPT || + current_backup->status == BACKUP_STATUS_RUNNING) && !rt->restore_no_validate) elog(WARNING, "Backup %s has status: %s", base36enc(current_backup->start_time), status2str(current_backup->status)); From c4259b856b9fa9aefe89bf30c9c21b7d28ba2fca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 20:09:09 +0300 Subject: [PATCH 0315/2107] PGPRO-1918 tests: minor fixes --- tests/locking.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/locking.py b/tests/locking.py index 9d885b16c..1c08c43c5 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -192,7 +192,7 @@ def test_locking_running_validate_2_specific_id(self): "Backup {0} has status RUNNING, change it " "to ERROR and skip validation".format( backup_id) in e.message and - "WARNING: Some backups are not valid" in + "ERROR: Backup {0} has status: ERROR".format(backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -203,6 +203,35 @@ def test_locking_running_validate_2_specific_id(self): self.assertEqual( 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has status: ERROR".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Backup {0} has status ERROR. Skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + # Clean after yourself self.del_test_dir(module_name, fname) From 8c9dcbec70a588ed58184cb0b9a2a6036575f799 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 20:16:45 +0300 Subject: [PATCH 0316/2107] PGPRO-1918: disable wal validation for RUNNING and ERROR backups --- src/restore.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index 99a163e2c..97e881f4b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -321,8 +321,12 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, continue; pgBackupValidate(tmp_backup); - /* Maybe we should be more paranoid and check for !BACKUP_STATUS_OK? */ - if (tmp_backup->status == BACKUP_STATUS_CORRUPT) + /* After pgBackupValidate() only following backup + * states are possible: ERROR, RUNNING, CORRUPT and OK. + * Validate WAL only for OK, because there is no point + * in WAL validation for corrupted, errored or running backups. + */ + if (tmp_backup->status != BACKUP_STATUS_OK) { corrupted_backup = tmp_backup; /* we need corrupted backup index from 'backups' not parent_chain From b826a0ae94d05b4802e4e3be429c4a5d220ff7e4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 20:38:39 +0300 Subject: [PATCH 0317/2107] PGPRO-1918: be more paranoid in instance validation, orphanize descendants of all not OK backups, not only CORRUPT --- src/validate.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/validate.c b/src/validate.c index ad3b24178..1fdb76e95 100644 --- a/src/validate.c +++ b/src/validate.c @@ -457,14 +457,14 @@ do_validate_instance(void) /* * Mark every descendant of corrupted backup as orphan */ - if (current_backup->status == BACKUP_STATUS_CORRUPT) + if (current_backup->status != BACKUP_STATUS_OK) { /* This is ridiculous but legal. - * PAGE1_2b <- OK - * PAGE1_2a <- OK - * PAGE1_1b <- ORPHAN - * PAGE1_1a <- CORRUPT - * FULL1 <- OK + * PAGE_b2 <- OK + * PAGE_a2 <- OK + * PAGE_b1 <- ORPHAN + * PAGE_a1 <- CORRUPT + * FULL <- OK */ corrupted_backup_found = true; @@ -503,14 +503,14 @@ do_validate_instance(void) pgBackup *tmp_backup = NULL; int result; - //PAGE3b ORPHAN - //PAGE2b ORPHAN ----- - //PAGE6a ORPHAN | - //PAGE5a CORRUPT | - //PAGE4a missing | - //PAGE3a missing | - //PAGE2a ORPHAN | - //PAGE1a OK <- we are here <-| + //PAGE_b2 ORPHAN + //PAGE_b1 ORPHAN ----- + //PAGE_a5 ORPHAN | + //PAGE_a4 CORRUPT | + //PAGE_a3 missing | + //PAGE_a2 missing | + //PAGE_a1 ORPHAN | + //PAGE OK <- we are here<-| //FULL OK if (is_parent(current_backup->start_time, backup, false)) From f7ed67a19e002baa354207ffb232f64a70fa8af2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 21:27:23 +0300 Subject: [PATCH 0318/2107] tests: added test_validate_error_intermediate_backups() --- tests/validate_test.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/validate_test.py b/tests/validate_test.py index c6d329066..a694491cc 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -411,6 +411,100 @@ def test_validate_corrupted_intermediate_backups(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_error_intermediate_backups(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + change backup status of FULL and PAGE1 to ERROR, + run validate on PAGE1 + purpouse of this test is to be sure that not only + CORRUPT backup descendants can be orphanized + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULL backup status to ERROR + control_path = os.path.join( + backup_dir, 'backups', 'node', backup_id_1, 'backup.control') + + with open(control_path, 'r') as f: + actual_control = f.read() + + print(actual_control) + + new_control_file = '' + for line in actual_control.splitlines(): + new_control_file += line.replace( + 'status = OK', 'status = ERROR') + new_control_file += '\n' + + with open(control_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + print('HELLO') + print(new_control_file) + + # Validate PAGE1 + try: + self.validate_pb( + backup_dir, 'node', backup_id=backup_id_2) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: ERROR'.format( + backup_id_2, backup_id_1) in e.message and + 'INFO: Validating parents for backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Backup {0} has status ERROR. Skip validation.'.format( + backup_id_1) and + 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "ERROR"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups_1(self): """ From 2329af38bc4164d16adc8641e56a61d9eeb92242 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Feb 2019 21:42:56 +0300 Subject: [PATCH 0319/2107] tests: added test_validate_specific_error_intermediate_backups() --- tests/validate_test.py | 94 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index a694491cc..c6738e578 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -412,12 +412,12 @@ def test_validate_corrupted_intermediate_backups(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_validate_error_intermediate_backups(self): + def test_validate_specific_error_intermediate_backups(self): """ make archive node, take FULL, PAGE1, PAGE2 backups, change backup status of FULL and PAGE1 to ERROR, run validate on PAGE1 - purpouse of this test is to be sure that not only + purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ fname = self.id().split('.')[3] @@ -450,8 +450,6 @@ def test_validate_error_intermediate_backups(self): with open(control_path, 'r') as f: actual_control = f.read() - print(actual_control) - new_control_file = '' for line in actual_control.splitlines(): new_control_file += line.replace( @@ -463,9 +461,6 @@ def test_validate_error_intermediate_backups(self): f.flush() f.close() - print('HELLO') - print(new_control_file) - # Validate PAGE1 try: self.validate_pb( @@ -505,6 +500,91 @@ def test_validate_error_intermediate_backups(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_error_intermediate_backups(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + change backup status of FULL and PAGE1 to ERROR, + run validate on instance + purpose of this test is to be sure that not only + CORRUPT backup descendants can be orphanized + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULL backup status to ERROR + control_path = os.path.join( + backup_dir, 'backups', 'node', backup_id_1, 'backup.control') + + with open(control_path, 'r') as f: + actual_control = f.read() + + new_control_file = '' + for line in actual_control.splitlines(): + new_control_file += line.replace( + 'status = OK', 'status = ERROR') + new_control_file += '\n' + + with open(control_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + # Validate instance + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Backup {0} is orphaned because " + "his parent {1} has status: ERROR".format( + backup_id_2, backup_id_1) in e.message and + 'WARNING: Backup {0} has status ERROR. Skip validation'.format( + backup_id_1) in e.message and + "WARNING: Some backups are not valid" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "ERROR"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups_1(self): """ From d9716e000c4ca5a523e555c3295b54f97ec689b7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 Feb 2019 15:24:57 +0300 Subject: [PATCH 0320/2107] tests: added test_locking_restore_locked() and test_locking_concurrent_vaidate_and_backup() --- tests/helpers/ptrack_helpers.py | 4 +- tests/locking.py | 92 +++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 9caf68d1f..97f3bc902 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -806,7 +806,7 @@ def show_pb( def validate_pb( self, backup_dir, instance=None, - backup_id=None, options=[], old_binary=False + backup_id=None, options=[], old_binary=False, gdb=False ): cmd_list = [ @@ -818,7 +818,7 @@ def validate_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb) def delete_pb( self, backup_dir, instance, diff --git a/tests/locking.py b/tests/locking.py index 1c08c43c5..fc1fbdf4e 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -305,3 +305,95 @@ def test_locking_running_3(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_locking_restore_locked(self): + """ + make node, take full backup, take two page backups, + launch validate on PAGE1 and stop it in the middle, + launch restore of PAGE2. + Expect restore to fail because validation of + intermediate backup is impossible + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.validate_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + node.cleanup() + + try: + self.restore_node(backup_dir, 'node', node) + self.assertEqual( + 1, 0, + "Expecting Error because restore without whole chain validation " + "is prohibited unless --no-validate provided.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "Insert expected error message".format( + backup_id) in e.message, + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_locking_concurrent_vaidate_and_backup(self): + """ + make node, take full backup, launch validate + and stop it in the middle, take page backup. + Expect PAGE backup to be successfully executed + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + # PAGE2 + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.validate_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + # This PAGE backup is expected to be successfull + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 883464fdfc4560667bdef96bfade77b0f39877e2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 Feb 2019 15:42:18 +0300 Subject: [PATCH 0321/2107] PGPRO-1918: added test_locking_restore_locked_without_validation() --- tests/locking.py | 60 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/tests/locking.py b/tests/locking.py index fc1fbdf4e..90a42ad2a 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -353,9 +353,63 @@ def test_locking_restore_locked(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "Insert expected error message".format( - backup_id) in e.message, - e.message, + "Insert expected error message" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_locking_restore_locked_without_validation(self): + """ + make node, take full backup, take page backup, + launch validate on FULL and stop it in the middle, + launch restore of PAGE. + Expect restore to fail because validation of + intermediate backup is impossible + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.validate_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because restore without whole chain validation " + "is prohibited unless --no-validate provided.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "Backup {0} is used without validation".format( + restore_id) in e.message and + 'is using backup {0} and still is running'.format( + backup_id) in e.message and + 'ERROR: Cannot lock backup directory' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 34065ad404579b33210215a3e63439424f2b611e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 20 Feb 2019 10:43:14 +0300 Subject: [PATCH 0322/2107] Create temporary files in tmpdir --- src/utils/file.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 2a6e346db..d8ba2fb3c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -721,7 +721,11 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location gzFile file; if (fio_is_remote(location)) { - char pattern[] = "gz.XXXXXX"; +#ifdef P_tmpdir + char pattern[] = P_tmpdir "/gz.XXXXXX"; +#else + char pattern[] = "/tmp/gz.XXXXXX"; +#endif int fd = mkstemp(pattern); if (fd < 0) return NULL; From ad5449f1b238d61b0aa9ee9314c831b577bd12a1 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 20 Feb 2019 13:29:16 +0300 Subject: [PATCH 0323/2107] PGPRO-1918: Add more informative messages --- src/backup.c | 3 ++- src/catalog.c | 8 ++++++-- src/delete.c | 4 ++++ src/restore.c | 12 +++++++++++- src/validate.c | 8 ++++++++ tests/locking.py | 4 ++-- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1426838f1..95a6bea41 100644 --- a/src/backup.c +++ b/src/backup.c @@ -978,7 +978,8 @@ do_backup(time_t start_time) if (pgBackupCreateDir(¤t)) elog(ERROR, "Cannot create backup directory"); if (!lock_backup(¤t)) - elog(ERROR, "Cannot lock backup directory"); + elog(ERROR, "Cannot lock backup %s directory", + base36enc(current.start_time)); write_backup(¤t); elog(LOG, "Backup destination is initialized"); diff --git a/src/catalog.c b/src/catalog.c index 6e7e70251..d4641a6f3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -433,8 +433,12 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) end_idx = Min(from_idx, to_idx); for (i = start_idx; i >= end_idx; i--) - if (!lock_backup((pgBackup *) parray_get(backup_list, i))) - elog(ERROR, "Cannot lock backup directory"); + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + if (!lock_backup(backup)) + elog(ERROR, "Cannot lock backup %s directory", + base36enc(backup->start_time)); + } } /* diff --git a/src/delete.c b/src/delete.c index a448a050b..d637b50e9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -207,7 +207,11 @@ do_retention_purge(void) * backup. */ if (!lock_backup(backup)) + { + elog(WARNING, "Cannot lock backup %s directory, skip purging", + base36enc(backup->start_time)); continue; + } /* Delete backup and update status to DELETED */ delete_backup_files(backup); diff --git a/src/restore.c b/src/restore.c index 97e881f4b..5aee4037c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -318,7 +318,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Do not interrupt, validate the next backup */ if (!lock_backup(tmp_backup)) - continue; + { + if (is_restore) + elog(ERROR, "Cannot lock backup %s directory", + base36enc(tmp_backup->start_time)); + else + { + elog(WARNING, "Cannot lock backup %s directory, skip validation", + base36enc(tmp_backup->start_time)); + continue; + } + } pgBackupValidate(tmp_backup); /* After pgBackupValidate() only following backup diff --git a/src/validate.c b/src/validate.c index 1fdb76e95..1ffa932de 100644 --- a/src/validate.c +++ b/src/validate.c @@ -444,7 +444,11 @@ do_validate_instance(void) /* Do not interrupt, validate the next backup */ if (!lock_backup(current_backup)) + { + elog(WARNING, "Cannot lock backup %s directory, skip validation", + base36enc(current_backup->start_time)); continue; + } /* Valiate backup files*/ pgBackupValidate(current_backup); @@ -532,7 +536,11 @@ do_validate_instance(void) { /* Do not interrupt, validate the next backup */ if (!lock_backup(backup)) + { + elog(WARNING, "Cannot lock backup %s directory, skip validation", + base36enc(backup->start_time)); continue; + } /* Revaliate backup files*/ pgBackupValidate(backup); diff --git a/tests/locking.py b/tests/locking.py index 90a42ad2a..44fff4de8 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -327,7 +327,7 @@ def test_locking_restore_locked(self): node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + full_id = self.backup_node(backup_dir, 'node', node) # PAGE1 backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -353,7 +353,7 @@ def test_locking_restore_locked(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "Insert expected error message" in e.message, + "ERROR: Cannot lock backup {0} directory\n".format(full_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 2ed13d7c7e0b378c5ee596f2bb436e450b17b510 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 20 Feb 2019 13:41:00 +0300 Subject: [PATCH 0324/2107] Remove temporary files --- src/utils/file.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index d8ba2fb3c..2b8aff91a 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -721,14 +721,11 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location gzFile file; if (fio_is_remote(location)) { -#ifdef P_tmpdir - char pattern[] = P_tmpdir "/gz.XXXXXX"; -#else char pattern[] = "/tmp/gz.XXXXXX"; -#endif int fd = mkstemp(pattern); if (fd < 0) return NULL; + unlink(pattern); /* remove file on close */ if (strcmp(mode, PG_BINARY_W) == 0) { *tmp_fd = fd; From 58f02c1af460f9c11d851ba66c1bd1504da8928c Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 20 Feb 2019 15:42:30 +0300 Subject: [PATCH 0325/2107] Create temporary files for gzopen either in working directory, either in /tmp --- src/utils/file.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 2b8aff91a..536dac50b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -721,11 +721,19 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location gzFile file; if (fio_is_remote(location)) { - char pattern[] = "/tmp/gz.XXXXXX"; - int fd = mkstemp(pattern); + char pattern1[] = "/tmp/gz.XXXXXX"; + char pattern2[] = "gz.XXXXXX"; + char* path = pattern1; + int fd = mkstemp(path); /* first try to create file in current directory */ if (fd < 0) - return NULL; - unlink(pattern); /* remove file on close */ + { + path = pattern2; + fd = mkstemp(path); /* if it is not possible, try to create it in /tmp/ */ + if (fd < 0) + return NULL; + } + unlink(path); /* delete file on close */ + if (strcmp(mode, PG_BINARY_W) == 0) { *tmp_fd = fd; From ff4d470b6fca93a986bb16ee3f3c1e3a44434f66 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 22 Feb 2019 15:12:54 +0300 Subject: [PATCH 0326/2107] PGPRO-1918: Report skipped backups because of locks --- src/validate.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/validate.c b/src/validate.c index 1ffa932de..7878dec9f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -19,6 +19,7 @@ static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); static bool corrupted_backup_found = false; +static bool skipped_due_to_lock = false; typedef struct { @@ -287,6 +288,9 @@ pgBackupValidateFiles(void *arg) int do_validate_all(void) { + corrupted_backup_found = false; + skipped_due_to_lock = false; + if (instance_name == NULL) { /* Show list of instances */ @@ -339,12 +343,16 @@ do_validate_all(void) do_validate_instance(); } + if (skipped_due_to_lock) + elog(WARNING, "Some backups weren't locked and they were skipped"); + if (corrupted_backup_found) { elog(WARNING, "Some backups are not valid"); return 1; } - else + + if (!skipped_due_to_lock && !corrupted_backup_found) elog(INFO, "All backups are valid"); return 0; @@ -447,6 +455,7 @@ do_validate_instance(void) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(current_backup->start_time)); + skipped_due_to_lock = true; continue; } /* Valiate backup files*/ @@ -539,6 +548,7 @@ do_validate_instance(void) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(backup->start_time)); + skipped_due_to_lock = true; continue; } /* Revaliate backup files*/ From d1cb537a9051ccea63728ec14bb63df35dea7ef3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Feb 2019 19:38:07 +0300 Subject: [PATCH 0327/2107] TODO about different exit codes for validation --- src/validate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/validate.c b/src/validate.c index 7878dec9f..031ead154 100644 --- a/src/validate.c +++ b/src/validate.c @@ -343,6 +343,14 @@ do_validate_all(void) do_validate_instance(); } + /* TODO: Probably we should have different exit code for every condition + * and they combination: + * 0 - all backups are valid + * 1 - some backups are corrup + * 2 - some backups where skipped due to concurrent locks + * 3 - some backups are corrupt and some are skipped due to concurrent locks + */ + if (skipped_due_to_lock) elog(WARNING, "Some backups weren't locked and they were skipped"); From 5b3a733b37266a44d5e29d4e907ce182d7b6e9bc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Feb 2019 21:19:56 +0300 Subject: [PATCH 0328/2107] tests for external directories: merge tests added --- tests/external.py | 244 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 230 insertions(+), 14 deletions(-) diff --git a/tests/external.py b/tests/external.py index 7de4e1abd..d346706ad 100644 --- a/tests/external.py +++ b/tests/external.py @@ -112,14 +112,14 @@ def test_external_none(self): # Delta backup without external directory self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--external-dirs=none']) + backup_dir, 'node', node, backup_type="delta") +# options=['--external-dirs=none']) + shutil.rmtree(external_dir, ignore_errors=True) pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs', 'somedirectory']) + node.base_dir, exclude_dirs=['logs']) node.cleanup() - shutil.rmtree(external_dir, ignore_errors=True) self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) @@ -161,15 +161,14 @@ def test_external_dir_mapping(self): external_dir1_old = self.get_tblspace_path(node, 'external_dir1') external_dir2_old = self.get_tblspace_path(node, 'external_dir2') - # copy postgresql.conf to external_directory - os.mkdir(external_dir1_old) - os.mkdir(external_dir2_old) - shutil.copyfile( - os.path.join(node.data_dir, 'postgresql.conf'), - os.path.join(external_dir1_old, 'postgresql.conf')) - shutil.copyfile( - os.path.join(node.data_dir, 'postgresql.conf'), - os.path.join(external_dir2_old, 'postgresql.conf')) + # Fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -228,7 +227,7 @@ def test_external_dir_mapping(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_backup_multiple_external(self): - """make node, """ + """check that cmdline has priority over config""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -331,6 +330,28 @@ def test_external_backward_compatibility(self): backup_dir, 'node', node, data_dir=external_dir2_old, options=["-j", "4"]) + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + # delta backup with external directories using new binary self.backup_node( backup_dir, 'node', node, backup_type="delta", @@ -364,6 +385,201 @@ def test_external_backward_compatibility(self): self.compare_pgdata(pgdata, pgdata_restored) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_1(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + # delta backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # delta backup without external directories + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + # external directory contain symlink to file # external directory contain symlink to directory # latest page backup without external_dir From ec2aa7f000085db34081f226fbe36aab39ba96d4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 24 Feb 2019 23:20:42 +0300 Subject: [PATCH 0329/2107] tests: move all ptrack tests to ptrack module --- tests/ptrack.py | 1874 +++++++++++++++++++++++- tests/ptrack_clean.py | 258 ---- tests/ptrack_cluster.py | 375 ----- tests/ptrack_empty.py | 183 --- tests/ptrack_move_to_tablespace.py | 74 - tests/ptrack_recovery.py | 72 - tests/ptrack_truncate.py | 179 --- tests/ptrack_vacuum.py | 206 --- tests/ptrack_vacuum_bits_frozen.py | 194 --- tests/ptrack_vacuum_bits_visibility.py | 90 -- tests/ptrack_vacuum_full.py | 201 --- tests/ptrack_vacuum_truncate.py | 192 --- 12 files changed, 1868 insertions(+), 2030 deletions(-) delete mode 100644 tests/ptrack_clean.py delete mode 100644 tests/ptrack_cluster.py delete mode 100644 tests/ptrack_empty.py delete mode 100644 tests/ptrack_move_to_tablespace.py delete mode 100644 tests/ptrack_recovery.py delete mode 100644 tests/ptrack_truncate.py delete mode 100644 tests/ptrack_vacuum.py delete mode 100644 tests/ptrack_vacuum_bits_frozen.py delete mode 100644 tests/ptrack_vacuum_bits_visibility.py delete mode 100644 tests/ptrack_vacuum_full.py delete mode 100644 tests/ptrack_vacuum_truncate.py diff --git a/tests/ptrack.py b/tests/ptrack.py index 9f52a4664..582d5422b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -1220,8 +1220,8 @@ def test_alter_table_set_tablespace_ptrack(self): # sys.exit(1) # PTRACK BACKUP - result = node.safe_psql( - "postgres", "select * from t_heap") + #result = node.safe_psql( + # "postgres", "select * from t_heap") self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -1265,10 +1265,10 @@ def test_alter_table_set_tablespace_ptrack(self): 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from t_heap") - - self.assertEqual(result, result_new, 'lost some data after restore') +# result_new = node_restored.safe_psql( +# "postgres", "select * from t_heap") +# +# self.assertEqual(result, result_new, 'lost some data after restore') # Clean after yourself self.del_test_dir(module_name, fname) @@ -1610,6 +1610,8 @@ def test_ptrack_multiple_segments(self): self.backup_node( backup_dir, 'node', node, backup_type='ptrack') + node.safe_psql("postgres", "checkpoint") + # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -1703,3 +1705,1863 @@ def test_atexit_fail(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean_replica(self): + """ + Take backups of every available types from + master and check that PTRACK on replica is clean + """ + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'archive_timeout': '30s'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--stream', '--master-host=localhost', + '--master-db=postgres', '--master-port={0}'.format( + master.port)]) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--stream', '--master-host=localhost', + '--master-db=postgres', '--master-port={0}'.format( + master.port)]) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_gist') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # Compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_empty(self): + """Take backups of every available types and check that PTRACK is clean""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector) " + "tablespace somedata") + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + tblspace1 = self.get_tblspace_path(node, 'somedata') + tblspace2 = self.get_tblspace_path(node_restored, 'somedata') + + # Take PTRACK backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, + options=[ + "-j", "4", + "-T{0}={1}".format(tblspace1, tblspace2)] + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_empty_replica(self): + """ + Take backups of every available types from master + and check that PTRACK on replica is clean + """ + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector)") + self.wait_until_replica_catch_with_master(master, replica) + + # Take FULL backup + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + + self.wait_until_replica_catch_with_master(master, replica) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + # Take PTRACK backup + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + if self.paranoia: + pgdata = self.pgdata_content(replica.data_dir) + + self.restore_node( + backup_dir, 'replica', node_restored, + backup_id=backup_id, + options=["-j", "4"] + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + self.create_tblspace_in_node(master, 'somedata') + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make full backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'truncate t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # Make full backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Make FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Take PTRACK backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} " + "using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off', + 'archive_timeout': '30s'} + ) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " + "tsvector from generate_series(0,256000) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream' + ] + ) + # TODO: check that all ptrack are nullified + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity, the most important part + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'delete from t_heap where id > 128;') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + fname = self.id().split('.')[3] + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'checkpoint_timeout': '30'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", "create index {0} on {1} " + "using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PTRACK backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream' + ] + ) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id > 128;') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files' + ) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'ptrack_enable': 'on', + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + diff --git a/tests/ptrack_clean.py b/tests/ptrack_clean.py deleted file mode 100644 index 3eeba1b76..000000000 --- a/tests/ptrack_clean.py +++ /dev/null @@ -1,258 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack -import time - - -module_name = 'ptrack_clean' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_clean(self): - """Take backups of every available types and check that PTRACK is clean""" - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, nextval('t_seq') as t_seq, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - options=['-j10', '--stream']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['-j10']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['-j10']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_clean_replica(self): - """Take backups of every available types from master and check that PTRACK on replica is clean""" - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'archive_timeout': '30s'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='ptrack', - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='page', - options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_cluster.py b/tests/ptrack_cluster.py deleted file mode 100644 index 7fe312b3b..000000000 --- a/tests/ptrack_cluster.py +++ /dev/null @@ -1,375 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack -from time import sleep -from sys import exit - - -module_name = 'ptrack_cluster' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_cluster_on_btree(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, nextval('t_seq') as t_seq, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'cluster t_heap using t_btree') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_gist(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'cluster t_heap using t_gist') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # Compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_btree_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'cluster t_heap using t_btree') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_gist_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'cluster t_heap using t_gist') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # Compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_empty.py b/tests/ptrack_empty.py deleted file mode 100644 index 35583ff16..000000000 --- a/tests/ptrack_empty.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack -import time - - -module_name = 'ptrack_clean' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_empty(self): - """Take backups of every available types and check that PTRACK is clean""" - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'autovacuum': 'off'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector) " - "tablespace somedata") - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - options=['-j10', '--stream']) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'checkpoint') - - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - - tblspace1 = self.get_tblspace_path(node, 'somedata') - tblspace2 = self.get_tblspace_path(node_restored, 'somedata') - - # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['-j10']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, - options=[ - "-j", "4", - "-T{0}={1}".format(tblspace1, tblspace2)] - ) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_empty_replica(self): - """Take backups of every available types from master and check that PTRACK on replica is clean""" - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector)") - self.wait_until_replica_catch_with_master(master, replica) - - # Take FULL backup - self.backup_node( - backup_dir, - 'replica', - replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - - self.wait_until_replica_catch_with_master(master, replica) - - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - - # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='ptrack', - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - if self.paranoia: - pgdata = self.pgdata_content(replica.data_dir) - - self.restore_node( - backup_dir, 'replica', node_restored, - backup_id=backup_id, - options=["-j", "4"] - ) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_move_to_tablespace.py b/tests/ptrack_move_to_tablespace.py deleted file mode 100644 index 2011d2a2d..000000000 --- a/tests/ptrack_move_to_tablespace.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_move_to_tablespace' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_recovery(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text,md5(repeat(i::text,10))::tsvector as " - "tsvector from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - # Move table and indexes and make checkpoint - for i in idx_ptrack: - if idx_ptrack[i]['type'] == 'heap': - node.safe_psql( - 'postgres', - 'alter table {0} set tablespace somedata;'.format(i)) - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - 'postgres', - 'alter index {0} set tablespace somedata'.format(i)) - node.safe_psql('postgres', 'checkpoint') - - # Check ptrack files - for i in idx_ptrack: - if idx_ptrack[i]['type'] == 'seq': - continue - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack has correct bits after recovery - self.check_ptrack_recovery(idx_ptrack[i]) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_recovery.py b/tests/ptrack_recovery.py deleted file mode 100644 index 068d7dd09..000000000 --- a/tests/ptrack_recovery.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import unittest -from sys import exit -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_recovery' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_recovery(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - for i in idx_ptrack: - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack has correct bits after recovery - self.check_ptrack_recovery(idx_ptrack[i]) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_truncate.py b/tests/ptrack_truncate.py deleted file mode 100644 index 45a431e0e..000000000 --- a/tests/ptrack_truncate.py +++ /dev/null @@ -1,179 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_truncate' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_truncate(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'truncate t_heap') - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make full backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.skip("skip") - def test_ptrack_truncate_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - self.create_tblspace_in_node(master, 'somedata') - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make full backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'truncate t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum.py b/tests/ptrack_vacuum.py deleted file mode 100644 index 9ecb3a793..000000000 --- a/tests/ptrack_vacuum.py +++ /dev/null @@ -1,206 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_vacuum' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # Make full backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - # Delete some rows, vacuum it and make checkpoint - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_vacuum_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Make FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - # Delete some rows, vacuum it and make checkpoint - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_frozen.py b/tests/ptrack_vacuum_bits_frozen.py deleted file mode 100644 index e6a72bc64..000000000 --- a/tests/ptrack_vacuum_bits_frozen.py +++ /dev/null @@ -1,194 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_vacuum_bits_frozen' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_bits_frozen(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'vacuum freeze t_heap') - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_vacuum_bits_frozen_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Take PTRACK backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'vacuum freeze t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_bits_visibility.py b/tests/ptrack_vacuum_bits_visibility.py deleted file mode 100644 index 53be4401c..000000000 --- a/tests/ptrack_vacuum_bits_visibility.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_vacuum_bits_visibility' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_bits_visibility(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_full.py b/tests/ptrack_vacuum_full.py deleted file mode 100644 index 826742fba..000000000 --- a/tests/ptrack_vacuum_full.py +++ /dev/null @@ -1,201 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack -import time - - -module_name = 'ptrack_vacuum_full' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_full(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} " - "using {2}({3}) tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'vacuum full t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity, the most important part - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_full_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'autovacuum': 'off', - 'archive_timeout': '30s'} - ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " - "tsvector from generate_series(0,256000) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream' - ] - ) - # TODO: check that all ptrack are nullified - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'vacuum full t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity, the most important part - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack_vacuum_truncate.py b/tests/ptrack_vacuum_truncate.py deleted file mode 100644 index c09913539..000000000 --- a/tests/ptrack_vacuum_truncate.py +++ /dev/null @@ -1,192 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack - - -module_name = 'ptrack_vacuum_truncate' - - -class SimpleTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_truncate(self): - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'delete from t_heap where id > 128;') - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_truncate_replica(self): - fname = self.id().split('.')[3] - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", "create index {0} on {1} " - "using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Take PTRACK backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream' - ] - ) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id > 128;') - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) From 94a8a24e34adf2e5ec818e84387f58cdc4b5e8af Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Feb 2019 17:12:02 +0300 Subject: [PATCH 0330/2107] minor bugfix: check that PGDATA is not empty, following https://github.com/postgrespro/pg_probackup/issues/48 --- src/backup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backup.c b/src/backup.c index 95a6bea41..07fb36bee 100644 --- a/src/backup.c +++ b/src/backup.c @@ -634,6 +634,14 @@ do_backup_instance(void) dir_list_file(backup_files_list, instance_config.pgdata, true, true, false); + /* Sanity check for backup_files_list, thank you, Windows: + * https://github.com/postgrespro/pg_probackup/issues/48 + */ + + if (parray_num(backup_files_list) == 0) + elog(ERROR, "PGDATA is empty. Either it was concrurrently deleted or " + "pg_probackup do not possess sufficient permissions to list PGDATA content"); + /* * Sort pathname ascending. It is necessary to create intermediate * directories sequentially. From c3aa3f0cff827027144ff8cd2e139431a60c436f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Feb 2019 17:13:10 +0300 Subject: [PATCH 0331/2107] minor spell fix --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 07fb36bee..c9a4225a3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -639,7 +639,7 @@ do_backup_instance(void) */ if (parray_num(backup_files_list) == 0) - elog(ERROR, "PGDATA is empty. Either it was concrurrently deleted or " + elog(ERROR, "PGDATA is empty. Either it was concurrently deleted or " "pg_probackup do not possess sufficient permissions to list PGDATA content"); /* From 20de566010571ee408e8a7d90e8ca97b478fc329 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Feb 2019 21:26:30 +0300 Subject: [PATCH 0332/2107] tests: minor fixes --- tests/__init__.py | 11 ++++------- tests/delta.py | 4 +--- tests/helpers/ptrack_helpers.py | 17 +++++++++-------- tests/page.py | 5 ++--- tests/ptrack.py | 6 ++---- tests/restore_test.py | 6 ++++-- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 181a579d7..db0a143f8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,13 +2,10 @@ from . import init_test, merge, option_test, show_test, compatibility, \ backup_test, delete_test, delta, restore_test, validate_test, \ - retention_test, ptrack_clean, ptrack_empty, ptrack_cluster, \ - ptrack_move_to_tablespace, ptrack_recovery, ptrack_truncate, \ - ptrack_vacuum, ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \ - ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \ - false_positive, replica, compression, page, ptrack, archive, \ - exclude, cfs_backup, cfs_restore, cfs_validate_backup, auth_test, \ - time_stamp, snapfs, logging, locking, remote + retention_test, pgpro560, pgpro589, false_positive, replica, \ + compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ + cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ + locking, remote def load_tests(loader, tests, pattern): diff --git a/tests/delta.py b/tests/delta.py index d51c374cb..6ad9617f1 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -557,9 +557,7 @@ def test_delta_vacuum_full(self): gdb = self.gdb_attach(pid) gdb.set_breakpoint('reform_and_rewrite_tuple') - if not gdb.continue_execution_until_running(): - print('Failed gdb continue') - exit(1) + gdb.continue_execution_until_running() acurs.execute("VACUUM FULL t_heap") diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 97f3bc902..207c825d6 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1361,18 +1361,17 @@ def run_until_break(self): def continue_execution_until_running(self): result = self._execute('continue') - running = False for line in result: - if line.startswith('*running'): - running = True - break + if line.startswith('*running') or line.startswith('^running'): + return if line.startswith('*stopped,reason="breakpoint-hit"'): - running = False continue if line.startswith('*stopped,reason="exited-normally"'): - running = False continue - return running + + raise GdbException( + 'Failed to continue execution until running.\n' + ) def continue_execution_until_exit(self): result = self._execute('continue', False) @@ -1383,10 +1382,11 @@ def continue_execution_until_exit(self): if line.startswith('*stopped,reason="breakpoint-hit"'): continue if ( - line.startswith('*stopped,reason="exited"') or + line.startswith('*stopped,reason="exited') or line == '*stopped\n' ): return + raise GdbException( 'Failed to continue execution until exit.\n' ) @@ -1420,6 +1420,7 @@ def continue_execution_until_break(self, ignore_count=0): return 'breakpoint-hit' if line.startswith('*stopped,reason="exited-normally"'): return 'exited-normally' + if running: return 'running' diff --git a/tests/page.py b/tests/page.py index 67473ed30..a0a554de3 100644 --- a/tests/page.py +++ b/tests/page.py @@ -691,7 +691,7 @@ def test_page_backup_with_lost_wal_segment(self): # delete last wal segment wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup')] + wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] wals = map(str, wals) file = os.path.join(wals_dir, max(wals)) os.remove(file) @@ -701,8 +701,7 @@ def test_page_backup_with_lost_wal_segment(self): # Single-thread PAGE backup try: self.backup_node( - backup_dir, 'node', node, - backup_type='page') + backup_dir, 'node', node, backup_type='page') self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " diff --git a/tests/ptrack.py b/tests/ptrack.py index 582d5422b..a3d11617b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -1,6 +1,6 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta import subprocess from testgres import QueryException @@ -236,9 +236,7 @@ def test_ptrack_vacuum_full(self): gdb = self.gdb_attach(pid) gdb.set_breakpoint('reform_and_rewrite_tuple') - if not gdb.continue_execution_until_running(): - print('Failed gdb continue') - exit(1) + gdb.continue_execution_until_running() acurs.execute("VACUUM FULL t_heap") diff --git a/tests/restore_test.py b/tests/restore_test.py index 0da40c5f4..4519fe07c 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1174,8 +1174,7 @@ def test_archive_node_backup_archive_pitr_2(self): backup_dir, 'node', node_restored, options=[ "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote" - ] + "--recovery-target-action=promote"] ), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -1184,6 +1183,9 @@ def test_archive_node_backup_archive_pitr_2(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + node_restored.slow_start() result = node_restored.psql("postgres", 'select * from t_heap') From 6c7ab25f06e37bd83c48c3eb411f1f04e7127b09 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 27 Feb 2019 12:05:35 +0300 Subject: [PATCH 0333/2107] Canonicalize PGDATA path --- src/pg_probackup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e5057686c..50a50c562 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -395,6 +395,8 @@ main(int argc, char *argv[]) * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. */ + if (instance_config.pgdata != NULL) + canonicalize_path(instance_config.pgdata); if (instance_config.pgdata != NULL && !is_absolute_path(instance_config.pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); From ef2abc31bfe92a43defde0b1b315e6327be5e211 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 27 Feb 2019 17:09:39 +0300 Subject: [PATCH 0334/2107] Write WAL files with O_EXCL during archive-get and archive-push --- src/data.c | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/data.c b/src/data.c index d2ebdd934..1dff690f5 100644 --- a/src/data.c +++ b/src/data.c @@ -1102,7 +1102,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite) { FILE *in = NULL; - FILE *out=NULL; + int out; char buf[XLOG_BLCKSZ]; const char *to_path_p; char to_path_temp[MAXPGPATH]; @@ -1142,7 +1142,13 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = gzopen(to_path_temp, PG_BINARY_W); + out = open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + gz_out = gzdopen(out, PG_BINARY_W); if (gzsetparams(gz_out, instance_config.compress_level, Z_DEFAULT_STRATEGY) != Z_OK) elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", instance_config.compress_level, to_path_temp, @@ -1153,9 +1159,10 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open destination WAL file \"%s\": %s", + out = open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); } @@ -1191,7 +1198,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fwrite(buf, 1, read_len, out) != read_len) + if (write(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1219,9 +1226,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fsync(out) != 0 || close(out) != 0) { errno_temp = errno; unlink(to_path_temp); @@ -1262,7 +1267,7 @@ void get_wal_file(const char *from_path, const char *to_path) { FILE *in = NULL; - FILE *out; + int out; char buf[XLOG_BLCKSZ]; const char *from_path_p = from_path; char to_path_temp[MAXPGPATH]; @@ -1312,10 +1317,11 @@ get_wal_file(const char *from_path, const char *to_path) /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - out = fopen(to_path_temp, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open destination WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + out = open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, + S_IRUSR | S_IWUSR); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); /* copy content */ for (;;) @@ -1349,7 +1355,7 @@ get_wal_file(const char *from_path, const char *to_path) if (read_len > 0) { - if (fwrite(buf, 1, read_len, out) != read_len) + if (write(out, buf, read_len) != read_len) { errno_temp = errno; unlink(to_path_temp); @@ -1373,9 +1379,7 @@ get_wal_file(const char *from_path, const char *to_path) } } - if (fflush(out) != 0 || - fsync(fileno(out)) != 0 || - fclose(out)) + if (fsync(out) != 0 || close(out) != 0) { errno_temp = errno; unlink(to_path_temp); From 47a7bed9318ec02208df526de7010fa523e100cf Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 27 Feb 2019 19:10:06 +0300 Subject: [PATCH 0335/2107] Fix compression in remote backup --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 536dac50b..2eb71779b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -974,7 +974,7 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); bph->compressed_size = BLCKSZ; } - hdr.size += bph->compressed_size; + hdr.size += MAXALIGN(bph->compressed_size); } IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); From ee6860f054a8d0fc365e869c6188c0da831ea5dd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Feb 2019 19:16:25 +0300 Subject: [PATCH 0336/2107] minor fix: use upper register for SQL --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index c9a4225a3..36ff52eb8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1291,7 +1291,7 @@ pg_ptrack_enable(void) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "show ptrack_enable", 0, NULL); + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) { @@ -1308,7 +1308,7 @@ pg_checksum_enable(void) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "show data_checksums", 0, NULL); + res_db = pgut_execute(backup_conn, "SHOW data_checksums", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) { From 2ce874c557435f8df4976c2ce7ba8e531ab7861b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 00:30:37 +0300 Subject: [PATCH 0337/2107] minor bugfix based on Issue 50: 'pg_log' in PG10 was renamed to 'log' --- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 50a50c562..bc8a67bf8 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -442,7 +442,7 @@ main(int argc, char *argv[]) for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ /* Set 'pg_log' in first empty slot */ - pgdata_exclude_dir[i] = "pg_log"; + pgdata_exclude_dir[i] = PG_LOG_DIR; } if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 921cbd25f..a22edfee2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -36,8 +36,10 @@ #define BACKUPS_DIR "backups" #if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" +#define PG_LOG_DIR "log" #else #define PG_XLOG_DIR "pg_xlog" +#define PG_LOG_DIR "pg_log" #endif #define PG_TBLSPC_DIR "pg_tblspc" #define PG_GLOBAL_DIR "global" From 5b69fe28339d091c8c1acc7b175ee160813ad5e7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 11:02:13 +0300 Subject: [PATCH 0338/2107] help fix for Issue 50 --- src/help.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index 409c8f825..1b0f19e88 100644 --- a/src/help.c +++ b/src/help.c @@ -215,7 +215,7 @@ help_backup(void) printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); printf(_(" --stream stream the transaction log and include it in the backup\n")); printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --backup-pg-log backup of pg_log directory\n")); + printf(_(" --backup-pg-log backup of '%s' directory\n"), PG_LOG_DIR); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); From ba8410686bf66ff72e7f9af9847b39dfdc220a7a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 09:17:54 +0300 Subject: [PATCH 0339/2107] Version 2.0.27 - Bugfix: reading of compressed WAL files was very slow, it happened because xlogreader was in habit of reading XLOG page twice, which breaks zlib optimization for sequential access. See https://commitfest.postgresql.org/22/1994 for additional info. Reported by Alex Ignatov. - Bugfix: previously path to PGDATA was not canonicalized. On Windows this could lead to producing empty backups. Additionally during investigation another bug was found: readdir() silently ignored 'permission denied' errors. See https://www.postgresql.org/message-id/2cad7829-8d66-e39c-b937-ac825db5203d%40postgrespro.ru for additional info. Reported by Yuri Kurenkov. - Bugfix: archive-push didn`t use O_EXCL flag when creating '.partial' file, in rare case of two PostgreSQL instances concurrently pushing the same WAL segment it could lead to corruption of pushed WAL segment. - Minor bugfix: disappeared during backup files were marked as 'not changed', now they just evicted from file list. - Minor bugfix: skip 'log' directory during backup for PG >= 10. Reported by triwada. - Improvement: previously locking was done at instance level, because of that concurrent operations were impossible, now it is done at the backup level, so restore of backup A won`t interfere with validation of backup B. - Improvement: previously restore was relying on sort order when restoring incremental backups, now it`s relying on parent chain. - Minor improvement: when using show command "Time" for RUNNING backups now calculated on the fly. - Minor improvement: START LSN of backup now written to backup meta right after receiving, not at the end of the backup. It will be needed for "resume" feature. --- src/pg_probackup.c | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index bc8a67bf8..e6516e920 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -19,7 +19,7 @@ #include "utils/thread.h" #include -const char *PROGRAM_VERSION = "2.0.26"; +const char *PROGRAM_VERSION = "2.0.27"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 17afa4fe1..1b495cda5 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.26 \ No newline at end of file +pg_probackup 2.0.27 \ No newline at end of file From 65099ca8d77a8e3288c29c24a46e62b659cbeaf1 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 28 Feb 2019 15:53:56 +0300 Subject: [PATCH 0340/2107] Fix checking unlink result --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index d4641a6f3..b73d97f99 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -37,7 +37,7 @@ unlink_lock_atexit(void) int res; res = unlink(lock_file); - if (res != 0 && res != ENOENT) + if (res != 0 && errno != ENOENT) elog(WARNING, "%s: %s", lock_file, strerror(errno)); } From b0f8b8725fc38f6da2b0e85fcaf47ef0dc5fa5b1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 16:31:46 +0300 Subject: [PATCH 0341/2107] minor fix in elog message about block validation --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 1dff690f5..fc2d5c38e 100644 --- a/src/data.c +++ b/src/data.c @@ -1569,7 +1569,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, pg_crc32 crc; bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - elog(VERBOSE, "validate relation blocks for file %s", file->name); + elog(VERBOSE, "validate relation blocks for file %s", file->path); in = fopen(file->path, PG_BINARY_R); if (in == NULL) From b2514b9a0aea6e4c6c74c8a40a05417ba99b08f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 16:55:11 +0300 Subject: [PATCH 0342/2107] allow user to specify compress-level=0 --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e6516e920..deee06e9f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -579,7 +579,7 @@ compress_init(void) elog(ERROR, "--compress-level value must be in the range from 0 to 9"); if (instance_config.compress_level == 0) - instance_config.compress_alg = NOT_DEFINED_COMPRESS; + elog(WARNING, "Compression level 0 will lead to data bloat!"); if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) { From b6c43f93d25c4e55c75f0651f88bb324dd6e438b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 17:06:33 +0300 Subject: [PATCH 0343/2107] tests: compression.test_uncompressable_pages() added --- tests/compression.py | 54 +++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/tests/compression.py b/tests/compression.py index 2cb440187..caae94c0e 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -493,7 +493,7 @@ def test_compression_wrong_algorithm(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_uncompressable_pages(self): """ make archive node, create table with uncompressable toast pages, @@ -507,53 +507,41 @@ def test_uncompressable_pages(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} - ) + 'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() -# node.safe_psql( -# "postgres", -# "create table t_heap as select i, " -# "repeat('1234567890abcdefghiyklmn', 1)::bytea, " -# "point(0,0) from generate_series(0,1) i") - - node.safe_psql( - "postgres", - "create table t as select i, " - "repeat(md5(i::text),5006056) as fat_attr " - "from generate_series(0,10) i;") - + # Full self.backup_node( backup_dir, 'node', node, - backup_type='full', options=[ - '--compress']) - - node.cleanup() + '--compress-algorithm=zlib', + '--compress-level=0']) - self.restore_node(backup_dir, 'node', node) - node.slow_start() + node.pgbench_init(scale=3) self.backup_node( backup_dir, 'node', node, - backup_type='full', + backup_type='delta', options=[ - '--compress']) + '--compress-algorithm=zlib', + '--compress-level=0']) - # Clean after yourself - # self.del_test_dir(module_name, fname) + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() -# create table t as select i, repeat(md5('1234567890'), 1)::bytea, point(0,0) from generate_series(0,1) i; + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + node.slow_start() -# create table t_bytea_1(file oid); -# INSERT INTO t_bytea_1 (file) -# VALUES (lo_import('/home/gsmol/git/postgres/contrib/pg_probackup/tests/expected/sample.random', 24593)); -# insert into t_bytea select string_agg(data,'') from pg_largeobject where pageno > 0; -# \ No newline at end of file + # Clean after yourself + self.del_test_dir(module_name, fname) From 093e597ac882ae54369f8ba445bab4d219be6eac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 17:07:41 +0300 Subject: [PATCH 0344/2107] tests: remove deprecated modules --- tests/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index db0a143f8..be994e1c5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -30,17 +30,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(page)) suite.addTests(loader.loadTestsFromModule(ptrack)) - suite.addTests(loader.loadTestsFromModule(ptrack_clean)) - suite.addTests(loader.loadTestsFromModule(ptrack_empty)) - suite.addTests(loader.loadTestsFromModule(ptrack_cluster)) - suite.addTests(loader.loadTestsFromModule(ptrack_move_to_tablespace)) - suite.addTests(loader.loadTestsFromModule(ptrack_recovery)) - suite.addTests(loader.loadTestsFromModule(ptrack_truncate)) - suite.addTests(loader.loadTestsFromModule(ptrack_vacuum)) - suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_frozen)) - suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_bits_visibility)) - suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_full)) - suite.addTests(loader.loadTestsFromModule(ptrack_vacuum_truncate)) suite.addTests(loader.loadTestsFromModule(remote)) suite.addTests(loader.loadTestsFromModule(replica)) suite.addTests(loader.loadTestsFromModule(restore_test)) From 7a34dbe5f85f355015dbabfa116a79c9406569b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 17:54:45 +0300 Subject: [PATCH 0345/2107] fix condition for elog message about compression level 0 --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index deee06e9f..bc5968488 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -578,7 +578,7 @@ compress_init(void) if (instance_config.compress_level < 0 || instance_config.compress_level > 9) elog(ERROR, "--compress-level value must be in the range from 0 to 9"); - if (instance_config.compress_level == 0) + if (instance_config.compress_alg == ZLIB_COMPRESS && instance_config.compress_level == 0) elog(WARNING, "Compression level 0 will lead to data bloat!"); if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) From e554d265a5c763f7c9e59b0e1a0fc0b9dd8f6583 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 28 Feb 2019 18:45:52 +0300 Subject: [PATCH 0346/2107] Assign end_time in MERGE --- src/merge.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/merge.c b/src/merge.c index 78e29aa6b..865ff3692 100644 --- a/src/merge.c +++ b/src/merge.c @@ -280,6 +280,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + to_backup->end_time = time(NULL); + /* * If one of the backups isn't "stream" backup then the target backup become * non-stream backup too. From 1692cbef617ba2940636287d28ba5a9427478428 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 28 Feb 2019 19:09:36 +0300 Subject: [PATCH 0347/2107] Add merge_time for backup --- src/catalog.c | 7 +++++++ src/merge.c | 3 +++ src/pg_probackup.h | 1 + src/show.c | 4 +++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index b73d97f99..87be436c4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -525,6 +525,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->start_time); fprintf(out, "start-time = '%s'\n", timestamp); + if (backup->merge_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->merge_time); + fprintf(out, "merge-time = '%s'\n", timestamp); + } if (backup->end_time > 0) { time2iso(timestamp, lengthof(timestamp), backup->end_time); @@ -632,6 +637,7 @@ readBackupControlFile(const char *path) {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, + {'t', 0, "merge-time", &backup->merge_time, SOURCE_FILE_STRICT}, {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, @@ -857,6 +863,7 @@ pgBackupInit(pgBackup *backup) backup->start_lsn = 0; backup->stop_lsn = 0; backup->start_time = (time_t) 0; + backup->merge_time = (time_t) 0; backup->end_time = (time_t) 0; backup->recovery_xid = 0; backup->recovery_time = (time_t) 0; diff --git a/src/merge.c b/src/merge.c index 865ff3692..67852bcbc 100644 --- a/src/merge.c +++ b/src/merge.c @@ -167,8 +167,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pthread_t *threads = NULL; merge_files_arg *threads_args = NULL; int i; + time_t merge_time; bool merge_isok = true; + merge_time = time(NULL); elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); /* @@ -280,6 +282,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->stop_lsn = from_backup->stop_lsn; to_backup->recovery_time = from_backup->recovery_time; to_backup->recovery_xid = from_backup->recovery_xid; + to_backup->merge_time = merge_time; to_backup->end_time = time(NULL); /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a22edfee2..a10547dc8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -223,6 +223,7 @@ struct pgBackup XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ time_t start_time; /* since this moment backup has status * BACKUP_STATUS_RUNNING */ + time_t merge_time; /* the moment when merge was started or 0 */ time_t end_time; /* the moment when backup was finished, or the moment * when we realized that backup is broken */ time_t recovery_time; /* Earliest moment for which you can restore diff --git a/src/show.c b/src/show.c index 9ff6243fc..ffcd0038e 100644 --- a/src/show.c +++ b/src/show.c @@ -388,7 +388,9 @@ show_instance_plain(parray *backup_list, bool show_name) if (backup->status == BACKUP_STATUS_RUNNING) snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, difftime(current_time, backup->start_time)); - + else if (backup->merge_time != (time_t) 0) + snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, + difftime(backup->end_time, backup->merge_time)); else if (backup->end_time != (time_t) 0) snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, difftime(backup->end_time, backup->start_time)); From 4aab7c488b2f0f80ff6fd4eeffe9196f10d511a9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 20:09:11 +0300 Subject: [PATCH 0348/2107] tests: added compatibility.test_backward_compatibility_merge() --- tests/compatibility.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index d8e1137a0..18a19120c 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -109,6 +109,9 @@ def test_backward_compatibility_page(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_delta(self): @@ -209,6 +212,9 @@ def test_backward_compatibility_delta(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_ptrack(self): @@ -309,6 +315,9 @@ def test_backward_compatibility_ptrack(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_compression(self): @@ -467,3 +476,64 @@ def test_backward_compatibility_compression(self): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, + old_binary=True) + + node.pgbench_init(scale=1) + + # PAGE backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.merge_backup(backup_dir, "node", backup_id) + + print(self.show_pb(backup_dir, as_text=True)) + + # restore OLD FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 75b0ff28812f72cc81e35d7544f367bc3c719fc6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 21:12:32 +0300 Subject: [PATCH 0349/2107] tests: minor fixes --- tests/compatibility.py | 4 +-- tests/merge.py | 68 +++++++++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 18a19120c..c2a7fe416 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -485,7 +485,7 @@ def test_backward_compatibility_compression(self): def test_backward_compatibility_merge(self): """ Create node, take FULL and PAGE backups with old binary, - merge them with + merge them with new binary """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -520,7 +520,7 @@ def test_backward_compatibility_merge(self): self.merge_backup(backup_dir, "node", backup_id) - print(self.show_pb(backup_dir, as_text=True)) + print(self.show_pb(backup_dir, as_text=True, as_json=False)) # restore OLD FULL with new binary node_restored = self.make_simple_node( diff --git a/tests/merge.py b/tests/merge.py index 4a0cb31cf..e3fbffa9b 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1224,28 +1224,23 @@ def test_continue_failed_merge_2(self): "postgres", "create table t_heap as select i as id," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i" - ) + " from generate_series(0,1000) i") # DELTA BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') node.safe_psql( "postgres", - "delete from t_heap" - ) + "delete from t_heap") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") # DELTA BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1285,6 +1280,59 @@ def test_continue_failed_merge_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_merge_different_compression_algo(self): + """ + Check that backups with different compression algorihtms can be merged + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--compress-algorithm=zlib']) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--compress-algorithm=pglz']) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + self.merge_backup(backup_dir, "node", backup_id) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 094dc8b16666047c72e9dc8fae4f9451d4d61385 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Feb 2019 23:38:38 +0300 Subject: [PATCH 0350/2107] mark master-* options as deprecated --- src/help.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/help.c b/src/help.c index 1b0f19e88..6574c76b5 100644 --- a/src/help.c +++ b/src/help.c @@ -267,10 +267,10 @@ help_backup(void) printf(_(" -W, --password force password prompt\n")); printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master\n")); - printf(_(" --master-db=db_name database to connect to master\n")); - printf(_(" --master-host=host_name database server host of master\n")); - printf(_(" --master-port=port database server port of master\n")); + printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); + printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); + printf(_(" --master-host=host_name database server host of master (deprecated)\n")); + printf(_(" --master-port=port database server port of master (deprecated)\n")); printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); } @@ -534,10 +534,10 @@ help_set_config(void) printf(_(" -p, --port=PORT database server port (default: 5432)\n")); printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master\n")); - printf(_(" --master-db=db_name database to connect to master\n")); - printf(_(" --master-host=host_name database server host of master\n")); - printf(_(" --master-port=port database server port of master\n")); + printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); + printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); + printf(_(" --master-host=host_name database server host of master (deprecated)\n")); + printf(_(" --master-port=port database server port of master (deprecated)\n")); printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); printf(_("\n Archive options:\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); From 6e9a5793ef4d14a53bf630eb6cae523afb9d540a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 02:41:13 +0300 Subject: [PATCH 0351/2107] mark --replica-timeout as deprecated --- src/help.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index 6574c76b5..3be757723 100644 --- a/src/help.c +++ b/src/help.c @@ -271,7 +271,7 @@ help_backup(void) printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); printf(_(" --master-host=host_name database server host of master (deprecated)\n")); printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n")); } static void @@ -533,14 +533,15 @@ help_set_config(void) printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_("\n Replica options:\n")); printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); printf(_(" --master-host=host_name database server host of master (deprecated)\n")); printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (default: 5min)\n")); - printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n")); } static void From 4097e1075410d9911b59179e8731885ba7447e1f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Mar 2019 11:18:00 +0300 Subject: [PATCH 0352/2107] Fix printing NULL values --- src/utils/configuration.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index fe50c4946..eabd35ebd 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -267,12 +267,24 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src) } } - if (isprint(opt->sname)) - elog(ERROR, "Option -%c, --%s should be %s: '%s'", - opt->sname, opt->lname, message, optarg); + if (optarg) + { + if (isprint(opt->sname)) + elog(ERROR, "Option -%c, --%s should be %s: '%s'", + opt->sname, opt->lname, message, optarg); + else + elog(ERROR, "Option --%s should be %s: '%s'", + opt->lname, message, optarg); + } else - elog(ERROR, "Option --%s should be %s: '%s'", - opt->lname, message, optarg); + { + if (isprint(opt->sname)) + elog(ERROR, "Option -%c, --%s should be %s", + opt->sname, opt->lname, message); + else + elog(ERROR, "Option --%s should be %s", + opt->lname, message); + } } static const char * From 42449fe89546141cc9e39c101585ad36b9f721b0 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Mar 2019 12:53:48 +0300 Subject: [PATCH 0353/2107] Use remote_host for pghost if last one was not specified --- src/pg_probackup.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1807c308a..0908509c3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -448,7 +448,10 @@ main(int argc, char *argv[]) elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); } - /* Setup stream options. They are used in streamutil.c. */ + if (!instance_config.pghost && instance_config.remote.host) + instance_config.pghost = instance_config.remote.host; + + /* Setup stream options. They are used in streamutil.c. */ if (instance_config.pghost != NULL) dbhost = pstrdup(instance_config.pghost); if (instance_config.pgport != NULL) From 98bc12d5359e73d6c2588a870fd154b9ef1903cb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 16:09:53 +0300 Subject: [PATCH 0354/2107] new flag '--temp-slot' allows user to use temporary slot for STREAM backups, if no slot name is specified via '--slot' option - default name 'pg_probackup_slot' is used --- src/backup.c | 7 +++++++ src/help.c | 13 +++++++------ src/pg_probackup.c | 2 ++ src/pg_probackup.h | 1 + 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index 36ff52eb8..2f143ea4a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2766,6 +2766,12 @@ StreamLog(void *arg) stream_stop_timeout = 0; stream_stop_begin = 0; +#if PG_VERSION_NUM >= 100000 + /* if slot name was not provided for temp slot, use default slot name */ + if (!replication_slot && temp_slot) + replication_slot = "pg_probackup_slot"; +#endif + /* * Start the replication */ @@ -2786,6 +2792,7 @@ StreamLog(void *arg) ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; + ctl.temp_slot = temp_slot; #else ctl.basedir = (char *) stream_arg->basedir; #endif diff --git a/src/help.c b/src/help.c index 3be757723..0068ecee2 100644 --- a/src/help.c +++ b/src/help.c @@ -97,9 +97,9 @@ help_pg_probackup(void) printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress]\n")); + printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [--backup-pg-log] [-j num-threads]\n")); + printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -186,9 +186,9 @@ static void help_backup(void) { printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name]] [--backup-pg-log]\n")); - printf(_(" [-j num-threads] [--archive-timeout=archive-timeout]\n")); - printf(_(" [--progress]\n")); + printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [--backup-pg-log] [-j num-threads]\n")); + printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -215,6 +215,7 @@ help_backup(void) printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); printf(_(" --stream stream the transaction log and include it in the backup\n")); printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --temp-slot use temporary replication slot\n")); printf(_(" --backup-pg-log backup of '%s' directory\n"), PG_LOG_DIR); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index bc5968488..a56658cfe 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -62,6 +62,7 @@ bool progress = false; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; #endif +bool temp_slot = false; /* backup options */ bool backup_logs = false; @@ -133,6 +134,7 @@ static ConfigOption cmd_options[] = { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, + { 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, /* TODO not completed feature. Make it unavailiable from user level diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a10547dc8..23db23541 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -359,6 +359,7 @@ extern bool progress; /* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; #endif +extern bool temp_slot; /* backup options */ extern bool smooth_checkpoint; From 4d8a9437ad7eba24712325d60d7f6b1de766e8d5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 1 Mar 2019 16:16:58 +0300 Subject: [PATCH 0355/2107] MERGE fix: Update program_version of the target backup, recalculate CRC of a unchanged file --- src/merge.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/merge.c b/src/merge.c index 67852bcbc..c8fc560cd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -277,6 +277,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * Update to_backup metadata. */ to_backup->status = BACKUP_STATUS_OK; + StrNCpy(to_backup->program_version, PROGRAM_VERSION, + sizeof(to_backup->program_version)); to_backup->parent_backup = INVALID_BACKUP_ID; to_backup->start_lsn = from_backup->start_lsn; to_backup->stop_lsn = from_backup->stop_lsn; @@ -393,6 +395,7 @@ merge_files(void *arg) pgFile *file = (pgFile *) parray_get(argument->files, i); pgFile *to_file; pgFile **res_file; + char to_file_path[MAXPGPATH]; /* Path of target file */ char from_file_path[MAXPGPATH]; char *prev_file_path; @@ -415,6 +418,8 @@ merge_files(void *arg) pgFileComparePathDesc); to_file = (res_file) ? *res_file : NULL; + join_path_components(to_file_path, argument->to_root, file->path); + /* * Skip files which haven't changed since previous backup. But in case * of DELTA backup we should consider n_blocks to truncate the target @@ -433,7 +438,15 @@ merge_files(void *arg) { file->compress_alg = to_file->compress_alg; file->write_size = to_file->write_size; - file->crc = to_file->crc; + + /* + * Recalculate crc for backup prior to 2.0.25. + */ + if (parse_program_version(from_backup->program_version) < 20025) + file->crc = pgFileGetCRC(to_file_path, true, true, NULL); + /* Otherwise just get it from the previous file */ + else + file->crc = to_file->crc; } continue; @@ -453,10 +466,6 @@ merge_files(void *arg) if (file->is_datafile && !file->is_cfs) { - char to_file_path[MAXPGPATH]; /* Path of target file */ - - join_path_components(to_file_path, argument->to_root, prev_file_path); - /* * We need more complicate algorithm if target file should be * compressed. From d9cb01a167d38bfc17ca39cde43adf6104aa364e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 16:29:47 +0300 Subject: [PATCH 0356/2107] tests: added backup_test.BackupTest.test_persistent_slot_for_stream_backup() and backup_test.BackupTest.test_temp_slot_for_stream_backup() --- tests/archive.py | 6 ++-- tests/backup_test.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 0881a0505..51e46ea0e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -728,8 +728,8 @@ def test_archive_pg_receivexlog(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_senders': '2', - 'checkpoint_timeout': '30s'} - ) + 'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -808,7 +808,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): self.add_instance(backup_dir, 'node', node) node.slow_start() if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL 10 for this test') + return unittest.skip('You need PostgreSQL >= 10 for this test') else: pg_receivexlog_path = self.get_bin_path('pg_receivewal') diff --git a/tests/backup_test.py b/tests/backup_test.py index 740f08d37..9570b3d3d 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -938,3 +938,74 @@ def test_drop_rel_during_backup_ptrack(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_persistent_slot_for_stream_backup(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '40MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "SELECT pg_create_physical_replication_slot('slot_1')") + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1']) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_temp_slot_for_stream_backup(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_size': '40MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL >= 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1', '--temp-slot']) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--temp-slot']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From c94c6d06f0754dca5e0a830b4c6d871ec090075e Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Fri, 1 Mar 2019 18:14:26 +0300 Subject: [PATCH 0357/2107] Fix merging external directories --- src/dir.c | 10 +++ src/merge.c | 18 ++--- src/pg_probackup.h | 1 + tests/external.py | 171 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 8 deletions(-) diff --git a/src/dir.c b/src/dir.c index 11bfc5441..4d4af0e41 100644 --- a/src/dir.c +++ b/src/dir.c @@ -383,6 +383,16 @@ pgFileComparePathDesc(const void *f1, const void *f2) return -pgFileComparePath(f1, f2); } +/* + * Compare two pgFile with their path and external_dir_num + * in descending order of ASCII code. + */ +int +pgFileComparePathWithExternalDesc(const void *f1, const void *f2) +{ + return -pgFileComparePathWithExternal(f1, f2); +} + /* Compare two pgFile with their linked directory path. */ int pgFileCompareLinked(const void *f1, const void *f2) diff --git a/src/merge.c b/src/merge.c index b0906a5c6..c71280991 100644 --- a/src/merge.c +++ b/src/merge.c @@ -372,12 +372,14 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); - char *dir_name = parray_get(to_external, file->external_dir_num - 1); - if (file->external_dir_num && - backup_contains_external(dir_name, from_external)) - /* Dir already removed*/ - continue; + if (file->external_dir_num && to_external) + { + char *dir_name = parray_get(to_external, file->external_dir_num - 1); + if (backup_contains_external(dir_name, from_external)) + /* Dir already removed*/ + continue; + } if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) { @@ -463,7 +465,7 @@ merge_files(void *arg) i + 1, num_files, file->path); res_file = parray_bsearch(argument->to_files, file, - pgFileComparePathDesc); + pgFileComparePathWithExternalDesc); to_file = (res_file) ? *res_file : NULL; /* @@ -611,8 +613,6 @@ merge_files(void *arg) file->crc = pgFileGetCRC(to_file_path, true, true, NULL); } } - else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(argument->from_root, argument->to_root, file); else if (file->external_dir_num) { char from_root[MAXPGPATH]; @@ -630,6 +630,8 @@ merge_files(void *arg) new_dir_num); copy_file(from_root, to_root, file); } + else if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(argument->from_root, argument->to_root, file); else copy_file(argument->from_root, argument->to_root, file); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 97a5ed60d..b435b4270 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -539,6 +539,7 @@ extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathWithExternal(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); +extern int pgFileComparePathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); diff --git a/tests/external.py b/tests/external.py index d346706ad..6469d39d0 100644 --- a/tests/external.py +++ b/tests/external.py @@ -580,6 +580,177 @@ def test_external_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_single(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup with changed data + backup_id = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_double(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + # delta backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # delta backup without external directories + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + # external directory contain symlink to file # external directory contain symlink to directory # latest page backup without external_dir From 423491994a992ef59eb003ddf086b8c6cc2c5149 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Fri, 1 Mar 2019 18:37:32 +0300 Subject: [PATCH 0358/2107] Add flag "--skip-external-dirs" to restore command --- src/help.c | 5 ++++- src/pg_probackup.c | 2 ++ src/pg_probackup.h | 1 + src/restore.c | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/help.c b/src/help.c index 4aea75eaa..8c2cf18bc 100644 --- a/src/help.c +++ b/src/help.c @@ -131,6 +131,7 @@ help_pg_probackup(void) printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--skip-external-dirs]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress]\n")); @@ -290,7 +291,8 @@ help_restore(void) printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--no-validate]\n")); - printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [--skip-block-validation]\n")); + printf(_(" [--skip-external-dirs]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -320,6 +322,7 @@ help_restore(void) printf(_(" to ease setting up a standby server\n")); printf(_(" --no-validate disable backup validation during restore\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index d5b51df3f..2677ab528 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -86,6 +86,7 @@ bool restore_as_replica = false; bool restore_no_validate = false; bool skip_block_validation = false; +bool skip_external_dirs = false; /* delete options */ bool delete_wal = false; @@ -153,6 +154,7 @@ static ConfigOption cmd_options[] = { 'b', 143, "no-validate", &restore_no_validate, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, + { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b435b4270..2388a2039 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -375,6 +375,7 @@ extern bool exclusive_backup; /* restore options */ extern bool restore_as_replica; extern bool skip_block_validation; +extern bool skip_external_dirs; /* delete options */ extern bool delete_wal; diff --git a/src/restore.c b/src/restore.c index 97505075b..0d10ac350 100644 --- a/src/restore.c +++ b/src/restore.c @@ -480,7 +480,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); create_data_directories(instance_config.pgdata, this_backup_path, true); - if(external_dir_str) + if(external_dir_str && !skip_external_dirs) { requested_external_dirs = make_external_directory_list(external_dir_str); for (i = 0; i < parray_num(requested_external_dirs); i++) @@ -514,7 +514,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) { pgFile *file = (pgFile *) parray_get(files, i); - /* if the entry was an external directory, create it in the backup */ + /* If the entry was an external directory, create it in the backup */ if (file->external_dir_num && S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; From 53bd83bfcd1b3aa4e5602abb8ea5f3a09218f3c6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 18:40:29 +0300 Subject: [PATCH 0359/2107] fix temp replication slot for PG >= 11 --- src/backup.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/backup.c b/src/backup.c index 2f143ea4a..5b5f30005 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2772,6 +2772,14 @@ StreamLog(void *arg) replication_slot = "pg_probackup_slot"; #endif + +#if PG_VERSION_NUM >= 110000 + /* Create temp repslot */ + if (temp_slot) + CreateReplicationSlot(stream_arg->conn, replication_slot, + NULL, temp_slot, true, true, false); +#endif + /* * Start the replication */ @@ -2792,7 +2800,9 @@ StreamLog(void *arg) ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; +#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 ctl.temp_slot = temp_slot; +#endif #else ctl.basedir = (char *) stream_arg->basedir; #endif From 322b3f0b2180b9177877949af3a9c5fef7f00345 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 19:19:56 +0300 Subject: [PATCH 0360/2107] tests: fixes for PG 9.5 --- tests/__init__.py | 2 +- tests/backup_test.py | 63 +++++++++------------ tests/validate_test.py | 125 ++++++++++++++++++----------------------- 3 files changed, 80 insertions(+), 110 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index be994e1c5..f827a4f32 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -29,7 +29,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(merge)) suite.addTests(loader.loadTestsFromModule(option_test)) suite.addTests(loader.loadTestsFromModule(page)) - suite.addTests(loader.loadTestsFromModule(ptrack)) +# suite.addTests(loader.loadTestsFromModule(ptrack)) suite.addTests(loader.loadTestsFromModule(remote)) suite.addTests(loader.loadTestsFromModule(replica)) suite.addTests(loader.loadTestsFromModule(restore_test)) diff --git a/tests/backup_test.py b/tests/backup_test.py index 9570b3d3d..ae42ec5d9 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -19,9 +19,8 @@ def test_backup_modes_archive(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on'} - ) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -89,9 +88,8 @@ def test_smooth_checkpoint(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -114,8 +112,8 @@ def test_incremental_backup_without_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -170,8 +168,8 @@ def test_incremental_backup_corrupt_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -235,8 +233,8 @@ def test_ptrack_threads(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -265,10 +263,9 @@ def test_ptrack_threads_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -295,8 +292,8 @@ def test_page_corruption_heal_via_ptrack_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -352,8 +349,8 @@ def test_page_corruption_heal_via_ptrack_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -425,8 +422,8 @@ def test_tablespace_in_pgdata_pgpro_1376(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -526,7 +523,6 @@ def test_tablespace_handling(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -639,7 +635,6 @@ def test_tablespace_handling_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -700,7 +695,6 @@ def test_tablespace_handling_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -751,9 +745,7 @@ def test_drop_rel_during_backup_delta(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -820,9 +812,7 @@ def test_drop_rel_during_backup_page(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -885,7 +875,6 @@ def test_drop_rel_during_backup_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'ptrack_enable': 'on'}) self.init_pb(backup_dir) @@ -949,7 +938,6 @@ def test_persistent_slot_for_stream_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_size': '40MB'}) self.init_pb(backup_dir) @@ -984,7 +972,6 @@ def test_temp_slot_for_stream_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_size': '40MB'}) self.init_pb(backup_dir) @@ -992,6 +979,11 @@ def test_temp_slot_for_stream_backup(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--temp-slot']) + if self.get_version(node) < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL >= 10 for this test') else: @@ -1002,10 +994,5 @@ def test_temp_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--slot=slot_1', '--temp-slot']) - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--temp-slot']) - # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/validate_test.py b/tests/validate_test.py index c6738e578..cb6247a1c 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -22,9 +22,8 @@ def test_validate_nullified_heap_page_backup(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -74,9 +73,8 @@ def test_validate_wal_unreal_values(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -214,9 +212,8 @@ def test_validate_corrupted_intermediate_backup(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -300,9 +297,8 @@ def test_validate_corrupted_intermediate_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -423,9 +419,8 @@ def test_validate_specific_error_intermediate_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -512,9 +507,8 @@ def test_validate_error_intermediate_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -596,9 +590,8 @@ def test_validate_corrupted_intermediate_backups_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -794,9 +787,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -979,9 +971,8 @@ def test_validate_instance_with_corrupted_page(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1126,9 +1117,8 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): try to restore backup with --no-validation option""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1222,9 +1212,8 @@ def test_validate_instance_with_corrupted_full(self): expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1316,9 +1305,8 @@ def test_validate_corrupt_wal_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1378,9 +1366,8 @@ def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" fname = self.id().split('.')[3] node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1450,9 +1437,8 @@ def test_validate_wal_lost_segment_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1527,9 +1513,8 @@ def test_validate_corrupt_wal_between_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1623,8 +1608,8 @@ def test_pgpro702_688(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1665,8 +1650,8 @@ def test_pgpro688(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1695,8 +1680,8 @@ def test_pgpro561(self): base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) @@ -1788,10 +1773,9 @@ def test_validate_corrupted_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'checkpoint_timeout': '30'} - ) + 'checkpoint_timeout': '30'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1913,8 +1897,8 @@ def test_validate_corrupted_full_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2025,8 +2009,8 @@ def test_validate_corrupted_full_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2375,8 +2359,8 @@ def test_validate_corrupted_full_missing(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2598,7 +2582,6 @@ def test_file_size_corruption_no_validate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), # initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} ) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2677,8 +2660,8 @@ def test_validate_specific_backup_with_missing_backup(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2819,8 +2802,8 @@ def test_validate_specific_backup_with_missing_backup_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2940,8 +2923,8 @@ def test_validate_with_missing_backup_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3128,8 +3111,8 @@ def test_validate_with_missing_backup_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3276,8 +3259,8 @@ def test_corrupt_pg_control_via_resetxlog(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 87dd195e5a2955a3fe05291200897cbde5028009 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 1 Mar 2019 19:34:55 +0300 Subject: [PATCH 0361/2107] Fix pgFileGetCRC call in merge.c --- src/merge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 8e91dea28..d522617ef 100644 --- a/src/merge.c +++ b/src/merge.c @@ -443,7 +443,7 @@ merge_files(void *arg) * Recalculate crc for backup prior to 2.0.25. */ if (parse_program_version(from_backup->program_version) < 20025) - file->crc = pgFileGetCRC(to_file_path, true, true, NULL); + file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST); /* Otherwise just get it from the previous file */ else file->crc = to_file->crc; From 236a9069d2044b83d469dbd87d52969c063d68ab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Mar 2019 22:44:57 +0300 Subject: [PATCH 0362/2107] tests: do not use remote archive-push/archive-get --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 59a1fe6f6..aea69771b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -943,8 +943,8 @@ def set_archiving( backup_dir.replace("\\","\\\\"), instance) - if self.remote: - archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' + # if self.remote: + # archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' if self.archive_compress or compress: archive_command = archive_command + '--compress ' From 807d46a49d341972e1874be3a53daade69325433 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Mar 2019 00:43:46 +0300 Subject: [PATCH 0363/2107] tests: fixes for test_external_merge_1() --- tests/external.py | 95 +++++++++++++++++++++++++++-------------------- tests/merge.py | 44 ++++++++++++++++++++++ 2 files changed, 99 insertions(+), 40 deletions(-) diff --git a/tests/external.py b/tests/external.py index 6469d39d0..49f21e234 100644 --- a/tests/external.py +++ b/tests/external.py @@ -24,7 +24,7 @@ def test_external_simple(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica'}) + 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') @@ -83,7 +83,7 @@ def test_external_none(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica'}) + 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') @@ -129,7 +129,7 @@ def test_external_none(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - # self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_external_dir_mapping(self): @@ -145,7 +145,6 @@ def test_external_dir_mapping(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -233,7 +232,7 @@ def test_backup_multiple_external(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica'}) + 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -292,7 +291,7 @@ def test_backup_multiple_external(self): # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility(self): - """Description in jira issue PGPRO-434""" + """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -385,6 +384,9 @@ def test_external_backward_compatibility(self): self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge(self): @@ -461,21 +463,24 @@ def test_external_merge(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') self.restore_node( - backup_dir, 'node', node_restored, + backup_dir, 'node', node, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) + node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_1(self): @@ -496,54 +501,55 @@ def test_external_merge_1(self): node.pgbench_init(scale=10) - # FULL backup with old binary without external dirs support - self.backup_node( + # FULL backup with old data + backup_id_1 = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) external_dir1_old = self.get_tblspace_path(node, 'external_dir1') external_dir2_old = self.get_tblspace_path(node, 'external_dir2') - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) - - # FULL backup - backup_id = self.backup_node( + # FULL backup with new data + backup_id_2 = self.backup_node( backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) + options=["-j", "4", "--stream"]) - # fill external directories with changed data + # fill external directories with old data self.restore_node( - backup_dir, 'node', node, + backup_dir, 'node', node, backup_id=backup_id_1, data_dir=external_dir1_old, options=["-j", "4"]) self.restore_node( - backup_dir, 'node', node, + backup_dir, 'node', node, backup_id=backup_id_1, data_dir=external_dir2_old, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # delta backup with external directories + # FULL backup with external directories self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_dir, 'node', node, options=[ "-j", "4", "--stream", "-E", "{0}:{1}".format( external_dir1_old, external_dir2_old)]) - # delta backup with external directories + # drop old external data + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # fill external directories with new data + self.restore_node( + backup_dir, 'node', node, backup_id=backup_id_2, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, backup_id=backup_id_2, + data_dir=external_dir2_old, options=["-j", "4"]) + + # drop now not needed backups + + # DELTA backup with external directories backup_id = self.backup_node( backup_dir, 'node', node, backup_type="delta", options=[ @@ -552,13 +558,13 @@ def test_external_merge_1(self): external_dir1_old, external_dir2_old)]) - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) - pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - # delta backup without external directories + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + # merge backups without external directories self.merge_backup(backup_dir, 'node', backup_id=backup_id) # RESTORE @@ -569,7 +575,7 @@ def test_external_merge_1(self): external_dir2_new = self.get_tblspace_path(node, 'external_dir2') self.restore_node( - backup_dir, 'node', node_restored, + backup_dir, 'node', node, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), @@ -580,6 +586,9 @@ def test_external_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_single(self): @@ -659,6 +668,9 @@ def test_external_merge_single(self): self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_double(self): @@ -751,6 +763,9 @@ def test_external_merge_double(self): self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself + self.del_test_dir(module_name, fname) + # external directory contain symlink to file # external directory contain symlink to directory # latest page backup without external_dir diff --git a/tests/merge.py b/tests/merge.py index e3fbffa9b..77e52c7db 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1333,6 +1333,50 @@ def test_merge_different_compression_algo(self): self.del_test_dir(module_name, fname) + def test_merge_different_wal_modes(self): + """ + Check that backups with different wal modes can be merged + correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'false', self.show_backup(backup_dir, 'node', backup_id)['stream']) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + self.merge(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'true', self.show_pb(backup_dir, 'node', backup_id)['stream']) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 459a4df987fec8b85c275b70a51fd2f3fb4a5232 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Mar 2019 02:05:36 +0300 Subject: [PATCH 0364/2107] tests: added external.ExternalTest.test_restore_skip_external() --- tests/external.py | 92 ++++++++++++++++++++++++++++++++++++++++++----- tests/merge.py | 18 +++++----- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/tests/external.py b/tests/external.py index 49f21e234..888188563 100644 --- a/tests/external.py +++ b/tests/external.py @@ -310,7 +310,7 @@ def test_external_backward_compatibility(self): node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=5) # FULL backup with old binary without external dirs support self.backup_node( @@ -405,7 +405,7 @@ def test_external_merge(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=5) # FULL backup with old binary without external dirs support self.backup_node( @@ -499,7 +499,7 @@ def test_external_merge_1(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=5) # FULL backup with old data backup_id_1 = self.backup_node( @@ -607,7 +607,7 @@ def test_external_merge_single(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=5) # FULL backup self.backup_node( @@ -689,7 +689,7 @@ def test_external_merge_double(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=5) # FULL backup self.backup_node( @@ -766,9 +766,85 @@ def test_external_merge_double(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_skip_external(self): + """ + Check that --skip-external-dirs works correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + node.pgbench_init(scale=3) + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # FULL backup with external directories + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1, + external_dir2)]) + + # delete first externals, so pgdata_compare + # will be capable of detecting redundant + # external files after restore + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--skip-external-dirs"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # external directory contain symlink to file # external directory contain symlink to directory - # latest page backup without external_dir + # latest page backup without external_dir + # multiple external directories + # --external-dirs=none + # --external-dirs point to a file @@ -776,5 +852,5 @@ def test_external_merge_double(self): # external directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories # check that not changed files are not copied by next backup + - # merge - # complex merge + # merge + + # complex merge + diff --git a/tests/merge.py b/tests/merge.py index 77e52c7db..23b3dceda 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1343,37 +1343,35 @@ def test_merge_different_wal_modes(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - # FULL backup + # FULL stream backup self.backup_node( backup_dir, 'node', node, options=['--stream']) - - # DELTA BACKUP + # DELTA archive backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta') - self.merge(backup_dir, 'node', backup_id=backup_id) + self.merge_backup(backup_dir, 'node', backup_id=backup_id) self.assertEqual( - 'false', self.show_backup(backup_dir, 'node', backup_id)['stream']) + 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) - # DELTA BACKUP + # DELTA stream backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream']) - self.merge(backup_dir, 'node', backup_id=backup_id) + self.merge_backup(backup_dir, 'node', backup_id=backup_id) self.assertEqual( - 'true', self.show_pb(backup_dir, 'node', backup_id)['stream']) + 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) self.del_test_dir(module_name, fname) From 2466e518715063dd93e528a46d0ce68a9dfa13e2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Mar 2019 02:29:58 +0300 Subject: [PATCH 0365/2107] tests: fix gdbobject method continue_execution_until_break() --- tests/__init__.py | 12 +----------- tests/delta.py | 4 +--- tests/helpers/ptrack_helpers.py | 11 ++++------- tests/locking.py | 12 ++++-------- tests/merge.py | 12 +++--------- tests/ptrack.py | 4 +--- 6 files changed, 14 insertions(+), 41 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index f827a4f32..fdba4164f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,7 +2,7 @@ from . import init_test, merge, option_test, show_test, compatibility, \ backup_test, delete_test, delta, restore_test, validate_test, \ - retention_test, pgpro560, pgpro589, false_positive, replica, \ + retention_test, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ locking, remote @@ -47,18 +47,8 @@ def load_tests(loader, tests, pattern): # ToDo: # archive: # discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work -# replica: -# backup should exit with correct error message if some master* option is missing -# --master* options shoukd not work when backuping master # logging: # https://jira.postgrespro.ru/browse/PGPRO-584 # https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md # archive: # immediate recovery and full recovery -# backward compatibility: -# previous version catalog must be readable by newer version -# incremental chain from previous version can be continued -# backups from previous version can be restored -# 10vanilla_1.3ptrack + -# 10vanilla+ -# 9.6vanilla_1.3ptrack + diff --git a/tests/delta.py b/tests/delta.py index 6ad9617f1..d5c3a03f8 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -562,9 +562,7 @@ def test_delta_vacuum_full(self): acurs.execute("VACUUM FULL t_heap") if gdb.stopped_in_breakpoint(): - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 207c825d6..bcce8448a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1412,17 +1412,14 @@ def continue_execution_until_break(self, ignore_count=0): else: result = self._execute('continue', False) - running = False for line in result: - if line.startswith('*running'): - running = True if line.startswith('*stopped,reason="breakpoint-hit"'): - return 'breakpoint-hit' + return if line.startswith('*stopped,reason="exited-normally"'): - return 'exited-normally' + break - if running: - return 'running' + raise GdbException( + 'Failed to continue execution until break.\n') def stopped_in_breakpoint(self): output = [] diff --git a/tests/locking.py b/tests/locking.py index 44fff4de8..b4b271f46 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -37,8 +37,7 @@ def test_locking_running_validate_1(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -93,8 +92,7 @@ def test_locking_running_validate_2(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() @@ -164,8 +162,7 @@ def test_locking_running_validate_2_specific_id(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() @@ -263,8 +260,7 @@ def test_locking_running_3(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() diff --git a/tests/merge.py b/tests/merge.py index e3fbffa9b..679bdb041 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1071,9 +1071,7 @@ def test_continue_failed_merge(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1154,9 +1152,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(2) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -1252,9 +1248,7 @@ def test_continue_failed_merge_2(self): gdb.set_breakpoint('pgFileDelete') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') diff --git a/tests/ptrack.py b/tests/ptrack.py index a3d11617b..b79528171 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -241,9 +241,7 @@ def test_ptrack_vacuum_full(self): acurs.execute("VACUUM FULL t_heap") if gdb.stopped_in_breakpoint(): - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, backup_type='ptrack') From dbb787f5c82b2b85b4466b797c85c8dade29cb20 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 2 Mar 2019 09:13:09 +0300 Subject: [PATCH 0366/2107] Propagate errno from remote agent --- src/utils/file.c | 28 ++++++++++++++++++++++------ src/utils/remote.c | 1 + 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index cd3097edb..16ae8bfb3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -535,7 +535,12 @@ int fio_fstat(int fd, struct stat* st) Assert(hdr.cop == FIO_FSTAT); IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - return hdr.arg; + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + return 0; } else { @@ -563,7 +568,12 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati Assert(hdr.cop == FIO_STAT); IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - return hdr.arg; + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + return 0; } else { @@ -589,7 +599,12 @@ int fio_access(char const* path, int mode, fio_location location) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_ACCESS); - return hdr.arg; + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + return 0; } else { @@ -1074,19 +1089,20 @@ void fio_communicate(int in, int out) break; case FIO_FSTAT: /* Get information about opened file */ hdr.size = sizeof(st); - hdr.arg = fstat(fd[hdr.handle], &st); + hdr.arg = fstat(fd[hdr.handle], &st) < 0 ? errno : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); - hdr.arg = hdr.arg ? stat(buf, &st) : lstat(buf, &st); + rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); + hdr.arg = rc < 0 ? errno : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; case FIO_ACCESS: /* Check presence of file with specified name */ hdr.size = 0; - hdr.arg = access(buf, hdr.arg); + hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_RENAME: /* Rename file */ diff --git a/src/utils/remote.c b/src/utils/remote.c index eced5f87d..51b1a336e 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -128,6 +128,7 @@ bool launch_agent(void) if (execvp(ssh_argv[0], ssh_argv) < 0) return false; } else { + elog(LOG, "Spawn agent %d", child_pid); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); /*atexit(kill_child);*/ From a50ffed5d4fd489c68b5d932862bfb19aaca1e73 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 3 Mar 2019 02:35:06 +0300 Subject: [PATCH 0367/2107] tests: add test_merge_different_wal_modes --- tests/merge.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 679bdb041..6e987a2a4 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1327,6 +1327,48 @@ def test_merge_different_compression_algo(self): self.del_test_dir(module_name, fname) + def test_merge_different_wal_modes(self): + """ + Check that backups with different wal modes can be merged + correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + # DELTA stream backup + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 04ff6f8fac9db44e5133488a53eec25f9c7a109b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 16:40:11 +0300 Subject: [PATCH 0368/2107] Inherit WAL mode from the PAGE backup --- src/merge.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/merge.c b/src/merge.c index c8fc560cd..30891d5d6 100644 --- a/src/merge.c +++ b/src/merge.c @@ -288,10 +288,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->end_time = time(NULL); /* - * If one of the backups isn't "stream" backup then the target backup become - * non-stream backup too. + * Target backup must inherit wal mode too. */ - to_backup->stream = to_backup->stream && from_backup->stream; + to_backup->stream = from_backup->stream; /* Compute summary of size of regular files in the backup */ to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) From e9190e6cc33ea8835c502e091525da2ee869a25f Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 16:47:21 +0300 Subject: [PATCH 0369/2107] tests: update help test --- src/help.c | 2 +- tests/expected/option_help.out | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index 0068ecee2..ddc477094 100644 --- a/src/help.c +++ b/src/help.c @@ -97,7 +97,7 @@ help_pg_probackup(void) printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [-C] [--stream [-S slot-name]] [--temp-slot]\n")); printf(_(" [--backup-pg-log] [-j num-threads]\n")); printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); printf(_(" [--log-level-console=log-level-console]\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index ecc59a893..abb5ce713 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -29,9 +29,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name - [-C] [--stream [-S slot-name]] [--backup-pg-log] - [-j num-threads] [--archive-timeout=archive-timeout] - [--progress] + [-C] [--stream [-S slot-name]] [--temp-slot] + [--backup-pg-log] [-j num-threads] + [--archive-timeout=archive-timeout] [--progress] [--log-level-console=log-level-console] [--log-level-file=log-level-file] [--log-filename=log-filename] From 38bd369145b7139de1fbdf4724c2643cf3d3b803 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 4 Mar 2019 17:08:50 +0300 Subject: [PATCH 0370/2107] PGPRO-2413. Use PG_LIBS_INTERNAL instead of PG_LIBS to link with libpq library within build tree --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 482fa2e96..8372e3e1a 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endif PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) -PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} +PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} all: checksrcdir $(INCLUDES); From 70b299c34b9057154e3cbeb048c36704cd999526 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 19:02:29 +0300 Subject: [PATCH 0371/2107] PGPRO-1973: Add support pthreads for VALIDATE command --- src/backup.c | 11 +- src/data.c | 4 +- src/help.c | 3 +- src/merge.c | 3 +- src/parsexlog.c | 1173 ++++++++++++++++++-------------- src/pg_probackup.h | 13 +- src/restore.c | 3 +- src/utils/logger.c | 7 +- src/utils/pgut.c | 2 +- src/utils/thread.c | 4 + src/utils/thread.h | 2 +- src/validate.c | 3 +- tests/expected/option_help.out | 2 +- 13 files changed, 710 insertions(+), 520 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5b5f30005..b61910b78 100644 --- a/src/backup.c +++ b/src/backup.c @@ -405,7 +405,7 @@ remote_backup_files(void *arg) instance_config.pguser); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during backup"); query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); @@ -621,6 +621,7 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; + thread_interrupted = false; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } @@ -678,8 +679,7 @@ do_backup_instance(void) * where this backup has started. */ extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, - prev_backup->start_lsn, current.start_lsn, - backup_files_list); + prev_backup->start_lsn, current.start_lsn); } else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -746,6 +746,7 @@ do_backup_instance(void) } /* Run threads */ + thread_interrupted = false; elog(INFO, "Start transfering data files"); for (i = 0; i < num_threads; i++) { @@ -2244,7 +2245,7 @@ backup_files(void *arg) elog(VERBOSE, "Copying file: \"%s\" ", file->path); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during backup"); if (progress) @@ -2689,7 +2690,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) static XLogRecPtr prevpos = InvalidXLogRecPtr; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during backup"); /* we assume that we get called once at the end of each segment */ diff --git a/src/data.c b/src/data.c index fc2d5c38e..36e0de28e 100644 --- a/src/data.c +++ b/src/data.c @@ -22,6 +22,8 @@ #include #endif +#include "utils/thread.h" + /* Union to ease operations on relation pages */ typedef union DataPage { @@ -318,7 +320,7 @@ prepare_page(backup_files_arg *arguments, BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during backup"); /* diff --git a/src/help.c b/src/help.c index ddc477094..b47ec56a6 100644 --- a/src/help.c +++ b/src/help.c @@ -131,7 +131,7 @@ help_pg_probackup(void) printf(_(" [--skip-block-validation]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); @@ -350,6 +350,7 @@ help_validate(void) printf(_(" -i, --backup-id=backup-id backup to validate\n")); printf(_(" --progress show progress\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); diff --git a/src/merge.c b/src/merge.c index 30891d5d6..2967f3608 100644 --- a/src/merge.c +++ b/src/merge.c @@ -245,6 +245,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pg_atomic_init_flag(&file->lock); } + thread_interrupted = false; for (i = 0; i < num_threads; i++) { merge_files_arg *arg = &(threads_args[i]); @@ -402,7 +403,7 @@ merge_files(void *arg) continue; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during merging backups"); /* Directories were created before */ diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a6258841..198f9c5aa 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -83,42 +83,61 @@ typedef struct xl_xact_abort /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ } xl_xact_abort; -static void extractPageInfo(XLogReaderState *record); -static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); +/* + * XLogRecTarget allows to track the last recovery targets. Currently used only + * within validate_wal(). + */ +typedef struct XLogRecTarget +{ + TimestampTz rec_time; + TransactionId rec_xid; + XLogRecPtr rec_lsn; +} XLogRecTarget; -typedef struct XLogPageReadPrivate +typedef struct XLogReaderData { int thread_num; - const char *archivedir; TimeLineID tli; - uint32 xlog_seg_size; + + XLogRecTarget cur_rec; + XLogSegNo xlogsegno; + bool xlogexists; char page_buf[XLOG_BLCKSZ]; uint32 prev_page_off; - bool manual_switch; bool need_switch; int xlogfile; - XLogSegNo xlogsegno; char xlogpath[MAXPGPATH]; - bool xlogexists; #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; #endif -} XLogPageReadPrivate; +} XLogReaderData; + +/* Function to process a WAL record */ +typedef void (*xlog_record_function) (XLogReaderState *record, + XLogReaderData *reader_data, + bool *stop_reading); /* An argument for a thread function */ typedef struct { - XLogPageReadPrivate private_data; + XLogReaderData reader_data; + + xlog_record_function process_record; XLogRecPtr startpoint; XLogRecPtr endpoint; XLogSegNo endSegNo; + /* + * The thread got the recovery target. + */ + bool got_target; + /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. @@ -130,14 +149,41 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); -static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, +static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, - TimeLineID tli, uint32 xlog_seg_size, + TimeLineID tli, uint32 segment_size, + bool manual_switch, + bool consistent_read, bool allocate_reader); +static bool RunXLogThreads(const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 segment_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool consistent_read, + xlog_record_function process_record, + XLogRecTarget *last_rec); +//static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg); +static bool SwitchThreadToNextWal(XLogReaderState *xlogreader, + xlog_thread_arg *arg); +static bool XLogWaitForConsistency(XLogReaderState *xlogreader); +static void *XLogThreadWorker(void *arg); static void CleanupXLogPageRead(XLogReaderState *xlogreader); -static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel); +static void PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel); + +static void extractPageInfo(XLogReaderState *record, + XLogReaderData *reader_data, bool *stop_reading); +static void validateXLogRecord(XLogReaderState *record, + XLogReaderData *reader_data, bool *stop_reading); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); -static XLogSegNo nextSegNoToRead = 0; +static XLogSegNo segno_start = 0; +/* Segment number where target record is located */ +static XLogSegNo segno_target = 0; +/* Next segment number to read by a thread */ +static XLogSegNo segno_next = 0; +/* Number of segments already read by threads */ +static uint32 segnum_read = 0; static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* copied from timestamp.c */ @@ -156,189 +202,25 @@ timestamptz_to_time_t(TimestampTz t) return result; } +static const char *wal_archivedir = NULL; +static uint32 wal_seg_size = 0; /* - * Do manual switch to the next WAL segment. - * - * Returns false if the reader reaches the end of a WAL segment list. + * If true a wal reader thread switches to the next segment using + * segno_next. */ -static bool -switchToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) -{ - XLogPageReadPrivate *private_data; - XLogRecPtr found; - - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - private_data->need_switch = false; - - /* Critical section */ - pthread_lock(&wal_segment_mutex); - Assert(nextSegNoToRead); - private_data->xlogsegno = nextSegNoToRead; - nextSegNoToRead++; - pthread_mutex_unlock(&wal_segment_mutex); - - /* We've reached the end */ - if (private_data->xlogsegno > arg->endSegNo) - return false; - - /* Adjust next record position */ - GetXLogRecPtr(private_data->xlogsegno, 0, - private_data->xlog_seg_size, arg->startpoint); - /* We need to close previously opened file if it wasn't closed earlier */ - CleanupXLogPageRead(xlogreader); - /* Skip over the page header and contrecord if any */ - found = XLogFindNextRecord(xlogreader, arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is - * absent or is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - PrintXLogCorruptionMsg(private_data, ERROR); - } - arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", - private_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - - return true; -} - +static bool wal_manual_switch = false; /* - * extractPageMap() worker. + * If true a wal reader thread waits for other threads if the thread met absent + * wal segment. */ -static void * -doExtractPageMap(void *arg) -{ - xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; - XLogPageReadPrivate *private_data; - XLogReaderState *xlogreader; - XLogSegNo nextSegNo = 0; - XLogRecPtr found; - char *errormsg; - - private_data = &extract_arg->private_data; -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, - &SimpleXLogPageRead, private_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); -#endif - if (xlogreader == NULL) - elog(ERROR, "Thread [%d]: out of memory", private_data->thread_num); - xlogreader->system_identifier = instance_config.system_identifier; - - found = XLogFindNextRecord(xlogreader, extract_arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is absent or - * is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); - PrintXLogCorruptionMsg(private_data, ERROR); - } - extract_arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); - - /* Switch WAL segment manually below without using SimpleXLogPageRead() */ - private_data->manual_switch = true; - - do - { - XLogRecord *record; - - if (interrupted) - elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - private_data->thread_num); - - /* - * We need to switch to the next WAL segment after reading previous - * record. It may happen if we read contrecord. - */ - if (private_data->need_switch) - { - if (!switchToNextWal(xlogreader, extract_arg)) - break; - } - - record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); - - if (record == NULL) - { - XLogRecPtr errptr; - - /* - * There is no record, try to switch to the next WAL segment. - * Usually SimpleXLogPageRead() does it by itself. But here we need - * to do it manually to support threads. - */ - if (private_data->need_switch && errormsg == NULL) - { - if (switchToNextWal(xlogreader, extract_arg)) - continue; - else - break; - } - - errptr = extract_arg->startpoint ? - extract_arg->startpoint : xlogreader->EndRecPtr; - - if (errormsg) - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", - private_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr)); - - /* - * If we don't have all WAL files from prev backup start_lsn to current - * start_lsn, we won't be able to build page map and PAGE backup will - * be incorrect. Stop it and throw an error. - */ - PrintXLogCorruptionMsg(private_data, ERROR); - } +static bool wal_consistent_read = false; - extractPageInfo(xlogreader); - - /* continue reading at next record */ - extract_arg->startpoint = InvalidXLogRecPtr; - - GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, - private_data->xlog_seg_size); - } while (nextSegNo <= extract_arg->endSegNo && - xlogreader->ReadRecPtr < extract_arg->endpoint); - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - /* Extracting is successful */ - extract_arg->ret = 0; - return NULL; -} +/* + * Variables used within validate_wal() and validateXLogRecord() to stop workers + */ +static time_t wal_target_time = 0; +static TransactionId wal_target_xid = InvalidTransactionId; +static XLogRecPtr wal_target_lsn = InvalidXLogRecPtr; /* * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the @@ -348,86 +230,20 @@ doExtractPageMap(void *arg) * file. */ void -extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint, parray *files) +extractPageMap(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint) { - int i; - int threads_need = 0; - XLogSegNo endSegNo; bool extract_isok = true; - pthread_t *threads; - xlog_thread_arg *thread_args; time_t start_time, end_time; elog(LOG, "Compiling pagemap"); - if (!XRecOffIsValid(startpoint)) - elog(ERROR, "Invalid startpoint value %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - - if (!XRecOffIsValid(endpoint)) - elog(ERROR, "Invalid endpoint value %X/%X", - (uint32) (endpoint >> 32), (uint32) (endpoint)); - - GetXLogSegNo(endpoint, endSegNo, seg_size); - - nextSegNoToRead = 0; time(&start_time); - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); - - /* - * Initialize thread args. - * - * Each thread works with its own WAL segment and we need to adjust - * startpoint value for each thread. - */ - for (i = 0; i < num_threads; i++) - { - InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, - seg_size, false); - thread_args[i].private_data.thread_num = i + 1; - - thread_args[i].startpoint = startpoint; - thread_args[i].endpoint = endpoint; - thread_args[i].endSegNo = endSegNo; - /* By default there is some error */ - thread_args[i].ret = 1; - - threads_need++; - - /* Adjust startpoint to the next thread */ - if (nextSegNoToRead == 0) - GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); - - nextSegNoToRead++; - /* - * If we need to read less WAL segments than num_threads, create less - * threads. - */ - if (nextSegNoToRead > endSegNo) - break; - GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); - } - - /* Run threads */ - for (i = 0; i < threads_need; i++) - { - elog(VERBOSE, "Start WAL reader thread: %d", i + 1); - pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); - } - - /* Wait for threads */ - for (i = 0; i < threads_need; i++) - { - pthread_join(threads[i], NULL); - if (thread_args[i].ret == 1) - extract_isok = false; - } - - pfree(threads); - pfree(thread_args); + extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, tli, wal_seg_size, + startpoint, endpoint, false, extractPageInfo, + NULL); time(&end_time); if (extract_isok) @@ -438,48 +254,26 @@ extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, } /* - * Ensure that the backup has all wal files needed for recovery to consistent state. + * Ensure that the backup has all wal files needed for recovery to consistent + * state. + * + * WAL records reading is processed using threads. Each thread reads single WAL + * file. */ static void validate_backup_wal_from_start_to_stop(pgBackup *backup, - char *backup_xlog_path, TimeLineID tli, + const char *archivedir, TimeLineID tli, uint32 xlog_seg_size) { - XLogRecPtr startpoint = backup->start_lsn; - XLogRecord *record; - XLogReaderState *xlogreader; - char *errormsg; - XLogPageReadPrivate private; - bool got_endpoint = false; - - xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, - xlog_seg_size, true); - - while (true) - { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + bool got_endpoint; - if (record == NULL) - { - if (errormsg) - elog(WARNING, "%s", errormsg); - - break; - } - - /* Got WAL record at stop_lsn */ - if (xlogreader->ReadRecPtr == backup->stop_lsn) - { - got_endpoint = true; - break; - } - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ - } + got_endpoint = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + false, NULL, NULL); if (!got_endpoint) { - PrintXLogCorruptionMsg(&private, WARNING); - /* * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. @@ -494,10 +288,6 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn)); } - - /* clean */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); } /* @@ -508,20 +298,12 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, - XLogRecPtr target_lsn, - TimeLineID tli, uint32 seg_size) + XLogRecPtr target_lsn, TimeLineID tli, uint32 wal_seg_size) { - XLogRecPtr startpoint = backup->start_lsn; const char *backup_id; - XLogRecord *record; - XLogReaderState *xlogreader; - char *errormsg; - XLogPageReadPrivate private; - TransactionId last_xid = InvalidTransactionId; - TimestampTz last_time = 0; + XLogRecTarget last_rec; char last_timestamp[100], target_timestamp[100]; - XLogRecPtr last_lsn = InvalidXLogRecPtr; bool all_wal = false; char backup_xlog_path[MAXPGPATH]; @@ -548,11 +330,11 @@ validate_wal(pgBackup *backup, const char *archivedir, DATABASE_DIR, PG_XLOG_DIR); validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, - seg_size); + wal_seg_size); } else validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, - seg_size); + wal_seg_size); if (backup->status == BACKUP_STATUS_CORRUPT) { @@ -563,7 +345,8 @@ validate_wal(pgBackup *backup, const char *archivedir, * If recovery target is provided check that we can restore backup to a * recovery target time or xid. */ - if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + if (!TransactionIdIsValid(target_xid) && target_time == 0 && + !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ elog(INFO, "Backup %s WAL segments are valid", backup_id); @@ -582,89 +365,41 @@ validate_wal(pgBackup *backup, const char *archivedir, * up to the given recovery target. * In any case we cannot restore to the point before stop_lsn. */ - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, - true); /* We can restore at least up to the backup end */ + last_rec.rec_time = 0; + last_rec.rec_xid = backup->recovery_xid; + last_rec.rec_lsn = backup->stop_lsn; + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); - last_xid = backup->recovery_xid; - last_lsn = backup->stop_lsn; - if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) || (target_time != 0 && backup->recovery_time >= target_time) - || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + || (XRecOffIsValid(target_lsn) && last_rec.rec_lsn >= target_lsn)) all_wal = true; - startpoint = backup->stop_lsn; - while (true) - { - bool timestamp_record; - - record = XLogReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - if (errormsg) - elog(WARNING, "%s", errormsg); - - break; - } - - timestamp_record = getRecordTimestamp(xlogreader, &last_time); - if (XLogRecGetXid(xlogreader) != InvalidTransactionId) - last_xid = XLogRecGetXid(xlogreader); - last_lsn = xlogreader->ReadRecPtr; - - /* Check target xid */ - if (TransactionIdIsValid(target_xid) && target_xid == last_xid) - { - all_wal = true; - break; - } - /* Check target time */ - else if (target_time != 0 && timestamp_record && - timestamptz_to_time_t(last_time) >= target_time) - { - all_wal = true; - break; - } - /* Check target lsn */ - else if (XRecOffIsValid(target_lsn) && last_lsn >= target_lsn) - { - all_wal = true; - break; - } - /* If there are no target xid, target time and target lsn */ - else if (!TransactionIdIsValid(target_xid) && target_time == 0 && - !XRecOffIsValid(target_lsn)) - { - all_wal = true; - /* We don't stop here. We want to get last_xid and last_time */ - } - - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ - } - - if (last_time > 0) + all_wal = all_wal || + RunXLogThreads(archivedir, target_time, target_xid, target_lsn, + tli, wal_seg_size, backup->stop_lsn, + InvalidXLogRecPtr, true, validateXLogRecord, &last_rec); + if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time)); + timestamptz_to_time_t(last_rec.rec_time)); /* There are all needed WAL records */ if (all_wal) elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_xid, - (uint32) (last_lsn >> 32), (uint32) last_lsn); + last_timestamp, last_rec.rec_xid, + (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); /* Some needed WAL records are absent */ else { - PrintXLogCorruptionMsg(&private, WARNING); - elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_xid, - (uint32) (last_lsn >> 32), (uint32) last_lsn); + last_timestamp, last_rec.rec_xid, + (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); @@ -678,10 +413,6 @@ validate_wal(pgBackup *backup, const char *archivedir, elog(ERROR, "not enough WAL records to lsn %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } - - /* clean */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); } /* @@ -690,13 +421,13 @@ validate_wal(pgBackup *backup, const char *archivedir, * pg_stop_backup(). */ bool -read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time, TransactionId *recovery_xid) { XLogRecPtr startpoint = stop_lsn; XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; bool res; if (!XRecOffIsValid(start_lsn)) @@ -707,7 +438,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, true, true); /* Read records from stop_lsn down to start_lsn */ do @@ -762,10 +494,10 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, */ bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 seg_size) + TimeLineID target_tli, uint32 wal_seg_size) { XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; char *errormsg; bool res; @@ -773,8 +505,8 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "Invalid target_lsn value %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, - true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli, + wal_seg_size, false, false, true); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ @@ -797,16 +529,16 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, - uint32 seg_size) + uint32 wal_seg_size) { XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; XLogRecPtr startpoint; XLogSegNo start_segno; XLogSegNo segno; XLogRecPtr res = InvalidXLogRecPtr; - GetXLogSegNo(stop_lsn, segno, seg_size); + GetXLogSegNo(stop_lsn, segno, wal_seg_size); if (segno <= 1) elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); @@ -814,19 +546,20 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, if (seek_prev_segment) segno = segno - 1; - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, false, true); /* * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. */ - GetXLogSegNo(start_lsn, start_segno, seg_size); + GetXLogSegNo(start_lsn, start_segno, wal_seg_size); if (start_segno == segno) startpoint = start_lsn; else { XLogRecPtr found; - GetXLogRecPtr(segno, 0, seg_size, startpoint); + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -838,7 +571,7 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, else elog(WARNING, "Could not read WAL record at %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); - PrintXLogCorruptionMsg(&private, ERROR); + PrintXLogCorruptionMsg(&reader_data, ERROR); } startpoint = found; } @@ -867,13 +600,13 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, else elog(WARNING, "Could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); - PrintXLogCorruptionMsg(&private, ERROR); + PrintXLogCorruptionMsg(&reader_data, ERROR); } /* continue reading at next record */ startpoint = InvalidXLogRecPtr; - GetXLogSegNo(xlogreader->EndRecPtr, next_segno, seg_size); + GetXLogSegNo(xlogreader->EndRecPtr, next_segno, wal_seg_size); if (next_segno > segno) break; @@ -919,25 +652,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) { - XLogPageReadPrivate *private_data; + XLogReaderData *reader_data; uint32 targetPageOff; - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - targetPageOff = targetPagePtr % private_data->xlog_seg_size; + reader_data = (XLogReaderData *) xlogreader->private_data; + targetPageOff = targetPagePtr % wal_seg_size; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - private_data->thread_num); + reader_data->thread_num); /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, - private_data->xlog_seg_size)) + if (!IsInXLogSeg(targetPagePtr, reader_data->xlogsegno, wal_seg_size)) { elog(VERBOSE, "Thread [%d]: Need to switch to the next WAL segment, page LSN %X/%X, record being read LSN %X/%X", - private_data->thread_num, + reader_data->thread_num, (uint32) (targetPagePtr >> 32), (uint32) (targetPagePtr), (uint32) (xlogreader->currRecPtr >> 32), (uint32) (xlogreader->currRecPtr )); @@ -954,52 +686,49 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* * Switch to the next WAL segment after reading contrecord. */ - if (private_data->manual_switch) - private_data->need_switch = true; + if (wal_manual_switch) + reader_data->need_switch = true; } else { CleanupXLogPageRead(xlogreader); /* - * Do not switch to next WAL segment in this function. Currently it is - * manually switched only in doExtractPageMap(). - */ - if (private_data->manual_switch) + * Do not switch to next WAL segment in this function. It is + * manually switched by a thread routine. + */ + if (wal_manual_switch) { - private_data->need_switch = true; + reader_data->need_switch = true; return -1; } } } - GetXLogSegNo(targetPagePtr, private_data->xlogsegno, - private_data->xlog_seg_size); + GetXLogSegNo(targetPagePtr, reader_data->xlogsegno, wal_seg_size); /* Try to switch to the next WAL segment */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) { char xlogfname[MAXFNAMELEN]; - GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, - private_data->xlog_seg_size); - snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", - private_data->archivedir, xlogfname); + GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, + wal_seg_size); + snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, + xlogfname); - if (fileExists(private_data->xlogpath)) + if (fileExists(reader_data->xlogpath)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", - private_data->thread_num, - private_data->xlogpath); + reader_data->thread_num, reader_data->xlogpath); - private_data->xlogexists = true; - private_data->xlogfile = open(private_data->xlogpath, - O_RDONLY | PG_BINARY, 0); + reader_data->xlogexists = true; + reader_data->xlogfile = open(reader_data->xlogpath, + O_RDONLY | PG_BINARY, 0); - if (private_data->xlogfile < 0) + if (reader_data->xlogfile < 0) { elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s", - private_data->thread_num, - private_data->xlogpath, + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } @@ -1008,21 +737,21 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* Try to open compressed WAL segment */ else { - snprintf(private_data->gz_xlogpath, - sizeof(private_data->gz_xlogpath), "%s.gz", - private_data->xlogpath); - if (fileExists(private_data->gz_xlogpath)) + snprintf(reader_data->gz_xlogpath, sizeof(reader_data->gz_xlogpath), + "%s.gz", reader_data->xlogpath); + if (fileExists(reader_data->gz_xlogpath)) { elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", - private_data->thread_num, private_data->gz_xlogpath); + reader_data->thread_num, reader_data->gz_xlogpath); - private_data->xlogexists = true; - private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, - "rb"); - if (private_data->gz_xlogfile == NULL) + reader_data->xlogexists = true; + reader_data->gz_xlogfile = gzopen(reader_data->gz_xlogpath, + "rb"); + if (reader_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", - private_data->thread_num, private_data->gz_xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->gz_xlogpath, + strerror(errno)); return -1; } } @@ -1030,69 +759,67 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #endif /* Exit without error if WAL segment doesn't exist */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) return -1; } /* * At this point, we have the right segment open. */ - Assert(private_data->xlogexists); + Assert(reader_data->xlogexists); /* * Do not read same page read earlier from the file, read it from the buffer */ - if (private_data->prev_page_off != 0 && - private_data->prev_page_off == targetPageOff) + if (reader_data->prev_page_off != 0 && + reader_data->prev_page_off == targetPageOff) { - memcpy(readBuf, private_data->page_buf, XLOG_BLCKSZ); - *pageTLI = private_data->tli; + memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); + *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } /* Read the requested page */ - if (private_data->xlogfile != -1) + if (reader_data->xlogfile != -1) { - if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + if (lseek(reader_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) { elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } - if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (read(reader_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } } #ifdef HAVE_LIBZ else { - if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (gzseek(reader_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); + reader_data->thread_num, reader_data->gz_xlogpath, + get_gz_error(reader_data->gz_xlogfile)); return -1; } - if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (gzread(reader_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); + reader_data->thread_num, reader_data->gz_xlogpath, + get_gz_error(reader_data->gz_xlogfile)); return -1; } } #endif - memcpy(private_data->page_buf, readBuf, XLOG_BLCKSZ); - private_data->prev_page_off = targetPageOff; - *pageTLI = private_data->tli; + memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); + reader_data->prev_page_off = targetPageOff; + *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } @@ -1100,89 +827,521 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, * Initialize WAL segments reading. */ static XLogReaderState * -InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, - TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, + TimeLineID tli, uint32 segment_size, bool manual_switch, + bool consistent_read, bool allocate_reader) { XLogReaderState *xlogreader = NULL; - MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); - private_data->archivedir = archivedir; - private_data->tli = tli; - private_data->xlog_seg_size = xlog_seg_size; - private_data->xlogfile = -1; + wal_archivedir = archivedir; + wal_seg_size = segment_size; + wal_manual_switch = manual_switch; + wal_consistent_read = consistent_read; + + MemSet(reader_data, 0, sizeof(XLogReaderData)); + reader_data->tli = tli; + reader_data->xlogfile = -1; if (allocate_reader) { #if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(xlog_seg_size, - &SimpleXLogPageRead, private_data); + xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); #else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); #endif if (xlogreader == NULL) - elog(ERROR, "out of memory"); + elog(ERROR, "Out of memory"); xlogreader->system_identifier = instance_config.system_identifier; } return xlogreader; } +/* + * Run WAL processing routines using threads. Start from startpoint up to + * endpoint. It is possible to send zero endpoint, threads will read WAL + * infinitely in this case. + */ +static bool +RunXLogThreads(const char *archivedir, time_t target_time, + TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, + uint32 segment_size, XLogRecPtr startpoint, XLogRecPtr endpoint, + bool consistent_read, xlog_record_function process_record, + XLogRecTarget *last_rec) +{ + pthread_t *threads; + xlog_thread_arg *thread_args; + int i; + int threads_need = 0; + XLogSegNo endSegNo = 0; + XLogSegNo errorSegNo = 0; + bool result = true; + + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XLogRecPtrIsInvalid(endpoint)) + { + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, segment_size); + } + + /* Initialize static variables for workers */ + wal_target_time = target_time; + wal_target_xid = target_xid; + wal_target_lsn = target_lsn; + + GetXLogSegNo(startpoint, segno_start, segment_size); + segno_target = 0; + GetXLogSegNo(startpoint, segno_next, segment_size); + segnum_read = 0; + + threads = (pthread_t *) pgut_malloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) pgut_malloc(sizeof(xlog_thread_arg) * num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + xlog_thread_arg *arg = &thread_args[i]; + + InitXLogPageRead(&arg->reader_data, archivedir, tli, segment_size, true, + consistent_read, false); + arg->reader_data.xlogsegno = segno_next; + arg->reader_data.thread_num = i + 1; + arg->process_record = process_record; + arg->startpoint = startpoint; + arg->endpoint = endpoint; + arg->endSegNo = endSegNo; + arg->got_target = false; + /* By default there is some error */ + arg->ret = 1; + + threads_need++; + segno_next++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (endSegNo != 0 && segno_next > endSegNo) + break; + GetXLogRecPtr(segno_next, 0, segment_size, startpoint); + } + + /* Run threads */ + thread_interrupted = false; + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i + 1); + pthread_create(&threads[i], NULL, XLogThreadWorker, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + result = false; + } + + if (last_rec) + for (i = 0; i < threads_need; i++) + { + XLogRecTarget *cur_rec; + + if (thread_args[i].ret != 0) + { + /* + * Save invalid segment number after which all segments are not + * valid. + */ + if (errorSegNo == 0 || + errorSegNo > thread_args[i].reader_data.xlogsegno) + errorSegNo = thread_args[i].reader_data.xlogsegno; + continue; + } + + /* Is this segment valid */ + if (errorSegNo != 0 && + thread_args[i].reader_data.xlogsegno > errorSegNo) + continue; + + cur_rec = &thread_args[i].reader_data.cur_rec; + /* + * If we got the target return minimum possible record. + */ + if (segno_target > 0) + { + if (thread_args[i].got_target && + thread_args[i].reader_data.xlogsegno == segno_target) + { + *last_rec = *cur_rec; + break; + } + } + /* + * Else return maximum possible record up to which restore is + * possible. + */ + else if (last_rec->rec_lsn < cur_rec->rec_lsn) + *last_rec = *cur_rec; + } + + pfree(threads); + pfree(thread_args); + + return result; +} + +/* + * WAL reader worker. + */ +void * +XLogThreadWorker(void *arg) +{ + xlog_thread_arg *thread_arg = (xlog_thread_arg *) arg; + XLogReaderData *reader_data = &thread_arg->reader_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + XLogRecPtr found; + uint32 prev_page_off = 0; + bool need_read = true; + +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); + xlogreader->system_identifier = instance_config.system_identifier; + + found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); + + /* + * We get invalid WAL record pointer usually when WAL segment is absent or + * is corrupted. + */ + if (XLogRecPtrIsInvalid(found)) + { + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + need_read = false; + else + { + if (xlogreader->errormsg_buf[0] != '\0') + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint), + xlogreader->errormsg_buf); + else + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint)); + PrintXLogCorruptionMsg(reader_data, ERROR); + } + } + + thread_arg->startpoint = found; + + elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint)); + + while (need_read) + { + XLogRecord *record; + char *errormsg; + bool stop_reading = false; + + if (interrupted || thread_interrupted) + elog(ERROR, "Thread [%d]: Interrupted during WAL reading", + reader_data->thread_num); + + /* + * We need to switch to the next WAL segment after reading previous + * record. It may happen if we read contrecord. + */ + if (reader_data->need_switch && + !SwitchThreadToNextWal(xlogreader, thread_arg)) + break; + + record = XLogReadRecord(xlogreader, thread_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * There is no record, try to switch to the next WAL segment. + * Usually SimpleXLogPageRead() does it by itself. But here we need + * to do it manually to support threads. + */ + if (reader_data->need_switch && errormsg == NULL) + { + if (SwitchThreadToNextWal(xlogreader, thread_arg)) + continue; + else + break; + } + + /* + * XLogWaitForConsistency() is normally used only with threads. + * Call it here for just in case. + */ + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + break; + + errptr = thread_arg->startpoint ? + thread_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + reader_data->thread_num, + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(reader_data, ERROR); + } + + getRecordTimestamp(xlogreader, &reader_data->cur_rec.rec_time); + if (TransactionIdIsValid(XLogRecGetXid(xlogreader))) + reader_data->cur_rec.rec_xid = XLogRecGetXid(xlogreader); + reader_data->cur_rec.rec_lsn = xlogreader->ReadRecPtr; + + if (thread_arg->process_record) + thread_arg->process_record(xlogreader, reader_data, &stop_reading); + if (stop_reading) + { + thread_arg->got_target = true; + + pthread_lock(&wal_segment_mutex); + /* We should store least target segment number */ + if (segno_target == 0 || segno_target > reader_data->xlogsegno) + segno_target = reader_data->xlogsegno; + pthread_mutex_unlock(&wal_segment_mutex); + + break; + } + + /* + * Check if other thread got the target segment. Check it not very + * often, only every WAL page. + */ + if (wal_consistent_read && prev_page_off != 0 && + prev_page_off != reader_data->prev_page_off) + { + XLogSegNo segno; + + pthread_lock(&wal_segment_mutex); + segno = segno_target; + pthread_mutex_unlock(&wal_segment_mutex); + + if (segno != 0 && segno < reader_data->xlogsegno) + break; + } + prev_page_off = reader_data->prev_page_off; + + /* continue reading at next record */ + thread_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, wal_seg_size); + + if (thread_arg->endSegNo != 0 && + !XLogRecPtrIsInvalid(thread_arg->endpoint) && + /* + * Consider thread_arg->endSegNo and thread_arg->endpoint only if + * they are valid. + */ + xlogreader->ReadRecPtr == thread_arg->endpoint && + nextSegNo > thread_arg->endSegNo) + break; + } + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + thread_arg->ret = 0; + return NULL; +} + +/* + * Do manual switch to the next WAL segment. + * + * Returns false if the reader reaches the end of a WAL segment list. + */ +static bool +SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) +{ + XLogReaderData *reader_data; + XLogRecPtr found; + + reader_data = (XLogReaderData *) xlogreader->private_data; + reader_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(segno_next); + reader_data->xlogsegno = segno_next; + segnum_read++; + segno_next++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We've reached the end */ + if (arg->endSegNo != 0 && reader_data->xlogsegno > arg->endSegNo) + return false; + + /* Adjust next record position */ + GetXLogRecPtr(reader_data->xlogsegno, 0, wal_seg_size, arg->startpoint); + /* We need to close previously opened file if it wasn't closed earlier */ + CleanupXLogPageRead(xlogreader); + /* Skip over the page header and contrecord if any */ + found = XLogFindNextRecord(xlogreader, arg->startpoint); + + /* + * We get invalid WAL record pointer usually when WAL segment is + * absent or is corrupted. + */ + if (XLogRecPtrIsInvalid(found)) + { + /* + * Check if we need to stop reading. We stop if other thread found a + * target segment. + */ + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + return false; + + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); + PrintXLogCorruptionMsg(reader_data, ERROR); + } + arg->startpoint = found; + + elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", + reader_data->thread_num, + (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); + + return true; +} + +/* + * Wait for other threads since the current thread couldn't read its segment. + * We need to decide is it fail or not. + * + * Returns true if there is no failure and previous target segment was found. + * Otherwise return false. + */ +static bool +XLogWaitForConsistency(XLogReaderState *xlogreader) +{ + uint32 segnum_need = 0; + XLogReaderData *reader_data =(XLogReaderData *) xlogreader->private_data; + + segnum_need = reader_data->xlogsegno - segno_start; + while (true) + { + uint32 segnum_current_read; + XLogSegNo segno; + + if (interrupted || thread_interrupted) + elog(ERROR, "Thread [%d]: Interrupted during WAL reading", + reader_data->thread_num); + + pthread_lock(&wal_segment_mutex); + segnum_current_read = segnum_read; + segno = segno_target; + pthread_mutex_unlock(&wal_segment_mutex); + + /* Other threads read all previous segments and didn't find target */ + if (segnum_need <= segnum_current_read) + return false; + + if (segno < reader_data->xlogsegno) + return true; + + pg_usleep(1000000L); /* 1000 ms */ + } + + /* We shouldn't reach it */ + return false; +} + /* * Cleanup after WAL segment reading. */ static void CleanupXLogPageRead(XLogReaderState *xlogreader) { - XLogPageReadPrivate *private_data; + XLogReaderData *reader_data; - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - if (private_data->xlogfile >= 0) + reader_data = (XLogReaderData *) xlogreader->private_data; + if (reader_data->xlogfile >= 0) { - close(private_data->xlogfile); - private_data->xlogfile = -1; + close(reader_data->xlogfile); + reader_data->xlogfile = -1; } #ifdef HAVE_LIBZ - else if (private_data->gz_xlogfile != NULL) + else if (reader_data->gz_xlogfile != NULL) { - gzclose(private_data->gz_xlogfile); - private_data->gz_xlogfile = NULL; + gzclose(reader_data->gz_xlogfile); + reader_data->gz_xlogfile = NULL; } #endif - private_data->prev_page_off = 0; - private_data->xlogexists = false; + reader_data->prev_page_off = 0; + reader_data->xlogexists = false; } static void -PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel) { - if (private_data->xlogpath[0] != 0) + if (reader_data->xlogpath[0] != 0) { /* * XLOG reader couldn't read WAL segment. * We throw a WARNING here to be able to update backup status. */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent", - private_data->thread_num, - private_data->xlogpath); - else if (private_data->xlogfile != -1) + reader_data->thread_num, reader_data->xlogpath); + else if (reader_data->xlogfile != -1) elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", - private_data->thread_num, - private_data->xlogpath); + reader_data->thread_num, reader_data->xlogpath); #ifdef HAVE_LIBZ - else if (private_data->gz_xlogfile != NULL) + else if (reader_data->gz_xlogfile != NULL) elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", - private_data->thread_num, - private_data->gz_xlogpath); + reader_data->thread_num, reader_data->gz_xlogpath); #endif } else { /* Cannot tell what happened specifically */ elog(elevel, "Thread [%d]: An error occured during WAL reading", - private_data->thread_num); + reader_data->thread_num); } } @@ -1190,7 +1349,8 @@ PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) * Extract information about blocks modified in this record. */ static void -extractPageInfo(XLogReaderState *record) +extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, + bool *stop_reading) { uint8 block_id; RmgrId rmid = XLogRecGetRmid(record); @@ -1258,6 +1418,27 @@ extractPageInfo(XLogReaderState *record) } } +/* + * Check the current read WAL record during validation. + */ +static void +validateXLogRecord(XLogReaderState *record, XLogReaderData *reader_data, + bool *stop_reading) +{ + /* Check target xid */ + if (TransactionIdIsValid(wal_target_xid) && + wal_target_xid == reader_data->cur_rec.rec_xid) + *stop_reading = true; + /* Check target time */ + else if (wal_target_time != 0 && + timestamptz_to_time_t(reader_data->cur_rec.rec_time) >= wal_target_time) + *stop_reading = true; + /* Check target lsn */ + else if (XRecOffIsValid(wal_target_lsn) && + reader_data->cur_rec.rec_lsn >= wal_target_lsn) + *stop_reading = true; +} + /* * Extract timestamp from WAL record. * diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 23db23541..78d60928d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -551,14 +551,11 @@ extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, /* parsexlog.c */ extern void extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint, - parray *files); -extern void validate_wal(pgBackup *backup, - const char *archivedir, - time_t target_time, - TransactionId target_xid, - XLogRecPtr target_lsn, - TimeLineID tli, uint32 seg_size); + XLogRecPtr startpoint, XLogRecPtr endpoint); +extern void validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli, + uint32 seg_size); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, diff --git a/src/restore.c b/src/restore.c index 5aee4037c..cdf607531 100644 --- a/src/restore.c +++ b/src/restore.c @@ -511,6 +511,7 @@ restore_backup(pgBackup *backup) } /* Restore files into target directory */ + thread_interrupted = false; for (i = 0; i < num_threads; i++) { restore_files_arg *arg = &(threads_args[i]); @@ -617,7 +618,7 @@ restore_files(void *arg) lengthof(from_root), DATABASE_DIR); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during restore database"); rel_path = GetRelativePath(file->path,from_root); diff --git a/src/utils/logger.c b/src/utils/logger.c index ba054a62c..72840aa87 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -121,9 +121,6 @@ exit_if_necessary(int elevel) { if (elevel > WARNING && !in_cleanup) { - /* Interrupt other possible routines */ - interrupted = true; - if (loggin_in_progress) { loggin_in_progress = false; @@ -132,11 +129,15 @@ exit_if_necessary(int elevel) /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) + { #ifdef WIN32 ExitThread(elevel); #else pthread_exit(NULL); #endif + /* Interrupt other possible routines */ + thread_interrupted = true; + } else exit(elevel); } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a7e12e913..cdd4b26d3 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -700,7 +700,7 @@ on_interrupt(void) int save_errno = errno; char errbuf[256]; - /* Set interruped flag */ + /* Set interrupted flag */ interrupted = true; /* diff --git a/src/utils/thread.c b/src/utils/thread.c index 0999a0d5b..f1624be98 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -7,8 +7,12 @@ *------------------------------------------------------------------------- */ +#include "postgres_fe.h" + #include "thread.h" +bool thread_interrupted = false; + #ifdef WIN32 DWORD main_tid = 0; #else diff --git a/src/utils/thread.h b/src/utils/thread.h index 6b8349bf5..a2948156b 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -34,7 +34,7 @@ extern DWORD main_tid; extern pthread_t main_tid; #endif - +extern bool thread_interrupted; extern int pthread_lock(pthread_mutex_t *mp); diff --git a/src/validate.c b/src/validate.c index 031ead154..6be5716fb 100644 --- a/src/validate.c +++ b/src/validate.c @@ -115,6 +115,7 @@ pgBackupValidate(pgBackup *backup) palloc(sizeof(validate_files_arg) * num_threads); /* Validate files */ + thread_interrupted = false; for (i = 0; i < num_threads; i++) { validate_files_arg *arg = &(threads_args[i]); @@ -184,7 +185,7 @@ pgBackupValidateFiles(void *arg) if (!pg_atomic_test_set_flag(&file->lock)) continue; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during validate"); /* Validate only regular files */ diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index abb5ce713..a83c39052 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -63,7 +63,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--skip-block-validation] pg_probackup validate -B backup-path [--instance=instance_name] - [-i backup-id] [--progress] + [-i backup-id] [--progress] [-j num-threads] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--recovery-target-name=target-name] [--timeline=timeline] From dac1d43899227d7aebf3584212199b2193736c74 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 5 Mar 2019 14:33:28 +0300 Subject: [PATCH 0372/2107] Fx handling files with zero length --- src/utils/file.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 16ae8bfb3..b664b9bcd 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -873,6 +873,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, req.arg.clevel = clevel; IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + file->compress_alg = calg; while (true) { @@ -886,7 +887,24 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, blknum = hdr.arg; if (hdr.size == 0) /* end of segment */ + { + if (n_blocks_read == 0) + { + BackupPageHeader ph; + ph.block = blknum; + ph.compressed_size = 0; + if (fio_fwrite(out, &ph, sizeof ph) != sizeof(ph)) + { + int errno_tmp = errno; + fio_fclose(out); + elog(ERROR, "File: %s, cannot write backup at block %u: %s", + file->path, blknum, strerror(errno_tmp)); + } + n_blocks_read++; + file->write_size = sizeof(ph); + } break; + } Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); @@ -900,7 +918,6 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, elog(ERROR, "File: %s, cannot write backup at block %u: %s", file->path, blknum, strerror(errno_tmp)); } - file->compress_alg = calg; file->read_size += BLCKSZ; file->write_size += hdr.size; n_blocks_read++; From eea78149aa1c32571cdf81e379067817eae45bf2 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 5 Mar 2019 18:35:31 +0300 Subject: [PATCH 0373/2107] PGPRO-1973: Fix a bug, improve validate tests --- src/parsexlog.c | 22 ++++++++++++++++++---- tests/validate_test.py | 17 ++++++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 198f9c5aa..5bea453ca 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1233,14 +1233,14 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) return false; - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); PrintXLogCorruptionMsg(reader_data, ERROR); } arg->startpoint = found; - elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", + elog(VERBOSE, "Thread [%d]: Switched to LSN %X/%X", reader_data->thread_num, (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); @@ -1257,8 +1257,9 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) static bool XLogWaitForConsistency(XLogReaderState *xlogreader) { - uint32 segnum_need = 0; + uint32 segnum_need; XLogReaderData *reader_data =(XLogReaderData *) xlogreader->private_data; + bool log_message = true; segnum_need = reader_data->xlogsegno - segno_start; while (true) @@ -1266,6 +1267,13 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) uint32 segnum_current_read; XLogSegNo segno; + if (log_message) + { + elog(VERBOSE, "Thread [%d]: Possible WAL corruption. Wait for other threads to decide is this a failure", + reader_data->thread_num); + log_message = false; + } + if (interrupted || thread_interrupted) elog(ERROR, "Thread [%d]: Interrupted during WAL reading", reader_data->thread_num); @@ -1277,9 +1285,15 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) /* Other threads read all previous segments and didn't find target */ if (segnum_need <= segnum_current_read) + { + /* Mark current segment as read even if it wasn't read actually */ + pthread_lock(&wal_segment_mutex); + segnum_read++; + pthread_mutex_unlock(&wal_segment_mutex); return false; + } - if (segno < reader_data->xlogsegno) + if (segno != 0 && segno < reader_data->xlogsegno) return true; pg_usleep(1000000L); /* 1000 ms */ diff --git a/tests/validate_test.py b/tests/validate_test.py index cb6247a1c..a2a9ba2df 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -58,7 +58,7 @@ def test_validate_nullified_heap_page_backup(self): '{0} blknum 1, empty page'.format(file_path) in f.read(), 'Failed to detect nullified block') - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -99,7 +99,7 @@ def test_validate_wal_unreal_values(self): "INFO: backup validation completed successfully", self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(target_time)]), + options=["--time={0}".format(target_time), "-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -108,7 +108,7 @@ def test_validate_wal_unreal_values(self): try: self.validate_pb( backup_dir, 'node', options=["--time={0}".format( - unreal_time_1)]) + unreal_time_1), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal time.\n " @@ -126,7 +126,7 @@ def test_validate_wal_unreal_values(self): try: self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(unreal_time_2)]) + options=["--time={0}".format(unreal_time_2), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal time.\n " @@ -150,7 +150,8 @@ def test_validate_wal_unreal_values(self): self.assertIn( "INFO: backup validation completed successfully", self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + backup_dir, 'node', options=["--xid={0}".format(target_xid), + "-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -158,7 +159,8 @@ def test_validate_wal_unreal_values(self): unreal_xid = int(target_xid) + 1000 try: self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + backup_dir, 'node', options=["--xid={0}".format(unreal_xid), + "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal xid.\n " @@ -171,7 +173,8 @@ def test_validate_wal_unreal_values(self): repr(e.message), self.cmd)) # Validate with backup ID - output = self.validate_pb(backup_dir, 'node', backup_id) + output = self.validate_pb(backup_dir, 'node', backup_id, + options=["-j", "4"]) self.assertIn( "INFO: Validating backup {0}".format(backup_id), output, From d4c80b9c77ab647983b478f2a7c74131feb22e66 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 6 Mar 2019 11:36:16 +0300 Subject: [PATCH 0374/2107] Fix backup of empty files --- src/utils/file.c | 86 ++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 50 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index b664b9bcd..1e7e2c28e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -872,9 +872,10 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, req.arg.calg = calg; req.arg.clevel = clevel; - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); file->compress_alg = calg; + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + while (true) { fio_header hdr; @@ -887,24 +888,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, blknum = hdr.arg; if (hdr.size == 0) /* end of segment */ - { - if (n_blocks_read == 0) - { - BackupPageHeader ph; - ph.block = blknum; - ph.compressed_size = 0; - if (fio_fwrite(out, &ph, sizeof ph) != sizeof(ph)) - { - int errno_tmp = errno; - fio_fclose(out); - elog(ERROR, "File: %s, cannot write backup at block %u: %s", - file->path, blknum, strerror(errno_tmp)); - } - n_blocks_read++; - file->write_size = sizeof(ph); - } break; - } Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); @@ -918,9 +902,15 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, elog(ERROR, "File: %s, cannot write backup at block %u: %s", file->path, blknum, strerror(errno_tmp)); } - file->read_size += BLCKSZ; file->write_size += hdr.size; n_blocks_read++; + + if (((BackupPageHeader*)buf)->compressed_size == PageIsTruncated) + { + blknum += 1; + break; + } + file->read_size += BLCKSZ; } *nBlocksSkipped = blknum - n_blocks_read; return blknum; @@ -939,25 +929,30 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) { int retry_attempts = PAGE_READ_ATTEMPTS; XLogRecPtr page_lsn = InvalidXLogRecPtr; - bool is_empty_page = false; - do + + while (true) { ssize_t rc = pread(fd, read_buffer, BLCKSZ, blknum*BLCKSZ); if (rc <= 0) { - hdr.size = 0; if (rc < 0) { hdr.arg = -errno; + hdr.size = 0; Assert(hdr.arg < 0); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } else { - /* This is the last page */ + BackupPageHeader bph; + bph.block = blknum; + bph.compressed_size = PageIsTruncated; hdr.arg = blknum; + hdr.size = sizeof(bph); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, &bph, sizeof(bph)), sizeof(bph)); } - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); return; } else if (rc == BLCKSZ) @@ -969,10 +964,7 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) /* Page is zeroed. No need to check header and checksum. */ if (i == BLCKSZ) - { - is_empty_page = true; break; - } } else if (!req->checksumVersion || pg_checksum_page(read_buffer, req->segBlockNum + blknum) == ((PageHeader)read_buffer)->pd_checksum) @@ -980,42 +972,36 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) break; } } - } while (--retry_attempts != 0); - if (retry_attempts == 0) - { - hdr.size = 0; - hdr.arg = PAGE_CHECKSUM_MISMATCH; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - return; + if (--retry_attempts == 0) + { + hdr.size = 0; + hdr.arg = PAGE_CHECKSUM_MISMATCH; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + return; + } } /* horizonLsn is not 0 for delta backup. As far as unsigned number are always greater or equal than zero, there is no sense to add more checks */ if (page_lsn >= req->horizonLsn) { char write_buffer[BLCKSZ*2]; BackupPageHeader* bph = (BackupPageHeader*)write_buffer; + const char *errormsg = NULL; hdr.arg = bph->block = blknum; hdr.size = sizeof(BackupPageHeader); - if (is_empty_page) + bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), sizeof(write_buffer) - sizeof(BackupPageHeader), + read_buffer, BLCKSZ, req->calg, req->clevel, + &errormsg); + if (bph->compressed_size <= 0 || bph->compressed_size >= BLCKSZ) { - bph->compressed_size = PageIsTruncated; - } - else - { - const char *errormsg = NULL; - bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), sizeof(write_buffer) - sizeof(BackupPageHeader), - read_buffer, BLCKSZ, req->calg, req->clevel, - &errormsg); - if (bph->compressed_size <= 0 || bph->compressed_size >= BLCKSZ) - { - /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); - bph->compressed_size = BLCKSZ; - } - hdr.size += MAXALIGN(bph->compressed_size); + /* Do not compress page */ + memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); + bph->compressed_size = BLCKSZ; } + hdr.size += MAXALIGN(bph->compressed_size); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); } From 481c42f2e57e2c3c2de1e0ea647d84d15d78d570 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Mar 2019 11:58:42 +0300 Subject: [PATCH 0375/2107] PGPRO-1973: Add more parallel tests --- tests/validate_test.py | 89 +++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index a2a9ba2df..0d21811d7 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -260,7 +260,7 @@ def test_validate_corrupted_intermediate_backup(self): # Simple validate try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -363,7 +363,7 @@ def test_validate_corrupted_intermediate_backups(self): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -462,7 +462,7 @@ def test_validate_specific_error_intermediate_backups(self): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because backup has status ERROR.\n " @@ -549,7 +549,7 @@ def test_validate_error_intermediate_backups(self): # Validate instance try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because backup has status ERROR.\n " @@ -688,7 +688,7 @@ def test_validate_corrupted_intermediate_backups_1(self): try: self.validate_pb( backup_dir, 'node', - backup_id=backup_id_4) + backup_id=backup_id_4, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n" @@ -888,7 +888,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.validate_pb( backup_dir, 'node', options=[ - '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + '-i', backup_id_4, '--xid={0}'.format(target_xid), + "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1029,8 +1030,7 @@ def test_validate_instance_with_corrupted_page(self): # Validate Instance try: - self.validate_pb( - backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1178,7 +1178,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1279,7 +1279,7 @@ def test_validate_instance_with_corrupted_full(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1337,7 +1337,7 @@ def test_validate_corrupt_wal_1(self): # Simple validate try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1407,7 +1407,7 @@ def test_validate_corrupt_wal_2(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid)]) + "--xid={0}".format(target_xid), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1464,7 +1464,7 @@ def test_validate_wal_lost_segment_1(self): file = file[:-3] try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n" @@ -1488,7 +1488,7 @@ def test_validate_wal_lost_segment_1(self): # Run validate again try: - self.validate_pb(backup_dir, 'node', backup_id) + self.validate_pb(backup_dir, 'node', backup_id, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup corruption.\n" @@ -1572,7 +1572,7 @@ def test_validate_corrupt_wal_between_backups(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid)]) + "--xid={0}".format(target_xid), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1626,7 +1626,7 @@ def test_pgpro702_688(self): try: self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(recovery_time)]) + options=["--time={0}".format(recovery_time), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " @@ -1666,7 +1666,8 @@ def test_pgpro688(self): backup_dir, 'node', backup_id)['recovery-time'] self.validate_pb( - backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + backup_dir, 'node', options=["--time={0}".format(recovery_time), + "-j", "4"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1823,7 +1824,7 @@ def test_validate_corrupted_full(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -1861,7 +1862,7 @@ def test_validate_corrupted_full(self): os.rename(file_new, file) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -1926,7 +1927,7 @@ def test_validate_corrupted_full_1(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -1966,7 +1967,7 @@ def test_validate_corrupted_full_1(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -2040,7 +2041,8 @@ def test_validate_corrupted_full_2(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2087,7 +2089,7 @@ def test_validate_corrupted_full_2(self): self.backup_node(backup_dir, 'node', node, backup_type='page') try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2155,7 +2157,8 @@ def test_validate_corrupted_full_2(self): # revalidate again try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2228,7 +2231,8 @@ def test_validate_corrupted_full_2(self): # Fix CORRUPT os.rename(file_new, file) - output = self.validate_pb(backup_dir, 'node', validate_id) + output = self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertIn( 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), @@ -2390,7 +2394,7 @@ def test_validate_corrupted_full_missing(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2432,7 +2436,7 @@ def test_validate_corrupted_full_missing(self): os.rename(old_directory, new_directory) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid', e.message, @@ -2476,7 +2480,7 @@ def test_validate_corrupted_full_missing(self): # second time must be provided with ID of missing backup try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid', e.message, @@ -2521,7 +2525,7 @@ def test_validate_corrupted_full_missing(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - output = self.validate_pb(backup_dir) + output = self.validate_pb(backup_dir, options=["-j", "4"]) self.assertIn( 'INFO: All backups are valid', @@ -2697,7 +2701,8 @@ def test_validate_specific_backup_with_missing_backup(self): os.rename(old_directory, new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2737,7 +2742,8 @@ def test_validate_specific_backup_with_missing_backup(self): self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2766,7 +2772,7 @@ def test_validate_specific_backup_with_missing_backup(self): os.rename(new_directory, old_directory) # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -2844,7 +2850,8 @@ def test_validate_specific_backup_with_missing_backup_1(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2887,7 +2894,7 @@ def test_validate_specific_backup_with_missing_backup_1(self): os.rename(full_new_directory, full_old_directory) # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -2967,7 +2974,8 @@ def test_validate_with_missing_backup_1(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3010,7 +3018,8 @@ def test_validate_with_missing_backup_1(self): # Revalidate backup chain try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3075,7 +3084,7 @@ def test_validate_with_missing_backup_1(self): os.rename(full_new_directory, full_old_directory) # Revalidate chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -3151,7 +3160,7 @@ def test_validate_with_missing_backup_2(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3200,7 +3209,7 @@ def test_validate_with_missing_backup_2(self): # Revalidate backup chain try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3309,7 +3318,7 @@ def test_corrupt_pg_control_via_resetxlog(self): # Validate backup try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of pg_control change.\n " From bd7b9bb4a1c7f04bee29665a84b6a2b255e1be71 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Mar 2019 14:21:20 +0300 Subject: [PATCH 0376/2107] PGPRO-2079: Use .partial file to write backup configuration files backup.control and backup_content.control --- src/catalog.c | 54 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 87be436c4..4b2e35878 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -569,22 +569,38 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) void write_backup(pgBackup *backup) { - FILE *fp = NULL; - char conf_path[MAXPGPATH]; + FILE *fp = NULL; + char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; + int errno_temp; + + pgBackupGetPath(backup, path, lengthof(path), BACKUP_CONTROL_FILE); + snprintf(path_temp, sizeof(path_temp), "%s.partial", path); - pgBackupGetPath(backup, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); - fp = fopen(conf_path, "wt"); + fp = fopen(path_temp, "wt"); if (fp == NULL) - elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, - strerror(errno)); + elog(ERROR, "Cannot open configuration file \"%s\": %s", + path_temp, strerror(errno)); pgBackupWriteControl(fp, backup); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) + { + errno_temp = errno; + unlink(path_temp); elog(ERROR, "Cannot write configuration file \"%s\": %s", - conf_path, strerror(errno)); + path_temp, strerror(errno_temp)); + } + + if (rename(path_temp, path) < 0) + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno_temp)); + } } /* @@ -595,20 +611,36 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root) { FILE *fp; char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; + int errno_temp; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + snprintf(path_temp, sizeof(path_temp), "%s.partial", path); - fp = fopen(path, "wt"); + fp = fopen(path_temp, "wt"); if (fp == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", path, - strerror(errno)); + elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, + strerror(errno)); print_file_list(fp, files, root); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) - elog(ERROR, "Cannot write file list \"%s\": %s", path, strerror(errno)); + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot write file list \"%s\": %s", + path_temp, strerror(errno)); + } + + if (rename(path_temp, path) < 0) + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno_temp)); + } } /* From b00243f05b5f82cb3eeea79c33143d06187b366b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 Mar 2019 20:23:17 +0300 Subject: [PATCH 0377/2107] bump version to 2.0.28 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 196898a8f..00470f4b8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -171,8 +171,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.0.27" -#define AGENT_PROTOCOL_VERSION 20026 +#define PROGRAM_VERSION "2.0.28" +#define AGENT_PROTOCOL_VERSION 20028 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1b495cda5..e98eea5d4 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.27 \ No newline at end of file +pg_probackup 2.0.28 \ No newline at end of file From 79087d38bf8c252f9bcb43da701857f645d1f73e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 7 Mar 2019 11:01:07 +0300 Subject: [PATCH 0378/2107] Fix problem with backuping non-data files --- src/backup.c | 2 +- src/data.c | 29 +++++++++++++++-------------- src/merge.c | 2 +- src/pg_probackup.h | 2 +- src/restore.c | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/backup.c b/src/backup.c index 86be9b1c0..e614c21c5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2333,7 +2333,7 @@ backup_files(void *arg) skip = true; /* ...skip copying file. */ } if (skip || - !copy_file(arguments->from_root, arguments->to_root, file, FIO_BACKUP_HOST)) + !copy_file(arguments->from_root, FIO_DB_HOST, arguments->to_root, FIO_BACKUP_HOST, file)) { /* disappeared file not to be confused with 'not changed' */ if (file->write_size != FILE_NOT_FOUND) diff --git a/src/data.c b/src/data.c index c339cea50..3c58f0fd8 100644 --- a/src/data.c +++ b/src/data.c @@ -912,7 +912,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * it is either small control file or already compressed cfs file. */ bool -copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location location) +copy_file(const char *from_root, fio_location from_location, + const char *to_root, fio_location to_location, pgFile *file) { char to_path[MAXPGPATH]; FILE *in; @@ -930,7 +931,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location file->write_size = 0; /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); + in = fio_fopen(file->path, PG_BINARY_R, from_location); if (in == NULL) { FIN_FILE_CRC32(true, crc); @@ -950,19 +951,19 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location /* open backup file for write */ join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - out = fio_fopen(to_path, PG_BINARY_W, location); + out = fio_fopen(to_path, PG_BINARY_W, to_location); if (out == NULL) { int errno_tmp = errno; - fclose(in); + fio_fclose(in); elog(ERROR, "cannot open destination file \"%s\": %s", to_path, strerror(errno_tmp)); } /* stat source file to change mode of destination file */ - if (fstat(fileno(in), &st) == -1) + if (fio_ffstat(in, &st) == -1) { - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot stat \"%s\": %s", file->path, strerror(errno)); @@ -973,14 +974,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location { read_len = 0; - if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf)) + if ((read_len = fio_fread(in, buf, sizeof(buf))) != sizeof(buf)) break; if (fio_fwrite(out, buf, read_len) != read_len) { errno_tmp = errno; /* oops */ - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); @@ -992,9 +993,9 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location } errno_tmp = errno; - if (!feof(in)) + if (read_len < 0) { - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot read backup mode file \"%s\": %s", file->path, strerror(errno_tmp)); @@ -1007,7 +1008,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location { errno_tmp = errno; /* oops */ - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot write to \"%s\": %s", to_path, strerror(errno_tmp)); @@ -1024,10 +1025,10 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location file->crc = crc; /* update file permission */ - if (fio_chmod(to_path, st.st_mode, location) == -1) + if (fio_chmod(to_path, st.st_mode, to_location) == -1) { errno_tmp = errno; - fclose(in); + fio_fclose(in); fio_fclose(out); elog(ERROR, "cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); @@ -1036,7 +1037,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location if (fio_fflush(out) != 0 || fio_fclose(out)) elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); - fclose(in); + fio_fclose(in); return true; } diff --git a/src/merge.c b/src/merge.c index bdb20effd..2ee1aa678 100644 --- a/src/merge.c +++ b/src/merge.c @@ -561,7 +561,7 @@ merge_files(void *arg) else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(argument->from_root, argument->to_root, file, FIO_LOCAL_HOST); else - copy_file(argument->from_root, argument->to_root, file, FIO_LOCAL_HOST); + copy_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); /* * We need to save compression algorithm type of the target backup to be diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 00470f4b8..f82dba08d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -570,7 +570,7 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern bool copy_file(const char *from_root, const char *to_root, pgFile *file, fio_location location); +extern bool copy_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); diff --git a/src/restore.c b/src/restore.c index 278195b20..f75962d2d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -676,7 +676,7 @@ restore_files(void *arg) else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(from_root, instance_config.pgdata, file, FIO_DB_HOST); else - copy_file(from_root, instance_config.pgdata, file, FIO_DB_HOST); + copy_file(from_root, FIO_BACKUP_HOST, instance_config.pgdata, FIO_DB_HOST, file); /* print size of restored file */ if (file->write_size != BYTES_INVALID) From f0cbfb71478f224e2f85859ed23beff85728f9f7 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 7 Mar 2019 12:21:27 +0300 Subject: [PATCH 0379/2107] Include version name in start agent log message --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 51b1a336e..e3ecffeb3 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -128,7 +128,7 @@ bool launch_agent(void) if (execvp(ssh_argv[0], ssh_argv) < 0) return false; } else { - elog(LOG, "Spawn agent %d", child_pid); + elog(LOG, "Spawn agent %d version %s", child_pid, PROGRAM_VERSION); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); /*atexit(kill_child);*/ From a1404d86b55df73825a5ec6937188c4a743d227c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 Mar 2019 16:17:04 +0300 Subject: [PATCH 0380/2107] tests: added delete_test.DeleteTest.test_delete_backup_with_empty_control_file() --- tests/delete_test.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/delete_test.py b/tests/delete_test.py index f49c01bf2..c7f52cfc4 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -209,3 +209,41 @@ def test_delete_orphaned_wal_segments(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_backup_with_empty_control_file(self): + """ + take backup, truncate its control file, + try to delete it via 'delete' command + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # page backup mode + self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + # page backup mode + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + + with open( + os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.control'), + 'wt') as f: + f.flush() + f.close() + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 49aa6e7ce48c6b2c246a788c157ae1ca5ddbc225 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 01:02:35 +0300 Subject: [PATCH 0381/2107] tests: added tests.external.ExternalTest.test_external_dir_is_symlink() --- tests/external.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/external.py b/tests/external.py index 888188563..1390eef31 100644 --- a/tests/external.py +++ b/tests/external.py @@ -842,8 +842,88 @@ def test_restore_skip_external(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_is_symlink(self): + """ + Check that backup works correctly if external dir is symlink, + symlink should be read correctly, but not restored + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + node.pgbench_init(scale=3) + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, module_name, fname) + simlinked_dir = os.path.join(core_dir, 'simlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=simlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + os.symlink(simlinked_dir, external_dir) + + # FULL backup with external directories + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # external directory contain symlink to file # external directory contain symlink to directory + # external directory is symlink + # latest page backup without external_dir + # multiple external directories + # --external-dirs=none + From 6e2e56ce5f3c0b0d9916a329d8857dd07067f4aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 01:06:37 +0300 Subject: [PATCH 0382/2107] tests: update comments in tests.external.ExternalTest.test_external_dir_is_symlink --- tests/external.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/external.py b/tests/external.py index 1390eef31..0b9e2c090 100644 --- a/tests/external.py +++ b/tests/external.py @@ -847,7 +847,8 @@ def test_restore_skip_external(self): def test_external_dir_is_symlink(self): """ Check that backup works correctly if external dir is symlink, - symlink should be read correctly, but not restored + symlink pointing to external dir should be followed, + but restored as directory """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -875,8 +876,6 @@ def test_external_dir_is_symlink(self): # fill some directory with data core_dir = os.path.join(self.tmp_path, module_name, fname) - simlinked_dir = os.path.join(core_dir, 'simlinked') - self.restore_node( backup_dir, 'node', node, data_dir=simlinked_dir, options=["-j", "4"]) From edcdbf5eb3148d5c782e2cebfe8e769e36e26fc7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 12:43:04 +0300 Subject: [PATCH 0383/2107] tests: added tests.merge.MergeTest.test_crash_after_opening_backup_control --- tests/merge.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 6e987a2a4..73c2f293e 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1369,6 +1369,56 @@ def test_merge_different_wal_modes(self): self.del_test_dir(module_name, fname) + def test_crash_after_opening_backup_control(self): + """ + check that crashing after opening backup.control + for writing will not result in losing backup metadata + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('write_backup') + gdb.continue_execution_until_break() + gdb.set_breakpoint('pgBackupWriteControl') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From d580907b6d6705da24b06662d2f030fac62f9f4c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 13:36:21 +0300 Subject: [PATCH 0384/2107] tests: added tests.merge.MergeTest.test_crash_after_opening_backup_control_2 --- tests/merge.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 73c2f293e..6e5355031 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1369,7 +1369,7 @@ def test_merge_different_wal_modes(self): self.del_test_dir(module_name, fname) - def test_crash_after_opening_backup_control(self): + def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control for writing will not result in losing backup metadata @@ -1419,6 +1419,89 @@ def test_crash_after_opening_backup_control(self): self.del_test_dir(module_name, fname) + def test_crash_after_opening_backup_control_2(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=3) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + vm_path = path + '_vm' + + print(vm_path) + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('print_file_list') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + os.remove( + os.path.join( + backup_dir, 'backups', 'node', + full_id, 'database', vm_path)) + + # Continue failed merge + self.merge_backup(backup_dir, "node", backup_id) + + node.cleanup() + + # restore merge backup + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 21434cdc6377c148333c48e119bd610858db2ef3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 14:25:50 +0300 Subject: [PATCH 0385/2107] tests: minor fixes for merge module --- tests/merge.py | 114 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 6e5355031..42f8b6f76 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1429,7 +1429,8 @@ def test_crash_after_opening_backup_control_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1444,16 +1445,18 @@ def test_crash_after_opening_backup_control_2(self): backup_dir, 'node', node, options=['--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '20', '-c', '2']) pgbench.wait() path = node.safe_psql( 'postgres', "select pg_relation_filepath('pgbench_accounts')").rstrip() - vm_path = path + '_vm' + fsm_path = path + '_fsm' - print(vm_path) + node.safe_psql( + 'postgres', + 'vacuum pgbench_accounts') # DELTA backup backup_id = self.backup_node( @@ -1483,10 +1486,14 @@ def test_crash_after_opening_backup_control_2(self): 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) # In to_backup drop file that comes from from_backup - os.remove( - os.path.join( - backup_dir, 'backups', 'node', - full_id, 'database', vm_path)) + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', fsm_path) + + print(file_to_remove) + + os.remove(file_to_remove) # Continue failed merge self.merge_backup(backup_dir, "node", backup_id) @@ -1502,6 +1509,97 @@ def test_crash_after_opening_backup_control_2(self): self.del_test_dir(module_name, fname) + def test_losing_file_after_failed_merge(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=3) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + vm_path = path + '_vm' + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('print_file_list') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', vm_path) + + print(file_to_remove) + + os.remove(file_to_remove) + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Merging of backup {0} failed".format( + backup_id) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node')[0]['status']) + + # self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From cb805f7e7c843745cb93bc926aba17819e65106c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 Mar 2019 14:53:32 +0300 Subject: [PATCH 0386/2107] tests: added tests.retention_test.RetentionTest.test_retention_window_3 --- tests/retention_test.py | 44 ++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/retention_test.py b/tests/retention_test.py index 34c026587..34a728d23 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -122,38 +122,31 @@ def test_retention_window_2(self): self.del_test_dir(module_name, fname) # @unittest.skip("123") - def test_retention_wal(self): - """purge backups using window-based retention policy""" + def test_retention_window_3(self): + """purge all backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # Take second FULL BACKUP + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Take third FULL BACKUP + backup_id_3 = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node) backups = os.path.join(backup_dir, 'backups', 'node') - days_delta = 5 for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue @@ -161,18 +154,15 @@ def test_retention_wal(self): os.path.join( backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=days_delta))) - days_delta -= 1 - - # Make backup to be keeped - self.backup_node(backup_dir, 'node', node, backup_type="page") - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + datetime.now() - timedelta(days=3))) # Purge backups self.delete_expired( - backup_dir, 'node', options=['--retention-window=2']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + backup_dir, 'node', options=['--retention-window=1']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) # Clean after yourself self.del_test_dir(module_name, fname) From 631590bec5ba2aee1931380ee2cab2aadc94d077 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 11 Mar 2019 21:34:58 +0300 Subject: [PATCH 0387/2107] Make possible to pass symlink as -E option --- src/dir.c | 2 +- tests/external.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/dir.c b/src/dir.c index 4d4af0e41..a2254f8e6 100644 --- a/src/dir.c +++ b/src/dir.c @@ -472,7 +472,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, false, external_dir_num); + file = pgFileNew(root, external_dir_num ? omit_symlink : false, omit_symlink); if (file == NULL) return; diff --git a/tests/external.py b/tests/external.py index 0b9e2c090..ca9226006 100644 --- a/tests/external.py +++ b/tests/external.py @@ -867,6 +867,7 @@ def test_external_dir_is_symlink(self): node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') + symlinked_dir = self.get_tblspace_path(node, 'symlinked_dir') node.pgbench_init(scale=3) @@ -878,13 +879,13 @@ def test_external_dir_is_symlink(self): core_dir = os.path.join(self.tmp_path, module_name, fname) self.restore_node( backup_dir, 'node', node, - data_dir=simlinked_dir, options=["-j", "4"]) + data_dir=symlinked_dir, options=["-j", "4"]) # drop temp FULL backup self.delete_pb(backup_dir, 'node', backup_id=backup_id) # create symlink to directory in external directory - os.symlink(simlinked_dir, external_dir) + os.symlink(symlinked_dir, external_dir) # FULL backup with external directories self.backup_node( @@ -894,9 +895,7 @@ def test_external_dir_is_symlink(self): "-E", "{0}".format( external_dir)]) - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - + symlink_data = self.pgdata_content(symlinked_dir) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -912,10 +911,9 @@ def test_external_dir_is_symlink(self): "-j", "4", "--external-mapping={0}={1}".format( external_dir, external_dir_new)]) - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) + restored_data = self.pgdata_content(external_dir_new) - self.compare_pgdata(pgdata, pgdata_restored) + self.compare_pgdata(symlink_data, restored_data) # Clean after yourself self.del_test_dir(module_name, fname) From 0ef24d0b7184032af07d17f9b6344823db0fe8b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 11:31:10 +0300 Subject: [PATCH 0388/2107] add "external-dirs" to show output in json format --- src/show.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/show.c b/src/show.c index ffcd0038e..083962e55 100644 --- a/src/show.c +++ b/src/show.c @@ -635,6 +635,10 @@ show_instance_json(parray *backup_list) json_add_value(buf, "primary_conninfo", backup->primary_conninfo, json_level, true); + if (backup->external_dir_str) + json_add_value(buf, "external-dirs", backup->external_dir_str, + json_level, true); + json_add_value(buf, "status", status2str(backup->status), json_level, true); From 8c365b67e8e17e4615ae84318fde32be54883e64 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 11:33:42 +0300 Subject: [PATCH 0389/2107] tests: fix test_external_dir_is_symlink --- tests/external.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/external.py b/tests/external.py index ca9226006..7ce7d6342 100644 --- a/tests/external.py +++ b/tests/external.py @@ -867,7 +867,6 @@ def test_external_dir_is_symlink(self): node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') - symlinked_dir = self.get_tblspace_path(node, 'symlinked_dir') node.pgbench_init(scale=3) @@ -877,6 +876,8 @@ def test_external_dir_is_symlink(self): # fill some directory with data core_dir = os.path.join(self.tmp_path, module_name, fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + self.restore_node( backup_dir, 'node', node, data_dir=symlinked_dir, options=["-j", "4"]) @@ -888,14 +889,16 @@ def test_external_dir_is_symlink(self): os.symlink(symlinked_dir, external_dir) # FULL backup with external directories - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, options=[ "-j", "4", "--stream", "-E", "{0}".format( external_dir)]) - symlink_data = self.pgdata_content(symlinked_dir) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -911,9 +914,17 @@ def test_external_dir_is_symlink(self): "-j", "4", "--external-mapping={0}={1}".format( external_dir, external_dir_new)]) - restored_data = self.pgdata_content(external_dir_new) + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + - self.compare_pgdata(symlink_data, restored_data) + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) # Clean after yourself self.del_test_dir(module_name, fname) From 5af689ea7470a3385acf558d1a7780749e57acbb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 12:09:28 +0300 Subject: [PATCH 0390/2107] tests: check external dir pointing to a file --- tests/external.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/external.py b/tests/external.py index 7ce7d6342..4c77c791b 100644 --- a/tests/external.py +++ b/tests/external.py @@ -20,6 +20,8 @@ def test_external_simple(self): external directory was successfully copied """ fname = self.id().split('.')[3] + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], @@ -35,6 +37,27 @@ def test_external_simple(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + # take FULL backup with external directory pointing to a file + file_path = os.path.join(core_dir, 'file') + open(file_path,"w+") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=[ + '--external-dirs={0}'.format(file_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir point to a file" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Insert correct message' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + # FULL backup self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -919,7 +942,6 @@ def test_external_dir_is_symlink(self): self.compare_pgdata(pgdata, pgdata_restored) - self.assertEqual( external_dir, self.show_pb( From 6a4b014f8de4f186d72d926170ec8e7ccd538ad9 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 12 Mar 2019 13:33:08 +0300 Subject: [PATCH 0391/2107] Fail if regular file passed as -E option --- src/dir.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index a2254f8e6..c503f5227 100644 --- a/src/dir.c +++ b/src/dir.c @@ -478,7 +478,11 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (!S_ISDIR(file->mode)) { - elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + if (external_dir_num) + elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", + file->path); + else + elog(WARNING, "Skip \"%s\": unexpected file format", file->path); return; } if (add_root) From 182fc7dfe14fad7589021b3ca7e7b0d0f42c76e7 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 12 Mar 2019 13:57:40 +0300 Subject: [PATCH 0392/2107] PGPRO-2521: Allow to delete backups without or emtpy .control file --- src/catalog.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 4b2e35878..59a0930e8 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -74,6 +74,14 @@ write_backup_status(pgBackup *backup, BackupStatus status) pgBackup *tmp; tmp = read_backup(backup->start_time); + if (!tmp) + { + /* + * Silently exit the function, since read_backup already logged the + * warning message. + */ + return; + } backup->status = status; tmp->status = backup->status; @@ -301,11 +309,10 @@ IsDir(const char *dirpath, const char *entry) parray * catalog_get_backup_list(time_t requested_backup_id) { - DIR *data_dir = NULL; - struct dirent *data_ent = NULL; - parray *backups = NULL; - pgBackup *backup = NULL; - int i; + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + int i; /* open backup instance backups directory */ data_dir = opendir(backup_instance_path); @@ -320,8 +327,9 @@ catalog_get_backup_list(time_t requested_backup_id) backups = parray_new(); for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) { - char backup_conf_path[MAXPGPATH]; - char data_path[MAXPGPATH]; + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + pgBackup *backup = NULL; /* skip not-directory entries and hidden entries */ if (!IsDir(backup_instance_path, data_ent->d_name) @@ -355,7 +363,6 @@ catalog_get_backup_list(time_t requested_backup_id) continue; } parray_append(backups, backup); - backup = NULL; if (errno && errno != ENOENT) { @@ -405,8 +412,6 @@ catalog_get_backup_list(time_t requested_backup_id) err_proc: if (data_dir) closedir(data_dir); - if (backup) - pgBackupFree(backup); if (backups) parray_walk(backups, pgBackupFree); parray_free(backups); From c2936e95b7f8e6069b9e771941262babf7605482 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 12 Mar 2019 14:58:00 +0300 Subject: [PATCH 0393/2107] PGPRO-2523: Allow to delete single FULL backup --- src/delete.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index d637b50e9..9ee9b2d1a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -126,7 +126,6 @@ do_retention_purge(void) size_t i; XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; - bool keep_next_backup = true; /* Do not delete first full backup */ bool backup_deleted = false; /* At least one backup was deleted */ if (delete_expired) @@ -158,6 +157,7 @@ do_retention_purge(void) (instance_config.retention_redundancy > 0 || instance_config.retention_window > 0)) { + bool keep_next_backup = false; /* Do not delete first full backup */ time_t days_threshold; uint32 backup_num = 0; @@ -216,6 +216,7 @@ do_retention_purge(void) /* Delete backup and update status to DELETED */ delete_backup_files(backup); backup_deleted = true; + keep_next_backup = false; /* reset it */ } } From ea270f6f960913f65a4887dbb3e67190cce47f20 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 17:46:30 +0300 Subject: [PATCH 0394/2107] tests: minor fix for test_delete_backup_with_empty_control_file --- tests/delete_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/delete_test.py b/tests/delete_test.py index c7f52cfc4..6c817f767 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -243,6 +243,9 @@ def test_delete_backup_with_empty_control_file(self): f.flush() f.close() + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 3) + self.delete_pb(backup_dir, 'node', backup_id=backup_id) # Clean after yourself From 25d9ce9d76aab044924ce4a6156b12d7bde24838 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 21:44:40 +0300 Subject: [PATCH 0395/2107] tests: added test_continue_failed_merge_3(), author Ilya Skvortsov --- tests/merge.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 42f8b6f76..b756ef646 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1274,6 +1274,102 @@ def test_continue_failed_merge_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_continue_failed_merge_3(self): + """ + Check that failed MERGE can`t be continued after target backup deleting + Create FULL and 2 PAGE backups + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create test data + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100000) i" + ) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # CREATE FEW PAGE BACKUP + i = 0 + + while i < 2: + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200000) i" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page' + ) + i = i + 1 + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id_merge = self.show_pb(backup_dir, "node")[2]["id"] + backup_id_delete = self.show_pb(backup_dir, "node")[1]["id"] + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # DELETE PAGE1 + shutil.rmtree( + os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id_merge) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Parent full backup for the given backup {0} was not found".format( + backup_id_merge) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + def test_merge_different_compression_algo(self): """ Check that backups with different compression algorihtms can be merged From 1f2760c1f7eadd42d480df5ced9891ad1bad70ad Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 22:16:58 +0300 Subject: [PATCH 0396/2107] tests: added tests.external.ExternalTest.test_external_dir_is_tablespace --- tests/external.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/external.py b/tests/external.py index 4c77c791b..c985522de 100644 --- a/tests/external.py +++ b/tests/external.py @@ -951,6 +951,59 @@ def test_external_dir_is_symlink(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_is_tablespace(self): + """ + Check that backup fails with error + if external directory points to tablespace + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + self.create_tblspace_in_node( + node, 'tblspace1', tblspc_path=external_dir) + + node.pgbench_init(scale=3, tablespace='tblspace1') + + # FULL backup with external directories + try: + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir points to the tablespace" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Insert correct message' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # external directory contain symlink to file # external directory contain symlink to directory # external directory is symlink + From 78a6ede9f0cd1047005ff48bde3afb3d0069759b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 Mar 2019 16:57:04 +0300 Subject: [PATCH 0397/2107] tests: minor fix in external.ExternalTest.test_external_simple --- tests/external.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/external.py b/tests/external.py index c985522de..77e242679 100644 --- a/tests/external.py +++ b/tests/external.py @@ -53,8 +53,10 @@ def test_external_simple(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'Insert correct message' in e.message, + self.assertIn( + 'ERROR: --external-dirs option "{0}": ' + 'directory or symbolic link expected\n'.format(file_path), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 98dba08d52b11e8cb2f5aba69daf7aea8fa0326a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 13 Mar 2019 17:16:05 +0300 Subject: [PATCH 0398/2107] PGPRO-1973: Report about absent or corrupted WALs only once --- src/parsexlog.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 5bea453ca..9b4323f7e 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -184,6 +184,8 @@ static XLogSegNo segno_target = 0; static XLogSegNo segno_next = 0; /* Number of segments already read by threads */ static uint32 segnum_read = 0; +/* Number of detected corrupted or absent segments */ +static uint32 segnum_corrupted = 0; static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* copied from timestamp.c */ @@ -900,6 +902,7 @@ RunXLogThreads(const char *archivedir, time_t target_time, segno_target = 0; GetXLogSegNo(startpoint, segno_next, segment_size); segnum_read = 0; + segnum_corrupted = 0; threads = (pthread_t *) pgut_malloc(sizeof(pthread_t) * num_threads); thread_args = (xlog_thread_arg *) pgut_malloc(sizeof(xlog_thread_arg) * num_threads); @@ -1103,6 +1106,20 @@ XLogThreadWorker(void *arg) */ if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) break; + else if (wal_consistent_read) + { + XLogSegNo segno_report; + + pthread_lock(&wal_segment_mutex); + segno_report = segno_start + segnum_read; + pthread_mutex_unlock(&wal_segment_mutex); + + /* + * Report error message if this is the first corrupted WAL. + */ + if (reader_data->xlogsegno > segno_report) + return NULL; /* otherwise just stop the thread */ + } errptr = thread_arg->startpoint ? thread_arg->startpoint : xlogreader->EndRecPtr; @@ -1232,6 +1249,20 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) */ if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) return false; + else if (wal_consistent_read) + { + XLogSegNo segno_report; + + pthread_lock(&wal_segment_mutex); + segno_report = segno_start + segnum_read; + pthread_mutex_unlock(&wal_segment_mutex); + + /* + * Report error message if this is the first corrupted WAL. + */ + if (reader_data->xlogsegno > segno_report) + return false; /* otherwise just stop the thread */ + } elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, @@ -1279,16 +1310,16 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) reader_data->thread_num); pthread_lock(&wal_segment_mutex); - segnum_current_read = segnum_read; + segnum_current_read = segnum_read + segnum_corrupted; segno = segno_target; pthread_mutex_unlock(&wal_segment_mutex); /* Other threads read all previous segments and didn't find target */ if (segnum_need <= segnum_current_read) { - /* Mark current segment as read even if it wasn't read actually */ + /* Mark current segment as corrupted */ pthread_lock(&wal_segment_mutex); - segnum_read++; + segnum_corrupted++; pthread_mutex_unlock(&wal_segment_mutex); return false; } @@ -1296,7 +1327,7 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) if (segno != 0 && segno < reader_data->xlogsegno) return true; - pg_usleep(1000000L); /* 1000 ms */ + pg_usleep(500000L); /* 500 ms */ } /* We shouldn't reach it */ From 89366b3fe8b4dca920e83d0b8e5089b1583578fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 Mar 2019 18:38:42 +0300 Subject: [PATCH 0399/2107] tests: added test_external_backward_compatibility_merge* tests --- tests/external.py | 243 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 231 insertions(+), 12 deletions(-) diff --git a/tests/external.py b/tests/external.py index 77e242679..dfc8fc937 100644 --- a/tests/external.py +++ b/tests/external.py @@ -76,9 +76,8 @@ def test_external_simple(self): options=[ '--external-dirs={0}'.format(external_dir)]) - if self.paranoia: - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) node.cleanup() shutil.rmtree(external_dir, ignore_errors=True) @@ -86,10 +85,9 @@ def test_external_simple(self): self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -316,7 +314,12 @@ def test_backup_multiple_external(self): # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility(self): - """""" + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + restore delta backup, check that incremental chain + restored correctly + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -357,15 +360,15 @@ def test_external_backward_compatibility(self): pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) - # FULL backup backup_id = self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) # fill external directories with changed data + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + self.restore_node( backup_dir, 'node', node, data_dir=external_dir1_old, options=["-j", "4"]) @@ -412,6 +415,222 @@ def test_external_backward_compatibility(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_1(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup ajd restore it + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=5) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain chain with new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_2(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup ajd restore it + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=5) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1_old = self.get_tblspace_path(node, 'external_dir1') + external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # Fill external dirs with changed data + shutil.rmtree(external_dir1_old, ignore_errors=True) + shutil.rmtree(external_dir2_old, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1_old, + options=['-j', '4', '--skip-external-dirs']) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2_old, + options=['-j', '4', '--skip-external-dirs']) + + # delta backup without external directories using old binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}:{1}".format( + external_dir1_old, + external_dir2_old)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain chain with new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge(self): @@ -1012,7 +1231,7 @@ def test_external_dir_is_tablespace(self): # latest page backup without external_dir + # multiple external directories + # --external-dirs=none + - # --external-dirs point to a file + # --external-dirs point to a file + # external directory in config and in command line + # external directory contain multuple directories, some of them my be empty + # forbid to external-dirs to point to tablespace directories From f025b90eb07915f21b64671819389434213a52c6 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 14 Mar 2019 13:58:02 +0300 Subject: [PATCH 0400/2107] Rewrite fio_gz* support --- src/data.c | 22 ++-- src/parsexlog.c | 9 +- src/utils/file.c | 261 +++++++++++++++++++++++++++++++++++++---------- src/utils/file.h | 8 +- 4 files changed, 226 insertions(+), 74 deletions(-) diff --git a/src/data.c b/src/data.c index 3c58f0fd8..5c5a30871 100644 --- a/src/data.c +++ b/src/data.c @@ -1102,7 +1102,6 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ char gz_to_path[MAXPGPATH]; gzFile gz_out = NULL; - int gz_tmp = -1; if (is_compress) { @@ -1135,14 +1134,10 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, &gz_tmp, FIO_BACKUP_HOST); + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); - if (gzsetparams(gz_out, instance_config.compress_level, Z_DEFAULT_STRATEGY) != Z_OK) - elog(ERROR, "Cannot set compression level %d to file \"%s\": %s", - instance_config.compress_level, to_path_temp, - get_gz_error(gz_out, errno)); } else #endif @@ -1176,7 +1171,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ if (is_compress) { - if (gzwrite(gz_out, buf, read_len) != read_len) + if (fio_gzwrite(gz_out, buf, read_len) != read_len) { errno_temp = errno; fio_unlink(to_path_temp, FIO_BACKUP_HOST); @@ -1204,7 +1199,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ if (is_compress) { - if (fio_gzclose(gz_out, to_path_temp, gz_tmp) != 0) + if (fio_gzclose(gz_out) != 0) { errno_temp = errno; fio_unlink(to_path_temp, FIO_BACKUP_HOST); @@ -1696,10 +1691,9 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) { char buf [1024]; gzFile gz_in = NULL; - int gz_tmp = -1; INIT_FILE_CRC32(true, crc2); - gz_in = fio_gzopen(path2, PG_BINARY_R, &gz_tmp, FIO_BACKUP_HOST); + gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST); if (gz_in == NULL) /* File cannot be read */ elog(ERROR, @@ -1709,20 +1703,20 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) for (;;) { size_t read_len = 0; - read_len = gzread(gz_in, buf, sizeof(buf)); - if (read_len != sizeof(buf) && !gzeof(gz_in)) + read_len = fio_gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !fio_gzeof(gz_in)) /* An error occurred while reading the file */ elog(ERROR, "Cannot compare WAL file \"%s\" with compressed \"%s\"", path1, path2); COMP_FILE_CRC32(true, crc2, buf, read_len); - if (gzeof(gz_in) || read_len == 0) + if (fio_gzeof(gz_in) || read_len == 0) break; } FIN_FILE_CRC32(true, crc2); - if (fio_gzclose(gz_in, path2, gz_tmp) != 0) + if (fio_gzclose(gz_in) != 0) elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", path2, get_gz_error(gz_in, errno)); } diff --git a/src/parsexlog.c b/src/parsexlog.c index 71c6f365e..8a8bcec80 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -108,7 +108,6 @@ typedef struct XLogPageReadPrivate #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; - int gz_tmp; #endif } XLogPageReadPrivate; @@ -1020,7 +1019,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, private_data->xlogexists = true; private_data->gz_xlogfile = fio_gzopen(private_data->gz_xlogpath, - "rb", &private_data->gz_tmp, private_data->location); + "rb", -1, private_data->location); if (private_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", @@ -1072,7 +1071,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #ifdef HAVE_LIBZ else { - if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (fio_gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", private_data->thread_num, @@ -1081,7 +1080,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, return -1; } - if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (fio_gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", private_data->thread_num, @@ -1147,7 +1146,7 @@ CleanupXLogPageRead(XLogReaderState *xlogreader) #ifdef HAVE_LIBZ else if (private_data->gz_xlogfile != NULL) { - fio_gzclose(private_data->gz_xlogfile, private_data->gz_xlogpath, private_data->gz_tmp); + fio_gzclose(private_data->gz_xlogfile); private_data->gz_xlogfile = NULL; } #endif diff --git a/src/utils/file.c b/src/utils/file.c index 1e7e2c28e..666a2d27b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -729,51 +729,69 @@ int fio_chmod(char const* path, int mode, fio_location location) } #ifdef HAVE_LIBZ -/* Open compressed file. In case of remove file, it is fetched to local temporary file in read-only mode or is written - * to temoporary file and rtansfered to remote host by fio_gzclose. */ -gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location location) + + +#define ZLIB_BUFFER_SIZE (64*1024) +#define MAX_WBITS 15 /* 32K LZ77 window */ +#define DEF_MEM_LEVEL 8 +#define FIO_GZ_REMOTE_MARKER 1 + +typedef struct fioGZFile { - gzFile file; + z_stream strm; + int fd; + bool compress; + bool eof; + Bytef buf[ZLIB_BUFFER_SIZE]; +} fioGZFile; + +gzFile +fio_gzopen(char const* path, char const* mode, int level, fio_location location) +{ + int rc; if (fio_is_remote(location)) { - char pattern1[] = "/tmp/gz.XXXXXX"; - char pattern2[] = "gz.XXXXXX"; - char* path = pattern1; - int fd = mkstemp(path); /* first try to create file in current directory */ - if (fd < 0) - { - path = pattern2; - fd = mkstemp(path); /* if it is not possible, try to create it in /tmp/ */ - if (fd < 0) - return NULL; - } - unlink(path); /* delete file on close */ + fioGZFile* gz = (fioGZFile*)malloc(sizeof(fioGZFile)); + memset(&gz->strm, 0, sizeof(gz->strm)); + gz->eof = 0; - if (strcmp(mode, PG_BINARY_W) == 0) + if (strcmp(mode, PG_BINARY_W) == 0) /* compress */ { - *tmp_fd = fd; + gz->strm.next_out = gz->buf; + gz->strm.avail_out = ZLIB_BUFFER_SIZE; + rc = deflateInit2(&gz->strm, + level, + Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY); + if (rc == Z_OK) + { + gz->compress = 1; + gz->fd = fio_open(path, O_WRONLY|O_CREAT|O_TRUNC, location); + } } else { - int rd = fio_open(path, O_RDONLY|PG_BINARY, location); - struct stat st; - void* buf; - if (rd < 0) { - return NULL; + gz->strm.next_in = gz->buf; + gz->strm.avail_in = ZLIB_BUFFER_SIZE; + rc = inflateInit2(&gz->strm, 15 + 16); + gz->strm.avail_in = 0; + if (rc == Z_OK) + { + gz->compress = 0; + gz->fd = fio_open(path, O_RDONLY, location); } - SYS_CHECK(fio_fstat(rd, &st)); - buf = malloc(st.st_size); - IO_CHECK(fio_read(rd, buf, st.st_size), st.st_size); - IO_CHECK(write(fd, buf, st.st_size), st.st_size); - SYS_CHECK(fio_close(rd)); - free(buf); - *tmp_fd = -1; } - file = gzdopen(fd, mode); + if (rc != Z_OK) + { + free(gz); + return NULL; + } + return (gzFile)((size_t)gz + FIO_GZ_REMOTE_MARKER); } else { - *tmp_fd = -1; + gzFile file; if (strcmp(mode, PG_BINARY_W) == 0) { int fd = open(path, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FILE_PERMISSIONS); @@ -783,43 +801,180 @@ gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location } else file = gzopen(path, mode); + if (file != NULL && level != Z_DEFAULT_COMPRESSION) + { + if (gzsetparams(file, level, Z_DEFAULT_STRATEGY) != Z_OK) + elog(ERROR, "Cannot set compression level %d: %s", + level, strerror(errno)); + } + return file; } - return file; } -/* Close compressed file. In case of writing remote file, content of temporary file is trasfered to remote host */ -int fio_gzclose(gzFile file, char const* path, int tmp_fd) +int +fio_gzread(gzFile f, void *buf, unsigned size) { - if (tmp_fd >= 0) + if ((size_t)f & FIO_GZ_REMOTE_MARKER) { - off_t size; - void* buf; - int fd; + int rc; + fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); + + if (gz->eof) + { + return 0; + } - SYS_CHECK(gzflush(file, Z_FINISH)); + gz->strm.next_out = (Bytef *)buf; + gz->strm.avail_out = size; - size = lseek(tmp_fd, 0, SEEK_END); - buf = malloc(size); + while (1) + { + if (gz->strm.avail_in != 0) /* If there is some data in receiver buffer, then decmpress it */ + { + rc = inflate(&gz->strm, Z_NO_FLUSH); + if (rc == Z_STREAM_END) + { + gz->eof = 1; + } + else if (rc != Z_OK) + { + return -1; + } + if (gz->strm.avail_out != size) + { + return size - gz->strm.avail_out; + } + if (gz->strm.avail_in == 0) + { + gz->strm.next_in = gz->buf; + } + } + else + { + gz->strm.next_in = gz->buf; + } + rc = read(gz->fd, gz->strm.next_in + gz->strm.avail_in, gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); + if (rc > 0) + { + gz->strm.avail_in += rc; + } + else + { + if (rc == 0) + { + gz->eof = 1; + } + return rc; + } + } + } + else + { + return gzread(f, buf, size); + } +} - lseek(tmp_fd, 0, SEEK_SET); - IO_CHECK(read(tmp_fd, buf, size), size); +int +fio_gzwrite(gzFile f, void const* buf, unsigned size) +{ + if ((size_t)f & FIO_GZ_REMOTE_MARKER) + { + int rc; + fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - SYS_CHECK(gzclose(file)); /* should close tmp_fd */ + gz->strm.next_in = (Bytef *)buf; + gz->strm.avail_in = size; - fd = fio_open(path, O_RDWR|O_CREAT|O_EXCL|PG_BINARY, FILE_PERMISSIONS); - if (fd < 0) { - free(buf); - return -1; + do + { + if (gz->strm.avail_out == ZLIB_BUFFER_SIZE) /* Compress buffer is empty */ + { + gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ + + if (gz->strm.avail_in != 0) /* Has something in input buffer */ + { + rc = deflate(&gz->strm, Z_NO_FLUSH); + Assert(rc == Z_OK); + gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of bufer */ + } + else + { + break; + } + } + rc = write(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + if (rc >= 0) + { + gz->strm.next_out += rc; + gz->strm.avail_out += rc; + } + else + { + return rc; + } + } while (gz->strm.avail_out != ZLIB_BUFFER_SIZE || gz->strm.avail_in != 0); + + return size; + } + else + { + return gzwrite(f, buf, size); + } +} + +int +fio_gzclose(gzFile f) +{ + if ((size_t)f & FIO_GZ_REMOTE_MARKER) + { + fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); + int rc; + if (gz->compress) + { + gz->strm.next_out = gz->buf; + rc = deflate(&gz->strm, Z_FINISH); + Assert(rc == Z_STREAM_END && gz->strm.avail_out != ZLIB_BUFFER_SIZE); + deflateEnd(&gz->strm); + rc = write(gz->fd, gz->buf, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + if (rc != ZLIB_BUFFER_SIZE - gz->strm.avail_out) + { + return -1; + } } - IO_CHECK(fio_write(fd, buf, size), size); - free(buf); - return fio_close(fd); + else + { + inflateEnd(&gz->strm); + } + rc = fio_close(gz->fd); + free(gz); + return rc; + } + else + { + return gzclose(f); + } +} + +int fio_gzeof(gzFile f) +{ + if ((size_t)f & FIO_GZ_REMOTE_MARKER) + { + fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); + return gz->eof; } else { - return gzclose(file); + return gzeof(f); } } + +z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) +{ + Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); + return gzseek(f, offset, whence); +} + + #endif /* Send file content */ diff --git a/src/utils/file.h b/src/utils/file.h index 8f27f480f..752bb69b3 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -105,8 +105,12 @@ extern FILE* fio_open_stream(char const* name, fio_location location); extern int fio_close_stream(FILE* f); #ifdef HAVE_LIBZ -extern gzFile fio_gzopen(char const* path, char const* mode, int* tmp_fd, fio_location location); -extern int fio_gzclose(gzFile file, char const* path, int tmp_fd); +extern gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location); +extern int fio_gzclose(gzFile file); +extern int fio_gzread(gzFile f, void *buf, unsigned size); +extern int fio_gzwrite(gzFile f, void const* buf, unsigned size); +extern int fio_gzeof(gzFile f); +extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); #endif #endif From cec20ccb8390106123c0c27a10abee840a178bc4 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 14 Mar 2019 14:05:22 +0300 Subject: [PATCH 0401/2107] [refer #PGPRO-1745] Rewrite fio_gz* support --- src/utils/file.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 666a2d27b..da99e8b8f 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -853,7 +853,7 @@ fio_gzread(gzFile f, void *buf, unsigned size) { gz->strm.next_in = gz->buf; } - rc = read(gz->fd, gz->strm.next_in + gz->strm.avail_in, gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); + rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in, gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); if (rc > 0) { gz->strm.avail_in += rc; @@ -902,7 +902,7 @@ fio_gzwrite(gzFile f, void const* buf, unsigned size) break; } } - rc = write(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + rc = fio_write(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); if (rc >= 0) { gz->strm.next_out += rc; @@ -935,7 +935,7 @@ fio_gzclose(gzFile f) rc = deflate(&gz->strm, Z_FINISH); Assert(rc == Z_STREAM_END && gz->strm.avail_out != ZLIB_BUFFER_SIZE); deflateEnd(&gz->strm); - rc = write(gz->fd, gz->buf, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + rc = fio_write(gz->fd, gz->buf, ZLIB_BUFFER_SIZE - gz->strm.avail_out); if (rc != ZLIB_BUFFER_SIZE - gz->strm.avail_out) { return -1; From a8dcbbf04f1aae0e450fc3d4e9a64059e15cfc61 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Mar 2019 14:24:18 +0300 Subject: [PATCH 0402/2107] PGPRO-1973: Improve XLogWaitForConsistency() verbose message --- src/parsexlog.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 9b4323f7e..630a15b83 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1300,8 +1300,13 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) if (log_message) { - elog(VERBOSE, "Thread [%d]: Possible WAL corruption. Wait for other threads to decide is this a failure", - reader_data->thread_num); + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, + wal_seg_size); + + elog(VERBOSE, "Thread [%d]: Possible WAL corruption in %s. Wait for other threads to decide is this a failure", + reader_data->thread_num, xlogfname); log_message = false; } From ed81839ab85bda10abec6dfcd37903cfa239c6ab Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 14 Mar 2019 16:41:19 +0300 Subject: [PATCH 0403/2107] Improve error reporting in remote backup --- src/data.c | 2 +- src/parsexlog.c | 2 +- src/utils/file.c | 19 ++++++++++++++++++- src/utils/file.h | 3 ++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index 5c5a30871..268f68342 100644 --- a/src/data.c +++ b/src/data.c @@ -1052,7 +1052,7 @@ get_gz_error(gzFile gzf, int errnum) int gz_errnum; const char *errmsg; - errmsg = gzerror(gzf, &gz_errnum); + errmsg = fio_gzerror(gzf, &gz_errnum); if (gz_errnum == Z_ERRNO) return strerror(errnum); else diff --git a/src/parsexlog.c b/src/parsexlog.c index 8a8bcec80..f0118a72e 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -906,7 +906,7 @@ get_gz_error(gzFile gzf) int errnum; const char *errmsg; - errmsg = gzerror(gzf, &errnum); + errmsg = fio_gzerror(gzf, &errnum); if (errnum == Z_ERRNO) return strerror(errno); else diff --git a/src/utils/file.c b/src/utils/file.c index da99e8b8f..fb89e9715 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -740,6 +740,7 @@ typedef struct fioGZFile { z_stream strm; int fd; + int errnum; bool compress; bool eof; Bytef buf[ZLIB_BUFFER_SIZE]; @@ -754,7 +755,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) fioGZFile* gz = (fioGZFile*)malloc(sizeof(fioGZFile)); memset(&gz->strm, 0, sizeof(gz->strm)); gz->eof = 0; - + gz->errnum = Z_OK; if (strcmp(mode, PG_BINARY_W) == 0) /* compress */ { gz->strm.next_out = gz->buf; @@ -838,6 +839,7 @@ fio_gzread(gzFile f, void *buf, unsigned size) } else if (rc != Z_OK) { + gz->errnum = rc; return -1; } if (gz->strm.avail_out != size) @@ -968,6 +970,21 @@ int fio_gzeof(gzFile f) } } +const char* fio_gzerror(gzFile f, int *errnum) +{ + if ((size_t)f & FIO_GZ_REMOTE_MARKER) + { + fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); + if (errnum) + *errnum = gz->errnum; + return gz->strm.msg; + } + else + { + return gzerror(f, errnum); + } +} + z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) { Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); diff --git a/src/utils/file.h b/src/utils/file.h index 752bb69b3..8ef40fdcb 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -49,7 +49,7 @@ typedef enum #define PAGE_CHECKSUM_MISMATCH (-256) #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc < 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) if (remote_agent) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc >= 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } else elog(ERROR, "Communication error: %s", _rc >= 0 ? "end of data" : strerror(errno)); } while (0) typedef struct { @@ -111,6 +111,7 @@ extern int fio_gzread(gzFile f, void *buf, unsigned size); extern int fio_gzwrite(gzFile f, void const* buf, unsigned size); extern int fio_gzeof(gzFile f); extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); +extern const char* fio_gzerror(gzFile file, int *errnum); #endif #endif From fa625dba02a269e23684f92ca3fbd51b9459e3fa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 Mar 2019 17:55:46 +0300 Subject: [PATCH 0404/2107] tests: minor fixes in page module --- tests/page.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/page.py b/tests/page.py index a0a554de3..b0eb79bf6 100644 --- a/tests/page.py +++ b/tests/page.py @@ -711,7 +711,7 @@ def test_page_backup_with_lost_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL segment "{0}" is absent\n'.format( file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -737,7 +737,7 @@ def test_page_backup_with_lost_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL segment "{0}" is absent\n'.format( file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -829,7 +829,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -855,7 +855,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -952,7 +952,7 @@ def test_page_backup_with_alien_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( @@ -979,7 +979,7 @@ def test_page_backup_with_alien_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( From e9c92c24e880bbb4755d0408fb74d86a566412bb Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Mar 2019 18:59:28 +0300 Subject: [PATCH 0405/2107] Make parsexlog messages using capital letter --- src/parsexlog.c | 24 ++++++++++++------------ tests/validate_test.py | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 630a15b83..d9d169d40 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -390,29 +390,29 @@ validate_wal(pgBackup *backup, const char *archivedir, /* There are all needed WAL records */ if (all_wal) - elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", + elog(INFO, "Backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", last_timestamp, last_rec.rec_xid, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); /* Some needed WAL records are absent */ else { - elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", + elog(WARNING, "Recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", last_timestamp, last_rec.rec_xid, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) time2iso(target_timestamp, lengthof(target_timestamp), target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) - elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + elog(ERROR, "Not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); else if (TransactionIdIsValid(target_xid)) - elog(ERROR, "not enough WAL records to xid " XID_FMT, + elog(ERROR, "Not enough WAL records to xid " XID_FMT, target_xid); else if (target_time != 0) - elog(ERROR, "not enough WAL records to time %s", + elog(ERROR, "Not enough WAL records to time %s", target_timestamp); else if (XRecOffIsValid(target_lsn)) - elog(ERROR, "not enough WAL records to lsn %X/%X", + elog(ERROR, "Not enough WAL records to lsn %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } } @@ -458,11 +458,11 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, errptr = startpoint ? startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", + elog(ERROR, "Could not read WAL record at %X/%X: %s", (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(ERROR, "could not read WAL record at %X/%X", + elog(ERROR, "Could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); } @@ -1042,13 +1042,13 @@ XLogThreadWorker(void *arg) else { if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", reader_data->thread_num, (uint32) (thread_arg->startpoint >> 32), (uint32) (thread_arg->startpoint), xlogreader->errormsg_buf); else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (thread_arg->startpoint >> 32), (uint32) (thread_arg->startpoint)); @@ -1125,12 +1125,12 @@ XLogThreadWorker(void *arg) thread_arg->startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", reader_data->thread_num, (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (errptr >> 32), (uint32) (errptr)); diff --git a/tests/validate_test.py b/tests/validate_test.py index 0d21811d7..13b87a0bd 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -96,7 +96,7 @@ def test_validate_wal_unreal_values(self): # Validate to real time self.assertIn( - "INFO: backup validation completed successfully", + "INFO: Backup validation completed successfully", self.validate_pb( backup_dir, 'node', options=["--time={0}".format(target_time), "-j", "4"]), @@ -134,7 +134,7 @@ def test_validate_wal_unreal_values(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to time' in e.message, + 'ERROR: Not enough WAL records to time' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -148,7 +148,7 @@ def test_validate_wal_unreal_values(self): self.switch_wal_segment(node) self.assertIn( - "INFO: backup validation completed successfully", + "INFO: Backup validation completed successfully", self.validate_pb( backup_dir, 'node', options=["--xid={0}".format(target_xid), "-j", "4"]), @@ -168,7 +168,7 @@ def test_validate_wal_unreal_values(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to xid' in e.message, + 'ERROR: Not enough WAL records to xid' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1580,9 +1580,9 @@ def test_validate_corrupt_wal_between_backups(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to xid' in e.message and - 'WARNING: recovery can be done up to time' in e.message and - "ERROR: not enough WAL records to xid {0}\n".format( + 'ERROR: Not enough WAL records to xid' in e.message and + 'WARNING: Recovery can be done up to time' in e.message and + "ERROR: Not enough WAL records to xid {0}\n".format( target_xid), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 8949abb2787c159430288f99b7372ea485aa8117 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 Mar 2019 18:25:03 +0300 Subject: [PATCH 0406/2107] tests: added retention.RetentionTest.test_retention_window_4 --- tests/retention_test.py | 63 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/retention_test.py b/tests/retention_test.py index 34a728d23..191438814 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -159,10 +159,73 @@ def test_retention_window_3(self): # Purge backups self.delete_expired( backup_dir, 'node', options=['--retention-window=1']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) + # count wal files in ARCHIVE + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_retention_window_4(self): + """purge all backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + # Take FULL BACKUPs + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + backup_id_3 = self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_pb(backup_dir, 'node', backup_id_2) + self.delete_pb(backup_dir, 'node', backup_id_3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # count wal files in ARCHIVE + wals_dir = os.path.join(backup_dir, 'wal', 'node') + n_wals = len(os.listdir(wals_dir)) + + self.assertTrue(n_wals > 0) + + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + # count again + n_wals = len(os.listdir(wals_dir)) + self.assertTrue(n_wals == 0) + # Clean after yourself self.del_test_dir(module_name, fname) From d6808f7f21a72864c5dab34848a1730e5c94d84f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 15 Mar 2019 18:48:54 +0300 Subject: [PATCH 0407/2107] Fix warnings in IO_CHECK macro --- src/utils/file.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.h b/src/utils/file.h index 8ef40fdcb..d8727fe72 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -49,7 +49,7 @@ typedef enum #define PAGE_CHECKSUM_MISMATCH (-256) #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) if (remote_agent) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc >= 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } else elog(ERROR, "Communication error: %s", _rc >= 0 ? "end of data" : strerror(errno)); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { if (remote_agent) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc >= 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } else elog(ERROR, "Communication error: %s", _rc >= 0 ? "end of data" : strerror(errno)); } } while (0) typedef struct { From 4f4c6b762372373e0ecb4c5f843ef1e010998eed Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 16 Mar 2019 08:37:39 +0300 Subject: [PATCH 0408/2107] [refer #PGPRO-1745] Add --remote-user option --- src/configure.c | 9 +++++++-- src/help.c | 24 ++++++++++++++---------- src/utils/remote.c | 4 ++++ src/utils/remote.h | 1 + 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/configure.c b/src/configure.c index 43b1f3408..de245f401 100644 --- a/src/configure.c +++ b/src/configure.c @@ -195,12 +195,17 @@ ConfigOption instance_options[] = OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 223, "ssh-options", + 's', 223, "remote-user", + &instance_config.remote.user, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 224, "ssh-options", &instance_config.remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "ssh-config", + 's', 225, "ssh-config", &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, diff --git a/src/help.c b/src/help.c index da5aec05c..a5ee373e3 100644 --- a/src/help.c +++ b/src/help.c @@ -120,7 +120,7 @@ help_pg_probackup(void) printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -133,7 +133,7 @@ help_pg_probackup(void) printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); @@ -155,7 +155,7 @@ help_pg_probackup(void) printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); @@ -169,14 +169,14 @@ help_pg_probackup(void) printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--overwrite]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); if ((PROGRAM_URL || PROGRAM_EMAIL)) @@ -224,7 +224,7 @@ help_backup(void) printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--skip-block-validation]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -298,6 +298,7 @@ help_backup(void) printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } @@ -313,7 +314,7 @@ help_restore(void) printf(_(" [--restore-as-replica] [--no-validate]\n\n")); printf(_(" [--skip-block-validation]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -370,6 +371,7 @@ help_restore(void) printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } @@ -599,7 +601,7 @@ help_add_instance(void) printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -612,6 +614,7 @@ help_add_instance(void) printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } @@ -635,7 +638,7 @@ help_archive_push(void) printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--overwrite]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -659,7 +662,7 @@ help_archive_get(void) printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -675,5 +678,6 @@ help_archive_get(void) printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); } diff --git a/src/utils/remote.c b/src/utils/remote.c index e3ecffeb3..4228fd244 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -73,6 +73,10 @@ bool launch_agent(void) ssh_argv[ssh_argc++] = "-p"; ssh_argv[ssh_argc++] = instance_config.remote.port; } + if (instance_config.remote.user != NULL) { + ssh_argv[ssh_argc++] = "-l"; + ssh_argv[ssh_argc++] = instance_config.remote.user; + } if (instance_config.remote.ssh_config != NULL) { ssh_argv[ssh_argc++] = "-F"; ssh_argv[ssh_argc++] = instance_config.remote.ssh_config; diff --git a/src/utils/remote.h b/src/utils/remote.h index 6705a7938..a0783671c 100644 --- a/src/utils/remote.h +++ b/src/utils/remote.h @@ -16,6 +16,7 @@ typedef struct RemoteConfig char* host; char* port; char* path; + char* user; char *ssh_config; char *ssh_options; } RemoteConfig; From c94b1829647b89abb3e0bc6168735ed37b8f8b7f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 16 Mar 2019 14:57:21 +0300 Subject: [PATCH 0409/2107] [refer #PGPRO-1745] Fix reading compressed file --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 268f68342..66993265c 100644 --- a/src/data.c +++ b/src/data.c @@ -1702,9 +1702,9 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) for (;;) { - size_t read_len = 0; + int read_len = 0; read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len != sizeof(buf) && !fio_gzeof(gz_in)) + if (read_len <= 0 && !fio_gzeof(gz_in)) /* An error occurred while reading the file */ elog(ERROR, "Cannot compare WAL file \"%s\" with compressed \"%s\"", From 0c0eefc8829b928756337f2c948d605322194686 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Mar 2019 11:36:46 +0300 Subject: [PATCH 0410/2107] tests: fix compatibility tests for remote backup --- tests/helpers/ptrack_helpers.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b5c5bc266..c2eb2561c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -636,7 +636,8 @@ def init_pb(self, backup_dir, options=[], old_binary=False): shutil.rmtree(backup_dir, ignore_errors=True) - if self.remote: + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: options = options + [ '--remote-proto=ssh', '--remote-host=localhost'] @@ -657,7 +658,8 @@ def add_instance(self, backup_dir, instance, node, old_binary=False, options=[]) '-D', node.data_dir ] - if self.remote: + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: options = options + [ '--remote-proto=ssh', '--remote-host=localhost'] @@ -710,7 +712,9 @@ def backup_node( '-d', 'postgres', '--instance={0}'.format(instance) ] - if self.remote: + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: options = options + [ '--remote-proto=ssh', '--remote-host=localhost'] @@ -736,6 +740,7 @@ def restore_node( self, backup_dir, instance, node=False, data_dir=None, backup_id=None, old_binary=False, options=[] ): + if data_dir is None: data_dir = node.data_dir @@ -745,7 +750,9 @@ def restore_node( '-D', data_dir, '--instance={0}'.format(instance) ] - if self.remote: + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: options = options + [ '--remote-proto=ssh', '--remote-host=localhost'] @@ -943,8 +950,9 @@ def set_archiving( backup_dir.replace("\\","\\\\"), instance) - # if self.remote: - # archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' if self.archive_compress or compress: archive_command = archive_command + '--compress ' @@ -1062,6 +1070,8 @@ def switch_wal_segment(self, node): else: node.execute('select pg_switch_xlog()') + sleep(1) + def wait_until_replica_catch_with_master(self, master, replica): if self.version_to_num( From a43a0e3289bd2fb2086f7a409e57c296cfecda1b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 16 Mar 2019 15:32:39 +0300 Subject: [PATCH 0411/2107] [refer #PGPRO-1745] Fix reading zeroed page --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index fb89e9715..2a5805dbf 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1154,7 +1154,7 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) } } /* horizonLsn is not 0 for delta backup. As far as unsigned number are always greater or equal than zero, there is no sense to add more checks */ - if (page_lsn >= req->horizonLsn) + if (page_lsn >= req->horizonLsn || page_lsn == InvalidXLogRecPtr) { char write_buffer[BLCKSZ*2]; BackupPageHeader* bph = (BackupPageHeader*)write_buffer; From 3ace1e7392fe44ef04e7b91090055c0cb2189b89 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Mar 2019 15:38:13 +0300 Subject: [PATCH 0412/2107] help: update help about --compress option --- src/help.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/help.c b/src/help.c index b47ec56a6..270686b0f 100644 --- a/src/help.c +++ b/src/help.c @@ -253,9 +253,9 @@ help_backup(void) printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_("\n Compression options:\n")); - printf(_(" --compress compress data files\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: none)\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -524,8 +524,9 @@ help_set_config(void) printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_("\n Compression options:\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -594,9 +595,9 @@ help_archive_push(void) printf(_(" relative path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the WAL file to retrieve from the server\n")); - printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','none'\n")); + printf(_(" available options: 'zlib', 'none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); printf(_(" --overwrite overwrite archived WAL file\n")); From 04f00976e1ebacf20196f4fd92c71f440df13b46 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 17 Mar 2019 09:13:07 +0300 Subject: [PATCH 0413/2107] [refer #PGPRO-1745] Check for existing file in fio_gzopen --- src/data.c | 7 +++---- src/utils/file.c | 11 +++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/data.c b/src/data.c index 66993265c..8aeeca5dc 100644 --- a/src/data.c +++ b/src/data.c @@ -1702,13 +1702,12 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) for (;;) { - int read_len = 0; - read_len = fio_gzread(gz_in, buf, sizeof(buf)); + int read_len = fio_gzread(gz_in, buf, sizeof(buf)); if (read_len <= 0 && !fio_gzeof(gz_in)) /* An error occurred while reading the file */ elog(ERROR, - "Cannot compare WAL file \"%s\" with compressed \"%s\"", - path1, path2); + "Cannot compare WAL file \"%s\" with compressed \"%s\": %d", + path1, path2, read_len); COMP_FILE_CRC32(true, crc2, buf, read_len); if (fio_gzeof(gz_in) || read_len == 0) diff --git a/src/utils/file.c b/src/utils/file.c index 2a5805dbf..01efc5a43 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -768,7 +768,14 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) if (rc == Z_OK) { gz->compress = 1; - gz->fd = fio_open(path, O_WRONLY|O_CREAT|O_TRUNC, location); + if (fio_access(path, F_OK, location) == 0) + { + elog(LOG, "File %s exists", path); + free(gz); + errno = EEXIST; + return NULL; + } + gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); } } else @@ -780,7 +787,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) if (rc == Z_OK) { gz->compress = 0; - gz->fd = fio_open(path, O_RDONLY, location); + gz->fd = fio_open(path, O_RDONLY | PG_BINARY, location); } } if (rc != Z_OK) From 46da0d851f54171fb93953f7b25d01ca3641b470 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sun, 17 Mar 2019 19:12:13 +0300 Subject: [PATCH 0414/2107] Replace error with warning in fileEqualCRC --- src/data.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 8aeeca5dc..2f0ff174b 100644 --- a/src/data.c +++ b/src/data.c @@ -1704,11 +1704,13 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) { int read_len = fio_gzread(gz_in, buf, sizeof(buf)); if (read_len <= 0 && !fio_gzeof(gz_in)) + { /* An error occurred while reading the file */ - elog(ERROR, + elog(WARNING, "Cannot compare WAL file \"%s\" with compressed \"%s\": %d", path1, path2, read_len); - + return false; + } COMP_FILE_CRC32(true, crc2, buf, read_len); if (fio_gzeof(gz_in) || read_len == 0) break; From a23386e17a28099355241897a87df3dcad5cfb54 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 Mar 2019 12:04:18 +0300 Subject: [PATCH 0415/2107] tests: fix test_validate_nullified_heap_page_backup and test_delta_nullified_heap_page_backup --- tests/delta.py | 24 +++++++++++------------- tests/validate_test.py | 20 +++++++++++++++----- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/tests/delta.py b/tests/delta.py index d5c3a03f8..8a091e423 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1254,9 +1254,8 @@ def test_delta_nullified_heap_page_backup(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1292,22 +1291,21 @@ def test_delta_nullified_heap_page_backup(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") - with open(log_file_path) as f: - self.assertTrue("LOG: File: {0} blknum 1, empty page".format( - file) in f.read()) - self.assertFalse("Skipping blknum: 1 in file: {0}".format( - file) in f.read()) + if not self.remote: + log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") + with open(log_file_path) as f: + self.assertTrue("LOG: File: {0} blknum 1, empty page".format( + file) in f.read()) + self.assertFalse("Skipping blknum: 1 in file: {0}".format( + file) in f.read()) # Restore DELTA backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored - ) + backup_dir, 'node', node_restored) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) diff --git a/tests/validate_test.py b/tests/validate_test.py index cb6247a1c..8ad304599 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -51,14 +51,24 @@ def test_validate_nullified_heap_page_backup(self): self.backup_node( backup_dir, 'node', node, options=['--log-level-file=verbose']) - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) - with open(log_file_path) as f: - self.assertTrue( - '{0} blknum 1, empty page'.format(file_path) in f.read(), - 'Failed to detect nullified block') + if not self.remote: + log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") + with open(log_file_path) as f: + self.assertTrue( + '{0} blknum 1, empty page'.format(file_path) in f.read(), + 'Failed to detect nullified block') self.validate_pb(backup_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From fc1ae6fba08030f39290744d0313536f1edf9429 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Tue, 19 Mar 2019 19:19:50 +0300 Subject: [PATCH 0416/2107] Check if external directory contains tablespace --- src/backup.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++ src/pg_probackup.h | 1 + tests/external.py | 2 +- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 67d0aac1a..25b78d6dd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -115,6 +115,7 @@ static void *StreamLog(void *arg); static void get_remote_pgdata_filelist(parray *files); static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); static void remote_copy_file(PGconn *conn, pgFile* file); +static void check_external_for_tablespaces(parray *external_list); /* Ptrack functions */ static void pg_ptrack_clear(void); @@ -487,7 +488,10 @@ do_backup_instance(void) elog(LOG, "Database backup start"); if(current.external_dir_str) + { external_dirs = make_external_directory_list(current.external_dir_str); + check_external_for_tablespaces(external_dirs); + } /* Initialize size summary */ current.data_bytes = 0; @@ -2996,3 +3000,46 @@ pg_ptrack_get_block(backup_files_arg *arguments, return result; } + +static void +check_external_for_tablespaces(parray *external_list) +{ + PGconn *conn; + PGresult *res; + int i = 0; + int j = 0; + char *tablespace_path = NULL; + char *query = "SELECT pg_catalog.pg_tablespace_location(oid)\n" + "FROM pg_tablespace\n" + "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; + + if (current.from_replica && exclusive_backup) + conn = master_conn; + else + conn = backup_conn; + + res = pgut_execute(conn, query, 0, NULL); + + /* Check successfull execution of query */ + if (!res) + elog(ERROR, "Failed to get list of tablespaces"); + + for (i = 0; i < res->ntups; i++) + { + tablespace_path = PQgetvalue(res, i, 0); + Assert (strlen(tablespace_path) > 0); + for (j = 0; j < parray_num(external_list); j++) + { + char *external_path = parray_get(external_list, j); + if (path_is_prefix_of_path(external_path, tablespace_path)) + elog(ERROR, "External directory path (-E option) \"%s\" " + "contains tablespace \"%s\"", + external_path, tablespace_path); + if (path_is_prefix_of_path(tablespace_path, external_path)) + elog(WARNING, "External directory path (-E option) \"%s\" " + "is in tablespace directory \"%s\"", + tablespace_path, external_path); + } + } + PQclear(res); +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a93bd4797..1bb08e325 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -12,6 +12,7 @@ #include "postgres_fe.h" #include "libpq-fe.h" +#include "libpq-int.h" #include "access/xlog_internal.h" #include "utils/pg_crc.h" diff --git a/tests/external.py b/tests/external.py index dfc8fc937..1327402cc 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1218,7 +1218,7 @@ def test_external_dir_is_tablespace(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'Insert correct message' in e.message, + 'External directory path (-E option)' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From ff86eb5b453ebc210f6b2f9e1963aa8385d1aa30 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 20 Mar 2019 14:20:19 +0300 Subject: [PATCH 0417/2107] Use pgut_malloc() instead of malloc() --- .gitignore | 2 ++ src/utils/file.c | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index da388019c..4fc21d916 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ /src/streamutil.c /src/streamutil.h /src/xlogreader.c +/src/walmethods.c +/src/walmethods.h diff --git a/src/utils/file.c b/src/utils/file.c index 01efc5a43..607072b52 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -112,7 +112,7 @@ FILE* fio_open_stream(char const* path, fio_location location) if (hdr.size > 0) { Assert(fio_stdin_buffer == NULL); - fio_stdin_buffer = malloc(hdr.size); + fio_stdin_buffer = pgut_malloc(hdr.size); IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); f = fmemopen(fio_stdin_buffer, hdr.size, "r"); } @@ -752,7 +752,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) int rc; if (fio_is_remote(location)) { - fioGZFile* gz = (fioGZFile*)malloc(sizeof(fioGZFile)); + fioGZFile* gz = (fioGZFile*) pgut_malloc(sizeof(fioGZFile)); memset(&gz->strm, 0, sizeof(gz->strm)); gz->eof = 0; gz->errnum = Z_OK; @@ -1014,7 +1014,7 @@ static void fio_send_file(int out, char const* path) if (fd >= 0) { off_t size = lseek(fd, 0, SEEK_END); - buf = malloc(size); + buf = pgut_malloc(size); lseek(fd, 0, SEEK_SET); IO_CHECK(fio_read_all(fd, buf, size), size); hdr.size = size; @@ -1202,7 +1202,7 @@ void fio_communicate(int in, int out) DIR* dir[FIO_FDMAX]; struct dirent* entry; size_t buf_size = 128*1024; - char* buf = (char*)malloc(buf_size); + char* buf = (char*)pgut_malloc(buf_size); fio_header hdr; struct stat st; int rc; From d064a64d13bdb2c1d0e4cf2d327566cdfaeef0f6 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 21 Mar 2019 19:11:00 +0300 Subject: [PATCH 0418/2107] List data directories at backup host using FIO_BACKUP_HOST --- src/dir.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/dir.c b/src/dir.c index e091912d9..f81a47ff4 100644 --- a/src/dir.c +++ b/src/dir.c @@ -126,7 +126,7 @@ static void dir_list_file_internal(parray *files, const char *root, bool omit_symlink, parray *black_list, fio_location location); static void list_data_directories(parray *files, const char *path, bool is_root, - bool exclude); + bool exclude, fio_location location); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; @@ -773,7 +773,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, */ static void list_data_directories(parray *files, const char *path, bool is_root, - bool exclude) + bool exclude, fio_location location) { DIR *dir; struct dirent *dent; @@ -781,7 +781,7 @@ list_data_directories(parray *files, const char *path, bool is_root, bool has_child_dirs = false; /* open directory and list contents */ - dir = fio_opendir(path, FIO_DB_HOST); + dir = fio_opendir(path, location); if (dir == NULL) elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); @@ -799,7 +799,7 @@ list_data_directories(parray *files, const char *path, bool is_root, join_path_components(child, path, dent->d_name); - if (fio_stat(child, &st, false, FIO_DB_HOST) == -1) + if (fio_stat(child, &st, false, location) == -1) elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) @@ -823,7 +823,7 @@ list_data_directories(parray *files, const char *path, bool is_root, continue; has_child_dirs = true; - list_data_directories(files, child, false, exclude); + list_data_directories(files, child, false, exclude, location); } /* List only full and last directories */ @@ -831,7 +831,7 @@ list_data_directories(parray *files, const char *path, bool is_root, { pgFile *dir; - dir = pgFileNew(path, false, FIO_LOCAL_HOST); + dir = pgFileNew(path, false, location); parray_append(files, dir); } @@ -982,7 +982,8 @@ create_data_directories(const char *data_dir, const char *backup_dir, } join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); - list_data_directories(dirs, backup_database_dir, true, false); + list_data_directories(dirs, backup_database_dir, true, false, + FIO_BACKUP_HOST); elog(LOG, "restore directories and symlinks..."); From adee1efc71612d7442ccdcb168b9b58faa021546 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 21 Mar 2019 22:55:38 +0300 Subject: [PATCH 0419/2107] Fix remote restore --- src/backup.c | 5 +++-- src/merge.c | 2 +- src/pg_probackup.h | 4 ++-- src/restore.c | 8 ++++++-- src/util.c | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index e614c21c5..220b04543 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2317,8 +2317,9 @@ backup_files(void *arg) } } else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(arguments->from_root, arguments->to_root, - file, FIO_BACKUP_HOST); + copy_pgcontrol_file(arguments->from_root, FIO_DB_HOST, + arguments->to_root, FIO_BACKUP_HOST, + file); else { bool skip = false; diff --git a/src/merge.c b/src/merge.c index 2ee1aa678..3cf7f8669 100644 --- a/src/merge.c +++ b/src/merge.c @@ -559,7 +559,7 @@ merge_files(void *arg) } } else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(argument->from_root, argument->to_root, file, FIO_LOCAL_HOST); + copy_pgcontrol_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); else copy_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f82dba08d..43407fb48 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -612,8 +612,8 @@ extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); extern uint32 get_xlog_seg_size(char *pgdata_path); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); -extern void copy_pgcontrol_file(const char *from_root, const char *to_root, - pgFile *file, fio_location location); +extern void copy_pgcontrol_file(const char *from_root, fio_location location, const char *to_root, fio_location to_location, + pgFile *file); extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); diff --git a/src/restore.c b/src/restore.c index f75962d2d..20b9e8ca9 100644 --- a/src/restore.c +++ b/src/restore.c @@ -674,9 +674,13 @@ restore_files(void *arg) parse_program_version(arguments->backup->program_version)); } else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(from_root, instance_config.pgdata, file, FIO_DB_HOST); + copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, + instance_config.pgdata, FIO_DB_HOST, + file); else - copy_file(from_root, FIO_BACKUP_HOST, instance_config.pgdata, FIO_DB_HOST, file); + copy_file(from_root, FIO_BACKUP_HOST, + instance_config.pgdata, FIO_DB_HOST, + file); /* print size of restored file */ if (file->write_size != BYTES_INVALID) diff --git a/src/util.c b/src/util.c index 6ded3a54e..661143c23 100644 --- a/src/util.c +++ b/src/util.c @@ -360,14 +360,14 @@ set_min_recovery_point(pgFile *file, const char *backup_path, * Copy pg_control file to backup. We do not apply compression to this file. */ void -copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file, fio_location location) +copy_pgcontrol_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file) { ControlFileData ControlFile; char *buffer; size_t size; char to_path[MAXPGPATH]; - buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false, from_location); digestControlFile(&ControlFile, buffer, size); @@ -376,7 +376,7 @@ copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file, fi file->write_size = size; join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); - writeControlFile(&ControlFile, to_path, location); + writeControlFile(&ControlFile, to_path, to_location); pg_free(buffer); } From aaf19f3ea329d23c7f202b4d0e5647b2be17300a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Mar 2019 22:14:53 +0300 Subject: [PATCH 0420/2107] tests: rename module retention_test to retention --- tests/{retention_test.py => retention.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{retention_test.py => retention.py} (100%) diff --git a/tests/retention_test.py b/tests/retention.py similarity index 100% rename from tests/retention_test.py rename to tests/retention.py From d1722ea172a66df9a56d734db3bf63e86d8e0526 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 Mar 2019 02:16:42 +0300 Subject: [PATCH 0421/2107] tests: added test_retention_interleaved_incremental_chains --- tests/helpers/ptrack_helpers.py | 23 ++++++ tests/retention.py | 119 +++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bcce8448a..f61e8d18d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -938,6 +938,29 @@ def set_replica( ) master.reload() + def change_backup_status(self, backup_dir, instance, backup_id, status): + + control_file_path = os.path.join( + backup_dir, 'backups', instance, backup_id, 'backup.control') + + with open(control_file_path, 'r') as f: + actual_control = f.read() + + new_control_file = '' + for line in actual_control.splitlines(): + if line.startswith('status'): + line = 'status = {0}'.format(status) + new_control_file += line + new_control_file += '\n' + + with open(control_file_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + with open(control_file_path, 'r') as f: + actual_control = f.read() + def wrong_wal_clean(self, node, wal_size): wals_dir = os.path.join(self.backup_dir(node), 'wal') wals = [ diff --git a/tests/retention.py b/tests/retention.py index 191438814..ac679ba22 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -67,7 +67,7 @@ def test_retention_redundancy_1(self): # Clean after yourself self.del_test_dir(module_name, fname) -# @unittest.skip("123") + #@unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -121,7 +121,7 @@ def test_retention_window_2(self): # Clean after yourself self.del_test_dir(module_name, fname) -# @unittest.skip("123") + #@unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -229,3 +229,118 @@ def test_retention_window_4(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_retention_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + # Take FULL BACKUPs + + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Change PAGEa2 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # Change PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From d48d4e8abe5427449b843fbf50da6eb8d1c61911 Mon Sep 17 00:00:00 2001 From: Sergey Cherkashin <4erkashin@list.ru> Date: Mon, 25 Mar 2019 16:37:53 +0300 Subject: [PATCH 0422/2107] Fix connection value in check_external_for_tablespaces() --- src/backup.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 25b78d6dd..2014a6efe 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3013,11 +3013,7 @@ check_external_for_tablespaces(parray *external_list) "FROM pg_tablespace\n" "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; - if (current.from_replica && exclusive_backup) - conn = master_conn; - else - conn = backup_conn; - + conn = backup_conn; res = pgut_execute(conn, query, 0, NULL); /* Check successfull execution of query */ From 5509a4bbe1dd5e059f137fe5a08dc079df2e7fa8 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 25 Mar 2019 16:30:38 +0300 Subject: [PATCH 0423/2107] PGPRO-421: Use strdup() call --- src/catalog.c | 10 +++------- src/dir.c | 13 +++++-------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 8734ad7e3..9d8b8adc4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -469,11 +469,8 @@ pgBackupCreateDir(pgBackup *backup) int i; char path[MAXPGPATH]; parray *subdirs = parray_new(); - char *temp; - temp = palloc(strlen(DATABASE_DIR) + 1); - sprintf(temp, "%s", DATABASE_DIR); - parray_append(subdirs, temp); + parray_append(subdirs, pg_strdup(DATABASE_DIR)); /* Add external dirs containers */ if (backup->external_dir_str) @@ -483,11 +480,10 @@ pgBackupCreateDir(pgBackup *backup) external_list = make_external_directory_list(backup->external_dir_str); for (int i = 0; i < parray_num(external_list); i++) { - /* 20 chars is enough to hold the externaldir number in string. */ - temp = palloc0(strlen(EXTERNAL_DIR) + 20); + char temp[MAXPGPATH]; /* Numeration of externaldirs starts with 1 */ makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); - parray_append(subdirs, temp); + parray_append(subdirs, pg_strdup(temp)); } free_dir_list(external_list); } diff --git a/src/dir.c b/src/dir.c index c503f5227..dcf1476d5 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1302,7 +1302,7 @@ check_external_dir_mapping(pgBackup *backup) "backup: \"%s\"", external_remap_list.head->old_dir); return; } - + external_dirs_to_restore = make_external_directory_list(backup->external_dir_str); for (cell = external_remap_list.head; cell; cell = cell->next) { @@ -1688,18 +1688,15 @@ make_external_directory_list(const char *colon_separated_dirs) { char *p; parray *list = parray_new(); - char *tmp = palloc0(strlen(colon_separated_dirs) + 1); + char *tmp = pg_strdup(colon_separated_dirs); - strcpy(tmp, colon_separated_dirs); p = strtok(tmp,":"); while(p!=NULL) { - char * dir = (char *)palloc0(strlen(p) + 1); - strcpy(dir,p); - if (is_absolute_path(dir)) - parray_append(list, dir); + if (is_absolute_path(p)) + parray_append(list, pg_strdup(p)); else - elog(ERROR, "Extra directory \"%s\" is not an absolute path", dir); + elog(ERROR, "External directory \"%s\" is not an absolute path", p); p=strtok(NULL,":"); } pfree(tmp); From a8be5ff27764a321c87eb55191e5a0cb32e81a5e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Mar 2019 02:29:58 +0300 Subject: [PATCH 0424/2107] tests: fix gdbobject method continue_execution_until_break() --- tests/__init__.py | 12 +----------- tests/delta.py | 4 +--- tests/helpers/ptrack_helpers.py | 11 ++++------- tests/locking.py | 12 ++++-------- tests/merge.py | 12 +++--------- tests/ptrack.py | 4 +--- 6 files changed, 14 insertions(+), 41 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d7587261e..3160d223e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,7 +2,7 @@ from . import init_test, merge, option_test, show_test, compatibility, \ backup_test, delete_test, delta, restore_test, validate_test, \ - retention_test, pgpro560, pgpro589, false_positive, replica, \ + retention_test, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ locking, remote, external @@ -48,18 +48,8 @@ def load_tests(loader, tests, pattern): # ToDo: # archive: # discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work -# replica: -# backup should exit with correct error message if some master* option is missing -# --master* options shoukd not work when backuping master # logging: # https://jira.postgrespro.ru/browse/PGPRO-584 # https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md # archive: # immediate recovery and full recovery -# backward compatibility: -# previous version catalog must be readable by newer version -# incremental chain from previous version can be continued -# backups from previous version can be restored -# 10vanilla_1.3ptrack + -# 10vanilla+ -# 9.6vanilla_1.3ptrack + diff --git a/tests/delta.py b/tests/delta.py index 6ad9617f1..d5c3a03f8 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -562,9 +562,7 @@ def test_delta_vacuum_full(self): acurs.execute("VACUUM FULL t_heap") if gdb.stopped_in_breakpoint(): - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 207c825d6..bcce8448a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1412,17 +1412,14 @@ def continue_execution_until_break(self, ignore_count=0): else: result = self._execute('continue', False) - running = False for line in result: - if line.startswith('*running'): - running = True if line.startswith('*stopped,reason="breakpoint-hit"'): - return 'breakpoint-hit' + return if line.startswith('*stopped,reason="exited-normally"'): - return 'exited-normally' + break - if running: - return 'running' + raise GdbException( + 'Failed to continue execution until break.\n') def stopped_in_breakpoint(self): output = [] diff --git a/tests/locking.py b/tests/locking.py index 44fff4de8..b4b271f46 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -37,8 +37,7 @@ def test_locking_running_validate_1(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) @@ -93,8 +92,7 @@ def test_locking_running_validate_2(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() @@ -164,8 +162,7 @@ def test_locking_running_validate_2_specific_id(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() @@ -263,8 +260,7 @@ def test_locking_running_3(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - self.AssertTrue(False, 'Failed to hit breakpoint') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() diff --git a/tests/merge.py b/tests/merge.py index 23b3dceda..6e987a2a4 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1071,9 +1071,7 @@ def test_continue_failed_merge(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1154,9 +1152,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - if gdb.continue_execution_until_break(2) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -1252,9 +1248,7 @@ def test_continue_failed_merge_2(self): gdb.set_breakpoint('pgFileDelete') gdb.run_until_break() - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') diff --git a/tests/ptrack.py b/tests/ptrack.py index a3d11617b..b79528171 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -241,9 +241,7 @@ def test_ptrack_vacuum_full(self): acurs.execute("VACUUM FULL t_heap") if gdb.stopped_in_breakpoint(): - if gdb.continue_execution_until_break(20) != 'breakpoint-hit': - print('Failed to hit breakpoint') - exit(1) + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, backup_type='ptrack') From 6f0d7beb9b0fffb261ebb3ca0bed0a8991576a82 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 3 Mar 2019 02:35:06 +0300 Subject: [PATCH 0425/2107] tests: add test_merge_different_wal_modes --- tests/merge.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 6e987a2a4..78610e75c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1369,6 +1369,48 @@ def test_merge_different_wal_modes(self): self.del_test_dir(module_name, fname) + def test_merge_different_wal_modes(self): + """ + Check that backups with different wal modes can be merged + correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + # DELTA stream backup + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 6db82d0bd96d1e8fa42f6f0376d9c530102a05ae Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 16:40:11 +0300 Subject: [PATCH 0426/2107] Inherit WAL mode from the PAGE backup --- src/merge.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/merge.c b/src/merge.c index 33d473e51..e5215cdcc 100644 --- a/src/merge.c +++ b/src/merge.c @@ -333,10 +333,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->end_time = time(NULL); /* - * If one of the backups isn't "stream" backup then the target backup become - * non-stream backup too. + * Target backup must inherit wal mode too. */ - to_backup->stream = to_backup->stream && from_backup->stream; + to_backup->stream = from_backup->stream; /* Compute summary of size of regular files in the backup */ to_backup->data_bytes = 0; for (i = 0; i < parray_num(files); i++) From 0253845a35daab39cbc1c0a5b611d3eb632aa2b5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 16:47:21 +0300 Subject: [PATCH 0427/2107] tests: update help test --- src/help.c | 2 +- tests/expected/option_help.out | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index 1d394b52b..2bedbfef2 100644 --- a/src/help.c +++ b/src/help.c @@ -97,7 +97,7 @@ help_pg_probackup(void) printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [-C] [--stream [-S slot-name]] [--temp-slot]\n")); printf(_(" [--backup-pg-log] [-j num-threads]\n")); printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); printf(_(" [--log-level-console=log-level-console]\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index ecc59a893..abb5ce713 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -29,9 +29,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name - [-C] [--stream [-S slot-name]] [--backup-pg-log] - [-j num-threads] [--archive-timeout=archive-timeout] - [--progress] + [-C] [--stream [-S slot-name]] [--temp-slot] + [--backup-pg-log] [-j num-threads] + [--archive-timeout=archive-timeout] [--progress] [--log-level-console=log-level-console] [--log-level-file=log-level-file] [--log-filename=log-filename] From 7acccb4220d745c6c96c10c7fcec10de0db3e8c3 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 4 Mar 2019 17:08:50 +0300 Subject: [PATCH 0428/2107] PGPRO-2413. Use PG_LIBS_INTERNAL instead of PG_LIBS to link with libpq library within build tree --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 482fa2e96..8372e3e1a 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endif PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) -PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS} +PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} all: checksrcdir $(INCLUDES); From 50f11a1fffab9ca0bac17bcbd28d0218121a958d Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Mar 2019 14:21:20 +0300 Subject: [PATCH 0429/2107] PGPRO-2079: Use .partial file to write backup configuration files backup.control and backup_content.control --- src/catalog.c | 54 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 9d8b8adc4..f658b9cac 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -592,22 +592,38 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) void write_backup(pgBackup *backup) { - FILE *fp = NULL; - char conf_path[MAXPGPATH]; + FILE *fp = NULL; + char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; + int errno_temp; + + pgBackupGetPath(backup, path, lengthof(path), BACKUP_CONTROL_FILE); + snprintf(path_temp, sizeof(path_temp), "%s.partial", path); - pgBackupGetPath(backup, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); - fp = fopen(conf_path, "wt"); + fp = fopen(path_temp, "wt"); if (fp == NULL) - elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path, - strerror(errno)); + elog(ERROR, "Cannot open configuration file \"%s\": %s", + path_temp, strerror(errno)); pgBackupWriteControl(fp, backup); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) + { + errno_temp = errno; + unlink(path_temp); elog(ERROR, "Cannot write configuration file \"%s\": %s", - conf_path, strerror(errno)); + path_temp, strerror(errno_temp)); + } + + if (rename(path_temp, path) < 0) + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno_temp)); + } } /* @@ -619,20 +635,36 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, { FILE *fp; char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; + int errno_temp; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + snprintf(path_temp, sizeof(path_temp), "%s.partial", path); - fp = fopen(path, "wt"); + fp = fopen(path_temp, "wt"); if (fp == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", path, - strerror(errno)); + elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, + strerror(errno)); print_file_list(fp, files, root, external_prefix, external_list); if (fflush(fp) != 0 || fsync(fileno(fp)) != 0 || fclose(fp)) - elog(ERROR, "Cannot write file list \"%s\": %s", path, strerror(errno)); + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot write file list \"%s\": %s", + path_temp, strerror(errno)); + } + + if (rename(path_temp, path) < 0) + { + errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno_temp)); + } } /* From 385246c49c7fd15cd95ca5b716f536111dbce056 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 Mar 2019 16:17:04 +0300 Subject: [PATCH 0430/2107] tests: added delete_test.DeleteTest.test_delete_backup_with_empty_control_file() --- tests/delete_test.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/delete_test.py b/tests/delete_test.py index f49c01bf2..c7f52cfc4 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -209,3 +209,41 @@ def test_delete_orphaned_wal_segments(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_backup_with_empty_control_file(self): + """ + take backup, truncate its control file, + try to delete it via 'delete' command + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # page backup mode + self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + # page backup mode + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + + with open( + os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.control'), + 'wt') as f: + f.flush() + f.close() + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 596f82faa2bde71c621f8b455e3c355cfca008f5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 12:43:04 +0300 Subject: [PATCH 0431/2107] tests: added tests.merge.MergeTest.test_crash_after_opening_backup_control --- tests/merge.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 78610e75c..96b571e9f 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1411,6 +1411,56 @@ def test_merge_different_wal_modes(self): self.del_test_dir(module_name, fname) + def test_crash_after_opening_backup_control(self): + """ + check that crashing after opening backup.control + for writing will not result in losing backup metadata + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('write_backup') + gdb.continue_execution_until_break() + gdb.set_breakpoint('pgBackupWriteControl') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From 58de0dbd3536709b450cae399f9d909a1dffa0f2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 13:36:21 +0300 Subject: [PATCH 0432/2107] tests: added tests.merge.MergeTest.test_crash_after_opening_backup_control_2 --- tests/merge.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 96b571e9f..e92272450 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1411,7 +1411,7 @@ def test_merge_different_wal_modes(self): self.del_test_dir(module_name, fname) - def test_crash_after_opening_backup_control(self): + def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control for writing will not result in losing backup metadata @@ -1461,6 +1461,89 @@ def test_crash_after_opening_backup_control(self): self.del_test_dir(module_name, fname) + def test_crash_after_opening_backup_control_2(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=3) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + vm_path = path + '_vm' + + print(vm_path) + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('print_file_list') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + os.remove( + os.path.join( + backup_dir, 'backups', 'node', + full_id, 'database', vm_path)) + + # Continue failed merge + self.merge_backup(backup_dir, "node", backup_id) + + node.cleanup() + + # restore merge backup + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From c02892217f593e5fe21e510e5b317d0e7db49eca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 Mar 2019 14:25:50 +0300 Subject: [PATCH 0433/2107] tests: minor fixes for merge module --- tests/merge.py | 114 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index e92272450..bccb8d140 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1421,7 +1421,8 @@ def test_crash_after_opening_backup_control_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1486,16 +1487,18 @@ def test_crash_after_opening_backup_control_2(self): backup_dir, 'node', node, options=['--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '20', '-c', '2']) pgbench.wait() path = node.safe_psql( 'postgres', "select pg_relation_filepath('pgbench_accounts')").rstrip() - vm_path = path + '_vm' + fsm_path = path + '_fsm' - print(vm_path) + node.safe_psql( + 'postgres', + 'vacuum pgbench_accounts') # DELTA backup backup_id = self.backup_node( @@ -1525,10 +1528,14 @@ def test_crash_after_opening_backup_control_2(self): 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) # In to_backup drop file that comes from from_backup - os.remove( - os.path.join( - backup_dir, 'backups', 'node', - full_id, 'database', vm_path)) + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', fsm_path) + + print(file_to_remove) + + os.remove(file_to_remove) # Continue failed merge self.merge_backup(backup_dir, "node", backup_id) @@ -1544,6 +1551,97 @@ def test_crash_after_opening_backup_control_2(self): self.del_test_dir(module_name, fname) + def test_losing_file_after_failed_merge(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=3) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench.wait() + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + vm_path = path + '_vm' + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('print_file_list') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', vm_path) + + print(file_to_remove) + + os.remove(file_to_remove) + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Merging of backup {0} failed".format( + backup_id) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node')[0]['status']) + + # self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From b888f6bdcd07e7b8b87f789de293d24be53bbc9c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 Mar 2019 14:53:32 +0300 Subject: [PATCH 0434/2107] tests: added tests.retention_test.RetentionTest.test_retention_window_3 --- tests/retention_test.py | 44 ++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/retention_test.py b/tests/retention_test.py index 34c026587..34a728d23 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -122,38 +122,31 @@ def test_retention_window_2(self): self.del_test_dir(module_name, fname) # @unittest.skip("123") - def test_retention_wal(self): - """purge backups using window-based retention policy""" + def test_retention_window_3(self): + """purge all backups using window-based retention policy""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # Take second FULL BACKUP + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Take third FULL BACKUP + backup_id_3 = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node) backups = os.path.join(backup_dir, 'backups', 'node') - days_delta = 5 for backup in os.listdir(backups): if backup == 'pg_probackup.conf': continue @@ -161,18 +154,15 @@ def test_retention_wal(self): os.path.join( backups, backup, "backup.control"), "a") as conf: conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=days_delta))) - days_delta -= 1 - - # Make backup to be keeped - self.backup_node(backup_dir, 'node', node, backup_type="page") - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + datetime.now() - timedelta(days=3))) # Purge backups self.delete_expired( - backup_dir, 'node', options=['--retention-window=2']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + backup_dir, 'node', options=['--retention-window=1']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) # Clean after yourself self.del_test_dir(module_name, fname) From 2ae1cf00076e0135d4d7ca525d1cbb129d47710a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 12 Mar 2019 13:57:40 +0300 Subject: [PATCH 0435/2107] PGPRO-2521: Allow to delete backups without or emtpy .control file --- src/catalog.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index f658b9cac..86b51c872 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -74,6 +74,14 @@ write_backup_status(pgBackup *backup, BackupStatus status) pgBackup *tmp; tmp = read_backup(backup->start_time); + if (!tmp) + { + /* + * Silently exit the function, since read_backup already logged the + * warning message. + */ + return; + } backup->status = status; tmp->status = backup->status; @@ -301,11 +309,10 @@ IsDir(const char *dirpath, const char *entry) parray * catalog_get_backup_list(time_t requested_backup_id) { - DIR *data_dir = NULL; - struct dirent *data_ent = NULL; - parray *backups = NULL; - pgBackup *backup = NULL; - int i; + DIR *data_dir = NULL; + struct dirent *data_ent = NULL; + parray *backups = NULL; + int i; /* open backup instance backups directory */ data_dir = opendir(backup_instance_path); @@ -320,8 +327,9 @@ catalog_get_backup_list(time_t requested_backup_id) backups = parray_new(); for (; (data_ent = readdir(data_dir)) != NULL; errno = 0) { - char backup_conf_path[MAXPGPATH]; - char data_path[MAXPGPATH]; + char backup_conf_path[MAXPGPATH]; + char data_path[MAXPGPATH]; + pgBackup *backup = NULL; /* skip not-directory entries and hidden entries */ if (!IsDir(backup_instance_path, data_ent->d_name) @@ -355,7 +363,6 @@ catalog_get_backup_list(time_t requested_backup_id) continue; } parray_append(backups, backup); - backup = NULL; if (errno && errno != ENOENT) { @@ -405,8 +412,6 @@ catalog_get_backup_list(time_t requested_backup_id) err_proc: if (data_dir) closedir(data_dir); - if (backup) - pgBackupFree(backup); if (backups) parray_walk(backups, pgBackupFree); parray_free(backups); From 012904b1f0cc93a441bb2ac801158e397c160a23 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 12 Mar 2019 14:58:00 +0300 Subject: [PATCH 0436/2107] PGPRO-2523: Allow to delete single FULL backup --- src/delete.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index f43d62c02..287e73776 100644 --- a/src/delete.c +++ b/src/delete.c @@ -126,7 +126,6 @@ do_retention_purge(void) size_t i; XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; - bool keep_next_backup = true; /* Do not delete first full backup */ bool backup_deleted = false; /* At least one backup was deleted */ if (delete_expired) @@ -158,6 +157,7 @@ do_retention_purge(void) (instance_config.retention_redundancy > 0 || instance_config.retention_window > 0)) { + bool keep_next_backup = false; /* Do not delete first full backup */ time_t days_threshold; uint32 backup_num = 0; @@ -216,6 +216,7 @@ do_retention_purge(void) /* Delete backup and update status to DELETED */ delete_backup_files(backup); backup_deleted = true; + keep_next_backup = false; /* reset it */ } } From db365f1d85796abfcb86336b39a77351ffd2f6e1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 17:46:30 +0300 Subject: [PATCH 0437/2107] tests: minor fix for test_delete_backup_with_empty_control_file --- tests/delete_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/delete_test.py b/tests/delete_test.py index c7f52cfc4..6c817f767 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -243,6 +243,9 @@ def test_delete_backup_with_empty_control_file(self): f.flush() f.close() + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 3) + self.delete_pb(backup_dir, 'node', backup_id=backup_id) # Clean after yourself From 001c0c267024ab3716e0e49c6c3fcce6d2563c2f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 Mar 2019 21:44:40 +0300 Subject: [PATCH 0438/2107] tests: added test_continue_failed_merge_3(), author Ilya Skvortsov --- tests/merge.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index bccb8d140..2c28a219a 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1274,6 +1274,102 @@ def test_continue_failed_merge_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_continue_failed_merge_3(self): + """ + Check that failed MERGE can`t be continued after target backup deleting + Create FULL and 2 PAGE backups + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create test data + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100000) i" + ) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # CREATE FEW PAGE BACKUP + i = 0 + + while i < 2: + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200000) i" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page' + ) + i = i + 1 + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id_merge = self.show_pb(backup_dir, "node")[2]["id"] + backup_id_delete = self.show_pb(backup_dir, "node")[1]["id"] + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # DELETE PAGE1 + shutil.rmtree( + os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id_merge) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Parent full backup for the given backup {0} was not found".format( + backup_id_merge) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + def test_merge_different_compression_algo(self): """ Check that backups with different compression algorihtms can be merged From 62a2850de0e9f8cd9ab89b6fb5b4b77226c847c7 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 4 Mar 2019 19:02:29 +0300 Subject: [PATCH 0439/2107] PGPRO-1973: Add support pthreads for VALIDATE command --- src/backup.c | 11 +- src/data.c | 4 +- src/help.c | 3 +- src/merge.c | 3 +- src/parsexlog.c | 1173 ++++++++++++++++++-------------- src/pg_probackup.h | 13 +- src/restore.c | 3 +- src/utils/logger.c | 7 +- src/utils/pgut.c | 2 +- src/utils/thread.c | 4 + src/utils/thread.h | 2 +- src/validate.c | 3 +- tests/expected/option_help.out | 2 +- 13 files changed, 710 insertions(+), 520 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2014a6efe..bd5ee5908 100644 --- a/src/backup.c +++ b/src/backup.c @@ -406,7 +406,7 @@ remote_backup_files(void *arg) instance_config.pguser); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during backup"); query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); @@ -631,6 +631,7 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; + thread_interrupted = false; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } @@ -698,8 +699,7 @@ do_backup_instance(void) * where this backup has started. */ extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, - prev_backup->start_lsn, current.start_lsn, - backup_files_list); + prev_backup->start_lsn, current.start_lsn); } else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -778,6 +778,7 @@ do_backup_instance(void) } /* Run threads */ + thread_interrupted = false; elog(INFO, "Start transfering data files"); for (i = 0; i < num_threads; i++) { @@ -2288,7 +2289,7 @@ backup_files(void *arg) elog(VERBOSE, "Copying file: \"%s\" ", file->path); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during backup"); if (progress) @@ -2757,7 +2758,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) static XLogRecPtr prevpos = InvalidXLogRecPtr; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during backup"); /* we assume that we get called once at the end of each segment */ diff --git a/src/data.c b/src/data.c index fc2d5c38e..36e0de28e 100644 --- a/src/data.c +++ b/src/data.c @@ -22,6 +22,8 @@ #include #endif +#include "utils/thread.h" + /* Union to ease operations on relation pages */ typedef union DataPage { @@ -318,7 +320,7 @@ prepare_page(backup_files_arg *arguments, BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during backup"); /* diff --git a/src/help.c b/src/help.c index 2bedbfef2..84b56678f 100644 --- a/src/help.c +++ b/src/help.c @@ -134,7 +134,7 @@ help_pg_probackup(void) printf(_(" [--skip-external-dirs]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-i backup-id] [--progress]\n")); + printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--timeline=timeline]\n")); @@ -361,6 +361,7 @@ help_validate(void) printf(_(" -i, --backup-id=backup-id backup to validate\n")); printf(_(" --progress show progress\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); diff --git a/src/merge.c b/src/merge.c index e5215cdcc..5726e36c1 100644 --- a/src/merge.c +++ b/src/merge.c @@ -284,6 +284,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pg_atomic_init_flag(&file->lock); } + thread_interrupted = false; for (i = 0; i < num_threads; i++) { merge_files_arg *arg = &(threads_args[i]); @@ -456,7 +457,7 @@ merge_files(void *arg) continue; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during merging backups"); /* Directories were created before */ diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a6258841..198f9c5aa 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -83,42 +83,61 @@ typedef struct xl_xact_abort /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ } xl_xact_abort; -static void extractPageInfo(XLogReaderState *record); -static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); +/* + * XLogRecTarget allows to track the last recovery targets. Currently used only + * within validate_wal(). + */ +typedef struct XLogRecTarget +{ + TimestampTz rec_time; + TransactionId rec_xid; + XLogRecPtr rec_lsn; +} XLogRecTarget; -typedef struct XLogPageReadPrivate +typedef struct XLogReaderData { int thread_num; - const char *archivedir; TimeLineID tli; - uint32 xlog_seg_size; + + XLogRecTarget cur_rec; + XLogSegNo xlogsegno; + bool xlogexists; char page_buf[XLOG_BLCKSZ]; uint32 prev_page_off; - bool manual_switch; bool need_switch; int xlogfile; - XLogSegNo xlogsegno; char xlogpath[MAXPGPATH]; - bool xlogexists; #ifdef HAVE_LIBZ gzFile gz_xlogfile; char gz_xlogpath[MAXPGPATH]; #endif -} XLogPageReadPrivate; +} XLogReaderData; + +/* Function to process a WAL record */ +typedef void (*xlog_record_function) (XLogReaderState *record, + XLogReaderData *reader_data, + bool *stop_reading); /* An argument for a thread function */ typedef struct { - XLogPageReadPrivate private_data; + XLogReaderData reader_data; + + xlog_record_function process_record; XLogRecPtr startpoint; XLogRecPtr endpoint; XLogSegNo endSegNo; + /* + * The thread got the recovery target. + */ + bool got_target; + /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. @@ -130,14 +149,41 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); -static XLogReaderState *InitXLogPageRead(XLogPageReadPrivate *private_data, +static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, - TimeLineID tli, uint32 xlog_seg_size, + TimeLineID tli, uint32 segment_size, + bool manual_switch, + bool consistent_read, bool allocate_reader); +static bool RunXLogThreads(const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, + TimeLineID tli, uint32 segment_size, + XLogRecPtr startpoint, XLogRecPtr endpoint, + bool consistent_read, + xlog_record_function process_record, + XLogRecTarget *last_rec); +//static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg); +static bool SwitchThreadToNextWal(XLogReaderState *xlogreader, + xlog_thread_arg *arg); +static bool XLogWaitForConsistency(XLogReaderState *xlogreader); +static void *XLogThreadWorker(void *arg); static void CleanupXLogPageRead(XLogReaderState *xlogreader); -static void PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel); +static void PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel); + +static void extractPageInfo(XLogReaderState *record, + XLogReaderData *reader_data, bool *stop_reading); +static void validateXLogRecord(XLogReaderState *record, + XLogReaderData *reader_data, bool *stop_reading); +static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); -static XLogSegNo nextSegNoToRead = 0; +static XLogSegNo segno_start = 0; +/* Segment number where target record is located */ +static XLogSegNo segno_target = 0; +/* Next segment number to read by a thread */ +static XLogSegNo segno_next = 0; +/* Number of segments already read by threads */ +static uint32 segnum_read = 0; static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* copied from timestamp.c */ @@ -156,189 +202,25 @@ timestamptz_to_time_t(TimestampTz t) return result; } +static const char *wal_archivedir = NULL; +static uint32 wal_seg_size = 0; /* - * Do manual switch to the next WAL segment. - * - * Returns false if the reader reaches the end of a WAL segment list. + * If true a wal reader thread switches to the next segment using + * segno_next. */ -static bool -switchToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) -{ - XLogPageReadPrivate *private_data; - XLogRecPtr found; - - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - private_data->need_switch = false; - - /* Critical section */ - pthread_lock(&wal_segment_mutex); - Assert(nextSegNoToRead); - private_data->xlogsegno = nextSegNoToRead; - nextSegNoToRead++; - pthread_mutex_unlock(&wal_segment_mutex); - - /* We've reached the end */ - if (private_data->xlogsegno > arg->endSegNo) - return false; - - /* Adjust next record position */ - GetXLogRecPtr(private_data->xlogsegno, 0, - private_data->xlog_seg_size, arg->startpoint); - /* We need to close previously opened file if it wasn't closed earlier */ - CleanupXLogPageRead(xlogreader); - /* Skip over the page header and contrecord if any */ - found = XLogFindNextRecord(xlogreader, arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is - * absent or is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - PrintXLogCorruptionMsg(private_data, ERROR); - } - arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", - private_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - - return true; -} - +static bool wal_manual_switch = false; /* - * extractPageMap() worker. + * If true a wal reader thread waits for other threads if the thread met absent + * wal segment. */ -static void * -doExtractPageMap(void *arg) -{ - xlog_thread_arg *extract_arg = (xlog_thread_arg *) arg; - XLogPageReadPrivate *private_data; - XLogReaderState *xlogreader; - XLogSegNo nextSegNo = 0; - XLogRecPtr found; - char *errormsg; - - private_data = &extract_arg->private_data; -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(private_data->xlog_seg_size, - &SimpleXLogPageRead, private_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); -#endif - if (xlogreader == NULL) - elog(ERROR, "Thread [%d]: out of memory", private_data->thread_num); - xlogreader->system_identifier = instance_config.system_identifier; - - found = XLogFindNextRecord(xlogreader, extract_arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is absent or - * is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); - PrintXLogCorruptionMsg(private_data, ERROR); - } - extract_arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", - private_data->thread_num, - (uint32) (extract_arg->startpoint >> 32), - (uint32) (extract_arg->startpoint)); - - /* Switch WAL segment manually below without using SimpleXLogPageRead() */ - private_data->manual_switch = true; - - do - { - XLogRecord *record; - - if (interrupted) - elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - private_data->thread_num); - - /* - * We need to switch to the next WAL segment after reading previous - * record. It may happen if we read contrecord. - */ - if (private_data->need_switch) - { - if (!switchToNextWal(xlogreader, extract_arg)) - break; - } - - record = XLogReadRecord(xlogreader, extract_arg->startpoint, &errormsg); - - if (record == NULL) - { - XLogRecPtr errptr; - - /* - * There is no record, try to switch to the next WAL segment. - * Usually SimpleXLogPageRead() does it by itself. But here we need - * to do it manually to support threads. - */ - if (private_data->need_switch && errormsg == NULL) - { - if (switchToNextWal(xlogreader, extract_arg)) - continue; - else - break; - } - - errptr = extract_arg->startpoint ? - extract_arg->startpoint : xlogreader->EndRecPtr; - - if (errormsg) - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", - private_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", - private_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr)); - - /* - * If we don't have all WAL files from prev backup start_lsn to current - * start_lsn, we won't be able to build page map and PAGE backup will - * be incorrect. Stop it and throw an error. - */ - PrintXLogCorruptionMsg(private_data, ERROR); - } +static bool wal_consistent_read = false; - extractPageInfo(xlogreader); - - /* continue reading at next record */ - extract_arg->startpoint = InvalidXLogRecPtr; - - GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, - private_data->xlog_seg_size); - } while (nextSegNo <= extract_arg->endSegNo && - xlogreader->ReadRecPtr < extract_arg->endpoint); - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - /* Extracting is successful */ - extract_arg->ret = 0; - return NULL; -} +/* + * Variables used within validate_wal() and validateXLogRecord() to stop workers + */ +static time_t wal_target_time = 0; +static TransactionId wal_target_xid = InvalidTransactionId; +static XLogRecPtr wal_target_lsn = InvalidXLogRecPtr; /* * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the @@ -348,86 +230,20 @@ doExtractPageMap(void *arg) * file. */ void -extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint, parray *files) +extractPageMap(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, + XLogRecPtr startpoint, XLogRecPtr endpoint) { - int i; - int threads_need = 0; - XLogSegNo endSegNo; bool extract_isok = true; - pthread_t *threads; - xlog_thread_arg *thread_args; time_t start_time, end_time; elog(LOG, "Compiling pagemap"); - if (!XRecOffIsValid(startpoint)) - elog(ERROR, "Invalid startpoint value %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - - if (!XRecOffIsValid(endpoint)) - elog(ERROR, "Invalid endpoint value %X/%X", - (uint32) (endpoint >> 32), (uint32) (endpoint)); - - GetXLogSegNo(endpoint, endSegNo, seg_size); - - nextSegNoToRead = 0; time(&start_time); - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - thread_args = (xlog_thread_arg *) palloc(sizeof(xlog_thread_arg)*num_threads); - - /* - * Initialize thread args. - * - * Each thread works with its own WAL segment and we need to adjust - * startpoint value for each thread. - */ - for (i = 0; i < num_threads; i++) - { - InitXLogPageRead(&thread_args[i].private_data, archivedir, tli, - seg_size, false); - thread_args[i].private_data.thread_num = i + 1; - - thread_args[i].startpoint = startpoint; - thread_args[i].endpoint = endpoint; - thread_args[i].endSegNo = endSegNo; - /* By default there is some error */ - thread_args[i].ret = 1; - - threads_need++; - - /* Adjust startpoint to the next thread */ - if (nextSegNoToRead == 0) - GetXLogSegNo(startpoint, nextSegNoToRead, seg_size); - - nextSegNoToRead++; - /* - * If we need to read less WAL segments than num_threads, create less - * threads. - */ - if (nextSegNoToRead > endSegNo) - break; - GetXLogRecPtr(nextSegNoToRead, 0, seg_size, startpoint); - } - - /* Run threads */ - for (i = 0; i < threads_need; i++) - { - elog(VERBOSE, "Start WAL reader thread: %d", i + 1); - pthread_create(&threads[i], NULL, doExtractPageMap, &thread_args[i]); - } - - /* Wait for threads */ - for (i = 0; i < threads_need; i++) - { - pthread_join(threads[i], NULL); - if (thread_args[i].ret == 1) - extract_isok = false; - } - - pfree(threads); - pfree(thread_args); + extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, tli, wal_seg_size, + startpoint, endpoint, false, extractPageInfo, + NULL); time(&end_time); if (extract_isok) @@ -438,48 +254,26 @@ extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, } /* - * Ensure that the backup has all wal files needed for recovery to consistent state. + * Ensure that the backup has all wal files needed for recovery to consistent + * state. + * + * WAL records reading is processed using threads. Each thread reads single WAL + * file. */ static void validate_backup_wal_from_start_to_stop(pgBackup *backup, - char *backup_xlog_path, TimeLineID tli, + const char *archivedir, TimeLineID tli, uint32 xlog_seg_size) { - XLogRecPtr startpoint = backup->start_lsn; - XLogRecord *record; - XLogReaderState *xlogreader; - char *errormsg; - XLogPageReadPrivate private; - bool got_endpoint = false; - - xlogreader = InitXLogPageRead(&private, backup_xlog_path, tli, - xlog_seg_size, true); - - while (true) - { - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + bool got_endpoint; - if (record == NULL) - { - if (errormsg) - elog(WARNING, "%s", errormsg); - - break; - } - - /* Got WAL record at stop_lsn */ - if (xlogreader->ReadRecPtr == backup->stop_lsn) - { - got_endpoint = true; - break; - } - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ - } + got_endpoint = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, tli, xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + false, NULL, NULL); if (!got_endpoint) { - PrintXLogCorruptionMsg(&private, WARNING); - /* * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. @@ -494,10 +288,6 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn)); } - - /* clean */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); } /* @@ -508,20 +298,12 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, - XLogRecPtr target_lsn, - TimeLineID tli, uint32 seg_size) + XLogRecPtr target_lsn, TimeLineID tli, uint32 wal_seg_size) { - XLogRecPtr startpoint = backup->start_lsn; const char *backup_id; - XLogRecord *record; - XLogReaderState *xlogreader; - char *errormsg; - XLogPageReadPrivate private; - TransactionId last_xid = InvalidTransactionId; - TimestampTz last_time = 0; + XLogRecTarget last_rec; char last_timestamp[100], target_timestamp[100]; - XLogRecPtr last_lsn = InvalidXLogRecPtr; bool all_wal = false; char backup_xlog_path[MAXPGPATH]; @@ -548,11 +330,11 @@ validate_wal(pgBackup *backup, const char *archivedir, DATABASE_DIR, PG_XLOG_DIR); validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, - seg_size); + wal_seg_size); } else validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, - seg_size); + wal_seg_size); if (backup->status == BACKUP_STATUS_CORRUPT) { @@ -563,7 +345,8 @@ validate_wal(pgBackup *backup, const char *archivedir, * If recovery target is provided check that we can restore backup to a * recovery target time or xid. */ - if (!TransactionIdIsValid(target_xid) && target_time == 0 && !XRecOffIsValid(target_lsn)) + if (!TransactionIdIsValid(target_xid) && target_time == 0 && + !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ elog(INFO, "Backup %s WAL segments are valid", backup_id); @@ -582,89 +365,41 @@ validate_wal(pgBackup *backup, const char *archivedir, * up to the given recovery target. * In any case we cannot restore to the point before stop_lsn. */ - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, - true); /* We can restore at least up to the backup end */ + last_rec.rec_time = 0; + last_rec.rec_xid = backup->recovery_xid; + last_rec.rec_lsn = backup->stop_lsn; + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); - last_xid = backup->recovery_xid; - last_lsn = backup->stop_lsn; - if ((TransactionIdIsValid(target_xid) && target_xid == last_xid) + if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) || (target_time != 0 && backup->recovery_time >= target_time) - || (XRecOffIsValid(target_lsn) && backup->stop_lsn >= target_lsn)) + || (XRecOffIsValid(target_lsn) && last_rec.rec_lsn >= target_lsn)) all_wal = true; - startpoint = backup->stop_lsn; - while (true) - { - bool timestamp_record; - - record = XLogReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - if (errormsg) - elog(WARNING, "%s", errormsg); - - break; - } - - timestamp_record = getRecordTimestamp(xlogreader, &last_time); - if (XLogRecGetXid(xlogreader) != InvalidTransactionId) - last_xid = XLogRecGetXid(xlogreader); - last_lsn = xlogreader->ReadRecPtr; - - /* Check target xid */ - if (TransactionIdIsValid(target_xid) && target_xid == last_xid) - { - all_wal = true; - break; - } - /* Check target time */ - else if (target_time != 0 && timestamp_record && - timestamptz_to_time_t(last_time) >= target_time) - { - all_wal = true; - break; - } - /* Check target lsn */ - else if (XRecOffIsValid(target_lsn) && last_lsn >= target_lsn) - { - all_wal = true; - break; - } - /* If there are no target xid, target time and target lsn */ - else if (!TransactionIdIsValid(target_xid) && target_time == 0 && - !XRecOffIsValid(target_lsn)) - { - all_wal = true; - /* We don't stop here. We want to get last_xid and last_time */ - } - - startpoint = InvalidXLogRecPtr; /* continue reading at next record */ - } - - if (last_time > 0) + all_wal = all_wal || + RunXLogThreads(archivedir, target_time, target_xid, target_lsn, + tli, wal_seg_size, backup->stop_lsn, + InvalidXLogRecPtr, true, validateXLogRecord, &last_rec); + if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_time)); + timestamptz_to_time_t(last_rec.rec_time)); /* There are all needed WAL records */ if (all_wal) elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_xid, - (uint32) (last_lsn >> 32), (uint32) last_lsn); + last_timestamp, last_rec.rec_xid, + (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); /* Some needed WAL records are absent */ else { - PrintXLogCorruptionMsg(&private, WARNING); - elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_xid, - (uint32) (last_lsn >> 32), (uint32) last_lsn); + last_timestamp, last_rec.rec_xid, + (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), - target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); @@ -678,10 +413,6 @@ validate_wal(pgBackup *backup, const char *archivedir, elog(ERROR, "not enough WAL records to lsn %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } - - /* clean */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); } /* @@ -690,13 +421,13 @@ validate_wal(pgBackup *backup, const char *archivedir, * pg_stop_backup(). */ bool -read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, +read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, time_t *recovery_time, TransactionId *recovery_xid) { XLogRecPtr startpoint = stop_lsn; XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; bool res; if (!XRecOffIsValid(start_lsn)) @@ -707,7 +438,8 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, elog(ERROR, "Invalid stop_lsn value %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, true, true); /* Read records from stop_lsn down to start_lsn */ do @@ -762,10 +494,10 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, */ bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 seg_size) + TimeLineID target_tli, uint32 wal_seg_size) { XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; char *errormsg; bool res; @@ -773,8 +505,8 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, elog(ERROR, "Invalid target_lsn value %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - xlogreader = InitXLogPageRead(&private, archivedir, target_tli, seg_size, - true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli, + wal_seg_size, false, false, true); res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ @@ -797,16 +529,16 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, - uint32 seg_size) + uint32 wal_seg_size) { XLogReaderState *xlogreader; - XLogPageReadPrivate private; + XLogReaderData reader_data; XLogRecPtr startpoint; XLogSegNo start_segno; XLogSegNo segno; XLogRecPtr res = InvalidXLogRecPtr; - GetXLogSegNo(stop_lsn, segno, seg_size); + GetXLogSegNo(stop_lsn, segno, wal_seg_size); if (segno <= 1) elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); @@ -814,19 +546,20 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, if (seek_prev_segment) segno = segno - 1; - xlogreader = InitXLogPageRead(&private, archivedir, tli, seg_size, true); + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, false, true); /* * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. */ - GetXLogSegNo(start_lsn, start_segno, seg_size); + GetXLogSegNo(start_lsn, start_segno, wal_seg_size); if (start_segno == segno) startpoint = start_lsn; else { XLogRecPtr found; - GetXLogRecPtr(segno, 0, seg_size, startpoint); + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -838,7 +571,7 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, else elog(WARNING, "Could not read WAL record at %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); - PrintXLogCorruptionMsg(&private, ERROR); + PrintXLogCorruptionMsg(&reader_data, ERROR); } startpoint = found; } @@ -867,13 +600,13 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, else elog(WARNING, "Could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); - PrintXLogCorruptionMsg(&private, ERROR); + PrintXLogCorruptionMsg(&reader_data, ERROR); } /* continue reading at next record */ startpoint = InvalidXLogRecPtr; - GetXLogSegNo(xlogreader->EndRecPtr, next_segno, seg_size); + GetXLogSegNo(xlogreader->EndRecPtr, next_segno, wal_seg_size); if (next_segno > segno) break; @@ -919,25 +652,24 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI) { - XLogPageReadPrivate *private_data; + XLogReaderData *reader_data; uint32 targetPageOff; - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - targetPageOff = targetPagePtr % private_data->xlog_seg_size; + reader_data = (XLogReaderData *) xlogreader->private_data; + targetPageOff = targetPagePtr % wal_seg_size; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - private_data->thread_num); + reader_data->thread_num); /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (!IsInXLogSeg(targetPagePtr, private_data->xlogsegno, - private_data->xlog_seg_size)) + if (!IsInXLogSeg(targetPagePtr, reader_data->xlogsegno, wal_seg_size)) { elog(VERBOSE, "Thread [%d]: Need to switch to the next WAL segment, page LSN %X/%X, record being read LSN %X/%X", - private_data->thread_num, + reader_data->thread_num, (uint32) (targetPagePtr >> 32), (uint32) (targetPagePtr), (uint32) (xlogreader->currRecPtr >> 32), (uint32) (xlogreader->currRecPtr )); @@ -954,52 +686,49 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* * Switch to the next WAL segment after reading contrecord. */ - if (private_data->manual_switch) - private_data->need_switch = true; + if (wal_manual_switch) + reader_data->need_switch = true; } else { CleanupXLogPageRead(xlogreader); /* - * Do not switch to next WAL segment in this function. Currently it is - * manually switched only in doExtractPageMap(). - */ - if (private_data->manual_switch) + * Do not switch to next WAL segment in this function. It is + * manually switched by a thread routine. + */ + if (wal_manual_switch) { - private_data->need_switch = true; + reader_data->need_switch = true; return -1; } } } - GetXLogSegNo(targetPagePtr, private_data->xlogsegno, - private_data->xlog_seg_size); + GetXLogSegNo(targetPagePtr, reader_data->xlogsegno, wal_seg_size); /* Try to switch to the next WAL segment */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) { char xlogfname[MAXFNAMELEN]; - GetXLogFileName(xlogfname, private_data->tli, private_data->xlogsegno, - private_data->xlog_seg_size); - snprintf(private_data->xlogpath, MAXPGPATH, "%s/%s", - private_data->archivedir, xlogfname); + GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, + wal_seg_size); + snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, + xlogfname); - if (fileExists(private_data->xlogpath)) + if (fileExists(reader_data->xlogpath)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", - private_data->thread_num, - private_data->xlogpath); + reader_data->thread_num, reader_data->xlogpath); - private_data->xlogexists = true; - private_data->xlogfile = open(private_data->xlogpath, - O_RDONLY | PG_BINARY, 0); + reader_data->xlogexists = true; + reader_data->xlogfile = open(reader_data->xlogpath, + O_RDONLY | PG_BINARY, 0); - if (private_data->xlogfile < 0) + if (reader_data->xlogfile < 0) { elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s", - private_data->thread_num, - private_data->xlogpath, + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } @@ -1008,21 +737,21 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* Try to open compressed WAL segment */ else { - snprintf(private_data->gz_xlogpath, - sizeof(private_data->gz_xlogpath), "%s.gz", - private_data->xlogpath); - if (fileExists(private_data->gz_xlogpath)) + snprintf(reader_data->gz_xlogpath, sizeof(reader_data->gz_xlogpath), + "%s.gz", reader_data->xlogpath); + if (fileExists(reader_data->gz_xlogpath)) { elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", - private_data->thread_num, private_data->gz_xlogpath); + reader_data->thread_num, reader_data->gz_xlogpath); - private_data->xlogexists = true; - private_data->gz_xlogfile = gzopen(private_data->gz_xlogpath, - "rb"); - if (private_data->gz_xlogfile == NULL) + reader_data->xlogexists = true; + reader_data->gz_xlogfile = gzopen(reader_data->gz_xlogpath, + "rb"); + if (reader_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", - private_data->thread_num, private_data->gz_xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->gz_xlogpath, + strerror(errno)); return -1; } } @@ -1030,69 +759,67 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, #endif /* Exit without error if WAL segment doesn't exist */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) return -1; } /* * At this point, we have the right segment open. */ - Assert(private_data->xlogexists); + Assert(reader_data->xlogexists); /* * Do not read same page read earlier from the file, read it from the buffer */ - if (private_data->prev_page_off != 0 && - private_data->prev_page_off == targetPageOff) + if (reader_data->prev_page_off != 0 && + reader_data->prev_page_off == targetPageOff) { - memcpy(readBuf, private_data->page_buf, XLOG_BLCKSZ); - *pageTLI = private_data->tli; + memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); + *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } /* Read the requested page */ - if (private_data->xlogfile != -1) + if (reader_data->xlogfile != -1) { - if (lseek(private_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) + if (lseek(reader_data->xlogfile, (off_t) targetPageOff, SEEK_SET) < 0) { elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } - if (read(private_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (read(reader_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", - private_data->thread_num, private_data->xlogpath, strerror(errno)); + reader_data->thread_num, reader_data->xlogpath, strerror(errno)); return -1; } } #ifdef HAVE_LIBZ else { - if (gzseek(private_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) + if (gzseek(reader_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) { elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); + reader_data->thread_num, reader_data->gz_xlogpath, + get_gz_error(reader_data->gz_xlogfile)); return -1; } - if (gzread(private_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + if (gzread(reader_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - private_data->thread_num, - private_data->gz_xlogpath, - get_gz_error(private_data->gz_xlogfile)); + reader_data->thread_num, reader_data->gz_xlogpath, + get_gz_error(reader_data->gz_xlogfile)); return -1; } } #endif - memcpy(private_data->page_buf, readBuf, XLOG_BLCKSZ); - private_data->prev_page_off = targetPageOff; - *pageTLI = private_data->tli; + memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); + reader_data->prev_page_off = targetPageOff; + *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } @@ -1100,89 +827,521 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, * Initialize WAL segments reading. */ static XLogReaderState * -InitXLogPageRead(XLogPageReadPrivate *private_data, const char *archivedir, - TimeLineID tli, uint32 xlog_seg_size, bool allocate_reader) +InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, + TimeLineID tli, uint32 segment_size, bool manual_switch, + bool consistent_read, bool allocate_reader) { XLogReaderState *xlogreader = NULL; - MemSet(private_data, 0, sizeof(XLogPageReadPrivate)); - private_data->archivedir = archivedir; - private_data->tli = tli; - private_data->xlog_seg_size = xlog_seg_size; - private_data->xlogfile = -1; + wal_archivedir = archivedir; + wal_seg_size = segment_size; + wal_manual_switch = manual_switch; + wal_consistent_read = consistent_read; + + MemSet(reader_data, 0, sizeof(XLogReaderData)); + reader_data->tli = tli; + reader_data->xlogfile = -1; if (allocate_reader) { #if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(xlog_seg_size, - &SimpleXLogPageRead, private_data); + xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); #else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, private_data); + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); #endif if (xlogreader == NULL) - elog(ERROR, "out of memory"); + elog(ERROR, "Out of memory"); xlogreader->system_identifier = instance_config.system_identifier; } return xlogreader; } +/* + * Run WAL processing routines using threads. Start from startpoint up to + * endpoint. It is possible to send zero endpoint, threads will read WAL + * infinitely in this case. + */ +static bool +RunXLogThreads(const char *archivedir, time_t target_time, + TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, + uint32 segment_size, XLogRecPtr startpoint, XLogRecPtr endpoint, + bool consistent_read, xlog_record_function process_record, + XLogRecTarget *last_rec) +{ + pthread_t *threads; + xlog_thread_arg *thread_args; + int i; + int threads_need = 0; + XLogSegNo endSegNo = 0; + XLogSegNo errorSegNo = 0; + bool result = true; + + if (!XRecOffIsValid(startpoint)) + elog(ERROR, "Invalid startpoint value %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + + if (!XLogRecPtrIsInvalid(endpoint)) + { + if (!XRecOffIsValid(endpoint)) + elog(ERROR, "Invalid endpoint value %X/%X", + (uint32) (endpoint >> 32), (uint32) (endpoint)); + + GetXLogSegNo(endpoint, endSegNo, segment_size); + } + + /* Initialize static variables for workers */ + wal_target_time = target_time; + wal_target_xid = target_xid; + wal_target_lsn = target_lsn; + + GetXLogSegNo(startpoint, segno_start, segment_size); + segno_target = 0; + GetXLogSegNo(startpoint, segno_next, segment_size); + segnum_read = 0; + + threads = (pthread_t *) pgut_malloc(sizeof(pthread_t) * num_threads); + thread_args = (xlog_thread_arg *) pgut_malloc(sizeof(xlog_thread_arg) * num_threads); + + /* + * Initialize thread args. + * + * Each thread works with its own WAL segment and we need to adjust + * startpoint value for each thread. + */ + for (i = 0; i < num_threads; i++) + { + xlog_thread_arg *arg = &thread_args[i]; + + InitXLogPageRead(&arg->reader_data, archivedir, tli, segment_size, true, + consistent_read, false); + arg->reader_data.xlogsegno = segno_next; + arg->reader_data.thread_num = i + 1; + arg->process_record = process_record; + arg->startpoint = startpoint; + arg->endpoint = endpoint; + arg->endSegNo = endSegNo; + arg->got_target = false; + /* By default there is some error */ + arg->ret = 1; + + threads_need++; + segno_next++; + /* + * If we need to read less WAL segments than num_threads, create less + * threads. + */ + if (endSegNo != 0 && segno_next > endSegNo) + break; + GetXLogRecPtr(segno_next, 0, segment_size, startpoint); + } + + /* Run threads */ + thread_interrupted = false; + for (i = 0; i < threads_need; i++) + { + elog(VERBOSE, "Start WAL reader thread: %d", i + 1); + pthread_create(&threads[i], NULL, XLogThreadWorker, &thread_args[i]); + } + + /* Wait for threads */ + for (i = 0; i < threads_need; i++) + { + pthread_join(threads[i], NULL); + if (thread_args[i].ret == 1) + result = false; + } + + if (last_rec) + for (i = 0; i < threads_need; i++) + { + XLogRecTarget *cur_rec; + + if (thread_args[i].ret != 0) + { + /* + * Save invalid segment number after which all segments are not + * valid. + */ + if (errorSegNo == 0 || + errorSegNo > thread_args[i].reader_data.xlogsegno) + errorSegNo = thread_args[i].reader_data.xlogsegno; + continue; + } + + /* Is this segment valid */ + if (errorSegNo != 0 && + thread_args[i].reader_data.xlogsegno > errorSegNo) + continue; + + cur_rec = &thread_args[i].reader_data.cur_rec; + /* + * If we got the target return minimum possible record. + */ + if (segno_target > 0) + { + if (thread_args[i].got_target && + thread_args[i].reader_data.xlogsegno == segno_target) + { + *last_rec = *cur_rec; + break; + } + } + /* + * Else return maximum possible record up to which restore is + * possible. + */ + else if (last_rec->rec_lsn < cur_rec->rec_lsn) + *last_rec = *cur_rec; + } + + pfree(threads); + pfree(thread_args); + + return result; +} + +/* + * WAL reader worker. + */ +void * +XLogThreadWorker(void *arg) +{ + xlog_thread_arg *thread_arg = (xlog_thread_arg *) arg; + XLogReaderData *reader_data = &thread_arg->reader_data; + XLogReaderState *xlogreader; + XLogSegNo nextSegNo = 0; + XLogRecPtr found; + uint32 prev_page_off = 0; + bool need_read = true; + +#if PG_VERSION_NUM >= 110000 + xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); +#else + xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); +#endif + if (xlogreader == NULL) + elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); + xlogreader->system_identifier = instance_config.system_identifier; + + found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); + + /* + * We get invalid WAL record pointer usually when WAL segment is absent or + * is corrupted. + */ + if (XLogRecPtrIsInvalid(found)) + { + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + need_read = false; + else + { + if (xlogreader->errormsg_buf[0] != '\0') + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint), + xlogreader->errormsg_buf); + else + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint)); + PrintXLogCorruptionMsg(reader_data, ERROR); + } + } + + thread_arg->startpoint = found; + + elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", + reader_data->thread_num, + (uint32) (thread_arg->startpoint >> 32), + (uint32) (thread_arg->startpoint)); + + while (need_read) + { + XLogRecord *record; + char *errormsg; + bool stop_reading = false; + + if (interrupted || thread_interrupted) + elog(ERROR, "Thread [%d]: Interrupted during WAL reading", + reader_data->thread_num); + + /* + * We need to switch to the next WAL segment after reading previous + * record. It may happen if we read contrecord. + */ + if (reader_data->need_switch && + !SwitchThreadToNextWal(xlogreader, thread_arg)) + break; + + record = XLogReadRecord(xlogreader, thread_arg->startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + /* + * There is no record, try to switch to the next WAL segment. + * Usually SimpleXLogPageRead() does it by itself. But here we need + * to do it manually to support threads. + */ + if (reader_data->need_switch && errormsg == NULL) + { + if (SwitchThreadToNextWal(xlogreader, thread_arg)) + continue; + else + break; + } + + /* + * XLogWaitForConsistency() is normally used only with threads. + * Call it here for just in case. + */ + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + break; + + errptr = thread_arg->startpoint ? + thread_arg->startpoint : xlogreader->EndRecPtr; + + if (errormsg) + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + reader_data->thread_num, + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (errptr >> 32), (uint32) (errptr)); + + /* + * If we don't have all WAL files from prev backup start_lsn to current + * start_lsn, we won't be able to build page map and PAGE backup will + * be incorrect. Stop it and throw an error. + */ + PrintXLogCorruptionMsg(reader_data, ERROR); + } + + getRecordTimestamp(xlogreader, &reader_data->cur_rec.rec_time); + if (TransactionIdIsValid(XLogRecGetXid(xlogreader))) + reader_data->cur_rec.rec_xid = XLogRecGetXid(xlogreader); + reader_data->cur_rec.rec_lsn = xlogreader->ReadRecPtr; + + if (thread_arg->process_record) + thread_arg->process_record(xlogreader, reader_data, &stop_reading); + if (stop_reading) + { + thread_arg->got_target = true; + + pthread_lock(&wal_segment_mutex); + /* We should store least target segment number */ + if (segno_target == 0 || segno_target > reader_data->xlogsegno) + segno_target = reader_data->xlogsegno; + pthread_mutex_unlock(&wal_segment_mutex); + + break; + } + + /* + * Check if other thread got the target segment. Check it not very + * often, only every WAL page. + */ + if (wal_consistent_read && prev_page_off != 0 && + prev_page_off != reader_data->prev_page_off) + { + XLogSegNo segno; + + pthread_lock(&wal_segment_mutex); + segno = segno_target; + pthread_mutex_unlock(&wal_segment_mutex); + + if (segno != 0 && segno < reader_data->xlogsegno) + break; + } + prev_page_off = reader_data->prev_page_off; + + /* continue reading at next record */ + thread_arg->startpoint = InvalidXLogRecPtr; + + GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, wal_seg_size); + + if (thread_arg->endSegNo != 0 && + !XLogRecPtrIsInvalid(thread_arg->endpoint) && + /* + * Consider thread_arg->endSegNo and thread_arg->endpoint only if + * they are valid. + */ + xlogreader->ReadRecPtr == thread_arg->endpoint && + nextSegNo > thread_arg->endSegNo) + break; + } + + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + /* Extracting is successful */ + thread_arg->ret = 0; + return NULL; +} + +/* + * Do manual switch to the next WAL segment. + * + * Returns false if the reader reaches the end of a WAL segment list. + */ +static bool +SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) +{ + XLogReaderData *reader_data; + XLogRecPtr found; + + reader_data = (XLogReaderData *) xlogreader->private_data; + reader_data->need_switch = false; + + /* Critical section */ + pthread_lock(&wal_segment_mutex); + Assert(segno_next); + reader_data->xlogsegno = segno_next; + segnum_read++; + segno_next++; + pthread_mutex_unlock(&wal_segment_mutex); + + /* We've reached the end */ + if (arg->endSegNo != 0 && reader_data->xlogsegno > arg->endSegNo) + return false; + + /* Adjust next record position */ + GetXLogRecPtr(reader_data->xlogsegno, 0, wal_seg_size, arg->startpoint); + /* We need to close previously opened file if it wasn't closed earlier */ + CleanupXLogPageRead(xlogreader); + /* Skip over the page header and contrecord if any */ + found = XLogFindNextRecord(xlogreader, arg->startpoint); + + /* + * We get invalid WAL record pointer usually when WAL segment is + * absent or is corrupted. + */ + if (XLogRecPtrIsInvalid(found)) + { + /* + * Check if we need to stop reading. We stop if other thread found a + * target segment. + */ + if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) + return false; + + elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + reader_data->thread_num, + (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); + PrintXLogCorruptionMsg(reader_data, ERROR); + } + arg->startpoint = found; + + elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", + reader_data->thread_num, + (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); + + return true; +} + +/* + * Wait for other threads since the current thread couldn't read its segment. + * We need to decide is it fail or not. + * + * Returns true if there is no failure and previous target segment was found. + * Otherwise return false. + */ +static bool +XLogWaitForConsistency(XLogReaderState *xlogreader) +{ + uint32 segnum_need = 0; + XLogReaderData *reader_data =(XLogReaderData *) xlogreader->private_data; + + segnum_need = reader_data->xlogsegno - segno_start; + while (true) + { + uint32 segnum_current_read; + XLogSegNo segno; + + if (interrupted || thread_interrupted) + elog(ERROR, "Thread [%d]: Interrupted during WAL reading", + reader_data->thread_num); + + pthread_lock(&wal_segment_mutex); + segnum_current_read = segnum_read; + segno = segno_target; + pthread_mutex_unlock(&wal_segment_mutex); + + /* Other threads read all previous segments and didn't find target */ + if (segnum_need <= segnum_current_read) + return false; + + if (segno < reader_data->xlogsegno) + return true; + + pg_usleep(1000000L); /* 1000 ms */ + } + + /* We shouldn't reach it */ + return false; +} + /* * Cleanup after WAL segment reading. */ static void CleanupXLogPageRead(XLogReaderState *xlogreader) { - XLogPageReadPrivate *private_data; + XLogReaderData *reader_data; - private_data = (XLogPageReadPrivate *) xlogreader->private_data; - if (private_data->xlogfile >= 0) + reader_data = (XLogReaderData *) xlogreader->private_data; + if (reader_data->xlogfile >= 0) { - close(private_data->xlogfile); - private_data->xlogfile = -1; + close(reader_data->xlogfile); + reader_data->xlogfile = -1; } #ifdef HAVE_LIBZ - else if (private_data->gz_xlogfile != NULL) + else if (reader_data->gz_xlogfile != NULL) { - gzclose(private_data->gz_xlogfile); - private_data->gz_xlogfile = NULL; + gzclose(reader_data->gz_xlogfile); + reader_data->gz_xlogfile = NULL; } #endif - private_data->prev_page_off = 0; - private_data->xlogexists = false; + reader_data->prev_page_off = 0; + reader_data->xlogexists = false; } static void -PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) +PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel) { - if (private_data->xlogpath[0] != 0) + if (reader_data->xlogpath[0] != 0) { /* * XLOG reader couldn't read WAL segment. * We throw a WARNING here to be able to update backup status. */ - if (!private_data->xlogexists) + if (!reader_data->xlogexists) elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent", - private_data->thread_num, - private_data->xlogpath); - else if (private_data->xlogfile != -1) + reader_data->thread_num, reader_data->xlogpath); + else if (reader_data->xlogfile != -1) elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", - private_data->thread_num, - private_data->xlogpath); + reader_data->thread_num, reader_data->xlogpath); #ifdef HAVE_LIBZ - else if (private_data->gz_xlogfile != NULL) + else if (reader_data->gz_xlogfile != NULL) elog(elevel, "Thread [%d]: Possible WAL corruption. " "Error has occured during reading WAL segment \"%s\"", - private_data->thread_num, - private_data->gz_xlogpath); + reader_data->thread_num, reader_data->gz_xlogpath); #endif } else { /* Cannot tell what happened specifically */ elog(elevel, "Thread [%d]: An error occured during WAL reading", - private_data->thread_num); + reader_data->thread_num); } } @@ -1190,7 +1349,8 @@ PrintXLogCorruptionMsg(XLogPageReadPrivate *private_data, int elevel) * Extract information about blocks modified in this record. */ static void -extractPageInfo(XLogReaderState *record) +extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, + bool *stop_reading) { uint8 block_id; RmgrId rmid = XLogRecGetRmid(record); @@ -1258,6 +1418,27 @@ extractPageInfo(XLogReaderState *record) } } +/* + * Check the current read WAL record during validation. + */ +static void +validateXLogRecord(XLogReaderState *record, XLogReaderData *reader_data, + bool *stop_reading) +{ + /* Check target xid */ + if (TransactionIdIsValid(wal_target_xid) && + wal_target_xid == reader_data->cur_rec.rec_xid) + *stop_reading = true; + /* Check target time */ + else if (wal_target_time != 0 && + timestamptz_to_time_t(reader_data->cur_rec.rec_time) >= wal_target_time) + *stop_reading = true; + /* Check target lsn */ + else if (XRecOffIsValid(wal_target_lsn) && + reader_data->cur_rec.rec_lsn >= wal_target_lsn) + *stop_reading = true; +} + /* * Extract timestamp from WAL record. * diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1bb08e325..2cceaaedb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -573,14 +573,11 @@ extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, /* parsexlog.c */ extern void extractPageMap(const char *archivedir, TimeLineID tli, uint32 seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint, - parray *files); -extern void validate_wal(pgBackup *backup, - const char *archivedir, - time_t target_time, - TransactionId target_xid, - XLogRecPtr target_lsn, - TimeLineID tli, uint32 seg_size); + XLogRecPtr startpoint, XLogRecPtr endpoint); +extern void validate_wal(pgBackup *backup, const char *archivedir, + time_t target_time, TransactionId target_xid, + XLogRecPtr target_lsn, TimeLineID tli, + uint32 seg_size); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, diff --git a/src/restore.c b/src/restore.c index a486fb50c..ae379e9f5 100644 --- a/src/restore.c +++ b/src/restore.c @@ -569,6 +569,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); /* Restore files into target directory */ + thread_interrupted = false; for (i = 0; i < num_threads; i++) { restore_files_arg *arg = &(threads_args[i]); @@ -680,7 +681,7 @@ restore_files(void *arg) lengthof(from_root), DATABASE_DIR); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during restore database"); rel_path = GetRelativePath(file->path,from_root); diff --git a/src/utils/logger.c b/src/utils/logger.c index ba054a62c..72840aa87 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -121,9 +121,6 @@ exit_if_necessary(int elevel) { if (elevel > WARNING && !in_cleanup) { - /* Interrupt other possible routines */ - interrupted = true; - if (loggin_in_progress) { loggin_in_progress = false; @@ -132,11 +129,15 @@ exit_if_necessary(int elevel) /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) + { #ifdef WIN32 ExitThread(elevel); #else pthread_exit(NULL); #endif + /* Interrupt other possible routines */ + thread_interrupted = true; + } else exit(elevel); } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a7e12e913..cdd4b26d3 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -700,7 +700,7 @@ on_interrupt(void) int save_errno = errno; char errbuf[256]; - /* Set interruped flag */ + /* Set interrupted flag */ interrupted = true; /* diff --git a/src/utils/thread.c b/src/utils/thread.c index 0999a0d5b..f1624be98 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -7,8 +7,12 @@ *------------------------------------------------------------------------- */ +#include "postgres_fe.h" + #include "thread.h" +bool thread_interrupted = false; + #ifdef WIN32 DWORD main_tid = 0; #else diff --git a/src/utils/thread.h b/src/utils/thread.h index 6b8349bf5..a2948156b 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -34,7 +34,7 @@ extern DWORD main_tid; extern pthread_t main_tid; #endif - +extern bool thread_interrupted; extern int pthread_lock(pthread_mutex_t *mp); diff --git a/src/validate.c b/src/validate.c index c19e319d6..7d5e94f43 100644 --- a/src/validate.c +++ b/src/validate.c @@ -117,6 +117,7 @@ pgBackupValidate(pgBackup *backup) palloc(sizeof(validate_files_arg) * num_threads); /* Validate files */ + thread_interrupted = false; for (i = 0; i < num_threads; i++) { validate_files_arg *arg = &(threads_args[i]); @@ -186,7 +187,7 @@ pgBackupValidateFiles(void *arg) if (!pg_atomic_test_set_flag(&file->lock)) continue; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during validate"); /* Validate only regular files */ diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index abb5ce713..a83c39052 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -63,7 +63,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--skip-block-validation] pg_probackup validate -B backup-path [--instance=instance_name] - [-i backup-id] [--progress] + [-i backup-id] [--progress] [-j num-threads] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] [--recovery-target-name=target-name] [--timeline=timeline] From 1e87dce5916ffb4cb8533d625a32a72ff9b013fa Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 5 Mar 2019 18:35:31 +0300 Subject: [PATCH 0440/2107] PGPRO-1973: Fix a bug, improve validate tests --- src/parsexlog.c | 22 ++++++++++++++++++---- tests/validate_test.py | 17 ++++++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 198f9c5aa..5bea453ca 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1233,14 +1233,14 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) return false; - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); PrintXLogCorruptionMsg(reader_data, ERROR); } arg->startpoint = found; - elog(VERBOSE, "Thread [%d]: switched to LSN %X/%X", + elog(VERBOSE, "Thread [%d]: Switched to LSN %X/%X", reader_data->thread_num, (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); @@ -1257,8 +1257,9 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) static bool XLogWaitForConsistency(XLogReaderState *xlogreader) { - uint32 segnum_need = 0; + uint32 segnum_need; XLogReaderData *reader_data =(XLogReaderData *) xlogreader->private_data; + bool log_message = true; segnum_need = reader_data->xlogsegno - segno_start; while (true) @@ -1266,6 +1267,13 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) uint32 segnum_current_read; XLogSegNo segno; + if (log_message) + { + elog(VERBOSE, "Thread [%d]: Possible WAL corruption. Wait for other threads to decide is this a failure", + reader_data->thread_num); + log_message = false; + } + if (interrupted || thread_interrupted) elog(ERROR, "Thread [%d]: Interrupted during WAL reading", reader_data->thread_num); @@ -1277,9 +1285,15 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) /* Other threads read all previous segments and didn't find target */ if (segnum_need <= segnum_current_read) + { + /* Mark current segment as read even if it wasn't read actually */ + pthread_lock(&wal_segment_mutex); + segnum_read++; + pthread_mutex_unlock(&wal_segment_mutex); return false; + } - if (segno < reader_data->xlogsegno) + if (segno != 0 && segno < reader_data->xlogsegno) return true; pg_usleep(1000000L); /* 1000 ms */ diff --git a/tests/validate_test.py b/tests/validate_test.py index cb6247a1c..a2a9ba2df 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -58,7 +58,7 @@ def test_validate_nullified_heap_page_backup(self): '{0} blknum 1, empty page'.format(file_path) in f.read(), 'Failed to detect nullified block') - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -99,7 +99,7 @@ def test_validate_wal_unreal_values(self): "INFO: backup validation completed successfully", self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(target_time)]), + options=["--time={0}".format(target_time), "-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -108,7 +108,7 @@ def test_validate_wal_unreal_values(self): try: self.validate_pb( backup_dir, 'node', options=["--time={0}".format( - unreal_time_1)]) + unreal_time_1), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal time.\n " @@ -126,7 +126,7 @@ def test_validate_wal_unreal_values(self): try: self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(unreal_time_2)]) + options=["--time={0}".format(unreal_time_2), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal time.\n " @@ -150,7 +150,8 @@ def test_validate_wal_unreal_values(self): self.assertIn( "INFO: backup validation completed successfully", self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(target_xid)]), + backup_dir, 'node', options=["--xid={0}".format(target_xid), + "-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -158,7 +159,8 @@ def test_validate_wal_unreal_values(self): unreal_xid = int(target_xid) + 1000 try: self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(unreal_xid)]) + backup_dir, 'node', options=["--xid={0}".format(unreal_xid), + "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of validation to unreal xid.\n " @@ -171,7 +173,8 @@ def test_validate_wal_unreal_values(self): repr(e.message), self.cmd)) # Validate with backup ID - output = self.validate_pb(backup_dir, 'node', backup_id) + output = self.validate_pb(backup_dir, 'node', backup_id, + options=["-j", "4"]) self.assertIn( "INFO: Validating backup {0}".format(backup_id), output, From 791837627b189b9c2d2f7852878f4fec005d4ff5 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 6 Mar 2019 11:58:42 +0300 Subject: [PATCH 0441/2107] PGPRO-1973: Add more parallel tests --- tests/validate_test.py | 89 +++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index a2a9ba2df..0d21811d7 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -260,7 +260,7 @@ def test_validate_corrupted_intermediate_backup(self): # Simple validate try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -363,7 +363,7 @@ def test_validate_corrupted_intermediate_backups(self): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -462,7 +462,7 @@ def test_validate_specific_error_intermediate_backups(self): # Validate PAGE1 try: self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2) + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because backup has status ERROR.\n " @@ -549,7 +549,7 @@ def test_validate_error_intermediate_backups(self): # Validate instance try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because backup has status ERROR.\n " @@ -688,7 +688,7 @@ def test_validate_corrupted_intermediate_backups_1(self): try: self.validate_pb( backup_dir, 'node', - backup_id=backup_id_4) + backup_id=backup_id_4, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n" @@ -888,7 +888,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.validate_pb( backup_dir, 'node', options=[ - '-i', backup_id_4, '--xid={0}'.format(target_xid)]) + '-i', backup_id_4, '--xid={0}'.format(target_xid), + "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1029,8 +1030,7 @@ def test_validate_instance_with_corrupted_page(self): # Validate Instance try: - self.validate_pb( - backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1178,7 +1178,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -1279,7 +1279,7 @@ def test_validate_instance_with_corrupted_full(self): # Validate Instance try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -1337,7 +1337,7 @@ def test_validate_corrupt_wal_1(self): # Simple validate try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1407,7 +1407,7 @@ def test_validate_corrupt_wal_2(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid)]) + "--xid={0}".format(target_xid), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1464,7 +1464,7 @@ def test_validate_wal_lost_segment_1(self): file = file[:-3] try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n" @@ -1488,7 +1488,7 @@ def test_validate_wal_lost_segment_1(self): # Run validate again try: - self.validate_pb(backup_dir, 'node', backup_id) + self.validate_pb(backup_dir, 'node', backup_id, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup corruption.\n" @@ -1572,7 +1572,7 @@ def test_validate_corrupt_wal_between_backups(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid)]) + "--xid={0}".format(target_xid), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" @@ -1626,7 +1626,7 @@ def test_pgpro702_688(self): try: self.validate_pb( backup_dir, 'node', - options=["--time={0}".format(recovery_time)]) + options=["--time={0}".format(recovery_time), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segment disappearance.\n " @@ -1666,7 +1666,8 @@ def test_pgpro688(self): backup_dir, 'node', backup_id)['recovery-time'] self.validate_pb( - backup_dir, 'node', options=["--time={0}".format(recovery_time)]) + backup_dir, 'node', options=["--time={0}".format(recovery_time), + "-j", "4"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1823,7 +1824,7 @@ def test_validate_corrupted_full(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -1861,7 +1862,7 @@ def test_validate_corrupted_full(self): os.rename(file_new, file) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -1926,7 +1927,7 @@ def test_validate_corrupted_full_1(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -1966,7 +1967,7 @@ def test_validate_corrupted_full_1(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid'.format( @@ -2040,7 +2041,8 @@ def test_validate_corrupted_full_2(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2087,7 +2089,7 @@ def test_validate_corrupted_full_2(self): self.backup_node(backup_dir, 'node', node, backup_type='page') try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2155,7 +2157,8 @@ def test_validate_corrupted_full_2(self): # revalidate again try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2228,7 +2231,8 @@ def test_validate_corrupted_full_2(self): # Fix CORRUPT os.rename(file_new, file) - output = self.validate_pb(backup_dir, 'node', validate_id) + output = self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertIn( 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), @@ -2390,7 +2394,7 @@ def test_validate_corrupted_full_missing(self): os.rename(file, file_new) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data file dissapearance.\n " @@ -2432,7 +2436,7 @@ def test_validate_corrupted_full_missing(self): os.rename(old_directory, new_directory) try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid', e.message, @@ -2476,7 +2480,7 @@ def test_validate_corrupted_full_missing(self): # second time must be provided with ID of missing backup try: - self.validate_pb(backup_dir) + self.validate_pb(backup_dir, options=["-j", "4"]) except ProbackupException as e: self.assertIn( 'WARNING: Some backups are not valid', e.message, @@ -2521,7 +2525,7 @@ def test_validate_corrupted_full_missing(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - output = self.validate_pb(backup_dir) + output = self.validate_pb(backup_dir, options=["-j", "4"]) self.assertIn( 'INFO: All backups are valid', @@ -2697,7 +2701,8 @@ def test_validate_specific_backup_with_missing_backup(self): os.rename(old_directory, new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2737,7 +2742,8 @@ def test_validate_specific_backup_with_missing_backup(self): self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2766,7 +2772,7 @@ def test_validate_specific_backup_with_missing_backup(self): os.rename(new_directory, old_directory) # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -2844,7 +2850,8 @@ def test_validate_specific_backup_with_missing_backup_1(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -2887,7 +2894,7 @@ def test_validate_specific_backup_with_missing_backup_1(self): os.rename(full_new_directory, full_old_directory) # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -2967,7 +2974,8 @@ def test_validate_with_missing_backup_1(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3010,7 +3018,8 @@ def test_validate_with_missing_backup_1(self): # Revalidate backup chain try: - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3075,7 +3084,7 @@ def test_validate_with_missing_backup_1(self): os.rename(full_new_directory, full_old_directory) # Revalidate chain - self.validate_pb(backup_dir, 'node', validate_id) + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') @@ -3151,7 +3160,7 @@ def test_validate_with_missing_backup_2(self): os.rename(full_old_directory, full_new_directory) try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3200,7 +3209,7 @@ def test_validate_with_missing_backup_2(self): # Revalidate backup chain try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of backup dissapearance.\n " @@ -3309,7 +3318,7 @@ def test_corrupt_pg_control_via_resetxlog(self): # Validate backup try: - self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of pg_control change.\n " From a0fa72eecaa253b60611201ff0ac5c02d654341c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 13 Mar 2019 17:16:05 +0300 Subject: [PATCH 0442/2107] PGPRO-1973: Report about absent or corrupted WALs only once --- src/parsexlog.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 5bea453ca..9b4323f7e 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -184,6 +184,8 @@ static XLogSegNo segno_target = 0; static XLogSegNo segno_next = 0; /* Number of segments already read by threads */ static uint32 segnum_read = 0; +/* Number of detected corrupted or absent segments */ +static uint32 segnum_corrupted = 0; static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; /* copied from timestamp.c */ @@ -900,6 +902,7 @@ RunXLogThreads(const char *archivedir, time_t target_time, segno_target = 0; GetXLogSegNo(startpoint, segno_next, segment_size); segnum_read = 0; + segnum_corrupted = 0; threads = (pthread_t *) pgut_malloc(sizeof(pthread_t) * num_threads); thread_args = (xlog_thread_arg *) pgut_malloc(sizeof(xlog_thread_arg) * num_threads); @@ -1103,6 +1106,20 @@ XLogThreadWorker(void *arg) */ if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) break; + else if (wal_consistent_read) + { + XLogSegNo segno_report; + + pthread_lock(&wal_segment_mutex); + segno_report = segno_start + segnum_read; + pthread_mutex_unlock(&wal_segment_mutex); + + /* + * Report error message if this is the first corrupted WAL. + */ + if (reader_data->xlogsegno > segno_report) + return NULL; /* otherwise just stop the thread */ + } errptr = thread_arg->startpoint ? thread_arg->startpoint : xlogreader->EndRecPtr; @@ -1232,6 +1249,20 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) */ if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) return false; + else if (wal_consistent_read) + { + XLogSegNo segno_report; + + pthread_lock(&wal_segment_mutex); + segno_report = segno_start + segnum_read; + pthread_mutex_unlock(&wal_segment_mutex); + + /* + * Report error message if this is the first corrupted WAL. + */ + if (reader_data->xlogsegno > segno_report) + return false; /* otherwise just stop the thread */ + } elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, @@ -1279,16 +1310,16 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) reader_data->thread_num); pthread_lock(&wal_segment_mutex); - segnum_current_read = segnum_read; + segnum_current_read = segnum_read + segnum_corrupted; segno = segno_target; pthread_mutex_unlock(&wal_segment_mutex); /* Other threads read all previous segments and didn't find target */ if (segnum_need <= segnum_current_read) { - /* Mark current segment as read even if it wasn't read actually */ + /* Mark current segment as corrupted */ pthread_lock(&wal_segment_mutex); - segnum_read++; + segnum_corrupted++; pthread_mutex_unlock(&wal_segment_mutex); return false; } @@ -1296,7 +1327,7 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) if (segno != 0 && segno < reader_data->xlogsegno) return true; - pg_usleep(1000000L); /* 1000 ms */ + pg_usleep(500000L); /* 500 ms */ } /* We shouldn't reach it */ From 95f4f50c86a7341d85d3ede5e45e68f8e97a22d2 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Mar 2019 14:24:18 +0300 Subject: [PATCH 0443/2107] PGPRO-1973: Improve XLogWaitForConsistency() verbose message --- src/parsexlog.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 9b4323f7e..630a15b83 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1300,8 +1300,13 @@ XLogWaitForConsistency(XLogReaderState *xlogreader) if (log_message) { - elog(VERBOSE, "Thread [%d]: Possible WAL corruption. Wait for other threads to decide is this a failure", - reader_data->thread_num); + char xlogfname[MAXFNAMELEN]; + + GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, + wal_seg_size); + + elog(VERBOSE, "Thread [%d]: Possible WAL corruption in %s. Wait for other threads to decide is this a failure", + reader_data->thread_num, xlogfname); log_message = false; } From be3b04b72810f7a0f68f51c2b178246e06e74f89 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 Mar 2019 17:55:46 +0300 Subject: [PATCH 0444/2107] tests: minor fixes in page module --- tests/page.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/page.py b/tests/page.py index a0a554de3..b0eb79bf6 100644 --- a/tests/page.py +++ b/tests/page.py @@ -711,7 +711,7 @@ def test_page_backup_with_lost_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL segment "{0}" is absent\n'.format( file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -737,7 +737,7 @@ def test_page_backup_with_lost_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL segment "{0}" is absent\n'.format( file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -829,7 +829,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -855,7 +855,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -952,7 +952,7 @@ def test_page_backup_with_alien_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( @@ -979,7 +979,7 @@ def test_page_backup_with_alien_wal_segment(self): self.assertTrue( 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and - 'could not read WAL record at' in e.message and + 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( From afa0081a41be6e0762fdc799cc6d154450a9adda Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 14 Mar 2019 18:59:28 +0300 Subject: [PATCH 0445/2107] Make parsexlog messages using capital letter --- src/parsexlog.c | 24 ++++++++++++------------ tests/validate_test.py | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 630a15b83..d9d169d40 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -390,29 +390,29 @@ validate_wal(pgBackup *backup, const char *archivedir, /* There are all needed WAL records */ if (all_wal) - elog(INFO, "backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", + elog(INFO, "Backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", last_timestamp, last_rec.rec_xid, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); /* Some needed WAL records are absent */ else { - elog(WARNING, "recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", + elog(WARNING, "Recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", last_timestamp, last_rec.rec_xid, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) time2iso(target_timestamp, lengthof(target_timestamp), target_time); if (TransactionIdIsValid(target_xid) && target_time != 0) - elog(ERROR, "not enough WAL records to time %s and xid " XID_FMT, + elog(ERROR, "Not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); else if (TransactionIdIsValid(target_xid)) - elog(ERROR, "not enough WAL records to xid " XID_FMT, + elog(ERROR, "Not enough WAL records to xid " XID_FMT, target_xid); else if (target_time != 0) - elog(ERROR, "not enough WAL records to time %s", + elog(ERROR, "Not enough WAL records to time %s", target_timestamp); else if (XRecOffIsValid(target_lsn)) - elog(ERROR, "not enough WAL records to lsn %X/%X", + elog(ERROR, "Not enough WAL records to lsn %X/%X", (uint32) (target_lsn >> 32), (uint32) (target_lsn)); } } @@ -458,11 +458,11 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, errptr = startpoint ? startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(ERROR, "could not read WAL record at %X/%X: %s", + elog(ERROR, "Could not read WAL record at %X/%X: %s", (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(ERROR, "could not read WAL record at %X/%X", + elog(ERROR, "Could not read WAL record at %X/%X", (uint32) (errptr >> 32), (uint32) (errptr)); } @@ -1042,13 +1042,13 @@ XLogThreadWorker(void *arg) else { if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", reader_data->thread_num, (uint32) (thread_arg->startpoint >> 32), (uint32) (thread_arg->startpoint), xlogreader->errormsg_buf); else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (thread_arg->startpoint >> 32), (uint32) (thread_arg->startpoint)); @@ -1125,12 +1125,12 @@ XLogThreadWorker(void *arg) thread_arg->startpoint : xlogreader->EndRecPtr; if (errormsg) - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X: %s", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", reader_data->thread_num, (uint32) (errptr >> 32), (uint32) (errptr), errormsg); else - elog(WARNING, "Thread [%d]: could not read WAL record at %X/%X", + elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", reader_data->thread_num, (uint32) (errptr >> 32), (uint32) (errptr)); diff --git a/tests/validate_test.py b/tests/validate_test.py index 0d21811d7..13b87a0bd 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -96,7 +96,7 @@ def test_validate_wal_unreal_values(self): # Validate to real time self.assertIn( - "INFO: backup validation completed successfully", + "INFO: Backup validation completed successfully", self.validate_pb( backup_dir, 'node', options=["--time={0}".format(target_time), "-j", "4"]), @@ -134,7 +134,7 @@ def test_validate_wal_unreal_values(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to time' in e.message, + 'ERROR: Not enough WAL records to time' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -148,7 +148,7 @@ def test_validate_wal_unreal_values(self): self.switch_wal_segment(node) self.assertIn( - "INFO: backup validation completed successfully", + "INFO: Backup validation completed successfully", self.validate_pb( backup_dir, 'node', options=["--xid={0}".format(target_xid), "-j", "4"]), @@ -168,7 +168,7 @@ def test_validate_wal_unreal_values(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to xid' in e.message, + 'ERROR: Not enough WAL records to xid' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1580,9 +1580,9 @@ def test_validate_corrupt_wal_between_backups(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: not enough WAL records to xid' in e.message and - 'WARNING: recovery can be done up to time' in e.message and - "ERROR: not enough WAL records to xid {0}\n".format( + 'ERROR: Not enough WAL records to xid' in e.message and + 'WARNING: Recovery can be done up to time' in e.message and + "ERROR: Not enough WAL records to xid {0}\n".format( target_xid), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 7acb3513389472ba36a98de6ec941411d0c015dd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 Mar 2019 18:25:03 +0300 Subject: [PATCH 0446/2107] tests: added retention.RetentionTest.test_retention_window_4 --- tests/retention_test.py | 63 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/retention_test.py b/tests/retention_test.py index 34a728d23..191438814 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -159,10 +159,73 @@ def test_retention_window_3(self): # Purge backups self.delete_expired( backup_dir, 'node', options=['--retention-window=1']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) + # count wal files in ARCHIVE + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_retention_window_4(self): + """purge all backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + # Take FULL BACKUPs + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + backup_id_3 = self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_pb(backup_dir, 'node', backup_id_2) + self.delete_pb(backup_dir, 'node', backup_id_3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # count wal files in ARCHIVE + wals_dir = os.path.join(backup_dir, 'wal', 'node') + n_wals = len(os.listdir(wals_dir)) + + self.assertTrue(n_wals > 0) + + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + # count again + n_wals = len(os.listdir(wals_dir)) + self.assertTrue(n_wals == 0) + # Clean after yourself self.del_test_dir(module_name, fname) From 038ffd6ace227c7eed062ba897f9246c9df80cb8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 16 Mar 2019 15:38:13 +0300 Subject: [PATCH 0447/2107] help: update help about --compress option --- src/help.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/help.c b/src/help.c index 84b56678f..549a1f28f 100644 --- a/src/help.c +++ b/src/help.c @@ -259,9 +259,9 @@ help_backup(void) printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_("\n Compression options:\n")); - printf(_(" --compress compress data files\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'pglz', 'none' (default: zlib)\n")); + printf(_(" available options: 'zlib', 'pglz', 'none' (default: none)\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -538,8 +538,9 @@ help_set_config(void) printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_("\n Compression options:\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none'\n")); + printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -611,9 +612,9 @@ help_archive_push(void) printf(_(" relative path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the WAL file to retrieve from the server\n")); - printf(_(" --compress compress WAL file during archiving\n")); + printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','none'\n")); + printf(_(" available options: 'zlib', 'none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); printf(_(" --overwrite overwrite archived WAL file\n")); From 1923560fd990ef848b7394ebf18b07e7f8905d05 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Mar 2019 22:14:53 +0300 Subject: [PATCH 0448/2107] tests: rename module retention_test to retention --- tests/{retention_test.py => retention.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{retention_test.py => retention.py} (100%) diff --git a/tests/retention_test.py b/tests/retention.py similarity index 100% rename from tests/retention_test.py rename to tests/retention.py From 87b6b9012a458a8da2302b93c60f75cd4b774a2d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 Mar 2019 02:16:42 +0300 Subject: [PATCH 0449/2107] tests: added test_retention_interleaved_incremental_chains --- tests/helpers/ptrack_helpers.py | 23 ++++++ tests/retention.py | 119 +++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bcce8448a..f61e8d18d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -938,6 +938,29 @@ def set_replica( ) master.reload() + def change_backup_status(self, backup_dir, instance, backup_id, status): + + control_file_path = os.path.join( + backup_dir, 'backups', instance, backup_id, 'backup.control') + + with open(control_file_path, 'r') as f: + actual_control = f.read() + + new_control_file = '' + for line in actual_control.splitlines(): + if line.startswith('status'): + line = 'status = {0}'.format(status) + new_control_file += line + new_control_file += '\n' + + with open(control_file_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + with open(control_file_path, 'r') as f: + actual_control = f.read() + def wrong_wal_clean(self, node, wal_size): wals_dir = os.path.join(self.backup_dir(node), 'wal') wals = [ diff --git a/tests/retention.py b/tests/retention.py index 191438814..ac679ba22 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -67,7 +67,7 @@ def test_retention_redundancy_1(self): # Clean after yourself self.del_test_dir(module_name, fname) -# @unittest.skip("123") + #@unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -121,7 +121,7 @@ def test_retention_window_2(self): # Clean after yourself self.del_test_dir(module_name, fname) -# @unittest.skip("123") + #@unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -229,3 +229,118 @@ def test_retention_window_4(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_retention_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + # Take FULL BACKUPs + + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Change PAGEa2 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # Change PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 092771792d02ebc4ac021fd34d5bad98d287da5c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Mar 2019 17:15:46 +0300 Subject: [PATCH 0450/2107] tests: for pgpro-2573 --- tests/helpers/ptrack_helpers.py | 2 +- tests/retention.py | 915 +++++++++++++++++++++++++++++++- 2 files changed, 909 insertions(+), 8 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f61e8d18d..0fc054440 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -837,7 +837,7 @@ def delete_pb( def delete_expired( self, backup_dir, instance, options=[], old_binary=False): cmd_list = [ - 'delete', '--expired', '--wal', + 'delete', '-B', backup_dir, '--instance={0}'.format(instance) ] diff --git a/tests/retention.py b/tests/retention.py index ac679ba22..bd76af26b 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -40,7 +40,8 @@ def test_retention_redundancy_1(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - log = self.delete_expired(backup_dir, 'node') + log = self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Check that WAL segments were deleted @@ -115,7 +116,7 @@ def test_retention_window_2(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) # Purge backups - self.delete_expired(backup_dir, 'node') + self.delete_expired(backup_dir, 'node', options=['--expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Clean after yourself @@ -158,7 +159,7 @@ def test_retention_window_3(self): # Purge backups self.delete_expired( - backup_dir, 'node', options=['--retention-window=1']) + backup_dir, 'node', options=['--retention-window=1', '--expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) @@ -207,7 +208,8 @@ def test_retention_window_4(self): # Purge backups self.delete_expired( - backup_dir, 'node', options=['--retention-window=1']) + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) @@ -221,7 +223,8 @@ def test_retention_window_4(self): self.assertTrue(n_wals > 0) self.delete_expired( - backup_dir, 'node', options=['--retention-window=1']) + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--wal']) # count again n_wals = len(os.listdir(wals_dir)) @@ -231,7 +234,7 @@ def test_retention_window_4(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_retention_interleaved_incremental_chains(self): + def test_window_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -335,7 +338,8 @@ def test_retention_interleaved_incremental_chains(self): self.delete_expired( - backup_dir, 'node', options=['--retention-window=1']) + backup_dir, 'node', + options=['--retention-window=1', '--expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) @@ -344,3 +348,900 @@ def test_retention_interleaved_incremental_chains(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_redundancy_expire_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Change PAGEa2 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # Change PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + self.delete_expired( + backup_dir, 'node', + options=['--retention-redundancy=1', '--expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_merge_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Change PAGEa2 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # Change PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--merge-expired']) + + self.assertIn( + "Merge incremental chain between FULL backup {0} and backup {1}".format( + backup_id_a, page_id_a2), + output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_a1, backup_id_a), output) + + self.assertIn( + "Rename {0} to {1}".format( + backup_id_a, page_id_a1), output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_a2, page_id_a1), output) + + self.assertIn( + "Rename {0} to {1}".format( + page_id_a1, page_id_a2), output) + + self.assertIn( + "Merge incremental chain between FULL backup {0} and backup {1}".format( + backup_id_b, page_id_b2), + output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_b1, backup_id_b), output) + + self.assertIn( + "Rename {0} to {1}".format( + backup_id_b, page_id_b1), output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_b2, page_id_b1), output) + + self.assertIn( + "Rename {0} to {1}".format( + page_id_b1, page_id_b2), output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_merge_interleaved_incremental_chains_1(self): + """ + PAGEb3 + PAGEb2 + PAGEb1 + PAGEa1 + FULLb + FULLa + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgdata_a1 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgdata_b3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench.wait() + + # PAGEb3 OK + # PAGEb2 OK + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb3 OK + # PAGEb2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a1, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--merge-expired']) + + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'FULL') + + node.cleanup() + + # Data correctness of PAGEa3 + self.restore_node(backup_dir, 'node', node, backup_id=page_id_a1) + pgdata_restored_a1 = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata_a1, pgdata_restored_a1) + + node.cleanup() + + # Data correctness of PAGEb3 + self.restore_node(backup_dir, 'node', node, backup_id=page_id_b3) + pgdata_restored_b3 = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata_b3, pgdata_restored_b3) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_merge_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + -----------------------------retention window + PAGEb2 / + | PAGEa2 / should be deleted + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change PAGEa1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # Change PAGEa2 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb2 and PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + print("Backups {0} and {1} are children of {2}".format( + page_id_a3, page_id_a2, page_id_a1)) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', '--expired', + '--merge-expired', '--log-level-console=log']) + + print(output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Merging chain A + self.assertIn( + "Merge incremental chain between FULL backup {0} and backup {1}".format( + backup_id_a, page_id_a3), + output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_a1, backup_id_a), output) + + self.assertIn( + "INFO: Rename {0} to {1}".format( + backup_id_a, page_id_a1), output) + + self.assertIn( + "WARNING: Backup {0} has multiple valid descendants. " + "Automatic merge is not possible.".format( + page_id_a1), output) + + # Merge chain B + self.assertIn( + "Merge incremental chain between FULL backup {0} and backup {1}".format( + backup_id_b, page_id_b3), + output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_b1, backup_id_b), output) + + self.assertIn( + "INFO: Rename {0} to {1}".format( + backup_id_b, page_id_b1), output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_b2, page_id_b1), output) + + self.assertIn( + "INFO: Rename {0} to {1}".format( + page_id_b1, page_id_b2), output) + + self.assertIn( + "Merging backup {0} with backup {1}".format( + page_id_b3, page_id_b2), output) + + self.assertIn( + "INFO: Rename {0} to {1}".format( + page_id_b2, page_id_b3), output) + + # this backup deleted because it is not guarded by retention + self.assertIn( + "INFO: Delete: {0}".format( + page_id_a1), output) + + + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_a3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'PAGE') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'FULL') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_chains(self): + """ + PAGE + -------window + PAGE + PAGE + FULL + PAGE + PAGE + FULL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Chain A + backup_id_a = self.backup_node(backup_dir, 'node', node) + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Chain B + backup_id_b = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', '--expired', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + def test_window_chains_1(self): + """ + PAGE + -------window + PAGE + PAGE + FULL + PAGE + PAGE + FULL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Chain A + backup_id_a = self.backup_node(backup_dir, 'node', node) + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Chain B + backup_id_b = self.backup_node(backup_dir, 'node', node) + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.assertIn( + "There are no backups to delete by retention policy", + output) + + self.assertIn( + "Retention merging finished", + output) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + self.assertIn( + "There are no backups to merge by retention policy", + output) + + self.assertIn( + "Purging finished", + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_error_backups(self): + """ + PAGE ERROR + -------window + PAGE ERROR + PAGE ERROR + PAGE ERROR + FULL ERROR + FULL + -------redundancy + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a1 = self.backup_node(backup_dir, 'node', node) + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULLb backup status to ERROR + #self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') From f7205ea1ff97a098d4fd223eb2b66ed996ecb796 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Mar 2019 17:26:14 +0300 Subject: [PATCH 0451/2107] PGPRO-2573: incrementally updated backups via merge --- src/backup.c | 4 +- src/catalog.c | 88 +++++++- src/delete.c | 521 +++++++++++++++++++++++++++++++++++++-------- src/merge.c | 5 +- src/pg_probackup.c | 17 +- src/pg_probackup.h | 8 + 6 files changed, 543 insertions(+), 100 deletions(-) diff --git a/src/backup.c b/src/backup.c index bd5ee5908..7b99e1af3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1068,8 +1068,8 @@ do_backup(time_t start_time) * After successfil backup completion remove backups * which are expired according to retention policies */ - if (delete_expired || delete_wal) - do_retention_purge(); + if (delete_expired || merge_expired || delete_wal) + do_retention(); return 0; } diff --git a/src/catalog.c b/src/catalog.c index 86b51c872..12bf2e1e0 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1016,6 +1016,53 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, make_native_path(path); } +/* + * Check if multiple backups consider target backup to be their direct parent + */ +bool +is_prolific(parray *backup_list, pgBackup *target_backup) +{ + int i; + int child_counter = 0; + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); + + /* consider only OK and DONE backups */ + if (tmp_backup->parent_backup == target_backup->start_time && + (tmp_backup->status == BACKUP_STATUS_OK || + tmp_backup->status == BACKUP_STATUS_DONE)) + child_counter++; + } + + if (child_counter > 1) + return true; + else + return false; +} + +/* + * Check if target_backup in backup_list. Use if bsearch cannot be trusted. + */ +bool +in_backup_list(parray *backup_list, pgBackup *target_backup) +{ + int i; + + if (!target_backup) + elog(ERROR, "Target backup cannot be NULL"); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); + + if (tmp_backup->start_time == target_backup->start_time) + return true; + } + return false; +} + /* * Find parent base FULL backup for current backup using parent_backup_link */ @@ -1025,6 +1072,7 @@ find_parent_full_backup(pgBackup *current_backup) pgBackup *base_full_backup = NULL; base_full_backup = current_backup; + /* sanity */ if (!current_backup) elog(ERROR, "Target backup cannot be NULL"); @@ -1034,12 +1082,44 @@ find_parent_full_backup(pgBackup *current_backup) } if (base_full_backup->backup_mode != BACKUP_MODE_FULL) - elog(ERROR, "Failed to find FULL backup parent for %s", + { + elog(WARNING, "Failed to find FULL backup parent for %s", base36enc(current_backup->start_time)); + return NULL; + } return base_full_backup; } +/* + * Find closest child of target_backup. If there are several direct + * offsprings in backup_list, then first win. + */ +pgBackup* +find_direct_child(parray *backup_list, pgBackup *target_backup) +{ + int i; + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); + + if (tmp_backup->backup_mode == BACKUP_MODE_FULL) + continue; + + /* Consider only OK and DONE children */ + if (tmp_backup->parent_backup == target_backup->start_time && + (tmp_backup->status == BACKUP_STATUS_OK || + tmp_backup->status == BACKUP_STATUS_DONE)) + { + return tmp_backup; + } + } + elog(WARNING, "Failed to find a direct child for backup %s", + base36enc(target_backup->start_time)); + return NULL; +} + /* * Interate over parent chain and look for any problems. * Return 0 if chain is broken. @@ -1146,6 +1226,6 @@ get_backup_index_number(parray *backup_list, pgBackup *backup) if (tmp_backup->start_time == backup->start_time) return i; } - elog(ERROR, "Failed to find backup %s", base36enc(backup->start_time)); - return 0; -} + elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time)); + return -1; +} \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 287e73776..78503fb0f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -16,6 +16,11 @@ static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, uint32 xlog_seg_size); +static void do_retention_internal(void); +static void do_retention_wal(void); + +static bool backup_deleted = false; /* At least one backup was deleted */ +static bool backup_merged = false; /* At least one merge was enacted */ void do_delete(time_t backup_id) @@ -57,6 +62,10 @@ do_delete(time_t backup_id) } else if (target_backup) { + /* TODO: Current algorithm is imperfect, it assume that backup + * can sire only one incremental chain + */ + if (backup->backup_mode != BACKUP_MODE_FULL && backup->parent_backup == parent_id) { @@ -115,142 +124,484 @@ do_delete(time_t backup_id) parray_free(backup_list); } -/* - * Remove backups by retention policy. Retention policy is configured by - * retention_redundancy and retention_window variables. - */ -int -do_retention_purge(void) +int do_retention(void) { - parray *backup_list; - size_t i; - XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TimeLineID oldest_tli = 0; - bool backup_deleted = false; /* At least one backup was deleted */ + bool retention_is_set = false; /* At least one retention policy is set */ - if (delete_expired) + if (delete_expired || merge_expired) { if (instance_config.retention_redundancy > 0) elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy); if (instance_config.retention_window > 0) elog(LOG, "WINDOW=%u", instance_config.retention_window); - if (instance_config.retention_redundancy == 0 - && instance_config.retention_window == 0) + if (instance_config.retention_redundancy == 0 && + instance_config.retention_window == 0) { + /* Retention is disabled but we still can cleanup + * failed backups and wal + */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) return 0; } + else + /* At least one retention policy is active */ + retention_is_set = true; } + if (retention_is_set && ((delete_expired || merge_expired) || dry_run)) + do_retention_internal(); + + if (delete_wal && !dry_run) + do_retention_wal(); + + if (!backup_merged) + elog(INFO, "There are no backups to merge by retention policy"); + + if (backup_deleted) + elog(INFO, "Purging finished"); + else + elog(INFO, "There are no backups to delete by retention policy"); + + return 0; + +} + +/* + * Merge and purge backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + * + * Invalid backups handled in Oracle style, so invalid backups are ignored + * for the purpose of retention fulfillment, + * i.e. CORRUPT full backup do not taken in account when deteremine + * which FULL backup should be keeped for redundancy obligation(only valid do), + * but if invalid backup is not guarded by retention - it is removed + */ +static void +do_retention_internal(void) +{ + parray *backup_list = NULL; + parray *to_purge_list = parray_new(); + parray *to_keep_list = parray_new(); + int i; + int j; + time_t current_time; + bool backup_list_is_empty = false; + + /* For retention calculation */ + uint32 n_full_backups = 0; + int cur_full_backup_num = 0; + time_t days_threshold = 0; + + /* For fancy reporting */ + float actual_window = 0; + /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) + backup_list_is_empty = true; + + if (backup_list_is_empty) { - elog(INFO, "backup list is empty, purging won't be executed"); - return 0; + elog(WARNING, "Backup list is empty, purging won't be executed"); + return; } - /* Find target backups to be deleted */ - if (delete_expired && - (instance_config.retention_redundancy > 0 || - instance_config.retention_window > 0)) + /* Get current time */ + current_time = time(NULL); + + /* Calculate n_full_backups and days_threshold */ + if (!backup_list_is_empty) { - bool keep_next_backup = false; /* Do not delete first full backup */ - time_t days_threshold; - uint32 backup_num = 0; + if (instance_config.retention_redundancy > 0) + { + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - days_threshold = time(NULL) - + /* Consider only valid backups for Redundancy */ + if (instance_config.retention_redundancy > 0 && + backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) + { + n_full_backups++; + } + } + } + + if (instance_config.retention_window > 0) + { + days_threshold = current_time - (instance_config.retention_window * 60 * 60 * 24); + } + } + + elog(INFO, "Evaluate backups by retention"); + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + { + + pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - for (i = 0; i < parray_num(backup_list); i++) + /* Remember the serial number of latest valid FULL backup */ + if (backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - uint32 backup_num_evaluate = backup_num; + cur_full_backup_num++; + } - /* Consider only validated and correct backups */ - if (backup->status != BACKUP_STATUS_OK) - continue; - /* - * When a valid full backup was found, we can delete the - * backup that is older than it using the number of generations. + /* Check if backup in needed by retention policy */ + if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) && + (instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num))) + { + /* This backup is not guarded by retention + * + * Redundancy = 1 + * FULL CORRUPT in retention (not count toward redundancy limit) + * FULL in retention + * ------retention redundancy ------- + * PAGE4 in retention + * ------retention window ----------- + * PAGE3 in retention + * PAGE2 out of retention + * PAGE1 out of retention + * FULL out of retention <- We are here + * FULL CORRUPT out of retention */ - if (backup->backup_mode == BACKUP_MODE_FULL) - backup_num++; - - /* Evaluate retention_redundancy if this backup is eligible for removal */ - if (keep_next_backup || - instance_config.retention_redundancy >= backup_num_evaluate + 1 || - (instance_config.retention_window > 0 && - backup->recovery_time >= days_threshold)) + + /* Add backup to purge_list */ + elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time)); + parray_append(to_purge_list, backup); + continue; + } + + /* Do not keep invalid backups by retention */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + continue; + + elog(VERBOSE, "Mark backup %s for retention.", base36enc(backup->start_time)); + parray_append(to_keep_list, backup); + } + + /* Message about retention state of backups + * TODO: Float is ugly, rewrite somehow. + */ + + /* sort keep_list and purge list */ + parray_qsort(to_keep_list, pgBackupCompareIdDesc); + parray_qsort(to_purge_list, pgBackupCompareIdDesc); + + cur_full_backup_num = 1; + for (i = 0; i < parray_num(backup_list); i++) + { + char *action = "Ignore"; + + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + + if (in_backup_list(to_keep_list, backup)) + action = "Keep"; + + if (in_backup_list(to_purge_list, backup)) + action = "Purge"; + + if (backup->recovery_time == 0) + actual_window = 0; + else + actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24); + + elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %.2fd/%ud. %s", + base36enc(backup->start_time), + pgBackupGetBackupMode(backup), + status2str(backup->status), + cur_full_backup_num, + instance_config.retention_redundancy, + actual_window, instance_config.retention_window, + action); + + if (backup->backup_mode == BACKUP_MODE_FULL) + cur_full_backup_num++; + } + + if (dry_run) + goto finish; + + /* Extreme example of keep_list + * + * FULLc <- keep + * PAGEb2 <- keep + * PAGEb1 <- keep + * PAGEa2 <- keep + * PAGEa1 <- keep + * FULLb <- in purge_list + * FULLa <- in purge_list + */ + + /* Go to purge */ + if (delete_expired && !merge_expired) + goto purge; + + /* IMPORTANT: we can merge to only those FULL backups, that are NOT + * guarded by retention and only from those incremental backups that + * are guarded by retention !!! + */ + + /* Merging happens here */ + for (i = 0; i < parray_num(to_keep_list); i++) + { + char *keep_backup_id = NULL; + pgBackup *full_backup = NULL; + parray *merge_list = NULL; + + pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); + + /* keep list may shrink during merge */ + if (!keep_backup) + break; + + elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time)); + + /* In keep list we are looking for incremental backups */ + if (keep_backup->backup_mode == BACKUP_MODE_FULL) + continue; + + /* Retain orphan backups in keep_list */ + if (!keep_backup->parent_backup_link) + continue; + + /* If parent of current backup is also in keep list, go to the next */ + if (in_backup_list(to_keep_list, keep_backup->parent_backup_link)) + { + /* make keep list a bit sparse */ + elog(INFO, "Sparsing keep list, remove %s", base36enc(keep_backup->start_time)); + parray_remove(to_keep_list, i); + i--; + continue; + } + + elog(INFO, "Lookup parents for backup %s", + base36enc(keep_backup->start_time)); + + /* Got valid incremental backup, find its FULL ancestor */ + full_backup = find_parent_full_backup(keep_backup); + + /* Failed to find parent */ + if (!full_backup) + { + elog(WARNING, "Failed to find FULL parent for %s", base36enc(keep_backup->start_time)); + continue; + } + + /* Check that ancestor is in purge_list */ + if (!in_backup_list(to_purge_list, full_backup)) + { + elog(WARNING, "Skip backup %s for merging, " + "because his FULL parent is not marked for purge", base36enc(keep_backup->start_time)); + continue; + } + + /* FULL backup in purge list, thanks to sparsing of keep_list current backup is + * final target for merge, but there could be intermediate incremental + * backups from purge_list. + */ + + keep_backup_id = base36enc_dup(keep_backup->start_time); + elog(INFO, "Merge incremental chain between FULL backup %s and backup %s", + base36enc(full_backup->start_time), keep_backup_id); + + merge_list = parray_new(); + + /* Form up a merge list */ + while(keep_backup->parent_backup_link) + { + parray_append(merge_list, keep_backup); + keep_backup = keep_backup->parent_backup_link; + } + + /* sanity */ + if (!merge_list) + continue; + + /* sanity */ + if (parray_num(merge_list) == 0) + { + parray_free(merge_list); + continue; + } + + /* In the end add FULL backup for easy locking */ + parray_append(merge_list, full_backup); + + /* Remove FULL backup from purge list */ + parray_rm(to_purge_list, full_backup, pgBackupCompareId); + + /* Lock merge chain */ + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); + + for (j = parray_num(merge_list) - 1; j > 0; j--) + { + pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j - 1 ); + + + /* Consider this extreme case */ + // PAGEa1 PAGEb1 both valid + // \ / + // FULL + + /* Check that FULL backup do not has multiple descendants */ + if (is_prolific(backup_list, full_backup)) { - /* Save LSN and Timeline to remove unnecessary WAL segments */ - oldest_lsn = backup->start_lsn; - oldest_tli = backup->tli; + elog(WARNING, "Backup %s has multiple valid descendants. " + "Automatic merge is not possible.", base36enc(full_backup->start_time)); + break; + } - /* Save parent backup of this incremental backup */ - if (backup->backup_mode != BACKUP_MODE_FULL) - keep_next_backup = true; - /* - * Previous incremental backup was kept or this is first backup - * so do not delete this backup. - */ - else - keep_next_backup = false; + merge_backups(full_backup, from_backup); + backup_merged = true; + + /* Try to remove merged incremental backup from both keep and purge lists */ + parray_rm(to_purge_list, from_backup, pgBackupCompareId); + + if (parray_rm(to_keep_list, from_backup, pgBackupCompareId) && (i >= 0)) + i--; + } + + /* Cleanup */ + parray_free(merge_list); + } + + elog(INFO, "Retention merging finished"); + + if (!delete_expired) + goto finish; +/* Do purging here */ +purge: + + /* Remove backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window + * Remove only backups, that do not have children guarded by retention + * + * TODO: We do not consider the situation if child is marked for purge + * but parent isn`t. Maybe something bad happened with time on server? + */ + + for (j = 0; j < parray_num(to_purge_list); j++) + { + bool purge = true; + + pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j); + + elog(LOG, "Consider backup %s for purge", + base36enc(delete_backup->start_time)); + + /* Evaluate marked for delete backup against every backup in keep list. + * If marked for delete backup is recognized as parent of one of those, + * then this backup should not be deleted. + */ + for (i = 0; i < parray_num(to_keep_list); i++) + { + char *keeped_backup_id; + + pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); + + /* Full backup cannot be a descendant */ + if (keep_backup->backup_mode == BACKUP_MODE_FULL) continue; - } - /* - * If the backup still is used do not interrupt go to the next - * backup. - */ - if (!lock_backup(backup)) + keeped_backup_id = base36enc_dup(keep_backup->start_time); + + elog(LOG, "Check if backup %s is parent of backup %s", + base36enc(delete_backup->start_time), keeped_backup_id); + + if (is_parent(delete_backup->start_time, keep_backup, true)) { - elog(WARNING, "Cannot lock backup %s directory, skip purging", - base36enc(backup->start_time)); - continue; + + /* We must not delete this backup, evict it from purge list */ + elog(LOG, "Retain backup %s from purge because his " + "descendant %s is guarded by retention", + base36enc(delete_backup->start_time), keeped_backup_id); + + purge = false; + break; } + } - /* Delete backup and update status to DELETED */ - delete_backup_files(backup); - backup_deleted = true; - keep_next_backup = false; /* reset it */ + /* Retain backup */ + if (!purge) + continue; + + /* Actual purge */ + if (!lock_backup(delete_backup)) + { + /* If the backup still is used, do not interrupt and go to the next */ + elog(WARNING, "Cannot lock backup %s directory, skip purging", + base36enc(delete_backup->start_time)); + continue; } + + /* Delete backup and update status to DELETED */ + delete_backup_files(delete_backup); + backup_deleted = true; + } - /* - * If oldest_lsn and oldest_tli weren`t set because previous step was skipped - * then set them now if we are going to purge WAL - */ - if (delete_wal && (XLogRecPtrIsInvalid(oldest_lsn))) +finish: + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + parray_free(to_keep_list); + parray_free(to_purge_list); + +} + +/* Purge WAL */ +static void +do_retention_wal(void) +{ + parray *backup_list = NULL; + + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TimeLineID oldest_tli = 0; + bool backup_list_is_empty = false; + + /* Get new backup_list. Should we */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + + if (parray_num(backup_list) == 0) + backup_list_is_empty = true; + + /* Save LSN and Timeline to remove unnecessary WAL segments */ + if (!backup_list_is_empty) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) - 1); - oldest_lsn = backup->start_lsn; + pgBackup *backup = NULL; + /* Get LSN and TLI of oldest alive backup */ + backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1); + oldest_tli = backup->tli; + oldest_lsn = backup->start_lsn; } /* Be paranoid */ - if (XLogRecPtrIsInvalid(oldest_lsn)) + if (!backup_list_is_empty && XLogRecPtrIsInvalid(oldest_lsn)) elog(ERROR, "Not going to purge WAL because LSN is invalid"); + /* If no backups left after retention, do not drop all WAL files, + * because of race conditions there is a risk of messing up + * concurrent backup wals. + * But it is sort of ok to drop all WAL files if backup_list is empty + * from the very beginning, because a risk of concurrent backup + * is much smaller. + */ + /* Purge WAL files */ - if (delete_wal) - { - delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); - } + delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); /* Cleanup */ parray_walk(backup_list, pgBackupFree); parray_free(backup_list); - - if (backup_deleted) - elog(INFO, "Purging finished"); - else - elog(INFO, "There are no backups to delete by retention policy"); - - return 0; } /* diff --git a/src/merge.c b/src/merge.c index 5726e36c1..1a1405485 100644 --- a/src/merge.c +++ b/src/merge.c @@ -34,7 +34,6 @@ typedef struct int ret; } merge_files_arg; -static void merge_backups(pgBackup *backup, pgBackup *next_backup); static void *merge_files(void *arg); static void reorder_external_dirs(pgBackup *to_backup, parray *to_external, @@ -159,7 +158,7 @@ do_merge(time_t backup_id) * - remove unnecessary directories and files from to_backup * - update metadata of from_backup, it becames FULL backup */ -static void +void merge_backups(pgBackup *to_backup, pgBackup *from_backup) { char *to_backup_id = base36enc_dup(to_backup->start_time), @@ -652,7 +651,7 @@ merge_files(void *arg) file->compress_alg = to_backup->compress_alg; if (file->write_size != BYTES_INVALID) - elog(LOG, "Merged file \"%s\": " INT64_FORMAT " bytes", + elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); /* Restore relative path */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 4f19f89e7..239b1ce53 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -92,7 +92,9 @@ bool skip_external_dirs = false; /* delete options */ bool delete_wal = false; bool delete_expired = false; +bool merge_expired = false; bool force_delete = false; +bool dry_run = false; /* compression options */ bool compress_shortcut = false; @@ -140,6 +142,8 @@ static ConfigOption cmd_options[] = { 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, + { 'b', 235, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, + { 'b', 237, "dry-run", &dry_run, SOURCE_CMD_STRICT }, /* TODO not completed feature. Make it unavailiable from user level { 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */ /* restore options */ @@ -509,12 +513,13 @@ main(int argc, char *argv[]) case DELETE_CMD: if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); - if (!delete_expired && !delete_wal && !backup_id_string) - elog(ERROR, "You must specify at least one of the delete options: --expired |--wal |--backup_id"); - if (delete_wal && !delete_expired && !backup_id_string) - return do_retention_purge(); - if (delete_expired) - return do_retention_purge(); + if (merge_expired && backup_id_string) + elog(ERROR, "You cannot specify --merge-expired and --backup-id options together"); + if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string) + elog(ERROR, "You must specify at least one of the delete options: " + "--expired |--wal |--merge-expired |--delete-invalid |--backup_id"); + if (!backup_id_string) + return do_retention(); else do_delete(current.backup_id); break; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2cceaaedb..796e1be36 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -385,7 +385,9 @@ extern bool skip_external_dirs; /* delete options */ extern bool delete_wal; extern bool delete_expired; +extern bool merge_expired; extern bool force_delete; +extern bool dry_run; /* compression options */ extern bool compress_shortcut; @@ -429,6 +431,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( /* in merge.c */ extern void do_merge(time_t backup_id); +extern void merge_backups(pgBackup *backup, pgBackup *next_backup); /* in init.c */ extern int do_init(void); @@ -451,6 +454,7 @@ extern int do_show(time_t requested_backup_id); /* in delete.c */ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); +extern int do_retention(void); extern int do_retention_purge(void); extern int do_delete_instance(void); @@ -496,10 +500,14 @@ extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); +extern int pgBackupCompareIdEqual(const void *l, const void *r); +extern pgBackup* find_direct_child(parray *backup_list, pgBackup *target_backup); extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); +extern bool is_prolific(parray *backup_list, pgBackup *target_backup); +extern bool in_backup_list(parray *backup_list, pgBackup *target_backup); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS From 281f34d770d286a3e5c74580d7370ee1eb061aec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Mar 2019 17:37:01 +0300 Subject: [PATCH 0452/2107] PGPRO-2573: cleanup comments --- src/delete.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/delete.c b/src/delete.c index 78503fb0f..45131b93e 100644 --- a/src/delete.c +++ b/src/delete.c @@ -588,14 +588,6 @@ do_retention_wal(void) if (!backup_list_is_empty && XLogRecPtrIsInvalid(oldest_lsn)) elog(ERROR, "Not going to purge WAL because LSN is invalid"); - /* If no backups left after retention, do not drop all WAL files, - * because of race conditions there is a risk of messing up - * concurrent backup wals. - * But it is sort of ok to drop all WAL files if backup_list is empty - * from the very beginning, because a risk of concurrent backup - * is much smaller. - */ - /* Purge WAL files */ delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); From 3256dd198895ad64b2218399d58da596e44fe620 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Mar 2019 17:47:02 +0300 Subject: [PATCH 0453/2107] PGPRO-2573: minor comment fixes --- src/delete.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/delete.c b/src/delete.c index 45131b93e..4ae1599e1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -138,8 +138,7 @@ int do_retention(void) if (instance_config.retention_redundancy == 0 && instance_config.retention_window == 0) { - /* Retention is disabled but we still can cleanup - * failed backups and wal + /* Retention is disabled but we still can cleanup wal */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) @@ -335,17 +334,29 @@ do_retention_internal(void) * PAGEb1 <- keep * PAGEa2 <- keep * PAGEa1 <- keep - * FULLb <- in purge_list - * FULLa <- in purge_list + * FULLb <- in purge_list + * FULLa <- in purge_list */ /* Go to purge */ if (delete_expired && !merge_expired) goto purge; - /* IMPORTANT: we can merge to only those FULL backups, that are NOT - * guarded by retention and only from those incremental backups that - * are guarded by retention !!! + /* IMPORTANT: we can merge to only those FULL backup, that is NOT + * guarded by retention and final targets of such merges must be + * incremental backup that is guarded by retention !!! + * + * + * PAGE4 E + * PAGE3 D + * --------retention window --- + * PAGE2 C + * PAGE1 B + * FULL A + * + * after retention merge: + * PAGE4 E + * FULL D */ /* Merging happens here */ From a27cf5911f2eba1fb744b7a8fe7fb576e623fd6b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Mar 2019 00:54:21 +0300 Subject: [PATCH 0454/2107] minor changes to --help about --external-dirs option --- src/help.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/help.c b/src/help.c index 549a1f28f..8843f1d23 100644 --- a/src/help.c +++ b/src/help.c @@ -226,7 +226,8 @@ help_backup(void) printf(_(" --progress show progress\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_(" -E --external-dirs=external-directory-path\n")); - printf(_(" backup some directory not from pgdata \n")); + printf(_(" backup some directories not from pgdata \n")); + printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -508,7 +509,8 @@ help_set_config(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -E --external-dirs=external-directory-path\n")); - printf(_(" backup some directory not from pgdata \n")); + printf(_(" backup some directories not from pgdata \n")); + printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -583,7 +585,8 @@ help_add_instance(void) printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --instance=instance_name name of the new instance\n")); printf(_(" -E --external-dirs=external-directory-path\n")); - printf(_(" backup some directory not from pgdata \n")); + printf(_(" backup some directories not from pgdata \n")); + printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); } static void From 081a0b317c701c0c01850459bcc5bf39bc385d05 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Mar 2019 00:55:30 +0300 Subject: [PATCH 0455/2107] tests: add pgpro2068 --- tests/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/__init__.py b/tests/__init__.py index 3160d223e..2b72b3b9a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,6 +39,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(validate_test)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(pgpro2068)) suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(external)) From 941b9c42679fdb64f742fc2cee2cfbbe08aa5d91 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Mar 2019 11:32:15 +0300 Subject: [PATCH 0456/2107] add TODO to restore comments --- src/restore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/restore.c b/src/restore.c index ae379e9f5..c84777744 100644 --- a/src/restore.c +++ b/src/restore.c @@ -414,6 +414,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required + * TODO: before restore - lock entire parent chain */ if (is_restore) { From c5c420ceab080b4dd0b3b94b4dc79b7ace0d7956 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Mar 2019 20:56:55 +0300 Subject: [PATCH 0457/2107] tests: rename module restore_test to restore --- tests/__init__.py | 8 ++++---- tests/{restore_test.py => restore.py} | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename tests/{restore_test.py => restore.py} (100%) diff --git a/tests/__init__.py b/tests/__init__.py index 2b72b3b9a..0c2d18d37 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,8 @@ import unittest from . import init_test, merge, option_test, show_test, compatibility, \ - backup_test, delete_test, delta, restore_test, validate_test, \ - retention_test, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ + backup_test, delete_test, delta, restore, validate_test, \ + retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ locking, remote, external @@ -32,8 +32,8 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(ptrack)) suite.addTests(loader.loadTestsFromModule(remote)) suite.addTests(loader.loadTestsFromModule(replica)) - suite.addTests(loader.loadTestsFromModule(restore_test)) - suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(restore)) + suite.addTests(loader.loadTestsFromModule(retention)) suite.addTests(loader.loadTestsFromModule(show_test)) suite.addTests(loader.loadTestsFromModule(snapfs)) suite.addTests(loader.loadTestsFromModule(validate_test)) diff --git a/tests/restore_test.py b/tests/restore.py similarity index 100% rename from tests/restore_test.py rename to tests/restore.py From 0726ca63db44dad7774ebdf1eb7b1fb514cf2d75 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Mar 2019 10:10:18 +0300 Subject: [PATCH 0458/2107] tests: minor fixes --- tests/retention.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index bd76af26b..e1104b3a8 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -218,13 +218,13 @@ def test_retention_window_4(self): # count wal files in ARCHIVE wals_dir = os.path.join(backup_dir, 'wal', 'node') - n_wals = len(os.listdir(wals_dir)) + # n_wals = len(os.listdir(wals_dir)) - self.assertTrue(n_wals > 0) + # self.assertTrue(n_wals > 0) - self.delete_expired( - backup_dir, 'node', - options=['--retention-window=1', '--expired', '--wal']) + # self.delete_expired( + # backup_dir, 'node', + # options=['--retention-window=1', '--expired', '--wal']) # count again n_wals = len(os.listdir(wals_dir)) From a6dc140702e8be7cf1c1a5050badb62cf3c4914d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Mar 2019 10:16:02 +0300 Subject: [PATCH 0459/2107] PGPRO-2573: minor changes --- src/delete.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/delete.c b/src/delete.c index 4ae1599e1..f79abfc81 100644 --- a/src/delete.c +++ b/src/delete.c @@ -451,9 +451,20 @@ do_retention_internal(void) /* Lock merge chain */ catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); - for (j = parray_num(merge_list) - 1; j > 0; j--) + + /* Merge list example: + * 0 PAGE3 + * 1 PAGE2 + * 2 PAGE1 + * 3 FULL + * + * Сonsequentially merge incremental backups from PAGE1 to PAGE3 + * into FULL. + */ + + for (j = parray_num(merge_list) - 2; j >= 0; j--) { - pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j - 1 ); + pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j); /* Consider this extreme case */ @@ -461,7 +472,9 @@ do_retention_internal(void) // \ / // FULL - /* Check that FULL backup do not has multiple descendants */ + /* Check that FULL backup do not has multiple descendants + * full_backup always point to current full_backup after merge + */ if (is_prolific(backup_list, full_backup)) { elog(WARNING, "Backup %s has multiple valid descendants. " From 3ff6a37d746fafb9c5095f47286dcecf36542c27 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 27 Mar 2019 14:07:49 +0300 Subject: [PATCH 0460/2107] Fix remote archive-get and restore --- src/backup.c | 4 ++-- src/data.c | 54 +++++++++++++++++++++++++--------------------- src/pg_probackup.c | 2 +- src/restore.c | 33 ++++++++++++++-------------- 4 files changed, 49 insertions(+), 44 deletions(-) diff --git a/src/backup.c b/src/backup.c index 220b04543..06baf08b4 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1617,13 +1617,13 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) { if (!file_exists) { - file_exists = fileExists(wal_segment_path, is_start_lsn ? FIO_DB_HOST : FIO_BACKUP_HOST); + file_exists = fileExists(wal_segment_path, FIO_BACKUP_HOST); /* Try to find compressed WAL file */ if (!file_exists) { #ifdef HAVE_LIBZ - file_exists = fileExists(gz_wal_segment_path, is_start_lsn ? FIO_DB_HOST : FIO_BACKUP_HOST); + file_exists = fileExists(gz_wal_segment_path, FIO_BACKUP_HOST); if (file_exists) elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); #endif diff --git a/src/data.c b/src/data.c index 2f0ff174b..51b5d1d57 100644 --- a/src/data.c +++ b/src/data.c @@ -1263,9 +1263,8 @@ get_wal_file(const char *from_path, const char *to_path) gzFile gz_in = NULL; #endif - /* open file for read */ - in = fopen(from_path, PG_BINARY_R); - if (in == NULL) + /* First check source file for existance */ + if (fio_access(from_path, F_OK, FIO_BACKUP_HOST) != 0) { #ifdef HAVE_LIBZ /* @@ -1273,19 +1272,7 @@ get_wal_file(const char *from_path, const char *to_path) * extension. */ snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - gz_in = gzopen(gz_from_path, PG_BINARY_R); - if (gz_in == NULL) - { - if (errno == ENOENT) - { - /* There is no compressed file too, raise an error below */ - } - /* Cannot open compressed file for some reason */ - else - elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", - gz_from_path, strerror(errno)); - } - else + if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0) { /* Found compressed file */ is_decompress = true; @@ -1294,9 +1281,28 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* Didn't find compressed file */ if (!is_decompress) + elog(ERROR, "Source WAL file \"%s\" doesn't exist", + from_path); + } + + /* open file for read */ + if (!is_decompress) + { + in = fio_fopen(from_path, PG_BINARY_R, FIO_BACKUP_HOST); + if (in == NULL) elog(ERROR, "Cannot open source WAL file \"%s\": %s", - from_path, strerror(errno)); + from_path, strerror(errno)); + } +#ifdef HAVE_LIBZ + else + { + gz_in = fio_gzopen(gz_from_path, PG_BINARY_R, Z_DEFAULT_COMPRESSION, + FIO_BACKUP_HOST); + if (gz_in == NULL) + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); } +#endif /* open backup file for write */ snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); @@ -1314,8 +1320,8 @@ get_wal_file(const char *from_path, const char *to_path) #ifdef HAVE_LIBZ if (is_decompress) { - read_len = gzread(gz_in, buf, sizeof(buf)); - if (read_len != sizeof(buf) && !gzeof(gz_in)) + read_len = fio_gzread(gz_in, buf, sizeof(buf)); + if (read_len != sizeof(buf) && !fio_gzeof(gz_in)) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); @@ -1326,7 +1332,7 @@ get_wal_file(const char *from_path, const char *to_path) else #endif { - read_len = fread(buf, 1, sizeof(buf), in); + read_len = fio_fread(in, buf, sizeof(buf)); if (ferror(in)) { errno_temp = errno; @@ -1351,13 +1357,13 @@ get_wal_file(const char *from_path, const char *to_path) #ifdef HAVE_LIBZ if (is_decompress) { - if (gzeof(gz_in) || read_len == 0) + if (fio_gzeof(gz_in) || read_len == 0) break; } else #endif { - if (feof(in) || read_len == 0) + if (/* feof(in) || */ read_len == 0) break; } } @@ -1373,7 +1379,7 @@ get_wal_file(const char *from_path, const char *to_path) #ifdef HAVE_LIBZ if (is_decompress) { - if (gzclose(gz_in) != 0) + if (fio_gzclose(gz_in) != 0) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); @@ -1384,7 +1390,7 @@ get_wal_file(const char *from_path, const char *to_path) else #endif { - if (fclose(in)) + if (fio_fclose(in)) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 7fdb32519..2a60e447a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -341,7 +341,7 @@ main(int argc, char *argv[]) canonicalize_path(backup_path); MyLocation = IsSshProtocol() - ? backup_subcmd == ARCHIVE_PUSH_CMD + ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) ? FIO_DB_HOST : FIO_BACKUP_HOST : FIO_LOCAL_HOST; diff --git a/src/restore.c b/src/restore.c index 20b9e8ca9..825db86aa 100644 --- a/src/restore.c +++ b/src/restore.c @@ -716,18 +716,18 @@ create_recovery_conf(time_t backup_id, elog(LOG, "creating recovery.conf"); snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); - fp = fopen(path, "wt"); + fp = fio_fopen(path, "wt", FIO_DB_HOST); if (fp == NULL) elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, strerror(errno)); - fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); + fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); if (need_restore_conf) { - fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " + fio_fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " "--wal-file-path %%p --wal-file-name %%f'\n", PROGRAM_NAME, backup_path, instance_name); @@ -736,42 +736,41 @@ create_recovery_conf(time_t backup_id, * exclusive options is specified, so the order of calls is insignificant. */ if (rt->recovery_target_name) - fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); if (rt->time_specified) - fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); if (rt->xid_specified) - fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); if (rt->recovery_target_lsn) - fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); if (rt->recovery_target_immediate) - fprintf(fp, "recovery_target = 'immediate'\n"); + fio_fprintf(fp, "recovery_target = 'immediate'\n"); if (rt->inclusive_specified) - fprintf(fp, "recovery_target_inclusive = '%s'\n", + fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", rt->recovery_target_inclusive?"true":"false"); if (rt->recovery_target_tli) - fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); if (rt->recovery_target_action) - fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); } if (restore_as_replica) { - fprintf(fp, "standby_mode = 'on'\n"); + fio_fprintf(fp, "standby_mode = 'on'\n"); if (backup->primary_conninfo) - fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); } - if (fflush(fp) != 0 || - fsync(fileno(fp)) != 0 || - fclose(fp)) + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, strerror(errno)); } From fefe8b14e81b33606d49a107a830c2852a0091de Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 27 Mar 2019 17:55:29 +0300 Subject: [PATCH 0461/2107] [refer #PGPRO-1745] Make remote mkdir operation synchronous --- src/utils/file.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 607072b52..11b93a9f7 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -697,7 +697,10 @@ int fio_mkdir(char const* path, int mode, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - return 0; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_MKDIR); + + return hdr.arg; } else { @@ -1297,7 +1300,9 @@ void fio_communicate(int in, int out) SYS_CHECK(remove(buf)); break; case FIO_MKDIR: /* Create direcory */ - SYS_CHECK(dir_create_dir(buf, hdr.arg)); + hdr.size = 0; + hdr.arg = dir_create_dir(buf, hdr.arg); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CHMOD: /* Change file mode */ SYS_CHECK(chmod(buf, hdr.arg)); From 867e9e583560f6a2e0181b89edd8eb4670c31569 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Mar 2019 18:15:15 +0300 Subject: [PATCH 0462/2107] tests: rename validate_test module to validate --- tests/__init__.py | 4 ++-- tests/{validate_test.py => validate.py} | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename tests/{validate_test.py => validate.py} (99%) diff --git a/tests/__init__.py b/tests/__init__.py index 0c2d18d37..b1fa5f5d2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ import unittest from . import init_test, merge, option_test, show_test, compatibility, \ - backup_test, delete_test, delta, restore, validate_test, \ + backup_test, delete_test, delta, restore, validate, \ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ @@ -36,7 +36,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(retention)) suite.addTests(loader.loadTestsFromModule(show_test)) suite.addTests(loader.loadTestsFromModule(snapfs)) - suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(validate)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) suite.addTests(loader.loadTestsFromModule(pgpro2068)) diff --git a/tests/validate_test.py b/tests/validate.py similarity index 99% rename from tests/validate_test.py rename to tests/validate.py index 13b87a0bd..3b4e26fbf 100644 --- a/tests/validate_test.py +++ b/tests/validate.py @@ -146,6 +146,7 @@ def test_validate_wal_unreal_values(self): con.commit() target_xid = res[0][0] self.switch_wal_segment(node) + time.sleep(5) self.assertIn( "INFO: Backup validation completed successfully", From 068218c54411242b14f175f6c33eb671ccc37dcd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Mar 2019 18:16:53 +0300 Subject: [PATCH 0463/2107] tests: added merge.MergeTest.test_merge_backup_from_future and restore.RestoreTest.test_restore_backup_from_future --- tests/merge.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++- tests/restore.py | 58 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index 0124daac1..3c34ec4b6 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -4,6 +4,7 @@ import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import shutil +from datetime import datetime, timedelta module_name = "merge" @@ -1695,7 +1696,68 @@ def test_losing_file_after_failed_merge(self): self.assertEqual( 'CORRUPT', self.show_pb(backup_dir, 'node')[0]['status']) - # self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_merge_backup_from_future(self): + """ + take FULL backup, table PAGE backup from future, + try to merge page with FULL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=3) + + # Take PAGE from future + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + with open( + os.path.join( + backup_dir, 'backups', 'node', + backup_id, "backup.control"), "a") as conf: + conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3))) + + # rename directory + new_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.rename( + os.path.join(backup_dir, 'backups', 'node', backup_id), + os.path.join(backup_dir, 'backups', 'node', new_id)) + + pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) + pgbench.wait() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, diff --git a/tests/restore.py b/tests/restore.py index 4519fe07c..3a5ec616e 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -5,6 +5,7 @@ from datetime import datetime import sys from time import sleep +from datetime import datetime, timedelta module_name = 'restore' @@ -1724,3 +1725,60 @@ def test_restore_chain_with_corrupted_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_backup_from_future(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica', + 'max_wal_senders': '2'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=3) + #pgbench = node.pgbench(options=['-T', '20', '-c', '2']) + #pgbench.wait() + + # Take PAGE from future + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + with open( + os.path.join( + backup_dir, 'backups', 'node', + backup_id, "backup.control"), "a") as conf: + conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3))) + + # rename directory + new_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.rename( + os.path.join(backup_dir, 'backups', 'node', backup_id), + os.path.join(backup_dir, 'backups', 'node', new_id)) + + pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) + pgbench.wait() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From b2cb9cf940906e015360bf7d4ee47336d1471077 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Mar 2019 22:29:19 +0300 Subject: [PATCH 0464/2107] PGPRO-2589: use parent_link instead of start-time sorting in merge, validate and restore --- src/catalog.c | 10 ++++-- src/merge.c | 85 ++++++++++++++++++++++++++------------------------ src/restore.c | 41 ++++++++++-------------- src/validate.c | 7 ++++- 4 files changed, 75 insertions(+), 68 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 86b51c872..e1aa0afa6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -393,7 +393,7 @@ catalog_get_backup_list(time_t requested_backup_id) if (curr->backup_mode == BACKUP_MODE_FULL) continue; - for (j = i+1; j < parray_num(backups); j++) + for (j = 0; j < parray_num(backups); j++) { pgBackup *ancestor = parray_get(backups, j); @@ -1034,8 +1034,12 @@ find_parent_full_backup(pgBackup *current_backup) } if (base_full_backup->backup_mode != BACKUP_MODE_FULL) - elog(ERROR, "Failed to find FULL backup parent for %s", - base36enc(current_backup->start_time)); + { + + elog(WARNING, "Backup %s is missing", + base36enc(base_full_backup->parent_backup)); + return NULL; + } return base_full_backup; } diff --git a/src/merge.c b/src/merge.c index 5726e36c1..e5861b396 100644 --- a/src/merge.c +++ b/src/merge.c @@ -53,12 +53,10 @@ void do_merge(time_t backup_id) { parray *backups; + parray *merge_list = parray_new(); pgBackup *dest_backup = NULL; pgBackup *full_backup = NULL; - time_t prev_parent = INVALID_BACKUP_ID; int i; - int dest_backup_idx = 0; - int full_backup_idx = 0; if (backup_id == INVALID_BACKUP_ID) elog(ERROR, "required parameter is not specified: --backup-id"); @@ -71,73 +69,79 @@ do_merge(time_t backup_id) /* Get list of all backups sorted in order of descending start time */ backups = catalog_get_backup_list(INVALID_BACKUP_ID); - /* Find destination and parent backups */ + /* Find destination backup first */ for (i = 0; i < parray_num(backups); i++) { pgBackup *backup = (pgBackup *) parray_get(backups, i); - if (backup->start_time > backup_id) - continue; - else if (backup->start_time == backup_id && !dest_backup) + /* found target */ + if (backup->start_time == backup_id) { + /* sanity */ if (backup->status != BACKUP_STATUS_OK && /* It is possible that previous merging was interrupted */ backup->status != BACKUP_STATUS_MERGING && backup->status != BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - base36enc(backup->start_time), status2str(backup->status)); + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); if (backup->backup_mode == BACKUP_MODE_FULL) elog(ERROR, "Backup %s is full backup", base36enc(backup->start_time)); dest_backup = backup; - dest_backup_idx = i; + break; } - else - { - if (dest_backup == NULL) - elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); - - if (backup->start_time != prev_parent) - continue; - - if (backup->status != BACKUP_STATUS_OK && - /* It is possible that previous merging was interrupted */ - backup->status != BACKUP_STATUS_MERGING) - elog(ERROR, "Backup %s has status: %s", - base36enc(backup->start_time), status2str(backup->status)); - - /* If we already found dest_backup, look for full backup */ - if (dest_backup && backup->backup_mode == BACKUP_MODE_FULL) - { - full_backup = backup; - full_backup_idx = i; - - /* Found target and full backups, so break the loop */ - break; - } - } - - prev_parent = backup->parent_backup; } + /* sanity */ if (dest_backup == NULL) elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); + + /* get full backup */ + full_backup = find_parent_full_backup(dest_backup); + + /* sanity */ if (full_backup == NULL) elog(ERROR, "Parent full backup for the given backup %s was not found", base36enc(backup_id)); - Assert(full_backup_idx != dest_backup_idx); + /* sanity */ + if (full_backup->status != BACKUP_STATUS_OK && + /* It is possible that previous merging was interrupted */ + full_backup->status != BACKUP_STATUS_MERGING) + elog(ERROR, "Backup %s has status: %s", + base36enc(full_backup->start_time), status2str(full_backup->status)); + + //Assert(full_backup_idx != dest_backup_idx); + + /* form merge list */ + while(dest_backup->parent_backup_link) + { + /* sanity */ + if (dest_backup->status != BACKUP_STATUS_OK && + /* It is possible that previous merging was interrupted */ + dest_backup->status != BACKUP_STATUS_MERGING && + dest_backup->status != BACKUP_STATUS_DELETING) + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); + + parray_append(merge_list, dest_backup); + dest_backup = dest_backup->parent_backup_link; + } + + /* Add FULL backup for easy locking */ + parray_append(merge_list, full_backup); - catalog_lock_backup_list(backups, full_backup_idx, dest_backup_idx); + /* Lock merge chain */ + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); /* * Found target and full backups, merge them and intermediate backups */ - for (i = full_backup_idx; i > dest_backup_idx; i--) + for (i = parray_num(merge_list) - 2; i >= 0; i--) { - pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1); + pgBackup *from_backup = (pgBackup *) parray_get(merge_list, i); merge_backups(full_backup, from_backup); } @@ -149,6 +153,7 @@ do_merge(time_t backup_id) /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); + parray_free(merge_list); elog(INFO, "Merge of backup %s completed", base36enc(backup_id)); } diff --git a/src/restore.c b/src/restore.c index c84777744..38fa11ed6 100644 --- a/src/restore.c +++ b/src/restore.c @@ -55,9 +55,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; pgBackup *corrupted_backup = NULL; - int dest_backup_index = 0; - int base_full_backup_index = 0; - int corrupted_backup_index = 0; char *action = is_restore ? "Restore":"Validate"; parray *parent_chain = NULL; @@ -179,8 +176,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (dest_backup == NULL) elog(ERROR, "Backup satisfying target options is not found."); - dest_backup_index = get_backup_index_number(backups, dest_backup); - /* If we already found dest_backup, look for full backup. */ if (dest_backup->backup_mode == BACKUP_MODE_FULL) base_full_backup = dest_backup; @@ -201,7 +196,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, missing_backup_start_time = tmp_backup->parent_backup; missing_backup_id = base36enc_dup(tmp_backup->parent_backup); - for (j = get_backup_index_number(backups, tmp_backup); j >= 0; j--) + for (j = 0; j < parray_num(backups); j++) { pgBackup *backup = (pgBackup *) parray_get(backups, j); @@ -235,7 +230,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* parent_backup_id contain human-readable backup ID of oldest invalid backup */ parent_backup_id = base36enc_dup(tmp_backup->start_time); - for (j = get_backup_index_number(backups, tmp_backup) - 1; j >= 0; j--) + for (j = 0; j < parray_num(backups); j++) { pgBackup *backup = (pgBackup *) parray_get(backups, j); @@ -261,6 +256,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } tmp_backup = find_parent_full_backup(dest_backup); + + /* sanity */ + if (!tmp_backup) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(dest_backup->start_time)); } /* @@ -276,8 +276,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (base_full_backup == NULL) elog(ERROR, "Full backup satisfying target options is not found."); - base_full_backup_index = get_backup_index_number(backups, base_full_backup); - /* * Ensure that directories provided in tablespace mapping are valid * i.e. empty or not exist. @@ -297,17 +295,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Take every backup that is a child of base_backup AND parent of dest_backup * including base_backup and dest_backup */ - for (i = base_full_backup_index; i >= dest_backup_index; i--) - { - tmp_backup = (pgBackup *) parray_get(backups, i); - if (is_parent(base_full_backup->start_time, tmp_backup, true) && - is_parent(tmp_backup->start_time, dest_backup, true)) - { - parray_append(parent_chain, tmp_backup); - } + tmp_backup = dest_backup; + while(tmp_backup->parent_backup_link) + { + parray_append(parent_chain, tmp_backup); + tmp_backup = tmp_backup->parent_backup_link; } + parray_append(parent_chain, base_full_backup); + /* for validation or restore with enabled validation */ if (!is_restore || !rt->restore_no_validate) { @@ -317,7 +314,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * Validate backups from base_full_backup to dest_backup. */ - for (i = 0; i < parray_num(parent_chain); i++) + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { tmp_backup = (pgBackup *) parray_get(parent_chain, i); @@ -344,10 +341,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (tmp_backup->status != BACKUP_STATUS_OK) { corrupted_backup = tmp_backup; - /* we need corrupted backup index from 'backups' not parent_chain - * so we can properly orphanize all its descendants - */ - corrupted_backup_index = get_backup_index_number(backups, corrupted_backup); break; } /* We do not validate WAL files of intermediate backups @@ -373,7 +366,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, char *corrupted_backup_id; corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); - for (j = corrupted_backup_index - 1; j >= 0; j--) + for (j = 0; j < parray_num(backups); j++) { pgBackup *backup = (pgBackup *) parray_get(backups, j); @@ -418,7 +411,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (is_restore) { - for (i = 0; i < parray_num(parent_chain); i++) + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); diff --git a/src/validate.c b/src/validate.c index 7d5e94f43..040ea7938 100644 --- a/src/validate.c +++ b/src/validate.c @@ -452,6 +452,11 @@ do_validate_instance(void) continue; } base_full_backup = find_parent_full_backup(current_backup); + + /* sanity */ + if (!base_full_backup) + elog(ERROR, "Parent full backup for the given backup %s was not found", + base36enc(current_backup->start_time)); } /* chain is whole, all parents are valid at first glance, * current backup validation can proceed @@ -568,7 +573,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_OK) { - //tmp_backup = find_parent_full_backup(dest_backup); + /* Revalidation successful, validate corresponding WAL files */ validate_wal(backup, arclog_path, 0, 0, 0, current_backup->tli, From 01207a708643f292b48236542679bd232f043d24 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 28 Mar 2019 13:42:58 +0300 Subject: [PATCH 0465/2107] Throw an error if pg_probackup.conf doesn't exist --- src/catalog.c | 6 +++--- src/configure.c | 22 +++++++++++++++++++--- src/init.c | 2 +- src/pg_probackup.c | 10 +++++++--- src/pg_probackup.h | 2 +- src/utils/configuration.c | 4 ++-- src/utils/configuration.h | 2 +- src/validate.c | 8 +++++++- 8 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 86b51c872..4c8ba3d43 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -603,7 +603,7 @@ write_backup(pgBackup *backup) int errno_temp; pgBackupGetPath(backup, path, lengthof(path), BACKUP_CONTROL_FILE); - snprintf(path_temp, sizeof(path_temp), "%s.partial", path); + snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); fp = fopen(path_temp, "wt"); if (fp == NULL) @@ -644,7 +644,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int errno_temp; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); - snprintf(path_temp, sizeof(path_temp), "%s.partial", path); + snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); fp = fopen(path_temp, "wt"); if (fp == NULL) @@ -728,7 +728,7 @@ readBackupControlFile(const char *path) return NULL; } - parsed_options = config_read_opt(path, options, WARNING, true); + parsed_options = config_read_opt(path, options, WARNING, true, true); if (parsed_options == 0) { diff --git a/src/configure.c b/src/configure.c index aea78244c..3a62831e3 100644 --- a/src/configure.c +++ b/src/configure.c @@ -9,6 +9,8 @@ #include "pg_probackup.h" +#include + #include "utils/configuration.h" #include "utils/json.h" @@ -213,16 +215,22 @@ do_show_config(void) * values into the file. */ void -do_set_config(void) +do_set_config(bool missing_ok) { char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; FILE *fp; int i; join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - fp = fopen(path, "wt"); + snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); + + if (!missing_ok && !fileExists(path)) + elog(ERROR, "Configuration file \"%s\" doesn't exist", path); + + fp = fopen(path_temp, "wt"); if (fp == NULL) - elog(ERROR, "cannot create %s: %s", + elog(ERROR, "Cannot create configuration file \"%s\": %s", BACKUP_CATALOG_CONF_FILE, strerror(errno)); current_group = NULL; @@ -253,6 +261,14 @@ do_set_config(void) } fclose(fp); + + if (rename(path_temp, path) < 0) + { + int errno_temp = errno; + unlink(path_temp); + elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno_temp)); + } } void diff --git a/src/init.c b/src/init.c index fb9b7bbb2..4fe1168cb 100644 --- a/src/init.c +++ b/src/init.c @@ -104,7 +104,7 @@ do_add_instance(void) config_set_opt(instance_options, &instance_config.xlog_seg_size, SOURCE_FILE); /* pgdata was set through command line */ - do_set_config(); + do_set_config(true); elog(INFO, "Instance '%s' successfully inited", instance_name); return 0; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 4f19f89e7..ee3cefde7 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -382,8 +382,12 @@ main(int argc, char *argv[]) config_get_opt_env(instance_options); /* Read options from configuration file */ - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(path, instance_options, ERROR, true); + if (backup_subcmd != ADD_INSTANCE_CMD) + { + join_path_components(path, backup_instance_path, + BACKUP_CATALOG_CONF_FILE); + config_read_opt(path, instance_options, ERROR, true, false); + } } /* Initialize logger */ @@ -525,7 +529,7 @@ main(int argc, char *argv[]) do_show_config(); break; case SET_CONFIG_CMD: - do_set_config(); + do_set_config(false); break; case NO_CMD: /* Should not happen */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2cceaaedb..e84b5e264 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -442,7 +442,7 @@ extern int do_archive_get(char *wal_file_path, char *wal_file_name); /* in configure.c */ extern void do_show_config(void); -extern void do_set_config(void); +extern void do_set_config(bool missing_ok); extern void init_config(InstanceConfig *config); /* in show.c */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c index eabd35ebd..ab2c91b63 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -521,7 +521,7 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], */ int config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict) + bool strict, bool missing_ok) { FILE *fp; char buf[1024]; @@ -532,7 +532,7 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, if (!options) return parsed_options; - if ((fp = pgut_fopen(path, "rt", true)) == NULL) + if ((fp = pgut_fopen(path, "rt", missing_ok)) == NULL) return parsed_options; while (fgets(buf, lengthof(buf), fp)) diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 9602f1d66..96e200472 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -78,7 +78,7 @@ struct ConfigOption extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], ConfigOption options[]); extern int config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict); + bool strict, bool missing_ok); extern void config_get_opt_env(ConfigOption options[]); extern void config_set_opt(ConfigOption options[], void *var, OptionSource source); diff --git a/src/validate.c b/src/validate.c index 7d5e94f43..b4c74786e 100644 --- a/src/validate.c +++ b/src/validate.c @@ -337,7 +337,13 @@ do_validate_all(void) sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); join_path_components(conf_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(conf_path, instance_options, ERROR, false); + if (config_read_opt(conf_path, instance_options, ERROR, false, + true) == 0) + { + elog(WARNING, "Configuration file \"%s\" is empty", conf_path); + corrupted_backup_found = true; + continue; + } do_validate_instance(); } From a0d65fa1c6b40f0093d3fd4fbe774891aeef559c Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 28 Mar 2019 15:56:27 +0300 Subject: [PATCH 0466/2107] Fix archive-get compressed error --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 51b5d1d57..36c040430 100644 --- a/src/data.c +++ b/src/data.c @@ -1315,13 +1315,13 @@ get_wal_file(const char *from_path, const char *to_path) /* copy content */ for (;;) { - size_t read_len = 0; + int read_len = 0; #ifdef HAVE_LIBZ if (is_decompress) { read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len != sizeof(buf) && !fio_gzeof(gz_in)) + if (read_len <= 0 && !fio_gzeof(gz_in)) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); From 65dd94d664a7ea7de10d3f41446637866907eb2a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 28 Mar 2019 16:58:14 +0300 Subject: [PATCH 0467/2107] PGPRO-2589: Binary search for catalog_get_backup_list() --- src/catalog.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index e1aa0afa6..a25e016bf 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -386,25 +386,18 @@ catalog_get_backup_list(time_t requested_backup_id) /* Link incremental backups with their ancestors.*/ for (i = 0; i < parray_num(backups); i++) { - pgBackup *curr = parray_get(backups, i); - - int j; + pgBackup *curr = parray_get(backups, i); + pgBackup **ancestor; + pgBackup key; if (curr->backup_mode == BACKUP_MODE_FULL) continue; - for (j = 0; j < parray_num(backups); j++) - { - pgBackup *ancestor = parray_get(backups, j); - - if (ancestor->start_time == curr->parent_backup) - { - curr->parent_backup_link = ancestor; - /* elog(INFO, "curr %s, ancestor %s j=%d", base36enc_dup(curr->start_time), - base36enc_dup(ancestor->start_time), j); */ - break; - } - } + key.start_time = curr->parent_backup; + ancestor = (pgBackup **) parray_bsearch(backups, &key, + pgBackupCompareIdDesc); + if (ancestor) + curr->parent_backup_link = *ancestor; } return backups; @@ -1035,9 +1028,12 @@ find_parent_full_backup(pgBackup *current_backup) if (base_full_backup->backup_mode != BACKUP_MODE_FULL) { - - elog(WARNING, "Backup %s is missing", - base36enc(base_full_backup->parent_backup)); + if (base_full_backup->parent_backup) + elog(WARNING, "Backup %s is missing", + base36enc(base_full_backup->parent_backup)); + else + elog(WARNING, "Failed to find parent FULL backup for %s", + base36enc(current_backup->start_time)); return NULL; } From e82b5daefcd2bceef118ef79e2d06194279ad912 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 17:10:28 +0300 Subject: [PATCH 0468/2107] tests: rename module delete_test to delete; added delete.DeleteTest.test_delete_interleaved_incremental_chains and delete.DeleteTest.test_delete_multiple_descendants --- tests/__init__.py | 4 +- tests/{delete_test.py => delete.py} | 299 ++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+), 2 deletions(-) rename tests/{delete_test.py => delete.py} (52%) diff --git a/tests/__init__.py b/tests/__init__.py index b1fa5f5d2..ad7f8a16e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ import unittest from . import init_test, merge, option_test, show_test, compatibility, \ - backup_test, delete_test, delta, restore, validate, \ + backup_test, delete, delta, restore, validate, \ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ @@ -19,7 +19,7 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) # suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(compression)) - suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(delete)) suite.addTests(loader.loadTestsFromModule(delta)) suite.addTests(loader.loadTestsFromModule(exclude)) suite.addTests(loader.loadTestsFromModule(false_positive)) diff --git a/tests/delete_test.py b/tests/delete.py similarity index 52% rename from tests/delete_test.py rename to tests/delete.py index 6c817f767..3c573a856 100644 --- a/tests/delete_test.py +++ b/tests/delete.py @@ -250,3 +250,302 @@ def test_delete_backup_with_empty_control_file(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Change PAGEa2 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # Change PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGEc1 OK + # FULLc OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Delete FULLb + self.delete_pb( + backup_dir, 'node', backup_id_b) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 5) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa should be deleted + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # Change PAGEa1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # Change PAGEa2 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb2 and PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + # Delete FULLa + self.delete_pb(backup_dir, 'node', backup_id_a) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) From d48f4024a8d775a2a3ca7884ffddfd9ae420816b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 17:19:33 +0300 Subject: [PATCH 0469/2107] PGPRO-2589: use parent_link when deleting backup chain via delete -i BACKUP_ID --- src/delete.c | 83 +++++++++++++++++++++---------------------------- tests/delete.py | 20 ------------ 2 files changed, 36 insertions(+), 67 deletions(-) diff --git a/src/delete.c b/src/delete.c index 287e73776..2b2f0d614 100644 --- a/src/delete.c +++ b/src/delete.c @@ -24,71 +24,57 @@ do_delete(time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; - time_t parent_id = 0; XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; /* Get complete list of backups */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - if (backup_id != 0) + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = 0; i < parray_num(backup_list); i++) { - delete_list = parray_new(); + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Find backup to be deleted and make increment backups array to be deleted */ - for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + if (backup->start_time == backup_id) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - - if (backup->start_time == backup_id) - { - parray_append(delete_list, backup); - - /* - * Do not remove next backups, if target backup was finished - * incorrectly. - */ - if (backup->status == BACKUP_STATUS_ERROR) - break; - - /* Save backup id to retreive increment backups */ - parent_id = backup->start_time; - target_backup = backup; - } - else if (target_backup) - { - if (backup->backup_mode != BACKUP_MODE_FULL && - backup->parent_backup == parent_id) - { - /* Append to delete list increment backup */ - parray_append(delete_list, backup); - /* Save backup id to retreive increment backups */ - parent_id = backup->start_time; - } - else - break; - } + target_backup = backup; + break; } + } - if (parray_num(delete_list) == 0) - elog(ERROR, "no backup found, cannot delete"); + /* sanity */ + if (!target_backup) + elog(ERROR, "Failed to find backup %s, cannot delete", base36enc(backup_id)); - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); + /* form delete list */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Delete backups from the end of list */ - for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + /* check if backup is descendant of delete target */ + if (is_parent(target_backup->start_time, backup, false)) + parray_append(delete_list, backup); + } + parray_append(delete_list, target_backup); - if (interrupted) - elog(ERROR, "interrupted during delete backup"); + /* Lock marked for delete backups */ + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); - delete_backup_files(backup); - } + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during delete backup"); - parray_free(delete_list); + delete_backup_files(backup); } + parray_free(delete_list); + /* Clean WAL segments */ if (delete_wal) { @@ -303,6 +289,9 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", i + 1, num_files, file->path); + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + pgFileDelete(file); } diff --git a/tests/delete.py b/tests/delete.py index 3c573a856..71919c869 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -388,16 +388,10 @@ def test_delete_multiple_descendants(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=3) - # Take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() backup_id_b = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -405,9 +399,6 @@ def test_delete_multiple_descendants(self): page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - # Change FULLb backup status to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') @@ -426,9 +417,6 @@ def test_delete_multiple_descendants(self): # FULLb OK # FULLa OK - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - # Change PAGEa1 backup status to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') @@ -443,9 +431,6 @@ def test_delete_multiple_descendants(self): page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK @@ -467,9 +452,6 @@ def test_delete_multiple_descendants(self): page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - # PAGEb2 OK # PAGEa2 ERROR # PAGEb1 OK @@ -490,8 +472,6 @@ def test_delete_multiple_descendants(self): page_id_a3 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() # PAGEa3 OK # PAGEb2 ERROR From 0ff607914bcbb1b7898ac07486a5cadbecbeeb48 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 18:08:49 +0300 Subject: [PATCH 0470/2107] tests: minor fixes to merge module --- tests/merge.py | 53 ++++++++++++++++---------------------------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 3c34ec4b6..03cd49d6a 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1203,11 +1203,7 @@ def test_continue_failed_merge_2(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica' - } - ) + set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1258,20 +1254,7 @@ def test_continue_failed_merge_2(self): backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] # Try to continue failed MERGE - try: - self.merge_backup(backup_dir, "node", backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Backup {0} has status: DELETING".format( - backup_id_deleted) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - + self.merge_backup(backup_dir, "node", backup_id) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1617,7 +1600,8 @@ def test_losing_file_after_failed_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1639,6 +1623,10 @@ def test_losing_file_after_failed_merge(self): 'postgres', "select pg_relation_filepath('pgbench_accounts')").rstrip() + node.safe_psql( + 'postgres', + "VACUUM pgbench_accounts") + vm_path = path + '_vm' # DELTA backup @@ -1674,27 +1662,20 @@ def test_losing_file_after_failed_merge(self): backup_dir, 'backups', 'node', full_id, 'database', vm_path) - print(file_to_remove) - os.remove(file_to_remove) # Try to continue failed MERGE - try: - self.merge_backup(backup_dir, "node", backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Merging of backup {0} failed".format( - backup_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.merge_backup(backup_dir, "node", backup_id) self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) self.del_test_dir(module_name, fname) From a94c228fcbd7a3caa70199d90009c114320c532e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 18:58:08 +0300 Subject: [PATCH 0471/2107] update help, add --merge-expired option --- src/help.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index 8843f1d23..ca5eabf1f 100644 --- a/src/help.c +++ b/src/help.c @@ -107,7 +107,7 @@ help_pg_probackup(void) printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); printf(_(" [--log-rotation-age=log-rotation-age]\n")); - printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--compress]\n")); @@ -145,7 +145,7 @@ help_pg_probackup(void) printf(_(" [--format=format]\n")); printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n")); printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id\n")); @@ -199,7 +199,7 @@ help_backup(void) printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); printf(_(" [--log-rotation-age=log-rotation-age]\n")); - printf(_(" [--delete-expired] [--delete-wal]\n")); + printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--compress]\n")); @@ -253,6 +253,8 @@ help_backup(void) printf(_("\n Retention options:\n")); printf(_(" --delete-expired delete backups expired according to current\n")); printf(_(" retention policy after successful backup completion\n")); + printf(_(" --merge-expired merge backups expired according to current\n")); + printf(_(" retention policy after successful backup completion\n")); printf(_(" --delete-wal remove redundant archived wal files\n")); printf(_(" --retention-redundancy=retention-redundancy\n")); printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); @@ -411,14 +413,18 @@ static void help_delete(void) { printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-i backup-id | --expired] [--wal]\n\n")); + printf(_(" [-i backup-id | --expired | --merge-expired] [--wal]\n")); + printf(_(" [-j num-threads]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to delete\n")); printf(_(" --expired delete backups expired according to current\n")); printf(_(" retention policy\n")); + printf(_(" --merge-expired merge backups expired according to current\n")); + printf(_(" retention policy\n")); printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From b80a1ffbb3d5b3ed94c327d39be85206949aca18 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 20:59:37 +0300 Subject: [PATCH 0472/2107] tests: minor changes to merge.MergeTest.test_losing_file_after_failed_merge --- tests/merge.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 03cd49d6a..8a812ac26 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1609,15 +1609,16 @@ def test_losing_file_after_failed_merge(self): node.slow_start() # Add data - node.pgbench_init(scale=3) + node.pgbench_init(scale=1) # FULL backup full_id = self.backup_node( backup_dir, 'node', node, options=['--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) - pgbench.wait() + node.safe_psql( + 'postgres', + "update pgbench_accounts set aid = aid + 1005000") path = node.safe_psql( 'postgres', @@ -1665,6 +1666,8 @@ def test_losing_file_after_failed_merge(self): os.remove(file_to_remove) # Try to continue failed MERGE + #print(backup_id) + #exit(1) self.merge_backup(backup_dir, "node", backup_id) self.assertEqual( From ca44e739a3350906b81ee5dc342e7a7a2e93f717 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 21:57:33 +0300 Subject: [PATCH 0473/2107] PGPRO-2573: use parray_bsearch --- src/catalog.c | 21 --------------------- src/delete.c | 14 +++++++++----- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index cf7a0a280..11ce830e8 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1035,27 +1035,6 @@ is_prolific(parray *backup_list, pgBackup *target_backup) return false; } -/* - * Check if target_backup in backup_list. Use if bsearch cannot be trusted. - */ -bool -in_backup_list(parray *backup_list, pgBackup *target_backup) -{ - int i; - - if (!target_backup) - elog(ERROR, "Target backup cannot be NULL"); - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); - - if (tmp_backup->start_time == target_backup->start_time) - return true; - } - return false; -} - /* * Find parent base FULL backup for current backup using parent_backup_link */ diff --git a/src/delete.c b/src/delete.c index c7aa2cae5..f4b9f919b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -282,11 +282,11 @@ do_retention_internal(void) pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (in_backup_list(to_keep_list, backup)) + if (parray_bsearch(to_keep_list, backup, pgBackupCompareIdDesc)) action = "Keep"; - if (in_backup_list(to_purge_list, backup)) - action = "Purge"; + if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) + action = "Keep"; if (backup->recovery_time == 0) actual_window = 0; @@ -365,7 +365,9 @@ do_retention_internal(void) continue; /* If parent of current backup is also in keep list, go to the next */ - if (in_backup_list(to_keep_list, keep_backup->parent_backup_link)) + if (parray_bsearch(to_keep_list, + keep_backup->parent_backup_link, + pgBackupCompareIdDesc)) { /* make keep list a bit sparse */ elog(INFO, "Sparsing keep list, remove %s", base36enc(keep_backup->start_time)); @@ -388,7 +390,9 @@ do_retention_internal(void) } /* Check that ancestor is in purge_list */ - if (!in_backup_list(to_purge_list, full_backup)) + if (!parray_bsearch(to_purge_list, + full_backup, + pgBackupCompareIdDesc)) { elog(WARNING, "Skip backup %s for merging, " "because his FULL parent is not marked for purge", base36enc(keep_backup->start_time)); From 0b26072ef6d2a996698d6f67c4fed4eff2913322 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 Mar 2019 21:58:15 +0300 Subject: [PATCH 0474/2107] tests: added merge.MergeTest.test_merge_multiple_descendants --- tests/merge.py | 182 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 8a812ac26..4608a34ce 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1743,6 +1743,188 @@ def test_merge_backup_from_future(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_merge_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Change PAGEa1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # Change PAGEa2 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEb2 and PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2 status to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + self.merge_backup( + backup_dir, 'node', page_id_a2, + options=['--merge-expired', '--log-level-console=log']) + + try: + self.merge_backup( + backup_dir, 'node', page_id_a3, + options=['--merge-expired', '--log-level-console=log']) + self.assertEqual( + 1, 0, + "Expecting Error because of parent FULL backup is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Parent full backup for the given " + "backup {0} was not found".format( + page_id_a3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, # it should not be possible to continue merge after that From e82201f5d792c2729b9dda0c113bcd3ae486d850 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 29 Mar 2019 16:49:02 +0300 Subject: [PATCH 0475/2107] Consider empty blocks within datafiles. RelationAddExtraBlocks() may extend a relation with empty blocks due to optimisation to avoid contentions. --- src/data.c | 64 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/src/data.c b/src/data.c index 36e0de28e..9cad2eabf 100644 --- a/src/data.c +++ b/src/data.c @@ -705,7 +705,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, in = fopen(file->path, PG_BINARY_R); if (in == NULL) { - elog(ERROR, "cannot open backup file \"%s\": %s", file->path, + elog(ERROR, "Cannot open backup file \"%s\": %s", file->path, strerror(errno)); } } @@ -722,7 +722,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, { int errno_tmp = errno; fclose(in); - elog(ERROR, "cannot open restore target file \"%s\": %s", + elog(ERROR, "Cannot open restore target file \"%s\": %s", to_path, strerror(errno_tmp)); } @@ -762,16 +762,22 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, break; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "odd size page found at block %u of \"%s\"", + "Odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(ERROR, "cannot read header of block %u of \"%s\": %s", + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); } + if (header.block == 0 && header.compressed_size == 0) + { + elog(VERBOSE, "Skip empty block of \"%s\"", file->path); + continue; + } + if (header.block < blknum) - elog(ERROR, "backup is broken at file->path %s block %u", - file->path, blknum); + elog(ERROR, "Backup is broken at block %u of \"%s\"", + blknum, file->path); blknum = header.block; @@ -792,7 +798,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, read_len = fread(compressed_page.data, 1, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "cannot read block %u of \"%s\" read %zu of %d", + elog(ERROR, "Cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); /* @@ -816,7 +822,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, errormsg); if (uncompressed_size != BLCKSZ) - elog(ERROR, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); } @@ -827,7 +833,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * Seek and write the restored page. */ if (fseek(out, write_pos, SEEK_SET) < 0) - elog(ERROR, "cannot seek block %u of \"%s\": %s", + elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_path, strerror(errno)); if (write_header) @@ -835,7 +841,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, /* We uncompressed the page, so its size is BLCKSZ */ header.compressed_size = BLCKSZ; if (fwrite(&header, 1, sizeof(header), out) != sizeof(header)) - elog(ERROR, "cannot write header of block %u of \"%s\": %s", + elog(ERROR, "Cannot write header of block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } @@ -846,14 +852,14 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, if (uncompressed_size == BLCKSZ) { if (fwrite(page.data, 1, BLCKSZ, out) != BLCKSZ) - elog(ERROR, "cannot write block %u of \"%s\": %s", + elog(ERROR, "Cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } else { /* */ if (fwrite(compressed_page.data, 1, BLCKSZ, out) != BLCKSZ) - elog(ERROR, "cannot write block %u of \"%s\": %s", + elog(ERROR, "Cannot write block %u of \"%s\": %s", blknum, file->path, strerror(errno)); } } @@ -891,7 +897,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * Truncate file to this length. */ if (ftruncate(fileno(out), write_pos) != 0) - elog(ERROR, "cannot truncate \"%s\": %s", + elog(ERROR, "Cannot truncate \"%s\": %s", file->path, strerror(errno)); elog(VERBOSE, "Delta truncate file %s to block %u", file->path, truncate_from); @@ -905,14 +911,14 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, if (in) fclose(in); fclose(out); - elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, strerror(errno_tmp)); } if (fflush(out) != 0 || fsync(fileno(out)) != 0 || fclose(out)) - elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + elog(ERROR, "Cannot write \"%s\": %s", to_path, strerror(errno)); if (in) fclose(in); } @@ -1571,7 +1577,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, pg_crc32 crc; bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - elog(VERBOSE, "validate relation blocks for file %s", file->path); + elog(VERBOSE, "Validate relation blocks for file %s", file->path); in = fopen(file->path, PG_BINARY_R); if (in == NULL) @@ -1582,7 +1588,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, return false; } - elog(ERROR, "cannot open file \"%s\": %s", + elog(ERROR, "Cannot open file \"%s\": %s", file->path, strerror(errno)); } @@ -1606,20 +1612,26 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, break; /* EOF found */ else if (read_len != 0 && feof(in)) elog(WARNING, - "odd size page found at block %u of \"%s\"", + "Odd size page found at block %u of \"%s\"", blknum, file->path); else - elog(WARNING, "cannot read header of block %u of \"%s\": %s", + elog(WARNING, "Cannot read header of block %u of \"%s\": %s", blknum, file->path, strerror(errno_tmp)); return false; } COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); + if (header.block == 0 && header.compressed_size == 0) + { + elog(VERBOSE, "Skip empty block of \"%s\"", file->path); + continue; + } + if (header.block < blknum) { - elog(WARNING, "backup is broken at file->path %s block %u", - file->path, blknum); + elog(WARNING, "Backup is broken at block %u of \"%s\"", + blknum, file->path); return false; } @@ -1627,8 +1639,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (header.compressed_size == PageIsTruncated) { - elog(LOG, "File %s, block %u is truncated", - file->path, blknum); + elog(LOG, "Block %u of \"%s\" is truncated", + blknum, file->path); continue; } @@ -1638,7 +1650,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, MAXALIGN(header.compressed_size), in); if (read_len != MAXALIGN(header.compressed_size)) { - elog(WARNING, "cannot read block %u of \"%s\" read %zu of %d", + elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", blknum, file->path, read_len, header.compressed_size); return false; } @@ -1668,7 +1680,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, is_valid = false; continue; } - elog(WARNING, "page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", file->path, uncompressed_size); return false; } @@ -1690,7 +1702,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (crc != file->crc) { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", file->path, file->crc, crc); is_valid = false; } From 0163a7e3fbca90285292183a310e4466edc15e70 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 29 Mar 2019 20:07:48 +0300 Subject: [PATCH 0476/2107] PGPRO-2573: minor change of elevel in message about sparsing keep list --- src/delete.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index f4b9f919b..b71ebf93c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -370,7 +370,7 @@ do_retention_internal(void) pgBackupCompareIdDesc)) { /* make keep list a bit sparse */ - elog(INFO, "Sparsing keep list, remove %s", base36enc(keep_backup->start_time)); + elog(LOG, "Sparsing keep list, remove %s", base36enc(keep_backup->start_time)); parray_remove(to_keep_list, i); i--; continue; From 912e6a548f4e29cd153b12a48c2a92cb48a6c1ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 29 Mar 2019 21:27:38 +0300 Subject: [PATCH 0477/2107] add --dry-run option to help --- src/help.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index ca5eabf1f..590282931 100644 --- a/src/help.c +++ b/src/help.c @@ -146,6 +146,7 @@ help_pg_probackup(void) printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n")); + printf(_(" [--dry-run]\n")); printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id\n")); @@ -260,6 +261,7 @@ help_backup(void) printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Compression options:\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); @@ -414,7 +416,7 @@ help_delete(void) { printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-i backup-id | --expired | --merge-expired] [--wal]\n")); - printf(_(" [-j num-threads]\n\n")); + printf(_(" [-j num-threads] [--dry-run]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -425,6 +427,7 @@ help_delete(void) printf(_(" retention policy\n")); printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From 6fef70277d885de99dfa22f6def0f78407813ee0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 29 Mar 2019 22:09:52 +0300 Subject: [PATCH 0478/2107] remove limitation about external files from README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a356b81b5..898c7d6a4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Taking backups from a standby server to avoid extra load on the master server * Extended logging settings * Custom commands to simplify WAL log archiving +* External to PGDATA directories, such as directories with config files and scripts, can be included in backup To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -40,7 +41,6 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * Creating backups from a remote server is currently not supported. * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. * Microsoft Windows operating system support is in beta stage. -* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately. ## Installation and Setup ### Linux Installation From 2647c378412f30d4b4f5828f0374e09b18ff0666 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 29 Mar 2019 22:10:24 +0300 Subject: [PATCH 0479/2107] tests: minor changes to logging and snapfs tests --- tests/logging.py | 42 ++++++++++++++++++++++++++++++++++++------ tests/snapfs.py | 2 +- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/tests/logging.py b/tests/logging.py index 173a977a1..210a6f4f6 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -1,7 +1,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - +import datetime module_name = 'logging' @@ -15,10 +15,8 @@ def test_log_rotation(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -41,4 +39,36 @@ def test_log_rotation(self): gdb.continue_execution_until_exit() # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) + + def test_log_filename_strftime(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE', + '--log-filename=pg_probackup-%a.log']) + + day_of_week = datetime.datetime.today().strftime("%a") + + path = os.path.join( + backup_dir, 'log', 'pg_probackup-{0}.log'.format(day_of_week)) + + self.assertTrue(os.path.isfile(path)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/snapfs.py b/tests/snapfs.py index a8acc8354..3ea05e8ed 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -9,8 +9,8 @@ class SnapFSTest(ProbackupTest, unittest.TestCase): - # @unittest.skip("skip") # @unittest.expectedFailure + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_snapfs_simple(self): """standart backup modes with ARCHIVE WAL method""" fname = self.id().split('.')[3] From 932fc5be776f8026fdba9576c9381fe983aa68ab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 29 Mar 2019 22:22:25 +0300 Subject: [PATCH 0480/2107] update help --- src/help.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/help.c b/src/help.c index 8843f1d23..6a5c2c9f6 100644 --- a/src/help.c +++ b/src/help.c @@ -122,9 +122,9 @@ help_pg_probackup(void) printf(_(" [--external-dirs=external-directory-path]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); @@ -146,8 +146,9 @@ help_pg_probackup(void) printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id\n")); + printf(_(" -i backup-id [--progress] [-j num-threads]\n")); printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); @@ -227,7 +228,7 @@ help_backup(void) printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); + printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -286,7 +287,7 @@ static void help_restore(void) { printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n")); + printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads] [--progress]\n")); printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); @@ -301,6 +302,7 @@ help_restore(void) printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -i, --backup-id=backup-id backup to restore\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); printf(_(" --time=time time stamp up to which recovery will proceed\n")); @@ -510,7 +512,7 @@ help_set_config(void) printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); + printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -586,7 +588,7 @@ help_add_instance(void) printf(_(" --instance=instance_name name of the new instance\n")); printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dirA:/tmp/dirB)\n")); + printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); } static void From 523457c91ddfd1a8d380b768893e9e1061365e2f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 30 Mar 2019 00:25:10 +0300 Subject: [PATCH 0481/2107] update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 898c7d6a4..b70ae8ad4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Implementing a single backup strategy for multi-server PostgreSQL clusters * Automatic data consistency checks and on-demand backup validation without actual data recovery * Managing backups in accordance with retention policy -* Running backup, restore, and validation processes on multiple parallel threads +* Merging incremental into full backups without actual data recovery +* Running backup, restore, merge and validation processes on multiple parallel threads * Storing backup data in a compressed state to save disk space * Taking backups from a standby server to avoid extra load on the master server * Extended logging settings From f0de44e0fa37141904d5932fc8fc588d79b588db Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 30 Mar 2019 01:48:57 +0300 Subject: [PATCH 0482/2107] tests: add module "config" --- tests/__init__.py | 3 ++- tests/config.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/config.py diff --git a/tests/__init__.py b/tests/__init__.py index ad7f8a16e..033ce535f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,7 +5,7 @@ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external + locking, remote, external, config def load_tests(loader, tests, pattern): @@ -14,6 +14,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup_test)) suite.addTests(loader.loadTestsFromModule(compatibility)) + suite.addTests(loader.loadTestsFromModule(config)) # suite.addTests(loader.loadTestsFromModule(cfs_backup)) # suite.addTests(loader.loadTestsFromModule(cfs_restore)) # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 000000000..4a382e13a --- /dev/null +++ b/tests/config.py @@ -0,0 +1,53 @@ +import unittest +import subprocess +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from sys import exit + +module_name = 'config' + + +class ConfigTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_remove_instance_config(self): + """remove pg_probackup.conf""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.show_pb(backup_dir) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + conf_file = os.path.join( + backup_dir, 'backups','node', 'pg_probackup.conf') + + os.unlink(os.path.join(backup_dir, 'backups','node', 'pg_probackup.conf')) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because pg_probackup.conf is missing. " + ".\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: could not open file "{0}": ' + 'No such file or directory'.format(conf_file), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) From fb4c3ffdc0a0ae9f5ca8985b0fd6dd53027e35d3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 Mar 2019 11:22:17 +0300 Subject: [PATCH 0483/2107] bugix: fix thread inturruption detection --- src/utils/logger.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 72840aa87..fb1005268 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -130,13 +130,13 @@ exit_if_necessary(int elevel) /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) { + /* Interrupt other possible routines */ + thread_interrupted = true; #ifdef WIN32 ExitThread(elevel); #else pthread_exit(NULL); #endif - /* Interrupt other possible routines */ - thread_interrupted = true; } else exit(elevel); From 702d13e4a547c9c587e1bdb2c391a0cd58406d0b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 Mar 2019 11:41:46 +0300 Subject: [PATCH 0484/2107] PGPRO-2065: allow checkdb to be used without BACKUP_DIR --- src/backup.c | 61 +++++++------ src/data.c | 208 +++++++++++++++++++++++++++------------------ src/pg_probackup.c | 16 ++-- 3 files changed, 171 insertions(+), 114 deletions(-) diff --git a/src/backup.c b/src/backup.c index 68fee4945..140f9ee5a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -940,7 +940,7 @@ do_block_validation(void) /* arrays with meta info for multi threaded backup */ pthread_t *threads; backup_files_arg *threads_args; - bool backup_isok = true; + bool check_isok = true; pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); @@ -1017,14 +1017,14 @@ do_block_validation(void) { pthread_join(threads[i], NULL); if (threads_args[i].ret > 0) - backup_isok = false; + check_isok = false; } /* TODO write better info message */ - if (backup_isok) - elog(INFO, "Data files are checked"); + if (check_isok) + elog(INFO, "Data files are valid"); else - elog(ERROR, "Data files checking failed"); + elog(ERROR, "Data files are corrupted"); if (backup_files_list) { @@ -1042,7 +1042,7 @@ do_amcheck(void) /* arrays with meta info for multi threaded backup */ pthread_t *threads; backup_files_arg *threads_args; - bool backup_isok = true; + bool check_isok = true; PGresult *res_db; int n_databases = 0; bool first_db_with_amcheck = true; @@ -1118,13 +1118,13 @@ do_amcheck(void) { pthread_join(threads[j], NULL); if (threads_args[j].ret == 1) - backup_isok = false; + check_isok = false; } pgut_disconnect(db_conn); } /* TODO write better info message */ - if (backup_isok) + if (check_isok) elog(INFO, "Indexes are checked"); else elog(ERROR, "Indexs checking failed"); @@ -1188,6 +1188,17 @@ pgdata_basic_setup(void) is_checksum_enabled = pg_checksum_enable(); + + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + /* TODO fix it for remote backup */ + + if (!is_remote_backup) + check_system_identifiers(); + if (is_checksum_enabled) elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " "Data block corruption will be detected"); @@ -1225,10 +1236,6 @@ do_backup(time_t start_time) current.compress_alg = instance_config.compress_alg; current.compress_level = instance_config.compress_level; - /* TODO fix it for remote backup*/ - if (!is_remote_backup) - current.checksum_version = get_data_checksum_version(true); - current.stream = stream_wal; is_ptrack_support = pg_ptrack_support(); @@ -1261,15 +1268,6 @@ do_backup(time_t start_time) instance_config.master_user); } - /* - * Ensure that backup directory was initialized for the same PostgreSQL - * instance we opened connection to. And that target backup database PGDATA - * belogns to the same instance. - */ - /* TODO fix it for remote backup */ - if (!is_remote_backup) - check_system_identifiers(); - /* Start backup. Update backup status. */ current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; @@ -1415,10 +1413,24 @@ check_system_identifiers(void) system_id_pgdata = get_system_identifier(instance_config.pgdata); system_id_conn = get_remote_system_identifier(backup_conn); + /* for checkdb check only system_id_pgdata and system_id_conn */ + if (current.backup_mode == BACKUP_MODE_INVALID) + { + if (system_id_conn != system_id_pgdata) + { + elog(ERROR, "Data directory initialized with system id " UINT64_FORMAT ", " + "but connected instance system id is " UINT64_FORMAT, + system_id_pgdata, system_id_conn); + } + return; + } + + if (system_id_conn != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but connected instance system id is " UINT64_FORMAT, instance_config.system_identifier, system_id_conn); + if (system_id_pgdata != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but target backup directory system id is " UINT64_FORMAT, @@ -2562,7 +2574,7 @@ check_files(void *arg) elog(VERBOSE, "Checking file: \"%s\" ", file->path); /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during checkdb"); if (progress) @@ -2579,7 +2591,6 @@ check_files(void *arg) * If file is not found, this is not en error. * It could have been deleted by concurrent postgres transaction. */ - file->write_size = BYTES_INVALID; elog(LOG, "File \"%s\" is not found", file->path); continue; } @@ -2597,7 +2608,7 @@ check_files(void *arg) if (S_ISREG(buf.st_mode)) { - /* check only uncompressed datafiles */ + /* check only uncompressed by cfs datafiles */ if (file->is_datafile && !file->is_cfs) { char to_path[MAXPGPATH]; @@ -2639,7 +2650,7 @@ check_indexes(void *arg) continue; /* check for interrupt */ - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "interrupted during checkdb --amcheck"); elog(VERBOSE, "Checking index number %d of %d : \"%s\" ", i,n_indexes, ind->name); diff --git a/src/data.c b/src/data.c index 58338de5b..275184cc6 100644 --- a/src/data.c +++ b/src/data.c @@ -181,6 +181,7 @@ typedef struct BackupPageHeader /* Special value for compressed_size field */ #define PageIsTruncated -2 #define SkipCurrentPage -3 +#define PageIsCorrupted -4 /* used by checkdb */ /* Verify page's header */ static bool @@ -278,7 +279,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) != ((PageHeader) page)->pd_checksum) { - elog(WARNING, "File: %s blknum %u have wrong checksum, try again", + elog(LOG, "File: %s blknum %u have wrong checksum, try again", file->path, blknum); return -1; } @@ -304,6 +305,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * Returns 0 if page was successfully retrieved * SkipCurrentPage(-3) if we need to skip this page * PageIsTruncated(-2) if the page was truncated + * PageIsCorrupted(-4) if the page check mismatch */ static int32 prepare_page(backup_files_arg *arguments, @@ -311,7 +313,8 @@ prepare_page(backup_files_arg *arguments, BlockNumber blknum, BlockNumber nblocks, FILE *in, int *n_skipped, BackupMode backup_mode, - Page page) + Page page, + bool strict) { XLogRecPtr page_lsn = 0; int try_again = 100; @@ -354,7 +357,7 @@ prepare_page(backup_files_arg *arguments, */ //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); - if (result == -1 && is_ptrack_support) + if (result == -1 && is_ptrack_support && strict) { elog(WARNING, "File %s, block %u, try to fetch via SQL", file->path, blknum); @@ -365,8 +368,24 @@ prepare_page(backup_files_arg *arguments, * If page is not valid after 100 attempts to read it * throw an error. */ - if(!page_is_valid && !is_ptrack_support) - elog(ERROR, "Data file checksum mismatch. Canceling backup"); + + if (!page_is_valid) + { + elog(WARNING, "CORRUPTION in file %s, block %u", + file->path, blknum); + + if (!is_ptrack_support && strict) + elog(ERROR, "Data file corruption. Canceling backup"); + } + + /* Checkdb not going futher */ + if (!strict) + { + if (page_is_valid) + return 0; + else + return PageIsCorrupted; + } } if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) @@ -507,67 +526,6 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->write_size += write_buffer_size; } -bool -check_data_file(backup_files_arg* arguments, - pgFile *file) -{ - FILE *in; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int n_blocks_skipped = 0; - int page_state; - char curr_page[BLCKSZ]; - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - - /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); - if (in == NULL) - { - FIN_FILE_CRC32(true, file->crc); - - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { - elog(LOG, "File \"%s\" is not found", file->path); - return false; - } - - elog(ERROR, "cannot open file \"%s\": %s", - file->path, strerror(errno)); - } - - if (file->size % BLCKSZ != 0) - { - fclose(in); - elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); - } - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; - - for (blknum = 0; blknum < nblocks; blknum++) - { - page_state = prepare_page(arguments, file, 0, //0 = InvalidXLogRecPtr - blknum, nblocks, in, &n_blocks_skipped, - BACKUP_MODE_FULL, curr_page); - if (page_state == PageIsTruncated) - break; - } - - return true; - fclose(in); -} - /* * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with @@ -672,7 +630,7 @@ backup_data_file(backup_files_arg* arguments, { page_state = prepare_page(arguments, file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page); + backup_mode, curr_page, true); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -695,7 +653,7 @@ backup_data_file(backup_files_arg* arguments, { page_state = prepare_page(arguments, file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page); + backup_mode, curr_page, true); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -1580,10 +1538,7 @@ validate_one_page(Page page, pgFile *file, /* Verify checksum */ if (checksum_version) { - /* - * If checksum is wrong, sleep a bit and then try again - * several times. If it didn't help, throw error - */ + /* Checksums are enabled, so check it. */ if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) == ((PageHeader) page)->pd_checksum) { @@ -1597,19 +1552,25 @@ validate_one_page(Page page, pgFile *file, } else { - /* Get lsn from page header. Ensure that page is from our time */ - lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (lsn > stop_lsn) - elog(WARNING, "File: %s, block %u, checksum is not enabled. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - else - return PAGE_IS_FOUND_AND_VALID; + if (stop_lsn > 0) + { + /* Get lsn from page header. Ensure that page is from our time. + * This is dangerous move, because we cannot be sure that + * lsn from page header is not a garbage. + */ + lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + if (lsn > stop_lsn) + elog(WARNING, "File: %s, block %u, checksum is not enabled. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", + file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, + (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + else + return PAGE_IS_FOUND_AND_VALID; + } } - if (checksum_is_ok) + if (checksum_is_ok && stop_lsn > 0) { /* Get lsn from page header. Ensure that page is from our time */ lsn = PageXLogRecPtrGet(phdr->pd_lsn); @@ -1627,6 +1588,87 @@ validate_one_page(Page page, pgFile *file, return PAGE_IS_FOUND_AND_NOT_VALID; } +bool +check_data_file(backup_files_arg* arguments, + pgFile *file) +{ + FILE *in; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; + int n_blocks_skipped = 0; + int page_state; + char curr_page[BLCKSZ]; + + bool is_valid = true; + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + + /* open backup mode file for read */ + in = fopen(file->path, PG_BINARY_R); + if (in == NULL) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + elog(LOG, "File \"%s\" is not found", file->path); + return true; + } + + elog(ERROR, "cannot open file \"%s\": %s", + file->path, strerror(errno)); + } + + if (file->size % BLCKSZ != 0) + { + fclose(in); + elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); + } + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + for (blknum = 0; blknum < nblocks; blknum++) + { + page_state = prepare_page(arguments, file, InvalidXLogRecPtr, //0 = InvalidXLogRecPtr + blknum, nblocks, in, &n_blocks_skipped, + BACKUP_MODE_FULL, curr_page, false); + + if (page_state == PageIsTruncated) + break; + + if (page_state == PageIsCorrupted) + { + /* Page is corrupted */ + //elog(WARNING, "File %s, block %u is CORRUPTED.", + // file->path, blknum); + is_valid = false; + continue; + } + + /* Page is found and this point, but it may not be 'sane' */ + if (validate_one_page(curr_page, file, blknum, + InvalidXLogRecPtr, + 0) == PAGE_IS_FOUND_AND_NOT_VALID) + { + /* Page is corrupted */ + is_valid = false; + } + + } + + fclose(in); + return is_valid; +} + /* Valiate pages of datafile in backup one by one */ bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 4c15f99e3..c24997f24 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -327,14 +327,14 @@ main(int argc, char *argv[]) help_command(command_name); /* backup_path is required for all pg_probackup commands except help and checkdb */ - if (backup_path == NULL && backup_subcmd != CHECKDB_CMD) + if (backup_path == NULL) { /* * If command line argument is not set, try to read BACKUP_PATH * from environment variable */ backup_path = getenv("BACKUP_PATH"); - if (backup_path == NULL) + if (backup_path == NULL && backup_subcmd != CHECKDB_CMD) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } @@ -353,10 +353,10 @@ main(int argc, char *argv[]) } /* Option --instance is required for all commands except init and show */ - if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) + if (instance_name == NULL) { - if (instance_name == NULL) + if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) elog(ERROR, "required parameter not specified: --instance"); } @@ -399,7 +399,11 @@ main(int argc, char *argv[]) { join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - config_read_opt(path, instance_options, ERROR, true, false); + + if (backup_subcmd == CHECKDB_CMD) + config_read_opt(path, instance_options, ERROR, true, true); + else + config_read_opt(path, instance_options, ERROR, true, false); } } From 0b549f317f314c038aec872a07368e3324c1bb8a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 Mar 2019 11:43:50 +0300 Subject: [PATCH 0485/2107] tests: added checkdb.CheckdbTest.test_checkdb_block_validation --- tests/checkdb.py | 63 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index ad154e77a..c4967bbae 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -14,7 +14,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): - def test_checkdb_index_loss(self): + @unittest.skip("skip") + def checkdb_index_loss(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" fname = self.id().split('.')[3] @@ -22,12 +23,8 @@ def test_checkdb_index_loss(self): node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - } - ) + initdb_params=['--data-checksums']) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -59,4 +56,54 @@ def test_checkdb_index_loss(self): gdb.continue_execution_until_exit() # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_checkdb_block_validation(self): + """make node, corrupt some pages, check that checkdb failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + #self.backup_node( + # backup_dir, 'node', node, + # backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(42000) + f.write(b"bla") + f.flush() + f.close + + print(self.checkdb_node('node', backup_dir, + data_dir=node.data_dir, options=['--block-validation'])) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 19405a95b3cd66ae276f8bd897d23a113df0ec2f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Apr 2019 01:23:50 +0300 Subject: [PATCH 0486/2107] PGPRO-2065: fix validate_one_page --- src/backup.c | 15 ++++++++------- src/data.c | 44 ++++++++++++++++++++++++++++++++++---------- src/pg_probackup.c | 20 +++++++++++++------- src/pg_probackup.h | 2 +- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/backup.c b/src/backup.c index 140f9ee5a..52bc72cf1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -997,8 +997,6 @@ do_block_validation(void) arg->ret = 1; } - pgut_atexit_push(threads_conn_disconnect, NULL); - /* TODO write better info message */ elog(INFO, "Start checking data files"); @@ -1141,12 +1139,17 @@ do_amcheck(void) /* TODO consider moving some code common with do_backup_instance * to separate function ot to pgdata_basic_setup */ int -do_checkdb(bool need_block_validation, bool need_amcheck) +do_checkdb(bool need_amcheck) { + + if (skip_block_validation && !need_amcheck) + elog(ERROR, "--skip-block-validation must be used with --amcheck option"); + pgdata_basic_setup(); - if (need_block_validation) + if (!skip_block_validation) do_block_validation(); + //if (need_amcheck) // do_amcheck(); @@ -1425,7 +1428,6 @@ check_system_identifiers(void) return; } - if (system_id_conn != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but connected instance system id is " UINT64_FORMAT, @@ -2536,7 +2538,7 @@ backup_disconnect(bool fatal, void *userdata) static void threads_conn_disconnect(bool fatal, void *userdata) { - int i; +// int i; elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); // for (i = 0; i < num_threads; i++) @@ -2624,7 +2626,6 @@ check_files(void *arg) elog(WARNING, "unexpected file type %d", buf.st_mode); } - /* Data files check is successful */ if (arguments->ret == 1) arguments->ret = 0; diff --git a/src/data.c b/src/data.c index 275184cc6..52e460ee7 100644 --- a/src/data.c +++ b/src/data.c @@ -369,14 +369,17 @@ prepare_page(backup_files_arg *arguments, * throw an error. */ - if (!page_is_valid) + if (!page_is_valid && + ((strict && !is_ptrack_support) || !strict)) { + /* show this message for checkdb or backup without ptrack support */ elog(WARNING, "CORRUPTION in file %s, block %u", file->path, blknum); + } - if (!is_ptrack_support && strict) + /* Backup with invalid block and without ptrack support must throw error */ + if (!page_is_valid && strict && !is_ptrack_support) elog(ERROR, "Data file corruption. Canceling backup"); - } /* Checkdb not going futher */ if (!strict) @@ -1488,6 +1491,7 @@ validate_one_page(Page page, pgFile *file, XLogRecPtr lsn; bool page_header_is_sane = false; bool checksum_is_ok = false; + bool lsn_from_future = false; /* new level of paranoia */ if (page == NULL) @@ -1518,10 +1522,11 @@ validate_one_page(Page page, pgFile *file, } /* Page is zeroed. No sense to check header and checksum. */ - page_header_is_sane = false; + return PAGE_IS_FOUND_AND_VALID; } else { + /* We should give more information about what exactly is looking fishy */ if (PageGetPageSize(phdr) == BLCKSZ && PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && @@ -1530,7 +1535,14 @@ validate_one_page(Page page, pgFile *file, phdr->pd_upper <= phdr->pd_special && phdr->pd_special <= BLCKSZ && phdr->pd_special == MAXALIGN(phdr->pd_special)) - page_header_is_sane = true; + page_header_is_sane = true; + else + { + /* Page does not looking good */ + page_header_is_sane = false; + elog(WARNING, "Page is not looking healthy: %s, block %i", + file->path, blknum); + } } if (page_header_is_sane) @@ -1548,10 +1560,12 @@ validate_one_page(Page page, pgFile *file, { elog(WARNING, "File: %s blknum %u have wrong checksum", file->path, blknum); + return PAGE_IS_FOUND_AND_NOT_VALID; } } else { + /* Checksums are disabled, so check lsn. */ if (stop_lsn > 0) { /* Get lsn from page header. Ensure that page is from our time. @@ -1561,31 +1575,41 @@ validate_one_page(Page page, pgFile *file, lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) + { elog(WARNING, "File: %s, block %u, checksum is not enabled. " "Page is from future: pageLSN %X/%X stopLSN %X/%X", file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - else - return PAGE_IS_FOUND_AND_VALID; + lsn_from_future = true; + } } + } + /* If checksum is ok, check that page is not from future */ if (checksum_is_ok && stop_lsn > 0) { /* Get lsn from page header. Ensure that page is from our time */ lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) + { elog(WARNING, "File: %s, block %u, checksum is correct. " "Page is from future: pageLSN %X/%X stopLSN %X/%X", file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - else - return PAGE_IS_FOUND_AND_VALID; + lsn_from_future = true; + } } + + if (lsn_from_future) + return PAGE_IS_FOUND_AND_NOT_VALID; + else + return PAGE_IS_FOUND_AND_VALID; } + else + return PAGE_IS_FOUND_AND_NOT_VALID; - return PAGE_IS_FOUND_AND_NOT_VALID; } bool diff --git a/src/pg_probackup.c b/src/pg_probackup.c index c24997f24..e34004d5d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -90,7 +90,7 @@ bool restore_no_validate = false; bool skip_block_validation = false; bool skip_external_dirs = false; -bool do_block_validation = false; +/* checkdb options */ bool do_amcheck = false; /* delete options */ @@ -162,8 +162,7 @@ static ConfigOption cmd_options[] = { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, /* checkdb options */ - { 'b', 157, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, - { 'b', 158, "block-validation", &do_block_validation, SOURCE_CMD_STRICT }, + { 'b', 157, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -411,12 +410,19 @@ main(int argc, char *argv[]) if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) config_get_opt_env(instance_options); + /* Sanity for checkdb, if backup_dir is provided but pgdata and instance are not */ + if (backup_subcmd == CHECKDB_CMD && + backup_path != NULL && + instance_name == NULL && + instance_config.pgdata == NULL) + elog(ERROR, "required parameter not specified: --instance"); + if (backup_subcmd == CHECKDB_CMD && (instance_config.logger.log_level_file != LOG_OFF) - && (&instance_config.logger.log_directory == NULL)) + && (instance_config.logger.log_directory == NULL)) elog(ERROR, "Cannot save checkdb logs to a file. " - "You must specify --log-directory option when running checkdb with " - "--log-level-file option enabled."); + "You must specify --log-directory option when running checkdb with " + "--log-level-file option enabled."); /* Initialize logger */ init_logger(backup_path, &instance_config.logger); @@ -560,7 +566,7 @@ main(int argc, char *argv[]) do_set_config(false); break; case CHECKDB_CMD: - do_checkdb(do_block_validation, do_amcheck); + do_checkdb(do_amcheck); break; case NO_CMD: /* Should not happen */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d8aa9dc92..09a9f30ef 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -416,7 +416,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); -extern int do_checkdb(bool need_block_validation, bool need_amcheck); +extern int do_checkdb(bool need_amcheck); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, From 030832c0baac12ecfc1dd9500e7a01676e464b74 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Apr 2019 03:45:15 +0300 Subject: [PATCH 0487/2107] PGPRO-2065: --amcheck minor fixes --- src/backup.c | 154 +++++++++++++++++++---------------------------- src/utils/pgut.c | 5 +- src/utils/pgut.h | 2 +- 3 files changed, 67 insertions(+), 94 deletions(-) diff --git a/src/backup.c b/src/backup.c index 52bc72cf1..b41cbdeb1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -109,7 +109,7 @@ static void do_amcheck(void); static void *check_files(void *arg); static void *check_indexes(void *arg); static parray* get_index_list(PGresult* res_db, int db_number, - bool* first_db_with_amcheck, PGconn* db_conn); + bool first_db_with_amcheck, PGconn* db_conn); static bool amcheck_one_index(backup_files_arg *arguments, pg_indexEntry *ind); @@ -1049,13 +1049,19 @@ do_amcheck(void) pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + res_db = pgut_execute(backup_conn, + "SELECT datname, oid, dattablespace " + "FROM pg_database " + "WHERE datname NOT IN ('template0', 'template1')", 0, NULL); + n_databases = PQntuples(res_db); /* TODO Warn user that one connection is used for snaphot */ - if (num_threads > 1) - num_threads--; + //if (num_threads > 1) + // num_threads--; + + elog(INFO, "Start checking instance with amcheck"); /* For each database check indexes. In parallel. */ for(i = 0; i < n_databases; i++) @@ -1069,7 +1075,8 @@ do_amcheck(void) } index_list = get_index_list(res_db, i, - &first_db_with_amcheck, db_conn); + first_db_with_amcheck, db_conn); + if (index_list == NULL) { if (db_conn) @@ -1077,6 +1084,8 @@ do_amcheck(void) continue; } + first_db_with_amcheck = false; + /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -1099,8 +1108,6 @@ do_amcheck(void) pgut_atexit_push(threads_conn_disconnect, NULL); - elog(INFO, "Start checking indexes with amcheck"); - /* Run threads */ for (j = 0; j < num_threads; j++) { @@ -1115,7 +1122,7 @@ do_amcheck(void) for (j = 0; j < num_threads; j++) { pthread_join(threads[j], NULL); - if (threads_args[j].ret == 1) + if (threads_args[j].ret > 0) check_isok = false; } pgut_disconnect(db_conn); @@ -1123,16 +1130,16 @@ do_amcheck(void) /* TODO write better info message */ if (check_isok) - elog(INFO, "Indexes are checked"); + elog(INFO, "Indexes are valid"); else - elog(ERROR, "Indexs checking failed"); - - if (backup_files_list) - { - parray_walk(backup_files_list, pgFileFree); - parray_free(backup_files_list); - backup_files_list = NULL; - } + elog(ERROR, "Some indexes are corrupted"); + + //if (backup_files_list) + //{ + // parray_walk(backup_files_list, pgFileFree); + // parray_free(backup_files_list); + // backup_files_list = NULL; + //} } /* Entry point of pg_probackup CHECKDB subcommand. */ @@ -1150,8 +1157,8 @@ do_checkdb(bool need_amcheck) if (!skip_block_validation) do_block_validation(); - //if (need_amcheck) - // do_amcheck(); + if (need_amcheck) + do_amcheck(); return 0; } @@ -2540,7 +2547,7 @@ threads_conn_disconnect(bool fatal, void *userdata) { // int i; - elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); +// elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); // for (i = 0; i < num_threads; i++) // { // backup_files_arg *arg = &(threads_args[i]); @@ -2654,7 +2661,9 @@ check_indexes(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "interrupted during checkdb --amcheck"); - elog(VERBOSE, "Checking index number %d of %d : \"%s\" ", i,n_indexes, ind->name); + if (progress) + elog(INFO, "Progress: (%d/%d). Processing index '%s.%s'", + i + 1, n_indexes, ind->amcheck_nspname, ind->name); if (arguments->backup_conn == NULL) { @@ -2666,26 +2675,11 @@ check_indexes(void *arg) ind->dbname, instance_config.pguser); arguments->cancel_conn = PQgetCancel(arguments->backup_conn); - - res = pgut_execute_parallel(arguments->backup_conn, - arguments->cancel_conn, - "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL, false); - PQclear(res); - - query = palloc(strlen("SET TRANSACTION SNAPSHOT '' ") + strlen(ind->snapshot)); - sprintf(query, "SET TRANSACTION SNAPSHOT '%s'", ind->snapshot); - - res = pgut_execute_parallel(arguments->backup_conn, - arguments->cancel_conn, - query, 0, NULL, false); - PQclear(res); - - free(query); } /* remember that we have a failed check */ if (!amcheck_one_index(arguments, ind)) - arguments->ret = 1; + arguments->ret = 2; } /* Close connection */ @@ -2693,7 +2687,8 @@ check_indexes(void *arg) pgut_disconnect(arguments->backup_conn); /* TODO where should we set arguments->ret to 1? */ - arguments->ret = 0; + if (arguments->ret == 1) + arguments->ret = 0; return NULL; } @@ -3410,7 +3405,7 @@ pg_ptrack_get_block(backup_files_arg *arguments, res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", - 4, (const char **)params, true); + 4, (const char **)params, true, false); if (PQnfields(res) != 1) { @@ -3478,10 +3473,10 @@ check_external_for_tablespaces(parray *external_list) PQclear(res); } -/* Clear ptrack files in all databases of the instance we connected to */ +/* Get index list for given database */ static parray* get_index_list(PGresult *res_db, int db_number, - bool *first_db_with_amcheck, PGconn *db_conn) + bool first_db_with_amcheck, PGconn *db_conn) { PGresult *res; char *nspname = NULL; @@ -3489,27 +3484,28 @@ get_index_list(PGresult *res_db, int db_number, int i; dbname = PQgetvalue(res_db, db_number, 0); - if (strcmp(dbname, "template0") == 0) - return NULL; db_conn = pgut_connect(instance_config.pghost, instance_config.pgport, dbname, instance_config.pguser); - res = pgut_execute(db_conn, "select extname, nspname, extversion from pg_namespace " - "n join pg_extension e on n.oid=e.extnamespace where e.extname='amcheck'", + res = pgut_execute(db_conn, "SELECT extname, nspname, extversion " + "FROM pg_namespace n " + "JOIN pg_extension e " + "ON n.oid=e.extnamespace " + "WHERE e.extname='amcheck'", 0, NULL); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); - elog(ERROR, "cannot check if amcheck is installed in database %s: %s", + elog(ERROR, "Cannot check if amcheck is installed in database %s: %s", dbname, PQerrorMessage(db_conn)); } if (PQntuples(res) < 1) { - elog(WARNING, "extension amcheck is not installed in database %s", dbname); + elog(WARNING, "Extension amcheck is not installed in database %s", dbname); return NULL; } @@ -3520,56 +3516,32 @@ get_index_list(PGresult *res_db, int db_number, * In order to avoid duplicates, select global indexes * (tablespace pg_global with oid 1664) only once */ - if (*first_db_with_amcheck) + if (first_db_with_amcheck) { - res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); - PQclear(res); - - res = pgut_execute(db_conn, "SELECT pg_export_snapshot();", 0, NULL); - - if (PQntuples(res) < 1) - elog(ERROR, "Failed to export snapshot for amcheck in database %s", dbname); - - snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); - strcpy(snapshot, PQgetvalue(res, 0, 0)); - - PQclear(res); + /* select only valid btree and persistent indexes */ res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" " FROM pg_index idx " " JOIN pg_class cls ON cls.oid=idx.indexrelid " " JOIN pg_am am ON am.oid=cls.relam " " WHERE am.amname='btree' AND cls.relpersistence != 't'" " AND idx.indisready AND idx.indisvalid; ", 0, NULL); - *first_db_with_amcheck = false; } else { - res = pgut_execute(db_conn, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", 0, NULL); - PQclear(res); - - res = pgut_execute(db_conn, "SELECT pg_export_snapshot();", 0, NULL); - - if (PQntuples(res) < 1) - elog(ERROR, "Failed to export snapshot for amcheck in database %s", dbname); - if (snapshot) - free(snapshot); - snapshot = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); - strcpy(snapshot, PQgetvalue(res, 0, 0)); - - PQclear(res); - - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" - " FROM pg_index idx " - " JOIN pg_class cls ON cls.oid=idx.indexrelid " - " JOIN pg_am am ON am.oid=cls.relam " - " WHERE am.amname='btree' AND cls.relpersistence != 't'" - " AND idx.indisready AND idx.indisvalid AND cls.reltablespace!=1664; ", 0, NULL); + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " + "FROM pg_index idx " + "JOIN pg_class cls ON cls.oid=idx.indexrelid " + "JOIN pg_am am ON am.oid=cls.relam " + "JOIN pg_tablespace tbl ON tbl.oid=cls.reltablespace " + "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "AND idx.indisready AND idx.indisvalid " + "AND tbl.spcname !='pg_global'", 0, NULL); } /* add info needed to check indexes into index_list */ - for(i = 0; i < PQntuples(res); i++) + for (i = 0; i < PQntuples(res); i++) { pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); char *name = NULL; @@ -3582,9 +3554,6 @@ get_index_list(PGresult *res_db, int db_number, ind->dbname = pgut_malloc(strlen(dbname) + 1); strcpy(ind->dbname, dbname); - ind->snapshot = pgut_malloc(strlen(snapshot) + 1); - strcpy(ind->snapshot, snapshot); - ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); strcpy(ind->amcheck_nspname, nspname); pg_atomic_clear_flag(&ind->lock); @@ -3598,6 +3567,8 @@ get_index_list(PGresult *res_db, int db_number, PQclear(res); + elog(INFO, "Amcheck database '%s'", dbname); + return index_list; } @@ -3613,27 +3584,26 @@ amcheck_one_index(backup_files_arg *arguments, sprintf(params[0], "%i", ind->indexrelid); - elog(VERBOSE, "amcheck index: '%s'", ind->name); - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, - query, 1, (const char **)params, true); + query, 1, (const char **)params, true, true); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - elog(VERBOSE, "amcheck failed for relation oid %u: %s", - ind->indexrelid, PQresultErrorMessage(res)); + elog(WARNING, "Amcheck failed for index: '%s.%s': %s", + ind->amcheck_nspname, + ind->name, PQresultErrorMessage(res)); pfree(params[0]); PQclear(res); return false; } else - elog(VERBOSE, "amcheck succeeded for relation oid %u", - ind->indexrelid); + elog(LOG, "Amcheck succeeded for index: '%s.%s'", + ind->amcheck_nspname, ind->name); pfree(params[0]); PQclear(res); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 7e36eaf8d..bdd99f2bb 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -359,7 +359,7 @@ PGresult * pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, const char *query, int nParams, const char **params, - bool text_result) + bool text_result, bool ok_error) { PGresult *res; @@ -405,6 +405,9 @@ pgut_execute_parallel(PGconn* conn, case PGRES_COPY_IN: break; default: + if (ok_error && PQresultStatus(res) == PGRES_FATAL_ERROR) + break; + elog(ERROR, "query failed: %squery was: %s", PQerrorMessage(conn), query); break; diff --git a/src/utils/pgut.h b/src/utils/pgut.h index ca44fb391..350b96726 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -57,7 +57,7 @@ extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nPar const char **params, bool text_result, bool ok_error); extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, const char *query, int nParams, - const char **params, bool text_result); + const char **params, bool text_result, bool ok_error); extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); extern void pgut_cancel(PGconn* conn); extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); From 47283e113ff8bb2e60899c5e60bbe139545bff2e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Apr 2019 03:57:04 +0300 Subject: [PATCH 0488/2107] PGPRO-2065: add thread number to elog messages --- src/backup.c | 18 ++++++++++-------- src/pg_probackup.h | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index b41cbdeb1..f2cd770a5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -49,7 +49,6 @@ const char *progname = "pg_probackup"; static parray *backup_files_list = NULL; static parray *index_list = NULL; - /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -1102,6 +1101,7 @@ do_amcheck(void) arg->prev_start_lsn = InvalidXLogRecPtr; arg->backup_conn = NULL; arg->cancel_conn = NULL; + arg->thread_num = j + 1; /* By default there are some error */ arg->ret = 1; } @@ -2659,11 +2659,13 @@ check_indexes(void *arg) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during checkdb --amcheck"); + elog(ERROR, "Thread [%d]: interrupted during checkdb --amcheck", + arguments->thread_num); if (progress) - elog(INFO, "Progress: (%d/%d). Processing index '%s.%s'", - i + 1, n_indexes, ind->amcheck_nspname, ind->name); + elog(INFO, "Thread [%d]. Progress: (%d/%d). Processing index '%s.%s'", + arguments->thread_num, i + 1, n_indexes, + ind->amcheck_nspname, ind->name); if (arguments->backup_conn == NULL) { @@ -3593,8 +3595,8 @@ amcheck_one_index(backup_files_arg *arguments, if (PQresultStatus(res) != PGRES_TUPLES_OK) { - elog(WARNING, "Amcheck failed for index: '%s.%s': %s", - ind->amcheck_nspname, + elog(WARNING, "Thread [%d]. Amcheck failed for index: '%s.%s': %s", + arguments->thread_num, ind->amcheck_nspname, ind->name, PQresultErrorMessage(res)); pfree(params[0]); @@ -3602,8 +3604,8 @@ amcheck_one_index(backup_files_arg *arguments, return false; } else - elog(LOG, "Amcheck succeeded for index: '%s.%s'", - ind->amcheck_nspname, ind->name); + elog(LOG, "Thread [%d]. Amcheck succeeded for index: '%s.%s'", + arguments->thread_num, ind->amcheck_nspname, ind->name); pfree(params[0]); PQclear(res); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 09a9f30ef..3f56abdb3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -318,6 +318,7 @@ typedef struct PGconn *backup_conn; PGcancel *cancel_conn; parray *index_list; + int thread_num; /* * Return value from the thread. From 8cf98dacaf31062ad2caa58bb1366bf9c770fdab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Apr 2019 10:26:21 +0300 Subject: [PATCH 0489/2107] PGPRO-2065: do not ignore indisready and indisvalid indexes --- src/backup.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index f2cd770a5..af24ff5fa 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1056,11 +1056,7 @@ do_amcheck(void) n_databases = PQntuples(res_db); - /* TODO Warn user that one connection is used for snaphot */ - //if (num_threads > 1) - // num_threads--; - - elog(INFO, "Start checking instance with amcheck"); + elog(INFO, "Start checking PostgreSQL instance with amcheck"); /* For each database check indexes. In parallel. */ for(i = 0; i < n_databases; i++) @@ -3522,12 +3518,13 @@ get_index_list(PGresult *res_db, int db_number, { /* select only valid btree and persistent indexes */ - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname" - " FROM pg_index idx " - " JOIN pg_class cls ON cls.oid=idx.indexrelid " - " JOIN pg_am am ON am.oid=cls.relam " - " WHERE am.amname='btree' AND cls.relpersistence != 't'" - " AND idx.indisready AND idx.indisvalid; ", 0, NULL); + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " + "FROM pg_index idx " + "JOIN pg_class cls ON cls.oid=idx.indexrelid " + "JOIN pg_am am ON am.oid=cls.relam " + "WHERE am.amname='btree' AND cls.relpersistence != 't'", + //"AND idx.indisready AND idx.indisvalid", + 0, NULL); } else { @@ -3538,8 +3535,8 @@ get_index_list(PGresult *res_db, int db_number, "JOIN pg_am am ON am.oid=cls.relam " "JOIN pg_tablespace tbl ON tbl.oid=cls.reltablespace " "WHERE am.amname='btree' AND cls.relpersistence != 't' " - "AND idx.indisready AND idx.indisvalid " - "AND tbl.spcname !='pg_global'", 0, NULL); + //"AND idx.indisready AND idx.indisvalid " + "AND tbl.spcname !='pg_global'",0, NULL); } /* add info needed to check indexes into index_list */ From b5d8a77a612ad50f1a12a7a0857330cdeec6d409 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 1 Apr 2019 14:27:59 +0300 Subject: [PATCH 0490/2107] Do not forget to release base36enc_dup()'s result --- src/restore.c | 2 ++ src/validate.c | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/restore.c b/src/restore.c index 38fa11ed6..7efd433a4 100644 --- a/src/restore.c +++ b/src/restore.c @@ -219,6 +219,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } } + pg_free(missing_backup_id); /* No point in doing futher */ elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time)); } @@ -255,6 +256,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } } + pg_free(parent_backup_id); tmp_backup = find_parent_full_backup(dest_backup); /* sanity */ diff --git a/src/validate.c b/src/validate.c index 56fa699bd..472f72463 100644 --- a/src/validate.c +++ b/src/validate.c @@ -382,7 +382,6 @@ do_validate_all(void) static void do_validate_instance(void) { - char *current_backup_id; int i; int j; parray *backups; @@ -397,7 +396,6 @@ do_validate_instance(void) for (i = 0; i < parray_num(backups); i++) { pgBackup *base_full_backup; - char *parent_backup_id; current_backup = (pgBackup *) parray_get(backups, i); @@ -412,6 +410,7 @@ do_validate_instance(void) /* chain is broken */ if (result == 0) { + char *parent_backup_id; /* determine missing backup ID */ parent_backup_id = base36enc_dup(tmp_backup->parent_backup); @@ -430,31 +429,31 @@ do_validate_instance(void) elog(WARNING, "Backup %s has missing parent %s", base36enc(current_backup->start_time), parent_backup_id); } + pg_free(parent_backup_id); continue; } /* chain is whole, but at least one parent is invalid */ else if (result == 1) { - /* determine corrupt backup ID */ - parent_backup_id = base36enc_dup(tmp_backup->start_time); - /* Oldest corrupt backup has a chance for revalidation */ if (current_backup->start_time != tmp_backup->start_time) { + char *backup_id = base36enc_dup(tmp_backup->start_time); /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(current_backup->start_time), parent_backup_id, + base36enc(current_backup->start_time), backup_id, status2str(tmp_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(current_backup->start_time),parent_backup_id, + base36enc(current_backup->start_time), backup_id, status2str(tmp_backup->status)); } + pg_free(backup_id); continue; } base_full_backup = find_parent_full_backup(current_backup); @@ -495,6 +494,7 @@ do_validate_instance(void) */ if (current_backup->status != BACKUP_STATUS_OK) { + char *current_backup_id; /* This is ridiculous but legal. * PAGE_b2 <- OK * PAGE_a2 <- OK From 66d03cc929fff0cfeacd9284544e00234117de51 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 11:55:59 +0300 Subject: [PATCH 0491/2107] PGPRO-2573: refactoring do_retention_internal() --- src/delete.c | 169 ++++++++++++++++++++++++--------------------- src/pg_probackup.h | 1 - 2 files changed, 90 insertions(+), 80 deletions(-) diff --git a/src/delete.c b/src/delete.c index b71ebf93c..a1ce4d931 100644 --- a/src/delete.c +++ b/src/delete.c @@ -16,7 +16,11 @@ static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, uint32 xlog_seg_size); -static void do_retention_internal(void); +static void do_retention_internal(parray *backup_list, parray *to_keep_list, + parray *to_purge_list); +static void do_retention_merge(parray *backup_list, parray *to_keep_list, + parray *to_purge_list); +static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); static void do_retention_wal(void); static bool backup_deleted = false; /* At least one backup was deleted */ @@ -108,8 +112,15 @@ do_delete(time_t backup_id) int do_retention(void) { + parray *backup_list = NULL; + parray *to_keep_list = parray_new(); + parray *to_purge_list = parray_new(); + bool retention_is_set = false; /* At least one retention policy is set */ + /* Get a complete list of backups. */ + backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + if (delete_expired || merge_expired) { if (instance_config.retention_redundancy > 0) @@ -131,9 +142,17 @@ int do_retention(void) retention_is_set = true; } - if (retention_is_set && ((delete_expired || merge_expired) || dry_run)) - do_retention_internal(); + /* show fancy message */ + if (retention_is_set) + do_retention_internal(backup_list, to_keep_list, to_purge_list); + + if (merge_expired && !dry_run) + do_retention_merge(backup_list, to_keep_list, to_purge_list); + + if (delete_expired && !dry_run) + do_retention_purge(to_keep_list, to_purge_list); + /* TODO: some sort of dry run for delete_wal */ if (delete_wal && !dry_run) do_retention_wal(); @@ -145,6 +164,12 @@ int do_retention(void) else elog(INFO, "There are no backups to delete by retention policy"); + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + parray_free(to_keep_list); + parray_free(to_purge_list); + return 0; } @@ -160,13 +185,9 @@ int do_retention(void) * but if invalid backup is not guarded by retention - it is removed */ static void -do_retention_internal(void) +do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list) { - parray *backup_list = NULL; - parray *to_purge_list = parray_new(); - parray *to_keep_list = parray_new(); int i; - int j; time_t current_time; bool backup_list_is_empty = false; @@ -178,8 +199,7 @@ do_retention_internal(void) /* For fancy reporting */ float actual_window = 0; - /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + /* sanity */ if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -257,36 +277,66 @@ do_retention_internal(void) parray_append(to_purge_list, backup); continue; } + } + + /* sort keep_list and purge list */ + parray_qsort(to_keep_list, pgBackupCompareIdDesc); + parray_qsort(to_purge_list, pgBackupCompareIdDesc); + + /* FULL + * PAGE + * PAGE <- Only such backups must go into keep list + ---------retention window ---- + * PAGE + * FULL + * PAGE + * FULL + */ + + for (i = 0; i < parray_num(backup_list); i++) + { + + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); /* Do not keep invalid backups by retention */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) continue; - elog(VERBOSE, "Mark backup %s for retention.", base36enc(backup->start_time)); - parray_append(to_keep_list, backup); + if (backup->backup_mode == BACKUP_MODE_FULL) + continue; + + /* orphan backup cannot be in keep list */ + if (!backup->parent_backup_link) + continue; + + if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) + continue; + + /* if parent in purge_list, add backup to keep list */ + if (parray_bsearch(to_purge_list, + backup->parent_backup_link, + pgBackupCompareIdDesc)) + { + /* make keep list a bit sparse */ + parray_append(to_keep_list, backup); + continue; + } } /* Message about retention state of backups * TODO: Float is ugly, rewrite somehow. */ - /* sort keep_list and purge list */ - parray_qsort(to_keep_list, pgBackupCompareIdDesc); - parray_qsort(to_purge_list, pgBackupCompareIdDesc); - cur_full_backup_num = 1; for (i = 0; i < parray_num(backup_list); i++) { - char *action = "Ignore"; + char *action = "Keep"; pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (parray_bsearch(to_keep_list, backup, pgBackupCompareIdDesc)) - action = "Keep"; - if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) - action = "Keep"; + action = "Purge"; if (backup->recovery_time == 0) actual_window = 0; @@ -305,33 +355,21 @@ do_retention_internal(void) if (backup->backup_mode == BACKUP_MODE_FULL) cur_full_backup_num++; } +} - if (dry_run) - goto finish; - - /* Extreme example of keep_list - * - * FULLc <- keep - * PAGEb2 <- keep - * PAGEb1 <- keep - * PAGEa2 <- keep - * PAGEa1 <- keep - * FULLb <- in purge_list - * FULLa <- in purge_list - */ - - /* Go to purge */ - if (delete_expired && !merge_expired) - goto purge; +static void +do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list) +{ + int i; + int j; /* IMPORTANT: we can merge to only those FULL backup, that is NOT * guarded by retention and final targets of such merges must be * incremental backup that is guarded by retention !!! * - * * PAGE4 E * PAGE3 D - * --------retention window --- + --------retention window --- * PAGE2 C * PAGE1 B * FULL A @@ -352,32 +390,9 @@ do_retention_internal(void) /* keep list may shrink during merge */ if (!keep_backup) - break; - - elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time)); - - /* In keep list we are looking for incremental backups */ - if (keep_backup->backup_mode == BACKUP_MODE_FULL) - continue; - - /* Retain orphan backups in keep_list */ - if (!keep_backup->parent_backup_link) continue; - /* If parent of current backup is also in keep list, go to the next */ - if (parray_bsearch(to_keep_list, - keep_backup->parent_backup_link, - pgBackupCompareIdDesc)) - { - /* make keep list a bit sparse */ - elog(LOG, "Sparsing keep list, remove %s", base36enc(keep_backup->start_time)); - parray_remove(to_keep_list, i); - i--; - continue; - } - - elog(INFO, "Lookup parents for backup %s", - base36enc(keep_backup->start_time)); + elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time)); /* Got valid incremental backup, find its FULL ancestor */ full_backup = find_parent_full_backup(keep_backup); @@ -473,9 +488,7 @@ do_retention_internal(void) /* Try to remove merged incremental backup from both keep and purge lists */ parray_rm(to_purge_list, from_backup, pgBackupCompareId); - - if (parray_rm(to_keep_list, from_backup, pgBackupCompareId) && (i >= 0)) - i--; + parray_set(to_keep_list, i, NULL); } /* Cleanup */ @@ -484,11 +497,13 @@ do_retention_internal(void) elog(INFO, "Retention merging finished"); - if (!delete_expired) - goto finish; +} -/* Do purging here */ -purge: +static void +do_retention_purge(parray *to_keep_list, parray *to_purge_list) +{ + int i; + int j; /* Remove backups by retention policy. Retention policy is configured by * retention_redundancy and retention_window @@ -517,6 +532,10 @@ do_retention_internal(void) pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); + /* item could have been nullified in merge */ + if (!keep_backup) + continue; + /* Full backup cannot be a descendant */ if (keep_backup->backup_mode == BACKUP_MODE_FULL) continue; @@ -557,14 +576,6 @@ do_retention_internal(void) backup_deleted = true; } - -finish: - /* Cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - parray_free(to_keep_list); - parray_free(to_purge_list); - } /* Purge WAL */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 56d1d31a6..85c1fd53f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -455,7 +455,6 @@ extern int do_show(time_t requested_backup_id); extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); extern int do_retention(void); -extern int do_retention_purge(void); extern int do_delete_instance(void); /* in fetch.c */ From fc16c52f0d80aff17c5d2c1ffc3aab2e9393a789 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 11:59:01 +0300 Subject: [PATCH 0492/2107] PGPRO-2573: minor changes --- src/delete.c | 53 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/delete.c b/src/delete.c index a1ce4d931..21653fe5f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -117,10 +117,15 @@ int do_retention(void) parray *to_purge_list = parray_new(); bool retention_is_set = false; /* At least one retention policy is set */ + bool backup_list_is_empty = false; /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + /* sanity */ + if (parray_num(backup_list) == 0) + backup_list_is_empty = true; + if (delete_expired || merge_expired) { if (instance_config.retention_redundancy > 0) @@ -142,14 +147,17 @@ int do_retention(void) retention_is_set = true; } + if (retention_is_set && backup_list_is_empty) + elog(WARNING, "Backup list is empty, retention purge and merge are problematic"); + /* show fancy message */ - if (retention_is_set) + if (retention_is_set && !backup_list_is_empty) do_retention_internal(backup_list, to_keep_list, to_purge_list); if (merge_expired && !dry_run) do_retention_merge(backup_list, to_keep_list, to_purge_list); - if (delete_expired && !dry_run) + if (delete_expired && !dry_run && !backup_list_is_empty) do_retention_purge(to_keep_list, to_purge_list); /* TODO: some sort of dry run for delete_wal */ @@ -203,40 +211,31 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg if (parray_num(backup_list) == 0) backup_list_is_empty = true; - if (backup_list_is_empty) - { - elog(WARNING, "Backup list is empty, purging won't be executed"); - return; - } - /* Get current time */ current_time = time(NULL); /* Calculate n_full_backups and days_threshold */ - if (!backup_list_is_empty) + if (instance_config.retention_redundancy > 0) { - if (instance_config.retention_redundancy > 0) + for (i = 0; i < parray_num(backup_list); i++) { - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Consider only valid backups for Redundancy */ - if (instance_config.retention_redundancy > 0 && - backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - { - n_full_backups++; - } + /* Consider only valid backups for Redundancy */ + if (instance_config.retention_redundancy > 0 && + backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) + { + n_full_backups++; } } + } - if (instance_config.retention_window > 0) - { - days_threshold = current_time - - (instance_config.retention_window * 60 * 60 * 24); - } + if (instance_config.retention_window > 0) + { + days_threshold = current_time - + (instance_config.retention_window * 60 * 60 * 24); } elog(INFO, "Evaluate backups by retention"); @@ -588,7 +587,7 @@ do_retention_wal(void) TimeLineID oldest_tli = 0; bool backup_list_is_empty = false; - /* Get new backup_list. Should we */ + /* Get list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) From 33ad94448d4c3032252bf25f19b768b82210a746 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 12:11:47 +0300 Subject: [PATCH 0493/2107] PGPRO-2573: minor changes to comments --- src/delete.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/delete.c b/src/delete.c index 21653fe5f..c3db6ab64 100644 --- a/src/delete.c +++ b/src/delete.c @@ -150,7 +150,7 @@ int do_retention(void) if (retention_is_set && backup_list_is_empty) elog(WARNING, "Backup list is empty, retention purge and merge are problematic"); - /* show fancy message */ + /* Populate purge and keep lists, and show retention state messages */ if (retention_is_set && !backup_list_is_empty) do_retention_internal(backup_list, to_keep_list, to_purge_list); @@ -197,7 +197,6 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { int i; time_t current_time; - bool backup_list_is_empty = false; /* For retention calculation */ uint32 n_full_backups = 0; @@ -207,10 +206,6 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* For fancy reporting */ float actual_window = 0; - /* sanity */ - if (parray_num(backup_list) == 0) - backup_list_is_empty = true; - /* Get current time */ current_time = time(NULL); @@ -294,7 +289,6 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg for (i = 0; i < parray_num(backup_list); i++) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); /* Do not keep invalid backups by retention */ @@ -302,6 +296,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg backup->status != BACKUP_STATUS_DONE) continue; + /* only incremental backups should be in keep list */ if (backup->backup_mode == BACKUP_MODE_FULL) continue; @@ -309,6 +304,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg if (!backup->parent_backup_link) continue; + /* skip if backup already in purge list */ if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) continue; From 9da90b6e8f5dfe774a73bcb152c8ed8a69818492 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 12:24:49 +0300 Subject: [PATCH 0494/2107] PGPRO-2573: rename 'Purge' and 'Keep' to 'Expired' and 'Active' for retention message, minor changes for comments --- src/delete.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/delete.c b/src/delete.c index c3db6ab64..fc05182a3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -110,6 +110,16 @@ do_delete(time_t backup_id) parray_free(backup_list); } +/* + * Merge and purge backups by retention policy. Retention policy is configured by + * retention_redundancy and retention_window variables. + * + * Invalid backups handled in Oracle style, so invalid backups are ignored + * for the purpose of retention fulfillment, + * i.e. CORRUPT full backup do not taken in account when deteremine + * which FULL backup should be keeped for redundancy obligation(only valid do), + * but if invalid backup is not guarded by retention - it is removed + */ int do_retention(void) { parray *backup_list = NULL; @@ -122,7 +132,6 @@ int do_retention(void) /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - /* sanity */ if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -136,14 +145,13 @@ int do_retention(void) if (instance_config.retention_redundancy == 0 && instance_config.retention_window == 0) { - /* Retention is disabled but we still can cleanup wal - */ + /* Retention is disabled but we still can cleanup wal */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) return 0; } else - /* At least one retention policy is active */ + /* At least one retention policy is active */ retention_is_set = true; } @@ -182,15 +190,9 @@ int do_retention(void) } -/* - * Merge and purge backups by retention policy. Retention policy is configured by - * retention_redundancy and retention_window variables. - * - * Invalid backups handled in Oracle style, so invalid backups are ignored - * for the purpose of retention fulfillment, - * i.e. CORRUPT full backup do not taken in account when deteremine - * which FULL backup should be keeped for redundancy obligation(only valid do), - * but if invalid backup is not guarded by retention - it is removed +/* Evaluate every backup by retention policies and populate purge and keep lists. + * Also for every backup print proposed action('Active' or 'Expired') according + * to active retention policies. */ static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list) @@ -326,12 +328,12 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg cur_full_backup_num = 1; for (i = 0; i < parray_num(backup_list); i++) { - char *action = "Keep"; + char *action = "Active"; pgBackup *backup = (pgBackup *) parray_get(backup_list, i); if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) - action = "Purge"; + action = "Expired"; if (backup->recovery_time == 0) actual_window = 0; @@ -352,6 +354,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg } } +/* Merge partially expired incremental chains */ static void do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list) { @@ -494,6 +497,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l } +/* Purge expired backups */ static void do_retention_purge(parray *to_keep_list, parray *to_purge_list) { From fcf8fc57a27c6327deaad26a36a85068886f2949 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 17:08:24 +0300 Subject: [PATCH 0495/2107] PGPRO-2065: add support for heapallindexed in newer versions of amcheck --- src/backup.c | 65 ++++++++++++++++++++++++++++++++-------------- src/help.c | 56 +++++++++++++++++++++++++++++++++++++++ src/pg_probackup.c | 2 ++ src/pg_probackup.h | 3 +++ 4 files changed, 107 insertions(+), 19 deletions(-) diff --git a/src/backup.c b/src/backup.c index af24ff5fa..abeead402 100644 --- a/src/backup.c +++ b/src/backup.c @@ -74,6 +74,7 @@ static int is_ptrack_enable = false; bool is_ptrack_support = false; bool is_checksum_enabled = false; bool exclusive_backup = false; +bool heapallindexed_is_supported = false; /* Backup connections */ static PGconn *backup_conn = NULL; @@ -1044,6 +1045,7 @@ do_amcheck(void) int n_databases = 0; bool first_db_with_amcheck = true; PGconn *db_conn = NULL; + bool db_skipped = false; pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); @@ -1076,6 +1078,7 @@ do_amcheck(void) { if (db_conn) pgut_disconnect(db_conn); + db_skipped = true; continue; } @@ -1125,17 +1128,14 @@ do_amcheck(void) } /* TODO write better info message */ - if (check_isok) + if (check_isok && !db_skipped) elog(INFO, "Indexes are valid"); - else + + if (!check_isok) elog(ERROR, "Some indexes are corrupted"); - //if (backup_files_list) - //{ - // parray_walk(backup_files_list, pgFileFree); - // parray_free(backup_files_list); - // backup_files_list = NULL; - //} + if (db_skipped) + elog(ERROR, "Some databases were not checked"); } /* Entry point of pg_probackup CHECKDB subcommand. */ @@ -1156,6 +1156,7 @@ do_checkdb(bool need_amcheck) if (need_amcheck) do_amcheck(); + /* need to exit with 1 if some corruption is found */ return 0; } @@ -3487,12 +3488,16 @@ get_index_list(PGresult *res_db, int db_number, dbname, instance_config.pguser); - res = pgut_execute(db_conn, "SELECT extname, nspname, extversion " + res = pgut_execute(db_conn, "SELECT " + "extname, nspname, extversion, " + "extversion::numeric in (1.1, 1) as old_version " "FROM pg_namespace n " "JOIN pg_extension e " "ON n.oid=e.extnamespace " - "WHERE e.extname='amcheck'", - 0, NULL); + "WHERE e.extname IN ('amcheck', 'amcheck_next') " + "ORDER BY extversion DESC " + "LIMIT 1", + 0, NULL); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3503,13 +3508,21 @@ get_index_list(PGresult *res_db, int db_number, if (PQntuples(res) < 1) { - elog(WARNING, "Extension amcheck is not installed in database %s", dbname); + elog(WARNING, "Extension 'amcheck' or 'amcheck_next' are not installed in database %s", dbname); return NULL; } nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); strcpy(nspname, PQgetvalue(res, 0, 1)); + /* heapallindexed_is_supported is database specific */ + if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && + strcmp(PQgetvalue(res, 0, 2), "1") != 0) + heapallindexed_is_supported = true; + + elog(INFO, "Checking database %s using module '%s' version %s from schema '%s'", + dbname, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + /* * In order to avoid duplicates, select global indexes * (tablespace pg_global with oid 1664) only once @@ -3566,8 +3579,6 @@ get_index_list(PGresult *res_db, int db_number, PQclear(res); - elog(INFO, "Amcheck database '%s'", dbname); - return index_list; } @@ -3577,18 +3588,34 @@ amcheck_one_index(backup_files_arg *arguments, pg_indexEntry *ind) { PGresult *res; - char *params[1]; - char *query; + char *params[2]; + char *query = NULL; + params[0] = palloc(64); + /* first argument is index oid */ sprintf(params[0], "%i", ind->indexrelid); + /* second argument is heapallindexed */ + params[1] = heapallindexed ? "true" : "false"; - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); - sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); + if (heapallindexed_is_supported) + { + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); + sprintf(query, "SELECT %s.bt_index_check($1, $2)", ind->amcheck_nspname); - res = pgut_execute_parallel(arguments->backup_conn, + res = pgut_execute_parallel(arguments->backup_conn, + arguments->cancel_conn, + query, 2, (const char **)params, true, true); + } + else + { + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); + sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); + + res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, query, 1, (const char **)params, true, true); + } if (PQresultStatus(res) != PGRES_TUPLES_OK) { diff --git a/src/help.c b/src/help.c index 6a5c2c9f6..0d540658f 100644 --- a/src/help.c +++ b/src/help.c @@ -140,6 +140,11 @@ help_pg_probackup(void) printf(_(" [--timeline=timeline]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); + printf(_(" [--amcheck] [--skip-block-validation]\n")); + printf(_(" [--heapallindexed] [--work-mem]\n")); + printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); printf(_(" [--format=format]\n")); @@ -396,6 +401,57 @@ help_validate(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } +static void +help_checkdb(void) +{ + printf(_("%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); + printf(_(" [--amcheck] [--skip-block-validation]\n")); + printf(_(" [--heapallindexed] [--work-mem=work_mem_size]\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + + printf(_(" --progress show progress\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --skip-block-validation skip file-level block checking\n")); + printf(_(" can be used only with --amcheck option\n")); + printf(_(" --amcheck in addition to file-level block checking\n")); + printf(_(" check btree indexes using 'amcheck' or 'amcheck_next' extension")); + printf(_(" --heapallindexed also check that heap is indexed\n")); + printf(_(" can be used only with --amcheck option\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); +} + static void help_show(void) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e34004d5d..8e55d056b 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -92,6 +92,7 @@ bool skip_external_dirs = false; /* checkdb options */ bool do_amcheck = false; +bool heapallindexed = false; /* delete options */ bool delete_wal = false; @@ -163,6 +164,7 @@ static ConfigOption cmd_options[] = { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 157, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, + { 'b', 158, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3f56abdb3..24f5cbb74 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -408,6 +408,9 @@ extern char *instance_name; /* show options */ extern ShowFormat show_format; +/* checkdb options */ +extern bool heapallindexed; + /* current settings */ extern pgBackup current; From 252176bb9b627e70186a7af5d557e3088416aa72 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 17:16:08 +0300 Subject: [PATCH 0496/2107] PGPRO-2065: minor fixes for checkdb --- src/backup.c | 3 --- src/help.c | 9 ++++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index abeead402..a96e2e0e0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2666,8 +2666,6 @@ check_indexes(void *arg) if (arguments->backup_conn == NULL) { - PGresult *res; - char *query; arguments->backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, @@ -3479,7 +3477,6 @@ get_index_list(PGresult *res_db, int db_number, { PGresult *res; char *nspname = NULL; - char *snapshot = NULL; int i; dbname = PQgetvalue(res_db, db_number, 0); diff --git a/src/help.c b/src/help.c index 0d540658f..b5a29cccb 100644 --- a/src/help.c +++ b/src/help.c @@ -22,6 +22,7 @@ static void help_add_instance(void); static void help_del_instance(void); static void help_archive_push(void); static void help_archive_get(void); +static void help_checkdb(void); void help_command(char *command) @@ -52,6 +53,8 @@ help_command(char *command) help_archive_push(); else if (strcmp(command, "archive-get") == 0) help_archive_get(); + else if (strcmp(command, "checkdb") == 0) + help_checkdb(); else if (strcmp(command, "--help") == 0 || strcmp(command, "help") == 0 || strcmp(command, "-?") == 0 @@ -416,11 +419,11 @@ help_checkdb(void) printf(_(" --progress show progress\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --skip-block-validation skip file-level block checking\n")); - printf(_(" can be used only with --amcheck option\n")); + printf(_(" can be used only with '--amcheck' option\n")); printf(_(" --amcheck in addition to file-level block checking\n")); - printf(_(" check btree indexes using 'amcheck' or 'amcheck_next' extension")); + printf(_(" check btree indexes using 'amcheck' or 'amcheck_next' extension\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); - printf(_(" can be used only with --amcheck option\n")); + printf(_(" can be used only with '--amcheck' option\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From 55b5656f77bf10dbd7867aadeebda64f2325a88e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 17:25:14 +0300 Subject: [PATCH 0497/2107] PGPRO-2065: move backup_mode check to pg_probackup.c --- src/backup.c | 4 ---- src/pg_probackup.c | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index a96e2e0e0..1d2ac1624 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1231,10 +1231,6 @@ do_backup(time_t start_time) pgdata_basic_setup(); /* below perform checks specific for backup command */ - if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "required parameter not specified: BACKUP_MODE " - "(-b, --backup-mode)"); - #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) elog(ERROR, "Failed to retreive wal_segment_size"); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 8e55d056b..df4a1c441 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -531,6 +531,11 @@ main(int argc, char *argv[]) PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); + /* sanity */ + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "required parameter not specified: BACKUP_MODE " + "(-b, --backup-mode)"); + return do_backup(start_time); } case RESTORE_CMD: From 79cc076a0c2845c811355cce2b4cfbfd6ba44d0e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 18:52:00 +0300 Subject: [PATCH 0498/2107] PGPRO-2065: add async mode to pgut_execute_parallel() --- src/backup.c | 19 +++++++++++------ src/utils/pgut.c | 54 ++++++++++++++++++++++++++++++++++++++++-------- src/utils/pgut.h | 2 +- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1d2ac1624..50be370c1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1021,8 +1021,15 @@ do_block_validation(void) /* TODO write better info message */ if (check_isok) elog(INFO, "Data files are valid"); - else - elog(ERROR, "Data files are corrupted"); + + if (!check_isok) + { + if (thread_interrupted || interrupted) + elog(ERROR, "Checkdb failed"); + else + elog(ERROR, "Data files are corrupted"); + } + if (backup_files_list) { @@ -1131,7 +1138,7 @@ do_amcheck(void) if (check_isok && !db_skipped) elog(INFO, "Indexes are valid"); - if (!check_isok) + if (!check_isok && !interrupted) elog(ERROR, "Some indexes are corrupted"); if (db_skipped) @@ -3398,7 +3405,7 @@ pg_ptrack_get_block(backup_files_arg *arguments, res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", - 4, (const char **)params, true, false); + 4, (const char **)params, true, false, false); if (PQnfields(res) != 1) { @@ -3598,7 +3605,7 @@ amcheck_one_index(backup_files_arg *arguments, res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, - query, 2, (const char **)params, true, true); + query, 2, (const char **)params, true, true, true); } else { @@ -3607,7 +3614,7 @@ amcheck_one_index(backup_files_arg *arguments, res = pgut_execute_parallel(arguments->backup_conn, arguments->cancel_conn, - query, 1, (const char **)params, true, true); + query, 1, (const char **)params, true, true, true); } if (PQresultStatus(res) != PGRES_TUPLES_OK) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index bdd99f2bb..a271333a3 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -359,9 +359,10 @@ PGresult * pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, const char *query, int nParams, const char **params, - bool text_result, bool ok_error) + bool text_result, bool ok_error, bool async) { PGresult *res; + int ret = 0; if (interrupted && !in_cleanup) elog(ERROR, "interrupted"); @@ -387,15 +388,50 @@ pgut_execute_parallel(PGconn* conn, } //on_before_exec(conn, thread_cancel_conn); - if (nParams == 0) - res = PQexec(conn, query); + if (async) + { + if (nParams == 0) + ret = PQsendQuery(conn, query); + else + ret = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + + /* wait for processing */ + while (true) + { + + if (interrupted) + elog(ERROR, "interrupted"); + + if (!PQconsumeInput(conn)) + elog(ERROR, "query failed: %squery was: %s", + PQerrorMessage(conn), query); + + /* query is no done */ + if (!PQisBusy(conn)) + break; + + usleep(1000); + } + + res = PQgetResult(conn); + } else - res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, - /* - * Specify zero to obtain results in text format, - * or one to obtain results in binary format. - */ - (text_result) ? 0 : 1); + { + if (nParams == 0) + res = PQexec(conn, query); + else + res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, + /* + * Specify zero to obtain results in text format, + * or one to obtain results in binary format. + */ + (text_result) ? 0 : 1); + } //on_after_exec(thread_cancel_conn); switch (PQresultStatus(res)) diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 350b96726..1fc3f1b3f 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -57,7 +57,7 @@ extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nPar const char **params, bool text_result, bool ok_error); extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, const char *query, int nParams, - const char **params, bool text_result, bool ok_error); + const char **params, bool text_result, bool ok_error, bool async); extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); extern void pgut_cancel(PGconn* conn); extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); From 2b969c43c758e1d07aa626931da850a05ac45ce0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 19:29:47 +0300 Subject: [PATCH 0499/2107] PGPRO-2065: cancel connection during interrupt --- src/utils/pgut.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a271333a3..a92317b4c 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -405,7 +405,10 @@ pgut_execute_parallel(PGconn* conn, { if (interrupted) + { + pgut_cancel(conn); elog(ERROR, "interrupted"); + } if (!PQconsumeInput(conn)) elog(ERROR, "query failed: %squery was: %s", From 7f814942901d6aa148abdd4098fc30a22cf116d1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Apr 2019 22:06:13 +0300 Subject: [PATCH 0500/2107] minor fixes: free base36enc_dup()'s results, comments cleanup --- src/delete.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/delete.c b/src/delete.c index fc05182a3..535c7a5f8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -162,7 +162,7 @@ int do_retention(void) if (retention_is_set && !backup_list_is_empty) do_retention_internal(backup_list, to_keep_list, to_purge_list); - if (merge_expired && !dry_run) + if (merge_expired && !dry_run && !backup_list_is_empty) do_retention_merge(backup_list, to_keep_list, to_purge_list); if (delete_expired && !dry_run && !backup_list_is_empty) @@ -191,7 +191,7 @@ int do_retention(void) } /* Evaluate every backup by retention policies and populate purge and keep lists. - * Also for every backup print proposed action('Active' or 'Expired') according + * Also for every backup print its status ('Active' or 'Expired') according * to active retention policies. */ static void @@ -259,9 +259,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg * FULL CORRUPT in retention (not count toward redundancy limit) * FULL in retention * ------retention redundancy ------- - * PAGE4 in retention - * ------retention window ----------- * PAGE3 in retention + * ------retention window ----------- * PAGE2 out of retention * PAGE1 out of retention * FULL out of retention <- We are here @@ -362,8 +361,8 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l int j; /* IMPORTANT: we can merge to only those FULL backup, that is NOT - * guarded by retention and final targets of such merges must be - * incremental backup that is guarded by retention !!! + * guarded by retention and final target of such merge must be + * an incremental backup that is guarded by retention !!! * * PAGE4 E * PAGE3 D @@ -420,6 +419,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l keep_backup_id = base36enc_dup(keep_backup->start_time); elog(INFO, "Merge incremental chain between FULL backup %s and backup %s", base36enc(full_backup->start_time), keep_backup_id); + pg_free(keep_backup_id); merge_list = parray_new(); @@ -553,8 +553,10 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) base36enc(delete_backup->start_time), keeped_backup_id); purge = false; + pg_free(keeped_backup_id); break; } + pg_free(keeped_backup_id); } /* Retain backup */ From 340e6944622165150c6e579a859c70412cb8d3d5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Apr 2019 03:58:23 +0300 Subject: [PATCH 0501/2107] PGPRO-2065: fix async pgut_execute_parallel() --- src/backup.c | 87 ++++++++++++++++++++++------------------------ src/data.c | 2 +- src/help.c | 4 ++- src/pg_probackup.c | 2 ++ src/utils/pgut.c | 85 ++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 126 insertions(+), 54 deletions(-) diff --git a/src/backup.c b/src/backup.c index 50be370c1..f40127f56 100644 --- a/src/backup.c +++ b/src/backup.c @@ -945,22 +945,18 @@ do_block_validation(void) pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); - /* initialize backup list */ + /* initialize file list */ backup_files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ dir_list_file(backup_files_list, instance_config.pgdata, true, true, false, 0); /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. + * Sort pathname ascending. * * For example: * 1 - create 'base' * 2 - create 'base/1' - * - * Sorted array is used at least in parse_backup_filelist_filenames(), - * extractPageMap(), make_pagemap_from_ptrack(). */ parray_qsort(backup_files_list, pgFileComparePath); /* Extract information about files in backup_list parsing their names:*/ @@ -976,7 +972,6 @@ do_block_validation(void) /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); - /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -1021,15 +1016,8 @@ do_block_validation(void) /* TODO write better info message */ if (check_isok) elog(INFO, "Data files are valid"); - - if (!check_isok) - { - if (thread_interrupted || interrupted) - elog(ERROR, "Checkdb failed"); - else - elog(ERROR, "Data files are corrupted"); - } - + else + elog(ERROR, "Checkdb failed"); if (backup_files_list) { @@ -1053,7 +1041,7 @@ do_amcheck(void) bool first_db_with_amcheck = true; PGconn *db_conn = NULL; bool db_skipped = false; - + pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); @@ -1065,7 +1053,7 @@ do_amcheck(void) n_databases = PQntuples(res_db); - elog(INFO, "Start checking PostgreSQL instance with amcheck"); + elog(INFO, "Start amchecking PostgreSQL instance"); /* For each database check indexes. In parallel. */ for(i = 0; i < n_databases; i++) @@ -1112,8 +1100,6 @@ do_amcheck(void) arg->ret = 1; } - pgut_atexit_push(threads_conn_disconnect, NULL); - /* Run threads */ for (j = 0; j < num_threads; j++) { @@ -1131,18 +1117,25 @@ do_amcheck(void) if (threads_args[j].ret > 0) check_isok = false; } + + /* cleanup */ pgut_disconnect(db_conn); + + if (interrupted) + break; } /* TODO write better info message */ - if (check_isok && !db_skipped) - elog(INFO, "Indexes are valid"); + if (db_skipped) + elog(WARNING, "Some databases were not checked"); - if (!check_isok && !interrupted) - elog(ERROR, "Some indexes are corrupted"); + if (!check_isok) + elog(ERROR, "Checkdb --amcheck failed"); + else + elog(INFO, "Checkdb --amcheck executed"); - if (db_skipped) - elog(ERROR, "Some databases were not checked"); + if (check_isok && !interrupted && !db_skipped) + elog(INFO, "Indexes are valid"); } /* Entry point of pg_probackup CHECKDB subcommand. */ @@ -1153,7 +1146,7 @@ do_checkdb(bool need_amcheck) { if (skip_block_validation && !need_amcheck) - elog(ERROR, "--skip-block-validation must be used with --amcheck option"); + elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); pgdata_basic_setup(); @@ -2545,19 +2538,17 @@ backup_disconnect(bool fatal, void *userdata) static void threads_conn_disconnect(bool fatal, void *userdata) { -// int i; - -// elog(VERBOSE, "threads_conn_disconnect, num_threads %d", num_threads); -// for (i = 0; i < num_threads; i++) -// { -// backup_files_arg *arg = &(threads_args[i]); -// -// if (arg->backup_conn) -// { -// pgut_cancel(arg->backup_conn); -// pgut_disconnect(arg->backup_conn); -// } -// } + int i; + + backup_files_arg *arguments = (backup_files_arg *) userdata; + + elog(VERBOSE, "threads_conn_disconnect, num_threads %d", arguments->thread_num); + + if (arguments->backup_conn) + { + pgut_cancel(arguments->backup_conn); + pgut_disconnect(arguments->backup_conn); + } } static void * @@ -2663,7 +2654,7 @@ check_indexes(void *arg) arguments->thread_num); if (progress) - elog(INFO, "Thread [%d]. Progress: (%d/%d). Processing index '%s.%s'", + elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'", arguments->thread_num, i + 1, n_indexes, ind->amcheck_nspname, ind->name); @@ -3489,8 +3480,7 @@ get_index_list(PGresult *res_db, int db_number, instance_config.pguser); res = pgut_execute(db_conn, "SELECT " - "extname, nspname, extversion, " - "extversion::numeric in (1.1, 1) as old_version " + "extname, nspname, extversion " "FROM pg_namespace n " "JOIN pg_extension e " "ON n.oid=e.extnamespace " @@ -3520,9 +3510,13 @@ get_index_list(PGresult *res_db, int db_number, strcmp(PQgetvalue(res, 0, 2), "1") != 0) heapallindexed_is_supported = true; - elog(INFO, "Checking database %s using module '%s' version %s from schema '%s'", + elog(INFO, "Amchecking database %s using module '%s' version %s from schema '%s'", dbname, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + if (!heapallindexed_is_supported && heapallindexed) + elog(WARNING, "Module '%s' verion %s in schema '%s' do not support 'heapallindexed' option", + PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + /* * In order to avoid duplicates, select global indexes * (tablespace pg_global with oid 1664) only once @@ -3549,7 +3543,7 @@ get_index_list(PGresult *res_db, int db_number, "JOIN pg_tablespace tbl ON tbl.oid=cls.reltablespace " "WHERE am.amname='btree' AND cls.relpersistence != 't' " //"AND idx.indisready AND idx.indisvalid " - "AND tbl.spcname !='pg_global'",0, NULL); + "AND tbl.spcname != 'pg_global'",0, NULL); } /* add info needed to check indexes into index_list */ @@ -3598,6 +3592,9 @@ amcheck_one_index(backup_files_arg *arguments, /* second argument is heapallindexed */ params[1] = heapallindexed ? "true" : "false"; + if (interrupted) + elog(ERROR, "Interrupted"); + if (heapallindexed_is_supported) { query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); diff --git a/src/data.c b/src/data.c index 52e460ee7..7e7468a57 100644 --- a/src/data.c +++ b/src/data.c @@ -324,7 +324,7 @@ prepare_page(backup_files_arg *arguments, /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during backup"); + elog(ERROR, "Interrupted during page reading"); /* * Read the page and verify its header and checksum. diff --git a/src/help.c b/src/help.c index b5a29cccb..b22d49bb7 100644 --- a/src/help.c +++ b/src/help.c @@ -421,7 +421,9 @@ help_checkdb(void) printf(_(" --skip-block-validation skip file-level block checking\n")); printf(_(" can be used only with '--amcheck' option\n")); printf(_(" --amcheck in addition to file-level block checking\n")); - printf(_(" check btree indexes using 'amcheck' or 'amcheck_next' extension\n")); + printf(_(" check btree indexes via function 'bt_index_check()'\n")); + printf(_(" using amcheck' or 'amcheck_next' extension\n")); + printf(_(" --parent use 'bt_index_parent_check()' instead of 'bt_index_check()'\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); printf(_(" can be used only with '--amcheck' option\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index df4a1c441..b98637ced 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -93,6 +93,7 @@ bool skip_external_dirs = false; /* checkdb options */ bool do_amcheck = false; bool heapallindexed = false; +bool amcheck_parent = false; /* delete options */ bool delete_wal = false; @@ -165,6 +166,7 @@ static ConfigOption cmd_options[] = /* checkdb options */ { 'b', 157, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, { 'b', 158, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, + { 'b', 159, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a92317b4c..ae7c3cd61 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -21,7 +21,7 @@ #include "logger.h" -const char *PROGRAM_NAME = NULL; +const char *PROGRAM_NAME = "pg_probackup"; static char *password = NULL; bool prompt_password = true; @@ -43,6 +43,9 @@ static void on_interrupt(void); static void on_cleanup(void); static pqsigfunc oldhandler = NULL; +//bool is_result_ready(PGconn *conn); +void discard_response(PGconn *conn); + void pgut_init(void) { @@ -363,6 +366,12 @@ pgut_execute_parallel(PGconn* conn, { PGresult *res; int ret = 0; + PGconn *connections[1]; + + struct timeval timeout; + + timeout.tv_sec = 100; + timeout.tv_usec = 200; if (interrupted && !in_cleanup) elog(ERROR, "interrupted"); @@ -390,10 +399,13 @@ pgut_execute_parallel(PGconn* conn, //on_before_exec(conn, thread_cancel_conn); if (async) { + /* clean any old data */ + discard_response(conn); + if (nParams == 0) - ret = PQsendQuery(conn, query); + PQsendQuery(conn, query); else - ret = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, + PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, /* * Specify zero to obtain results in text format, * or one to obtain results in binary format. @@ -401,24 +413,35 @@ pgut_execute_parallel(PGconn* conn, (text_result) ? 0 : 1); /* wait for processing */ - while (true) +// while(!is_result_ready(conn)) +// { +// if (interrupted) +// { +// pgut_cancel(conn); +// pgut_disconnect(conn); +// elog(ERROR, "Interrupted"); +// } +// } + + /* wait for processing, TODO: timeout */ + for (;;) { - if (interrupted) { pgut_cancel(conn); + pgut_disconnect(conn); elog(ERROR, "interrupted"); } if (!PQconsumeInput(conn)) elog(ERROR, "query failed: %squery was: %s", - PQerrorMessage(conn), query); + PQerrorMessage(conn), query); /* query is no done */ if (!PQisBusy(conn)) break; - usleep(1000); + usleep(10000); } res = PQgetResult(conn); @@ -1023,3 +1046,51 @@ select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, con } #endif /* WIN32 */ + +void +discard_response(PGconn *conn) +{ + PGresult *res; + + do + { + res = PQgetResult(conn); + if (res) + PQclear(res); + } while (res); +} + +//bool is_result_ready(PGconn * conn) +//{ +// int sock; +// struct timeval timeout; +// fd_set read_mask; +// +// if (!PQisBusy(conn)) +// return true; +// +// sock = PQsocket(conn); +// +// timeout.tv_sec = (time_t)1; +// timeout.tv_usec = 0; +// +// FD_ZERO(&read_mask); +// FD_SET(sock, &read_mask); +// +// if (select(sock + 1, &read_mask, NULL, NULL, &timeout) == 0) +// return false; +// else if (FD_ISSET(sock, &read_mask)) +// { +// if (PQconsumeInput(conn)) +// { +// if (PQisBusy(conn)) +// return false; +// else +// return true; +// } +// else +// return false; +// } +// else +// return false; +//} From 2022487f3b4ff6d4e053811e5c749f5592b8b1b1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Apr 2019 04:00:36 +0300 Subject: [PATCH 0502/2107] PGPRO-2065: cleanup --- src/backup.c | 31 +++++++++++++++---------------- src/utils/pgut.c | 7 ------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/backup.c b/src/backup.c index f40127f56..109e25d81 100644 --- a/src/backup.c +++ b/src/backup.c @@ -95,7 +95,7 @@ static bool pg_stop_backup_is_sent = false; */ static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -static void threads_conn_disconnect(bool fatal, void *userdata); +//static void threads_conn_disconnect(bool fatal, void *userdata); static void pgdata_basic_setup(void); @@ -2535,21 +2535,20 @@ backup_disconnect(bool fatal, void *userdata) /* * Disconnect checkdb connections created in threads during quit pg_probackup. */ -static void -threads_conn_disconnect(bool fatal, void *userdata) -{ - int i; - - backup_files_arg *arguments = (backup_files_arg *) userdata; - - elog(VERBOSE, "threads_conn_disconnect, num_threads %d", arguments->thread_num); - - if (arguments->backup_conn) - { - pgut_cancel(arguments->backup_conn); - pgut_disconnect(arguments->backup_conn); - } -} +//static void +//threads_conn_disconnect(bool fatal, void *userdata) +//{ +// +// backup_files_arg *arguments = (backup_files_arg *) userdata; +// +// elog(VERBOSE, "threads_conn_disconnect, num_threads %d", arguments->thread_num); +// +// if (arguments->backup_conn) +// { +// pgut_cancel(arguments->backup_conn); +// pgut_disconnect(arguments->backup_conn); +// } +//} static void * check_files(void *arg) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ae7c3cd61..c262f6a74 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -365,13 +365,6 @@ pgut_execute_parallel(PGconn* conn, bool text_result, bool ok_error, bool async) { PGresult *res; - int ret = 0; - PGconn *connections[1]; - - struct timeval timeout; - - timeout.tv_sec = 100; - timeout.tv_usec = 200; if (interrupted && !in_cleanup) elog(ERROR, "interrupted"); From 140b33cc94a640407412ccb8b189cd289f59ecb6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Apr 2019 04:25:40 +0300 Subject: [PATCH 0503/2107] PGPRO-2065: fix comments --- src/backup.c | 2 +- src/help.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index e2563bb27..61a780530 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1156,7 +1156,7 @@ do_checkdb(bool need_amcheck) if (need_amcheck) do_amcheck(); - /* need to exit with 1 if some corruption is found */ + /* TODO: need to exit with 1 if some corruption is found */ return 0; } diff --git a/src/help.c b/src/help.c index 2dcf79421..e7daabae5 100644 --- a/src/help.c +++ b/src/help.c @@ -426,7 +426,7 @@ help_checkdb(void) printf(_(" can be used only with '--amcheck' option\n")); printf(_(" --amcheck in addition to file-level block checking\n")); printf(_(" check btree indexes via function 'bt_index_check()'\n")); - printf(_(" using amcheck' or 'amcheck_next' extension\n")); + printf(_(" using 'amcheck' or 'amcheck_next' extensions\n")); printf(_(" --parent use 'bt_index_parent_check()' instead of 'bt_index_check()'\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); printf(_(" can be used only with '--amcheck' option\n")); From 01fedaf04cc79247b1803d0ba80067d70d50c59f Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 3 Apr 2019 16:35:57 +0300 Subject: [PATCH 0504/2107] pgpro-2605. Use absolute path of pg_probackup binary in recovery.conf generated by pg_probackup --- src/pg_probackup.c | 9 +++++++++ src/restore.c | 2 +- src/utils/pgut.h | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 67db3bb16..24ba2759e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -22,6 +22,7 @@ const char *PROGRAM_VERSION = "2.0.27"; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; +const char *PROGRAM_FULL_PATH = NULL; typedef enum ProbackupSubcmd { @@ -201,6 +202,14 @@ main(int argc, char *argv[]) init_config(&instance_config); PROGRAM_NAME = get_progname(argv[0]); + PROGRAM_FULL_PATH = palloc0(MAXPGPATH); + + if (find_my_exec(argv[0],(char *) PROGRAM_FULL_PATH) < 0) + { + fprintf(stderr, _("%s: could not find own program executable\n"), PROGRAM_NAME); + exit(1); + } + set_pglocale_pgservice(argv[0], "pgscripts"); #if PG_VERSION_NUM >= 110000 diff --git a/src/restore.c b/src/restore.c index 7efd433a4..bd8f5b72b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -796,7 +796,7 @@ create_recovery_conf(time_t backup_id, fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " "--wal-file-path %%p --wal-file-name %%f'\n", - PROGRAM_NAME, backup_path, instance_name); + PROGRAM_FULL_PATH, backup_path, instance_name); /* * We've already checked that only one of the four following mutually diff --git a/src/utils/pgut.h b/src/utils/pgut.h index bde5dee1c..41a261d26 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -20,6 +20,7 @@ typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); * pgut client variables and functions */ extern const char *PROGRAM_NAME; +extern const char *PROGRAM_FULL_PATH; extern const char *PROGRAM_VERSION; extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; From 9091bb540504f36aab39aa57103a7e0816c92c7b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Apr 2019 18:34:18 +0300 Subject: [PATCH 0505/2107] tests: additional tests for external directories --- tests/external.py | 173 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/external.py b/tests/external.py index 1327402cc..e085b153b 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1172,6 +1172,179 @@ def test_external_dir_is_symlink(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_contain_symlink_on_dir(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + dir_in_external_dir = os.path.join(external_dir, 'dir') + + node.pgbench_init(scale=3) + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, module_name, fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + print(symlinked_dir) + print(dir_in_external_dir) + os.mkdir(external_dir) + os.symlink(symlinked_dir, dir_in_external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_contain_symlink_on_file(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + file_in_external_dir = os.path.join(external_dir, 'file') + + node.pgbench_init(scale=3) + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, module_name, fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + src_file = os.path.join(symlinked_dir, 'postgresql.conf') + os.mkdir(external_dir) + os.symlink(src_file, file_in_external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_is_tablespace(self): From 6b6491e3b788241cecf5320a7cc565673db17eab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Apr 2019 22:25:58 +0300 Subject: [PATCH 0506/2107] fix symlink handling for external directories --- src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index dcf1476d5..ad118b126 100644 --- a/src/dir.c +++ b/src/dir.c @@ -472,7 +472,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, external_dir_num ? omit_symlink : false, omit_symlink); + file = pgFileNew(root, external_dir_num ? true : false, external_dir_num); if (file == NULL) return; From 0fbbc22c6af35427b657e4bf1caac8df2b1d33a3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 4 Apr 2019 13:30:30 +0300 Subject: [PATCH 0507/2107] Little optimization, remove unnecessary find_direct_child() --- src/catalog.c | 40 ++++++---------------------------------- src/delete.c | 5 ++++- src/pg_probackup.h | 1 - 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 11ce830e8..becdbf3ba 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1026,13 +1026,14 @@ is_prolific(parray *backup_list, pgBackup *target_backup) if (tmp_backup->parent_backup == target_backup->start_time && (tmp_backup->status == BACKUP_STATUS_OK || tmp_backup->status == BACKUP_STATUS_DONE)) + { child_counter++; + if (child_counter > 1) + return true; + } } - if (child_counter > 1) - return true; - else - return false; + return false; } /* @@ -1067,35 +1068,6 @@ find_parent_full_backup(pgBackup *current_backup) return base_full_backup; } -/* - * Find closest child of target_backup. If there are several direct - * offsprings in backup_list, then first win. - */ -pgBackup* -find_direct_child(parray *backup_list, pgBackup *target_backup) -{ - int i; - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); - - if (tmp_backup->backup_mode == BACKUP_MODE_FULL) - continue; - - /* Consider only OK and DONE children */ - if (tmp_backup->parent_backup == target_backup->start_time && - (tmp_backup->status == BACKUP_STATUS_OK || - tmp_backup->status == BACKUP_STATUS_DONE)) - { - return tmp_backup; - } - } - elog(WARNING, "Failed to find a direct child for backup %s", - base36enc(target_backup->start_time)); - return NULL; -} - /* * Interate over parent chain and look for any problems. * Return 0 if chain is broken. @@ -1204,4 +1176,4 @@ get_backup_index_number(parray *backup_list, pgBackup *backup) } elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time)); return -1; -} \ No newline at end of file +} diff --git a/src/delete.c b/src/delete.c index 535c7a5f8..14c0f1bcf 100644 --- a/src/delete.c +++ b/src/delete.c @@ -129,6 +129,9 @@ int do_retention(void) bool retention_is_set = false; /* At least one retention policy is set */ bool backup_list_is_empty = false; + backup_deleted = false; + backup_merged = false; + /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -411,7 +414,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l continue; } - /* FULL backup in purge list, thanks to sparsing of keep_list current backup is + /* FULL backup in purge list, thanks to sparsing of keep_list current backup is * final target for merge, but there could be intermediate incremental * backups from purge_list. */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 85c1fd53f..ff4ae8664 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -501,7 +501,6 @@ extern int pgBackupCompareId(const void *f1, const void *f2); extern int pgBackupCompareIdDesc(const void *f1, const void *f2); extern int pgBackupCompareIdEqual(const void *l, const void *r); -extern pgBackup* find_direct_child(parray *backup_list, pgBackup *target_backup); extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); From 9a5c8074dbabf5afc0eb2396be65636e8f1d5e44 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Apr 2019 14:39:59 +0300 Subject: [PATCH 0508/2107] tests: fix test_delta_archive --- tests/delta.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/delta.py b/tests/delta.py index d5c3a03f8..e4c8305d1 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -352,17 +352,11 @@ def test_delta_archive(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30s' - } - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.slow_start() # FULL BACKUP @@ -372,8 +366,7 @@ def test_delta_archive(self): "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") full_result = node.execute("postgres", "SELECT * FROM t_heap") full_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream']) + backup_dir, 'node', node, backup_type='full') # delta BACKUP node.safe_psql( @@ -382,8 +375,7 @@ def test_delta_archive(self): "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") delta_result = node.execute("postgres", "SELECT * FROM t_heap") delta_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) + backup_dir, 'node', node, backup_type='delta') # Drop Node node.cleanup() From 514a01dc15e252f1dcf2cf3c0cc2134b65e78bcf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Apr 2019 18:36:35 +0300 Subject: [PATCH 0509/2107] another symlink fix --- src/backup.c | 4 ++-- src/dir.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 7b99e1af3..e7c20e2df 100644 --- a/src/backup.c +++ b/src/backup.c @@ -659,8 +659,8 @@ do_backup_instance(void) * https://github.com/postgrespro/pg_probackup/issues/48 */ - if (parray_num(backup_files_list) == 0) - elog(ERROR, "PGDATA is empty. Either it was concurrently deleted or " + if (parray_num(backup_files_list) < 100) + elog(ERROR, "PGDATA is almost empty. Either it was concurrently deleted or " "pg_probackup do not possess sufficient permissions to list PGDATA content"); /* diff --git a/src/dir.c b/src/dir.c index ad118b126..ec32d877c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -472,7 +472,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, external_dir_num ? true : false, external_dir_num); + file = pgFileNew(root, omit_symlink, external_dir_num); if (file == NULL) return; From 1746b549780b0fde570b935d54dedd3be96c640c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 4 Apr 2019 20:26:36 +0300 Subject: [PATCH 0510/2107] Make changes for C89 --- src/catalog.c | 2 +- src/merge.c | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index becdbf3ba..b0e6a5e70 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -476,7 +476,7 @@ pgBackupCreateDir(pgBackup *backup) parray *external_list; external_list = make_external_directory_list(backup->external_dir_str); - for (int i = 0; i < parray_num(external_list); i++) + for (i = 0; i < parray_num(external_list); i++) { char temp[MAXPGPATH]; /* Numeration of externaldirs starts with 1 */ diff --git a/src/merge.c b/src/merge.c index 3ac95ea2b..bd3654089 100644 --- a/src/merge.c +++ b/src/merge.c @@ -673,10 +673,12 @@ merge_files(void *arg) static void remove_dir_with_files(const char *path) { - parray *files = parray_new(); + parray *files = parray_new(); + int i; + dir_list_file(files, path, true, true, true, 0); parray_qsort(files, pgFileComparePathDesc); - for (int i = 0; i < parray_num(files); i++) + for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); @@ -689,9 +691,11 @@ remove_dir_with_files(const char *path) static int get_external_index(const char *key, const parray *list) { + int i; + if (!list) /* Nowhere to search */ return -1; - for (int i = 0; i < parray_num(list); i++) + for (i = 0; i < parray_num(list); i++) { if (strcmp(key, parray_get(list, i)) == 0) return i + 1; @@ -704,11 +708,12 @@ static void reorder_external_dirs(pgBackup *to_backup, parray *to_external, parray *from_external) { - char externaldir_template[MAXPGPATH]; + char externaldir_template[MAXPGPATH]; + int i; pgBackupGetPath(to_backup, externaldir_template, lengthof(externaldir_template), EXTERNAL_DIR); - for (int i = 0; i < parray_num(to_external); i++) + for (i = 0; i < parray_num(to_external); i++) { int from_num = get_external_index(parray_get(to_external, i), from_external); From 049f81830e27e2b63b010abaf87b463b5853d535 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Apr 2019 00:57:06 +0300 Subject: [PATCH 0511/2107] minor fix for del-instance --- src/delete.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index 14c0f1bcf..63c7aac33 100644 --- a/src/delete.c +++ b/src/delete.c @@ -835,8 +835,9 @@ do_delete_instance(void) if (rmdir(backup_instance_path) != 0) elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, strerror(errno)); + if (rmdir(arclog_path) != 0) - elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + elog(ERROR, "can't remove \"%s\": %s", arclog_path, strerror(errno)); elog(INFO, "Instance '%s' successfully deleted", instance_name); From 6b49402e9fe98ad88b62c13646972db907fee293 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Apr 2019 12:03:05 +0300 Subject: [PATCH 0512/2107] tests: added delete.DeleteTest.test_del_instance_archive --- tests/delete.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/delete.py b/tests/delete.py index 71919c869..8209e3793 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -55,6 +55,36 @@ def test_delete_full_backups(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_del_instance_archive(self): + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + # full backup + self.backup_node(backup_dir, 'node', node) + + # restore + self.restore_node( + backup_dir, 'node', node) + + self.del_instance(backup_dir, 'node') + + # Clean after yourself + #self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_archive_mix_compress_and_non_compressed_segments(self): From 8df3a8cf4be5ece19da66d55ff740d621af818b2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Apr 2019 12:09:09 +0300 Subject: [PATCH 0513/2107] tests: minor fixes --- tests/delete.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/delete.py b/tests/delete.py index 8209e3793..1359aaf04 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -76,10 +76,15 @@ def test_del_instance_archive(self): # full backup self.backup_node(backup_dir, 'node', node) + node.cleanup() + + # restore self.restore_node( backup_dir, 'node', node) + node.slow_start() + self.del_instance(backup_dir, 'node') # Clean after yourself From 64a7fac7a2d4fed3225c3574a9656d63e963394a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Apr 2019 13:54:29 +0300 Subject: [PATCH 0514/2107] tests: minor fixes for del_instance --- tests/delete.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index 1359aaf04..ac5a42841 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -76,19 +76,16 @@ def test_del_instance_archive(self): # full backup self.backup_node(backup_dir, 'node', node) - node.cleanup() - - # restore - self.restore_node( - backup_dir, 'node', node) - + node.cleanup() + self.restore_node(backup_dir, 'node', node) node.slow_start() + # Delete instance self.del_instance(backup_dir, 'node') # Clean after yourself - #self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure From df0cd6507b37c6f048ab1ac086a87f85b3db3c23 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Apr 2019 17:39:48 +0300 Subject: [PATCH 0515/2107] bugfix: it is not a fatal error to backup a file, whose size is not multiple of BLCKSZ --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 9cad2eabf..32057e796 100644 --- a/src/data.c +++ b/src/data.c @@ -577,7 +577,7 @@ backup_data_file(backup_files_arg* arguments, if (file->size % BLCKSZ != 0) { fclose(in); - elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); + elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size); } /* From 77bb25410c2823efd593f54103de338e022a57f5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Apr 2019 23:10:15 +0300 Subject: [PATCH 0516/2107] Issue 52: --no-validate flag for 'backup' command, also allow DONE backup to be a parent of incremental backup --- src/backup.c | 5 +++-- src/catalog.c | 3 ++- src/help.c | 5 +++-- src/pg_probackup.c | 8 ++++---- src/pg_probackup.h | 6 +++--- src/restore.c | 16 ++++++++-------- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index e7c20e2df..315352da9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -922,7 +922,7 @@ do_backup_instance(void) * Entry point of pg_probackup BACKUP subcommand. */ int -do_backup(time_t start_time) +do_backup(time_t start_time, bool no_validate) { /* PGDATA and BACKUP_MODE are always required */ if (instance_config.pgdata == NULL) @@ -1060,7 +1060,8 @@ do_backup(time_t start_time) //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", // current.data_bytes); - pgBackupValidate(¤t); + if (!no_validate) + pgBackupValidate(¤t); elog(INFO, "Backup %s completed", base36enc(current.start_time)); diff --git a/src/catalog.c b/src/catalog.c index b0e6a5e70..766afb95f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -453,7 +453,8 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) { backup = (pgBackup *) parray_get(backup_list, (size_t) i); - if (backup->status == BACKUP_STATUS_OK && backup->tli == tli) + if ((backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) && backup->tli == tli) return backup; } diff --git a/src/help.c b/src/help.c index 214a15ed8..32a9a3ba1 100644 --- a/src/help.c +++ b/src/help.c @@ -118,7 +118,7 @@ help_pg_probackup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--skip-block-validation]\n")); + printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [--external-dirs=external-directory-path]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -212,7 +212,7 @@ help_backup(void) printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--skip-block-validation]\n")); + printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-E external-dirs=external-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -226,6 +226,7 @@ help_backup(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); + printf(_(" --no-validate disable validation after backup\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_(" -E --external-dirs=external-directory-path\n")); printf(_(" backup some directories not from pgdata \n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 24ba2759e..0f4931f88 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -85,7 +85,7 @@ static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; bool restore_as_replica = false; -bool restore_no_validate = false; +bool no_validate = false; bool skip_block_validation = false; bool skip_external_dirs = false; @@ -158,7 +158,7 @@ static ConfigOption cmd_options[] = { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, - { 'b', 143, "no-validate", &restore_no_validate, SOURCE_CMD_STRICT }, + { 'b', 143, "no-validate", &no_validate, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, @@ -474,7 +474,7 @@ main(int argc, char *argv[]) /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, target_inclusive, target_tli, target_lsn, target_immediate, - target_name, target_action, restore_no_validate); + target_name, target_action, no_validate); } if (num_threads < 1) @@ -508,7 +508,7 @@ main(int argc, char *argv[]) PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, stream_wal ? "true" : "false", is_remote_backup ? "true" : "false"); - return do_backup(start_time); + return do_backup(start_time, no_validate); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ff4ae8664..1f66194d8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -291,7 +291,7 @@ typedef struct pgRecoveryTarget bool recovery_target_immediate; const char *recovery_target_name; const char *recovery_target_action; - bool restore_no_validate; + bool no_validate; } pgRecoveryTarget; typedef struct @@ -406,7 +406,7 @@ extern pgBackup current; extern const char *pgdata_exclude_dir[]; /* in backup.c */ -extern int do_backup(time_t start_time); +extern int do_backup(time_t start_time, bool no_validate); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, @@ -427,7 +427,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, bool target_immediate, const char *target_name, - const char *target_action, bool restore_no_validate); + const char *target_action, bool no_validate); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index bd8f5b72b..f1fea48d2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -125,7 +125,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup->status == BACKUP_STATUS_ORPHAN || current_backup->status == BACKUP_STATUS_CORRUPT || current_backup->status == BACKUP_STATUS_RUNNING) - && !rt->restore_no_validate) + && !rt->no_validate) elog(WARNING, "Backup %s has status: %s", base36enc(current_backup->start_time), status2str(current_backup->status)); else @@ -308,7 +308,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray_append(parent_chain, base_full_backup); /* for validation or restore with enabled validation */ - if (!is_restore || !rt->restore_no_validate) + if (!is_restore || !rt->no_validate) { if (dest_backup->backup_mode != BACKUP_MODE_FULL) elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); @@ -395,7 +395,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (dest_backup->status == BACKUP_STATUS_OK) { - if (rt->restore_no_validate) + if (rt->no_validate) elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); else elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); @@ -425,7 +425,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Backup was locked during validation if no-validate wasn't * specified. */ - if (rt->restore_no_validate && !lock_backup(backup)) + if (rt->no_validate && !lock_backup(backup)) elog(ERROR, "Cannot lock backup directory"); restore_backup(backup, dest_backup->external_dir_str); @@ -993,7 +993,7 @@ parseRecoveryTargetOptions(const char *target_time, bool target_immediate, const char *target_name, const char *target_action, - bool restore_no_validate) + bool no_validate) { time_t dummy_time; TransactionId dummy_xid; @@ -1022,7 +1022,7 @@ parseRecoveryTargetOptions(const char *target_time, rt->recovery_target_immediate = false; rt->recovery_target_name = NULL; rt->recovery_target_action = NULL; - rt->restore_no_validate = false; + rt->no_validate = false; /* parse given options */ if (target_time) @@ -1080,9 +1080,9 @@ parseRecoveryTargetOptions(const char *target_time, rt->recovery_target_immediate = target_immediate; } - if (restore_no_validate) + if (no_validate) { - rt->restore_no_validate = restore_no_validate; + rt->no_validate = no_validate; } if (target_name) From 1ff905989efe5c2d13c2ec6a749e272fe1877130 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 9 Apr 2019 18:05:36 +0300 Subject: [PATCH 0517/2107] Replaced the function strftime with pg_strftime for logs in Windows --- gen_probackup_project.pl | 1 + src/utils/logger.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 999a0d55a..88a1a948e 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -182,6 +182,7 @@ sub build_pgprobackup $probackup->AddFile("$pgsrc/src/bin/pg_rewind/datapagemap.c"); $probackup->AddFile("$pgsrc/src/interfaces/libpq/pthread-win32.c"); + $probackup->AddFile("$pgsrc/src/timezone/strftime.c"); $probackup->AddIncludeDir("$pgsrc/src/bin/pg_basebackup"); $probackup->AddIncludeDir("$pgsrc/src/bin/pg_rewind"); diff --git a/src/utils/logger.c b/src/utils/logger.c index fb1005268..10f23105f 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -470,7 +470,11 @@ logfile_getname(const char *format, time_t timestamp) len = strlen(filename); /* Treat log_filename as a strftime pattern */ +#ifdef WIN32 + if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) +#else if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) +#endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); return filename; From d87149c68f930efc1105259bb583d20176d8b1af Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 10 Apr 2019 15:57:32 +0300 Subject: [PATCH 0518/2107] PBCKP-36: Create recovery.conf if --latest specified --- src/help.c | 7 ++++--- src/pg_probackup.c | 5 ++++- src/pg_probackup.h | 3 ++- src/restore.c | 12 +++++++----- tests/expected/option_help.out | 16 ++++++++++------ tests/option_test.py | 2 +- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/help.c b/src/help.c index 214a15ed8..8e8e3f709 100644 --- a/src/help.c +++ b/src/help.c @@ -123,7 +123,7 @@ help_pg_probackup(void) printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); @@ -292,7 +292,7 @@ help_restore(void) { printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads] [--progress]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]]\n")); printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); @@ -312,7 +312,8 @@ help_restore(void) printf(_(" --time=time time stamp up to which recovery will proceed\n")); printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); + printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --latest recovery into a latest possible state\n")); printf(_(" --timeline=timeline recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 24ba2759e..e07a280ca 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -81,6 +81,7 @@ static TimeLineID target_tli; static bool target_immediate; static char *target_name = NULL; static char *target_action = NULL; +static bool target_latest = false; static pgRecoveryTarget *recovery_target_options = NULL; @@ -152,6 +153,7 @@ static ConfigOption cmd_options[] = { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, + { 'b', 157, "latest", &target_latest, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, @@ -474,7 +476,8 @@ main(int argc, char *argv[]) /* parse all recovery target options into recovery_target_options structure */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, target_inclusive, target_tli, target_lsn, target_immediate, - target_name, target_action, restore_no_validate); + target_name, target_action, restore_no_validate, + target_latest); } if (num_threads < 1) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ff4ae8664..c50e1a8d8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -292,6 +292,7 @@ typedef struct pgRecoveryTarget const char *recovery_target_name; const char *recovery_target_action; bool restore_no_validate; + bool latest; } pgRecoveryTarget; typedef struct @@ -427,7 +428,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, bool target_immediate, const char *target_name, - const char *target_action, bool restore_no_validate); + const char *target_action, bool restore_no_validate, bool latest); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index bd8f5b72b..e1a65dd7e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -769,11 +769,11 @@ create_recovery_conf(time_t backup_id, { char path[MAXPGPATH]; FILE *fp; - bool need_restore_conf = false; + bool need_restore_conf; - if (!backup->stream - || (rt->time_specified || rt->xid_specified)) - need_restore_conf = true; + need_restore_conf = !backup->stream || + (rt->time_specified || rt->xid_specified || rt->lsn_specified) || + rt->latest; /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || restore_as_replica)) @@ -993,7 +993,7 @@ parseRecoveryTargetOptions(const char *target_time, bool target_immediate, const char *target_name, const char *target_action, - bool restore_no_validate) + bool restore_no_validate, bool latest) { time_t dummy_time; TransactionId dummy_xid; @@ -1085,6 +1085,8 @@ parseRecoveryTargetOptions(const char *target_time, rt->restore_no_validate = restore_no_validate; } + rt->latest = latest; + if (target_name) { recovery_target_specified++; diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index a83c39052..23f3cf912 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -39,7 +39,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--log-directory=log-directory] [--log-rotation-size=log-rotation-size] [--log-rotation-age=log-rotation-age] - [--delete-expired] [--delete-wal] + [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] [--compress] @@ -51,16 +51,19 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] [--skip-block-validation] + [--external-dirs=external-directory-path] pg_probackup restore -B backup-path --instance=instance_name - [-D pgdata-path] [-i backup-id] [--progress] - [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + [-D pgdata-path] [-i backup-id] [-j num-threads] + [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]] + [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] [--skip-block-validation] + [--skip-external-dirs] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] @@ -74,10 +77,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--format=format] pg_probackup delete -B backup-path --instance=instance_name - [--wal] [-i backup-id | --expired] + [--wal] [-i backup-id | --expired | --merge-expired] + [--dry-run] pg_probackup merge -B backup-path --instance=instance_name - -i backup-id + -i backup-id [--progress] [-j num-threads] pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name diff --git a/tests/option_test.py b/tests/option_test.py index 5a64a8b6f..092c79d99 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -95,7 +95,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertEqual(e.message, - 'ERROR: You must specify at least one of the delete options: --expired |--wal |--backup_id\n', + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--merge-expired |--delete-invalid |--backup_id\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From bc52046ac65ab47d1c0fd045920e8bf2cb52177b Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 10 Apr 2019 17:02:00 +0300 Subject: [PATCH 0519/2107] Do not wait for the next segment --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index d9d169d40..87323431b 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1191,8 +1191,8 @@ XLogThreadWorker(void *arg) * Consider thread_arg->endSegNo and thread_arg->endpoint only if * they are valid. */ - xlogreader->ReadRecPtr == thread_arg->endpoint && - nextSegNo > thread_arg->endSegNo) + xlogreader->ReadRecPtr >= thread_arg->endpoint && + nextSegNo >= thread_arg->endSegNo) break; } From 8240bc2dc4c23be4969d3c7e87adc670deb2faae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Apr 2019 22:27:07 +0300 Subject: [PATCH 0520/2107] tests: fix archive.ArchiveTest.test_replica_archive --- tests/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 51e46ea0e..685e69869 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -422,10 +422,10 @@ def test_replica_archive(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'archive_timeout': '10s', - 'max_wal_size': '1GB'} - ) + 'checkpoint_timeout': '30s', + 'max_wal_size': '16MB'}) + self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) From 9ca020c1278096dded34360dee61846513dcedb8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Apr 2019 22:31:52 +0300 Subject: [PATCH 0521/2107] tests: update option_help.out --- tests/expected/option_help.out | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index a83c39052..6cb277585 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -39,7 +39,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--log-directory=log-directory] [--log-rotation-size=log-rotation-size] [--log-rotation-age=log-rotation-age] - [--delete-expired] [--delete-wal] + [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] [--compress] @@ -50,17 +50,20 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-db=db_name] [--master-host=host_name] [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] - [--skip-block-validation] + [--no-validate] [--skip-block-validation] + [--external-dirs=external-directory-path] pg_probackup restore -B backup-path --instance=instance_name - [-D pgdata-path] [-i backup-id] [--progress] + [-D pgdata-path] [-i backup-id] [-j num-threads] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] [--skip-block-validation] + [--skip-external-dirs] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] @@ -74,10 +77,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--format=format] pg_probackup delete -B backup-path --instance=instance_name - [--wal] [-i backup-id | --expired] + [--wal] [-i backup-id | --expired | --merge-expired] + [--dry-run] pg_probackup merge -B backup-path --instance=instance_name - -i backup-id + -i backup-id [--progress] [-j num-threads] pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name From af340b982c14ed7a0fd1b7928d69469f8a28a7e9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Apr 2019 23:43:30 +0300 Subject: [PATCH 0522/2107] tests: added delta.DeltaTest.test_delta_backup_from_past --- tests/delta.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/tests/delta.py b/tests/delta.py index e4c8305d1..262c7fdcb 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -711,14 +711,12 @@ def test_create_db(self): 1, 0, "Expecting Error because we are connecting to deleted database" "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) + repr(self.output), self.cmd)) except QueryException as e: self.assertTrue( 'FATAL: database "db1" does not exist' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd) - ) + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1307,3 +1305,53 @@ def test_delta_nullified_heap_page_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_delta_backup_from_past(self): + """ + make node, take FULL stream backup, take DELTA stream backup, + restore FULL backup, try to take second DELTA stream backup + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=3) + + # First DELTA + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Restore FULL backup + node.cleanup() + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + node.slow_start() + + # Second DELTA backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are backing up an instance from the past" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertTrue( + 'Insert error message' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 9c83463f914646d7c1323431058efb3092235706 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 11 Apr 2019 12:35:55 +0300 Subject: [PATCH 0523/2107] Consider DELETING status within merge_backups() --- src/merge.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/merge.c b/src/merge.c index bd3654089..8e91babb3 100644 --- a/src/merge.c +++ b/src/merge.c @@ -81,8 +81,8 @@ do_merge(time_t backup_id) /* It is possible that previous merging was interrupted */ backup->status != BACKUP_STATUS_MERGING && backup->status != BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - base36enc(backup->start_time), status2str(backup->status)); + elog(ERROR, "Backup %s has status: %s", + base36enc(backup->start_time), status2str(backup->status)); if (backup->backup_mode == BACKUP_MODE_FULL) elog(ERROR, "Backup %s is full backup", @@ -109,10 +109,8 @@ do_merge(time_t backup_id) if (full_backup->status != BACKUP_STATUS_OK && /* It is possible that previous merging was interrupted */ full_backup->status != BACKUP_STATUS_MERGING) - elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), status2str(full_backup->status)); - - //Assert(full_backup_idx != dest_backup_idx); + elog(ERROR, "Backup %s has status: %s", + base36enc(full_backup->start_time), status2str(full_backup->status)); /* form merge list */ while(dest_backup->parent_backup_link) @@ -122,8 +120,8 @@ do_merge(time_t backup_id) /* It is possible that previous merging was interrupted */ dest_backup->status != BACKUP_STATUS_MERGING && dest_backup->status != BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup->start_time), status2str(dest_backup->status)); + elog(ERROR, "Backup %s has status: %s", + base36enc(dest_backup->start_time), status2str(dest_backup->status)); parray_append(merge_list, dest_backup); dest_backup = dest_backup->parent_backup_link; @@ -205,7 +203,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * BACKUP_STATUS_MERGING status. */ Assert(from_backup->status == BACKUP_STATUS_OK || - from_backup->status == BACKUP_STATUS_MERGING); + from_backup->status == BACKUP_STATUS_MERGING || + from_backup->status == BACKUP_STATUS_DELETING); pgBackupValidate(from_backup); if (from_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Interrupt merging"); From 8aeddc98924cec57b1d4ad658a9c6181f9c4ee93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Apr 2019 16:33:16 +0300 Subject: [PATCH 0524/2107] tests: minor fixes --- tests/__init__.py | 2 +- tests/expected/option_help.out | 14 +++++++++----- tests/merge.py | 4 +++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 033ce535f..bc787ca4d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -18,7 +18,7 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(cfs_backup)) # suite.addTests(loader.loadTestsFromModule(cfs_restore)) # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) -# suite.addTests(loader.loadTestsFromModule(logging)) + suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(compression)) suite.addTests(loader.loadTestsFromModule(delete)) suite.addTests(loader.loadTestsFromModule(delta)) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index a83c39052..cac631c17 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -39,7 +39,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--log-directory=log-directory] [--log-rotation-size=log-rotation-size] [--log-rotation-age=log-rotation-age] - [--delete-expired] [--delete-wal] + [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] [--compress] @@ -51,16 +51,19 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--master-port=port] [--master-user=user_name] [--replica-timeout=timeout] [--skip-block-validation] + [--external-dirs=external-directory-path] pg_probackup restore -B backup-path --instance=instance_name - [-D pgdata-path] [-i backup-id] [--progress] + [-D pgdata-path] [-i backup-id] [-j num-threads] [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] + [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] [--immediate] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] [--skip-block-validation] + [--skip-external-dirs] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] @@ -74,10 +77,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--format=format] pg_probackup delete -B backup-path --instance=instance_name - [--wal] [-i backup-id | --expired] + [--wal] [-i backup-id | --expired | --merge-expired] + [--dry-run] pg_probackup merge -B backup-path --instance=instance_name - -i backup-id + -i backup-id [--progress] [-j num-threads] pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name diff --git a/tests/merge.py b/tests/merge.py index 4608a34ce..ee8cf2fbd 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1197,7 +1197,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): def test_continue_failed_merge_2(self): """ - Check that failed MERGE on delete can`t be continued + Check that failed MERGE on delete can be continued """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1253,6 +1253,8 @@ def test_continue_failed_merge_2(self): backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] + # TODO check that full backup has meta info is equal to DELETTING + # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) # Clean after yourself From 06add7d59f6379d320b2e8a06a1a182eed88cb7b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Apr 2019 17:00:44 +0300 Subject: [PATCH 0525/2107] tests: minor fix for replica.ReplicaTest.test_replica_archive_page_backup --- tests/replica.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 9ab49a6ea..358d97d79 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -141,10 +141,10 @@ def test_replica_archive_page_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'archive_timeout': '10s'} - ) + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s', + 'max_wal_size': '16MB'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) From 6d8bad7293ebcf5c5232da888d54bfb0b15c0213 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Apr 2019 19:14:33 +0300 Subject: [PATCH 0526/2107] bugfix: explicitly forbid incremental backing up instance from the past --- src/backup.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backup.c b/src/backup.c index e7c20e2df..f4e930429 100644 --- a/src/backup.c +++ b/src/backup.c @@ -582,6 +582,15 @@ do_backup_instance(void) strlen(" with pg_probackup")); pg_start_backup(label, smooth_checkpoint, ¤t); + /* For incremental backup check that start_lsn is not from the past */ + if (current.backup_mode != BACKUP_MODE_FULL && + prev_backup->start_lsn > current.start_lsn) + elog(ERROR, "Current START LSN %X/%X is lower than START LSN %X/%X of previous backup %s. " + "It may indicate that we are trying to backup PostgreSQL instance from the past.", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), + base36enc(prev_backup->start_time)); + /* Update running backup meta with START LSN */ write_backup(¤t); From c2d6824569f27b0d046d0e2d31ea743eddea1b65 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Apr 2019 19:41:26 +0300 Subject: [PATCH 0527/2107] Issue 52: treat DONE backups as OK --- src/delete.c | 2 +- src/merge.c | 7 ++++++- src/restore.c | 26 ++++++++++++++++---------- src/validate.c | 9 ++++++--- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/delete.c b/src/delete.c index 63c7aac33..a516150bf 100644 --- a/src/delete.c +++ b/src/delete.c @@ -94,7 +94,7 @@ do_delete(time_t backup_id) { pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - if (backup->status == BACKUP_STATUS_OK) + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { oldest_lsn = backup->start_lsn; oldest_tli = backup->tli; diff --git a/src/merge.c b/src/merge.c index bd3654089..f4a477f21 100644 --- a/src/merge.c +++ b/src/merge.c @@ -78,6 +78,7 @@ do_merge(time_t backup_id) { /* sanity */ if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ backup->status != BACKUP_STATUS_MERGING && backup->status != BACKUP_STATUS_DELETING) @@ -107,6 +108,7 @@ do_merge(time_t backup_id) /* sanity */ if (full_backup->status != BACKUP_STATUS_OK && + full_backup->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ full_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Backup %s has status: %s", @@ -119,6 +121,7 @@ do_merge(time_t backup_id) { /* sanity */ if (dest_backup->status != BACKUP_STATUS_OK && + dest_backup->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ dest_backup->status != BACKUP_STATUS_MERGING && dest_backup->status != BACKUP_STATUS_DELETING) @@ -193,7 +196,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * BACKUP_STATUS_MERGING status then it isn't valid backup until merging * finished. */ - if (to_backup->status == BACKUP_STATUS_OK) + if (to_backup->status == BACKUP_STATUS_OK || + to_backup->status == BACKUP_STATUS_DONE) { pgBackupValidate(to_backup); if (to_backup->status == BACKUP_STATUS_CORRUPT) @@ -205,6 +209,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) * BACKUP_STATUS_MERGING status. */ Assert(from_backup->status == BACKUP_STATUS_OK || + from_backup->status == BACKUP_STATUS_DONE || from_backup->status == BACKUP_STATUS_MERGING); pgBackupValidate(from_backup); if (from_backup->status == BACKUP_STATUS_CORRUPT) diff --git a/src/restore.c b/src/restore.c index f1fea48d2..04dff24c3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -94,7 +94,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore && target_backup_id == INVALID_BACKUP_ID && - current_backup->status != BACKUP_STATUS_OK) + (current_backup->status != BACKUP_STATUS_OK && + current_backup->status != BACKUP_STATUS_DONE)) { elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", base36enc(current_backup->start_time), status2str(current_backup->status)); @@ -110,7 +111,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { /* backup is not ok, - * but in case of CORRUPT, ORPHAN or DONE revalidation is possible + * but in case of CORRUPT or ORPHAN revalidation is possible * unless --no-validate is used, * in other cases throw an error. */ @@ -119,10 +120,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, // 3. restore -i INVALID_ID <- allowed revalidate and restore // 4. restore <- impossible // 5. restore --no-validate <- forbidden - if (current_backup->status != BACKUP_STATUS_OK) + if (current_backup->status != BACKUP_STATUS_OK && + current_backup->status != BACKUP_STATUS_DONE) { - if ((current_backup->status == BACKUP_STATUS_DONE || - current_backup->status == BACKUP_STATUS_ORPHAN || + if ((current_backup->status == BACKUP_STATUS_ORPHAN || current_backup->status == BACKUP_STATUS_CORRUPT || current_backup->status == BACKUP_STATUS_RUNNING) && !rt->no_validate) @@ -205,7 +206,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (is_parent(missing_backup_start_time, backup, false)) { - if (backup->status == BACKUP_STATUS_OK) + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN); @@ -238,7 +240,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_parent(tmp_backup->start_time, backup, false)) { - if (backup->status == BACKUP_STATUS_OK) + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN); @@ -374,7 +377,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_parent(corrupted_backup->start_time, backup, false)) { - if (backup->status == BACKUP_STATUS_OK) + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN); @@ -393,7 +397,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * If dest backup is corrupted or was orphaned in previous check * produce corresponding error message */ - if (dest_backup->status == BACKUP_STATUS_OK) + if (dest_backup->status == BACKUP_STATUS_OK || + dest_backup->status == BACKUP_STATUS_DONE) { if (rt->no_validate) elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); @@ -473,7 +478,8 @@ restore_backup(pgBackup *backup, const char *external_dir_str) bool restore_isok = true; - if (backup->status != BACKUP_STATUS_OK) + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) elog(ERROR, "Backup %s cannot be restored because it is not valid", base36enc(backup->start_time)); diff --git a/src/validate.c b/src/validate.c index 472f72463..b7825f4dd 100644 --- a/src/validate.c +++ b/src/validate.c @@ -417,7 +417,8 @@ do_validate_instance(void) corrupted_backup_found = true; /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) + if (current_backup->status == BACKUP_STATUS_OK || + current_backup->status == BACKUP_STATUS_DONE) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", @@ -440,7 +441,8 @@ do_validate_instance(void) { char *backup_id = base36enc_dup(tmp_backup->start_time); /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK) + if (current_backup->status == BACKUP_STATUS_OK || + current_backup->status == BACKUP_STATUS_DONE) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", @@ -512,7 +514,8 @@ do_validate_instance(void) if (is_parent(current_backup->start_time, backup, false)) { - if (backup->status == BACKUP_STATUS_OK) + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN); From db179af048de8511f4a35c16a6128f3411742814 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 12 Apr 2019 14:48:53 +0300 Subject: [PATCH 0528/2107] PBCKP-36: Fix recovery-target options --- src/help.c | 65 +++++++------ src/pg_probackup.c | 39 +++++--- src/pg_probackup.h | 36 ++++---- src/restore.c | 162 ++++++++++++++++----------------- tests/expected/option_help.out | 12 ++- 5 files changed, 166 insertions(+), 148 deletions(-) diff --git a/src/help.c b/src/help.c index 8e8e3f709..9997c862c 100644 --- a/src/help.c +++ b/src/help.c @@ -123,10 +123,12 @@ help_pg_probackup(void) printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-timeline=timeline]\n")); + printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target=immediate|latest]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); @@ -135,9 +137,9 @@ help_pg_probackup(void) printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--timeline=timeline]\n")); + printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--skip-block-validation]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); @@ -291,15 +293,18 @@ static void help_restore(void) { printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads] [--progress]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline] [-T OLDDIR=NEWDIR]\n")); + printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-timeline=timeline]\n")); + printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--immediate] [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target=immediate|latest]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica] [--no-validate]\n")); + printf(_(" [--restore-as-replica]\n")); + printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [--skip-external-dirs]\n\n")); + printf(_(" [--skip-external-dirs]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -309,18 +314,21 @@ help_restore(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --latest recovery into a latest possible state\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); + printf(_(" --recovery-target-xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --recovery-target-inclusive=boolean\n")); + printf(_(" whether we stop just after the recovery target\n")); + printf(_(" --recovery-target-timeline=timeline\n")); + printf(_(" recovering into a particular timeline\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); - printf(_(" --immediate end recovery as soon as a consistent state is reached\n")); + printf(_(" --recovery-target=immediate|latest\n")); + printf(_(" end recovery as soon as a consistent state is reached or as late as possible\n")); + printf(_(" (default: immediate)\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --recovery-target-action=pause|promote|shutdown\n")); @@ -359,10 +367,11 @@ static void help_validate(void) { printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-i backup-id] [--progress]\n")); - printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n")); - printf(_(" [--timeline=timeline]\n\n")); - printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); + printf(_(" [--recovery-target-timeline=timeline]\n")); + printf(_(" [--skip-block-validation]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -370,11 +379,13 @@ help_validate(void) printf(_(" --progress show progress\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --time=time time stamp up to which recovery will proceed\n")); - printf(_(" --xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --inclusive=boolean whether we stop just after the recovery target\n")); - printf(_(" --timeline=timeline recovering into a particular timeline\n")); + printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); + printf(_(" --recovery-target-xid=xid transaction ID up to which recovery will proceed\n")); + printf(_(" --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); + printf(_(" --recovery-target-inclusive=boolean\n")); + printf(_(" whether we stop just after the recovery target\n")); + printf(_(" --recovery-target-timeline=timeline\n")); + printf(_(" recovering into a particular timeline\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e07a280ca..2a5851c2b 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,10 +78,10 @@ static char *target_xid = NULL; static char *target_lsn = NULL; static char *target_inclusive = NULL; static TimeLineID target_tli; +static char *target_stop; static bool target_immediate; static char *target_name = NULL; static char *target_action = NULL; -static bool target_latest = false; static pgRecoveryTarget *recovery_target_options = NULL; @@ -149,19 +149,18 @@ static ConfigOption cmd_options[] = /* TODO not completed feature. Make it unavailiable from user level { 'b', 18, "remote", &is_remote_backup, SOURCE_CMD_STRICT, }, */ /* restore options */ - { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, - { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, - { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, - { 'b', 157, "latest", &target_latest, SOURCE_CMD_STRICT }, + { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, + { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, + { 's', 144, "recovery-target-lsn", &target_lsn, SOURCE_CMD_STRICT }, + { 's', 138, "recovery-target-inclusive", &target_inclusive, SOURCE_CMD_STRICT }, + { 'u', 139, "recovery-target-timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 157, "recovery-target", &target_stop, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, - { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 'b', 143, "no-validate", &restore_no_validate, SOURCE_CMD_STRICT }, - { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, /* delete options */ @@ -182,6 +181,15 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, + + /* options for backward compatibility */ + { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, + { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, + { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, + { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, + { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, + { 0 } }; @@ -473,11 +481,16 @@ main(int argc, char *argv[]) if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { - /* parse all recovery target options into recovery_target_options structure */ - recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_lsn, target_immediate, - target_name, target_action, restore_no_validate, - target_latest); + /* + * Parse all recovery target options into recovery_target_options + * structure. + */ + recovery_target_options = + parseRecoveryTargetOptions(target_time, target_xid, + target_inclusive, target_tli, target_lsn, + (target_stop != NULL) ? target_stop : + (target_immediate) ? "immediate" : NULL, + target_name, target_action, restore_no_validate); } if (num_threads < 1) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c50e1a8d8..80c8fa91b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -273,26 +273,22 @@ struct pgBackup /* Recovery target for restore and validate subcommands */ typedef struct pgRecoveryTarget { - bool time_specified; - time_t recovery_target_time; - /* add one more field in order to avoid deparsing recovery_target_time back */ - const char *target_time_string; - bool xid_specified; - TransactionId recovery_target_xid; - /* add one more field in order to avoid deparsing recovery_target_xid back */ - const char *target_xid_string; - bool lsn_specified; - XLogRecPtr recovery_target_lsn; - /* add one more field in order to avoid deparsing recovery_target_lsn back */ - const char *target_lsn_string; - TimeLineID recovery_target_tli; - bool recovery_target_inclusive; + time_t target_time; + /* add one more field in order to avoid deparsing target_time back */ + const char *time_string; + TransactionId target_xid; + /* add one more field in order to avoid deparsing target_xid back */ + const char *xid_string; + XLogRecPtr target_lsn; + /* add one more field in order to avoid deparsing target_lsn back */ + const char *lsn_string; + TimeLineID target_tli; + bool target_inclusive; bool inclusive_specified; - bool recovery_target_immediate; - const char *recovery_target_name; - const char *recovery_target_action; + const char *target_stop; + const char *target_name; + const char *target_action; bool restore_no_validate; - bool latest; } pgRecoveryTarget; typedef struct @@ -427,8 +423,8 @@ extern bool satisfy_recovery_target(const pgBackup *backup, extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, - bool target_immediate, const char *target_name, - const char *target_action, bool restore_no_validate, bool latest); + const char *target_stop, const char *target_name, + const char *target_action, bool restore_no_validate); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index e1a65dd7e..0c0237c7d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -133,13 +133,13 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(current_backup->start_time), status2str(current_backup->status)); } - if (rt->recovery_target_tli) + if (rt->target_tli) { parray *timelines; - elog(LOG, "target timeline ID = %u", rt->recovery_target_tli); + elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ - timelines = read_timeline_history(rt->recovery_target_tli); + timelines = read_timeline_history(rt->target_tli); if (!satisfy_timeline(timelines, current_backup)) { @@ -358,8 +358,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * We pass base_full_backup timeline as last argument to this function, * because it's needed to form the name of xlog file. */ - validate_wal(dest_backup, arclog_path, rt->recovery_target_time, - rt->recovery_target_xid, rt->recovery_target_lsn, + validate_wal(dest_backup, arclog_path, rt->target_time, + rt->target_xid, rt->target_lsn, base_full_backup->tli, instance_config.xlog_seg_size); } /* Orphinize every OK descendant of corrupted backup */ @@ -417,9 +417,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - if (rt->lsn_specified && parse_server_version(backup->server_version) < 100000) + if (rt->lsn_string && + parse_server_version(backup->server_version) < 100000) elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", - base36enc(dest_backup->start_time), dest_backup->server_version); + base36enc(dest_backup->start_time), + dest_backup->server_version); /* * Backup was locked during validation if no-validate wasn't @@ -772,8 +774,8 @@ create_recovery_conf(time_t backup_id, bool need_restore_conf; need_restore_conf = !backup->stream || - (rt->time_specified || rt->xid_specified || rt->lsn_specified) || - rt->latest; + (rt->time_string || rt->xid_string || rt->lsn_string) || + (rt->target_stop != NULL && strcmp(rt->target_stop, "latest") == 0); /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || restore_as_replica)) @@ -802,30 +804,30 @@ create_recovery_conf(time_t backup_id, * We've already checked that only one of the four following mutually * exclusive options is specified, so the order of calls is insignificant. */ - if (rt->recovery_target_name) - fprintf(fp, "recovery_target_name = '%s'\n", rt->recovery_target_name); + if (rt->target_name) + fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); - if (rt->time_specified) - fprintf(fp, "recovery_target_time = '%s'\n", rt->target_time_string); + if (rt->time_string) + fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); - if (rt->xid_specified) - fprintf(fp, "recovery_target_xid = '%s'\n", rt->target_xid_string); + if (rt->xid_string) + fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); - if (rt->recovery_target_lsn) - fprintf(fp, "recovery_target_lsn = '%s'\n", rt->target_lsn_string); + if (rt->lsn_string) + fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - if (rt->recovery_target_immediate) - fprintf(fp, "recovery_target = 'immediate'\n"); + if (rt->target_stop) + fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); if (rt->inclusive_specified) fprintf(fp, "recovery_target_inclusive = '%s'\n", - rt->recovery_target_inclusive?"true":"false"); + rt->target_inclusive ? "true" : "false"); - if (rt->recovery_target_tli) - fprintf(fp, "recovery_target_timeline = '%u'\n", rt->recovery_target_tli); + if (rt->target_tli) + fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); - if (rt->recovery_target_action) - fprintf(fp, "recovery_target_action = '%s'\n", rt->recovery_target_action); + if (rt->target_action) + fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); } if (restore_as_replica) @@ -951,14 +953,14 @@ read_timeline_history(TimeLineID targetTLI) bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) { - if (rt->xid_specified) - return backup->recovery_xid <= rt->recovery_target_xid; + if (rt->xid_string) + return backup->recovery_xid <= rt->target_xid; - if (rt->time_specified) - return backup->recovery_time <= rt->recovery_target_time; + if (rt->time_string) + return backup->recovery_time <= rt->target_time; - if (rt->lsn_specified) - return backup->stop_lsn <= rt->recovery_target_lsn; + if (rt->lsn_string) + return backup->stop_lsn <= rt->target_lsn; return true; } @@ -990,15 +992,12 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_inclusive, TimeLineID target_tli, const char *target_lsn, - bool target_immediate, + const char *target_stop, const char *target_name, const char *target_action, - bool restore_no_validate, bool latest) + bool restore_no_validate) { - time_t dummy_time; - TransactionId dummy_xid; bool dummy_bool; - XLogRecPtr dummy_lsn; /* * count the number of the mutually exclusive options which may specify * recovery target. If final value > 1, throw an error. @@ -1007,114 +1006,111 @@ parseRecoveryTargetOptions(const char *target_time, pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); /* fill all options with default values */ - rt->time_specified = false; - rt->xid_specified = false; - rt->inclusive_specified = false; - rt->lsn_specified = false; - rt->recovery_target_time = 0; - rt->recovery_target_xid = 0; - rt->recovery_target_lsn = InvalidXLogRecPtr; - rt->target_time_string = NULL; - rt->target_xid_string = NULL; - rt->target_lsn_string = NULL; - rt->recovery_target_inclusive = false; - rt->recovery_target_tli = 0; - rt->recovery_target_immediate = false; - rt->recovery_target_name = NULL; - rt->recovery_target_action = NULL; - rt->restore_no_validate = false; + MemSet(rt, 0, sizeof(pgRecoveryTarget)); /* parse given options */ if (target_time) { + time_t dummy_time; + recovery_target_specified++; - rt->time_specified = true; - rt->target_time_string = target_time; + rt->time_string = target_time; if (parse_time(target_time, &dummy_time, false)) - rt->recovery_target_time = dummy_time; + rt->target_time = dummy_time; else - elog(ERROR, "Invalid value of --time option %s", target_time); + elog(ERROR, "Invalid value for --recovery-target-time option %s", + target_time); } if (target_xid) { + TransactionId dummy_xid; + recovery_target_specified++; - rt->xid_specified = true; - rt->target_xid_string = target_xid; + rt->xid_string = target_xid; #ifdef PGPRO_EE if (parse_uint64(target_xid, &dummy_xid, 0)) #else if (parse_uint32(target_xid, &dummy_xid, 0)) #endif - rt->recovery_target_xid = dummy_xid; + rt->target_xid = dummy_xid; else - elog(ERROR, "Invalid value of --xid option %s", target_xid); + elog(ERROR, "Invalid value for --recovery-target-xid option %s", + target_xid); } if (target_lsn) { + XLogRecPtr dummy_lsn; + recovery_target_specified++; - rt->lsn_specified = true; - rt->target_lsn_string = target_lsn; + rt->lsn_string = target_lsn; if (parse_lsn(target_lsn, &dummy_lsn)) - rt->recovery_target_lsn = dummy_lsn; + rt->target_lsn = dummy_lsn; else - elog(ERROR, "Invalid value of --lsn option %s", target_lsn); + elog(ERROR, "Invalid value of --ecovery-target-lsn option %s", + target_lsn); } if (target_inclusive) { rt->inclusive_specified = true; if (parse_bool(target_inclusive, &dummy_bool)) - rt->recovery_target_inclusive = dummy_bool; + rt->target_inclusive = dummy_bool; else - elog(ERROR, "Invalid value of --inclusive option %s", target_inclusive); + elog(ERROR, "Invalid value for --recovery-target-inclusive option %s", + target_inclusive); } - rt->recovery_target_tli = target_tli; - if (target_immediate) + rt->target_tli = target_tli; + if (target_stop) { - recovery_target_specified++; - rt->recovery_target_immediate = target_immediate; - } + if ((strcmp(target_stop, "immediate") != 0) + && (strcmp(target_stop, "latest") != 0)) + elog(ERROR, "Invalid value for --recovery-target option %s", + target_stop); - if (restore_no_validate) - { - rt->restore_no_validate = restore_no_validate; + recovery_target_specified++; + rt->target_stop = target_stop; } - rt->latest = latest; + rt->restore_no_validate = restore_no_validate; if (target_name) { recovery_target_specified++; - rt->recovery_target_name = target_name; + rt->target_name = target_name; } if (target_action) { - rt->recovery_target_action = target_action; - if ((strcmp(target_action, "pause") != 0) && (strcmp(target_action, "promote") != 0) && (strcmp(target_action, "shutdown") != 0)) - elog(ERROR, "Invalid value of --recovery-target-action option %s", target_action); + elog(ERROR, "Invalid value for --recovery-target-action option %s", + target_action); + + rt->target_action = target_action; } else { /* Default recovery target action is pause */ - rt->recovery_target_action = "pause"; + rt->target_action = "pause"; } /* More than one mutually exclusive option was defined. */ if (recovery_target_specified > 1) - elog(ERROR, "At most one of --immediate, --target-name, --time, --xid, or --lsn can be used"); + elog(ERROR, "At most one of --recovery-target, --recovery-target-name, --recovery-target-time, --recovery-target-xid, or --recovery-target-lsn can be specified"); - /* If none of the options is defined, '--inclusive' option is meaningless */ - if (!(rt->xid_specified || rt->time_specified || rt->lsn_specified) && rt->recovery_target_inclusive) - elog(ERROR, "--inclusive option applies when either --time or --xid is specified"); + /* + * If none of the options is defined, '--recovery-target-inclusive' option + * is meaningless. + */ + if (!(rt->xid_string || rt->time_string || rt->lsn_string) && + rt->target_inclusive) + elog(ERROR, "--recovery-target-inclusive option applies when either --recovery-target-time, --recovery-target-xid or --recovery-target-lsn is specified"); return rt; } diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 23f3cf912..4a722bb7f 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -55,10 +55,12 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [-j num-threads] - [--time=time|--xid=xid|--lsn=lsn|--latest [--inclusive=boolean]] - [--timeline=timeline] [-T OLDDIR=NEWDIR] [--progress] + [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] + [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] - [--immediate] [--recovery-target-name=target-name] + [--recovery-target=immediate|latest] + [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] @@ -67,9 +69,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] - [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]] + [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-name=target-name] - [--timeline=timeline] + [--recovery-target-timeline=timeline] [--skip-block-validation] pg_probackup show -B backup-path From dd108817659255375b5f6583e6043f2115254b33 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 12 Apr 2019 15:06:58 +0300 Subject: [PATCH 0529/2107] [refer #PBCKP-3] Reset MyLocation after reading config file --- src/pg_probackup.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f4f53dc05..0b144e725 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -184,6 +184,15 @@ static ConfigOption cmd_options[] = { 0 } }; +static void +setMyLocation(void) +{ + MyLocation = IsSshProtocol() + ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) + ? FIO_DB_HOST : FIO_BACKUP_HOST + : FIO_LOCAL_HOST; +} + /* * Entry point of pg_probackup command. */ @@ -358,10 +367,7 @@ main(int argc, char *argv[]) } canonicalize_path(backup_path); - MyLocation = IsSshProtocol() - ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST : FIO_BACKUP_HOST - : FIO_LOCAL_HOST; + setMyLocation(); /* Ensure that backup_path is a path to a directory */ rc = fio_stat(backup_path, &stat_buf, true, FIO_BACKUP_HOST); @@ -423,6 +429,7 @@ main(int argc, char *argv[]) BACKUP_CATALOG_CONF_FILE); config_read_opt(path, instance_options, ERROR, true, false); } + setMyLocation(); } /* Initialize logger */ From df8fe86d6d1dc6a0c6126dad0a2d408e5f4789cc Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 12 Apr 2019 18:41:33 +0300 Subject: [PATCH 0530/2107] code cleanup --- src/backup.c | 52 ++++++++++++++++++---------------------------- src/data.c | 18 ++++++++-------- src/pg_probackup.h | 1 - src/utils/pgut.c | 47 ----------------------------------------- 4 files changed, 29 insertions(+), 89 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1e3cdebea..25adb3f86 100644 --- a/src/backup.c +++ b/src/backup.c @@ -47,6 +47,7 @@ const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; +/* list of indexes for use in checkdb --amcheck */ static parray *index_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ @@ -95,7 +96,6 @@ static bool pg_stop_backup_is_sent = false; */ static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -//static void threads_conn_disconnect(bool fatal, void *userdata); static void pgdata_basic_setup(void); @@ -941,6 +941,7 @@ do_backup_instance(void) backup_files_list = NULL; } +/* collect list of files and run threads to check files in the instance */ static void do_block_validation(void) { @@ -1022,18 +1023,19 @@ do_block_validation(void) check_isok = false; } - /* TODO write better info message */ - if (check_isok) - elog(INFO, "Data files are valid"); - else - elog(ERROR, "Checkdb failed"); - + /* cleanup */ if (backup_files_list) { parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); backup_files_list = NULL; } + + /* TODO write better info message */ + if (check_isok) + elog(INFO, "Data files are valid"); + else + elog(ERROR, "Checkdb failed"); } static void @@ -1113,9 +1115,7 @@ do_amcheck(void) for (j = 0; j < num_threads; j++) { backup_files_arg *arg = &(threads_args[j]); - elog(VERBOSE, "Start thread num: %i", j); - pthread_create(&threads[j], NULL, check_indexes, arg); } @@ -1143,13 +1143,14 @@ do_amcheck(void) else elog(INFO, "Checkdb --amcheck executed"); + /* FIXME We already wrote message about skipped databases. + * Maybe we shouldn't check db_skipped here ? + */ if (check_isok && !interrupted && !db_skipped) elog(INFO, "Indexes are valid"); } /* Entry point of pg_probackup CHECKDB subcommand. */ -/* TODO consider moving some code common with do_backup_instance - * to separate function ot to pgdata_basic_setup */ int do_checkdb(bool need_amcheck) { @@ -1173,7 +1174,6 @@ do_checkdb(bool need_amcheck) * Common code for CHECKDB and BACKUP commands. * Ensure that we're able to connect to the instance * check compatibility and fill basic info. - * TODO maybe move it to pg_probackup.c */ static void pgdata_basic_setup(void) @@ -1441,7 +1441,6 @@ check_system_identifiers(void) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but connected instance system id is " UINT64_FORMAT, instance_config.system_identifier, system_id_conn); - if (system_id_pgdata != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but target backup directory system id is " UINT64_FORMAT, @@ -2542,23 +2541,11 @@ backup_disconnect(bool fatal, void *userdata) } /* - * Disconnect checkdb connections created in threads during quit pg_probackup. + * Check files in PGDATA. + * Read all files listed in backup_files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. */ -//static void -//threads_conn_disconnect(bool fatal, void *userdata) -//{ -// -// backup_files_arg *arguments = (backup_files_arg *) userdata; -// -// elog(VERBOSE, "threads_conn_disconnect, num_threads %d", arguments->thread_num); -// -// if (arguments->backup_conn) -// { -// pgut_cancel(arguments->backup_conn); -// pgut_disconnect(arguments->backup_conn); -// } -//} - static void * check_files(void *arg) { @@ -2605,7 +2592,7 @@ check_files(void *arg) else { elog(ERROR, - "can't stat file to backup \"%s\": %s", + "can't stat file to check \"%s\": %s", file->path, strerror(errno)); } } @@ -2625,19 +2612,21 @@ check_files(void *arg) file->path + strlen(arguments->from_root) + 1); if (!check_data_file(arguments, file)) - arguments->ret = 2; + arguments->ret = 2; /* FIXME what does 2 mean? */ } } else elog(WARNING, "unexpected file type %d", buf.st_mode); } + /* FIXME why do we reset return code here? */ if (arguments->ret == 1) arguments->ret = 0; return NULL; } +/* Check indexes with amcheck */ static void * check_indexes(void *arg) { @@ -2645,7 +2634,6 @@ check_indexes(void *arg) backup_files_arg *arguments = (backup_files_arg *) arg; int n_indexes = 0; - /* Check indexes with amcheck */ if (arguments->index_list) n_indexes = parray_num(arguments->index_list); diff --git a/src/data.c b/src/data.c index 303036ab0..5cd1b5d22 100644 --- a/src/data.c +++ b/src/data.c @@ -1612,6 +1612,12 @@ validate_one_page(Page page, pgFile *file, } +/* + * Valiate pages of datafile in PGDATA one by one. + * + * returns true if the file is valid + * also returns true if the file was not found + */ bool check_data_file(backup_files_arg* arguments, pgFile *file) @@ -1622,14 +1628,8 @@ check_data_file(backup_files_arg* arguments, int n_blocks_skipped = 0; int page_state; char curr_page[BLCKSZ]; + bool is_valid = true; - bool is_valid = true; - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - - /* open backup mode file for read */ in = fopen(file->path, PG_BINARY_R); if (in == NULL) { @@ -1662,7 +1662,7 @@ check_data_file(backup_files_arg* arguments, for (blknum = 0; blknum < nblocks; blknum++) { - page_state = prepare_page(arguments, file, InvalidXLogRecPtr, //0 = InvalidXLogRecPtr + page_state = prepare_page(arguments, file, InvalidXLogRecPtr, blknum, nblocks, in, &n_blocks_skipped, BACKUP_MODE_FULL, curr_page, false); @@ -1672,6 +1672,7 @@ check_data_file(backup_files_arg* arguments, if (page_state == PageIsCorrupted) { /* Page is corrupted */ + // TODO why this message is commented? //elog(WARNING, "File %s, block %u is CORRUPTED.", // file->path, blknum); is_valid = false; @@ -1686,7 +1687,6 @@ check_data_file(backup_files_arg* arguments, /* Page is corrupted */ is_valid = false; } - } fclose(in); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ab20f774e..b48e81a09 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -139,7 +139,6 @@ typedef struct pg_indexEntry Oid indexrelid; char *name; char *dbname; - char *snapshot; /* snapshot for index check */ char *amcheck_nspname; /* schema where amcheck extention is located */ volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ } pg_indexEntry; diff --git a/src/utils/pgut.c b/src/utils/pgut.c index c262f6a74..5a3301449 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -43,7 +43,6 @@ static void on_interrupt(void); static void on_cleanup(void); static pqsigfunc oldhandler = NULL; -//bool is_result_ready(PGconn *conn); void discard_response(PGconn *conn); void @@ -405,17 +404,6 @@ pgut_execute_parallel(PGconn* conn, */ (text_result) ? 0 : 1); - /* wait for processing */ -// while(!is_result_ready(conn)) -// { -// if (interrupted) -// { -// pgut_cancel(conn); -// pgut_disconnect(conn); -// elog(ERROR, "Interrupted"); -// } -// } - /* wait for processing, TODO: timeout */ for (;;) { @@ -1052,38 +1040,3 @@ discard_response(PGconn *conn) PQclear(res); } while (res); } - -//bool is_result_ready(PGconn * conn) -//{ -// int sock; -// struct timeval timeout; -// fd_set read_mask; -// -// if (!PQisBusy(conn)) -// return true; -// -// sock = PQsocket(conn); -// -// timeout.tv_sec = (time_t)1; -// timeout.tv_usec = 0; -// -// FD_ZERO(&read_mask); -// FD_SET(sock, &read_mask); -// -// if (select(sock + 1, &read_mask, NULL, NULL, &timeout) == 0) -// return false; -// else if (FD_ISSET(sock, &read_mask)) -// { -// if (PQconsumeInput(conn)) -// { -// if (PQisBusy(conn)) -// return false; -// else -// return true; -// } -// else -// return false; -// } -// else -// return false; -//} From 290df2637313207e68919562023ceaf415baf579 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 15:14:20 +0300 Subject: [PATCH 0531/2107] tests: minor fixes --- tests/external.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/external.py b/tests/external.py index e085b153b..5c82fca6b 100644 --- a/tests/external.py +++ b/tests/external.py @@ -253,9 +253,7 @@ def test_backup_multiple_external(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -326,9 +324,7 @@ def test_external_backward_compatibility(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -429,9 +425,7 @@ def test_external_backward_compatibility_merge_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -523,9 +517,7 @@ def test_external_backward_compatibility_merge_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -641,12 +633,10 @@ def test_external_merge(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.add_instance(backup_dir, 'node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=5) From aa1b350526152f533d181392ce4ee49d55b94242 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 16:16:35 +0300 Subject: [PATCH 0532/2107] help fix: add closing bracket to parenthesis with example of log-filename option --- src/help.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/help.c b/src/help.c index 9997c862c..1d5eb0c80 100644 --- a/src/help.c +++ b/src/help.c @@ -242,7 +242,7 @@ help_backup(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); @@ -350,7 +350,7 @@ help_restore(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); @@ -399,7 +399,7 @@ help_validate(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); @@ -452,7 +452,7 @@ help_delete(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); @@ -494,7 +494,7 @@ help_merge(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); @@ -544,7 +544,7 @@ help_set_config(void) printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); From eb767ed0ad0dcf98d69374f821cc2cae8a2e6485 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 16:50:50 +0300 Subject: [PATCH 0533/2107] tests: added test_restore_target_immediate_stream, test_restore_target_immediate_archive and test_restore_target_latest_archive in "restore" module --- tests/restore.py | 150 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index 3a5ec616e..fdab8049d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -6,6 +6,7 @@ import sys from time import sleep from datetime import datetime, timedelta +import hashlib module_name = 'restore' @@ -1782,3 +1783,152 @@ def test_restore_backup_from_future(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_target_immediate_stream(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Take delta + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--immediate']) + + # For stream backup with immediate recovery target there is no need to + # create recovery.conf. Is it wise? + self.assertFalse( + os.path.isfile(recovery_conf)) + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--recovery-target=immediate']) + + # For stream backup with immediate recovery target there is no need to + # create recovery.conf. Is it wise? + self.assertFalse( + os.path.isfile(recovery_conf)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_target_immediate_archive(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + # Take delta + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--immediate']) + + # For archive backup with immediate recovery target + # recovery.conf is mandatory + with open(recovery_conf, 'r') as f: + self.assertIn("recovery_target = 'immediate'", f.read()) + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--recovery-target=immediate']) + + # For archive backup with immediate recovery target + # recovery.conf is mandatory + with open(recovery_conf, 'r') as f: + self.assertIn("recovery_target = 'immediate'", f.read()) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_target_latest_archive(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore + node.cleanup() + self.restore_node( + backup_dir, 'node', node) + + with open(recovery_conf, 'r') as f: + print(f.read()) + + hash_1 = hashlib.md5( + open(recovery_conf, 'rb').read()).hexdigest() + + # restore + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--recovery-target=latest']) + + with open(recovery_conf, 'r') as f: + print(f.read()) + + hash_2 = hashlib.md5( + open(recovery_conf, 'rb').read()).hexdigest() + + self.assertEqual(hash_1, hash_2) + + # Clean after yourself + self.del_test_dir(module_name, fname) From e5e16320fd7f71967550b957724b9788865c285a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 21:27:22 +0300 Subject: [PATCH 0534/2107] checkdb: minor fixes, point out by a.lubennikova@postgrespro.ru --- src/backup.c | 12 ++++++++---- src/data.c | 18 +++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/backup.c b/src/backup.c index 25adb3f86..dd8a007c8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1143,8 +1143,8 @@ do_amcheck(void) else elog(INFO, "Checkdb --amcheck executed"); - /* FIXME We already wrote message about skipped databases. - * Maybe we shouldn't check db_skipped here ? + /* We cannot state that all indexes are ok + * without checking indexes in all databases */ if (check_isok && !interrupted && !db_skipped) elog(INFO, "Indexes are valid"); @@ -2612,14 +2612,18 @@ check_files(void *arg) file->path + strlen(arguments->from_root) + 1); if (!check_data_file(arguments, file)) - arguments->ret = 2; /* FIXME what does 2 mean? */ + arguments->ret = 2; /* corruption found */ } } else elog(WARNING, "unexpected file type %d", buf.st_mode); } - /* FIXME why do we reset return code here? */ + /* Ret values: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ if (arguments->ret == 1) arguments->ret = 0; diff --git a/src/data.c b/src/data.c index 5cd1b5d22..667991ac5 100644 --- a/src/data.c +++ b/src/data.c @@ -1643,14 +1643,15 @@ check_data_file(backup_files_arg* arguments, return true; } - elog(ERROR, "cannot open file \"%s\": %s", + elog(WARNING, "cannot open file \"%s\": %s", file->path, strerror(errno)); + return false; } if (file->size % BLCKSZ != 0) { fclose(in); - elog(ERROR, "File: %s, invalid file size %zu", file->path, file->size); + elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size); } /* @@ -1671,15 +1672,18 @@ check_data_file(backup_files_arg* arguments, if (page_state == PageIsCorrupted) { - /* Page is corrupted */ - // TODO why this message is commented? - //elog(WARNING, "File %s, block %u is CORRUPTED.", - // file->path, blknum); + /* Page is corrupted, no need to elog about it, + * prepare_page() already done that + */ is_valid = false; continue; } - /* Page is found and this point, but it may not be 'sane' */ + /* At this point page is found and its checksum is ok, if any + * but could be 'insane' + * TODO: between prepare_page and validate_one_page we + * compute and compare checksum twice, it`s ineffective + */ if (validate_one_page(curr_page, file, blknum, InvalidXLogRecPtr, 0) == PAGE_IS_FOUND_AND_NOT_VALID) From 2e639a81fe352ca35ea94b3d2fa4708afc736472 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 22:33:04 +0300 Subject: [PATCH 0535/2107] checkdb: close initial connection, skipped db in amcheck now counts as fatal error --- src/backup.c | 15 ++++++++------- src/pg_probackup.h | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index dd8a007c8..676b56ce6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1063,6 +1063,7 @@ do_amcheck(void) 0, NULL); n_databases = PQntuples(res_db); + PQclear(res_db); elog(INFO, "Start amchecking PostgreSQL instance"); @@ -1134,14 +1135,17 @@ do_amcheck(void) break; } + /* close initial connection to pgdatabase */ + pgut_disconnect(backup_conn); + /* TODO write better info message */ if (db_skipped) elog(WARNING, "Some databases were not checked"); - if (!check_isok) + if (!check_isok || db_skipped) elog(ERROR, "Checkdb --amcheck failed"); - else - elog(INFO, "Checkdb --amcheck executed"); + + elog(INFO, "Checkdb --amcheck executed successfully"); /* We cannot state that all indexes are ok * without checking indexes in all databases @@ -1151,7 +1155,7 @@ do_amcheck(void) } /* Entry point of pg_probackup CHECKDB subcommand. */ -int +void do_checkdb(bool need_amcheck) { @@ -1165,9 +1169,6 @@ do_checkdb(bool need_amcheck) if (need_amcheck) do_amcheck(); - - /* TODO: need to exit with 1 if some corruption is found */ - return 0; } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b48e81a09..34efcad89 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -421,7 +421,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time); -extern int do_checkdb(bool need_amcheck); +extern void do_checkdb(bool need_amcheck); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, From c290624765110d605ea788a343c10b9ffe90491b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 22:50:39 +0300 Subject: [PATCH 0536/2107] remove is_checksum_enabled variable --- src/backup.c | 10 +++++----- src/data.c | 2 +- src/pg_probackup.h | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 676b56ce6..f16294c69 100644 --- a/src/backup.c +++ b/src/backup.c @@ -73,7 +73,6 @@ static StreamThreadArg stream_thread_arg = {"", NULL, 1}; static int is_ptrack_enable = false; bool is_ptrack_support = false; -bool is_checksum_enabled = false; bool exclusive_backup = false; bool heapallindexed_is_supported = false; @@ -1201,9 +1200,10 @@ pgdata_basic_setup(void) /* Confirm that this server version is supported */ check_server_version(); - current.checksum_version = get_data_checksum_version(true); - - is_checksum_enabled = pg_checksum_enable(); + if (pg_checksum_enable()) + current.checksum_version = 1; + else + current.checksum_version = 0; /* @@ -1216,7 +1216,7 @@ pgdata_basic_setup(void) if (!is_remote_backup) check_system_identifiers(); - if (is_checksum_enabled) + if (current.checksum_version) elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " "Data block corruption will be detected"); else diff --git a/src/data.c b/src/data.c index 667991ac5..9bb97fad2 100644 --- a/src/data.c +++ b/src/data.c @@ -419,7 +419,7 @@ prepare_page(backup_files_arg *arguments, */ memcpy(page, ptrack_page, BLCKSZ); free(ptrack_page); - if (is_checksum_enabled) + if (current.checksum_version) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* get lsn from page, provided by pg_ptrack_get_block() */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 34efcad89..b497ea310 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -385,7 +385,6 @@ extern bool smooth_checkpoint; extern bool is_remote_backup; extern bool is_ptrack_support; -extern bool is_checksum_enabled; extern bool exclusive_backup; /* restore options */ From 367f885a9b82b4c57f1d061692f2fb42a2d57cb6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 23:17:49 +0300 Subject: [PATCH 0537/2107] checkdb: error at the end of do_amcheck if execution was interrupted --- src/backup.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index f16294c69..6a88f19ff 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1062,7 +1062,6 @@ do_amcheck(void) 0, NULL); n_databases = PQntuples(res_db); - PQclear(res_db); elog(INFO, "Start amchecking PostgreSQL instance"); @@ -1134,14 +1133,11 @@ do_amcheck(void) break; } - /* close initial connection to pgdatabase */ - pgut_disconnect(backup_conn); - /* TODO write better info message */ if (db_skipped) elog(WARNING, "Some databases were not checked"); - if (!check_isok || db_skipped) + if (!check_isok || db_skipped || interrupted) elog(ERROR, "Checkdb --amcheck failed"); elog(INFO, "Checkdb --amcheck executed successfully"); @@ -3511,7 +3507,7 @@ get_index_list(PGresult *res_db, int db_number, strcmp(PQgetvalue(res, 0, 2), "1") != 0) heapallindexed_is_supported = true; - elog(INFO, "Amchecking database %s using module '%s' version %s from schema '%s'", + elog(INFO, "Amchecking database '%s' using module '%s' version %s from schema '%s'", dbname, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); if (!heapallindexed_is_supported && heapallindexed) From abd4ed01bfdfc89e53a5ebb830d43c4245bf3999 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Apr 2019 23:31:19 +0300 Subject: [PATCH 0538/2107] checkdb: for amcheck only mode do compare system ID, because all work is done by server --- src/backup.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6a88f19ff..8a8d040c6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -96,7 +96,7 @@ static bool pg_stop_backup_is_sent = false; static void backup_cleanup(bool fatal, void *userdata); static void backup_disconnect(bool fatal, void *userdata); -static void pgdata_basic_setup(void); +static void pgdata_basic_setup(bool amcheck_only); static void *backup_files(void *arg); static void *remote_backup_files(void *arg); @@ -1147,17 +1147,24 @@ do_amcheck(void) */ if (check_isok && !interrupted && !db_skipped) elog(INFO, "Indexes are valid"); + + /* close initial connection to pgdatabase */ + pgut_disconnect(backup_conn); } /* Entry point of pg_probackup CHECKDB subcommand. */ void do_checkdb(bool need_amcheck) { + bool amcheck_only = false; if (skip_block_validation && !need_amcheck) elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); - pgdata_basic_setup(); + if (skip_block_validation && need_amcheck) + amcheck_only = true; + + pgdata_basic_setup(amcheck_only); if (!skip_block_validation) do_block_validation(); @@ -1170,12 +1177,17 @@ do_checkdb(bool need_amcheck) * Common code for CHECKDB and BACKUP commands. * Ensure that we're able to connect to the instance * check compatibility and fill basic info. + * For checkdb launched in amcheck mode with pgdata validation + * do not check system ID, it gives user an opportunity to + * check remote PostgreSQL instance. + * Also checking system ID in this case serves no purpose, because + * all work is done by server. */ static void -pgdata_basic_setup(void) +pgdata_basic_setup(bool amcheck_only) { - /* PGDATA is always required */ - if (instance_config.pgdata == NULL) + /* PGDATA is always required unless running checkdb in amcheck only mode */ + if (instance_config.pgdata == NULL && !amcheck_only) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); @@ -1209,7 +1221,7 @@ pgdata_basic_setup(void) */ /* TODO fix it for remote backup */ - if (!is_remote_backup) + if (!is_remote_backup && !amcheck_only) check_system_identifiers(); if (current.checksum_version) @@ -1234,7 +1246,7 @@ do_backup(time_t start_time) * setup backup_conn, do some compatibility checks and * fill basic info about instance */ - pgdata_basic_setup(); + pgdata_basic_setup(false); /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 From fff9318401f50d3885da50ff3f8f5cca6e87de07 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 15 Apr 2019 11:32:09 +0300 Subject: [PATCH 0539/2107] PBCKP-36: Do not write 'latest' into recovery.conf --- src/help.c | 1 - src/restore.c | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index 1d5eb0c80..623bc025f 100644 --- a/src/help.c +++ b/src/help.c @@ -328,7 +328,6 @@ help_restore(void) printf(_(" --recovery-target=immediate|latest\n")); printf(_(" end recovery as soon as a consistent state is reached or as late as possible\n")); - printf(_(" (default: immediate)\n")); printf(_(" --recovery-target-name=target-name\n")); printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --recovery-target-action=pause|promote|shutdown\n")); diff --git a/src/restore.c b/src/restore.c index 0c0237c7d..23545151b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -772,10 +772,12 @@ create_recovery_conf(time_t backup_id, char path[MAXPGPATH]; FILE *fp; bool need_restore_conf; + bool target_latest; + target_latest = rt->target_stop != NULL && + strcmp(rt->target_stop, "latest") == 0; need_restore_conf = !backup->stream || - (rt->time_string || rt->xid_string || rt->lsn_string) || - (rt->target_stop != NULL && strcmp(rt->target_stop, "latest") == 0); + (rt->time_string || rt->xid_string || rt->lsn_string) || target_latest; /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || restore_as_replica)) @@ -816,7 +818,7 @@ create_recovery_conf(time_t backup_id, if (rt->lsn_string) fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - if (rt->target_stop) + if (rt->target_stop && !target_latest) fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); if (rt->inclusive_specified) From 60f89246c1b61d7625b497b046dd979247fb901a Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 15 Apr 2019 15:04:49 +0300 Subject: [PATCH 0540/2107] [refer #PBCKP-3] Set location to FIO_LOCAL_HOST for all commands except backup/resotre/archive --- src/pg_probackup.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0b144e725..9f66265c3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -189,7 +189,10 @@ setMyLocation(void) { MyLocation = IsSshProtocol() ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST : FIO_BACKUP_HOST + ? FIO_DB_HOST + : (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD) + ? FIO_BACKUP_HOST + : FIO_LOCAL_HOST : FIO_LOCAL_HOST; } From 97e6182ca1a1abc53d301f4222c40002374d6825 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 17:54:30 +0300 Subject: [PATCH 0541/2107] tests: fix header size for EE versions >= 10 --- tests/helpers/ptrack_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0fc054440..2fd34c235 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -395,7 +395,10 @@ def get_md5_per_page_for_fork(self, file, size_in_pages): def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): if self.get_pgpro_edition(node) == 'enterprise': - header_size = 48 + if self.get_version(node) < self.version_to_num('10.0'): + header_size = 48 + else + header_size = 24 else: header_size = 24 ptrack_bits_for_fork = [] From 44065395283ec7c5eaab5d95b006637b92efc14a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 19:02:07 +0300 Subject: [PATCH 0542/2107] tests: added restore.RestoreTest.test_restore_target_new_options --- tests/restore.py | 176 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 1 deletion(-) diff --git a/tests/restore.py b/tests/restore.py index fdab8049d..341ba3aa0 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1012,7 +1012,7 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) node.safe_psql("postgres", "create table t_heap(a int)") - node.safe_psql("postgres", "select pg_switch_xlog()") + node.stop() node.cleanup() @@ -1932,3 +1932,177 @@ def test_restore_target_latest_archive(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_target_new_options(self): + """more complex test_restore_chain()""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.safe_psql( + "postgres", + "CREATE TABLE tbl0005 (a text)") + + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + + target_name = 'savepoint' + + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + if self.get_version(node) > self.version_to_num('10.0'): + res = con.execute("SELECT pg_current_wal_lsn()") + else: + res = con.execute("SELECT pg_current_xlog_location()") + + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + # Restore with recovery target time + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-time={0}'.format(target_time), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + print(recovery_conf_content) + + self.assertIn( + "recovery_target_time = '{0}'".format(target_time), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target xid + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + print(recovery_conf_content) + + self.assertIn( + "recovery_target_xid = '{0}'".format(target_xid), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target lsn + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + print(recovery_conf_content) + + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target name + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-name={0}'.format(target_name), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + print(recovery_conf_content) + + self.assertIn( + "recovery_target_name = '{0}'".format(target_name), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Clean after yourself + self.del_test_dir(module_name, fname) From e208e7f191cd7e2fa9cddf0d42c19aff485a9cb2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 19:53:38 +0300 Subject: [PATCH 0543/2107] tests: add more comments in restore module --- tests/restore.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index 341ba3aa0..80336b2a9 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1786,7 +1786,10 @@ def test_restore_backup_from_future(self): # @unittest.skip("skip") def test_restore_target_immediate_stream(self): - """more complex test_restore_chain()""" + """ + correct handling of immediate recovery target + for STREAM backups + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1836,7 +1839,10 @@ def test_restore_target_immediate_stream(self): # @unittest.skip("skip") def test_restore_target_immediate_archive(self): - """more complex test_restore_chain()""" + """ + correct handling of immediate recovery target + for ARCHIVE backups + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1887,7 +1893,10 @@ def test_restore_target_immediate_archive(self): # @unittest.skip("skip") def test_restore_target_latest_archive(self): - """more complex test_restore_chain()""" + """ + make sure that recovery_target 'latest' + is default recovery target + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1935,7 +1944,10 @@ def test_restore_target_latest_archive(self): # @unittest.skip("skip") def test_restore_target_new_options(self): - """more complex test_restore_chain()""" + """ + check that new --recovery-target-* + options are working correctly + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), From f393133d5a064737950041989c9c25e0c2aa0638 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 20:32:58 +0300 Subject: [PATCH 0544/2107] checkdb: minor changes to error messages in do_amcheck() --- src/backup.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 8a8d040c6..77a3aebed 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1133,13 +1133,13 @@ do_amcheck(void) break; } - /* TODO write better info message */ - if (db_skipped) - elog(WARNING, "Some databases were not checked"); - - if (!check_isok || db_skipped || interrupted) + /* Inform user about amcheck results */ + if (!check_isok || interrupted) elog(ERROR, "Checkdb --amcheck failed"); + if (db_skipped) + elog(ERROR, "Some databases were not amchecked"); + elog(INFO, "Checkdb --amcheck executed successfully"); /* We cannot state that all indexes are ok @@ -1148,7 +1148,8 @@ do_amcheck(void) if (check_isok && !interrupted && !db_skipped) elog(INFO, "Indexes are valid"); - /* close initial connection to pgdatabase */ + /* cleanup */ + PQclear(res_db); pgut_disconnect(backup_conn); } @@ -3519,11 +3520,11 @@ get_index_list(PGresult *res_db, int db_number, strcmp(PQgetvalue(res, 0, 2), "1") != 0) heapallindexed_is_supported = true; - elog(INFO, "Amchecking database '%s' using module '%s' version %s from schema '%s'", + elog(INFO, "Amchecking database '%s' using extension '%s' version %s from schema '%s'", dbname, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); if (!heapallindexed_is_supported && heapallindexed) - elog(WARNING, "Module '%s' verion %s in schema '%s' do not support 'heapallindexed' option", + elog(WARNING, "Extension '%s' verion %s in schema '%s' do not support 'heapallindexed' option", PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); /* From 771c01df4fdbb17df2ddc9844f6dd8df43609c5d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 20:37:12 +0300 Subject: [PATCH 0545/2107] checkdb: revert block from future changes in validate_one_page() --- src/data.c | 62 +++++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/src/data.c b/src/data.c index 9bb97fad2..07ef98a9c 100644 --- a/src/data.c +++ b/src/data.c @@ -1491,7 +1491,6 @@ validate_one_page(Page page, pgFile *file, XLogRecPtr lsn; bool page_header_is_sane = false; bool checksum_is_ok = false; - bool lsn_from_future = false; /* new level of paranoia */ if (page == NULL) @@ -1522,11 +1521,10 @@ validate_one_page(Page page, pgFile *file, } /* Page is zeroed. No sense to check header and checksum. */ - return PAGE_IS_FOUND_AND_VALID; + page_header_is_sane = false; } else { - /* We should give more information about what exactly is looking fishy */ if (PageGetPageSize(phdr) == BLCKSZ && PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && @@ -1535,14 +1533,7 @@ validate_one_page(Page page, pgFile *file, phdr->pd_upper <= phdr->pd_special && phdr->pd_special <= BLCKSZ && phdr->pd_special == MAXALIGN(phdr->pd_special)) - page_header_is_sane = true; - else - { - /* Page does not looking good */ - page_header_is_sane = false; - elog(WARNING, "Page is not looking healthy: %s, block %i", - file->path, blknum); - } + page_header_is_sane = true; } if (page_header_is_sane) @@ -1550,7 +1541,10 @@ validate_one_page(Page page, pgFile *file, /* Verify checksum */ if (checksum_version) { - /* Checksums are enabled, so check it. */ + /* + * If checksum is wrong, sleep a bit and then try again + * several times. If it didn't help, throw error + */ if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) == ((PageHeader) page)->pd_checksum) { @@ -1560,56 +1554,38 @@ validate_one_page(Page page, pgFile *file, { elog(WARNING, "File: %s blknum %u have wrong checksum", file->path, blknum); - return PAGE_IS_FOUND_AND_NOT_VALID; } } else { - /* Checksums are disabled, so check lsn. */ - if (stop_lsn > 0) - { - /* Get lsn from page header. Ensure that page is from our time. - * This is dangerous move, because we cannot be sure that - * lsn from page header is not a garbage. - */ - lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (lsn > stop_lsn) - { - elog(WARNING, "File: %s, block %u, checksum is not enabled. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - lsn_from_future = true; - } - } + /* Get lsn from page header. Ensure that page is from our time */ + lsn = PageXLogRecPtrGet(phdr->pd_lsn); + if (lsn > stop_lsn) + elog(WARNING, "File: %s, block %u, checksum is not enabled. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", + file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, + (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + else + return PAGE_IS_FOUND_AND_VALID; } - /* If checksum is ok, check that page is not from future */ - if (checksum_is_ok && stop_lsn > 0) + if (checksum_is_ok) { /* Get lsn from page header. Ensure that page is from our time */ lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) - { elog(WARNING, "File: %s, block %u, checksum is correct. " "Page is from future: pageLSN %X/%X stopLSN %X/%X", file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - lsn_from_future = true; - } + else + return PAGE_IS_FOUND_AND_VALID; } - - if (lsn_from_future) - return PAGE_IS_FOUND_AND_NOT_VALID; - else - return PAGE_IS_FOUND_AND_VALID; } - else - return PAGE_IS_FOUND_AND_NOT_VALID; + return PAGE_IS_FOUND_AND_NOT_VALID; } /* From 3fce25d740404397bc260403e54ee77705ef84ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Apr 2019 21:35:56 +0300 Subject: [PATCH 0546/2107] refactoring of validate_one_page() --- src/data.c | 110 ++++++++++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/src/data.c b/src/data.c index 07ef98a9c..8344dd0a7 100644 --- a/src/data.c +++ b/src/data.c @@ -1490,7 +1490,6 @@ validate_one_page(Page page, pgFile *file, PageHeader phdr; XLogRecPtr lsn; bool page_header_is_sane = false; - bool checksum_is_ok = false; /* new level of paranoia */ if (page == NULL) @@ -1520,72 +1519,71 @@ validate_one_page(Page page, pgFile *file, file->path, blknum); } - /* Page is zeroed. No sense to check header and checksum. */ - page_header_is_sane = false; + /* Page is zeroed. No sense in checking header and checksum. */ + return PAGE_IS_FOUND_AND_VALID; } - else + + /* Verify checksum */ + if (checksum_version) { - if (PageGetPageSize(phdr) == BLCKSZ && - PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && - (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && - phdr->pd_lower >= SizeOfPageHeaderData && - phdr->pd_lower <= phdr->pd_upper && - phdr->pd_upper <= phdr->pd_special && - phdr->pd_special <= BLCKSZ && - phdr->pd_special == MAXALIGN(phdr->pd_special)) + /* Checksums are enabled, so check them. */ + if (!(pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) + == ((PageHeader) page)->pd_checksum)) + { + elog(WARNING, "File: %s blknum %u have wrong checksum", + file->path, blknum); + return PAGE_IS_FOUND_AND_NOT_VALID; + } + } + + /* Check page for the sights of insanity. + * TODO: We should give more information about what exactly is looking "wrong" + */ + if (PageGetPageSize(phdr) == BLCKSZ && + PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && + phdr->pd_lower >= SizeOfPageHeaderData && + phdr->pd_lower <= phdr->pd_upper && + phdr->pd_upper <= phdr->pd_special && + phdr->pd_special <= BLCKSZ && + phdr->pd_special == MAXALIGN(phdr->pd_special)) + /* Page header is sane */ page_header_is_sane = true; + else + { + /* Page does not looking good */ + page_header_is_sane = false; + elog(WARNING, "Page is looking insane: %s, block %i", + file->path, blknum); } - if (page_header_is_sane) + /* Page is insane, no need going further */ + if (!page_header_is_sane) + return PAGE_IS_FOUND_AND_NOT_VALID; + + /* At this point page header is sane, if checksums are enabled - the`re ok. + * Check that page is not from future. + */ + if (stop_lsn > 0) { - /* Verify checksum */ - if (checksum_version) - { - /* - * If checksum is wrong, sleep a bit and then try again - * several times. If it didn't help, throw error - */ - if (pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) - == ((PageHeader) page)->pd_checksum) - { - checksum_is_ok = true; - } - else - { - elog(WARNING, "File: %s blknum %u have wrong checksum", - file->path, blknum); - } - } - else - { - /* Get lsn from page header. Ensure that page is from our time */ - lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (lsn > stop_lsn) - elog(WARNING, "File: %s, block %u, checksum is not enabled. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - else - return PAGE_IS_FOUND_AND_VALID; - } + /* Get lsn from page header. Ensure that page is from our time. + * This could be a dangerous move, because in case of disabled checksum we + * cannot be sure that lsn from page header is not a garbage. + */ + lsn = PageXLogRecPtrGet(phdr->pd_lsn); - if (checksum_is_ok) + if (lsn > stop_lsn) { - /* Get lsn from page header. Ensure that page is from our time */ - lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (lsn > stop_lsn) - elog(WARNING, "File: %s, block %u, checksum is correct. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - else - return PAGE_IS_FOUND_AND_VALID; + elog(WARNING, "File: %s, block %u, checksum is %s. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", + file->path, blknum, checksum_version ? "correct" : "not enabled", + (uint32) (lsn >> 32), (uint32) lsn, + (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + return PAGE_IS_FOUND_AND_NOT_VALID; } } - return PAGE_IS_FOUND_AND_NOT_VALID; + return PAGE_IS_FOUND_AND_VALID; } /* From 592df2e606cef30ac0924e938a829b63242fd010 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 16 Apr 2019 10:54:05 +0300 Subject: [PATCH 0547/2107] PBCKP-36: Fix help message for RESTORE and VALIDATE --- src/help.c | 22 +++++++++++----------- tests/expected/option_help.out | 6 +++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/help.c b/src/help.c index 623bc025f..75b5035d0 100644 --- a/src/help.c +++ b/src/help.c @@ -125,21 +125,21 @@ help_pg_probackup(void) printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); - printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); + printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); @@ -296,14 +296,14 @@ help_restore(void) printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); - printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica]\n")); printf(_(" [--no-validate]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); + printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -321,11 +321,6 @@ help_restore(void) printf(_(" whether we stop just after the recovery target\n")); printf(_(" --recovery-target-timeline=timeline\n")); printf(_(" recovering into a particular timeline\n")); - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); - printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); - printf(_(" --recovery-target=immediate|latest\n")); printf(_(" end recovery as soon as a consistent state is reached or as late as possible\n")); printf(_(" --recovery-target-name=target-name\n")); @@ -338,6 +333,11 @@ help_restore(void) printf(_(" to ease setting up a standby server\n")); printf(_(" --no-validate disable backup validation during restore\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_("\n Logging options:\n")); @@ -368,8 +368,8 @@ help_validate(void) printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); + printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 4a722bb7f..4da9667aa 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -57,21 +57,21 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-timeline=timeline] - [-T OLDDIR=NEWDIR] [--progress] - [--external-mapping=OLDDIR=NEWDIR] [--recovery-target=immediate|latest] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--no-validate] [--skip-block-validation] + [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] - [--recovery-target-name=target-name] [--recovery-target-timeline=timeline] + [--recovery-target-name=target-name] [--skip-block-validation] pg_probackup show -B backup-path From baa6aa8e350f2e912a280c391b76107cd9fb2dcb Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 16 Apr 2019 13:34:57 +0300 Subject: [PATCH 0548/2107] Fix help messages --- src/help.c | 17 +++++++++-------- tests/expected/option_help.out | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/help.c b/src/help.c index 3d52fdda0..455211a15 100644 --- a/src/help.c +++ b/src/help.c @@ -160,6 +160,7 @@ help_pg_probackup(void) printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); + printf(_(" [--external-dirs=external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -229,10 +230,10 @@ help_backup(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [--external-dirs=external-directory-path]\n\n")); + printf(_(" [-E external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -408,7 +409,7 @@ help_validate(void) printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--skip-block-validation]\n")); + printf(_(" [--skip-block-validation]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -564,7 +565,7 @@ help_set_config(void) printf(_(" [--master-port=port] [--master-user=user_name]\n")); printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [-E external-dirs=external-directory-path]\n\n")); + printf(_(" [-E external-directory-path]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -639,10 +640,10 @@ help_add_instance(void) { printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); - printf(_(" --remote-proto --remote-host\n")); - printf(_(" --remote-port --remote-path --remote-user\n")); - printf(_(" --ssh-options\n")); - printf(_(" -E external-dirs=external-directory-path\n\n")); + printf(_(" [-E external-directory-path]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 97d1015c3..76f2afa7e 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -52,6 +52,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--replica-timeout=timeout] [--no-validate] [--skip-block-validation] [--external-dirs=external-directory-path] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [-j num-threads] @@ -65,6 +68,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] @@ -86,6 +92,10 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name + [--external-dirs=external-directory-path] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] pg_probackup del-instance -B backup-path --instance=instance_name @@ -97,10 +107,16 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] [--overwrite] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] pg_probackup archive-get -B backup-path --instance=instance_name --wal-file-path=wal-file-path --wal-file-name=wal-file-name + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] Read the website for details. Report bugs to . From 5b4685eb3ab662a00102fcac45bb0d6667e7dd56 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Apr 2019 02:13:56 +0300 Subject: [PATCH 0549/2107] tests: fix missing _ptrack fork for 0 sized relations --- tests/helpers/ptrack_helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index fe730354d..55eafc2c2 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -431,7 +431,12 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): header_size = 24 ptrack_bits_for_fork = [] + # TODO: use macro instead of hard coded 8KB page_body_size = 8192-header_size + # Check that if main fork file size is 0, it`s ok + # to not having a _ptrack fork + if os.path.getsize(file) == 0: + return ptrack_bits_for_fork byte_size = os.path.getsize(file + '_ptrack') npages = byte_size/8192 if byte_size % 8192 != 0: From 2efce08756839ae1d5d009437455e04891b8244a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Apr 2019 14:37:39 +0300 Subject: [PATCH 0550/2107] tests: fix some ptrack tests --- tests/ptrack.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index b79528171..e192b9070 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -2554,7 +2554,7 @@ def test_ptrack_truncate(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_ptrack_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( @@ -2563,9 +2563,9 @@ def test_ptrack_truncate_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'checkpoint_timeout': '30'}) + 'max_wal_size': '16MB', + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2620,6 +2620,7 @@ def test_ptrack_truncate_replica(self): backup_dir, 'replica', replica, options=[ '-j10', + '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) @@ -2656,8 +2657,7 @@ def test_ptrack_truncate_replica(self): success = False self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + success, 'Ptrack has failed to register changes in data files') # Clean after yourself self.del_test_dir(module_name, fname) @@ -3493,8 +3493,7 @@ def test_ptrack_vacuum_truncate_replica(self): success = False self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + success, 'Ptrack has failed to register changes in data files') # Clean after yourself self.del_test_dir(module_name, fname) From cb9134d067b9616bdac4c126f8eff2b5d5d1b172 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Apr 2019 01:51:38 +0300 Subject: [PATCH 0551/2107] validate: interrupt more frequently during file validation --- src/data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/data.c b/src/data.c index 995e27a8d..b70c4dd97 100644 --- a/src/data.c +++ b/src/data.c @@ -1592,6 +1592,9 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, BackupPageHeader header; BlockNumber blknum = 0; + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during data file validation"); + /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); if (read_len != sizeof(header)) From e7c3031f7f7266c9357a19f075c525f30fc00f18 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Apr 2019 16:10:31 +0300 Subject: [PATCH 0552/2107] Issue 56: added logging.LogTest.test_truncated_rotation_file --- tests/logging.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/logging.py b/tests/logging.py index 210a6f4f6..54462547f 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -72,3 +72,50 @@ def test_log_filename_strftime(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_truncated_rotation_file(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + # self.assertTrue(os.path.isfile(rotation_file_path)) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE', + '--log-filename=pg_probackup.log']) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # truncate .rotation file + with open(rotation_file_path, "rb+", 0) as f: + f.truncate() + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE', + '--log-filename=pg_probackup.log']) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From c00bd3cd77243e137891397399888e28b3a71c91 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 18 Apr 2019 16:12:57 +0300 Subject: [PATCH 0553/2107] code and comments cleanup --- src/backup.c | 27 ++++++++++++++++++++------- src/data.c | 21 +++++---------------- src/help.c | 3 +-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/backup.c b/src/backup.c index 92e77c19a..05793916e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1040,6 +1040,17 @@ do_block_validation(void) elog(ERROR, "Checkdb failed"); } +/* + * Entry point of checkdb --amcheck. + * + * Connect to all databases in the cluster + * and get list of persistent indexes, + * then run parallel threads to perform bt_index_check() + * for all indexes from the list. + * + * If amcheck extension is not installed in the database, + * skip this database and report it via warning message. + */ static void do_amcheck(void) { @@ -2680,14 +2691,18 @@ check_indexes(void *arg) /* remember that we have a failed check */ if (!amcheck_one_index(arguments, ind)) - arguments->ret = 2; + arguments->ret = 2; /* corruption found */ } /* Close connection */ if (arguments->backup_conn) pgut_disconnect(arguments->backup_conn); - /* TODO where should we set arguments->ret to 1? */ + /* Ret values: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ if (arguments->ret == 1) arguments->ret = 0; @@ -3531,18 +3546,18 @@ get_index_list(PGresult *res_db, int db_number, /* * In order to avoid duplicates, select global indexes - * (tablespace pg_global with oid 1664) only once + * (tablespace pg_global with oid 1664) only once. + * + * select only persistent btree indexes. */ if (first_db_with_amcheck) { - /* select only valid btree and persistent indexes */ res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " "FROM pg_index idx " "JOIN pg_class cls ON cls.oid=idx.indexrelid " "JOIN pg_am am ON am.oid=cls.relam " "WHERE am.amname='btree' AND cls.relpersistence != 't'", - //"AND idx.indisready AND idx.indisvalid", 0, NULL); } else @@ -3554,7 +3569,6 @@ get_index_list(PGresult *res_db, int db_number, "JOIN pg_am am ON am.oid=cls.relam " "JOIN pg_tablespace tbl ON tbl.oid=cls.reltablespace " "WHERE am.amname='btree' AND cls.relpersistence != 't' " - //"AND idx.indisready AND idx.indisvalid " "AND tbl.spcname != 'pg_global'",0, NULL); } @@ -3579,7 +3593,6 @@ get_index_list(PGresult *res_db, int db_number, if (index_list == NULL) index_list = parray_new(); -// elog(WARNING, "add to index_list index '%s' dbname '%s'",ind->name, ind->dbname); parray_append(index_list, ind); } diff --git a/src/data.c b/src/data.c index f851a5aab..f6572680f 100644 --- a/src/data.c +++ b/src/data.c @@ -1477,7 +1477,6 @@ validate_one_page(Page page, pgFile *file, { PageHeader phdr; XLogRecPtr lsn; - bool page_header_is_sane = false; /* new level of paranoia */ if (page == NULL) @@ -1527,37 +1526,27 @@ validate_one_page(Page page, pgFile *file, /* Check page for the sights of insanity. * TODO: We should give more information about what exactly is looking "wrong" */ - if (PageGetPageSize(phdr) == BLCKSZ && + if (!(PageGetPageSize(phdr) == BLCKSZ && PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && phdr->pd_lower <= phdr->pd_upper && phdr->pd_upper <= phdr->pd_special && phdr->pd_special <= BLCKSZ && - phdr->pd_special == MAXALIGN(phdr->pd_special)) - /* Page header is sane */ - page_header_is_sane = true; - else + phdr->pd_special == MAXALIGN(phdr->pd_special))) { /* Page does not looking good */ - page_header_is_sane = false; - elog(WARNING, "Page is looking insane: %s, block %i", + elog(WARNING, "Page header is looking insane: %s, block %i", file->path, blknum); - } - - /* Page is insane, no need going further */ - if (!page_header_is_sane) return PAGE_IS_FOUND_AND_NOT_VALID; + } /* At this point page header is sane, if checksums are enabled - the`re ok. * Check that page is not from future. */ if (stop_lsn > 0) { - /* Get lsn from page header. Ensure that page is from our time. - * This could be a dangerous move, because in case of disabled checksum we - * cannot be sure that lsn from page header is not a garbage. - */ + /* Get lsn from page header. Ensure that page is from our time. */ lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (lsn > stop_lsn) diff --git a/src/help.c b/src/help.c index a3aba9f9f..22a579ce3 100644 --- a/src/help.c +++ b/src/help.c @@ -153,7 +153,7 @@ help_pg_probackup(void) printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed] [--work-mem]\n")); + printf(_(" [--heapallindexed]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); @@ -477,7 +477,6 @@ help_checkdb(void) printf(_(" --amcheck in addition to file-level block checking\n")); printf(_(" check btree indexes via function 'bt_index_check()'\n")); printf(_(" using 'amcheck' or 'amcheck_next' extensions\n")); - printf(_(" --parent use 'bt_index_parent_check()' instead of 'bt_index_check()'\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); printf(_(" can be used only with '--amcheck' option\n")); From 9a785d6fa0a9c8afe64ceb3eb986efeefa4df039 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 18 Apr 2019 17:20:22 +0300 Subject: [PATCH 0554/2107] fix compiler warning. TODO: fix checkdb test --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index f6572680f..b41209dd3 100644 --- a/src/data.c +++ b/src/data.c @@ -1576,7 +1576,7 @@ check_data_file(backup_files_arg* arguments, FILE *in; BlockNumber blknum = 0; BlockNumber nblocks = 0; - int n_blocks_skipped = 0; + BlockNumber n_blocks_skipped = 0; int page_state; char curr_page[BLCKSZ]; bool is_valid = true; From cf20e9f3ba32284f6feedfc92861e1d713fb0a38 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Apr 2019 19:23:44 +0300 Subject: [PATCH 0555/2107] checkdb: minor fixes --- src/backup.c | 4 ++-- src/pg_probackup.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 05793916e..52ea15268 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1202,7 +1202,7 @@ static void pgdata_basic_setup(bool amcheck_only) { /* PGDATA is always required unless running checkdb in amcheck only mode */ - if (instance_config.pgdata == NULL && !amcheck_only) + if (!instance_config.pgdata && !amcheck_only) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); @@ -1234,7 +1234,7 @@ pgdata_basic_setup(bool amcheck_only) * belogns to the same instance. */ /* TODO fix it for remote backup */ - if (!IsReplicationProtocol()) + if (!IsReplicationProtocol() && !amcheck_only) check_system_identifiers(); if (current.checksum_version) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1f16e46e2..a1e9652ee 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -96,7 +96,7 @@ bool skip_block_validation = false; bool skip_external_dirs = false; /* checkdb options */ -bool do_amcheck = false; +bool need_amcheck = false; bool heapallindexed = false; bool amcheck_parent = false; @@ -171,9 +171,9 @@ static ConfigOption cmd_options[] = { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, /* checkdb options */ - { 'b', 157, "amcheck", &do_amcheck, SOURCE_CMD_STRICT }, - { 'b', 158, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, - { 'b', 159, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, + { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, + { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, + { 'b', 197, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -409,7 +409,7 @@ main(int argc, char *argv[]) } /* Ensure that backup_path is an absolute path */ - if (!is_absolute_path(backup_path)) + if (backup_path && !is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); @@ -644,7 +644,7 @@ main(int argc, char *argv[]) do_set_config(false); break; case CHECKDB_CMD: - do_checkdb(do_amcheck); + do_checkdb(need_amcheck); break; case NO_CMD: /* Should not happen */ From d5b074728823e2b5f64ff92833d0956f5b79dfb1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Apr 2019 19:40:35 +0300 Subject: [PATCH 0556/2107] checkdb: fix disconnection after successfull checkdb execution --- src/backup.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 52ea15268..132ad228a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1164,7 +1164,6 @@ do_amcheck(void) /* cleanup */ PQclear(res_db); - pgut_disconnect(backup_conn); } /* Entry point of pg_probackup CHECKDB subcommand. */ From 56a909da0c5ff32b389fb734d31363df4678a3bc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 01:50:45 +0300 Subject: [PATCH 0557/2107] checkdb: fix index query and logging error condition --- src/backup.c | 16 +++++++++------- src/pg_probackup.c | 11 ++++++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index 132ad228a..6acbda709 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3554,8 +3554,8 @@ get_index_list(PGresult *res_db, int db_number, res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " "FROM pg_index idx " - "JOIN pg_class cls ON cls.oid=idx.indexrelid " - "JOIN pg_am am ON am.oid=cls.relam " + "JOIN pg_class cls ON idx.indexrelid=cls.oid " + "JOIN pg_am am ON cls.relam=am.oid " "WHERE am.amname='btree' AND cls.relpersistence != 't'", 0, NULL); } @@ -3564,11 +3564,13 @@ get_index_list(PGresult *res_db, int db_number, res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " "FROM pg_index idx " - "JOIN pg_class cls ON cls.oid=idx.indexrelid " - "JOIN pg_am am ON am.oid=cls.relam " - "JOIN pg_tablespace tbl ON tbl.oid=cls.reltablespace " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " - "AND tbl.spcname != 'pg_global'",0, NULL); + "JOIN pg_class cls ON idx.indexrelid=cls.oid " + "JOIN pg_am am ON cls.relam=am.oid " + "LEFT JOIN pg_tablespace tbl " + "ON cls.reltablespace=tbl.oid " + "AND tbl.spcname <> 'pg_global' " + "WHERE am.amname='btree' AND cls.relpersistence != 't'", + 0, NULL); } /* add info needed to check indexes into index_list */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a1e9652ee..dc40fe1cf 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -480,9 +480,14 @@ main(int argc, char *argv[]) instance_config.pgdata == NULL) elog(ERROR, "required parameter not specified: --instance"); - if (backup_subcmd == CHECKDB_CMD - && (instance_config.logger.log_level_file != LOG_OFF) - && (instance_config.logger.log_directory == NULL)) + /* Usually checkdb for file logging requires log_directory + * to be specified explicitly, but if backup_dir and instance name are provided, + * checkdb can use the tusual default values or values from config + */ + if (backup_subcmd == CHECKDB_CMD && + (instance_config.logger.log_level_file != LOG_OFF && + instance_config.logger.log_directory == NULL) && + (!instance_config.pgdata || !instance_name)) elog(ERROR, "Cannot save checkdb logs to a file. " "You must specify --log-directory option when running checkdb with " "--log-level-file option enabled."); From d27ab77920d5ed9497031c288d3b5616bcf017e4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 01:52:41 +0300 Subject: [PATCH 0558/2107] tests: checkdb tests added --- tests/checkdb.py | 377 ++++++++++++++++++++++++++++++-- tests/helpers/ptrack_helpers.py | 19 +- 2 files changed, 371 insertions(+), 25 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index c4967bbae..876126338 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -14,10 +14,9 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): - @unittest.skip("skip") - def checkdb_index_loss(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" + # @unittest.skip("skip") + def checkdb_amcheck_only_sanity(self): + """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -27,39 +26,329 @@ def checkdb_index_loss(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() node.safe_psql( "postgres", "create table t_heap as select i" - " as id from generate_series(0,100) i" - ) + " as id from generate_series(0,100) i") node.safe_psql( "postgres", - "create index on t_heap(id)" - ) + "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create extension amcheck") + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + # simple sanity + try: + self.checkdb_node( + options=['--skip-block-validation']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because --amcheck option is missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Option '--skip-block-validation' must be " + "used with '--amcheck' option", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # simple sanity + output = self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + + self.assertIn( + 'INFO: Checkdb --amcheck executed successfully', + output) + self.assertIn( + 'INFO: Indexes are valid', + output) + + # logging to file sanity + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres','-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because log_directory missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Cannot save checkdb logs to a file. " + "You must specify --log-directory option when " + "running checkdb with --log-level-file option enabled", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # If backup_dir provided, then instance name must be + # provided too + try: + self.checkdb_node( + backup_dir, + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because log_directory missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: required parameter not specified: --instance", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # checkdb can use default or set in config values, + # if backup_dir and instance name are provided + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres', '-p', str(node.port)]) + + # check that file present and full of messages + os.path.isfile(log_file_path) + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'INFO: Checkdb --amcheck executed successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + os.unlink(log_file_path) + + # log-level-file and log-directory are provided + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) + + # check that file present and full of messages + os.path.isfile(log_file_path) + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'INFO: Checkdb --amcheck executed successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + os.unlink(log_file_path) gdb = self.checkdb_node( - 'node', data_dir=node.data_dir, - gdb=True, options=['-d', 'postgres', '--amcheck', '-p', str(node.port)]) + gdb=True, + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) gdb.set_breakpoint('amcheck_one_index') gdb.run_until_break() node.safe_psql( "postgres", - "drop table t_heap" - ) + "drop table t_heap") + + gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() + # check that message about missing index is present + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'ERROR: Checkdb --amcheck failed', + log_file_content) + self.assertIn( + "WARNING: Thread [1]. Amcheck failed for index: 'public.t_heap_id_idx':", + log_file_content) + self.assertIn( + 'ERROR: could not open relation with OID', + log_file_content) + # Clean after yourself self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_checkdb_block_validation(self): + def checkdb_amcheck_only_sanity_1(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # init pgbench in two databases and corrupt both indexes + node.safe_psql( + "postgres", + "create database db1") + + node.safe_psql( + "db1", + "create extension amcheck") + + node.safe_psql( + "postgres", + "create database db2") + + node.safe_psql( + "db2", + "create extension amcheck") + + node.pgbench_init( + scale=5, dbname='db1') + + node.pgbench_init( + scale=5, dbname='db2') + + node.safe_psql( + "db2", + "alter index pgbench_accounts_pkey rename to some_index") + + index_path_1 = os.path.join( + node.data_dir, + node.safe_psql( + "db1", + "select pg_relation_filepath('pgbench_accounts_pkey')").rstrip()) + + index_path_2 = os.path.join( + node.data_dir, + node.safe_psql( + "db2", + "select pg_relation_filepath('some_index')").rstrip()) + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because some db was not amchecked" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Some databases were not amchecked", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.stop() + + # Let`s do index corruption + with open(index_path_1, "rb+", 0) as f: + #f.truncate(os.path.getsize(index_path_1) - 4096) + #f.flush() + #f.close + + f.seek(42000) + f.write(b"blablahblahs") + f.flush() + f.close + + with open(index_path_2, "rb+", 0) as f: + #f.truncate(os.path.getsize(index_path_2) - 4096) + #f.flush() + #f.close + + f.seek(42000) + f.write(b"blablahblahs") + f.flush() + f.close + + node.slow_start() + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because some db was not amchecked" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Checkdb --amcheck failed", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # corruption of both indexes in db1 and db2 must be detected + # also the that amcheck is not installed in 'postgres' + # musted be logged + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + "WARNING: Thread [1]. Amcheck failed for index: 'public.pgbench_accounts_pkey':", + log_file_content) + + self.assertIn( + "WARNING: Thread [1]. Amcheck failed for index: 'public.some_index':", + log_file_content) + + self.assertIn( + "ERROR: Checkdb --amcheck failed", + log_file_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_checkdb_block_validation_sanity(self): """make node, corrupt some pages, check that checkdb failed""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -73,10 +362,6 @@ def test_checkdb_block_validation(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - #self.backup_node( - # backup_dir, 'node', node, - # backup_type="full", options=["-j", "4", "--stream"]) - node.safe_psql( "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, " @@ -90,20 +375,68 @@ def test_checkdb_block_validation(self): "postgres", "select pg_relation_filepath('t_heap')").rstrip() - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + # sanity + try: + self.checkdb_node() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pgdata must be specified\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: required parameter not specified: PGDATA (-D, --pgdata)", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.checkdb_node( + data_dir=node.data_dir, + options=['-d', 'postgres', '-p', str(node.port)]) + + self.checkdb_node( + backup_dir, 'node', + options=['-d', 'postgres', '-p', str(node.port)]) + + heap_full_path = os.path.join(node.data_dir, heap_path) + + with open(heap_full_path, "rb+", 0) as f: f.seek(9000) f.write(b"bla") f.flush() f.close - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + with open(heap_full_path, "rb+", 0) as f: f.seek(42000) f.write(b"bla") f.flush() f.close - print(self.checkdb_node('node', backup_dir, - data_dir=node.data_dir, options=['--block-validation'])) + try: + self.checkdb_node( + backup_dir, 'node', + options=['-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of data corruption\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Checkdb failed", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: CORRUPTION in file {0}, block 1".format(heap_full_path), + e.message) + + self.assertIn( + "WARNING: CORRUPTION in file {0}, block 5".format(heap_full_path), + e.message) # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 128ac8541..f2ae21e20 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -733,7 +733,7 @@ def backup_node( return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def checkdb_node( - self, instance, backup_dir=False, data_dir=False, + self, backup_dir=False, instance=False, data_dir=False, options=[], async=False, gdb=False, old_binary=False ): @@ -742,11 +742,12 @@ def checkdb_node( if backup_dir: cmd_list += ["-B", backup_dir] + if instance: + cmd_list += ["--instance={0}".format(instance)] + if data_dir: cmd_list += ["-D", data_dir] - cmd_list += ["--instance={0}".format(instance)] - return self.run_pb(cmd_list + options, async, gdb, old_binary) def merge_backup( @@ -1458,6 +1459,18 @@ def set_breakpoint(self, location): 'Failed to set breakpoint.\n Output:\n {0}'.format(result) ) + def remove_all_breakpoints(self): + + result = self._execute('delete') + for line in result: + + if line.startswith('^done'): + return + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + def run_until_break(self): result = self._execute('run', False) for line in result: From e04abf12f5599457d1d270c1ec5f5cb5252980d2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 10:59:52 +0300 Subject: [PATCH 0559/2107] tests: minor changes to checkdb module --- tests/checkdb.py | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 876126338..80dc3352d 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -220,27 +220,15 @@ def checkdb_amcheck_only_sanity_1(self): node.slow_start() # init pgbench in two databases and corrupt both indexes - node.safe_psql( - "postgres", - "create database db1") + node.safe_psql("postgres", "create database db1") - node.safe_psql( - "db1", - "create extension amcheck") + node.safe_psql("db1", "create extension amcheck") - node.safe_psql( - "postgres", - "create database db2") + node.safe_psql("postgres", "create database db2") + node.safe_psql("db2", "create extension amcheck") - node.safe_psql( - "db2", - "create extension amcheck") - - node.pgbench_init( - scale=5, dbname='db1') - - node.pgbench_init( - scale=5, dbname='db2') + node.pgbench_init(scale=5, dbname='db1') + node.pgbench_init(scale=5, dbname='db2') node.safe_psql( "db2", @@ -281,20 +269,12 @@ def checkdb_amcheck_only_sanity_1(self): # Let`s do index corruption with open(index_path_1, "rb+", 0) as f: - #f.truncate(os.path.getsize(index_path_1) - 4096) - #f.flush() - #f.close - f.seek(42000) f.write(b"blablahblahs") f.flush() f.close with open(index_path_2, "rb+", 0) as f: - #f.truncate(os.path.getsize(index_path_2) - 4096) - #f.flush() - #f.close - f.seek(42000) f.write(b"blablahblahs") f.flush() From aae69835e2b9cb35c470c5f86e4fff1cde9fab9b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 11:11:48 +0300 Subject: [PATCH 0560/2107] help: minor fixes for checkdb --- src/help.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/help.c b/src/help.c index 22a579ce3..d26c4f8be 100644 --- a/src/help.c +++ b/src/help.c @@ -208,14 +208,14 @@ help_pg_probackup(void) static void help_init(void) { - printf(_("%s init -B backup-path\n\n"), PROGRAM_NAME); + printf(_("\n%s init -B backup-path\n\n"), PROGRAM_NAME); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); } static void help_backup(void) { - printf(_("%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); printf(_(" [--backup-pg-log] [-j num-threads]\n")); printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); @@ -328,7 +328,7 @@ help_backup(void) static void help_restore(void) { - printf(_("%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); @@ -412,7 +412,7 @@ help_restore(void) static void help_validate(void) { - printf(_("%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); @@ -461,10 +461,10 @@ help_validate(void) static void help_checkdb(void) { - printf(_("%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed] [--work-mem=work_mem_size]\n")); + printf(_(" [--heapallindexed]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -513,7 +513,7 @@ help_checkdb(void) static void help_show(void) { - printf(_("%s show -B backup-path\n"), PROGRAM_NAME); + printf(_("\n%s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); printf(_(" [--format=format]\n\n")); @@ -526,7 +526,7 @@ help_show(void) static void help_delete(void) { - printf(_("%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-i backup-id | --expired | --merge-expired] [--wal]\n")); printf(_(" [-j num-threads] [--dry-run]\n\n")); @@ -566,7 +566,7 @@ help_delete(void) static void help_merge(void) { - printf(_("%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id [-j num-threads] [--progress]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); @@ -608,7 +608,7 @@ help_merge(void) static void help_set_config(void) { - printf(_("%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -687,7 +687,7 @@ help_set_config(void) static void help_show_config(void) { - printf(_("%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); @@ -698,7 +698,7 @@ help_show_config(void) static void help_add_instance(void) { - printf(_("%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); + printf(_("\n%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); printf(_(" [-E external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -725,7 +725,7 @@ help_add_instance(void) static void help_del_instance(void) { - printf(_("%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + printf(_("\n%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance to delete\n")); @@ -734,7 +734,7 @@ help_del_instance(void) static void help_archive_push(void) { - printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--compress]\n")); @@ -762,7 +762,7 @@ help_archive_push(void) static void help_archive_get(void) { - printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--remote-proto] [--remote-host]\n")); From ec2e526ad8996cc208c1d7f7e9302b09a685f189 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 11:12:19 +0300 Subject: [PATCH 0561/2107] tests: checkdb minor fixes --- tests/checkdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 80dc3352d..6366a39a4 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -219,14 +219,14 @@ def checkdb_amcheck_only_sanity_1(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - # init pgbench in two databases and corrupt both indexes + # create two databases node.safe_psql("postgres", "create database db1") - node.safe_psql("db1", "create extension amcheck") node.safe_psql("postgres", "create database db2") node.safe_psql("db2", "create extension amcheck") + # init pgbench in two databases and corrupt both indexes node.pgbench_init(scale=5, dbname='db1') node.pgbench_init(scale=5, dbname='db2') From 6aea5520425dae3b8ccb68c0771e088a850624b5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 11:17:20 +0300 Subject: [PATCH 0562/2107] tests: more minor fixes --- tests/checkdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 6366a39a4..5a133cf93 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -15,7 +15,7 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") - def checkdb_amcheck_only_sanity(self): + def test_checkdb_amcheck_only_sanity(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -207,7 +207,7 @@ def checkdb_amcheck_only_sanity(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def checkdb_amcheck_only_sanity_1(self): + def test_checkdb_amcheck_only_sanity_1(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From cd6b175301e5923c1c6a7f855208deb8a5a61170 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 11:24:27 +0300 Subject: [PATCH 0563/2107] fix unnessesary tabulations --- src/utils/logger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index acd9c4762..8a7848fa3 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -134,7 +134,7 @@ exit_if_necessary(int elevel) if (remote_agent) sleep(1); /* Let parent receive sent messages */ - + /* If this is not the main thread then don't call exit() */ if (main_tid != pthread_self()) { From 45e05ba1261b2b5786d6ff2f9e5f696765719bce Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 11:36:04 +0300 Subject: [PATCH 0564/2107] bugfix: do not check page layout version, it may give false-positive result --- src/data.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index f9cb66307..04a896efe 100644 --- a/src/data.c +++ b/src/data.c @@ -139,7 +139,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) /* First check if page header is valid (it seems to be fast enough check) */ if (!(PageGetPageSize(phdr) == BLCKSZ && - PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && phdr->pd_lower <= phdr->pd_upper && @@ -178,7 +178,7 @@ parse_page(Page page, XLogRecPtr *lsn) *lsn = PageXLogRecPtrGet(phdr->pd_lsn); if (PageGetPageSize(phdr) == BLCKSZ && - PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && phdr->pd_lower <= phdr->pd_upper && @@ -1527,7 +1527,7 @@ validate_one_page(Page page, pgFile *file, * TODO: We should give more information about what exactly is looking "wrong" */ if (!(PageGetPageSize(phdr) == BLCKSZ && - PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && + // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && phdr->pd_lower <= phdr->pd_upper && From e0d6c6bfc056c851385e51950d8cf5c114dba8a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 15:55:03 +0300 Subject: [PATCH 0565/2107] tests: added test_unlink_rotation_file --- tests/helpers/ptrack_helpers.py | 8 ++-- tests/logging.py | 81 +++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f2ae21e20..e7d89156f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -579,7 +579,7 @@ def check_ptrack_clean(self, idx_dict, size): ) ) - def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False): + def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True): if not self.probackup_old_path and old_binary: print('PGPROBACKUPBIN_OLD is not set') exit(1) @@ -608,7 +608,7 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False): stderr=subprocess.STDOUT, env=self.test_env ).decode('utf-8') - if command[0] == 'backup': + if command[0] == 'backup' and return_id: # return backup ID for line in self.output.splitlines(): if 'INFO: Backup' and 'completed' in line: @@ -700,7 +700,7 @@ def clean_pb(self, backup_dir): def backup_node( self, backup_dir, instance, node, data_dir=False, backup_type='full', options=[], asynchronous=False, gdb=False, - old_binary=False + old_binary=False, return_id=True ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -730,7 +730,7 @@ def backup_node( if backup_type: cmd_list += ['-b', backup_type] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id) def checkdb_node( self, backup_dir=False, instance=False, data_dir=False, diff --git a/tests/logging.py b/tests/logging.py index 54462547f..0333d9517 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -73,7 +73,7 @@ def test_log_filename_strftime(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_truncated_rotation_file(self): + def test_truncate_rotation_file(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -97,8 +97,7 @@ def test_truncated_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE', - '--log-filename=pg_probackup.log']) + '--log-level-file=VERBOSE']) self.assertTrue(os.path.isfile(rotation_file_path)) @@ -108,14 +107,86 @@ def test_truncated_rotation_file(self): f.flush() f.close + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertIn( + 'WARNING: cannot read creation timestamp from rotation file', + output) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertNotIn( + 'WARNING:', + output) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_unlink_rotation_file(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + self.backup_node( backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE', - '--log-filename=pg_probackup.log']) + '--log-level-file=VERBOSE']) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # unlink .rotation file + os.unlink(rotation_file_path) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertIn( + 'WARNING: missing rotation file:', + output) self.assertTrue(os.path.isfile(rotation_file_path)) + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertNotIn( + 'WARNING:', + output) + # Clean after yourself self.del_test_dir(module_name, fname) From ac41c53f45792124bbed460e8c3f54f4fb2ad2ec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 16:25:30 +0300 Subject: [PATCH 0566/2107] tests: added logging.LogTest.test_garbage_in_rotation_file --- tests/logging.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/logging.py b/tests/logging.py index 0333d9517..1bbe3bc78 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -190,3 +190,63 @@ def test_unlink_rotation_file(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_garbage_in_rotation_file(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE']) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # mangle .rotation file + with open(rotation_file_path, "wtb", 0) as f: + f.write(b"blah") + f.flush() + f.close + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertIn( + 'WARNING: rotation file "{0}" has wrong ' + 'creation timestamp'.format(rotation_file_path), + output) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertNotIn( + 'WARNING:', + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) From ec2d74dbbe8a646274faa3b7c98bdb21eb28afef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 17:05:14 +0300 Subject: [PATCH 0567/2107] tests: added backup.BackupTest.test_pg_11_adjusted_wal_segment_size --- tests/backup_test.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index ae42ec5d9..900247299 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -996,3 +996,94 @@ def test_temp_slot_for_stream_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pg_11_adjusted_wal_segment_size(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--wal-segsize=64'], + pg_options={ + 'min_wal_size': '128MB', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.get_version(node) < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + + node.pgbench_init(scale=5) + + # FULL STREAM backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # PAGE STREAM backup + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # DELTA STREAM backup + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # FULL ARCHIVE backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # PAGE ARCHIVE backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # DELTA ARCHIVE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + pgdata = self.pgdata_content(node.data_dir) + + # delete + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--expired', + '--delete-wal', + '--retention-redundancy=1']) + + print(output) + + # validate + self.validate_pb(backup_dir) + + # merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # restore + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From fc8453b6c7f3376c51032534398b7c6fb0c29245 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 17:11:58 +0300 Subject: [PATCH 0568/2107] Issue 56: be more lenient to .rotation file been empty or containing garbage: issue WARNING instead of ERROR and rotate file immediately. Also issue WARNING if .rotation file is missing, previously such condition was ignored --- src/utils/logger.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 8a7848fa3..04f406024 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -565,11 +565,18 @@ open_logfile(FILE **file, const char *filename_format) { struct stat control_st; - if (stat(control, &control_st) == -1) + if (stat(control, &control_st) < 0) { if (errno != ENOENT) elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", control, strerror(errno)); + else + { + /* file not found, force rotation */ + elog_stderr(WARNING, "missing rotation file: \"%s\"", + control); + rotation_requested = true; + } } else { @@ -585,18 +592,28 @@ open_logfile(FILE **file, const char *filename_format) time_t creation_time; if (!parse_int64(buf, (int64 *) &creation_time, 0)) - elog_stderr(ERROR, "rotation file \"%s\" has wrong " + { + /* Inability to parse value from .rotation file is + * concerning but not a critical error + */ + elog_stderr(WARNING, "rotation file \"%s\" has wrong " "creation timestamp \"%s\"", control, buf); - /* Parsed creation time */ - - rotation_requested = (cur_time - creation_time) > - /* convert to seconds from milliseconds */ - logger_config.log_rotation_age / 1000; + rotation_requested = true; + } + else + /* Parsed creation time */ + rotation_requested = (cur_time - creation_time) > + /* convert to seconds from milliseconds */ + logger_config.log_rotation_age / 1000; } else - elog_stderr(ERROR, "cannot read creation timestamp from " + { + /* truncated .rotation file is not a critical error */ + elog_stderr(WARNING, "cannot read creation timestamp from " "rotation file \"%s\"", control); + rotation_requested = true; + } fclose(control_file); } From c93e732864534090e83d255a4c084a57b9ad5777 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 18:40:57 +0300 Subject: [PATCH 0569/2107] Issue 56: do not force log file rotation if '.rotation' file is missing or mangled, just recreate '.rotation file' --- src/utils/logger.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 04f406024..784f262f4 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -535,7 +535,8 @@ open_logfile(FILE **file, const char *filename_format) FILE *control_file; time_t cur_time = time(NULL); bool rotation_requested = false, - logfile_exists = false; + logfile_exists = false, + rotation_file_exists = false; filename = logfile_getname(filename_format, cur_time); @@ -567,19 +568,17 @@ open_logfile(FILE **file, const char *filename_format) if (stat(control, &control_st) < 0) { - if (errno != ENOENT) - elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", - control, strerror(errno)); - else - { - /* file not found, force rotation */ + if (errno == ENOENT) + /* '.rotation' file is not found, force its recreation */ elog_stderr(WARNING, "missing rotation file: \"%s\"", control); - rotation_requested = true; - } + else + elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", + control, strerror(errno)); } else { + /* rotation file exists */ char buf[1024]; control_file = fopen(control, "r"); @@ -587,6 +586,8 @@ open_logfile(FILE **file, const char *filename_format) elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", control, strerror(errno)); + rotation_file_exists = true; + if (fgets(buf, lengthof(buf), control_file)) { time_t creation_time; @@ -599,7 +600,7 @@ open_logfile(FILE **file, const char *filename_format) elog_stderr(WARNING, "rotation file \"%s\" has wrong " "creation timestamp \"%s\"", control, buf); - rotation_requested = true; + rotation_file_exists = false; } else /* Parsed creation time */ @@ -612,7 +613,7 @@ open_logfile(FILE **file, const char *filename_format) /* truncated .rotation file is not a critical error */ elog_stderr(WARNING, "cannot read creation timestamp from " "rotation file \"%s\"", control); - rotation_requested = true; + rotation_file_exists = false; } fclose(control_file); @@ -634,7 +635,7 @@ open_logfile(FILE **file, const char *filename_format) pfree(filename); /* Rewrite rotation control file */ - if (rotation_requested || !logfile_exists) + if (rotation_requested || !logfile_exists || !rotation_file_exists) { time_t timestamp = time(NULL); From ea50f4e9cf22183963c1914a0466b7fb42bade28 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 18:42:07 +0300 Subject: [PATCH 0570/2107] tests: check that log file do not gets rotated if '.rotation' file is missing or mangled --- tests/logging.py | 75 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/tests/logging.py b/tests/logging.py index 1bbe3bc78..4038a218b 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -88,17 +88,20 @@ def test_truncate_rotation_file(self): backup_dir, 'node', options=['--log-rotation-age=1d']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - - # self.assertTrue(os.path.isfile(rotation_file_path)) - self.backup_node( backup_dir, 'node', node, options=[ '--stream', '--log-level-file=VERBOSE']) + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + self.assertTrue(os.path.isfile(rotation_file_path)) # truncate .rotation file @@ -111,9 +114,14 @@ def test_truncate_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE'], + '--log-level-file=LOG'], return_id=False) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + self.assertIn( 'WARNING: cannot read creation timestamp from rotation file', output) @@ -122,9 +130,14 @@ def test_truncate_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE'], + '--log-level-file=LOG'], return_id=False) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + self.assertNotIn( 'WARNING:', output) @@ -149,9 +162,6 @@ def test_unlink_rotation_file(self): backup_dir, 'node', options=['--log-rotation-age=1d']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - self.backup_node( backup_dir, 'node', node, @@ -159,6 +169,14 @@ def test_unlink_rotation_file(self): '--stream', '--log-level-file=VERBOSE']) + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + self.assertTrue(os.path.isfile(rotation_file_path)) # unlink .rotation file @@ -168,9 +186,14 @@ def test_unlink_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE'], + '--log-level-file=LOG'], return_id=False) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + self.assertIn( 'WARNING: missing rotation file:', output) @@ -188,6 +211,11 @@ def test_unlink_rotation_file(self): 'WARNING:', output) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -206,15 +234,20 @@ def test_garbage_in_rotation_file(self): backup_dir, 'node', options=['--log-rotation-age=1d']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - self.backup_node( backup_dir, 'node', node, options=[ '--stream', '--log-level-file=VERBOSE']) + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + self.assertTrue(os.path.isfile(rotation_file_path)) # mangle .rotation file @@ -227,9 +260,14 @@ def test_garbage_in_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE'], + '--log-level-file=LOG'], return_id=False) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + self.assertIn( 'WARNING: rotation file "{0}" has wrong ' 'creation timestamp'.format(rotation_file_path), @@ -241,12 +279,17 @@ def test_garbage_in_rotation_file(self): backup_dir, 'node', node, options=[ '--stream', - '--log-level-file=VERBOSE'], + '--log-level-file=LOG'], return_id=False) self.assertNotIn( 'WARNING:', output) + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + # Clean after yourself self.del_test_dir(module_name, fname) From 9ef51ae27e728c276f5c64973f8b05a20c379eec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 19:15:02 +0300 Subject: [PATCH 0571/2107] tests: added backup_test.BackupTest.test_sigterm_hadnling and backup_test.BackupTest.test_sigint_hadnling --- tests/backup_test.py | 76 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index 900247299..5690d5a02 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1087,3 +1087,79 @@ def test_pg_11_adjusted_wal_segment_size(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_sigint_hadnling(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=verbose']) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGINT') + gdb.continue_execution_until_exit() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_sigterm_hadnling(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=verbose']) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGTERM') + gdb.continue_execution_until_exit() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 9e26a6d6c97963f674611db5be6cdb63271999a9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 19:24:25 +0300 Subject: [PATCH 0572/2107] update Readme about remote mode via ssh --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b70ae8ad4..e2b11b836 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Extended logging settings * Custom commands to simplify WAL log archiving * External to PGDATA directories, such as directories with config files and scripts, can be included in backup +* Remote backup, restore, archive-push and archive-get operations via ssh (beta) To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -39,8 +40,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Limitations `pg_probackup` currently has the following limitations: -* Creating backups from a remote server is currently not supported. * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. +* Remote mode is in beta stage. * Microsoft Windows operating system support is in beta stage. ## Installation and Setup From 54998dce445eb5a3b8d6ff26c8140842a7622f8d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 19:30:36 +0300 Subject: [PATCH 0573/2107] tests: added backup_test.Backuptest.test_sigquit_hadnling --- tests/backup_test.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index 5690d5a02..94014e3f5 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1163,3 +1163,41 @@ def test_sigterm_hadnling(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_sigquit_hadnling(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=verbose']) + + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGQUIT') + gdb.continue_execution_until_exit() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) From ca717e2d4cb0efb08c616934cb9b0ab9c9167105 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Apr 2019 19:32:02 +0300 Subject: [PATCH 0574/2107] tests: spelling fix --- tests/backup_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 94014e3f5..ff8e1c3e4 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1089,7 +1089,7 @@ def test_pg_11_adjusted_wal_segment_size(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_sigint_hadnling(self): + def test_sigint_handling(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1127,7 +1127,7 @@ def test_sigint_hadnling(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_sigterm_hadnling(self): + def test_sigterm_handling(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1165,7 +1165,7 @@ def test_sigterm_hadnling(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_sigquit_hadnling(self): + def test_sigquit_handling(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From cd0548da67b89581e60501b7b2447fc7594356ce Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 00:23:45 +0300 Subject: [PATCH 0575/2107] bugfix: replace fprintf() with fio_fprintf() --- src/dir.c | 2 +- src/restore.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/dir.c b/src/dir.c index b2c2876eb..2bc11e776 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1378,7 +1378,7 @@ print_file_list(FILE *out, const parray *files, const char *root, fio_fprintf(out, ",\"segno\":\"%d\"", file->segno); if (file->linked) - fprintf(out, ",\"linked\":\"%s\"", file->linked); + fio_fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) fio_fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); diff --git a/src/restore.c b/src/restore.c index 9432731b3..0bd6e967d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -817,29 +817,29 @@ create_recovery_conf(time_t backup_id, * exclusive options is specified, so the order of calls is insignificant. */ if (rt->target_name) - fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); + fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); if (rt->time_string) - fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); + fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); if (rt->xid_string) - fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); + fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); if (rt->lsn_string) - fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); + fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); if (rt->target_stop && !target_latest) - fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); + fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); if (rt->inclusive_specified) - fprintf(fp, "recovery_target_inclusive = '%s'\n", + fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", rt->target_inclusive ? "true" : "false"); if (rt->target_tli) - fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); if (rt->target_action) - fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); + fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); } if (restore_as_replica) From 1d31908b1e0aef7e450c8a92f0666c5e99a826bd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 00:42:59 +0300 Subject: [PATCH 0576/2107] README: add checkdb --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e2b11b836..391ab1d04 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Custom commands to simplify WAL log archiving * External to PGDATA directories, such as directories with config files and scripts, can be included in backup * Remote backup, restore, archive-push and archive-get operations via ssh (beta) +* Checking running PostgreSQL instance for the sights of corruption in read-only mode via `checkdb` command. To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. From a7d074dbffb3a738069282be947d62a4df8bc6b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 02:50:47 +0300 Subject: [PATCH 0577/2107] README: change wording about WAL delivery strategies --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 391ab1d04..d62f3da0f 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Using `pg_probackup`, you can take full or incremental backups: * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode could impose read-only I/O pressure equal to `Full` backup. * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies: +Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following strategies of WAL delivery: * `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. * `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). From fd4ad9d2cb95e27f7a0f5ef627e9aab97d895f10 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 03:44:51 +0300 Subject: [PATCH 0578/2107] tests: fixes for remote backup and PG11 --- tests/archive.py | 2 +- tests/backup_test.py | 36 ++++++++++++++++++++++++------------ tests/delta.py | 44 ++++++++++++++++++++++++++++++-------------- tests/ptrack.py | 2 +- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 685e69869..a79034f41 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -424,7 +424,7 @@ def test_replica_archive(self): pg_options={ 'archive_timeout': '10s', 'checkpoint_timeout': '30s', - 'max_wal_size': '16MB'}) + 'max_wal_size': '32MB'}) self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' diff --git a/tests/backup_test.py b/tests/backup_test.py index ff8e1c3e4..be962ca0b 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -394,18 +394,30 @@ def test_page_corruption_heal_via_ptrack_2(self): " Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - "WARNING: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via SQL" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + if self.remote: + self.assertTrue( + "WARNING: File" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) self.assertTrue( self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', diff --git a/tests/delta.py b/tests/delta.py index cd72d0dc1..a77773605 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1216,18 +1216,30 @@ def test_page_corruption_heal_via_ptrack_2(self): " Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - "WARNING: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via SQL" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + if self.remote: + self.assertTrue( + "WARNING: File" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertTrue( + "WARNING: File" in e.message and + "blknum" in e.message and + "have wrong checksum" in e.message and + "try to fetch via SQL" in e.message and + "WARNING: page verification failed, " + "calculated checksum" in e.message and + "ERROR: query failed: " + "ERROR: invalid page in block" in e.message and + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) self.assertTrue( self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', @@ -1345,9 +1357,13 @@ def test_delta_backup_from_past(self): "Expecting Error because we are backing up an instance from the past" "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) - except QueryException as e: + except ProbackupException as e: self.assertTrue( - 'Insert error message' in e.message, + 'ERROR: Current START LSN ' in e.message and + 'is lower than START LSN ' in e.message and + 'of previous backup ' in e.message and + 'It may indicate that we are trying ' + 'to backup PostgreSQL instance from the past' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/ptrack.py b/tests/ptrack.py index e192b9070..075df9bbb 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -2563,7 +2563,7 @@ def test_ptrack_truncate_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_size': '16MB', + 'max_wal_size': '32MB', 'archive_timeout': '10s', 'checkpoint_timeout': '30s'}) From 1630d73becbc03a26c6587c3a20920d8a5289985 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 03:46:00 +0300 Subject: [PATCH 0579/2107] bugfix: don`t use ferror() in get_wal_file() --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 04a896efe..0c3b86027 100644 --- a/src/data.c +++ b/src/data.c @@ -1362,7 +1362,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif { read_len = fio_fread(in, buf, sizeof(buf)); - if (ferror(in)) + if (read_len < 0) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); From 58a3d2dcb0cd16499d29fc1d8c39a755269716f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 12:41:48 +0300 Subject: [PATCH 0580/2107] help: minor fix --- src/help.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index d26c4f8be..8c428ebdf 100644 --- a/src/help.c +++ b/src/help.c @@ -150,7 +150,7 @@ help_pg_probackup(void) printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); printf(_(" [--heapallindexed]\n")); From 380d6c655fd53c965044a0eef083af1f39c99d2c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 12:42:17 +0300 Subject: [PATCH 0581/2107] tests: minor fixes --- tests/replica.py | 2 +- tests/validate.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index 358d97d79..5f417f2bf 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -143,7 +143,7 @@ def test_replica_archive_page_backup(self): pg_options={ 'archive_timeout': '10s', 'checkpoint_timeout': '30s', - 'max_wal_size': '16MB'}) + 'max_wal_size': '32MB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) diff --git a/tests/validate.py b/tests/validate.py index 16a2a03c6..fc3a01d46 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1753,8 +1753,28 @@ def test_pgpro561(self): # if result == "": # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + node1.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10) i") + self.switch_wal_segment(node1) + +# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] +# wals = map(str, wals) +# print(wals) + self.switch_wal_segment(node2) + +# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] +# wals = map(str, wals) +# print(wals) + time.sleep(5) log_file = os.path.join(node2.logs_dir, 'postgresql.log') From a1e24cafc666bc1fec822894719b022359d7334c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 15:28:33 +0300 Subject: [PATCH 0582/2107] tests: fixes for false_positive module --- tests/false_positive.py | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/tests/false_positive.py b/tests/false_positive.py index e67b75988..558283a7c 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -112,7 +112,6 @@ def test_incremental_backup_corrupt_full_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_ptrack_concurrent_get_and_clear_1(self): """make node, make full and ptrack stream backups," @@ -190,13 +189,11 @@ def test_ptrack_concurrent_get_and_clear_1(self): # Logical comparison self.assertEqual( result, - node.safe_psql("postgres", "SELECT * FROM t_heap") - ) + node.safe_psql("postgres", "SELECT * FROM t_heap")) # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_ptrack_concurrent_get_and_clear_2(self): """make node, make full and ptrack stream backups," @@ -294,36 +291,3 @@ def test_ptrack_concurrent_get_and_clear_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - @unittest.expectedFailure - def test_multiple_delete(self): - """delete multiple backups""" - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") - # first full backup - backup_1_id = self.backup_node(backup_dir, 'node', node) - # second full backup - backup_2_id = self.backup_node(backup_dir, 'node', node) - # third full backup - backup_3_id = self.backup_node(backup_dir, 'node', node) - node.stop() - - self.delete_pb(backup_dir, 'node', options= - ["-i {0}".format(backup_1_id), "-i {0}".format(backup_2_id), "-i {0}".format(backup_3_id)]) - - # Clean after yourself - self.del_test_dir(module_name, fname) From 58a7922b91eaa51b14b2454b3e7fd56eb26a2419 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 15:41:14 +0300 Subject: [PATCH 0583/2107] help: shufle options a bit --- src/help.c | 171 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 66 deletions(-) diff --git a/src/help.c b/src/help.c index 8c428ebdf..d803b72ca 100644 --- a/src/help.c +++ b/src/help.c @@ -79,6 +79,8 @@ help_pg_probackup(void) printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path]\n")); + printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -90,19 +92,21 @@ help_pg_probackup(void) printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); - printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--archive-timeout=timeout]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n")); printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name]] [--temp-slot]\n")); - printf(_(" [--backup-pg-log] [-j num-threads]\n")); - printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); + printf(_(" [-D pgdata-path] [-C]\n")); + printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); + printf(_(" [--no-validate] [--skip-block-validation]\n")); + printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -116,20 +120,17 @@ help_pg_probackup(void) printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--archive-timeout=archive-timeout]\n")); printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); - printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [--external-dirs=external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); + printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); @@ -145,7 +146,8 @@ help_pg_probackup(void) printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); + printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); @@ -168,7 +170,7 @@ help_pg_probackup(void) printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); - printf(_(" [--external-dirs=external-directory-path]\n")); + printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -179,10 +181,10 @@ help_pg_probackup(void) printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--overwrite]\n")); printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--overwrite]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -209,16 +211,18 @@ static void help_init(void) { printf(_("\n%s init -B backup-path\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n\n")); } static void help_backup(void) { printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-C] [--stream [-S slot-name] [--temp-slot]\n")); - printf(_(" [--backup-pg-log] [-j num-threads]\n")); - printf(_(" [--archive-timeout=archive-timeout] [--progress]\n")); + printf(_(" [-D pgdata-path] [-C]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); + printf(_(" [--no-validate] [--skip-block-validation]\n")); + printf(_(" [-E external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -232,13 +236,9 @@ help_backup(void) printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); + printf(_(" [--archive-timeout=archive-timeout]\n")); printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); - printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [-E external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); @@ -246,17 +246,17 @@ help_backup(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); printf(_(" --stream stream the transaction log and include it in the backup\n")); printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); printf(_(" --temp-slot use temporary replication slot\n")); printf(_(" --backup-pg-log backup of '%s' directory\n"), PG_LOG_DIR); printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_(" --progress show progress\n")); printf(_(" --no-validate disable validation after backup\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - printf(_(" -E --external-dirs=external-directory-path\n")); + printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); @@ -300,6 +300,9 @@ help_backup(void) printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_("\n Connection options:\n")); printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); @@ -308,21 +311,23 @@ help_backup(void) printf(_(" -w, --no-password never prompt for password\n")); printf(_(" -W, --password force password prompt\n")); - printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); - printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); - printf(_(" --master-host=host_name database server host of master (deprecated)\n")); - printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n")); - printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: none)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); + + printf(_("\n Replica options:\n")); + printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); + printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); + printf(_(" --master-host=host_name database server host of master (deprecated)\n")); + printf(_(" --master-port=port database server port of master (deprecated)\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); } static void @@ -330,7 +335,8 @@ help_restore(void) { printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); + printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); @@ -404,9 +410,11 @@ help_restore(void) printf(_(" available options: 'ssh', 'none' (default: none)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } static void @@ -414,7 +422,8 @@ help_validate(void) { printf(_("\n%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); + printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); + printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n\n")); @@ -455,7 +464,7 @@ help_validate(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); } static void @@ -472,7 +481,7 @@ help_checkdb(void) printf(_(" --progress show progress\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --skip-block-validation skip file-level block checking\n")); + printf(_(" --skip-block-validation skip file-level checking\n")); printf(_(" can be used only with '--amcheck' option\n")); printf(_(" --amcheck in addition to file-level block checking\n")); printf(_(" check btree indexes via function 'bt_index_check()'\n")); @@ -507,7 +516,7 @@ help_checkdb(void) printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); printf(_(" -p, --port=PORT database server port (default: 5432)\n")); printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n")); + printf(_(" -W, --password force password prompt\n\n")); } static void @@ -520,7 +529,7 @@ help_show(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific intstance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); - printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --format=format show format=PLAIN|JSON\n\n")); } static void @@ -560,7 +569,7 @@ help_delete(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); } static void @@ -602,13 +611,15 @@ help_merge(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); } static void help_set_config(void) { printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-D pgdata-path]\n")); + printf(_(" [-E external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -620,16 +631,16 @@ help_set_config(void) printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--master-db=db_name] [--master-host=host_name]\n")); - printf(_(" [--master-port=port] [--master-user=user_name]\n")); - printf(_(" [--replica-timeout=timeout]\n")); printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [-E external-directory-path]\n\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -E --external-dirs=external-directory-path\n")); + printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); + printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); @@ -667,21 +678,32 @@ help_set_config(void) printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_("\n Archive options:\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_("\n Connection options:\n")); printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); printf(_(" -p, --port=PORT database server port (default: 5432)\n")); - printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); printf(_("\n Replica options:\n")); printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); printf(_(" --master-host=host_name database server host of master (deprecated)\n")); printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n")); + printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); } static void @@ -692,7 +714,7 @@ help_show_config(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --format=format show format=PLAIN|JSON\n\n")); } static void @@ -709,7 +731,7 @@ help_add_instance(void) printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --instance=instance_name name of the new instance\n")); - printf(_(" -E --external-dirs=external-directory-path\n")); + printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_("\n Remote options:\n")); @@ -717,18 +739,20 @@ help_add_instance(void) printf(_(" available options: 'ssh', 'none' (default: none)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } static void help_del_instance(void) { - printf(_("\n%s del-instance -B backup-path --instance=instance_name\n\n"), PROGRAM_NAME); + printf(_("\n%s del-instance -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" --instance=instance_name name of the instance to delete\n\n")); } static void @@ -737,10 +761,10 @@ help_archive_push(void) printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--overwrite]\n")); printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--overwrite]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); @@ -751,12 +775,25 @@ help_archive_push(void) printf(_(" relative path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" --overwrite overwrite archived WAL file\n")); + + printf(_("\n Compression options:\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'none' (default: 'none')\n")); + printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); - printf(_(" --overwrite overwrite archived WAL file\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } static void @@ -781,7 +818,9 @@ help_archive_get(void) printf(_(" available options: 'ssh', 'none' (default: none)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to pg_probackup binary on remote host (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default current user)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } From 809ccbe7bce316761c1946ec690050cc8e3d6e99 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Apr 2019 15:48:52 +0300 Subject: [PATCH 0584/2107] bump version to 2.1.0 and update expected help --- src/pg_probackup.h | 4 ++-- tests/expected/option_help.out | 39 ++++++++++++++++++------------- tests/expected/option_version.out | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1ad5ad0ce..71857ee8f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -184,8 +184,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.0.28" -#define AGENT_PROTOCOL_VERSION 20028 +#define PROGRAM_VERSION "2.1.0" +#define AGENT_PROTOCOL_VERSION 20100 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 76f2afa7e..539e0d307 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -8,6 +8,8 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup init -B backup-path pg_probackup set-config -B backup-path --instance=instance_name + [-D pgdata-path] + [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] [--log-level-file=log-level-file] [--log-filename=log-filename] @@ -19,19 +21,21 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--retention-window=retention-window] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] - [-d dbname] [-h host] [-p port] [-U username] - [--master-db=db_name] [--master-host=host_name] - [--master-port=port] [--master-user=user_name] - [--replica-timeout=timeout] [--archive-timeout=timeout] + [-d dbname] [-h host] [-p port] [-U username] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] pg_probackup show-config -B backup-path --instance=instance_name [--format=format] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name - [-C] [--stream [-S slot-name]] [--temp-slot] - [--backup-pg-log] [-j num-threads] - [--archive-timeout=archive-timeout] [--progress] + [-D pgdata-path] [-C] + [--stream [-S slot-name]] [--temp-slot] + [--backup-pg-log] [-j num-threads] [--progress] + [--no-validate] [--skip-block-validation] + [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] [--log-level-file=log-level-file] [--log-filename=log-filename] @@ -45,20 +49,17 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--compress] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] + [--archive-timeout=archive-timeout] [-d dbname] [-h host] [-p port] [-U username] [-w --no-password] [-W --password] - [--master-db=db_name] [--master-host=host_name] - [--master-port=port] [--master-user=user_name] - [--replica-timeout=timeout] - [--no-validate] [--skip-block-validation] - [--external-dirs=external-directory-path] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [-j num-threads] - [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-timeline=timeline] [--recovery-target=immediate|latest] [--recovery-target-name=target-name] @@ -74,11 +75,17 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] - [--recovery-target-time=time|--recovery-target-xid=xid|--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-timeline=timeline] [--recovery-target-name=target-name] [--skip-block-validation] + pg_probackup checkdb [-B backup-path] [--instance=instance_name] + [-D pgdata-path] [--progress] [-j num-threads] + [--amcheck] [--skip-block-validation] + [--heapallindexed] + pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] [--format=format] @@ -92,7 +99,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name - [--external-dirs=external-directory-path] + [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] @@ -103,10 +110,10 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup archive-push -B backup-path --instance=instance_name --wal-file-path=wal-file-path --wal-file-name=wal-file-name + [--overwrite] [--compress] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] - [--overwrite] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e98eea5d4..7ed9f224a 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.0.28 \ No newline at end of file +pg_probackup 2.1.0 \ No newline at end of file From da21861bfde8c38e70a6a2c59c61c8072dc89f8a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Apr 2019 12:55:30 +0300 Subject: [PATCH 0585/2107] fix remote mode for add-instance --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index dc40fe1cf..5828b5e86 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -210,7 +210,7 @@ setMyLocation(void) MyLocation = IsSshProtocol() ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) ? FIO_DB_HOST - : (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD) + : (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == ADD_INSTANCE_CMD) ? FIO_BACKUP_HOST : FIO_LOCAL_HOST : FIO_LOCAL_HOST; From 1eeff192536ba77949b384a7e483549426d95ab4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Apr 2019 20:52:00 +0300 Subject: [PATCH 0586/2107] tests: fixes for PG 9.5 --- tests/archive.py | 8 ++- tests/auth_test.py | 6 +-- tests/cfs_backup.py | 1 - tests/cfs_restore.py | 2 - tests/checkdb.py | 29 +++++++++-- tests/compression.py | 10 ++-- tests/delete.py | 14 +++-- tests/delta.py | 23 +++------ tests/exclude.py | 9 ++-- tests/external.py | 3 ++ tests/locking.py | 21 +++----- tests/logging.py | 5 ++ tests/merge.py | 28 +++------- tests/option_test.py | 1 - tests/page.py | 25 +++------ tests/pgpro560.py | 9 ++-- tests/pgpro589.py | 5 +- tests/ptrack.py | 47 ++--------------- tests/remote.py | 3 +- tests/replica.py | 18 +++---- tests/restore.py | 118 +++++++++++++++---------------------------- tests/retention.py | 13 +++-- tests/show_test.py | 18 +++---- tests/snapfs.py | 4 +- tests/time_stamp.py | 10 ++-- 25 files changed, 158 insertions(+), 272 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index a79034f41..634f2019a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -324,8 +324,12 @@ def test_arhive_push_file_exists(self): ) self.assertFalse('pg_probackup archive-push completed successfully' in log_content) - wal_src = os.path.join( - node.data_dir, 'pg_wal', '000000010000000000000001') + if self.get_version(node) < 100000: + wal_src = os.path.join( + node.data_dir, 'pg_xlog', '000000010000000000000001') + else: + wal_src = os.path.join( + node.data_dir, 'pg_wal', '000000010000000000000001') if self.archive_compress: with open(wal_src, 'rb') as f_in, gzip.open( diff --git a/tests/auth_test.py b/tests/auth_test.py index 153044e10..8b8378348 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -36,7 +36,6 @@ def test_backup_via_unpriviledged_user(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'} ) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -193,10 +192,7 @@ def setUpClass(cls): cls.node = cls.pb.make_simple_node( base_dir="{}/node".format(module_name), set_replication=True, - initdb_params=['--data-checksums', '--auth-host=md5'], - pg_options={ - 'wal_level': 'replica' - } + initdb_params=['--data-checksums', '--auth-host=md5'] ) modify_pg_hba(cls.node) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 76d4a0a11..5a3665518 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -22,7 +22,6 @@ def setUp(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'ptrack_enable': 'on', 'cfs_encryption': 'off', 'max_wal_senders': '2', diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index a18d0903e..f59c092af 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -32,10 +32,8 @@ def setUp(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', # 'ptrack_enable': 'on', 'cfs_encryption': 'off', - 'max_wal_senders': '2' } ) diff --git a/tests/checkdb.py b/tests/checkdb.py index 5a133cf93..6bbc96c54 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -37,9 +37,14 @@ def test_checkdb_amcheck_only_sanity(self): "postgres", "create index on t_heap(id)") - node.safe_psql( - "postgres", - "create extension amcheck") + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") log_file_path = os.path.join( backup_dir, 'log', 'pg_probackup.log') @@ -221,10 +226,24 @@ def test_checkdb_amcheck_only_sanity_1(self): # create two databases node.safe_psql("postgres", "create database db1") - node.safe_psql("db1", "create extension amcheck") + try: + node.safe_psql( + "db1", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "db1", + "create extension amcheck_next") node.safe_psql("postgres", "create database db2") - node.safe_psql("db2", "create extension amcheck") + try: + node.safe_psql( + "db2", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "db2", + "create extension amcheck_next") # init pgbench in two databases and corrupt both indexes node.pgbench_init(scale=5, dbname='db1') diff --git a/tests/compression.py b/tests/compression.py index caae94c0e..fd86d15a3 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -13,7 +13,10 @@ class CompressionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_compression_stream_zlib(self): - """make archive node, make full and page stream backups, check data correctness in restored instance""" + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -462,7 +465,6 @@ def test_compression_wrong_algorithm(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} @@ -505,9 +507,7 @@ def test_uncompressable_pages(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/delete.py b/tests/delete.py index ac5a42841..8c0994f82 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -17,8 +17,7 @@ def test_delete_full_backups(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -98,8 +97,7 @@ def test_delete_increment_page(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -140,7 +138,7 @@ def test_delete_increment_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + pg_options={'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -180,8 +178,7 @@ def test_delete_orphaned_wal_segments(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -252,7 +249,8 @@ def test_delete_backup_with_empty_control_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'}) + set_replication=True, + pg_options={'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/delta.py b/tests/delta.py index a77773605..a56652390 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -27,7 +27,6 @@ def test_delta_vacuum_truncate_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off'}) @@ -107,7 +106,6 @@ def test_delta_vacuum_truncate_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' @@ -198,7 +196,6 @@ def test_delta_vacuum_truncate_3(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' @@ -268,7 +265,6 @@ def test_delta_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s' } @@ -428,7 +424,6 @@ def test_delta_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'fsync': 'off', 'shared_buffers': '1GB', @@ -511,7 +506,6 @@ def test_delta_vacuum_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s' } @@ -603,7 +597,6 @@ def test_create_db(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_size': '10GB', 'max_wal_senders': '2', 'checkpoint_timeout': '5min', @@ -734,7 +727,6 @@ def test_exists_in_previous_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_size': '10GB', 'max_wal_senders': '2', 'checkpoint_timeout': '5min', @@ -843,7 +835,6 @@ def test_alter_table_set_tablespace_delta(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -934,9 +925,9 @@ def test_alter_database_set_tablespace_delta(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + set_replication=True, + initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -1030,7 +1021,6 @@ def test_delta_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -1112,8 +1102,8 @@ def test_delta_corruption_heal_via_ptrack_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1171,8 +1161,8 @@ def test_page_corruption_heal_via_ptrack_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + pg_options={'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1324,6 +1314,7 @@ def test_delta_backup_from_past(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/exclude.py b/tests/exclude.py index b290a3b63..21272bcaa 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -22,9 +22,8 @@ def test_exclude_temp_tables(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', - 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'} - ) + 'max_wal_senders': '2', + 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -113,12 +112,10 @@ def test_exclude_unlogged_tables_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', "shared_buffers": "10MB", "fsync": "off", - 'ptrack_enable': 'on'} - ) + 'ptrack_enable': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/external.py b/tests/external.py index 5c82fca6b..8630f6788 100644 --- a/tests/external.py +++ b/tests/external.py @@ -25,6 +25,7 @@ def test_external_simple(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], + set_replication=True, pg_options={ 'max_wal_senders': '2'}) @@ -105,6 +106,7 @@ def test_external_none(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], + set_replication=True, pg_options={ 'max_wal_senders': '2'}) @@ -253,6 +255,7 @@ def test_backup_multiple_external(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/locking.py b/tests/locking.py index b4b271f46..918509c60 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -20,8 +20,7 @@ def test_locking_running_validate_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -75,8 +74,7 @@ def test_locking_running_validate_2(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -145,8 +143,7 @@ def test_locking_running_validate_2_specific_id(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -243,8 +240,7 @@ def test_locking_running_3(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -313,8 +309,7 @@ def test_locking_restore_locked(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -367,8 +362,7 @@ def test_locking_restore_locked_without_validation(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -421,8 +415,7 @@ def test_locking_concurrent_vaidate_and_backup(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/logging.py b/tests/logging.py index 4038a218b..c47d68dc1 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -15,6 +15,7 @@ def test_log_rotation(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -45,6 +46,7 @@ def test_log_filename_strftime(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -77,6 +79,7 @@ def test_truncate_rotation_file(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -151,6 +154,7 @@ def test_unlink_rotation_file(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -223,6 +227,7 @@ def test_garbage_in_rotation_file(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/merge.py b/tests/merge.py index 71bf8de36..56dc077c8 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -485,7 +485,6 @@ def test_merge_tablespaces(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'autovacuum': 'off' } @@ -565,7 +564,6 @@ def test_merge_tablespaces_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'autovacuum': 'off' } @@ -650,7 +648,6 @@ def test_merge_page_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' @@ -745,7 +742,6 @@ def test_merge_delta_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' @@ -840,7 +836,6 @@ def test_merge_ptrack_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off', @@ -935,7 +930,6 @@ def test_merge_delta_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -1020,12 +1014,10 @@ def test_continue_failed_merge(self): fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica' - } - ) + base_dir=os.path.join( + module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1072,7 +1064,7 @@ def test_continue_failed_merge(self): gdb.set_breakpoint('copy_file') gdb.run_until_break() - gdb.continue_execution_until_break(20) + gdb.continue_execution_until_break(5) gdb._execute('signal SIGKILL') @@ -1097,11 +1089,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica' - } - ) + set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1365,8 +1353,7 @@ def test_merge_different_compression_algo(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1696,7 +1683,6 @@ def test_merge_backup_from_future(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'autovacuum': 'off'}) diff --git a/tests/option_test.py b/tests/option_test.py index 092c79d99..d2c352275 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -120,7 +120,6 @@ def test_options_5(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), pg_options={ - 'wal_level': 'logical', 'max_wal_senders': '2'}) self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), diff --git a/tests/page.py b/tests/page.py index b0eb79bf6..d9395c96e 100644 --- a/tests/page.py +++ b/tests/page.py @@ -27,7 +27,6 @@ def test_page_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' @@ -120,7 +119,6 @@ def test_page_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) @@ -209,7 +207,6 @@ def test_page_archive(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) @@ -304,7 +301,6 @@ def test_page_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'fsync': 'off', 'shared_buffers': '1GB', @@ -387,7 +383,6 @@ def test_page_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -467,7 +462,6 @@ def test_page_delete_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -674,9 +668,8 @@ def test_page_backup_with_lost_wal_segment(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -763,9 +756,8 @@ def test_page_backup_with_corrupted_wal_segment(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -884,9 +876,8 @@ def test_page_backup_with_alien_wal_segment(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + alien_node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'alien_node')) @@ -1003,9 +994,7 @@ def test_multithread_page_backup_with_toast(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index ed0110741..7ebb2131d 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -23,8 +23,7 @@ def test_pgpro560_control_file_loss(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -59,15 +58,13 @@ def test_pgpro560_systemid_mismatch(self): node1 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) node1.slow_start() node2 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node2'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) node2.slow_start() diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 11809528e..3ed398725 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -20,9 +20,8 @@ def test_pgpro589(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/ptrack.py b/tests/ptrack.py index 075df9bbb..76d7d3ba8 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -24,7 +24,6 @@ def test_ptrack_enable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s' } @@ -73,7 +72,6 @@ def test_ptrack_disable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on' @@ -134,7 +132,6 @@ def test_ptrack_uncommited_xact(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' @@ -196,7 +193,6 @@ def test_ptrack_vacuum_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' @@ -288,7 +284,6 @@ def test_ptrack_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on', @@ -374,7 +369,6 @@ def test_ptrack_simple(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' @@ -450,7 +444,6 @@ def test_ptrack_get_block(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' @@ -522,7 +515,6 @@ def test_ptrack_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -617,7 +609,6 @@ def test_ptrack_archive(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -724,8 +715,6 @@ def test_ptrack_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': - 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -808,7 +797,6 @@ def test_page_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -880,7 +868,6 @@ def test_full_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -957,7 +944,6 @@ def test_create_db(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_size': '10GB', 'max_wal_senders': '2', 'checkpoint_timeout': '5min', @@ -1084,7 +1070,6 @@ def test_create_db_on_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -1184,7 +1169,6 @@ def test_alter_table_set_tablespace_ptrack(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -1282,7 +1266,6 @@ def test_alter_database_set_tablespace_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -1353,7 +1336,6 @@ def test_drop_tablespace(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', @@ -1437,10 +1419,9 @@ def test_ptrack_alter_tablespace(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', + 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1550,7 +1531,7 @@ def test_ptrack_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2', + 'max_wal_senders': '2', 'ptrack_enable': 'on', 'fsync': 'off', 'autovacuum': 'off', 'full_page_writes': 'off' @@ -1660,9 +1641,9 @@ def test_atexit_fail(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'max_connections': '15'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1713,7 +1694,6 @@ def test_ptrack_clean(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1825,7 +1805,6 @@ def test_ptrack_clean_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'archive_timeout': '30s'}) @@ -1962,7 +1941,6 @@ def test_ptrack_cluster_on_btree(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2043,7 +2021,6 @@ def test_ptrack_cluster_on_gist(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2121,7 +2098,6 @@ def test_ptrack_cluster_on_btree_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2221,7 +2197,6 @@ def test_ptrack_cluster_on_gist_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2327,7 +2302,6 @@ def test_ptrack_empty(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2408,8 +2382,8 @@ def test_ptrack_empty_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -2502,7 +2476,6 @@ def test_ptrack_truncate(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2672,7 +2645,6 @@ def test_ptrack_vacuum(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2759,7 +2731,6 @@ def test_ptrack_vacuum_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30'}) @@ -2869,7 +2840,6 @@ def test_ptrack_vacuum_bits_frozen(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2949,7 +2919,6 @@ def test_ptrack_vacuum_bits_frozen_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3054,7 +3023,6 @@ def test_ptrack_vacuum_bits_visibility(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3135,7 +3103,6 @@ def test_ptrack_vacuum_full(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3216,7 +3183,6 @@ def test_ptrack_vacuum_full_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'autovacuum': 'off', 'archive_timeout': '30s'} @@ -3326,7 +3292,6 @@ def test_ptrack_vacuum_truncate(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3407,7 +3372,6 @@ def test_ptrack_vacuum_truncate_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30'}) @@ -3508,7 +3472,6 @@ def test_ptrack_recovery(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/remote.py b/tests/remote.py index 8230d40d7..50697b3e5 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -16,8 +16,7 @@ def test_remote_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/replica.py b/tests/replica.py index 5f417f2bf..116dbdd98 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -25,10 +25,8 @@ def test_replica_stream_ptrack_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'ptrack_enable': 'on'} - ) + 'ptrack_enable': 'on'}) + master.slow_start() self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -292,10 +290,8 @@ def test_make_replica_via_restore(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'archive_timeout': '10s'} - ) + 'archive_timeout': '10s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -353,10 +349,8 @@ def test_take_backup_from_delayed_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'max_wal_senders': '2', - 'archive_timeout': '10s'} - ) + 'archive_timeout': '10s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) diff --git a/tests/restore.py b/tests/restore.py index 80336b2a9..256746aba 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -21,9 +21,7 @@ def test_restore_full_to_latest(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -69,9 +67,8 @@ def test_restore_full_page_to_latest(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -117,9 +114,8 @@ def test_restore_to_specific_timeline(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -186,9 +182,8 @@ def test_restore_to_time(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -235,9 +230,8 @@ def test_restore_to_xid_inclusive(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -298,10 +292,8 @@ def test_restore_to_xid_not_inclusive(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -361,9 +353,7 @@ def test_restore_to_lsn_inclusive(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): self.del_test_dir(module_name, fname) @@ -434,9 +424,7 @@ def test_restore_to_lsn_not_inclusive(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): self.del_test_dir(module_name, fname) @@ -509,8 +497,8 @@ def test_restore_full_ptrack_archive(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -557,8 +545,8 @@ def test_restore_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'ptrack_enable': 'on'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -613,11 +601,8 @@ def test_restore_full_ptrack_stream(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + pg_options={'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -669,10 +654,8 @@ def test_restore_full_ptrack_under_load(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -735,10 +718,8 @@ def test_restore_full_under_load_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -799,10 +780,8 @@ def test_restore_with_tablespace_mapping_1(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'ptrack_enable': 'on', - 'max_wal_senders': '2'} - ) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -917,9 +896,8 @@ def test_restore_with_tablespace_mapping_2(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1000,9 +978,8 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1050,9 +1027,8 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1099,9 +1075,8 @@ def test_archive_node_backup_stream_pitr(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica', 'max_wal_senders': '2'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1147,9 +1122,8 @@ def test_archive_node_backup_archive_pitr_2(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1207,9 +1181,8 @@ def test_archive_restore_to_restore_point(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1256,9 +1229,8 @@ def test_zags_block_corrupt(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1312,9 +1284,7 @@ def test_zags_block_corrupt(self): node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) node_restored.cleanup() @@ -1336,7 +1306,6 @@ def test_zags_block_corrupt_1(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'autovacuum': 'off', 'full_page_writes': 'on'} ) @@ -1404,9 +1373,7 @@ def test_zags_block_corrupt_1(self): node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) pgdata = self.pgdata_content(node.data_dir) @@ -1462,7 +1429,6 @@ def test_restore_chain(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1553,7 +1519,6 @@ def test_restore_chain_with_corrupted_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1736,7 +1701,6 @@ def test_restore_backup_from_future(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', 'max_wal_senders': '2'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/retention.py b/tests/retention.py index e1104b3a8..e0429439e 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -16,9 +16,8 @@ def test_retention_redundancy_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -74,9 +73,8 @@ def test_retention_window_2(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1047,7 +1045,8 @@ def test_window_chains(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/show_test.py b/tests/show_test.py index 201839a77..43d4848b7 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -16,8 +16,7 @@ def test_show_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -43,8 +42,7 @@ def test_show_json(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -70,8 +68,7 @@ def test_corrupt_2(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -115,8 +112,7 @@ def test_no_control_file(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -145,8 +141,7 @@ def test_empty_control_file(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -177,8 +172,7 @@ def test_corrupt_control_file(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/snapfs.py b/tests/snapfs.py index 3ea05e8ed..a7f926c4c 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -16,9 +16,7 @@ def test_snapfs_simple(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 8f0355524..adc6cd3e6 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -12,10 +12,12 @@ def test_start_time_format(self): We should convert local time in UTC format""" # Create simple node fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'} - ) + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 99c55b6cff04dd340e4a659060353a0405034308 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Apr 2019 10:06:27 +0300 Subject: [PATCH 0587/2107] tests: fix checkdb.CheckdbTest.test_checkdb_amcheck_only_sanity_1 --- tests/checkdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 6bbc96c54..29674ff31 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -218,7 +218,8 @@ def test_checkdb_amcheck_only_sanity_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True) + set_replication=True, + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 5dfa59fa9e7cd796b2a22581f4b0ec914d911d0c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 12:00:53 +0300 Subject: [PATCH 0588/2107] dir_is_empty() should be aware about MyLocation --- src/catalog.c | 2 +- src/dir.c | 12 ++++++------ src/parsexlog.c | 2 +- src/pg_probackup.h | 2 +- src/restore.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 693d46fe8..7a662b713 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -489,7 +489,7 @@ pgBackupCreateDir(pgBackup *backup) pgBackupGetPath(backup, path, lengthof(path), NULL); - if (!dir_is_empty(path)) + if (!dir_is_empty(path, FIO_BACKUP_HOST)) elog(ERROR, "backup destination is not empty \"%s\"", path); fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); diff --git a/src/dir.c b/src/dir.c index 2bc11e776..fb7bcc7c0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1277,7 +1277,7 @@ check_tablespace_mapping(pgBackup *backup) elog(ERROR, "tablespace directory is not an absolute path: %s\n", linked_path); - if (!dir_is_empty(linked_path)) + if (!dir_is_empty(linked_path, FIO_DB_HOST)) elog(ERROR, "restore tablespace destination is not empty: \"%s\"", linked_path); } @@ -1619,12 +1619,12 @@ dir_read_file_list(const char *root, const char *external_prefix, * Check if directory empty. */ bool -dir_is_empty(const char *path) +dir_is_empty(const char *path, fio_location location) { DIR *dir; struct dirent *dir_ent; - dir = opendir(path); + dir = fio_opendir(path, location); if (dir == NULL) { /* Directory in path doesn't exist */ @@ -1634,7 +1634,7 @@ dir_is_empty(const char *path) } errno = 0; - while ((dir_ent = readdir(dir))) + while ((dir_ent = fio_readdir(dir))) { /* Skip entries point current dir or parent dir */ if (strcmp(dir_ent->d_name, ".") == 0 || @@ -1642,13 +1642,13 @@ dir_is_empty(const char *path) continue; /* Directory is not empty */ - closedir(dir); + fio_closedir(dir); return false; } if (errno) elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); - closedir(dir); + fio_closedir(dir); return true; } diff --git a/src/parsexlog.c b/src/parsexlog.c index 075bece48..a2682083a 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -359,7 +359,7 @@ validate_wal(pgBackup *backup, const char *archivedir, * If recovery target is provided, ensure that archive files exist in * archive directory. */ - if (dir_is_empty(archivedir)) + if (dir_is_empty(archivedir, FIO_BACKUP_HOST)) elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 71857ee8f..72d95fedd 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -583,7 +583,7 @@ extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, extern bool backup_contains_external(const char *dir, parray *dirs_list); extern int dir_create_dir(const char *path, mode_t mode); -extern bool dir_is_empty(const char *path); +extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); diff --git a/src/restore.c b/src/restore.c index 0bd6e967d..80111fc29 100644 --- a/src/restore.c +++ b/src/restore.c @@ -64,7 +64,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(ERROR, "required parameter not specified: PGDATA (-D, --pgdata)"); /* Check if restore destination empty */ - if (!dir_is_empty(instance_config.pgdata)) + if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) elog(ERROR, "restore destination is not empty: \"%s\"", instance_config.pgdata); } From 0baf9764243ccdbc11403e8b3971b600de186f99 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 12:15:32 +0300 Subject: [PATCH 0589/2107] Create external directories by fio_mkdir() --- src/restore.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index 80111fc29..82a877314 100644 --- a/src/restore.c +++ b/src/restore.c @@ -512,7 +512,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) { char *external_path = parray_get(requested_external_dirs, i); external_path = get_external_remap(external_path); - dir_create_dir(external_path, DIR_PERMISSION); + fio_mkdir(external_path, DIR_PERMISSION, FIO_DB_HOST); } } @@ -562,7 +562,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) dir_name = GetRelativePath(file->path, container_dir); elog(VERBOSE, "Create directory \"%s\"", dir_name); join_path_components(dirpath, external_path, dir_name); - dir_create_dir(dirpath, DIR_PERMISSION); + fio_mkdir(dirpath, DIR_PERMISSION, FIO_DB_HOST); } } From 9ea7634e66f8245138303b868e048a7f8ce16edf Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 13:34:39 +0300 Subject: [PATCH 0590/2107] Remote: Handle case when dir[hdr.handle] is NULL --- src/utils/file.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 11b93a9f7..f582a4817 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1227,14 +1227,24 @@ void fio_communicate(int in, int out) case FIO_OPENDIR: /* Open directory for traversal */ dir[hdr.handle] = opendir(buf); break; - case FIO_READDIR: /* Get next direcrtory entry */ - entry = readdir(dir[hdr.handle]); + case FIO_READDIR: /* Get next directory entry */ hdr.cop = FIO_SEND; - if (entry != NULL) { - hdr.size = sizeof(*entry); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); - } else { + entry = NULL; + + if (dir[hdr.handle] != NULL) + { + entry = readdir(dir[hdr.handle]); + if (entry != NULL) + { + hdr.size = sizeof(*entry); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); + } + } + + /* We didn't manage to read the directory */ + if (entry == NULL) + { hdr.size = 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } From da6a1126f39e13e4f9cecaf522406a1ffb181de4 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 14:15:02 +0300 Subject: [PATCH 0591/2107] Remote: Handle case when dir[hdr.handle] is NULL --- src/utils/file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index f582a4817..73148a274 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1250,7 +1250,8 @@ void fio_communicate(int in, int out) } break; case FIO_CLOSEDIR: /* Finish directory traversal */ - SYS_CHECK(closedir(dir[hdr.handle])); + if (dir[hdr.handle] != NULL) + SYS_CHECK(closedir(dir[hdr.handle])); break; case FIO_OPEN: /* Open file */ SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); From 9386d5cdccab2f47c0cb45ba586b6f744a021df3 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 14:52:26 +0300 Subject: [PATCH 0592/2107] Handle errno by FIO_OPENDIR and FIO_OPEN --- src/utils/file.c | 50 +++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 73148a274..1b63a764e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -162,6 +162,13 @@ DIR* fio_opendir(char const* path, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return NULL; + } dir = (DIR*)(size_t)(i + 1); } else @@ -243,6 +250,13 @@ int fio_open(char const* path, int mode, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } fd = i | FIO_PIPE_MARKER; } else @@ -255,7 +269,8 @@ int fio_open(char const* path, int mode, fio_location location) /* Open stdio file */ FILE* fio_fopen(char const* path, char const* mode, fio_location location) { - FILE* f; + FILE *f = NULL + if (fio_is_remote(location)) { int flags = O_RDWR|O_CREAT; @@ -268,7 +283,8 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) flags |= O_APPEND; } fd = fio_open(path, flags, location); - f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); + if (fd != NULL) + f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); } else { @@ -1226,35 +1242,33 @@ void fio_communicate(int in, int out) break; case FIO_OPENDIR: /* Open directory for traversal */ dir[hdr.handle] = opendir(buf); + hdr.arg = dir[hdr.handle] == NULL ? errno : 0; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_READDIR: /* Get next directory entry */ hdr.cop = FIO_SEND; - entry = NULL; - - if (dir[hdr.handle] != NULL) + entry = readdir(dir[hdr.handle]); + if (entry != NULL) { - entry = readdir(dir[hdr.handle]); - if (entry != NULL) - { - hdr.size = sizeof(*entry); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); - } + hdr.size = sizeof(*entry); + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); } - - /* We didn't manage to read the directory */ - if (entry == NULL) + else { hdr.size = 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } break; case FIO_CLOSEDIR: /* Finish directory traversal */ - if (dir[hdr.handle] != NULL) - SYS_CHECK(closedir(dir[hdr.handle])); + SYS_CHECK(closedir(dir[hdr.handle])); break; case FIO_OPEN: /* Open file */ - SYS_CHECK(fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS)); + fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS); + hdr.arg = fd[hdr.handle] < 0 ? errno : 0; + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CLOSE: /* Close file */ SYS_CHECK(close(fd[hdr.handle])); From 5d240242dfe39fcafa0cd45f0d1d2546f33ead63 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 14:53:29 +0300 Subject: [PATCH 0593/2107] Fix typos --- src/utils/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 1b63a764e..9438fe3f0 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -269,7 +269,7 @@ int fio_open(char const* path, int mode, fio_location location) /* Open stdio file */ FILE* fio_fopen(char const* path, char const* mode, fio_location location) { - FILE *f = NULL + FILE *f = NULL; if (fio_is_remote(location)) { @@ -283,7 +283,7 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) flags |= O_APPEND; } fd = fio_open(path, flags, location); - if (fd != NULL) + if (fd >= 0) f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); } else From 6b9aca1c41911312b8a0093fef32303408bf51cc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Apr 2019 17:21:30 +0300 Subject: [PATCH 0594/2107] tests: added external.ExternalTest.test_restore_external_dir_not_empty --- tests/external.py | 68 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/tests/external.py b/tests/external.py index 8630f6788..8228ced4a 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1391,16 +1391,58 @@ def test_external_dir_is_tablespace(self): # Clean after yourself self.del_test_dir(module_name, fname) - # external directory contain symlink to file - # external directory contain symlink to directory - # external directory is symlink + - # latest page backup without external_dir + - # multiple external directories + - # --external-dirs=none + - # --external-dirs point to a file + - # external directory in config and in command line + - # external directory contain multuple directories, some of them my be empty + - # forbid to external-dirs to point to tablespace directories - # check that not changed files are not copied by next backup + - # merge + - # complex merge + + def test_restore_external_dir_not_empty(self): + """ + Check that backup fails with error + if external directory points to tablespace + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + node.cleanup() + + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external diris not empty" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Insert correct error message' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From a480a5d275f9970b910f2a54fb451ca3c0a88f31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Apr 2019 17:55:04 +0300 Subject: [PATCH 0595/2107] tests: add more "not empty" tests for tablespaces and externals, remapping included --- tests/backup_test.py | 15 ++++----------- tests/external.py | 30 ++++++++++++++++++++++++++++-- tests/restore.py | 33 ++++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index be962ca0b..9081d10b5 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -433,8 +433,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -533,9 +532,7 @@ def test_tablespace_handling(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -645,9 +642,7 @@ def test_tablespace_handling_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -705,9 +700,7 @@ def test_tablespace_handling_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/external.py b/tests/external.py index 8228ced4a..fa0d0e957 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1394,7 +1394,8 @@ def test_external_dir_is_tablespace(self): def test_restore_external_dir_not_empty(self): """ Check that backup fails with error - if external directory points to tablespace + if external directory point to not empty tablespace and + if remapped directory also isn`t empty """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1435,7 +1436,32 @@ def test_restore_external_dir_not_empty(self): # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because external diris not empty" + "Expecting Error because external dir is not empty" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Insert correct error message' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + external_dir_new = self.get_tblspace_path(node, 'external_dir_new') + + # create empty file in directory, which will be a target of + # remapping + os.mkdir(external_dir_new) + with open(os.path.join(external_dir_new, 'file1'), 'w+') as f: + f.close() + + try: + self.restore_node( + backup_dir, 'node', node, + options=['--external-mapping={0}={1}'.format( + external_dir, external_dir_new)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped external dir is not empty" "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: diff --git a/tests/restore.py b/tests/restore.py index 256746aba..3019d552d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -290,9 +290,7 @@ def test_restore_to_xid_not_inclusive(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -778,9 +776,7 @@ def test_restore_with_tablespace_mapping_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -839,7 +835,30 @@ def test_restore_with_tablespace_mapping_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # 3 - Restore using tablespace-mapping + # 3 - Restore using tablespace-mapping to not empty directory + tblspc_path_temp = os.path.join(node.base_dir, "tblspc_temp") + os.mkdir(tblspc_path_temp) + with open(os.path.join(tblspc_path_temp, 'file'), 'w+') as f: + f.close() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["-T", "%s=%s" % (tblspc_path, tblspc_path_temp)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: restore tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 4 - Restore using tablespace-mapping tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), From 130bb7ee481638832c990d20306d168928c82f6e Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 23 Apr 2019 18:37:50 +0300 Subject: [PATCH 0596/2107] Check for existance and emptiness of external directories --- src/dir.c | 28 +++++++++++++++++++++++----- tests/external.py | 6 +++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/dir.c b/src/dir.c index fb7bcc7c0..6d0b0fc18 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1292,7 +1292,6 @@ check_external_dir_mapping(pgBackup *backup) { TablespaceListCell *cell; parray *external_dirs_to_restore; - bool found; int i; if (!backup->external_dir_str) @@ -1305,16 +1304,22 @@ check_external_dir_mapping(pgBackup *backup) } external_dirs_to_restore = make_external_directory_list(backup->external_dir_str); + /* 1 - each OLDDIR must have an entry in external_dirs_to_restore */ for (cell = external_remap_list.head; cell; cell = cell->next) { - char *old_dir = cell->old_dir; + bool found = false; - found = false; for (i = 0; i < parray_num(external_dirs_to_restore); i++) { - char *external_dir = parray_get(external_dirs_to_restore, i); - if (strcmp(old_dir, external_dir) == 0) + char *external_dir = parray_get(external_dirs_to_restore, i); + + if (strcmp(cell->old_dir, external_dir) == 0) { + /* Swap new dir name with old one, it is used by 2-nd step */ + parray_set(external_dirs_to_restore, i, + pgut_strdup(cell->new_dir)); + pfree(external_dir); + found = true; break; } @@ -1324,6 +1329,19 @@ check_external_dir_mapping(pgBackup *backup) "have an entry in list of external directories of current " "backup: \"%s\"", cell->old_dir); } + + /* 2 - all linked directories must be empty */ + for (i = 0; i < parray_num(external_dirs_to_restore); i++) + { + char *external_dir = (char *) parray_get(external_dirs_to_restore, + i); + + if (!dir_is_empty(external_dir, FIO_DB_HOST)) + elog(ERROR, "External directory is not empty: \"%s\"", + external_dir); + } + + free_dir_list(external_dirs_to_restore); } char * diff --git a/tests/external.py b/tests/external.py index fa0d0e957..d22a78923 100644 --- a/tests/external.py +++ b/tests/external.py @@ -304,7 +304,7 @@ def test_backup_multiple_external(self): self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) - + pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) @@ -1441,7 +1441,7 @@ def test_restore_external_dir_not_empty(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'Insert correct error message' in e.message, + 'External directory is not empty' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1466,7 +1466,7 @@ def test_restore_external_dir_not_empty(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'Insert correct error message' in e.message, + 'External directory is not empty' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 6ad66cca0ca97178181e6e782704c1c3e9ca636d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 11:24:59 +0300 Subject: [PATCH 0597/2107] Use pg_attribute_printf instead of __attribute__ --- src/utils/file.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.h b/src/utils/file.h index d8727fe72..f168d6cbe 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -71,7 +71,7 @@ extern FILE* fio_fopen(char const* name, char const* mode, fio_location locati extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); -extern int fio_fprintf(FILE* f, char const* arg, ...) __attribute__((format(printf, 2, 3))); +extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); extern int fio_ftruncate(FILE* f, off_t size); From f6e1bcdb288061d99dd9986ec264111d117ab7ab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Apr 2019 19:46:08 +0300 Subject: [PATCH 0598/2107] add elog message about external directories --- src/dir.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dir.c b/src/dir.c index 6d0b0fc18..781f08ac0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1243,10 +1243,8 @@ check_tablespace_mapping(pgBackup *backup) /* Sort links by the path of a linked file*/ parray_qsort(links, pgFileCompareLinked); - if (logger_config.log_level_console <= LOG || - logger_config.log_level_file <= LOG) - elog(LOG, "check tablespace directories of backup %s", - base36enc(backup->start_time)); + elog(LOG, "check tablespace directories of backup %s", + base36enc(backup->start_time)); /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ for (cell = tablespace_dirs.head; cell; cell = cell->next) @@ -1294,6 +1292,9 @@ check_external_dir_mapping(pgBackup *backup) parray *external_dirs_to_restore; int i; + elog(LOG, "check external directories of backup %s", + base36enc(backup->start_time)); + if (!backup->external_dir_str) { if (external_remap_list.head) From 7412b74d18c597a03417635fed4d12c89a1bcb08 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Apr 2019 23:25:46 +0300 Subject: [PATCH 0599/2107] README: minor update of archive backup description --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d62f3da0f..7e2c601b2 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Using `pg_probackup`, you can take full or incremental backups: * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following strategies of WAL delivery: -* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. -* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). +* `Autonomous backups` streams via replication protocol all the WAL files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +* `Archive backups` rely on continuous archiving. `PTRACK` backup support provided via following options: * vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) From 3578ef9f5e15eaf63f7b7bd094e9ef81d816c1a7 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 13:57:38 +0300 Subject: [PATCH 0600/2107] Fix Win32 build problems --- src/pg_probackup.h | 5 +++++ src/utils/logger.c | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 72d95fedd..85c6a7f39 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -68,6 +68,11 @@ #define XID_FMT "%u" #endif +#ifndef STDIN_FILENO +#define STDIN_FILENO 0 +#define STDOUT_FILENO 1 +#endif + /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) diff --git a/src/utils/logger.c b/src/utils/logger.c index 784f262f4..5cd41ae8c 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -484,11 +484,7 @@ logfile_getname(const char *format, time_t timestamp) len = strlen(filename); /* Treat log_filename as a strftime pattern */ -#ifdef WIN32 - if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#else if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); return filename; From 3be11bc8996966a498d31e3be603b60bb3f85b07 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Apr 2019 14:51:28 +0300 Subject: [PATCH 0601/2107] get pg_strftime() backup --- src/utils/logger.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/logger.c b/src/utils/logger.c index 5cd41ae8c..784f262f4 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -484,7 +484,11 @@ logfile_getname(const char *format, time_t timestamp) len = strlen(filename); /* Treat log_filename as a strftime pattern */ +#ifdef WIN32 + if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) +#else if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) +#endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); return filename; From af41c5de4ada2085d2f33f0bcf416a850cbaffd5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Apr 2019 18:42:34 +0300 Subject: [PATCH 0602/2107] fix windows build --- src/catalog.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 7a662b713..a3180eaff 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -8,12 +8,13 @@ *------------------------------------------------------------------------- */ +#include "pg_probackup.h" + #include #include #include #include -#include "pg_probackup.h" #include "utils/file.h" #include "utils/configuration.h" From a32a744d6bef5e5d80f212dbd1b3a7634c66ec1f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 19:06:49 +0300 Subject: [PATCH 0603/2107] Fix build at Windows --- src/utils/file.c | 7 ++++++- src/utils/remote.c | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 9438fe3f0..43ee86a58 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1,8 +1,13 @@ #include #include -#include #include +#ifdef WIN32 +#define __thread +#else +#include +#endif + #include "pg_probackup.h" #include "file.h" #include "storage/checksum.h" diff --git a/src/utils/remote.c b/src/utils/remote.c index 4228fd244..d56cf36f2 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -1,3 +1,14 @@ +#ifdef WIN32 + +#include "pg_probackup.h" + +bool launch_agent(void) +{ + return false; +} + +#else + #include #include #include @@ -142,3 +153,4 @@ bool launch_agent(void) return true; } +#endif From bf31d7181169f9d39e2abe9d963392832c037595 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 19:28:29 +0300 Subject: [PATCH 0604/2107] Fix build at Windows --- src/utils/file.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/file.c b/src/utils/file.c index 43ee86a58..f373dc629 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -50,6 +50,16 @@ static bool fio_is_remote_fd(int fd) return (fd & FIO_PIPE_MARKER) != 0; } +#ifdef WIN32 +static ssize_t pread(int fd, void* buf, size_t size, off_t off) +{ + off_t rc = lseek(fd, SEEK_SET, off); + if (rc != off) + return -1; + return read(fd, buf, size); +} +#endif + /* Check if specified location is local for current node */ static bool fio_is_remote(fio_location location) { @@ -119,7 +129,13 @@ FILE* fio_open_stream(char const* path, fio_location location) Assert(fio_stdin_buffer == NULL); fio_stdin_buffer = pgut_malloc(hdr.size); IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); +#ifdef WIN32 + f = tmpfile(); + IO_CHECK(fwrite(f, 1, hdr.size, fio_stdin_buffer), hdr.size); + SYS_CHECK(fseek(f, 0, SEEK_SET)); +#else f = fmemopen(fio_stdin_buffer, hdr.size, "r"); +#endif } else { From 98ff7303e70e377263fdc82f6e502de1dfd9b0fe Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 19:34:37 +0300 Subject: [PATCH 0605/2107] Fix build at Windows --- gen_probackup_project.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 88a1a948e..0fe27fd74 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -160,6 +160,8 @@ sub build_pgprobackup $probackup->AddFiles( "$currpath/src/utils", 'configuration.c', + 'file.c', + 'remote.c', 'json.c', 'logger.c', 'parray.c', From 350439cb078a6c1cf95f41d0076e6063084aa95f Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 24 Apr 2019 22:05:58 +0300 Subject: [PATCH 0606/2107] Fix bug in pseek implementation --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index f373dc629..64e7f4cd9 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -53,7 +53,7 @@ static bool fio_is_remote_fd(int fd) #ifdef WIN32 static ssize_t pread(int fd, void* buf, size_t size, off_t off) { - off_t rc = lseek(fd, SEEK_SET, off); + off_t rc = lseek(fd, off, SEEK_SET); if (rc != off) return -1; return read(fd, buf, size); From 6e0d087eb5fdae2672995f45be6c64150e608a99 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Apr 2019 23:39:11 +0300 Subject: [PATCH 0607/2107] tests: minor fixes --- tests/backup_test.py | 6 +++--- tests/restore.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 9081d10b5..e0e12c997 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -384,8 +384,8 @@ def test_page_corruption_heal_via_ptrack_2(self): try: self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", '--log-level-console=LOG']) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -407,7 +407,7 @@ def test_page_corruption_heal_via_ptrack_2(self): repr(e.message), self.cmd)) else: self.assertTrue( - "WARNING: File" in e.message and + "LOG: File" in e.message and "blknum" in e.message and "have wrong checksum" in e.message and "try to fetch via SQL" in e.message and diff --git a/tests/restore.py b/tests/restore.py index 3019d552d..b2146b3c3 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1720,7 +1720,7 @@ def test_restore_backup_from_future(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2'}) + 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 37ae88e9b260f6d607876d6d7454bf52c6d76557 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 25 Apr 2019 12:47:01 +0300 Subject: [PATCH 0608/2107] Use pgFileDelete() within delete_backup_files() --- src/delete.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index 4383489c5..e406aa081 100644 --- a/src/delete.c +++ b/src/delete.c @@ -674,7 +674,7 @@ delete_backup_files(pgBackup *backup) if (interrupted) elog(ERROR, "interrupted during delete backup"); - fio_unlink(file->path, FIO_BACKUP_HOST); + pgFileDelete(file); } parray_walk(files, pgFileFree); From e5c6bb477a644f277805664f0aa9fdff893c5c11 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 12:59:20 +0300 Subject: [PATCH 0609/2107] Fix remove dir under Windows --- src/utils/file.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 64e7f4cd9..cf25b8db6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -58,6 +58,17 @@ static ssize_t pread(int fd, void* buf, size_t size, off_t off) return -1; return read(fd, buf, size); } +static int remove_file_or_dir(char const* path) +{ + int rc = remove(path); +#ifdef WIN32 + if (rc < 0 && errno == EACCESS) + rc = rmdir(path); +#endif + return rc; +} +#else +#define remove_file_or_dir(path) remove(path) #endif /* Check if specified location is local for current node */ @@ -715,7 +726,7 @@ int fio_unlink(char const* path, fio_location location) } else { - return remove(path); + return remove_file_or_dir(path); } } @@ -1343,7 +1354,7 @@ void fio_communicate(int in, int out) SYS_CHECK(symlink(buf, buf + strlen(buf) + 1)); break; case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ - SYS_CHECK(remove(buf)); + SYS_CHECK(remove_file_or_dir(buf)); break; case FIO_MKDIR: /* Create direcory */ hdr.size = 0; From 75447d71f28d13cc81c3dab25d4e807e7bcee646 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Apr 2019 13:29:43 +0300 Subject: [PATCH 0610/2107] tests: minor fix --- tests/retention.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/retention.py b/tests/retention.py index e0429439e..dd84e92ba 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -614,7 +614,8 @@ def test_window_merge_interleaved_incremental_chains_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options=['autovacuum': 'off']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 8ffe2ba59727e2f2de8d9d4953525a342a9df48e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Apr 2019 13:31:40 +0300 Subject: [PATCH 0611/2107] tests: minor fix --- tests/retention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/retention.py b/tests/retention.py index dd84e92ba..322c78f16 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -615,7 +615,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options=['autovacuum': 'off']) + pg_options={'autovacuum':'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From d46179c682c1383272e561b1b3a02e78e07f4365 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 25 Apr 2019 15:47:24 +0300 Subject: [PATCH 0612/2107] Update copyrights --- LICENSE | 2 +- src/archive.c | 2 +- src/backup.c | 2 +- src/catalog.c | 2 +- src/configure.c | 2 +- src/data.c | 2 +- src/delete.c | 2 +- src/dir.c | 2 +- src/help.c | 2 +- src/init.c | 2 +- src/merge.c | 2 +- src/parsexlog.c | 2 +- src/pg_probackup.c | 2 +- src/pg_probackup.h | 2 +- src/restore.c | 2 +- src/show.c | 2 +- src/util.c | 2 +- src/utils/configuration.c | 2 +- src/utils/configuration.h | 2 +- src/utils/json.c | 2 +- src/utils/json.h | 2 +- src/utils/logger.c | 2 +- src/utils/logger.h | 2 +- src/utils/pgut.c | 2 +- src/utils/pgut.h | 2 +- src/utils/remote.h | 2 +- src/utils/thread.c | 2 +- src/utils/thread.h | 2 +- src/validate.c | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/LICENSE b/LICENSE index 3969eaa89..dc4e8b8d5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2018, Postgres Professional +Copyright (c) 2015-2019, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/src/archive.c b/src/archive.c index add41f640..e570a0202 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2018, Postgres Professional + * Portions Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/backup.c b/src/backup.c index 6acbda709..f60c85710 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3,7 +3,7 @@ * backup.c: backup DB cluster, archived WAL * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/catalog.c b/src/catalog.c index a3180eaff..d186980eb 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -3,7 +3,7 @@ * catalog.c: backup catalog operation * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/configure.c b/src/configure.c index ace6fb1ba..4726503e1 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,7 +2,7 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/data.c b/src/data.c index 0c3b86027..d15938901 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/delete.c b/src/delete.c index e406aa081..05aa73d8a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -3,7 +3,7 @@ * delete.c: delete backup files. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/dir.c b/src/dir.c index 781f08ac0..c285ae7e5 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,7 +3,7 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/help.c b/src/help.c index d803b72ca..b52483d63 100644 --- a/src/help.c +++ b/src/help.c @@ -2,7 +2,7 @@ * * help.c * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/init.c b/src/init.c index 4fe1168cb..4e52292aa 100644 --- a/src/init.c +++ b/src/init.c @@ -3,7 +3,7 @@ * init.c: - initialize backup catalog. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/merge.c b/src/merge.c index bf93143f6..4e0d6d80f 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2,7 +2,7 @@ * * merge.c: merge FULL and incremental backups * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/parsexlog.c b/src/parsexlog.c index a2682083a..d05bca459 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,7 +5,7 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5828b5e86..581454dc9 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -3,7 +3,7 @@ * pg_probackup.c: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 85c6a7f39..8eef32671 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/restore.c b/src/restore.c index 82a877314..4c509c3f6 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/show.c b/src/show.c index 083962e55..7fe163f93 100644 --- a/src/show.c +++ b/src/show.c @@ -3,7 +3,7 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/util.c b/src/util.c index 661143c23..a279959c1 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c index d4fb89506..45866846f 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -3,7 +3,7 @@ * configuration.c: - function implementations to work with pg_probackup * configurations. * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 96e200472..46b5d6c1b 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -3,7 +3,7 @@ * configuration.h: - prototypes of functions and structures for * configuration. * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/json.c b/src/utils/json.c index a2608304b..9f13a958f 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -2,7 +2,7 @@ * * json.c: - make json document. * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/json.h b/src/utils/json.h index 0344a5d75..cc9f1168d 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -2,7 +2,7 @@ * * json.h: - prototypes of json output functions. * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.c b/src/utils/logger.c index 784f262f4..e9e4a7445 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -2,7 +2,7 @@ * * logger.c: - log events into log file or stderr. * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/logger.h b/src/utils/logger.h index dba4489e9..37b6ff095 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -2,7 +2,7 @@ * * logger.h: - prototypes of logger functions. * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6b2147ae4..501a89b61 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2018, Postgres Professional + * Portions Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/pgut.h b/src/utils/pgut.h index b527c760d..2fb13cf4c 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2018, Postgres Professional + * Portions Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/remote.h b/src/utils/remote.h index a0783671c..dc98644ab 100644 --- a/src/utils/remote.h +++ b/src/utils/remote.h @@ -2,7 +2,7 @@ * * remote.h: - prototypes of remote functions. * - * Copyright (c) 2017-2018, Postgres Professional + * Copyright (c) 2017-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.c b/src/utils/thread.c index f1624be98..5ceee068d 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -2,7 +2,7 @@ * * thread.c: - multi-platform pthread implementations. * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/utils/thread.h b/src/utils/thread.h index a2948156b..2eaa5fb45 100644 --- a/src/utils/thread.h +++ b/src/utils/thread.h @@ -2,7 +2,7 @@ * * thread.h: - multi-platform pthread implementations. * - * Copyright (c) 2018, Postgres Professional + * Copyright (c) 2018-2019, Postgres Professional * *------------------------------------------------------------------------- */ diff --git a/src/validate.c b/src/validate.c index 646a89cf7..0185a7dbb 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,7 +3,7 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2019, Postgres Professional * *------------------------------------------------------------------------- */ From 766571c979a6d2d299f0b5f1ac5789cedf6ed998 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 25 Apr 2019 17:27:27 +0300 Subject: [PATCH 0613/2107] Use ; as delimiter under Windows, ensure that external directories are canonicalized --- src/dir.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/dir.c b/src/dir.c index c285ae7e5..f71ed91d2 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1710,14 +1710,25 @@ make_external_directory_list(const char *colon_separated_dirs) parray *list = parray_new(); char *tmp = pg_strdup(colon_separated_dirs); - p = strtok(tmp,":"); +#ifndef WIN32 +#define EXTERNAL_DIRECTORY_DELIMITER ":" +#else +#define EXTERNAL_DIRECTORY_DELIMITER ";" +#endif + + p = strtok(tmp, EXTERNAL_DIRECTORY_DELIMITER); while(p!=NULL) { - if (is_absolute_path(p)) - parray_append(list, pg_strdup(p)); + char *external_path = pg_strdup(p); + + canonicalize_path(external_path); + if (is_absolute_path(external_path)) + parray_append(list, external_path); else - elog(ERROR, "External directory \"%s\" is not an absolute path", p); - p=strtok(NULL,":"); + elog(ERROR, "External directory \"%s\" is not an absolute path", + external_path); + + p = strtok(NULL, EXTERNAL_DIRECTORY_DELIMITER); } pfree(tmp); parray_qsort(list, BlackListCompare); From 10b4e1ac9a154c0e795e5c34acff1ddc9c107d42 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 17:36:28 +0300 Subject: [PATCH 0614/2107] Support remote SSH at windows --- src/pg_probackup.c | 4 ++++ src/pg_probackup.h | 5 ----- src/utils/remote.c | 45 +++++++++++++++++++++++++++++---------------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 581454dc9..f845ada40 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -301,7 +301,11 @@ main(int argc, char *argv[]) "Agent version %s doesn't match master pg_probackup version %s", remote_agent, PROGRAM_VERSION); } +#ifdef WIN32 + fio_communicate(atoi(argv[3]), atoi(argv[4])); +#else fio_communicate(STDIN_FILENO, STDOUT_FILENO); +#endif return 0; } else if (strcmp(argv[1], "--help") == 0 || diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8eef32671..31d397c28 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -68,11 +68,6 @@ #define XID_FMT "%u" #endif -#ifndef STDIN_FILENO -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#endif - /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) diff --git a/src/utils/remote.c b/src/utils/remote.c index d56cf36f2..7bb16ce04 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -1,14 +1,3 @@ -#ifdef WIN32 - -#include "pg_probackup.h" - -bool launch_agent(void) -{ - return false; -} - -#else - #include #include #include @@ -22,6 +11,7 @@ bool launch_agent(void) #define MAX_CMDLINE_LENGTH 4096 #define MAX_CMDLINE_OPTIONS 256 #define ERR_BUF_SIZE 4096 +#define PIPE_SIZE (64*1024) static int split_options(int argc, char* argv[], int max_options, char* options) { @@ -111,6 +101,33 @@ bool launch_agent(void) ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; +#ifdef WIN32 + SYS_CHECK(_pipe(infd, PIPE_SIZE, BINARY)) ; + SYS_CHECK(_pipe(outfd, PIPE_SIZE, O_BINARY)); + + if (instance_config.remote.path) + { + char* sep = strrchr(pg_probackup, '\\'); + if (sep != NULL) { + pg_probackup = sep + 1; + } + snprintf(cmd, sizeof(cmd), "%s\\%s agent %s %d %d", + instance_config.remote.path, pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); + } + else + { + snprintf(cmd, sizeof(cmd), "%s agent %s %d %d", + pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); + } + { + intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv) < 0) + if (pid < 0) + return false; + child_pid = GetProcessId(pid); +#else + SYS_CHECK(pipe(infd)); + SYS_CHECK(pipe(outfd)); + if (instance_config.remote.path) { char* sep = strrchr(pg_probackup, '/'); @@ -123,9 +140,6 @@ bool launch_agent(void) snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } - SYS_CHECK(pipe(infd)); - SYS_CHECK(pipe(outfd)); - SYS_CHECK(child_pid = fork()); if (child_pid == 0) { /* child */ @@ -143,6 +157,7 @@ bool launch_agent(void) if (execvp(ssh_argv[0], ssh_argv) < 0) return false; } else { +#endif elog(LOG, "Spawn agent %d version %s", child_pid, PROGRAM_VERSION); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); @@ -152,5 +167,3 @@ bool launch_agent(void) } return true; } - -#endif From 2b95e621c42d6b861131cfcfe3f3dd281f239621 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Apr 2019 17:43:20 +0300 Subject: [PATCH 0615/2107] tests: added external.ExternalTest.test_restore_external_dir_is_empty --- tests/external.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/external.py b/tests/external.py index d22a78923..2e906c763 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1472,3 +1472,76 @@ def test_restore_external_dir_not_empty(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_restore_external_dir_is_empty(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup + restore page backup, check that restored + external directory is empty + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + # drop external directory + shutil.rmtree(external_dir, ignore_errors=True) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path(node_restored, 'external_dir_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 87fab3c16a9221410d563089383ca3e41e6a7fb9 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 18:01:35 +0300 Subject: [PATCH 0616/2107] Support remote SSH at windows --- src/utils/remote.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 7bb16ce04..1a5186692 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -102,7 +102,7 @@ bool launch_agent(void) ssh_argv[ssh_argc] = NULL; #ifdef WIN32 - SYS_CHECK(_pipe(infd, PIPE_SIZE, BINARY)) ; + SYS_CHECK(_pipe(infd, PIPE_SIZE, O_BINARY)) ; SYS_CHECK(_pipe(outfd, PIPE_SIZE, O_BINARY)); if (instance_config.remote.path) @@ -120,7 +120,7 @@ bool launch_agent(void) pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); } { - intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv) < 0) + intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); if (pid < 0) return false; child_pid = GetProcessId(pid); From f466a04a309d25446bcf2a6c641e9ac396834874 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 18:04:39 +0300 Subject: [PATCH 0617/2107] Support remote SSH at windows --- src/utils/remote.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 1a5186692..fb3210c99 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -110,8 +110,13 @@ bool launch_agent(void) char* sep = strrchr(pg_probackup, '\\'); if (sep != NULL) { pg_probackup = sep + 1; + } else { + sep = strrchr(pg_probackup, '/'); + if (sep != NULL) { + pg_probackup = sep + 1; + } } - snprintf(cmd, sizeof(cmd), "%s\\%s agent %s %d %d", + snprintf(cmd, sizeof(cmd), "%s/%s agent %s %d %d", instance_config.remote.path, pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); } else @@ -123,7 +128,7 @@ bool launch_agent(void) intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); if (pid < 0) return false; - child_pid = GetProcessId(pid); + child_pid = GetProcessId((HANDLE)pid); #else SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); From b95710c4b2e986803e2e45f75b5bcd0aa3bf3959 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 18:56:52 +0300 Subject: [PATCH 0618/2107] Rewrite remote ssh support for Windows --- src/pg_probackup.c | 8 +++---- src/pg_probackup.h | 11 ++++++++- src/utils/remote.c | 56 +++++++++++++++++++++------------------------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f845ada40..339c282fa 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -291,6 +291,10 @@ main(int argc, char *argv[]) backup_subcmd = SHOW_CONFIG_CMD; else if (strcmp(argv[1], "checkdb") == 0) backup_subcmd = CHECKDB_CMD; +#ifdef WIN32 + else if (strcmp(argv[1], "ssh") == 0) + launch_ssh(argv); +#endif else if (strcmp(argv[1], "agent") == 0 && argc > 2) { remote_agent = argv[2]; @@ -301,11 +305,7 @@ main(int argc, char *argv[]) "Agent version %s doesn't match master pg_probackup version %s", remote_agent, PROGRAM_VERSION); } -#ifdef WIN32 - fio_communicate(atoi(argv[3]), atoi(argv[4])); -#else fio_communicate(STDIN_FILENO, STDOUT_FILENO); -#endif return 0; } else if (strcmp(argv[1], "--help") == 0 || diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 31d397c28..6b69b0014 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2018, Postgres Professional * *------------------------------------------------------------------------- */ @@ -68,6 +68,11 @@ #define XID_FMT "%u" #endif +#ifndef STDIN_FILENO +#define STDIN_FILENO 0 +#define STDOUT_FILENO 1 +#endif + /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) @@ -440,6 +445,9 @@ extern bool heapallindexed; /* current settings */ extern pgBackup current; +/* argv of the process */ +extern char** commands_args; + /* in dir.c */ /* exclude directory list for $PGDATA file listing */ extern const char *pgdata_exclude_dir[]; @@ -549,6 +557,7 @@ extern bool is_prolific(parray *backup_list, pgBackup *target_backup); extern bool in_backup_list(parray *backup_list, pgBackup *target_backup); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); extern bool launch_agent(void); +extern void launch_ssh(char* argv[]); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS #define COMPRESS_LEVEL_DEFAULT 1 diff --git a/src/utils/remote.c b/src/utils/remote.c index fb3210c99..4d5a9059f 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -60,6 +60,16 @@ static void kill_child(void) } #endif +#ifdef WIN32 +void launch_ssh(char* argv[]) +{ + SYS_CHECK(dup2(atoi(argv[2]), 0)); + SYS_CHECK(dup2(atoi(argv[3]), 1)); + SYS_CHECK(execvp(ssh_argv[4], ssh_argv+4)); +} +#endif + + bool launch_agent(void) { char cmd[MAX_CMDLINE_LENGTH]; @@ -70,6 +80,10 @@ bool launch_agent(void) ssh_argc = 0; ssh_argv[ssh_argc++] = instance_config.remote.proto; +#ifdef WIN32 + ssh_argv[ssh_argc] = "ssh"; + ssh_argc += 3; +#endif if (instance_config.remote.port != NULL) { ssh_argv[ssh_argc++] = "-p"; ssh_argv[ssh_argc++] = instance_config.remote.port; @@ -101,31 +115,25 @@ bool launch_agent(void) ssh_argv[ssh_argc++] = cmd; ssh_argv[ssh_argc] = NULL; -#ifdef WIN32 - SYS_CHECK(_pipe(infd, PIPE_SIZE, O_BINARY)) ; - SYS_CHECK(_pipe(outfd, PIPE_SIZE, O_BINARY)); - if (instance_config.remote.path) { - char* sep = strrchr(pg_probackup, '\\'); + char* sep = strrchr(pg_probackup, '/'); if (sep != NULL) { pg_probackup = sep + 1; - } else { - sep = strrchr(pg_probackup, '/'); - if (sep != NULL) { - pg_probackup = sep + 1; - } } - snprintf(cmd, sizeof(cmd), "%s/%s agent %s %d %d", - instance_config.remote.path, pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); - } - else - { - snprintf(cmd, sizeof(cmd), "%s agent %s %d %d", - pg_probackup, PROGRAM_VERSION, outfd[0], infd[1]); + snprintf(cmd, sizeof(cmd), "%s/%s agent %s", + instance_config.remote.path, pg_probackup, PROGRAM_VERSION); + } else { + snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } + +#ifdef WIN32 + SYS_CHECK(_pipe(infd, PIPE_SIZE, O_BINARY)) ; + SYS_CHECK(_pipe(outfd, PIPE_SIZE, O_BINARY)); + ssh_argv[2] = psprintf("%d", outfd[0]); + ssh_argv[3] = psprintf("%d", infd[1]); { - intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); + intptr_t pid = _spawnvp(_P_NOWAIT, pg_probackup, ssh_argv); if (pid < 0) return false; child_pid = GetProcessId((HANDLE)pid); @@ -133,18 +141,6 @@ bool launch_agent(void) SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); - if (instance_config.remote.path) - { - char* sep = strrchr(pg_probackup, '/'); - if (sep != NULL) { - pg_probackup = sep + 1; - } - snprintf(cmd, sizeof(cmd), "%s/%s agent %s", - instance_config.remote.path, pg_probackup, PROGRAM_VERSION); - } else { - snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); - } - SYS_CHECK(child_pid = fork()); if (child_pid == 0) { /* child */ From be52f7d2067583f14b59550c9e5682170ecaacad Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 19:04:34 +0300 Subject: [PATCH 0619/2107] Rewrite remote ssh support for Windows --- src/utils/remote.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 4d5a9059f..1e028c93f 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -65,7 +65,7 @@ void launch_ssh(char* argv[]) { SYS_CHECK(dup2(atoi(argv[2]), 0)); SYS_CHECK(dup2(atoi(argv[3]), 1)); - SYS_CHECK(execvp(ssh_argv[4], ssh_argv+4)); + SYS_CHECK(execvp(argv[4], argv+4)); } #endif @@ -79,11 +79,12 @@ bool launch_agent(void) int infd[2]; ssh_argc = 0; - ssh_argv[ssh_argc++] = instance_config.remote.proto; #ifdef WIN32 - ssh_argv[ssh_argc] = "ssh"; - ssh_argc += 3; + ssh_argv[ssh_argc++] = pg_probackup; + ssh_argv[ssh_argc++] = "ssh"; + ssh_argc += 2; /* reserve space for pipe descriptors */ #endif + ssh_argv[ssh_argc++] = instance_config.remote.proto; if (instance_config.remote.port != NULL) { ssh_argv[ssh_argc++] = "-p"; ssh_argv[ssh_argc++] = instance_config.remote.port; @@ -133,7 +134,7 @@ bool launch_agent(void) ssh_argv[2] = psprintf("%d", outfd[0]); ssh_argv[3] = psprintf("%d", infd[1]); { - intptr_t pid = _spawnvp(_P_NOWAIT, pg_probackup, ssh_argv); + intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); if (pid < 0) return false; child_pid = GetProcessId((HANDLE)pid); From 576ffcee8d43813eec250d86a349b8c01c1aabaa Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 19:10:57 +0300 Subject: [PATCH 0620/2107] Rewrite remote ssh support for Windows --- src/utils/remote.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 1e028c93f..034b58958 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -118,12 +118,21 @@ bool launch_agent(void) if (instance_config.remote.path) { - char* sep = strrchr(pg_probackup, '/'); + char const* probackup = pg_probackup; + char* sep = strrchr(probackup, '/'); if (sep != NULL) { - pg_probackup = sep + 1; + probackup = sep + 1; } +#ifdef WIN32 + else { + sep = strrchr(probackup, '\\'); + if (sep != NULL) { + probackup = sep + 1; + } + } +#endif snprintf(cmd, sizeof(cmd), "%s/%s agent %s", - instance_config.remote.path, pg_probackup, PROGRAM_VERSION); + instance_config.remote.path, probackup, PROGRAM_VERSION); } else { snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } From cc66172785869e1e255f68a245de207e879cec2a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Apr 2019 19:14:15 +0300 Subject: [PATCH 0621/2107] minor bugfix: do not check external directories if their restore is not requested --- src/restore.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 4c509c3f6..f50da01c0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -288,7 +288,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore) { check_tablespace_mapping(dest_backup); - check_external_dir_mapping(dest_backup); + + /* no point in checking external directories if their restore is not resquested */ + if (!skip_external_dirs) + check_external_dir_mapping(dest_backup); } /* At this point we are sure that parent chain is whole From 8aba8e8940414cb36385bc3801a88905eb9666f0 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 19:31:15 +0300 Subject: [PATCH 0622/2107] Rewrite remote ssh support for Windows --- src/utils/remote.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 034b58958..8a5725e6f 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -63,8 +63,12 @@ static void kill_child(void) #ifdef WIN32 void launch_ssh(char* argv[]) { - SYS_CHECK(dup2(atoi(argv[2]), 0)); - SYS_CHECK(dup2(atoi(argv[3]), 1)); + SYS_CHECK(close(STDIN_FILENO)); + SYS_CHECK(close(STDOUT_FILENO)); + + SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); + SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + SYS_CHECK(execvp(argv[4], argv+4)); } #endif From 66d33a188e85b54810aa4a251af1b3edbd284fed Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 19:34:11 +0300 Subject: [PATCH 0623/2107] Rewrite remote ssh support for Windows --- src/utils/remote.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 8a5725e6f..139d4e4ad 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -63,11 +63,14 @@ static void kill_child(void) #ifdef WIN32 void launch_ssh(char* argv[]) { + int infd = atoi(argv[2]); + int outfd = atoi(argv[3]); + SYS_CHECK(close(STDIN_FILENO)); SYS_CHECK(close(STDOUT_FILENO)); - SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); - SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + SYS_CHECK(dup2(infd, STDIN_FILENO)); + SYS_CHECK(dup2(outfd, STDOUT_FILENO)); SYS_CHECK(execvp(argv[4], argv+4)); } From 982cf6ac63b274edb18515006d00e8ac9a45165c Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Apr 2019 19:53:35 +0300 Subject: [PATCH 0624/2107] Fix error in forming remote path for Windows --- src/pg_probackup.c | 1 + src/utils/remote.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 339c282fa..3946f0c94 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -298,6 +298,7 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "agent") == 0 && argc > 2) { remote_agent = argv[2]; + sleep(10); if (strcmp(remote_agent, PROGRAM_VERSION) != 0) { uint32 agent_version = parse_program_version(remote_agent); diff --git a/src/utils/remote.c b/src/utils/remote.c index 139d4e4ad..ab27bf712 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -137,9 +137,12 @@ bool launch_agent(void) probackup = sep + 1; } } -#endif + snprintf(cmd, sizeof(cmd), "%s\\%s agent %s", + instance_config.remote.path, probackup, PROGRAM_VERSION); +#else snprintf(cmd, sizeof(cmd), "%s/%s agent %s", instance_config.remote.path, probackup, PROGRAM_VERSION); +#endif } else { snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } From d489cc700e5a63ad5c3f320704b4eb613b741ae9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 26 Apr 2019 01:16:42 +0300 Subject: [PATCH 0625/2107] tests: rename external.ExternalTest.test_restore_external_dir_is_empty to external.ExternalTest.test_restore_external_dir_is_missing --- tests/external.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/external.py b/tests/external.py index 2e906c763..f07379be5 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1473,10 +1473,10 @@ def test_restore_external_dir_not_empty(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_restore_external_dir_is_empty(self): + def test_restore_external_dir_is_missing(self): """ take FULL backup with not empty external directory - drop external directory content + drop external directory take DELTA backup restore page backup, check that restored external directory is empty From 32d283f712a607b7c081814f4faf9581e78babbc Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 26 Apr 2019 11:33:34 +0300 Subject: [PATCH 0626/2107] Check for existance of an external directory --- src/backup.c | 3 ++- src/dir.c | 17 ++++++++++++----- src/restore.c | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index f60c85710..1a06af87b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -888,7 +888,8 @@ do_backup_instance(void) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0, + FIO_BACKUP_HOST); for (i = 0; i < parray_num(xlog_files_list); i++) { diff --git a/src/dir.c b/src/dir.c index f71ed91d2..991fb3add 100644 --- a/src/dir.c +++ b/src/dir.c @@ -163,13 +163,14 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink, int external_dir_num, fio_location location) +pgFileNew(const char *path, bool omit_symlink, int external_dir_num, + fio_location location) { struct stat st; pgFile *file; - /* stat the file */ - if (fio_stat(path, &st, omit_symlink, location) < 0) + /* stat the file */ + if (fio_stat(path, &st, omit_symlink, location) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -475,11 +476,17 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, file = pgFileNew(root, omit_symlink, external_dir_num, location); if (file == NULL) - return; + { + /* For external directory this is not ok */ + if (external_dir_num > 0) + elog(ERROR, "Exteral directory is not found: %s", root); + else + return; + } if (!S_ISDIR(file->mode)) { - if (external_dir_num) + if (external_dir_num > 0) elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", file->path); else diff --git a/src/restore.c b/src/restore.c index f50da01c0..6af43c7dd 100644 --- a/src/restore.c +++ b/src/restore.c @@ -640,7 +640,8 @@ remove_deleted_files(pgBackup *backup) /* Get list of files actually existing in target database */ files_restored = parray_new(); - dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0, FIO_BACKUP_HOST); + dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0, + FIO_BACKUP_HOST); /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); From 977ef811c18be996ffa115e0fd2fd9ef05174dc5 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 26 Apr 2019 11:36:56 +0300 Subject: [PATCH 0627/2107] Remove debug sleep --- src/pg_probackup.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3946f0c94..339c282fa 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -298,7 +298,6 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "agent") == 0 && argc > 2) { remote_agent = argv[2]; - sleep(10); if (strcmp(remote_agent, PROGRAM_VERSION) != 0) { uint32 agent_version = parse_program_version(remote_agent); From 26cb867d0f37638285f6d3ea71b8f89691c995cf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 26 Apr 2019 14:13:46 +0300 Subject: [PATCH 0628/2107] fix typo --- src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 991fb3add..9ede00ebd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -479,7 +479,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, { /* For external directory this is not ok */ if (external_dir_num > 0) - elog(ERROR, "Exteral directory is not found: %s", root); + elog(ERROR, "External directory is not found: %s", root); else return; } From 6974eb89580f1aadd7b3dca39db40c288053b682 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 26 Apr 2019 17:37:29 +0300 Subject: [PATCH 0629/2107] Remove external directories files from previous backups --- src/backup.c | 3 +- src/catalog.c | 3 +- src/dir.c | 21 +++++- src/merge.c | 6 +- src/pg_probackup.h | 3 +- src/restore.c | 162 ++++++++++++++++++++++++++++++++------------- tests/external.py | 4 +- 7 files changed, 145 insertions(+), 57 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1a06af87b..f62dd0646 100644 --- a/src/backup.c +++ b/src/backup.c @@ -504,7 +504,8 @@ do_backup_instance(void) elog(LOG, "Database backup start"); if(current.external_dir_str) { - external_dirs = make_external_directory_list(current.external_dir_str); + external_dirs = make_external_directory_list(current.external_dir_str, + false); check_external_for_tablespaces(external_dirs); } diff --git a/src/catalog.c b/src/catalog.c index d186980eb..363c49efc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -477,7 +477,8 @@ pgBackupCreateDir(pgBackup *backup) { parray *external_list; - external_list = make_external_directory_list(backup->external_dir_str); + external_list = make_external_directory_list(backup->external_dir_str, + false); for (i = 0; i < parray_num(external_list); i++) { char temp[MAXPGPATH]; diff --git a/src/dir.c b/src/dir.c index 9ede00ebd..8e0d96d9f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1311,7 +1311,9 @@ check_external_dir_mapping(pgBackup *backup) return; } - external_dirs_to_restore = make_external_directory_list(backup->external_dir_str); + external_dirs_to_restore = make_external_directory_list( + backup->external_dir_str, + false); /* 1 - each OLDDIR must have an entry in external_dirs_to_restore */ for (cell = external_remap_list.head; cell; cell = cell->next) { @@ -1707,11 +1709,11 @@ pgFileSize(const char *path) } /* - * Construct parray containing external directories paths + * Construct parray containing remmaped external directories paths * from string like /path1:/path2 */ parray * -make_external_directory_list(const char *colon_separated_dirs) +make_external_directory_list(const char *colon_separated_dirs, bool remap) { char *p; parray *list = parray_new(); @@ -1730,7 +1732,20 @@ make_external_directory_list(const char *colon_separated_dirs) canonicalize_path(external_path); if (is_absolute_path(external_path)) + { + if (remap) + { + char *full_path = get_external_remap(external_path); + + if (full_path != external_path) + { + full_path = pg_strdup(full_path); + pfree(external_path); + external_path = full_path; + } + } parray_append(list, external_path); + } else elog(ERROR, "External directory \"%s\" is not an absolute path", external_path); diff --git a/src/merge.c b/src/merge.c index 4e0d6d80f..1f87eb202 100644 --- a/src/merge.c +++ b/src/merge.c @@ -262,9 +262,11 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* Create external directories lists */ if (to_backup->external_dir_str) - to_external = make_external_directory_list(to_backup->external_dir_str); + to_external = make_external_directory_list(to_backup->external_dir_str, + false); if (from_backup->external_dir_str) - from_external = make_external_directory_list(from_backup->external_dir_str); + from_external = make_external_directory_list(from_backup->external_dir_str, + false); /* * Rename external directoties in to_backup (if exists) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6b69b0014..6e4a9137a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -585,7 +585,8 @@ extern void print_file_list(FILE *out, const parray *files, const char *root, const char *external_prefix, parray *external_list); extern parray *dir_read_file_list(const char *root, const char *external_prefix, const char *file_txt, fio_location location); -extern parray *make_external_directory_list(const char *colon_separated_dirs); +extern parray *make_external_directory_list(const char *colon_separated_dirs, + bool remap); extern void free_dir_list(parray *list); extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); diff --git a/src/restore.c b/src/restore.c index 6af43c7dd..e86779a9a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,8 +21,8 @@ typedef struct { parray *files; pgBackup *backup; - parray *req_external_dirs; - parray *cur_external_dirs; + parray *dest_external_dirs; + parray *external_dirs; char *external_prefix; /* @@ -32,13 +32,14 @@ typedef struct int ret; } restore_files_arg; -static void restore_backup(pgBackup *backup, const char *external_dir_str); +static void restore_backup(pgBackup *backup, const char *external_dir_str, + parray **external_dirs); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); -static void remove_deleted_files(pgBackup *backup); +static void remove_deleted_files(pgBackup *backup, parray *external_dirs); /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. @@ -421,6 +422,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (is_restore) { + parray *dest_external_dirs = NULL; + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -438,7 +441,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (rt->no_validate && !lock_backup(backup)) elog(ERROR, "Cannot lock backup directory"); - restore_backup(backup, dest_backup->external_dir_str); + restore_backup(backup, dest_backup->external_dir_str, + &dest_external_dirs); } /* @@ -446,7 +450,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * deleted between previous and current backup are not in the list. */ if (dest_backup->backup_mode != BACKUP_MODE_FULL) - remove_deleted_files(dest_backup); + remove_deleted_files(dest_backup, dest_external_dirs); + + if (dest_external_dirs != NULL) + free_dir_list(dest_external_dirs); /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup); @@ -466,7 +473,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup, const char *external_dir_str) +restore_backup(pgBackup *backup, const char *external_dir_str, + parray **external_dirs) { char timestamp[100]; char this_backup_path[MAXPGPATH]; @@ -474,15 +482,13 @@ restore_backup(pgBackup *backup, const char *external_dir_str) char external_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *requested_external_dirs = NULL; - parray *current_external_dirs = NULL; + parray *backup_external_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; restore_files_arg *threads_args; bool restore_isok = true; - if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) elog(ERROR, "Backup %s cannot be restored because it is not valid", @@ -506,21 +512,26 @@ restore_backup(pgBackup *backup, const char *external_dir_str) * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id */ pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - create_data_directories(instance_config.pgdata, this_backup_path, true, FIO_DB_HOST); + create_data_directories(instance_config.pgdata, this_backup_path, true, + FIO_DB_HOST); - if(external_dir_str && !skip_external_dirs) + if (external_dir_str && !skip_external_dirs && + /* Make external directories only once */ + *external_dirs == NULL) { - requested_external_dirs = make_external_directory_list(external_dir_str); - for (i = 0; i < parray_num(requested_external_dirs); i++) + *external_dirs = make_external_directory_list(external_dir_str, true); + for (i = 0; i < parray_num(*external_dirs); i++) { - char *external_path = parray_get(requested_external_dirs, i); - external_path = get_external_remap(external_path); + char *external_path = parray_get(*external_dirs, i); + fio_mkdir(external_path, DIR_PERMISSION, FIO_DB_HOST); } } - if(backup->external_dir_str) - current_external_dirs = make_external_directory_list(backup->external_dir_str); + if (backup->external_dir_str) + backup_external_dirs = make_external_directory_list( + backup->external_dir_str, + true); /* * Get list of files which need to be restored. @@ -529,7 +540,8 @@ restore_backup(pgBackup *backup, const char *external_dir_str) pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, external_prefix, list_path, FIO_BACKUP_HOST); + files = dir_read_file_list(database_path, external_prefix, list_path, + FIO_BACKUP_HOST); /* Restore directories in do_backup_instance way */ parray_qsort(files, pgFileComparePath); @@ -540,28 +552,30 @@ restore_backup(pgBackup *backup, const char *external_dir_str) */ for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(files, i); - /* If the entry was an external directory, create it in the backup */ - if (file->external_dir_num && S_ISDIR(file->mode)) + /* + * If the entry was an external directory, create it in the backup. + */ + if (!skip_external_dirs && + file->external_dir_num && S_ISDIR(file->mode)) { - char dirpath[MAXPGPATH]; - char *dir_name; char *external_path; - if (!current_external_dirs || - parray_num(current_external_dirs) < file->external_dir_num - 1) + if (!backup_external_dirs || + parray_num(backup_external_dirs) < file->external_dir_num - 1) elog(ERROR, "Inconsistent external directory backup metadata"); - external_path = parray_get(current_external_dirs, - file->external_dir_num - 1); - if (backup_contains_external(external_path, requested_external_dirs)) + external_path = parray_get(backup_external_dirs, + file->external_dir_num - 1); + if (backup_contains_external(external_path, *external_dirs)) { char container_dir[MAXPGPATH]; + char dirpath[MAXPGPATH]; + char *dir_name; - external_path = get_external_remap(external_path); makeExternalDirPathByNum(container_dir, external_prefix, - file->external_dir_num); + file->external_dir_num); dir_name = GetRelativePath(file->path, container_dir); elog(VERBOSE, "Create directory \"%s\"", dir_name); join_path_components(dirpath, external_path, dir_name); @@ -573,7 +587,8 @@ restore_backup(pgBackup *backup, const char *external_dir_str) pg_atomic_clear_flag(&file->lock); } threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg)*num_threads); + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * + num_threads); /* Restore files into target directory */ thread_interrupted = false; @@ -583,8 +598,8 @@ restore_backup(pgBackup *backup, const char *external_dir_str) arg->files = files; arg->backup = backup; - arg->req_external_dirs = requested_external_dirs; - arg->cur_external_dirs = current_external_dirs; + arg->dest_external_dirs = *external_dirs; + arg->external_dirs = backup_external_dirs; arg->external_prefix = external_prefix; /* By default there are some error */ threads_args[i].ret = 1; @@ -612,6 +627,9 @@ restore_backup(pgBackup *backup, const char *external_dir_str) parray_walk(files, pgFileFree); parray_free(files); + if (backup_external_dirs != NULL) + free_dir_list(backup_external_dirs); + if (logger_config.log_level_console <= LOG || logger_config.log_level_file <= LOG) elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); @@ -624,7 +642,7 @@ restore_backup(pgBackup *backup, const char *external_dir_str) * are not in the backup's filelist. */ static void -remove_deleted_files(pgBackup *backup) +remove_deleted_files(pgBackup *backup, parray *external_dirs) { parray *files; parray *files_restored; @@ -632,16 +650,61 @@ remove_deleted_files(pgBackup *backup) char external_prefix[MAXPGPATH]; int i; - pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), DATABASE_FILE_LIST); - pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); + pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), + DATABASE_FILE_LIST); + pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), + EXTERNAL_DIR); /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, external_prefix, filelist_path, FIO_BACKUP_HOST); + files = dir_read_file_list(instance_config.pgdata, external_prefix, + filelist_path, FIO_BACKUP_HOST); + /* Replace external prefix with full path */ + if (external_dirs) + { + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if (file->external_dir_num > 0) + { + char *external_path = parray_get(external_dirs, + file->external_dir_num - 1); + char container_dir[MAXPGPATH]; + char *rel_path; + char new_path[MAXPGPATH]; + + makeExternalDirPathByNum(container_dir, external_prefix, + file->external_dir_num); + rel_path = GetRelativePath(file->path, container_dir); + join_path_components(new_path, external_path, rel_path); + + pfree(file->path); + file->path = pg_strdup(new_path); + } + } + } parray_qsort(files, pgFileComparePathDesc); /* Get list of files actually existing in target database */ files_restored = parray_new(); dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0, - FIO_BACKUP_HOST); + FIO_DB_HOST); + if (external_dirs) + for (i = 0; i < parray_num(external_dirs); i++) + { + parray *external_files; + + /* + * external_dirs paths already remmaped. + */ + + external_files = parray_new(); + dir_list_file(external_files, (char *) parray_get(external_dirs, i), + true, true, false, 0, FIO_DB_HOST); + if (parray_num(external_files) > 0) + parray_concat(files_restored, external_files); + + parray_free(external_files); + } /* To delete from leaf, sort in reversed order */ parray_qsort(files_restored, pgFileComparePathDesc); @@ -715,14 +778,21 @@ restore_files(void *arg) /* Directories were created before */ if (S_ISDIR(file->mode)) { - elog(VERBOSE, "directory, skip"); + elog(VERBOSE, "Directory, skip"); continue; } /* Do not restore tablespace_map file */ if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) { - elog(VERBOSE, "skip tablespace_map"); + elog(VERBOSE, "Skip tablespace_map"); + continue; + } + + /* Do no restore external directory file if a user doesn't want */ + if (skip_external_dirs && file->external_dir_num > 0) + { + elog(VERBOSE, "Skip external directory file"); continue; } @@ -747,14 +817,12 @@ restore_files(void *arg) } else if (file->external_dir_num) { - char *external_path = parray_get(arguments->cur_external_dirs, + char *external_path = parray_get(arguments->external_dirs, file->external_dir_num - 1); if (backup_contains_external(external_path, - arguments->req_external_dirs)) - { - external_path = get_external_remap(external_path); - copy_file(arguments->external_prefix, FIO_BACKUP_HOST, external_path, FIO_DB_HOST, file); - } + arguments->dest_external_dirs)) + copy_file(arguments->external_prefix, FIO_BACKUP_HOST, + external_path, FIO_DB_HOST, file); } else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, diff --git a/tests/external.py b/tests/external.py index f07379be5..8514fdd16 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1513,8 +1513,8 @@ def test_restore_external_dir_is_missing(self): "-E", "{0}".format( external_dir)]) - # drop external directory - shutil.rmtree(external_dir, ignore_errors=True) + # drop external file only + os.remove(os.path.join(external_dir, 'file')) self.backup_node( backup_dir, 'node', node, From e51398e25610a0ea65781c146b1c7df467d15686 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Apr 2019 01:04:21 +0300 Subject: [PATCH 0630/2107] add double quotes to elog message about missing external directory --- src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 8e0d96d9f..1a27c84a2 100644 --- a/src/dir.c +++ b/src/dir.c @@ -479,7 +479,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, { /* For external directory this is not ok */ if (external_dir_num > 0) - elog(ERROR, "External directory is not found: %s", root); + elog(ERROR, "External directory is not found: \"%s\"", root); else return; } From 10229f710f3aa223e840e0cbc82a79e39aced2f0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Apr 2019 01:06:15 +0300 Subject: [PATCH 0631/2107] tests: fix test_restore_external_dir_is_missing and test_restore_external_dir_is_empty and add test_restore_external_dir_string_order --- tests/external.py | 222 +++++++++++++++++++++++++++++--- tests/helpers/ptrack_helpers.py | 2 + 2 files changed, 205 insertions(+), 19 deletions(-) diff --git a/tests/external.py b/tests/external.py index 8514fdd16..eb2f11405 100644 --- a/tests/external.py +++ b/tests/external.py @@ -227,7 +227,10 @@ def test_external_dir_mapping(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format(external_dir1_old, external_dir2_old)]) + "-E", "{0}{1}{2}".format( + external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2_old)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -383,8 +386,9 @@ def test_external_backward_compatibility(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgdata = self.pgdata_content( @@ -472,8 +476,9 @@ def test_external_backward_compatibility_merge_1(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgdata = self.pgdata_content( @@ -565,8 +570,9 @@ def test_external_backward_compatibility_merge_2(self): backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgbench = node.pgbench(options=['-T', '30', '-c', '1']) @@ -592,8 +598,9 @@ def test_external_backward_compatibility_merge_2(self): backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgdata = self.pgdata_content( @@ -687,8 +694,9 @@ def test_external_merge(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) self.merge_backup(backup_dir, 'node', backup_id=backup_id) @@ -767,8 +775,9 @@ def test_external_merge_1(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) # drop old external data @@ -791,8 +800,9 @@ def test_external_merge_1(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgdata = self.pgdata_content( @@ -877,8 +887,9 @@ def test_external_merge_single(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) self.merge_backup(backup_dir, 'node', backup_id=backup_id) @@ -959,8 +970,9 @@ def test_external_merge_double(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) # delta backup with external directories @@ -968,8 +980,9 @@ def test_external_merge_double(self): backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1_old, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2_old)]) pgdata = self.pgdata_content( @@ -1049,8 +1062,9 @@ def test_restore_skip_external(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}:{1}".format( + "-E", "{0}{1}{2}".format( external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) # delete first externals, so pgdata_compare @@ -1476,9 +1490,95 @@ def test_restore_external_dir_not_empty(self): def test_restore_external_dir_is_missing(self): """ take FULL backup with not empty external directory - drop external directory - take DELTA backup - restore page backup, check that restored + delete external directory + take DELTA backup with external directory, which + should fail + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + + # drop external directory + shutil.rmtree(external_dir, ignore_errors=True) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}".format( + external_dir)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: External directory is not found: "{0}"'.format(external_dir) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + # take DELTA without external directories + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_restore_external_dir_is_empty(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored external directory is empty """ fname = self.id().split('.')[3] @@ -1513,9 +1613,10 @@ def test_restore_external_dir_is_missing(self): "-E", "{0}".format( external_dir)]) - # drop external file only + # make external directory empty os.remove(os.path.join(external_dir, 'file')) + # take DELTA backup with empty external directory self.backup_node( backup_dir, 'node', node, backup_type='delta', @@ -1530,14 +1631,16 @@ def test_restore_external_dir_is_missing(self): # Restore Delta backup node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() - external_dir_new = self.get_tblspace_path(node_restored, 'external_dir_new') + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') self.restore_node( backup_dir, 'node', node_restored, - options=["--external-mapping={0}={1}".format( - external_dir, external_dir_new)]) + options=['--external-mapping={0}={1}'.format( + external_dir, external_dir_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -1545,3 +1648,84 @@ def test_restore_external_dir_is_missing(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_restore_external_dir_string_order(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored + external directory is empty + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_senders': '2', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + # create empty file in external directory + os.mkdir(external_dir_1) + with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: + f.close() + + os.mkdir(external_dir_2) + with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_2)]) + + with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: + f.close() + + with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: + f.close() + + # take DELTA backup and swap external_dir_2 and external_dir_1 + # in external_dir_str + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_1)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e7d89156f..03fa81e16 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -245,10 +245,12 @@ def __init__(self, *args, **kwargs): exit(1) if os.name == 'posix': + self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( self.probackup_path) + ':' + os.environ['PATH'] elif os.name == 'nt': + self.EXTERNAL_DIRECTORY_DELIMITER = ';' os.environ['PATH'] = os.path.dirname( self.probackup_path) + ';' + os.environ['PATH'] From 16fff38156a430fd871285c9fdca6fd85369e6af Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Apr 2019 23:28:35 +0300 Subject: [PATCH 0632/2107] tests: fixes to "external" and "merge" modules --- tests/delta.py | 6 +- tests/external.py | 1013 +++++++++++++++++++++++++++++++++------------ 2 files changed, 754 insertions(+), 265 deletions(-) diff --git a/tests/delta.py b/tests/delta.py index a56652390..3c146d975 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1196,8 +1196,8 @@ def test_page_corruption_heal_via_ptrack_2(self): try: self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream", "--log-level-console=LOG"]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -1208,7 +1208,7 @@ def test_page_corruption_heal_via_ptrack_2(self): except ProbackupException as e: if self.remote: self.assertTrue( - "WARNING: File" in e.message and + "LOG: File" in e.message and "try to fetch via SQL" in e.message and "WARNING: page verification failed, " "calculated checksum" in e.message and diff --git a/tests/external.py b/tests/external.py index eb2f11405..bb9945ccf 100644 --- a/tests/external.py +++ b/tests/external.py @@ -25,9 +25,7 @@ def test_external_simple(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={ - 'max_wal_senders': '2'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') @@ -81,13 +79,14 @@ def test_external_simple(self): node.base_dir, exclude_dirs=['logs']) node.cleanup() - shutil.rmtree(external_dir, ignore_errors=True) + shutil.rmtree(node.base_dir, ignore_errors=True) self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -106,9 +105,7 @@ def test_external_none(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={ - 'max_wal_senders': '2'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') @@ -137,14 +134,15 @@ def test_external_none(self): # Delta backup without external directory self.backup_node( - backup_dir, 'node', node, backup_type="delta") -# options=['--external-dirs=none']) + backup_dir, 'node', node, backup_type="delta", + options=['--external-dirs=none']) shutil.rmtree(external_dir, ignore_errors=True) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) self.restore_node( backup_dir, 'node', node, options=["-j", "4"]) @@ -168,9 +166,7 @@ def test_external_dir_mapping(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -182,17 +178,17 @@ def test_external_dir_mapping(self): backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # Fill external directories with data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -207,9 +203,9 @@ def test_external_dir_mapping(self): options=[ "-j", "4", "--external-mapping={0}={1}".format( - external_dir1_old, external_dir1_new), + external_dir1, external_dir1_new), "--external-mapping={0}={1}".format( - external_dir2_old, external_dir2_new)]) + external_dir2, external_dir2_new)]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -228,9 +224,9 @@ def test_external_dir_mapping(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -240,9 +236,9 @@ def test_external_dir_mapping(self): options=[ "-j", "4", "--external-mapping={0}={1}".format( - external_dir1_old, external_dir1_new), + external_dir1, external_dir1_new), "--external-mapping={0}={1}".format( - external_dir2_old, external_dir2_new)]) + external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -264,11 +260,10 @@ def test_backup_multiple_external(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # FULL backup self.backup_node( @@ -278,31 +273,29 @@ def test_backup_multiple_external(self): # fill external directories with data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.set_config( backup_dir, 'node', - options=[ - '-E', external_dir1_old]) + options=['-E', external_dir1]) # cmdline option MUST override options in config self.backup_node( backup_dir, 'node', node, backup_type="delta", options=[ "-j", "4", "--stream", - "-E", "{0}".format(external_dir2_old)]) + "-E", external_dir2]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs', 'external_dir1']) node.cleanup() - - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) self.restore_node( backup_dir, 'node', node, @@ -340,24 +333,24 @@ def test_external_backward_compatibility(self): node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # FULL backup with old binary without external dirs support self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() @@ -368,16 +361,16 @@ def test_external_backward_compatibility(self): old_binary=True, options=["-j", "4", "--stream"]) # fill external directories with changed data - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.delete_pb(backup_dir, 'node', backup_id=backup_id) @@ -387,9 +380,9 @@ def test_external_backward_compatibility(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -407,8 +400,8 @@ def test_external_backward_compatibility(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -442,24 +435,24 @@ def test_external_backward_compatibility_merge_1(self): node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # tmp FULL backup with old binary tmp_id = self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.delete_pb(backup_dir, 'node', backup_id=tmp_id) @@ -477,9 +470,9 @@ def test_external_backward_compatibility_merge_1(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -500,8 +493,8 @@ def test_external_backward_compatibility_merge_1(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -535,24 +528,24 @@ def test_external_backward_compatibility_merge_2(self): node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # tmp FULL backup with old binary tmp_id = self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.delete_pb(backup_dir, 'node', backup_id=tmp_id) @@ -571,25 +564,25 @@ def test_external_backward_compatibility_merge_2(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgbench = node.pgbench(options=['-T', '30', '-c', '1']) pgbench.wait() # Fill external dirs with changed data - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, + data_dir=external_dir1, options=['-j', '4', '--skip-external-dirs']) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, + data_dir=external_dir2, options=['-j', '4', '--skip-external-dirs']) # delta backup without external directories using old binary @@ -599,9 +592,9 @@ def test_external_backward_compatibility_merge_2(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -622,8 +615,8 @@ def test_external_backward_compatibility_merge_2(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -649,45 +642,34 @@ def test_external_merge(self): self.add_instance(backup_dir, 'node', node, old_binary=True) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # FULL backup with old binary without external dirs support - self.backup_node( + tmp_id = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir2, options=["-j", "4"]) - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - # FULL backup - backup_id = self.backup_node( + # FULL backup with old binary without external dirs support + self.backup_node( backup_dir, 'node', node, old_binary=True, options=["-j", "4", "--stream"]) - # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() # delta backup with external directories using new binary backup_id = self.backup_node( @@ -695,15 +677,16 @@ def test_external_merge(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) - - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) + # Merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + # RESTORE node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) @@ -715,8 +698,8 @@ def test_external_merge(self): backup_dir, 'node', node, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -728,7 +711,7 @@ def test_external_merge(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_external_merge_1(self): + def test_external_merge_skip_external_dirs(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -737,38 +720,35 @@ def test_external_merge_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # FULL backup with old data - backup_id_1 = self.backup_node( + tmp_id = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup with new data - backup_id_2 = self.backup_node( - backup_dir, 'node', node, - options=["-j", "4", "--stream"]) + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with old data self.restore_node( - backup_dir, 'node', node, backup_id=backup_id_1, - data_dir=external_dir1_old, options=["-j", "4"]) + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( - backup_dir, 'node', node, backup_id=backup_id_1, - data_dir=external_dir2_old, options=["-j", "4"]) + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() # FULL backup with external directories self.backup_node( @@ -776,24 +756,24 @@ def test_external_merge_1(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) # drop old external data - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) # fill external directories with new data self.restore_node( - backup_dir, 'node', node, backup_id=backup_id_2, - data_dir=external_dir1_old, options=["-j", "4"]) + backup_dir, 'node', node, + data_dir=external_dir1, + options=["-j", "4", "--skip-external-dirs"]) self.restore_node( - backup_dir, 'node', node, backup_id=backup_id_2, - data_dir=external_dir2_old, options=["-j", "4"]) - - # drop now not needed backups + backup_dir, 'node', node, + data_dir=external_dir2, + options=["-j", "4", "--skip-external-dirs"]) # DELTA backup with external directories backup_id = self.backup_node( @@ -801,32 +781,25 @@ def test_external_merge_1(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) - # merge backups without external directories - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.merge_backup( + backup_dir, 'node', + backup_id=backup_id, options=['--skip-external-dirs']) # RESTORE node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - external_dir1_new = self.get_tblspace_path(node, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - self.restore_node( backup_dir, 'node', node, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + options=["-j", "4"]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -847,21 +820,20 @@ def test_external_merge_single(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # FULL backup self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() @@ -874,11 +846,11 @@ def test_external_merge_single(self): # fill external directories with changed data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.delete_pb(backup_dir, 'node', backup_id=backup_id) @@ -888,9 +860,9 @@ def test_external_merge_single(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) self.merge_backup(backup_dir, 'node', backup_id=backup_id) @@ -908,8 +880,8 @@ def test_external_merge_single(self): backup_dir, 'node', node, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -930,21 +902,20 @@ def test_external_merge_double(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=3) # FULL backup self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) - external_dir1_old = self.get_tblspace_path(node, 'external_dir1') - external_dir2_old = self.get_tblspace_path(node, 'external_dir2') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() @@ -957,11 +928,11 @@ def test_external_merge_double(self): # fill external directories with changed data self.restore_node( backup_dir, 'node', node, - data_dir=external_dir1_old, options=["-j", "4"]) + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=external_dir2_old, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) self.delete_pb(backup_dir, 'node', backup_id=backup_id) @@ -971,9 +942,9 @@ def test_external_merge_double(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) # delta backup with external directories backup_id = self.backup_node( @@ -981,15 +952,15 @@ def test_external_merge_double(self): options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( - external_dir1_old, + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2_old)]) + external_dir2)]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - shutil.rmtree(external_dir1_old, ignore_errors=True) - shutil.rmtree(external_dir2_old, ignore_errors=True) + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) # delta backup without external directories self.merge_backup(backup_dir, 'node', backup_id=backup_id) @@ -1005,8 +976,8 @@ def test_external_merge_double(self): backup_dir, 'node', node, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1_old, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2_old, external_dir2_new)]) + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1016,13 +987,10 @@ def test_external_merge_double(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") - def test_restore_skip_external(self): - """ - Check that --skip-external-dirs works correctly - """ + def test_restore_external_changed_data(self): + """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1030,21 +998,22 @@ def test_restore_skip_external(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + node.pgbench_init(scale=2) + + # set externals external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') - node.pgbench_init(scale=3) - - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) # fill external directories with data self.restore_node( @@ -1055,10 +1024,14 @@ def test_restore_skip_external(self): backup_dir, 'node', node, data_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - # FULL backup with external directories - self.backup_node( + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( backup_dir, 'node', node, options=[ "-j", "4", "--stream", @@ -1067,23 +1040,44 @@ def test_restore_skip_external(self): self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) - # delete first externals, so pgdata_compare - # will be capable of detecting redundant - # external files after restore + # fill external directories with changed data shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Delta backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - # RESTORE + # Restore node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) self.restore_node( backup_dir, 'node', node, - options=[ - "-j", "4", "--skip-external-dirs"]) + options=["-j", "4"]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1095,75 +1089,357 @@ def test_restore_skip_external(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_external_dir_is_symlink(self): - """ - Check that backup works correctly if external dir is symlink, - symlink pointing to external dir should be followed, - but restored as directory - """ + def test_restore_external_changed_data_1(self): + """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) - shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', - 'autovacuum': 'off'}) + 'autovacuum': 'off', + 'max_wal_size': '32MB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - external_dir = self.get_tblspace_path(node, 'external_dir') + node.pgbench_init(scale=1) - node.pgbench_init(scale=3) + # set externals + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) - # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) self.restore_node( backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) + data_dir=external_dir2, options=["-j", "4"]) - # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - # create symlink to directory in external directory - os.symlink(symlinked_dir, external_dir) + # change data a bit + pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) + pgbench.wait() - # FULL backup with external directories + # FULL backup backup_id = self.backup_node( backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) - # RESTORE - node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) - external_dir_new = self.get_tblspace_path( - node_restored, 'external_dir') + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() - self.restore_node( - backup_dir, 'node', node_restored, + # Delta backup with only one external directory + self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=[ - "-j", "4", "--external-mapping={0}={1}".format( - external_dir, external_dir_new)]) + "-j", "4", "--stream", + "-E", external_dir1]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir2']) + + # Restore + node.cleanup() + shutil.rmtree(node._base_dir) + + # create empty file in external_dir2 + os.mkdir(node._base_dir) + os.mkdir(external_dir2) + with open(os.path.join(external_dir2, 'file'), 'w+') as f: + f.close() + + output = self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + self.assertNotIn( + 'externaldir2', + output) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir2']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_merge_external_changed_data(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'max_wal_size': '32MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + + # set externals + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Delta backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_skip_external(self): + """ + Check that --skip-external-dirs works correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # FULL backup with external directories + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # delete first externals, so pgdata_compare + # will be capable of detecting redundant + # external files after restore + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--skip-external-dirs"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_is_symlink(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, module_name, fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + os.symlink(symlinked_dir, external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -1203,8 +1479,6 @@ def test_external_dir_contain_symlink_on_dir(self): external_dir = self.get_tblspace_path(node, 'external_dir') dir_in_external_dir = os.path.join(external_dir, 'dir') - node.pgbench_init(scale=3) - # temp FULL backup backup_id = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) @@ -1221,8 +1495,6 @@ def test_external_dir_contain_symlink_on_dir(self): self.delete_pb(backup_dir, 'node', backup_id=backup_id) # create symlink to directory in external directory - print(symlinked_dir) - print(dir_in_external_dir) os.mkdir(external_dir) os.symlink(symlinked_dir, dir_in_external_dir) @@ -1231,8 +1503,7 @@ def test_external_dir_contain_symlink_on_dir(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1290,8 +1561,6 @@ def test_external_dir_contain_symlink_on_file(self): external_dir = self.get_tblspace_path(node, 'external_dir') file_in_external_dir = os.path.join(external_dir, 'file') - node.pgbench_init(scale=3) - # temp FULL backup backup_id = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) @@ -1317,8 +1586,7 @@ def test_external_dir_contain_symlink_on_file(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1368,7 +1636,6 @@ def test_external_dir_is_tablespace(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) @@ -1380,7 +1647,7 @@ def test_external_dir_is_tablespace(self): self.create_tblspace_in_node( node, 'tblspace1', tblspc_path=external_dir) - node.pgbench_init(scale=3, tablespace='tblspace1') + node.pgbench_init(scale=1, tablespace='tblspace1') # FULL backup with external directories try: @@ -1388,8 +1655,7 @@ def test_external_dir_is_tablespace(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -1420,7 +1686,6 @@ def test_restore_external_dir_not_empty(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) @@ -1440,8 +1705,7 @@ def test_restore_external_dir_not_empty(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) node.cleanup() @@ -1503,7 +1767,6 @@ def test_restore_external_dir_is_missing(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) @@ -1523,8 +1786,7 @@ def test_restore_external_dir_is_missing(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) # drop external directory shutil.rmtree(external_dir, ignore_errors=True) @@ -1535,8 +1797,7 @@ def test_restore_external_dir_is_missing(self): backup_type='delta', options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -1573,6 +1834,95 @@ def test_restore_external_dir_is_missing(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_merge_external_dir_is_missing(self): + """ + take FULL backup with not empty external directory + delete external directory + take DELTA backup with external directory, which + should fail, + take DELTA backup without external directory, + merge it into FULL, restore and check + data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # drop external directory + shutil.rmtree(external_dir, ignore_errors=True) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: External directory is not found: "{0}"'.format(external_dir) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + # take DELTA without external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + def test_restore_external_dir_is_empty(self): """ take FULL backup with not empty external directory @@ -1590,7 +1940,6 @@ def test_restore_external_dir_is_empty(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) @@ -1610,8 +1959,7 @@ def test_restore_external_dir_is_empty(self): backup_dir, 'node', node, options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) # make external directory empty os.remove(os.path.join(external_dir, 'file')) @@ -1622,28 +1970,87 @@ def test_restore_external_dir_is_empty(self): backup_type='delta', options=[ "-j", "4", "--stream", - "-E", "{0}".format( - external_dir)]) + "-E", external_dir]) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) # Restore Delta backup - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) - node_restored.cleanup() + self.restore_node(backup_dir, 'node', node) - external_dir_new = self.get_tblspace_path( - node_restored, 'external_dir') + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) - self.restore_node( - backup_dir, 'node', node_restored, - options=['--external-mapping={0}={1}'.format( - external_dir, external_dir_new)]) + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_merge_external_dir_is_empty(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + merge backups and restore FULL, check that restored + external directory is empty + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # make external directory empty + os.remove(os.path.join(external_dir, 'file')) + + # take DELTA backup with empty external directory + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) + node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -1666,7 +2073,6 @@ def test_restore_external_dir_string_order(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir) @@ -1729,3 +2135,86 @@ def test_restore_external_dir_string_order(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_merge_external_dir_string_order(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored + external directory is empty + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + core_dir = os.path.join(self.tmp_path, module_name, fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + # create empty file in external directory + os.mkdir(external_dir_1) + with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: + f.close() + + os.mkdir(external_dir_2) + with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_2)]) + + with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: + f.close() + + with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: + f.close() + + # take DELTA backup and swap external_dir_2 and external_dir_1 + # in external_dir_str + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_1)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge backups + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From a490120c16a7245b43bfc65c328b9c82bebc9b01 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 29 Apr 2019 10:56:40 +0300 Subject: [PATCH 0633/2107] Correctly set binary mode for SSH pipe --- src/pg_probackup.h | 12 ------------ src/utils/file.c | 9 +++++++-- src/utils/remote.c | 4 ++-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6e4a9137a..346a7ebaf 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -679,16 +679,4 @@ extern bool parse_page(Page page, XLogRecPtr *lsn); int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); -#ifdef WIN32 -#ifdef _DEBUG -#define lseek _lseek -#define open _open -#define fstat _fstat -#define read _read -#define close _close -#define write _write -#define mkdir(dir,mode) _mkdir(dir) -#endif -#endif - #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/file.c b/src/utils/file.c index cf25b8db6..07a646d45 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -3,7 +3,7 @@ #include #ifdef WIN32 -#define __thread +#define __thread __declspec(thread) #else #include #endif @@ -1258,7 +1258,12 @@ void fio_communicate(int in, int out) struct stat st; int rc; - /* Main loop until command of processing master command */ +#ifdef WIN32 + SYS_CHECK(setmode(in, _O_BINARY)); + SYS_CHECK(setmode(out, _O_BINARY)); +#endif + + /* Main loop until command of processing master command */ while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { if (hdr.size != 0) { if (hdr.size > buf_size) { diff --git a/src/utils/remote.c b/src/utils/remote.c index ab27bf712..2f75e9d33 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -148,8 +148,8 @@ bool launch_agent(void) } #ifdef WIN32 - SYS_CHECK(_pipe(infd, PIPE_SIZE, O_BINARY)) ; - SYS_CHECK(_pipe(outfd, PIPE_SIZE, O_BINARY)); + SYS_CHECK(_pipe(infd, PIPE_SIZE, _O_BINARY)) ; + SYS_CHECK(_pipe(outfd, PIPE_SIZE, _O_BINARY)); ssh_argv[2] = psprintf("%d", outfd[0]); ssh_argv[3] = psprintf("%d", infd[1]); { From d66bca9fffe163560a464e2db0fd0ac97e933dda Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 29 Apr 2019 15:23:59 +0300 Subject: [PATCH 0634/2107] Remove '\' handling code --- src/utils/configuration.c | 47 +-------------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 45866846f..cf5436f29 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -310,52 +310,7 @@ get_next_token(const char *src, char *dst, const char *line) s++; for (i = 0, j = 0; s[i] != '\0'; i++) { - if (s[i] == '\\') - { - i++; - switch (s[i]) - { - case 'b': - dst[j] = '\b'; - break; - case 'f': - dst[j] = '\f'; - break; - case 'n': - dst[j] = '\n'; - break; - case 'r': - dst[j] = '\r'; - break; - case 't': - dst[j] = '\t'; - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - { - int k; - long octVal = 0; - - for (k = 0; - s[i + k] >= '0' && s[i + k] <= '7' && k < 3; - k++) - octVal = (octVal << 3) + (s[i + k] - '0'); - i += k - 1; - dst[j] = ((char) octVal); - } - break; - default: - dst[j] = s[i]; - break; - } - } - else if (s[i] == '\'') + if (s[i] == '\'') { i++; /* doubled quote becomes just one quote */ From cbfb496d5bebfb1d9e5bbd1eeeefc25779112d45 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Apr 2019 15:59:19 +0300 Subject: [PATCH 0635/2107] tests: minor fix in "external" module --- tests/external.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/external.py b/tests/external.py index bb9945ccf..cfc7cf83c 100644 --- a/tests/external.py +++ b/tests/external.py @@ -52,10 +52,9 @@ def test_external_simple(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'ERROR: --external-dirs option "{0}": ' - 'directory or symbolic link expected\n'.format(file_path), - e.message, + self.assertTrue( + 'ERROR: --external-dirs option' in e.message and + 'directory or symbolic link expected' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 1b02f87c1200d666ef541b982a19c3b7ac070619 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Apr 2019 16:08:26 +0300 Subject: [PATCH 0636/2107] tests: additional fixes for "external" module --- tests/external.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/external.py b/tests/external.py index cfc7cf83c..b80beb805 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1662,8 +1662,9 @@ def test_external_dir_is_tablespace(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'External directory path (-E option)' in e.message, + self.assertIn( + 'External directory path (-E option)', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1717,8 +1718,9 @@ def test_restore_external_dir_not_empty(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'External directory is not empty' in e.message, + self.assertIn( + 'External directory is not empty', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1742,8 +1744,9 @@ def test_restore_external_dir_not_empty(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'External directory is not empty' in e.message, + self.assertIn( + 'External directory is not empty', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1804,8 +1807,9 @@ def test_restore_external_dir_is_missing(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: External directory is not found: "{0}"'.format(external_dir) in e.message, + self.assertIn( + 'ERROR: External directory is not found:', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1890,8 +1894,9 @@ def test_merge_external_dir_is_missing(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: External directory is not found: "{0}"'.format(external_dir) in e.message, + self.assertIn( + 'ERROR: External directory is not found:', + external_dir, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From b2464d946b2c3af680867dae16ecf8980ff69354 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 29 Apr 2019 19:11:01 +0300 Subject: [PATCH 0637/2107] Canonicalize external mapping paths --- src/dir.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dir.c b/src/dir.c index 1a27c84a2..0b9518e57 100644 --- a/src/dir.c +++ b/src/dir.c @@ -988,6 +988,8 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, if (!*cell->old_dir || !*cell->new_dir) elog(ERROR, "invalid %s mapping format \"%s\", " "must be \"OLDDIR=NEWDIR\"", type, arg); + canonicalize_path(cell->old_dir); + canonicalize_path(cell->new_dir); /* * This check isn't absolutely necessary. But all tablespaces are created From f154be4c0f4a79d16e1a707ff43727072f0daab9 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 29 Apr 2019 19:19:44 +0300 Subject: [PATCH 0638/2107] Canonicalize some other paths --- src/archive.c | 4 ++++ src/utils/logger.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/archive.c b/src/archive.c index e570a0202..fe42a728d 100644 --- a/src/archive.c +++ b/src/archive.c @@ -39,6 +39,8 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + canonicalize_path(wal_file_path); + if (!getcwd(current_dir, sizeof(current_dir))) elog(ERROR, "getcwd() error"); @@ -97,6 +99,8 @@ do_archive_get(char *wal_file_path, char *wal_file_name) if (wal_file_path == NULL) elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + canonicalize_path(wal_file_path); + if (!getcwd(current_dir, sizeof(current_dir))) elog(ERROR, "getcwd() error"); diff --git a/src/utils/logger.c b/src/utils/logger.c index e9e4a7445..0e0cba29d 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -84,6 +84,9 @@ init_logger(const char *root_path, LoggerConfig *config) root_path, LOG_DIRECTORY_DEFAULT); } + if (config->log_directory != NULL) + canonicalize_path(config->log_directory); + logger_config = *config; } From c3451c5604b2397a8b6748ba8b1a2715b2691c34 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Apr 2019 19:40:59 +0300 Subject: [PATCH 0639/2107] tests: module "external" fixes for Windows --- tests/external.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/external.py b/tests/external.py index b80beb805..0c05059a8 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1380,6 +1380,9 @@ def test_external_dir_is_symlink(self): symlink pointing to external dir should be followed, but restored as directory """ + if os.name == 'nt': + return unittest.skip('Skipped for Windows') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') core_dir = os.path.join(self.tmp_path, module_name, fname) @@ -1462,6 +1465,9 @@ def test_external_dir_contain_symlink_on_dir(self): symlink pointing to external dir should be followed, but restored as directory """ + if os.name == 'nt': + return unittest.skip('Skipped for Windows') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') core_dir = os.path.join(self.tmp_path, module_name, fname) @@ -1544,6 +1550,9 @@ def test_external_dir_contain_symlink_on_file(self): symlink pointing to external dir should be followed, but restored as directory """ + if os.name == 'nt': + return unittest.skip('Skipped for Windows') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') core_dir = os.path.join(self.tmp_path, module_name, fname) @@ -1896,7 +1905,7 @@ def test_merge_external_dir_is_missing(self): except ProbackupException as e: self.assertIn( 'ERROR: External directory is not found:', - external_dir, + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 7d30e5b33fe125c62b8ef06ca43b756268ef063c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Apr 2019 22:45:44 +0300 Subject: [PATCH 0640/2107] minor bugfix: ensure that tablespace directories are canonicalized --- src/backup.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index f62dd0646..8160db275 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3461,8 +3461,8 @@ check_external_for_tablespaces(parray *external_list) int i = 0; int j = 0; char *tablespace_path = NULL; - char *query = "SELECT pg_catalog.pg_tablespace_location(oid)\n" - "FROM pg_tablespace\n" + char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " + "FROM pg_catalog.pg_tablespace " "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; conn = backup_conn; @@ -3476,9 +3476,13 @@ check_external_for_tablespaces(parray *external_list) { tablespace_path = PQgetvalue(res, i, 0); Assert (strlen(tablespace_path) > 0); + + canonicalize_path(tablespace_path); + for (j = 0; j < parray_num(external_list); j++) { char *external_path = parray_get(external_list, j); + if (path_is_prefix_of_path(external_path, tablespace_path)) elog(ERROR, "External directory path (-E option) \"%s\" " "contains tablespace \"%s\"", From d042b8e0db8ccfc068471b61ed9aed74ec3a260e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Apr 2019 23:53:24 +0300 Subject: [PATCH 0641/2107] minor bugfix: canonicalize tablespace path in read_tablespace_map() --- src/dir.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dir.c b/src/dir.c index 0b9518e57..6636158ec 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1223,6 +1223,8 @@ read_tablespace_map(parray *files, const char *backup_dir) file->linked = pgut_malloc(strlen(path) + 1); strcpy(file->linked, path); + canonicalize_path(file->linked); + parray_append(files, file); } From a71288249e053dd1cab44d8944b9d842bfa662a8 Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Tue, 30 Apr 2019 00:24:44 +0300 Subject: [PATCH 0642/2107] canonicalize_path() within pgFileInit() --- src/dir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dir.c b/src/dir.c index 6636158ec..a572e0498 100644 --- a/src/dir.c +++ b/src/dir.c @@ -217,6 +217,7 @@ pgFileInit(const char *path) file->path = pgut_malloc(strlen(path) + 1); strcpy(file->path, path); /* enough buffer size guaranteed */ + canonicalize_path(file->path); /* Get file name from the path */ file_name = last_dir_separator(file->path); From 091b7c6e4b9178db70932301f8b4bc879899eff6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 00:30:25 +0300 Subject: [PATCH 0643/2107] tests: minor fixes for Windows --- tests/archive.py | 17 +++++++++++------ tests/backup_test.py | 18 +++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 634f2019a..4f6a6e210 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -296,9 +296,11 @@ def test_arhive_push_file_exists(self): wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - file = os.path.join(wals_dir, '000000010000000000000001.gz') + filename = '000000010000000000000001.gz' + file = os.path.join(wals_dir, filename) else: - file = os.path.join(wals_dir, '000000010000000000000001') + filename = '000000010000000000000001' + file = os.path.join(wals_dir, filename) with open(file, 'a') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") @@ -319,7 +321,8 @@ def test_arhive_push_file_exists(self): 'LOG: archive command failed with exit code 1' in log_content and 'DETAIL: The failed archive command was:' in log_content and 'INFO: pg_probackup archive-push from' in log_content and - 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + 'ERROR: WAL segment ' in log_content and + '{0}" already exists.'.format(filename) in log_content, 'Expecting error messages about failed archive_command' ) self.assertFalse('pg_probackup archive-push completed successfully' in log_content) @@ -369,9 +372,11 @@ def test_arhive_push_file_exists_overwrite(self): wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - file = os.path.join(wals_dir, '000000010000000000000001.gz') + filename = '000000010000000000000001.gz' + file = os.path.join(wals_dir, filename) else: - file = os.path.join(wals_dir, '000000010000000000000001') + filename = '000000010000000000000001' + file = os.path.join(wals_dir, filename) with open(file, 'a') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") @@ -392,7 +397,7 @@ def test_arhive_push_file_exists_overwrite(self): 'LOG: archive command failed with exit code 1' in log_content and 'DETAIL: The failed archive command was:' in log_content and 'INFO: pg_probackup archive-push from' in log_content and - 'ERROR: WAL segment "{0}" already exists.'.format(file) in log_content, + '{0}" already exists.'.format(filename) in log_content, 'Expecting error messages about failed archive_command' ) self.assertFalse('pg_probackup archive-push completed successfully' in log_content) diff --git a/tests/backup_test.py b/tests/backup_test.py index e0e12c997..d54a03119 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -610,17 +610,17 @@ def test_tablespace_handling(self): "drop tablespace some_lame_tablespace") self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream"]) + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"]) self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace1_new_path), - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace2_new_path)]) + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) From 5362c2a783c9151a53871f180ca890bd53756a3c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 00:38:19 +0300 Subject: [PATCH 0644/2107] canonicalize tablespace path --- src/dir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dir.c b/src/dir.c index a572e0498..13c0f7de5 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1225,6 +1225,7 @@ read_tablespace_map(parray *files, const char *backup_dir) strcpy(file->linked, path); canonicalize_path(file->linked); + canonicalize_path(file->path); parray_append(files, file); } From d8086219c72784e9fab239326e009daea70224d7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 01:49:14 +0300 Subject: [PATCH 0645/2107] tests: fix checkdb for Windows --- tests/checkdb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 29674ff31..038fb7cff 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -431,11 +431,13 @@ def test_checkdb_block_validation_sanity(self): repr(e.message), self.cmd)) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 1".format(heap_full_path), + "WARNING: CORRUPTION in file {0}, block 1".format( + os.path.normpath(heap_full_path)), e.message) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 5".format(heap_full_path), + "WARNING: CORRUPTION in file {0}, block 5".format( + os.path.normpath(heap_full_path)), e.message) # Clean after yourself From ee90c5410d4b341dfca14c88e500a142bfdf9162 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Tue, 30 Apr 2019 12:28:26 +0300 Subject: [PATCH 0646/2107] Remove make_native_path() from pgBackupGetPath2() --- src/catalog.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 363c49efc..43ebdf791 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1004,8 +1004,6 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, else snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, base36enc(backup->start_time), subdir1, subdir2); - - make_native_path(path); } /* From e771c97aa2241cb72da8e334c7c71b48a0390182 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 19:44:06 +0300 Subject: [PATCH 0647/2107] tests: minor fixes for "page" module --- tests/page.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/page.py b/tests/page.py index d9395c96e..648081cbc 100644 --- a/tests/page.py +++ b/tests/page.py @@ -705,8 +705,7 @@ def test_page_backup_with_lost_wal_segment(self): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'Could not read WAL record at' in e.message and - 'WAL segment "{0}" is absent\n'.format( - file) in e.message, + 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -823,8 +822,7 @@ def test_page_backup_with_corrupted_wal_segment(self): 'in archived WAL segment' in e.message and 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( - file) in e.message, + 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1093,14 +1091,14 @@ def test_page_create_db(self): # DROP DATABASE DB1 node.safe_psql( "postgres", "drop database db1") - # SECOND PTRACK BACKUP + # SECOND PAGE BACKUP backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - # RESTORE SECOND PTRACK BACKUP + # RESTORE SECOND PAGE BACKUP self.restore_node( backup_dir, 'node', node_restored, backup_id=backup_id, options=["-j", "4"] From b7a9a20b9965cf86f498e554b5756211d770c56f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 20:18:40 +0300 Subject: [PATCH 0648/2107] tests: fixes for Windows --- tests/delete.py | 7 +++---- tests/delta.py | 16 +++++++++++----- tests/merge.py | 7 ------- tests/page.py | 9 ++++----- tests/retention.py | 3 ++- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index 8c0994f82..e2a996b7a 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -148,9 +148,9 @@ def test_delete_increment_ptrack(self): # full backup mode self.backup_node(backup_dir, 'node', node) - # page backup mode + # ptrack backup mode self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # page backup mode + # ptrack backup mode self.backup_node(backup_dir, 'node', node, backup_type="ptrack") # full backup mode self.backup_node(backup_dir, 'node', node) @@ -249,8 +249,7 @@ def test_delete_backup_with_empty_control_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={'ptrack_enable': 'on'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/delta.py b/tests/delta.py index 3c146d975..2c030fc03 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1268,7 +1268,9 @@ def test_delta_nullified_heap_page_backup(self): backup_dir, 'node', node) # Nullify some block in PostgreSQL - file = os.path.join(node.data_dir, file_path) + file = os.path.join(node.data_dir, file_path).replace("\\","/") + if os.name == 'nt': + file = file.replace("\\","/") with open(file, 'r+b', 0) as f: f.seek(8192) @@ -1286,10 +1288,14 @@ def test_delta_nullified_heap_page_backup(self): if not self.remote: log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") with open(log_file_path) as f: - self.assertTrue("LOG: File: {0} blknum 1, empty page".format( - file) in f.read()) - self.assertFalse("Skipping blknum: 1 in file: {0}".format( - file) in f.read()) + content = f.read() + + self.assertIn( + "LOG: File: {0} blknum 1, empty page".format(file), + content) + self.assertNotIn( + "Skipping blknum: 1 in file: {0}".format(file), + content) # Restore DELTA backup node_restored = self.make_simple_node( diff --git a/tests/merge.py b/tests/merge.py index 56dc077c8..4ae8c0ba4 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -485,7 +485,6 @@ def test_merge_tablespaces(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off' } ) @@ -564,7 +563,6 @@ def test_merge_tablespaces_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off' } ) @@ -648,7 +646,6 @@ def test_merge_page_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' } @@ -742,7 +739,6 @@ def test_merge_delta_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' } @@ -836,7 +832,6 @@ def test_merge_ptrack_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off', 'ptrack_enable': 'on' @@ -930,7 +925,6 @@ def test_merge_delta_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' } @@ -1683,7 +1677,6 @@ def test_merge_backup_from_future(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/page.py b/tests/page.py index 648081cbc..a06170969 100644 --- a/tests/page.py +++ b/tests/page.py @@ -911,7 +911,7 @@ def test_page_backup_with_alien_wal_segment(self): "create table t_heap_alien as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i;") + "from generate_series(0,100000) i;") # copy lastest wal segment wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') @@ -925,6 +925,7 @@ def test_page_backup_with_alien_wal_segment(self): # file = os.path.join(wals_dir, '000000010000000000000004') print(file) print(file_destination) + os.remove(file_destination) os.rename(file, file_destination) # Single-thread PAGE backup @@ -944,8 +945,7 @@ def test_page_backup_with_alien_wal_segment(self): 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( - file_destination) in e.message, + 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -971,8 +971,7 @@ def test_page_backup_with_alien_wal_segment(self): 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( - file_destination) in e.message, + 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/retention.py b/tests/retention.py index 322c78f16..363e61460 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -762,7 +762,8 @@ def test_window_merge_multiple_descendants(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From b70b9a232b0d81cd77877eede929697819a3a13f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Apr 2019 20:20:41 +0300 Subject: [PATCH 0649/2107] tests: fixes for Windows --- tests/page.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/page.py b/tests/page.py index a06170969..62f7e0546 100644 --- a/tests/page.py +++ b/tests/page.py @@ -730,8 +730,7 @@ def test_page_backup_with_lost_wal_segment(self): 'INFO: Wait for LSN' in e.message and 'in archived WAL segment' in e.message and 'Could not read WAL record at' in e.message and - 'WAL segment "{0}" is absent\n'.format( - file) in e.message, + 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 04af42c241c4173d5f72751069c03659dce53057 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:08:51 +0300 Subject: [PATCH 0650/2107] tests: changes in "init" and "restore" modules for Windows --- tests/{init_test.py => init.py} | 29 ++++++++++++++++++----------- tests/restore.py | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) rename tests/{init_test.py => init.py} (89%) diff --git a/tests/init_test.py b/tests/init.py similarity index 89% rename from tests/init_test.py rename to tests/init.py index ff416e244..059e24d95 100644 --- a/tests/init_test.py +++ b/tests/init.py @@ -21,7 +21,9 @@ def test_success(self): ['backups', 'wal'] ) self.add_instance(backup_dir, 'node', node) - self.assertEqual("INFO: Instance 'node' successfully deleted\n", self.del_instance(backup_dir, 'node'), + self.assertIn( + "INFO: Instance 'node' successfully deleted", + self.del_instance(backup_dir, 'node'), '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) # Show non-existing instance @@ -30,8 +32,9 @@ def test_success(self): self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - "ERROR: Instance 'node' does not exist in this backup catalog\n", + self.assertIn( + "ERROR: Instance 'node' does not exist in this backup catalog", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) # Delete non-existing instance @@ -40,8 +43,9 @@ def test_success(self): self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - "ERROR: Instance 'node1' does not exist in this backup catalog\n", + self.assertIn( + "ERROR: Instance 'node1' does not exist in this backup catalog", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) # Add instance without pgdata @@ -54,8 +58,9 @@ def test_success(self): self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)\n", + self.assertIn( + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) # Clean after yourself @@ -73,8 +78,9 @@ def test_already_exist(self): self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - "ERROR: Instance 'node' does not exist in this backup catalog\n", + self.assertIn( + "ERROR: Instance 'node' does not exist in this backup catalog", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself @@ -91,8 +97,9 @@ def test_abs_path(self): self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - "ERROR: -B, --backup-path must be an absolute path\n", + self.assertIn( + "ERROR: -B, --backup-path must be an absolute path", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself diff --git a/tests/restore.py b/tests/restore.py index b2146b3c3..a2e2ff8a6 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -810,7 +810,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore destination is not empty: "{0}"'.format(node.data_dir), + 'ERROR: restore destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 642eb1a0969d9c57799c85bc534f868c7324a862 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:20:08 +0300 Subject: [PATCH 0651/2107] tests: fixes for Windows in "option" modules --- tests/{option_test.py => option.py} | 65 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 24 deletions(-) rename tests/{option_test.py => option.py} (87%) diff --git a/tests/option_test.py b/tests/option.py similarity index 87% rename from tests/option_test.py rename to tests/option.py index d2c352275..9cf0b8f36 100644 --- a/tests/option_test.py +++ b/tests/option.py @@ -42,7 +42,9 @@ def test_without_backup_path_3(self): self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)\n', + self.assertIn( + 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -63,8 +65,9 @@ def test_options_4(self): self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: required parameter not specified: --instance\n', + self.assertIn( + 'ERROR: required parameter not specified: --instance', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # backup command failure without backup mode option @@ -73,7 +76,8 @@ def test_options_4(self): self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn('ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + self.assertIn( + 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -83,8 +87,9 @@ def test_options_4(self): self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: invalid backup-mode "bad"\n', + self.assertIn( + 'ERROR: invalid backup-mode "bad"', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # delete failure without delete options @@ -94,8 +99,9 @@ def test_options_4(self): self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: You must specify at least one of the delete options: --expired |--wal |--merge-expired |--delete-invalid |--backup_id\n', + self.assertIn( + 'ERROR: You must specify at least one of the delete options: --expired |--wal |--merge-expired |--delete-invalid |--backup_id', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -106,7 +112,9 @@ def test_options_4(self): self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue("option requires an argument -- 'i'" in e.message, + self.assertIn( + "option requires an argument -- 'i'", + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself @@ -118,12 +126,16 @@ def test_options_5(self): fname = self.id().split(".")[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - pg_options={ - 'max_wal_senders': '2'}) + base_dir=os.path.join(module_name, fname, 'node')) - self.assertEqual("INFO: Backup catalog '{0}' successfully inited\n".format(backup_dir), - self.init_pb(backup_dir)) + output = self.init_pb(backup_dir) + self.assertIn( + "INFO: Backup catalog", + output) + + self.assertIn( + "successfully inited", + output) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -138,8 +150,9 @@ def test_options_5(self): self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: Syntax error in " = INFINITE"\n', + self.assertIn( + 'ERROR: Syntax error in " = INFINITE', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -156,8 +169,9 @@ def test_options_5(self): self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: Invalid option "BACKUP_MODE" in file "{0}"\n'.format(conf_file), + self.assertIn( + 'ERROR: Invalid option "BACKUP_MODE" in file', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -177,8 +191,9 @@ def test_options_5(self): self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: Option system-identifier cannot be specified in command line\n', + self.assertIn( + 'ERROR: Option system-identifier cannot be specified in command line', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # invalid value in pg_probackup.conf @@ -191,8 +206,9 @@ def test_options_5(self): self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file "{0}"\n'.format(conf_file), + self.assertIn( + 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) self.clean_pb(backup_dir) @@ -209,8 +225,9 @@ def test_options_5(self): self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertEqual(e.message, - 'ERROR: Invalid option "TIMELINEID" in file "{0}"\n'.format(conf_file), + self.assertIn( + 'ERROR: Invalid option "TIMELINEID" in file', + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # Clean after yourself From 613a5b3cc0ef6a070411f45a4a24111716471f05 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:32:16 +0300 Subject: [PATCH 0652/2107] tests: Windows fixes for "show" module --- tests/{show_test.py => show.py} | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) rename tests/{show_test.py => show.py} (91%) diff --git a/tests/show_test.py b/tests/show.py similarity index 91% rename from tests/show_test.py rename to tests/show.py index 43d4848b7..6044e15b8 100644 --- a/tests/show_test.py +++ b/tests/show.py @@ -88,14 +88,14 @@ def test_corrupt_2(self): # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because backup corrupted.\n" + "Expecting Error because backup corrupted." " Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd ) ) except ProbackupException as e: self.assertIn( - 'data files are corrupted\n', + 'data files are corrupted', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd) @@ -127,9 +127,15 @@ def test_no_control_file(self): backup_id, "backup.control") os.remove(file) + output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + + self.assertIn( + 'Control file', + output) + self.assertIn( - 'Control file "{0}" doesn\'t exist'.format(file), - self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) + 'doesn\'t exist', + output) # Clean after yourself self.del_test_dir(module_name, fname) @@ -157,9 +163,15 @@ def test_empty_control_file(self): fd = open(file, 'w') fd.close() + output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + + self.assertIn( + 'Control file', + output) + self.assertIn( - 'Control file "{0}" is empty'.format(file), - self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) + 'is empty', + output) # Clean after yourself self.del_test_dir(module_name, fname) @@ -190,7 +202,7 @@ def test_corrupt_control_file(self): fd.close() self.assertIn( - 'WARNING: Invalid option "statuss" in file'.format(file), + 'WARNING: Invalid option "statuss" in file', self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) # Clean after yourself From 04541c50293ddab839cd2c755079e9be27df8c82 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:39:08 +0300 Subject: [PATCH 0653/2107] tests: final Windows fixes --- tests/Readme.md | 6 +++++- tests/__init__.py | 22 +++++++++------------- tests/exclude.py | 3 +-- tests/logging.py | 9 ++++++--- tests/merge.py | 3 ++- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 315710481..61d000fcc 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,7 +1,11 @@ [см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` -Note: For now there are tests only for Linix +Note: For now these are works on Linix and "kinda" works on Windows +``` + +``` +Windows Note: For tablespaceses tests to work on Windows, you should explicitly(!) grant current user full access to tmp_dirs ``` diff --git a/tests/__init__.py b/tests/__init__.py index 5e006f41a..dd4ebc3bf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,6 @@ import unittest -from . import init_test, merge, option_test, show_test, compatibility, \ +from . import init, merge, option, show, compatibility, \ backup_test, delete, delta, restore, validate, \ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ @@ -19,40 +19,36 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(cfs_backup)) # suite.addTests(loader.loadTestsFromModule(cfs_restore)) # suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) - suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(compression)) suite.addTests(loader.loadTestsFromModule(delete)) suite.addTests(loader.loadTestsFromModule(delta)) suite.addTests(loader.loadTestsFromModule(exclude)) + suite.addTests(loader.loadTestsFromModule(external)) suite.addTests(loader.loadTestsFromModule(false_positive)) - suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(init)) suite.addTests(loader.loadTestsFromModule(locking)) suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(merge)) - suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(option)) suite.addTests(loader.loadTestsFromModule(page)) # suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(pgpro560)) + suite.addTests(loader.loadTestsFromModule(pgpro589)) + suite.addTests(loader.loadTestsFromModule(pgpro2068)) suite.addTests(loader.loadTestsFromModule(remote)) suite.addTests(loader.loadTestsFromModule(replica)) suite.addTests(loader.loadTestsFromModule(restore)) suite.addTests(loader.loadTestsFromModule(retention)) - suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(show)) suite.addTests(loader.loadTestsFromModule(snapfs)) - suite.addTests(loader.loadTestsFromModule(validate)) - suite.addTests(loader.loadTestsFromModule(pgpro560)) - suite.addTests(loader.loadTestsFromModule(pgpro589)) - suite.addTests(loader.loadTestsFromModule(pgpro2068)) suite.addTests(loader.loadTestsFromModule(time_stamp)) - suite.addTests(loader.loadTestsFromModule(external)) + suite.addTests(loader.loadTestsFromModule(validate)) return suite # test_pgpro434_2 unexpected success # ToDo: -# archive: -# discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work # logging: -# https://jira.postgrespro.ru/browse/PGPRO-584 # https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md # archive: # immediate recovery and full recovery diff --git a/tests/exclude.py b/tests/exclude.py index 21272bcaa..5e641bada 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -112,9 +112,8 @@ def test_exclude_unlogged_tables_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', + 'autovacuum': 'off', "shared_buffers": "10MB", - "fsync": "off", 'ptrack_enable': 'on'}) self.init_pb(backup_dir) diff --git a/tests/logging.py b/tests/logging.py index c47d68dc1..c905695bd 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -256,7 +256,7 @@ def test_garbage_in_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) # mangle .rotation file - with open(rotation_file_path, "wtb", 0) as f: + with open(rotation_file_path, "wt", 0) as f: f.write(b"blah") f.flush() f.close @@ -274,8 +274,11 @@ def test_garbage_in_rotation_file(self): log_file_size) self.assertIn( - 'WARNING: rotation file "{0}" has wrong ' - 'creation timestamp'.format(rotation_file_path), + 'WARNING: rotation file', + output) + + self.assertIn( + 'has wrong creation timestamp', output) self.assertTrue(os.path.isfile(rotation_file_path)) diff --git a/tests/merge.py b/tests/merge.py index 4ae8c0ba4..c06e15060 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1224,7 +1224,8 @@ def test_continue_failed_merge_2(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('fio_unlink') + gdb.set_breakpoint('pgFileDelete') + gdb.run_until_break() gdb.continue_execution_until_break(20) From 481faf0b5be6edfe1cc856de16f0d2a2f18490e9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:41:27 +0300 Subject: [PATCH 0654/2107] tests: rename module "backup_test" to "backup" --- tests/__init__.py | 4 ++-- tests/{backup_test.py => backup.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/{backup_test.py => backup.py} (100%) diff --git a/tests/__init__.py b/tests/__init__.py index dd4ebc3bf..495d4bdbb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ import unittest from . import init, merge, option, show, compatibility, \ - backup_test, delete, delta, restore, validate, \ + backup, delete, delta, restore, validate, \ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ @@ -12,7 +12,7 @@ def load_tests(loader, tests, pattern): suite = unittest.TestSuite() # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) - suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(backup)) suite.addTests(loader.loadTestsFromModule(compatibility)) suite.addTests(loader.loadTestsFromModule(checkdb)) suite.addTests(loader.loadTestsFromModule(config)) diff --git a/tests/backup_test.py b/tests/backup.py similarity index 100% rename from tests/backup_test.py rename to tests/backup.py From a7650c07c291148723a55c6701f7235220db1c6e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:43:52 +0300 Subject: [PATCH 0655/2107] stub for remote operations on Windows --- src/pg_probackup.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 339c282fa..6f474820a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -207,6 +207,12 @@ static ConfigOption cmd_options[] = static void setMyLocation(void) { + +#ifdef WIN32 + if (IsSshProtocol()) + elog(ERROR, "Remote operations on Windows are not possible at this moment"); +#endif + MyLocation = IsSshProtocol() ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) ? FIO_DB_HOST From 559390aa9eb2eec6452e0db2262cc8baa48c8b3e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 02:45:17 +0300 Subject: [PATCH 0656/2107] change stub wording --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6f474820a..73d511a4d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -210,7 +210,7 @@ setMyLocation(void) #ifdef WIN32 if (IsSshProtocol()) - elog(ERROR, "Remote operations on Windows are not possible at this moment"); + elog(ERROR, "Currently remote operations on Windows are not supported"); #endif MyLocation = IsSshProtocol() From 323add28aaf4c2e398394519c4545e749e921f93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 16:53:49 +0300 Subject: [PATCH 0657/2107] tests: added external.ExternalTest.test_external_dirs_overlapping --- tests/external.py | 57 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/external.py b/tests/external.py index 0c05059a8..d74527965 100644 --- a/tests/external.py +++ b/tests/external.py @@ -112,7 +112,6 @@ def test_external_none(self): # create directory in external_directory self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() # FULL backup @@ -134,7 +133,7 @@ def test_external_none(self): # Delta backup without external directory self.backup_node( backup_dir, 'node', node, backup_type="delta", - options=['--external-dirs=none']) + options=['--external-dirs=none', '--stream']) shutil.rmtree(external_dir, ignore_errors=True) pgdata = self.pgdata_content( @@ -153,6 +152,60 @@ def test_external_none(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_external_dirs_overlapping(self): + """ + make node, create directory, + take backup with two external directories pointing to + the same directory, backup should fail + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # create directory in external_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + os.mkdir(external_dir1) + os.mkdir(external_dir2) + + # Full backup with external dirs + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}{1}{0}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir1)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: External directory path (-E option)' in e.message and + 'contain another external directory' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_external_dir_mapping(self): """ From c509ae42b966a99d17070fa7319d7222c7ab9283 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 16:55:48 +0300 Subject: [PATCH 0658/2107] backup: sanity check that external directories do not overlap --- src/backup.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backup.c b/src/backup.c index 8160db275..528fd1045 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3494,6 +3494,30 @@ check_external_for_tablespaces(parray *external_list) } } PQclear(res); + + /* Check that external directories do not overlap */ + if (parray_num(external_list) < 2) + return; + + for (i = 0; i < parray_num(external_list); i++) + { + char *external_path = parray_get(external_list, i); + + for (j = 0; j < parray_num(external_list); j++) + { + char *tmp_external_path = parray_get(external_list, j); + + /* skip yourself */ + if (j == i) + continue; + + if (path_is_prefix_of_path(external_path, tmp_external_path)) + elog(ERROR, "External directory path (-E option) \"%s\" " + "contain another external directory \"%s\"", + external_path, tmp_external_path); + + } + } } /* Get index list for given database */ From d2129fdd12f2142b099b5a7d298635ae8b533de2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 17:53:29 +0300 Subject: [PATCH 0659/2107] change elog level to WARNING in message about restoring backup without validation --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index e86779a9a..b23fa073b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -405,7 +405,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup->status == BACKUP_STATUS_DONE) { if (rt->no_validate) - elog(INFO, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); else elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); } From f84b21ca24a98fcb125b9733fb479f01495c75ed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 May 2019 18:01:56 +0300 Subject: [PATCH 0660/2107] Release 2.1.1 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 346a7ebaf..f759773cf 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -189,8 +189,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.0" -#define AGENT_PROTOCOL_VERSION 20100 +#define PROGRAM_VERSION "2.1.1" +#define AGENT_PROTOCOL_VERSION 20101 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 7ed9f224a..3d8a86e73 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.0 \ No newline at end of file +pg_probackup 2.1.1 \ No newline at end of file From 9dacdaad8fccdd1629ccb2474a46aa7c10a64c66 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 6 May 2019 11:16:00 +0300 Subject: [PATCH 0661/2107] remove unused code related to remote backup via replication protocol, which was not implemented --- src/backup.c | 380 ++------------------------------------------- src/pg_probackup.h | 1 - 2 files changed, 10 insertions(+), 371 deletions(-) diff --git a/src/backup.c b/src/backup.c index 528fd1045..5dacc3bfc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -81,7 +81,6 @@ bool heapallindexed_is_supported = false; /* Backup connections */ static PGconn *backup_conn = NULL; static PGconn *master_conn = NULL; -static PGconn *backup_conn_replication = NULL; /* PostgreSQL server version from "backup_conn" */ static int server_version = 0; @@ -101,7 +100,6 @@ static void backup_disconnect(bool fatal, void *userdata); static void pgdata_basic_setup(bool amcheck_only); static void *backup_files(void *arg); -static void *remote_backup_files(void *arg); static void do_backup_instance(void); @@ -127,9 +125,6 @@ static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); static void make_pagemap_from_ptrack(parray *files); static void *StreamLog(void *arg); -static void get_remote_pgdata_filelist(parray *files); -static void ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum); -static void remote_copy_file(PGconn *conn, pgFile* file); static void check_external_for_tablespaces(parray *external_list); /* Ptrack functions */ @@ -157,324 +152,6 @@ static void set_cfs_datafiles(parray *files, const char *root, char *relative, s exit(code); \ } -/* Fill "files" with data about all the files to backup */ -static void -get_remote_pgdata_filelist(parray *files) -{ - PGresult *res; - int resultStatus; - int i; - - backup_conn_replication = pgut_connect_replication(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - - if (PQsendQuery(backup_conn_replication, "FILE_BACKUP FILELIST") == 0) - elog(ERROR,"%s: could not send replication command \"%s\": %s", - PROGRAM_NAME, "FILE_BACKUP", PQerrorMessage(backup_conn_replication)); - - res = PQgetResult(backup_conn_replication); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - resultStatus = PQresultStatus(res); - PQclear(res); - elog(ERROR, "cannot start getting FILE_BACKUP filelist: %s, result_status %d", - PQerrorMessage(backup_conn_replication), resultStatus); - } - - if (PQntuples(res) < 1) - elog(ERROR, "%s: no data returned from server", PROGRAM_NAME); - - for (i = 0; i < PQntuples(res); i++) - { - ReceiveFileList(files, backup_conn_replication, res, i); - } - - res = PQgetResult(backup_conn_replication); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - elog(ERROR, "%s: final receive failed: %s", - PROGRAM_NAME, PQerrorMessage(backup_conn_replication)); - } - - PQfinish(backup_conn_replication); -} - -/* - * workhorse for get_remote_pgdata_filelist(). - * Parse received message into pgFile structure. - */ -static void -ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) -{ - char filename[MAXPGPATH]; - pgoff_t current_len_left = 0; - bool basetablespace; - char *copybuf = NULL; - pgFile *pgfile; - - /* What for do we need this basetablespace field?? */ - basetablespace = PQgetisnull(res, rownum, 0); - if (basetablespace) - elog(LOG,"basetablespace"); - else - elog(LOG, "basetablespace %s", PQgetvalue(res, rownum, 1)); - - res = PQgetResult(conn); - - if (PQresultStatus(res) != PGRES_COPY_OUT) - elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(conn)); - - while (1) - { - int r; - int filemode; - - if (copybuf != NULL) - { - PQfreemem(copybuf); - copybuf = NULL; - } - - r = PQgetCopyData(conn, ©buf, 0); - - if (r == -2) - elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); - - /* end of copy */ - if (r == -1) - break; - - /* This must be the header for a new file */ - if (r != 512) - elog(ERROR, "Invalid tar block header size: %d\n", r); - - current_len_left = read_tar_number(©buf[124], 12); - - /* Set permissions on the file */ - filemode = read_tar_number(©buf[100], 8); - - /* First part of header is zero terminated filename */ - snprintf(filename, sizeof(filename), "%s", copybuf); - - pgfile = pgFileInit(filename); - pgfile->size = current_len_left; - pgfile->mode |= filemode; - - if (filename[strlen(filename) - 1] == '/') - { - /* Symbolic link or directory has size zero */ - Assert (pgfile->size == 0); - /* Ends in a slash means directory or symlink to directory */ - if (copybuf[156] == '5') - { - /* Directory */ - pgfile->mode |= S_IFDIR; - } - else if (copybuf[156] == '2') - { - /* Symlink */ -#ifndef WIN32 - pgfile->mode |= S_IFLNK; -#else - pgfile->mode |= S_IFDIR; -#endif - } - else - elog(ERROR, "Unrecognized link indicator \"%c\"\n", - copybuf[156]); - } - else - { - /* regular file */ - pgfile->mode |= S_IFREG; - } - - parray_append(files, pgfile); - } - - if (copybuf != NULL) - PQfreemem(copybuf); -} - -/* read one file via replication protocol - * and write it to the destination subdir in 'backup_path' */ -static void -remote_copy_file(PGconn *conn, pgFile* file) -{ - PGresult *res; - char *copybuf = NULL; - char buf[32768]; - FILE *out; - char database_path[MAXPGPATH]; - char to_path[MAXPGPATH]; - bool skip_padding = false; - - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - join_path_components(to_path, database_path, file->path); - - out = fopen(to_path, PG_BINARY_W); - if (out == NULL) - { - int errno_tmp = errno; - elog(ERROR, "cannot open destination file \"%s\": %s", - to_path, strerror(errno_tmp)); - } - - INIT_FILE_CRC32(true, file->crc); - - /* read from stream and write to backup file */ - while (1) - { - int row_length; - int errno_tmp; - int write_buffer_size = 0; - if (copybuf != NULL) - { - PQfreemem(copybuf); - copybuf = NULL; - } - - row_length = PQgetCopyData(conn, ©buf, 0); - - if (row_length == -2) - elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(conn)); - - if (row_length == -1) - break; - - if (!skip_padding) - { - write_buffer_size = Min(row_length, sizeof(buf)); - memcpy(buf, copybuf, write_buffer_size); - COMP_FILE_CRC32(true, file->crc, buf, write_buffer_size); - - /* TODO calc checksum*/ - if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size) - { - errno_tmp = errno; - /* oops */ - FIN_FILE_CRC32(true, file->crc); - fclose(out); - PQfinish(conn); - elog(ERROR, "cannot write to \"%s\": %s", to_path, - strerror(errno_tmp)); - } - - file->read_size += write_buffer_size; - } - if (file->read_size >= file->size) - { - skip_padding = true; - } - } - - res = PQgetResult(conn); - - /* File is not found. That's normal. */ - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - elog(ERROR, "final receive failed: status %d ; %s",PQresultStatus(res), PQerrorMessage(conn)); - } - - file->write_size = (int64) file->read_size; - FIN_FILE_CRC32(true, file->crc); - - fclose(out); -} - -/* - * Take a remote backup of the PGDATA at a file level. - * Copy all directories and files listed in backup_files_list. - */ -static void * -remote_backup_files(void *arg) -{ - int i; - backup_files_arg *arguments = (backup_files_arg *) arg; - int n_backup_files_list = parray_num(arguments->files_list); - PGconn *file_backup_conn = NULL; - - for (i = 0; i < n_backup_files_list; i++) - { - char *query_str; - PGresult *res; - char *copybuf = NULL; - pgFile *file; - int row_length; - - file = (pgFile *) parray_get(arguments->files_list, i); - - /* We have already copied all directories */ - if (S_ISDIR(file->mode)) - continue; - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - file_backup_conn = pgut_connect_replication(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during backup"); - - query_str = psprintf("FILE_BACKUP FILEPATH '%s'",file->path); - - if (PQsendQuery(file_backup_conn, query_str) == 0) - elog(ERROR,"%s: could not send replication command \"%s\": %s", - PROGRAM_NAME, query_str, PQerrorMessage(file_backup_conn)); - - res = PQgetResult(file_backup_conn); - - /* File is not found. That's normal. */ - if (PQresultStatus(res) == PGRES_COMMAND_OK) - { - PQclear(res); - PQfinish(file_backup_conn); - continue; - } - - if (PQresultStatus(res) != PGRES_COPY_OUT) - { - PQclear(res); - PQfinish(file_backup_conn); - elog(ERROR, "Could not get COPY data stream: %s", PQerrorMessage(file_backup_conn)); - } - - /* read the header of the file */ - row_length = PQgetCopyData(file_backup_conn, ©buf, 0); - - if (row_length == -2) - elog(ERROR, "Could not read COPY data: %s", PQerrorMessage(file_backup_conn)); - - /* end of copy TODO handle it */ - if (row_length == -1) - elog(ERROR, "Unexpected end of COPY data"); - - if(row_length != 512) - elog(ERROR, "Invalid tar block header size: %d\n", row_length); - file->size = read_tar_number(©buf[124], 12); - - /* receive the data from stream and write to backup file */ - remote_copy_file(file_backup_conn, file); - - elog(VERBOSE, "File \"%s\". Copied " INT64_FORMAT " bytes", - file->path, file->write_size); - PQfinish(file_backup_conn); - } - - /* Data files transferring is successful */ - arguments->ret = 0; - - return NULL; -} - /* * Take a backup of a single postgresql instance. * Move files from 'pgdata' to a subdirectory in 'backup_path'. @@ -513,32 +190,7 @@ do_backup_instance(void) current.data_bytes = 0; /* Obtain current timeline */ - if (IsReplicationProtocol()) - { - char *sysidentifier; - TimeLineID starttli; - XLogRecPtr startpos; - - backup_conn_replication = pgut_connect_replication(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - - /* Check replication prorocol connection */ - if (!RunIdentifySystem(backup_conn_replication, &sysidentifier, &starttli, &startpos, NULL)) - elog(ERROR, "Failed to send command for remote backup"); - -// TODO implement the check -// if (&sysidentifier != instance_config.system_identifier) -// elog(ERROR, "Backup data directory was initialized for system id %ld, but target backup directory system id is %ld", -// instance_config.system_identifier, sysidentifier); - - current.tli = starttli; - - PQfinish(backup_conn_replication); - } - else - current.tli = get_current_timeline(false); + current.tli = get_current_timeline(false); /* * In incremental backup mode ensure that already-validated @@ -664,12 +316,8 @@ do_backup_instance(void) backup_files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - - if (IsReplicationProtocol()) - get_remote_pgdata_filelist(backup_files_list); - else - dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, 0, FIO_DB_HOST); + dir_list_file(backup_files_list, instance_config.pgdata, + true, true, false, 0, FIO_DB_HOST); /* * Append to backup list all files and directories @@ -749,15 +397,12 @@ do_backup_instance(void) char dirpath[MAXPGPATH]; char *dir_name; - if (!IsReplicationProtocol()) - if (file->external_dir_num) - dir_name = GetRelativePath(file->path, - parray_get(external_dirs, - file->external_dir_num - 1)); - else - dir_name = GetRelativePath(file->path, instance_config.pgdata); + if (file->external_dir_num) + dir_name = GetRelativePath(file->path, + parray_get(external_dirs, + file->external_dir_num - 1)); else - dir_name = file->path; + dir_name = GetRelativePath(file->path, instance_config.pgdata); elog(VERBOSE, "Create directory \"%s\"", dir_name); @@ -812,11 +457,7 @@ do_backup_instance(void) backup_files_arg *arg = &(threads_args[i]); elog(VERBOSE, "Start thread num: %i", i); - - if (!IsReplicationProtocol()) - pthread_create(&threads[i], NULL, backup_files, arg); - else - pthread_create(&threads[i], NULL, remote_backup_files, arg); + pthread_create(&threads[i], NULL, backup_files, arg); } /* Wait threads */ @@ -1234,8 +875,7 @@ pgdata_basic_setup(bool amcheck_only) * instance we opened connection to. And that target backup database PGDATA * belogns to the same instance. */ - /* TODO fix it for remote backup */ - if (!IsReplicationProtocol() && !amcheck_only) + if (!amcheck_only) check_system_identifiers(); if (current.checksum_version) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f759773cf..b17917669 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -391,7 +391,6 @@ typedef struct BackupPageHeader #endif #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) -#define IsReplicationProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "replication") == 0) /* directory options */ extern char *pg_probackup; From 14418a200ffa2a60c2d5f729f4813a65705ab6ba Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 6 May 2019 11:27:42 +0300 Subject: [PATCH 0662/2107] refactoring. move WAL related functions to archive.c --- src/archive.c | 446 +++++++++++++++++++++++++++++++++++++++++++++ src/data.c | 431 ------------------------------------------- src/pg_probackup.h | 3 - 3 files changed, 446 insertions(+), 434 deletions(-) diff --git a/src/archive.c b/src/archive.c index fe42a728d..5a1053cd1 100644 --- a/src/archive.c +++ b/src/archive.c @@ -12,6 +12,19 @@ #include +static void push_wal_file(const char *from_path, const char *to_path, + bool is_compress, bool overwrite); +static void get_wal_file(const char *from_path, const char *to_path); +#ifdef HAVE_LIBZ +static const char *get_gz_error(gzFile gzf, int errnum); +#endif +static bool fileEqualCRC(const char *path1, const char *path2, + bool path2_is_compressed); +static void copy_file_attributes(const char *from_path, + fio_location from_location, + const char *to_path, fio_location to_location, + bool unlink_on_error); + /* * pg_probackup specific archive command for archive backups * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup @@ -114,3 +127,436 @@ do_archive_get(char *wal_file_path, char *wal_file_name) return 0; } + +/* ------------- INTERNAL FUNCTIONS ---------- */ +/* + * Copy WAL segment from pgdata to archive catalog with possible compression. + */ +void +push_wal_file(const char *from_path, const char *to_path, bool is_compress, + bool overwrite) +{ + FILE *in = NULL; + int out = -1; + char buf[XLOG_BLCKSZ]; + const char *to_path_p; + char to_path_temp[MAXPGPATH]; + int errno_temp; + +#ifdef HAVE_LIBZ + char gz_to_path[MAXPGPATH]; + gzFile gz_out = NULL; + + if (is_compress) + { + snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); + to_path_p = gz_to_path; + } + else +#endif + to_path_p = to_path; + + /* open file for read */ + in = fio_fopen(from_path, PG_BINARY_R, FIO_DB_HOST); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, + strerror(errno)); + + /* Check if possible to skip copying */ + if (fileExists(to_path_p, FIO_BACKUP_HOST)) + { + if (fileEqualCRC(from_path, to_path_p, is_compress)) + return; + /* Do not copy and do not rise error. Just quit as normal. */ + else if (!overwrite) + elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p); + } + + /* open backup file for write */ +#ifdef HAVE_LIBZ + if (is_compress) + { + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); + if (gz_out == NULL) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + else +#endif + { + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + + /* copy content */ + for (;;) + { + ssize_t read_len = 0; + + read_len = fio_fread(in, buf, sizeof(buf)); + + if (read_len < 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, + "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + if (read_len > 0) + { +#ifdef HAVE_LIBZ + if (is_compress) + { + if (fio_gzwrite(gz_out, buf, read_len) != read_len) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fio_write(out, buf, read_len) != read_len) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + } + + if (read_len == 0) + break; + } + +#ifdef HAVE_LIBZ + if (is_compress) + { + if (fio_gzclose(gz_out) != 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + to_path_temp, get_gz_error(gz_out, errno_temp)); + } + } + else +#endif + { + if (fio_flush(out) != 0 || fio_close(out) != 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + } + + if (fio_fclose(in)) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + + /* update file permission. */ + copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); + + if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path_p, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_compress) + elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); +#endif +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + */ +void +get_wal_file(const char *from_path, const char *to_path) +{ + FILE *in = NULL; + int out; + char buf[XLOG_BLCKSZ]; + const char *from_path_p = from_path; + char to_path_temp[MAXPGPATH]; + int errno_temp; + bool is_decompress = false; + +#ifdef HAVE_LIBZ + char gz_from_path[MAXPGPATH]; + gzFile gz_in = NULL; +#endif + + /* First check source file for existance */ + if (fio_access(from_path, F_OK, FIO_BACKUP_HOST) != 0) + { +#ifdef HAVE_LIBZ + /* + * Maybe we need to decompress the file. Check it with .gz + * extension. + */ + snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); + if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0) + { + /* Found compressed file */ + is_decompress = true; + from_path_p = gz_from_path; + } +#endif + /* Didn't find compressed file */ + if (!is_decompress) + elog(ERROR, "Source WAL file \"%s\" doesn't exist", + from_path); + } + + /* open file for read */ + if (!is_decompress) + { + in = fio_fopen(from_path, PG_BINARY_R, FIO_BACKUP_HOST); + if (in == NULL) + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); + } +#ifdef HAVE_LIBZ + else + { + gz_in = fio_gzopen(gz_from_path, PG_BINARY_R, Z_DEFAULT_COMPRESSION, + FIO_BACKUP_HOST); + if (gz_in == NULL) + elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", + gz_from_path, strerror(errno)); + } +#endif + + /* open backup file for write */ + snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + + out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_DB_HOST); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + + /* copy content */ + for (;;) + { + int read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = fio_gzread(gz_in, buf, sizeof(buf)); + if (read_len <= 0 && !fio_gzeof(gz_in)) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + read_len = fio_fread(in, buf, sizeof(buf)); + if (read_len < 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + if (read_len > 0) + { + if (fio_write(out, buf, read_len) != read_len) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, + strerror(errno_temp)); + } + } + + /* Check for EOF */ +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (fio_gzeof(gz_in) || read_len == 0) + break; + } + else +#endif + { + if (/* feof(in) || */ read_len == 0) + break; + } + } + + if (fio_flush(out) != 0 || fio_close(out) != 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot write WAL file \"%s\": %s", + to_path_temp, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + { + if (fio_gzclose(gz_in) != 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + gz_from_path, get_gz_error(gz_in, errno_temp)); + } + } + else +#endif + { + if (fio_fclose(in)) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot close source WAL file \"%s\": %s", + from_path, strerror(errno_temp)); + } + } + + /* update file permission. */ + copy_file_attributes(from_path_p, FIO_BACKUP_HOST, to_path_temp, FIO_DB_HOST, true); + + if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) + { + errno_temp = errno; + fio_unlink(to_path_temp, FIO_DB_HOST); + elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", + to_path_temp, to_path, strerror(errno_temp)); + } + +#ifdef HAVE_LIBZ + if (is_decompress) + elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); +#endif +} + +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = fio_gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* + * compare CRC of two WAL files. + * If necessary, decompress WAL file from path2 + */ +static bool +fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) +{ + pg_crc32 crc1; + pg_crc32 crc2; + + /* Get checksum of backup file */ +#ifdef HAVE_LIBZ + if (path2_is_compressed) + { + char buf [1024]; + gzFile gz_in = NULL; + + INIT_FILE_CRC32(true, crc2); + gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST); + if (gz_in == NULL) + /* File cannot be read */ + elog(ERROR, + "Cannot compare WAL file \"%s\" with compressed \"%s\"", + path1, path2); + + for (;;) + { + int read_len = fio_gzread(gz_in, buf, sizeof(buf)); + if (read_len <= 0 && !fio_gzeof(gz_in)) + { + /* An error occurred while reading the file */ + elog(WARNING, + "Cannot compare WAL file \"%s\" with compressed \"%s\": %d", + path1, path2, read_len); + return false; + } + COMP_FILE_CRC32(true, crc2, buf, read_len); + if (fio_gzeof(gz_in) || read_len == 0) + break; + } + FIN_FILE_CRC32(true, crc2); + + if (fio_gzclose(gz_in) != 0) + elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", + path2, get_gz_error(gz_in, errno)); + } + else +#endif + { + crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST); + } + + /* Get checksum of original file */ + crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST); + + return EQ_CRC32C(crc1, crc2); +} + +/* Copy file attributes */ +static void +copy_file_attributes(const char *from_path, fio_location from_location, + const char *to_path, fio_location to_location, + bool unlink_on_error) +{ + struct stat st; + + if (fio_stat(from_path, &st, true, from_location) == -1) + { + if (unlink_on_error) + fio_unlink(to_path, to_location); + elog(ERROR, "Cannot stat file \"%s\": %s", + from_path, strerror(errno)); + } + + if (fio_chmod(to_path, st.st_mode, to_location) == -1) + { + if (unlink_on_error) + fio_unlink(to_path, to_location); + elog(ERROR, "Cannot change mode of file \"%s\": %s", + to_path, strerror(errno)); + } +} diff --git a/src/data.c b/src/data.c index d15938901..3480cdd62 100644 --- a/src/data.c +++ b/src/data.c @@ -31,9 +31,6 @@ typedef union DataPage char data[BLCKSZ]; } DataPage; -static bool -fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed); - #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ static int32 @@ -1071,380 +1068,6 @@ copy_file(const char *from_root, fio_location from_location, return true; } -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf, int errnum) -{ - int gz_errnum; - const char *errmsg; - - errmsg = fio_gzerror(gzf, &gz_errnum); - if (gz_errnum == Z_ERRNO) - return strerror(errnum); - else - return errmsg; -} -#endif - -/* - * Copy file attributes - */ -static void -copy_meta(const char *from_path, fio_location from_location, const char *to_path, fio_location to_location, bool unlink_on_error) -{ - struct stat st; - - if (fio_stat(from_path, &st, true, from_location) == -1) - { - if (unlink_on_error) - fio_unlink(to_path, to_location); - elog(ERROR, "Cannot stat file \"%s\": %s", - from_path, strerror(errno)); - } - - if (fio_chmod(to_path, st.st_mode, to_location) == -1) - { - if (unlink_on_error) - fio_unlink(to_path, to_location); - elog(ERROR, "Cannot change mode of file \"%s\": %s", - to_path, strerror(errno)); - } -} - -/* - * Copy WAL segment from pgdata to archive catalog with possible compression. - */ -void -push_wal_file(const char *from_path, const char *to_path, bool is_compress, - bool overwrite) -{ - FILE *in = NULL; - int out = -1; - char buf[XLOG_BLCKSZ]; - const char *to_path_p; - char to_path_temp[MAXPGPATH]; - int errno_temp; - -#ifdef HAVE_LIBZ - char gz_to_path[MAXPGPATH]; - gzFile gz_out = NULL; - - if (is_compress) - { - snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); - to_path_p = gz_to_path; - } - else -#endif - to_path_p = to_path; - - /* open file for read */ - in = fio_fopen(from_path, PG_BINARY_R, FIO_DB_HOST); - if (in == NULL) - elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, - strerror(errno)); - - /* Check if possible to skip copying */ - if (fileExists(to_path_p, FIO_BACKUP_HOST)) - { - if (fileEqualCRC(from_path, to_path_p, is_compress)) - return; - /* Do not copy and do not rise error. Just quit as normal. */ - else if (!overwrite) - elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p); - } - - /* open backup file for write */ -#ifdef HAVE_LIBZ - if (is_compress) - { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); - - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); - if (gz_out == NULL) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); - } - else -#endif - { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); - } - - /* copy content */ - for (;;) - { - ssize_t read_len = 0; - - read_len = fio_fread(in, buf, sizeof(buf)); - - if (read_len < 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, - "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); - } - - if (read_len > 0) - { -#ifdef HAVE_LIBZ - if (is_compress) - { - if (fio_gzwrite(gz_out, buf, read_len) != read_len) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", - to_path_temp, get_gz_error(gz_out, errno_temp)); - } - } - else -#endif - { - if (fio_write(out, buf, read_len) != read_len) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); - } - } - } - - if (read_len == 0) - break; - } - -#ifdef HAVE_LIBZ - if (is_compress) - { - if (fio_gzclose(gz_out) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - to_path_temp, get_gz_error(gz_out, errno_temp)); - } - } - else -#endif - { - if (fio_flush(out) != 0 || fio_close(out) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); - } - } - - if (fio_fclose(in)) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); - } - - /* update file permission. */ - copy_meta(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); - - if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", - to_path_temp, to_path_p, strerror(errno_temp)); - } - -#ifdef HAVE_LIBZ - if (is_compress) - elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); -#endif -} - -/* - * Copy WAL segment from archive catalog to pgdata with possible decompression. - */ -void -get_wal_file(const char *from_path, const char *to_path) -{ - FILE *in = NULL; - int out; - char buf[XLOG_BLCKSZ]; - const char *from_path_p = from_path; - char to_path_temp[MAXPGPATH]; - int errno_temp; - bool is_decompress = false; - -#ifdef HAVE_LIBZ - char gz_from_path[MAXPGPATH]; - gzFile gz_in = NULL; -#endif - - /* First check source file for existance */ - if (fio_access(from_path, F_OK, FIO_BACKUP_HOST) != 0) - { -#ifdef HAVE_LIBZ - /* - * Maybe we need to decompress the file. Check it with .gz - * extension. - */ - snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0) - { - /* Found compressed file */ - is_decompress = true; - from_path_p = gz_from_path; - } -#endif - /* Didn't find compressed file */ - if (!is_decompress) - elog(ERROR, "Source WAL file \"%s\" doesn't exist", - from_path); - } - - /* open file for read */ - if (!is_decompress) - { - in = fio_fopen(from_path, PG_BINARY_R, FIO_BACKUP_HOST); - if (in == NULL) - elog(ERROR, "Cannot open source WAL file \"%s\": %s", - from_path, strerror(errno)); - } -#ifdef HAVE_LIBZ - else - { - gz_in = fio_gzopen(gz_from_path, PG_BINARY_R, Z_DEFAULT_COMPRESSION, - FIO_BACKUP_HOST); - if (gz_in == NULL) - elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", - gz_from_path, strerror(errno)); - } -#endif - - /* open backup file for write */ - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); - - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_DB_HOST); - if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); - - /* copy content */ - for (;;) - { - int read_len = 0; - -#ifdef HAVE_LIBZ - if (is_decompress) - { - read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len <= 0 && !fio_gzeof(gz_in)) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", - gz_from_path, get_gz_error(gz_in, errno_temp)); - } - } - else -#endif - { - read_len = fio_fread(in, buf, sizeof(buf)); - if (read_len < 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); - } - } - - if (read_len > 0) - { - if (fio_write(out, buf, read_len) != read_len) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, - strerror(errno_temp)); - } - } - - /* Check for EOF */ -#ifdef HAVE_LIBZ - if (is_decompress) - { - if (fio_gzeof(gz_in) || read_len == 0) - break; - } - else -#endif - { - if (/* feof(in) || */ read_len == 0) - break; - } - } - - if (fio_flush(out) != 0 || fio_close(out) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot write WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); - } - -#ifdef HAVE_LIBZ - if (is_decompress) - { - if (fio_gzclose(gz_in) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - gz_from_path, get_gz_error(gz_in, errno_temp)); - } - } - else -#endif - { - if (fio_fclose(in)) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot close source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); - } - } - - /* update file permission. */ - copy_meta(from_path_p, FIO_BACKUP_HOST, to_path_temp, FIO_DB_HOST, true); - - if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", - to_path_temp, to_path, strerror(errno_temp)); - } - -#ifdef HAVE_LIBZ - if (is_decompress) - elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); -#endif -} - /* * Calculate checksum of various files which are not copied from PGDATA, * but created in process of backup, such as stream XLOG files, @@ -1794,57 +1417,3 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, return is_valid; } - -static bool -fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) -{ - pg_crc32 crc1; - pg_crc32 crc2; - - /* Get checksum of backup file */ -#ifdef HAVE_LIBZ - if (path2_is_compressed) - { - char buf [1024]; - gzFile gz_in = NULL; - - INIT_FILE_CRC32(true, crc2); - gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST); - if (gz_in == NULL) - /* File cannot be read */ - elog(ERROR, - "Cannot compare WAL file \"%s\" with compressed \"%s\"", - path1, path2); - - for (;;) - { - int read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len <= 0 && !fio_gzeof(gz_in)) - { - /* An error occurred while reading the file */ - elog(WARNING, - "Cannot compare WAL file \"%s\" with compressed \"%s\": %d", - path1, path2, read_len); - return false; - } - COMP_FILE_CRC32(true, crc2, buf, read_len); - if (fio_gzeof(gz_in) || read_len == 0) - break; - } - FIN_FILE_CRC32(true, crc2); - - if (fio_gzclose(gz_in) != 0) - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - path2, get_gz_error(gz_in, errno)); - } - else -#endif - { - crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST); - } - - /* Get checksum of original file */ - crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST); - - return EQ_CRC32C(crc1, crc2); -} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b17917669..72d41ba3d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -624,9 +624,6 @@ extern void restore_data_file(const char *to_path, uint32 backup_version); extern bool copy_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); -extern void push_wal_file(const char *from_path, const char *to_path, - bool is_compress, bool overwrite); -extern void get_wal_file(const char *from_path, const char *to_path); extern void calc_file_checksum(pgFile *file, fio_location location); From 109af56a50c8b75a706ca8a65f6043b7b29734d9 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 6 May 2019 11:32:34 +0300 Subject: [PATCH 0663/2107] refactoring. remove unneeded calc_file_checksum() function --- src/backup.c | 20 ++++++++++++++++---- src/data.c | 14 -------------- src/pg_probackup.h | 2 -- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5dacc3bfc..b4c329028 100644 --- a/src/backup.c +++ b/src/backup.c @@ -537,7 +537,11 @@ do_backup_instance(void) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); if (S_ISREG(file->mode)) - calc_file_checksum(file, FIO_BACKUP_HOST); + { + file->crc = pgFileGetCRC(file->path, true, false, + &file->read_size, FIO_BACKUP_HOST); + file->write_size = file->read_size; + } /* Remove file path root prefix*/ if (strstr(file->path, database_path) == file->path) { @@ -2022,7 +2026,9 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { file = pgFileNew(backup_label, true, 0, FIO_BACKUP_HOST); - calc_file_checksum(file, FIO_BACKUP_HOST); + file->crc = pgFileGetCRC(file->path, true, false, + &file->read_size, FIO_BACKUP_HOST); + file->write_size = file->read_size; free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); @@ -2066,7 +2072,11 @@ pg_stop_backup(pgBackup *backup) { file = pgFileNew(tablespace_map, true, 0, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) - calc_file_checksum(file, FIO_BACKUP_HOST); + { + file->crc = pgFileGetCRC(file->path, true, false, + &file->read_size, FIO_BACKUP_HOST); + file->write_size = file->read_size; + } free(file->path); file->path = strdup(PG_TABLESPACE_MAP_FILE); parray_append(backup_files_list, file); @@ -2475,7 +2485,9 @@ backup_files(void *arg) if (prev_file && file->exists_in_prev && buf.st_mtime < current.parent_backup) { - calc_file_checksum(file, FIO_DB_HOST); + file->crc = pgFileGetCRC(file->path, true, false, + &file->read_size, FIO_DB_HOST); + file->write_size = file->read_size; /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc)) skip = true; /* ...skip copying file. */ diff --git a/src/data.c b/src/data.c index 3480cdd62..e0e0fd010 100644 --- a/src/data.c +++ b/src/data.c @@ -1068,20 +1068,6 @@ copy_file(const char *from_root, fio_location from_location, return true; } -/* - * Calculate checksum of various files which are not copied from PGDATA, - * but created in process of backup, such as stream XLOG files, - * PG_TABLESPACE_MAP_FILE and PG_BACKUP_LABEL_FILE. - */ -void -calc_file_checksum(pgFile *file, fio_location location) -{ - Assert(S_ISREG(file->mode)); - - file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, location); - file->write_size = file->read_size; -} - /* * Validate given page. * diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 72d41ba3d..f76b7da0f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -625,8 +625,6 @@ extern void restore_data_file(const char *to_path, extern bool copy_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); -extern void calc_file_checksum(pgFile *file, fio_location location); - extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ From c411a8cffdbe2e854472e1639a9271c1d15dc8ad Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 6 May 2019 11:43:26 +0300 Subject: [PATCH 0664/2107] refactoring. remove unused move_file(), remove_trailing_space(), remove_not_digit() functions --- src/pg_probackup.h | 8 +++----- src/util.c | 38 -------------------------------------- 2 files changed, 3 insertions(+), 43 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f76b7da0f..07bcb0ed0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -622,8 +622,9 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern bool copy_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); -extern void move_file(const char *from_root, const char *to_root, pgFile *file); +extern bool copy_file(const char *from_root, fio_location from_location, + const char *to_root, fio_location to_location, + pgFile *file); extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); @@ -659,11 +660,8 @@ extern void set_min_recovery_point(pgFile *file, const char *backup_path, extern void copy_pgcontrol_file(const char *from_root, fio_location location, const char *to_root, fio_location to_location, pgFile *file); -extern void sanityChecks(void); extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); -extern void remove_trailing_space(char *buf, int comment_mark); -extern void remove_not_digit(char *buf, size_t len, const char *str); extern const char *base36enc(long unsigned int value); extern char *base36enc_dup(long unsigned int value); extern long unsigned int base36dec(const char *text); diff --git a/src/util.c b/src/util.c index a279959c1..6f7e32816 100644 --- a/src/util.c +++ b/src/util.c @@ -457,41 +457,3 @@ status2str(BackupStatus status) return statusName[status]; } - -void -remove_trailing_space(char *buf, int comment_mark) -{ - int i; - char *last_char = NULL; - - for (i = 0; buf[i]; i++) - { - if (buf[i] == comment_mark || buf[i] == '\n' || buf[i] == '\r') - { - buf[i] = '\0'; - break; - } - } - for (i = 0; buf[i]; i++) - { - if (!isspace(buf[i])) - last_char = buf + i; - } - if (last_char != NULL) - *(last_char + 1) = '\0'; - -} - -void -remove_not_digit(char *buf, size_t len, const char *str) -{ - int i, j; - - for (i = 0, j = 0; str[i] && j < len; i++) - { - if (!isdigit(str[i])) - continue; - buf[j++] = str[i]; - } - buf[j] = '\0'; -} From 7f1fb4777a197d1eb38f8fd781e6c39f31a29bdc Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 6 May 2019 12:32:43 +0300 Subject: [PATCH 0665/2107] Issue #63: Do not copy unnecessary files --- src/backup.c | 15 ++- src/data.c | 6 +- src/dir.c | 192 +++++++++++++++++++----------------- src/merge.c | 7 +- src/pg_probackup.h | 15 ++- src/restore.c | 240 ++++++++++++++++----------------------------- src/util.c | 5 +- 7 files changed, 211 insertions(+), 269 deletions(-) diff --git a/src/backup.c b/src/backup.c index 528fd1045..e7acdd2e1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -259,7 +259,7 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum) /* First part of header is zero terminated filename */ snprintf(filename, sizeof(filename), "%s", copybuf); - pgfile = pgFileInit(filename); + pgfile = pgFileInit(filename, filename); pgfile->size = current_len_left; pgfile->mode |= filemode; @@ -2381,7 +2381,8 @@ pg_stop_backup(pgBackup *backup) */ if (backup_files_list) { - file = pgFileNew(backup_label, true, 0, FIO_BACKUP_HOST); + file = pgFileNew(backup_label, backup_label, true, 0, + FIO_BACKUP_HOST); calc_file_checksum(file, FIO_BACKUP_HOST); free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); @@ -2424,7 +2425,8 @@ pg_stop_backup(pgBackup *backup) if (backup_files_list) { - file = pgFileNew(tablespace_map, true, 0, FIO_BACKUP_HOST); + file = pgFileNew(tablespace_map, tablespace_map, true, 0, + FIO_BACKUP_HOST); if (S_ISREG(file->mode)) calc_file_checksum(file, FIO_BACKUP_HOST); free(file->path); @@ -2826,7 +2828,6 @@ backup_files(void *arg) file); else { - const char *src; const char *dst; bool skip = false; char external_dst[MAXPGPATH]; @@ -2846,16 +2847,12 @@ backup_files(void *arg) makeExternalDirPathByNum(external_dst, arguments->external_prefix, file->external_dir_num); - src = external_path; dst = external_dst; } else - { - src = arguments->from_root; dst = arguments->to_root; - } if (skip || - !copy_file(src, FIO_DB_HOST, dst, FIO_BACKUP_HOST, file)) + !copy_file(FIO_DB_HOST, dst, FIO_BACKUP_HOST, file)) { /* disappeared file not to be confused with 'not changed' */ if (file->write_size != FILE_NOT_FOUND) diff --git a/src/data.c b/src/data.c index d15938901..cf020a1ed 100644 --- a/src/data.c +++ b/src/data.c @@ -941,8 +941,8 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * it is either small control file or already compressed cfs file. */ bool -copy_file(const char *from_root, fio_location from_location, - const char *to_root, fio_location to_location, pgFile *file) +copy_file(fio_location from_location, const char *to_root, + fio_location to_location, pgFile *file) { char to_path[MAXPGPATH]; FILE *in; @@ -979,7 +979,7 @@ copy_file(const char *from_root, fio_location from_location, } /* open backup file for write */ - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + join_path_components(to_path, to_root, file->rel_path); out = fio_fopen(to_path, PG_BINARY_W, to_location); if (out == NULL) { diff --git a/src/dir.c b/src/dir.c index 13c0f7de5..b2afa3f69 100644 --- a/src/dir.c +++ b/src/dir.c @@ -120,13 +120,14 @@ typedef struct TablespaceCreatedList static int BlackListCompare(const void *str1, const void *str2); -static char dir_check_file(const char *root, pgFile *file); -static void dir_list_file_internal(parray *files, const char *root, - pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, int external_dir_num, fio_location location); - -static void list_data_directories(parray *files, const char *path, bool is_root, - bool exclude, fio_location location); +static char dir_check_file(pgFile *file); +static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list, + int external_dir_num, fio_location location); + +static void list_data_directories(parray *files, const char *root, + const char *rel_path, bool exclude, + fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -163,8 +164,8 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, bool omit_symlink, int external_dir_num, - fio_location location) +pgFileNew(const char *path, const char *rel_path, bool omit_symlink, + int external_dir_num, fio_location location) { struct stat st; pgFile *file; @@ -179,7 +180,7 @@ pgFileNew(const char *path, bool omit_symlink, int external_dir_num, strerror(errno)); } - file = pgFileInit(path); + file = pgFileInit(path, rel_path); file->size = st.st_size; file->mode = st.st_mode; file->external_dir_num = external_dir_num; @@ -188,37 +189,23 @@ pgFileNew(const char *path, bool omit_symlink, int external_dir_num, } pgFile * -pgFileInit(const char *path) +pgFileInit(const char *path, const char *rel_path) { pgFile *file; char *file_name; file = (pgFile *) pgut_malloc(sizeof(pgFile)); + MemSet(file, 0, sizeof(pgFile)); - file->name = NULL; - - file->size = 0; - file->mode = 0; - file->read_size = 0; - file->write_size = 0; - file->crc = 0; - file->is_datafile = false; - file->linked = NULL; - file->pagemap.bitmap = NULL; - file->pagemap.bitmapsize = PageBitmapIsEmpty; - file->pagemap_isabsent = false; - file->tblspcOid = 0; - file->dbOid = 0; - file->relOid = 0; - file->segno = 0; - file->is_database = false; file->forkName = pgut_malloc(MAXPGPATH); file->forkName[0] = '\0'; - file->path = pgut_malloc(strlen(path) + 1); - strcpy(file->path, path); /* enough buffer size guaranteed */ + file->path = pgut_strdup(path); canonicalize_path(file->path); + file->rel_path = pgut_strdup(rel_path); + canonicalize_path(file->rel_path); + /* Get file name from the path */ file_name = last_dir_separator(file->path); if (file_name == NULL) @@ -229,12 +216,9 @@ pgFileInit(const char *path) file->name = file_name; } - file->is_cfs = false; - file->exists_in_prev = false; /* can change only in Incremental backup. */ /* Number of blocks readed during backup */ file->n_blocks = BLOCKNUM_INVALID; - file->compress_alg = NOT_DEFINED_COMPRESS; - file->external_dir_num = 0; + return file; } @@ -341,8 +325,9 @@ pgFileFree(void *file) if (file_ptr->forkName) free(file_ptr->forkName); - free(file_ptr->path); - free(file); + pfree(file_ptr->path); + pfree(file_ptr->rel_path); + pfree(file); } /* Compare two pgFile with their path in ascending order of ASCII code. */ @@ -379,6 +364,30 @@ pgFileComparePathWithExternal(const void *f1, const void *f2) return res; } +/* + * Compare two pgFile with their relative path and external_dir_num in ascending + * order of ASСII code. + */ +int +pgFileCompareRelPathWithExternal(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + int res; + + res = strcmp(f1p->rel_path, f2p->rel_path); + if (res == 0) + { + if (f1p->external_dir_num > f2p->external_dir_num) + return 1; + else if (f1p->external_dir_num < f2p->external_dir_num) + return -1; + else + return 0; + } + return res; +} + /* Compare two pgFile with their path in descending order of ASCII code. */ int pgFileComparePathDesc(const void *f1, const void *f2) @@ -445,7 +454,8 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, join_path_components(path, backup_instance_path, PG_BLACK_LIST); /* List files with black list */ if (root && instance_config.pgdata && - strcmp(root, instance_config.pgdata) == 0 && fileExists(path, FIO_BACKUP_HOST)) + strcmp(root, instance_config.pgdata) == 0 && + fileExists(path, FIO_BACKUP_HOST)) { FILE *black_list_file = NULL; char buf[MAXPGPATH * 2]; @@ -475,7 +485,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, omit_symlink, external_dir_num, location); + file = pgFileNew(root, "", omit_symlink, external_dir_num, location); if (file == NULL) { /* For external directory this is not ok */ @@ -497,7 +507,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (add_root) parray_append(files, file); - dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list, + dir_list_file_internal(files, file, exclude, omit_symlink, black_list, external_dir_num, location); if (!add_root) @@ -528,15 +538,13 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, * - datafiles */ static char -dir_check_file(const char *root, pgFile *file) +dir_check_file(pgFile *file) { - const char *rel_path; int i; int sscanf_res; bool in_tablespace = false; - rel_path = GetRelativePath(file->path, root); - in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path); + in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path); /* Check if we need to exclude file by name */ if (S_ISREG(file->mode)) @@ -607,9 +615,9 @@ dir_check_file(const char *root, pgFile *file) * Valid path for the tablespace is * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY */ - if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path)) + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) return CHECK_FALSE; - sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s", + sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s", &tblspcOid, tmp_rel_path); if (sscanf_res == 0) return CHECK_FALSE; @@ -619,7 +627,7 @@ dir_check_file(const char *root, pgFile *file) { char tmp_rel_path[MAXPGPATH]; - sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", &(file->tblspcOid), tmp_rel_path, &(file->dbOid)); @@ -634,18 +642,18 @@ dir_check_file(const char *root, pgFile *file) strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) file->is_database = true; } - else if (path_is_prefix_of_path("global", rel_path)) + else if (path_is_prefix_of_path("global", file->rel_path)) { file->tblspcOid = GLOBALTABLESPACE_OID; if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) file->is_database = true; } - else if (path_is_prefix_of_path("base", rel_path)) + else if (path_is_prefix_of_path("base", file->rel_path)) { file->tblspcOid = DEFAULTTABLESPACE_OID; - sscanf(rel_path, "base/%u/", &(file->dbOid)); + sscanf(file->rel_path, "base/%u/", &(file->dbOid)); if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) file->is_database = true; @@ -712,12 +720,13 @@ dir_check_file(const char *root, pgFile *file) } /* - * List files in "root" directory. If "exclude" is true do not add into "files" - * files from pgdata_exclude_files and directories from pgdata_exclude_dir. + * List files in parent->path directory. If "exclude" is true do not add into + * "files" files from pgdata_exclude_files and directories from + * pgdata_exclude_dir. */ static void -dir_list_file_internal(parray *files, const char *root, pgFile *parent, - bool exclude, bool omit_symlink, parray *black_list, +dir_list_file_internal(parray *files, pgFile *parent, bool exclude, + bool omit_symlink, parray *black_list, int external_dir_num, fio_location location) { DIR *dir; @@ -735,7 +744,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, /* Maybe the directory was removed */ return; } - elog(ERROR, "cannot open directory \"%s\": %s", + elog(ERROR, "Cannot open directory \"%s\": %s", parent->path, strerror(errno)); } @@ -744,11 +753,14 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, { pgFile *file; char child[MAXPGPATH]; + char rel_child[MAXPGPATH]; char check_res; join_path_components(child, parent->path, dent->d_name); + join_path_components(rel_child, parent->rel_path, dent->d_name); - file = pgFileNew(child, omit_symlink, external_dir_num, location); + file = pgFileNew(child, rel_child, omit_symlink, external_dir_num, + location); if (file == NULL) continue; @@ -782,7 +794,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, if (exclude) { - check_res = dir_check_file(root, file); + check_res = dir_check_file(file); if (check_res == CHECK_FALSE) { /* Skip */ @@ -804,7 +816,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, * recursively. */ if (S_ISDIR(file->mode)) - dir_list_file_internal(files, root, file, exclude, omit_symlink, + dir_list_file_internal(files, file, exclude, omit_symlink, black_list, external_dir_num, location); } @@ -812,32 +824,35 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent, { int errno_tmp = errno; fio_closedir(dir); - elog(ERROR, "cannot read directory \"%s\": %s", + elog(ERROR, "Cannot read directory \"%s\": %s", parent->path, strerror(errno_tmp)); } fio_closedir(dir); } /* - * List data directories excluding directories from - * pgdata_exclude_dir array. + * List data directories excluding directories from pgdata_exclude_dir array. * - * **is_root** is a little bit hack. We exclude only first level of directories - * and on the first level we check all files and directories. + * We exclude first level of directories and on the first level we check all + * files and directories. */ static void -list_data_directories(parray *files, const char *path, bool is_root, +list_data_directories(parray *files, const char *root, const char *rel_path, bool exclude, fio_location location) { + char full_path[MAXPGPATH]; DIR *dir; struct dirent *dent; int prev_errno; bool has_child_dirs = false; + join_path_components(full_path, root, rel_path); + /* open directory and list contents */ - dir = fio_opendir(path, location); + dir = fio_opendir(full_path, location); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", full_path, + strerror(errno)); errno = 0; while ((dent = fio_readdir(dir))) @@ -851,16 +866,17 @@ list_data_directories(parray *files, const char *path, bool is_root, strcmp(dent->d_name, "..") == 0) continue; - join_path_components(child, path, dent->d_name); + /* Make full child path */ + join_path_components(child, full_path, dent->d_name); if (fio_stat(child, &st, false, location) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; /* Check for exclude for the first level of listing */ - if (is_root && exclude) + if (exclude && rel_path[0] == '\0') { int i; @@ -877,24 +893,22 @@ list_data_directories(parray *files, const char *path, bool is_root, continue; has_child_dirs = true; - list_data_directories(files, child, false, exclude, location); + /* Make relative child path */ + join_path_components(child, rel_path, dent->d_name); + list_data_directories(files, root, child, exclude, location); } /* List only full and last directories */ - if (!is_root && !has_child_dirs) - { - pgFile *dir; - - dir = pgFileNew(path, false, 0, location); - parray_append(files, dir); - } + if (rel_path[0] != '\0' && !has_child_dirs) + parray_append(files, + pgFileNew(full_path, rel_path, false, 0, location)); prev_errno = errno; fio_closedir(dir); if (prev_errno && prev_errno != ENOENT) - elog(ERROR, "cannot read directory \"%s\": %s", - path, strerror(prev_errno)); + elog(ERROR, "Cannot read directory \"%s\": %s", + full_path, strerror(prev_errno)); } /* @@ -1053,23 +1067,22 @@ create_data_directories(const char *data_dir, const char *backup_dir, } join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); - list_data_directories(dirs, backup_database_dir, true, false, + list_data_directories(dirs, backup_database_dir, "", false, FIO_BACKUP_HOST); - elog(LOG, "restore directories and symlinks..."); + elog(LOG, "Restore directories and symlinks..."); for (i = 0; i < parray_num(dirs); i++) { pgFile *dir = (pgFile *) parray_get(dirs, i); - char *relative_ptr = GetRelativePath(dir->path, backup_database_dir); Assert(S_ISDIR(dir->mode)); /* Try to create symlink and linked directory if necessary */ if (extract_tablespaces && - path_is_prefix_of_path(PG_TBLSPC_DIR, relative_ptr)) + path_is_prefix_of_path(PG_TBLSPC_DIR, dir->rel_path)) { - char *link_ptr = GetRelativePath(relative_ptr, PG_TBLSPC_DIR), + char *link_ptr = GetRelativePath(dir->rel_path, PG_TBLSPC_DIR), *link_sep, *tmp_ptr; char link_name[MAXPGPATH]; @@ -1130,10 +1143,10 @@ create_data_directories(const char *data_dir, const char *backup_dir, if (link_sep) elog(VERBOSE, "create directory \"%s\" and symbolic link \"%.*s\"", linked_path, - (int) (link_sep - relative_ptr), relative_ptr); + (int) (link_sep - dir->rel_path), dir->rel_path); else elog(VERBOSE, "create directory \"%s\" and symbolic link \"%s\"", - linked_path, relative_ptr); + linked_path, dir->rel_path); /* Firstly, create linked directory */ fio_mkdir(linked_path, DIR_PERMISSION, location); @@ -1163,10 +1176,10 @@ create_data_directories(const char *data_dir, const char *backup_dir, } create_directory: - elog(VERBOSE, "create directory \"%s\"", relative_ptr); + elog(VERBOSE, "create directory \"%s\"", dir->rel_path); /* This is not symlink, create directory */ - join_path_components(to_path, data_dir, relative_ptr); + join_path_components(to_path, data_dir, dir->rel_path); fio_mkdir(to_path, DIR_PERMISSION, location); } @@ -1619,7 +1632,7 @@ dir_read_file_list(const char *root, const char *external_prefix, else strcpy(filepath, path); - file = pgFileInit(filepath); + file = pgFileInit(filepath, path); file->write_size = (int64) write_size; file->mode = (mode_t) mode; @@ -1634,7 +1647,10 @@ dir_read_file_list(const char *root, const char *external_prefix, */ if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + { file->linked = pgut_strdup(linked); + canonicalize_path(file->linked); + } if (get_control_value(buf, "segno", NULL, &segno, false)) file->segno = (int) segno; diff --git a/src/merge.c b/src/merge.c index 1f87eb202..607f44fba 100644 --- a/src/merge.c +++ b/src/merge.c @@ -635,7 +635,6 @@ merge_files(void *arg) } else if (file->external_dir_num) { - char from_root[MAXPGPATH]; char to_root[MAXPGPATH]; int new_dir_num; char *file_external_path = parray_get(argument->from_external, @@ -644,16 +643,14 @@ merge_files(void *arg) Assert(argument->from_external); new_dir_num = get_external_index(file_external_path, argument->from_external); - makeExternalDirPathByNum(from_root, argument->from_external_prefix, - file->external_dir_num); makeExternalDirPathByNum(to_root, argument->to_external_prefix, new_dir_num); - copy_file(from_root, FIO_LOCAL_HOST, to_root, FIO_LOCAL_HOST, file); + copy_file(FIO_LOCAL_HOST, to_root, FIO_LOCAL_HOST, file); } else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); else - copy_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); + copy_file(FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); /* * We need to save compression algorithm type of the target backup to be diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f759773cf..abd13ee62 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -123,7 +123,8 @@ typedef struct pgFile pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ - char *path; /* absolute path of the file */ + char *path; /* absolute path of the file */ + char *rel_path; /* relative path of the file */ Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ Oid dbOid; /* dbOid extracted from path, if applicable */ Oid relOid; /* relOid extracted from path, if applicable */ @@ -354,6 +355,8 @@ typedef struct BackupPageHeader /* * return pointer that exceeds the length of prefix from character string. * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". + * + * Deprecated. Do not use this in new code. */ #define GetRelativePath(str, prefix) \ ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) @@ -598,14 +601,17 @@ extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); -extern pgFile *pgFileNew(const char *path, bool omit_symlink, int external_dir_num, fio_location location); -extern pgFile *pgFileInit(const char *path); +extern pgFile *pgFileNew(const char *path, const char *rel_path, + bool omit_symlink, int external_dir_num, + fio_location location); +extern pgFile *pgFileInit(const char *path, const char *rel_path); extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read, fio_location location); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathWithExternal(const void *f1, const void *f2); +extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileComparePathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); @@ -623,7 +629,8 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern bool copy_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); +extern bool copy_file(fio_location from_location, const char *to_root, + fio_location to_location, pgFile *file); extern void move_file(const char *from_root, const char *to_root, pgFile *file); extern void push_wal_file(const char *from_path, const char *to_path, bool is_compress, bool overwrite); diff --git a/src/restore.c b/src/restore.c index b23fa073b..ec7a38822 100644 --- a/src/restore.c +++ b/src/restore.c @@ -21,9 +21,10 @@ typedef struct { parray *files; pgBackup *backup; - parray *dest_external_dirs; parray *external_dirs; char *external_prefix; + parray *dest_external_dirs; + parray *dest_files; /* * Return value from the thread. @@ -32,8 +33,8 @@ typedef struct int ret; } restore_files_arg; -static void restore_backup(pgBackup *backup, const char *external_dir_str, - parray **external_dirs); +static void restore_backup(pgBackup *backup, parray *dest_external_dirs, + parray *dest_files); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); @@ -423,7 +424,47 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (is_restore) { parray *dest_external_dirs = NULL; + parray *dest_files; + char control_file[MAXPGPATH], + dest_backup_path[MAXPGPATH]; + int i; + + /* + * Preparations for actual restoring. + */ + pgBackupGetPath(dest_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + dest_files = dir_read_file_list(NULL, NULL, control_file, + FIO_BACKUP_HOST); + parray_qsort(dest_files, pgFileCompareRelPathWithExternal); + /* + * Restore dest_backup internal directories. + */ + pgBackupGetPath(dest_backup, dest_backup_path, + lengthof(dest_backup_path), NULL); + create_data_directories(instance_config.pgdata, dest_backup_path, true, + FIO_DB_HOST); + + /* + * Restore dest_backup external directories. + */ + if (dest_backup->external_dir_str && !skip_external_dirs) + { + dest_external_dirs = make_external_directory_list( + dest_backup->external_dir_str, + true); + if (parray_num(dest_external_dirs) > 0) + elog(LOG, "Restore external directories"); + + for (i = 0; i < parray_num(dest_external_dirs); i++) + fio_mkdir(parray_get(dest_external_dirs, i), + DIR_PERMISSION, FIO_DB_HOST); + } + + /* + * At least restore backups files starting from the parent backup. + */ for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -441,20 +482,15 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (rt->no_validate && !lock_backup(backup)) elog(ERROR, "Cannot lock backup directory"); - restore_backup(backup, dest_backup->external_dir_str, - &dest_external_dirs); + restore_backup(backup, dest_external_dirs, dest_files); } - /* - * Delete files which are not in dest backup file list. Files which were - * deleted between previous and current backup are not in the list. - */ - if (dest_backup->backup_mode != BACKUP_MODE_FULL) - remove_deleted_files(dest_backup, dest_external_dirs); - if (dest_external_dirs != NULL) free_dir_list(dest_external_dirs); + parray_walk(dest_files, pgFileFree); + parray_free(dest_files); + /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup); } @@ -473,16 +509,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup, const char *external_dir_str, - parray **external_dirs) +restore_backup(pgBackup *backup, parray *dest_external_dirs, parray *dest_files) { char timestamp[100]; - char this_backup_path[MAXPGPATH]; char database_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; char list_path[MAXPGPATH]; parray *files; - parray *backup_external_dirs = NULL; + parray *external_dirs = NULL; int i; /* arrays with meta info for multi threaded backup */ pthread_t *threads; @@ -505,33 +539,11 @@ restore_backup(pgBackup *backup, const char *external_dir_str, backup->wal_block_size, XLOG_BLCKSZ); time2iso(timestamp, lengthof(timestamp), backup->start_time); - elog(LOG, "restoring database from backup %s", timestamp); - - /* - * Restore backup directories. - * this_backup_path = $BACKUP_PATH/backups/instance_name/backup_id - */ - pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - create_data_directories(instance_config.pgdata, this_backup_path, true, - FIO_DB_HOST); - - if (external_dir_str && !skip_external_dirs && - /* Make external directories only once */ - *external_dirs == NULL) - { - *external_dirs = make_external_directory_list(external_dir_str, true); - for (i = 0; i < parray_num(*external_dirs); i++) - { - char *external_path = parray_get(*external_dirs, i); - - fio_mkdir(external_path, DIR_PERMISSION, FIO_DB_HOST); - } - } + elog(LOG, "Restoring database from backup %s", timestamp); if (backup->external_dir_str) - backup_external_dirs = make_external_directory_list( - backup->external_dir_str, - true); + external_dirs = make_external_directory_list(backup->external_dir_str, + true); /* * Get list of files which need to be restored. @@ -558,17 +570,19 @@ restore_backup(pgBackup *backup, const char *external_dir_str, * If the entry was an external directory, create it in the backup. */ if (!skip_external_dirs && - file->external_dir_num && S_ISDIR(file->mode)) + file->external_dir_num && S_ISDIR(file->mode) && + /* Do not create unnecessary external directories */ + parray_bsearch(dest_files, file, pgFileCompareRelPathWithExternal)) { char *external_path; - if (!backup_external_dirs || - parray_num(backup_external_dirs) < file->external_dir_num - 1) + if (!external_dirs || + parray_num(external_dirs) < file->external_dir_num - 1) elog(ERROR, "Inconsistent external directory backup metadata"); - external_path = parray_get(backup_external_dirs, - file->external_dir_num - 1); - if (backup_contains_external(external_path, *external_dirs)) + external_path = parray_get(external_dirs, + file->external_dir_num - 1); + if (backup_contains_external(external_path, dest_external_dirs)) { char container_dir[MAXPGPATH]; char dirpath[MAXPGPATH]; @@ -598,9 +612,10 @@ restore_backup(pgBackup *backup, const char *external_dir_str, arg->files = files; arg->backup = backup; - arg->dest_external_dirs = *external_dirs; - arg->external_dirs = backup_external_dirs; + arg->external_dirs = external_dirs; arg->external_prefix = external_prefix; + arg->dest_external_dirs = dest_external_dirs; + arg->dest_files = dest_files; /* By default there are some error */ threads_args[i].ret = 1; @@ -627,107 +642,10 @@ restore_backup(pgBackup *backup, const char *external_dir_str, parray_walk(files, pgFileFree); parray_free(files); - if (backup_external_dirs != NULL) - free_dir_list(backup_external_dirs); - - if (logger_config.log_level_console <= LOG || - logger_config.log_level_file <= LOG) - elog(LOG, "restore %s backup completed", base36enc(backup->start_time)); -} - -/* - * Delete files which are not in backup's file list from target pgdata. - * It is necessary to restore incremental backup correctly. - * Files which were deleted between previous and current backup - * are not in the backup's filelist. - */ -static void -remove_deleted_files(pgBackup *backup, parray *external_dirs) -{ - parray *files; - parray *files_restored; - char filelist_path[MAXPGPATH]; - char external_prefix[MAXPGPATH]; - int i; - - pgBackupGetPath(backup, filelist_path, lengthof(filelist_path), - DATABASE_FILE_LIST); - pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), - EXTERNAL_DIR); - /* Read backup's filelist using target database path as base path */ - files = dir_read_file_list(instance_config.pgdata, external_prefix, - filelist_path, FIO_BACKUP_HOST); - /* Replace external prefix with full path */ - if (external_dirs) - { - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - if (file->external_dir_num > 0) - { - char *external_path = parray_get(external_dirs, - file->external_dir_num - 1); - char container_dir[MAXPGPATH]; - char *rel_path; - char new_path[MAXPGPATH]; + if (external_dirs != NULL) + free_dir_list(external_dirs); - makeExternalDirPathByNum(container_dir, external_prefix, - file->external_dir_num); - rel_path = GetRelativePath(file->path, container_dir); - join_path_components(new_path, external_path, rel_path); - - pfree(file->path); - file->path = pg_strdup(new_path); - } - } - } - parray_qsort(files, pgFileComparePathDesc); - - /* Get list of files actually existing in target database */ - files_restored = parray_new(); - dir_list_file(files_restored, instance_config.pgdata, true, true, false, 0, - FIO_DB_HOST); - if (external_dirs) - for (i = 0; i < parray_num(external_dirs); i++) - { - parray *external_files; - - /* - * external_dirs paths already remmaped. - */ - - external_files = parray_new(); - dir_list_file(external_files, (char *) parray_get(external_dirs, i), - true, true, false, 0, FIO_DB_HOST); - if (parray_num(external_files) > 0) - parray_concat(files_restored, external_files); - - parray_free(external_files); - } - /* To delete from leaf, sort in reversed order */ - parray_qsort(files_restored, pgFileComparePathDesc); - - for (i = 0; i < parray_num(files_restored); i++) - { - pgFile *file = (pgFile *) parray_get(files_restored, i); - - /* If the file is not in the file list, delete it */ - if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) - { - pgFileDelete(file); - if (logger_config.log_level_console <= LOG || - logger_config.log_level_file <= LOG) - elog(LOG, "deleted %s", GetRelativePath(file->path, - instance_config.pgdata)); - } - } - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); - parray_walk(files_restored, pgFileFree); - parray_free(files_restored); + elog(LOG, "Restore %s backup completed", base36enc(backup->start_time)); } /* @@ -742,7 +660,6 @@ restore_files(void *arg) for (i = 0; i < parray_num(arguments->files); i++) { char from_root[MAXPGPATH]; - char *rel_path; pgFile *file = (pgFile *) parray_get(arguments->files, i); if (!pg_atomic_test_set_flag(&file->lock)) @@ -753,13 +670,12 @@ restore_files(void *arg) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during restore database"); - - rel_path = GetRelativePath(file->path,from_root); + elog(ERROR, "Interrupted during restore database"); if (progress) elog(INFO, "Progress: (%d/%lu). Process file %s ", - i + 1, (unsigned long) parray_num(arguments->files), rel_path); + i + 1, (unsigned long) parray_num(arguments->files), + file->rel_path); /* * For PAGE and PTRACK backups skip files which haven't changed @@ -783,7 +699,7 @@ restore_files(void *arg) } /* Do not restore tablespace_map file */ - if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path)) + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) { elog(VERBOSE, "Skip tablespace_map"); continue; @@ -796,6 +712,14 @@ restore_files(void *arg) continue; } + /* Skip unnecessary file */ + if (parray_bsearch(arguments->dest_files, file, + pgFileCompareRelPathWithExternal) == NULL) + { + elog(VERBOSE, "Skip removed file"); + continue; + } + /* * restore the file. * We treat datafiles separately, cause they were backed up block by @@ -809,7 +733,7 @@ restore_files(void *arg) char to_path[MAXPGPATH]; join_path_components(to_path, instance_config.pgdata, - file->path + strlen(from_root) + 1); + file->rel_path); restore_data_file(to_path, file, arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, false, @@ -821,7 +745,7 @@ restore_files(void *arg) file->external_dir_num - 1); if (backup_contains_external(external_path, arguments->dest_external_dirs)) - copy_file(arguments->external_prefix, FIO_BACKUP_HOST, + copy_file(FIO_BACKUP_HOST, external_path, FIO_DB_HOST, file); } else if (strcmp(file->name, "pg_control") == 0) @@ -829,7 +753,7 @@ restore_files(void *arg) instance_config.pgdata, FIO_DB_HOST, file); else - copy_file(from_root, FIO_BACKUP_HOST, + copy_file(FIO_BACKUP_HOST, instance_config.pgdata, FIO_DB_HOST, file); diff --git a/src/util.c b/src/util.c index a279959c1..788f5c9dc 100644 --- a/src/util.c +++ b/src/util.c @@ -360,7 +360,8 @@ set_min_recovery_point(pgFile *file, const char *backup_path, * Copy pg_control file to backup. We do not apply compression to this file. */ void -copy_pgcontrol_file(const char *from_root, fio_location from_location, const char *to_root, fio_location to_location, pgFile *file) +copy_pgcontrol_file(const char *from_root, fio_location from_location, + const char *to_root, fio_location to_location, pgFile *file) { ControlFileData ControlFile; char *buffer; @@ -375,7 +376,7 @@ copy_pgcontrol_file(const char *from_root, fio_location from_location, const cha file->read_size = size; file->write_size = size; - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + join_path_components(to_path, to_root, file->rel_path); writeControlFile(&ControlFile, to_path, to_location); pg_free(buffer); From b6dcb82d463a1401d72178038a3cc87b75cb93ac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 16:12:56 +0300 Subject: [PATCH 0666/2107] README: Windows is considered stable --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7e2c601b2..c8314180e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. * Remote mode is in beta stage. -* Microsoft Windows operating system support is in beta stage. ## Installation and Setup ### Linux Installation From 87f28e76fdf8df1f2cb988bcedc516699eab545d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 16:14:24 +0300 Subject: [PATCH 0667/2107] tests: added restore.RestoreTest.test_smart_restore --- tests/helpers/ptrack_helpers.py | 29 ++++++++++++ tests/restore.py | 80 ++++++++++++++++++++++++++++++--- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 03fa81e16..e5c6736ed 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -551,6 +551,35 @@ def check_ptrack_sanity(self, idx_dict): # success, 'Ptrack has failed to register changes in data files' # ) + def get_backup_filelist(self, backup_dir, instance, backup_id): + + filelist_path = os.path.join( + backup_dir, 'backups', + instance, backup_id, 'backup_content.control') + + with open(filelist_path, 'r') as f: + filelist_raw = f.read() + + filelist_splitted = filelist_raw.splitlines() + + filelist = {} + for line in filelist_splitted: + line = json.loads(line) + filelist[line['path']] = line + + return filelist + + # return dict of files from filelist A, + # which are not exists in filelist_B + def get_backup_filelist_diff(self, filelist_A, filelist_B): + + filelist_diff = {} + for file in filelist_A: + if file not in filelist_B: + filelist_diff[file] = filelist_A[file] + + return filelist_diff + def check_ptrack_recovery(self, idx_dict): size = idx_dict['size'] for PageNum in range(size): diff --git a/tests/restore.py b/tests/restore.py index a2e2ff8a6..2d4d5601a 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1732,11 +1732,12 @@ def test_restore_backup_from_future(self): self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=3) - #pgbench = node.pgbench(options=['-T', '20', '-c', '2']) - #pgbench.wait() + # pgbench = node.pgbench(options=['-T', '20', '-c', '2']) + # pgbench.wait() # Take PAGE from future - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') with open( os.path.join( @@ -1755,7 +1756,8 @@ def test_restore_backup_from_future(self): pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.cleanup() @@ -1964,7 +1966,8 @@ def test_restore_target_new_options(self): target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() target_xid = res[0][0] @@ -2101,3 +2104,70 @@ def test_restore_target_new_options(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smart_restore(self): + """ + make node, create database, take full backup, drop database, + take incremental backup and restore it, + make sure that files from dropped database are not + copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # restore PAGE backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=['--no-validate', '--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + for file in filelist_diff: + self.assertNotIn(file, logfile_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + +# smart restore of external dirs From f07c0ae2b8e727bb992720e652543d85099f04d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 18:04:49 +0300 Subject: [PATCH 0668/2107] tests: added tests.merge.MergeTest.test_failed_merge_after_delete and tests.merge.MergeTest.test_failed_merge_after_delete_1 --- tests/merge.py | 206 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index c06e15060..29d10cfe8 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1666,6 +1666,152 @@ def test_losing_file_after_failed_merge(self): self.del_test_dir(module_name, fname) + def test_failed_merge_after_delete(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + dboid = node.safe_psql( + "postgres", + "select oid from pg_database where datname = 'testdb'").rstrip() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # drop database + node.safe_psql( + 'postgres', + 'DROP DATABASE testdb') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + gdb.set_breakpoint('parray_bsearch') + gdb.continue_execution_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(30) + + gdb._execute('signal SIGKILL') + + # backup half-merged + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + db_path = os.path.join( + backup_dir, 'backups', 'node', + full_id, 'database', 'base', dboid) + + self.assertNotTrue( + os.path.isdir(db_path)) + + exit(1) + + # try to continue failed MERGE + self.merge_backup(backup_dir, "node", backup_id) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.del_test_dir(module_name, fname) + + def test_failed_merge_after_delete_1(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.pgbench_init(scale=1) + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # drop database + pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + gdb.set_breakpoint('parray_bsearch') + gdb.continue_execution_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(30) + + gdb._execute('signal SIGKILL') + + # backup half-merged + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + # restore + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_backup_from_future(self): """ @@ -1906,6 +2052,66 @@ def test_merge_multiple_descendants(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_smart_merge(self): + """ + make node, create database, take full backup, drop database, + take PAGE backup and merge it into FULL, + make sure that files from dropped database are not + copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + # merge PAGE backup + self.merge_backup( + backup_dir, 'node', page_id, + options=['--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + # Clean after yourself + self.del_test_dir(module_name, fname) + # 1. always use parent link when merging (intermediates may be from different chain) # 2. page backup we are merging with may disappear after failed merge, From 5b539457516416f51c81ce29b46257edeeb21e04 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 19:18:19 +0300 Subject: [PATCH 0669/2107] tests: fixed tests.merge.MergeTest.test_failed_merge_after_delete --- tests/merge.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 29d10cfe8..dd7da403f 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1707,6 +1707,7 @@ def test_failed_merge_after_delete(self): gdb = self.merge_backup( backup_dir, 'node', page_id, gdb=True, options=['--log-level-console=verbose']) + gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() @@ -1714,7 +1715,7 @@ def test_failed_merge_after_delete(self): gdb.continue_execution_until_break() gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(30) + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1729,23 +1730,10 @@ def test_failed_merge_after_delete(self): backup_dir, 'backups', 'node', full_id, 'database', 'base', dboid) - self.assertNotTrue( - os.path.isdir(db_path)) - - exit(1) - - # try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + self.assertFalse( + os.path.isdir(db_path), + 'Directory {0} should not exists'.format( + db_path, full_id)) self.del_test_dir(module_name, fname) @@ -1785,6 +1773,7 @@ def test_failed_merge_after_delete_1(self): gdb = self.merge_backup( backup_dir, 'node', page_id, gdb=True, options=['--log-level-console=verbose']) + gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() From c9c55590b0bbf531035abedb2b91a8df3d080322 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 19:21:22 +0300 Subject: [PATCH 0670/2107] tests: fix typo --- tests/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/merge.py b/tests/merge.py index dd7da403f..0ba3d27eb 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1732,7 +1732,7 @@ def test_failed_merge_after_delete(self): self.assertFalse( os.path.isdir(db_path), - 'Directory {0} should not exists'.format( + 'Directory {0} should not exist'.format( db_path, full_id)) self.del_test_dir(module_name, fname) From 1e1daa0079fb714593542f615e35f41f2b63fd3e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 May 2019 21:22:16 +0300 Subject: [PATCH 0671/2107] tests: added tests.external.ExternalTest.test_smart_restore_externals --- tests/external.py | 81 +++++++++++++++++++++++++++++++++++++++++++++- tests/restore.py | 3 -- tests/retention.py | 24 ++++---------- 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/tests/external.py b/tests/external.py index d74527965..9407c1b7c 100644 --- a/tests/external.py +++ b/tests/external.py @@ -38,7 +38,7 @@ def test_external_simple(self): # take FULL backup with external directory pointing to a file file_path = os.path.join(core_dir, 'file') - open(file_path,"w+") + open(file_path, "w+") try: self.backup_node( @@ -2284,3 +2284,82 @@ def test_merge_external_dir_string_order(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_smart_restore_externals(self): + """ + make node, create database, take full backup with externals, + take incremental backup without externals and restore it, + make sure that files from externals are not copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # fill external directories with data + tmp_id = self.backup_node(backup_dir, 'node', node) + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir_1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir_2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # restore PAGE backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=['--no-validate', '--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + for file in filelist_diff: + self.assertNotIn(file, logfile_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/restore.py b/tests/restore.py index 2d4d5601a..27829fd5d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2168,6 +2168,3 @@ def test_smart_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) - - -# smart restore of external dirs diff --git a/tests/retention.py b/tests/retention.py index 363e61460..21ee76962 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -67,7 +67,7 @@ def test_retention_redundancy_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - #@unittest.skip("skip") + # @unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -120,7 +120,7 @@ def test_retention_window_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - #@unittest.skip("skip") + # @unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" fname = self.id().split('.')[3] @@ -134,8 +134,7 @@ def test_retention_window_3(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - - # Take FULL BACKUP + # take FULL BACKUP backup_id_1 = self.backup_node(backup_dir, 'node', node) # Take second FULL BACKUP @@ -144,7 +143,6 @@ def test_retention_window_3(self): # Take third FULL BACKUP backup_id_3 = self.backup_node(backup_dir, 'node', node) - backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): if backup == 'pg_probackup.conf': @@ -183,8 +181,7 @@ def test_retention_window_4(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - - # Take FULL BACKUPs + # take FULL BACKUPs backup_id_1 = self.backup_node(backup_dir, 'node', node) backup_id_2 = self.backup_node(backup_dir, 'node', node) @@ -245,9 +242,7 @@ def test_window_expire_interleaved_incremental_chains(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - - # Take FULL BACKUPs - + # take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) @@ -334,7 +329,6 @@ def test_window_expire_interleaved_incremental_chains(self): conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=3))) - self.delete_expired( backup_dir, 'node', options=['--retention-window=1', '--expired']) @@ -361,8 +355,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - - # Take FULL BACKUPs + # take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) @@ -710,7 +703,6 @@ def test_window_merge_interleaved_incremental_chains_1(self): backup_dir, 'node', options=['--retention-window=1', '--expired', '--merge-expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) self.assertEqual( @@ -1004,7 +996,6 @@ def test_window_merge_multiple_descendants(self): "INFO: Delete: {0}".format( page_id_a1), output) - self.assertEqual( self.show_pb(backup_dir, 'node')[2]['id'], page_id_b3) @@ -1119,7 +1110,6 @@ def test_window_chains(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_chains_1(self): """ @@ -1245,4 +1235,4 @@ def test_window_error_backups(self): backup_dir, 'node', node, backup_type='page') # Change FULLb backup status to ERROR - #self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') From 16dca32efd2b59c5d016373dfd899e264809ff33 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 May 2019 00:09:27 +0300 Subject: [PATCH 0672/2107] tests minor cleanup --- tests/external.py | 1 + tests/restore.py | 28 ++++++---------------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/tests/external.py b/tests/external.py index 9407c1b7c..54efb5f26 100644 --- a/tests/external.py +++ b/tests/external.py @@ -128,6 +128,7 @@ def test_external_none(self): self.backup_node( backup_dir, 'node', node, options=[ + '--stream', '--external-dirs={0}'.format(external_dir)]) # Delta backup without external directory diff --git a/tests/restore.py b/tests/restore.py index 27829fd5d..5b388682f 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1344,10 +1344,6 @@ def test_zags_block_corrupt_1(self): 'postgres', 'insert into tbl select i from generate_series(0,100000) as i') - print(node.safe_psql( - 'postgres', - "select pg_relation_size('idx')")) - node.safe_psql( 'postgres', 'delete from tbl where i%2 = 0') @@ -1364,10 +1360,6 @@ def test_zags_block_corrupt_1(self): 'postgres', 'create extension pageinspect') - print(node.safe_psql( - 'postgres', - "select * from bt_page_stats('idx',1)")) - node.safe_psql( 'postgres', 'checkpoint') @@ -1413,8 +1405,8 @@ def test_zags_block_corrupt_1(self): if 'selected new timeline ID' in f.read(): break - with open(node_restored.pg_log_file, 'r') as f: - print(f.read()) + # with open(node_restored.pg_log_file, 'r') as f: + # print(f.read()) pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -1905,8 +1897,8 @@ def test_restore_target_latest_archive(self): self.restore_node( backup_dir, 'node', node) - with open(recovery_conf, 'r') as f: - print(f.read()) + # with open(recovery_conf, 'r') as f: + # print(f.read()) hash_1 = hashlib.md5( open(recovery_conf, 'rb').read()).hexdigest() @@ -1916,8 +1908,8 @@ def test_restore_target_latest_archive(self): self.restore_node( backup_dir, 'node', node, options=['--recovery-target=latest']) - with open(recovery_conf, 'r') as f: - print(f.read()) + # with open(recovery_conf, 'r') as f: + # print(f.read()) hash_2 = hashlib.md5( open(recovery_conf, 'rb').read()).hexdigest() @@ -1999,8 +1991,6 @@ def test_restore_target_new_options(self): with open(recovery_conf, 'r') as f: recovery_conf_content = f.read() - print(recovery_conf_content) - self.assertIn( "recovery_target_time = '{0}'".format(target_time), recovery_conf_content) @@ -2028,8 +2018,6 @@ def test_restore_target_new_options(self): with open(recovery_conf, 'r') as f: recovery_conf_content = f.read() - print(recovery_conf_content) - self.assertIn( "recovery_target_xid = '{0}'".format(target_xid), recovery_conf_content) @@ -2057,8 +2045,6 @@ def test_restore_target_new_options(self): with open(recovery_conf, 'r') as f: recovery_conf_content = f.read() - print(recovery_conf_content) - self.assertIn( "recovery_target_lsn = '{0}'".format(target_lsn), recovery_conf_content) @@ -2086,8 +2072,6 @@ def test_restore_target_new_options(self): with open(recovery_conf, 'r') as f: recovery_conf_content = f.read() - print(recovery_conf_content) - self.assertIn( "recovery_target_name = '{0}'".format(target_name), recovery_conf_content) From f697d57c9ae68ba7a42611cc42f013ba39356354 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 13 May 2019 19:44:20 +0300 Subject: [PATCH 0673/2107] bugfix: do not use find_my_exec(), because it is relying on PATH, which can be unavailable. More details: https://github.com/postgrespro/pg_probackup/issues/39 --- src/pg_probackup.c | 24 ++++++++++++++---------- src/pg_probackup.h | 6 ++++++ src/restore.c | 3 ++- src/utils/pgut.c | 3 +-- src/utils/pgut.h | 9 --------- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 73d511a4d..5da9afca5 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -20,9 +20,10 @@ #include "utils/thread.h" #include -const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; -const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; -const char *PROGRAM_FULL_PATH = NULL; +const char *PROGRAM_NAME = NULL; +const char *PROGRAM_FULL_PATH = NULL; +const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; +const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; typedef enum ProbackupSubcmd { @@ -245,13 +246,7 @@ main(int argc, char *argv[]) PROGRAM_NAME = get_progname(argv[0]); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); - if (find_my_exec(argv[0],(char *) PROGRAM_FULL_PATH) < 0) - { - fprintf(stderr, _("%s: could not find own program executable\n"), PROGRAM_NAME); - exit(1); - } - - set_pglocale_pgservice(argv[0], "pgscripts"); + //set_pglocale_pgservice(argv[0], "pgscripts"); #if PG_VERSION_NUM >= 110000 /* @@ -514,6 +509,15 @@ main(int argc, char *argv[]) command = NULL; } + /* For archive-push and archive-get skip full path lookup */ + if ((backup_subcmd != ARCHIVE_GET_CMD && + backup_subcmd != ARCHIVE_PUSH_CMD) && + (find_my_exec(argv[0],(char *) PROGRAM_FULL_PATH) < 0)) + { + PROGRAM_FULL_PATH = NULL; + elog(WARNING, "%s: could not find a full path to executable", PROGRAM_NAME); + } + /* * We have read pgdata path from command line or from configuration file. * Ensure that pgdata is an absolute path. diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e07eccf5e..573ed3999 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -34,6 +34,12 @@ #include "datapagemap.h" +/* pgut client variables and full path */ +extern const char *PROGRAM_NAME; +extern const char *PROGRAM_FULL_PATH; +extern const char *PROGRAM_URL; +extern const char *PROGRAM_EMAIL; + /* Directory/File names */ #define DATABASE_DIR "database" #define BACKUPS_DIR "backups" diff --git a/src/restore.c b/src/restore.c index ce2424854..516f8c27a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -805,7 +805,8 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " "--wal-file-path %%p --wal-file-name %%f'\n", - PROGRAM_FULL_PATH, backup_path, instance_name); + PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, + backup_path, instance_name); /* * We've already checked that only one of the four following mutually diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 501a89b61..f51980f0c 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -8,6 +8,7 @@ *------------------------------------------------------------------------- */ +#include "pg_probackup.h" #include "postgres_fe.h" #include "getopt_long.h" @@ -22,8 +23,6 @@ #include "file.h" -const char *PROGRAM_NAME = "pg_probackup"; - static char *password = NULL; bool prompt_password = true; bool force_password = false; diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 2fb13cf4c..588df7f9f 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -16,15 +16,6 @@ typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); -/* - * pgut client variables and functions - */ -extern const char *PROGRAM_NAME; -extern const char *PROGRAM_FULL_PATH; -extern const char *PROGRAM_VERSION; -extern const char *PROGRAM_URL; -extern const char *PROGRAM_EMAIL; - extern void pgut_help(bool details); /* From e1c4f8fcd08c73f74fddf68bfaf4a54ef12c75d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 13 May 2019 20:23:21 +0300 Subject: [PATCH 0674/2107] remove noninformative elog messages in restore --- src/restore.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/restore.c b/src/restore.c index 516f8c27a..1b124483a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -686,16 +686,13 @@ restore_files(void *arg) (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) { - elog(VERBOSE, "The file didn`t change. Skip restore: %s", file->path); + elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", file->path); continue; } /* Directories were created before */ if (S_ISDIR(file->mode)) - { - elog(VERBOSE, "Directory, skip"); continue; - } /* Do not restore tablespace_map file */ if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) @@ -706,18 +703,12 @@ restore_files(void *arg) /* Do no restore external directory file if a user doesn't want */ if (skip_external_dirs && file->external_dir_num > 0) - { - elog(VERBOSE, "Skip external directory file"); continue; - } /* Skip unnecessary file */ if (parray_bsearch(arguments->dest_files, file, pgFileCompareRelPathWithExternal) == NULL) - { - elog(VERBOSE, "Skip removed file"); continue; - } /* * restore the file. From c88cf22c13df8a3c5e86364228d7af0dc95532c6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 14 May 2019 02:28:37 +0300 Subject: [PATCH 0675/2107] tests: added restore.RestoreTest.test_pg_11_group_access --- tests/helpers/ptrack_helpers.py | 43 +++++++++++++++++++++++++++++++-- tests/restore.py | 41 +++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e5c6736ed..a86912d5e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1222,7 +1222,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): directory_dict = {} directory_dict['pgdata'] = pgdata directory_dict['files'] = {} - directory_dict['dirs'] = [] + directory_dict['dirs'] = {} for root, dirs, files in os.walk(pgdata, followlinks=True): dirs[:] = [d for d in dirs if d not in dirs_to_ignore] for file in files: @@ -1267,7 +1267,18 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): break if not found: - directory_dict['dirs'].append(directory_relpath) + directory_dict['dirs'][directory_relpath] = {} + + # get permissions for every file and directory + for file in directory_dict['dirs']: + full_path = os.path.join(pgdata, file) + directory_dict['dirs'][file]['mode'] = os.stat( + full_path).st_mode + + for file in directory_dict['files']: + full_path = os.path.join(pgdata, file) + directory_dict['files'][file]['mode'] = os.stat( + full_path).st_mode return directory_dict @@ -1283,6 +1294,21 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): error_message += '\nDirectory was not present' error_message += ' in original PGDATA: {0}\n'.format( os.path.join(restored_pgdata['pgdata'], directory)) + else: + if ( + restored_pgdata['dirs'][directory]['mode'] != + original_pgdata['dirs'][directory]['mode'] + ): + fail = True + error_message += '\nDir permissions mismatch:\n' + error_message += ' Dir old: {0} Permissions: {1}\n'.format( + os.path.join(original_pgdata['pgdata'], directory), + original_pgdata['dirs'][directory]['mode']) + error_message += ' Dir new: {0} Permissions: {1}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory), + restored_pgdata['dirs'][directory]['mode']) + + for directory in original_pgdata['dirs']: if directory not in restored_pgdata['dirs']: @@ -1305,6 +1331,19 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): for file in original_pgdata['files']: if file in restored_pgdata['files']: + if ( + restored_pgdata['files'][file]['mode'] != + original_pgdata['files'][file]['mode'] + ): + fail = True + error_message += '\nFile permissions mismatch:\n' + error_message += ' File_old: {0} Permissions: {1}\n'.format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['mode']) + error_message += ' File_new: {0} Permissions: {1}\n'.format( + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['mode']) + if ( original_pgdata['files'][file]['md5'] != restored_pgdata['files'][file]['md5'] diff --git a/tests/restore.py b/tests/restore.py index 5b388682f..35d39972d 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2152,3 +2152,44 @@ def test_smart_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pg_11_group_access(self): + """ + test group access for PG >= 11 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--allow-group-access']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if self.get_version(node) < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + + # take FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # restore backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + # compare pgdata permissions + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 83773d260e8ed3cbd98b855c9cfffd4badd09676 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 17 May 2019 18:07:15 +0300 Subject: [PATCH 0676/2107] README: add link to Windows installers --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c8314180e..101db189f 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ yum install pg_probackup-{11,10,9.6,9.5} yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` +### Windows Installation +[Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L) + Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). ## Building from source @@ -96,6 +99,10 @@ SET PATH=%PATH%;C:\msys64\usr\bin gen_probackup_project.pl C:\path_to_postgresql_source_tree ``` +## Current release + +[2.1.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.1) + ## Documentation Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). From a02fec70d44b666b3519cc0d9f9bd8dc0cef49c4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 17 May 2019 18:09:59 +0300 Subject: [PATCH 0677/2107] Readme: small fix --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 101db189f..fed987db3 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * Remote mode is in beta stage. ## Installation and Setup +### Windows Installation +[Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L) + ### Linux Installation ```shell #DEB Ubuntu|Debian Packages @@ -73,9 +76,6 @@ yum install pg_probackup-{11,10,9.6,9.5} yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` -### Windows Installation -[Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L) - Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). ## Building from source From aa1812eec1d22b337ac774602484c2fc3f29cb46 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 May 2019 18:44:05 +0300 Subject: [PATCH 0678/2107] tests: added tests.delete.DeleteTest.test_delete_wal_between_multiple_timelines --- tests/delete.py | 61 ++++++++++++++++++++++++++++++++- tests/helpers/ptrack_helpers.py | 13 +++---- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index e2a996b7a..5ad4c2508 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -174,7 +174,11 @@ def test_delete_increment_ptrack(self): # @unittest.skip("skip") def test_delete_orphaned_wal_segments(self): - """make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option""" + """ + make archive node, make three full backups, + delete second backup without --wal option, + then delete orphaned wals via --wal option + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -239,6 +243,61 @@ def test_delete_orphaned_wal_segments(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_delete_wal_between_multiple_timelines(self): + """ + /-------B1-- + A1----------------A2---- + + delete A1 backup, check that WAL segments on [A1, A2) and + [A1, B1) are deleted and backups B1 and A2 keep + their WAL + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + A1 = self.backup_node(backup_dir, 'node', node) + + # load some data to node + node.pgbench_init(scale=3) + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + + self.restore_node(backup_dir, 'node', node2) + node2.append_conf( + 'postgresql.auto.conf', "port = {0}".format(node2.port)) + node2.slow_start() + + # load some more data to node + node.pgbench_init(scale=3) + + # take A2 + A2 = self.backup_node(backup_dir, 'node', node) + + # load some more data to node2 + node2.pgbench_init(scale=2) + + B1 = self.backup_node( + backup_dir, 'node', + node2, data_dir=node2.data_dir) + + self.delete_pb(backup_dir, 'node', backup_id=A1, options=['--wal']) + + self.validate_pb(backup_dir) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_delete_backup_with_empty_control_file(self): """ diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a86912d5e..1ed69f498 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -737,21 +737,18 @@ def backup_node( print('You must provide ether node or data_dir for backup') exit(1) - if node: - pgdata = node.data_dir - - if data_dir: - pgdata = data_dir - cmd_list = [ 'backup', '-B', backup_dir, + '--instance={0}'.format(instance), # "-D", pgdata, '-p', '%i' % node.port, - '-d', 'postgres', - '--instance={0}'.format(instance) + '-d', 'postgres' ] + if data_dir: + cmd_list += ['-D', data_dir] + # don`t forget to kill old_binary after remote ssh release if self.remote and not old_binary: options = options + [ From ebeebcaffa416d76b557bbeb6cad7cd0b71df5f4 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Mon, 20 May 2019 11:28:51 +0300 Subject: [PATCH 0679/2107] Get last valid WAL record correctly using sorted thread_args array --- src/parsexlog.c | 55 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index d05bca459..bd8585f5a 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -860,6 +860,18 @@ InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, return xlogreader; } +/* + * Comparison function to sort xlog_thread_arg array. + */ +static int +xlog_thread_arg_comp(const void *a1, const void *a2) +{ + const xlog_thread_arg *arg1 = a1; + const xlog_thread_arg *arg2 = a2; + + return arg1->reader_data.xlogsegno - arg2->reader_data.xlogsegno; +} + /* * Run WAL processing routines using threads. Start from startpoint up to * endpoint. It is possible to send zero endpoint, threads will read WAL @@ -877,7 +889,6 @@ RunXLogThreads(const char *archivedir, time_t target_time, int i; int threads_need = 0; XLogSegNo endSegNo = 0; - XLogSegNo errorSegNo = 0; bool result = true; if (!XRecOffIsValid(startpoint)) @@ -956,28 +967,27 @@ RunXLogThreads(const char *archivedir, time_t target_time, result = false; } + /* Release threads here, use thread_args only below */ + pfree(threads); + threads = NULL; + if (last_rec) + { + /* + * We need to sort xlog_thread_arg array by xlogsegno to return latest + * possible record up to which restore is possible. We need to sort to + * detect failed thread between start segment and target segment. + * + * Loop stops on first failed thread. + */ + if (threads_need > 1) + qsort((void *) thread_args, threads_need, sizeof(xlog_thread_arg), + xlog_thread_arg_comp); + for (i = 0; i < threads_need; i++) { XLogRecTarget *cur_rec; - if (thread_args[i].ret != 0) - { - /* - * Save invalid segment number after which all segments are not - * valid. - */ - if (errorSegNo == 0 || - errorSegNo > thread_args[i].reader_data.xlogsegno) - errorSegNo = thread_args[i].reader_data.xlogsegno; - continue; - } - - /* Is this segment valid */ - if (errorSegNo != 0 && - thread_args[i].reader_data.xlogsegno > errorSegNo) - continue; - cur_rec = &thread_args[i].reader_data.cur_rec; /* * If we got the target return minimum possible record. @@ -997,9 +1007,16 @@ RunXLogThreads(const char *archivedir, time_t target_time, */ else if (last_rec->rec_lsn < cur_rec->rec_lsn) *last_rec = *cur_rec; + + /* + * We reached failed thread, so stop here. We cannot use following + * WAL records after failed segment. + */ + if (thread_args[i].ret != 0) + break; } + } - pfree(threads); pfree(thread_args); return result; From 0493eabfd6183c9221bf50da5550b980eef58dc5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 May 2019 17:54:23 +0300 Subject: [PATCH 0680/2107] tests: added backup.BackupTest.test_backup_concurrent_drop_table and compatibility.CompatibilityTest.test_backup_concurrent_drop_table --- tests/backup.py | 45 +++++++++++++++++++++++++++++++++++++++++ tests/compatibility.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index d54a03119..6adfe190b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1002,6 +1002,51 @@ def test_temp_slot_for_stream_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_backup_concurrent_drop_table(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress', '--log-level-file=VERBOSE'], + gdb=True) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + node.safe_psql( + 'postgres', + 'DROP TABLE pgbench_accounts') + + # do checkpoint to guarantee filenode removal + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): """""" diff --git a/tests/compatibility.py b/tests/compatibility.py index c2a7fe416..1eca57499 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -537,3 +537,49 @@ def test_backward_compatibility_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_concurrent_drop_table(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=5) + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress', '--log-level-file=VERBOSE'], + gdb=True, old_binary=True) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + node.safe_psql( + 'postgres', + 'DROP TABLE pgbench_accounts') + + # do checkpoint to guarantee filenode removal + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + # show_backup = self.show_pb(backup_dir, 'node')[0] + # self.assertEqual(show_backup['status'], "OK") + + # validate with fresh binary, it MUST be successful + self.validate_pb(backup_dir) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 18cfcdd2367c1f28d04d3e2000fe59d20f022979 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 May 2019 17:56:22 +0300 Subject: [PATCH 0681/2107] tests: minor fix --- tests/backup.py | 2 +- tests/compatibility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 6adfe190b..9d844a1f8 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1017,7 +1017,7 @@ def test_backup_concurrent_drop_table(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=1) # FULL backup gdb = self.backup_node( diff --git a/tests/compatibility.py b/tests/compatibility.py index 1eca57499..b4d66af37 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -552,7 +552,7 @@ def test_backup_concurrent_drop_table(self): self.add_instance(backup_dir, 'node', node, old_binary=True) node.slow_start() - node.pgbench_init(scale=5) + node.pgbench_init(scale=1) # FULL backup gdb = self.backup_node( From 92a73d1ce2de246a60e463de645d070972b05b87 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 May 2019 19:28:00 +0300 Subject: [PATCH 0682/2107] tests: added restore.RestoreTest.test_pg_10_waldir --- tests/restore.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index 35d39972d..666bf399f 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -7,6 +7,7 @@ from time import sleep from datetime import datetime, timedelta import hashlib +import shutil module_name = 'restore' @@ -2193,3 +2194,52 @@ def test_pg_11_group_access(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pg_10_waldir(self): + """ + test group access for PG >= 11 + """ + fname = self.id().split('.')[3] + wal_dir = os.path.join( + os.path.join(self.tmp_path, module_name, fname), 'wal_dir') + shutil.rmtree(wal_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--waldir={0}'.format(wal_dir)]) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL >= 10 for this test') + + # take FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # restore backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + # compare pgdata permissions + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertTrue( + os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), + 'pg_wal should be symlink') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 486cdc274b67f6787b7812d884eec5f320e1ec32 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 May 2019 19:29:49 +0300 Subject: [PATCH 0683/2107] tests: added restore.RestoreTest.test_restore_concurrent_drop_table --- tests/backup.py | 2 +- tests/restore.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index 9d844a1f8..1317829d4 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1022,7 +1022,7 @@ def test_backup_concurrent_drop_table(self): # FULL backup gdb = self.backup_node( backup_dir, 'node', node, - options=['--stream', '--compress', '--log-level-file=VERBOSE'], + options=['--stream', '--compress'], gdb=True) gdb.set_breakpoint('backup_data_file') diff --git a/tests/restore.py b/tests/restore.py index 666bf399f..60d837594 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2243,3 +2243,58 @@ def test_pg_10_waldir(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_concurrent_drop_table(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress']) + + # DELTA backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress', '--no-validate'], + gdb=True) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + node.safe_psql( + 'postgres', + 'DROP TABLE pgbench_accounts') + + # do checkpoint to guarantee filenode removal + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 0bac131d7a91d27e401c20ddec18f1b99f0ce7ed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 May 2019 11:34:21 +0300 Subject: [PATCH 0684/2107] tests: added external.ExternalTest.test_external_merge_3 --- tests/external.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/external.py b/tests/external.py index 54efb5f26..37d8ea3e0 100644 --- a/tests/external.py +++ b/tests/external.py @@ -58,6 +58,8 @@ def test_external_simple(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + sleep(1) + # FULL backup self.backup_node( backup_dir, 'node', node, backup_type="full", @@ -944,6 +946,98 @@ def test_external_merge_single(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_3(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["-j", "4"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # page backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=[ + "-j", "4", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # page backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=[ + "-j", "4", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_double(self): From 21230cbb6c7ba7b0828b0facc35f4ed9e31cdcce Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 20 May 2019 15:33:02 +0300 Subject: [PATCH 0685/2107] Release 2.1.2 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 573ed3999..4527a33e1 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -196,8 +196,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.1" -#define AGENT_PROTOCOL_VERSION 20101 +#define PROGRAM_VERSION "2.1.2" +#define AGENT_PROTOCOL_VERSION 20102 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 3d8a86e73..18458de4e 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.1 \ No newline at end of file +pg_probackup 2.1.2 \ No newline at end of file From ad20823ccf610799386c6c30369b10d471a5706a Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 22 May 2019 14:41:21 +0300 Subject: [PATCH 0686/2107] Use pgFileComparePathWithExternalDesc for to_files during merging files --- src/merge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 607f44fba..3ac73cf77 100644 --- a/src/merge.c +++ b/src/merge.c @@ -235,7 +235,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) DATABASE_FILE_LIST); to_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); /* To delete from leaf, sort in reversed order */ - parray_qsort(to_files, pgFileComparePathDesc); + parray_qsort(to_files, pgFileComparePathWithExternalDesc); /* * Get list of files which need to be moved. */ From 5cf1556984cc486ad83cbb48b1f8dd7e2a15d8f2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 May 2019 22:15:40 +0300 Subject: [PATCH 0687/2107] tests: added restore.RestoreTest.test_lost_non_data_file --- tests/restore.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index 60d837594..28a6c61a2 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2298,3 +2298,46 @@ def test_restore_concurrent_drop_table(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_lost_non_data_file(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + os.remove(file) + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because of non-data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Insert correct error message', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 00cbcc81604ceaaf739aa3c6ae8065eb0949fc3a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:29:00 +0300 Subject: [PATCH 0688/2107] tests: fixes --- tests/external.py | 9 +++++---- tests/merge.py | 21 +++++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/external.py b/tests/external.py index 37d8ea3e0..df5695261 100644 --- a/tests/external.py +++ b/tests/external.py @@ -866,7 +866,7 @@ def test_external_merge_skip_external_dirs(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_external_merge_single(self): + def test_external_merge_1(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1012,7 +1012,9 @@ def test_external_merge_3(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.merge_backup( + backup_dir, 'node', backup_id=backup_id, + options=['--log-level-file=verbose']) # RESTORE node.cleanup() @@ -1040,7 +1042,7 @@ def test_external_merge_3(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_external_merge_double(self): + def test_external_merge_2(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1446,7 +1448,6 @@ def test_merge_external_changed_data(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_skip_external(self): diff --git a/tests/merge.py b/tests/merge.py index 0ba3d27eb..63f2e59ca 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1721,7 +1721,7 @@ def test_failed_merge_after_delete(self): # backup half-merged self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) self.assertEqual( full_id, self.show_pb(backup_dir, 'node')[0]['id']) @@ -1753,16 +1753,21 @@ def test_failed_merge_after_delete_1(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - # add database - node.pgbench_init(scale=1) - # take FULL backup full_id = self.backup_node( backup_dir, 'node', node, options=['--stream']) + node.pgbench_init(scale=1) + + page_1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_1, 'ERROR') + pgdata = self.pgdata_content(node.data_dir) - # drop database + # add data pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) pgbench.wait() @@ -1770,6 +1775,9 @@ def test_failed_merge_after_delete_1(self): page_id = self.backup_node( backup_dir, 'node', node, backup_type='page') + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_1, 'OK') + gdb = self.merge_backup( backup_dir, 'node', page_id, gdb=True, options=['--log-level-console=verbose']) @@ -1827,7 +1835,8 @@ def test_merge_backup_from_future(self): node.pgbench_init(scale=3) # Take PAGE from future - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') with open( os.path.join( From d27835dd7ddc243d2072b426ed36741b6d0c920e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:42:12 +0300 Subject: [PATCH 0689/2107] bugfix: disappeared during backup files could give a false-positive alarm by validation. Issue #72 --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 87829d9de..1077cea0c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -479,9 +479,9 @@ do_backup_instance(void) if (tmp_file->write_size == FILE_NOT_FOUND) { - pg_atomic_clear_flag(&tmp_file->lock); pgFileFree(tmp_file); parray_remove(backup_files_list, i); + i--; } } From ac4ea5e731595266500ceb7e674911e4ce82de76 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:51:03 +0300 Subject: [PATCH 0690/2107] bugfix: exit with error if validation after backup found corruption --- src/backup.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1077cea0c..d47e7a1a7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -996,10 +996,14 @@ do_backup(time_t start_time, bool no_validate) if (!no_validate) pgBackupValidate(¤t); - elog(INFO, "Backup %s completed", base36enc(current.start_time)); + if (current.status == BACKUP_STATUS_OK || + current.status == BACKUP_STATUS_DONE) + elog(INFO, "Backup %s completed", base36enc(current.start_time)); + else + elog(ERROR, "Backup %s failed", base36enc(current.start_time)); /* - * After successfil backup completion remove backups + * After successful backup completion remove backups * which are expired according to retention policies */ if (delete_expired || merge_expired || delete_wal) From 7f21f63bb038acfbf06c220a6b8a7c653bb6b3a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:53:32 +0300 Subject: [PATCH 0691/2107] bugfix: allow restore of DELTA backup to skip unchanged non-data files --- src/restore.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/restore.c b/src/restore.c index 1b124483a..0e1fde486 100644 --- a/src/restore.c +++ b/src/restore.c @@ -677,17 +677,23 @@ restore_files(void *arg) file->rel_path); /* - * For PAGE and PTRACK backups skip files which haven't changed + * For PAGE and PTRACK backups skip datafiles which haven't changed * since previous backup and thus were not backed up. * We cannot do the same when restoring DELTA backup because we need information - * about every file to correctly truncate them. + * about every datafile to correctly truncate them. */ - if (file->write_size == BYTES_INVALID && - (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE - || arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) + if (file->write_size == BYTES_INVALID) { - elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", file->path); - continue; + /* data file, only PAGE and PTRACK can skip */ + if (((file->is_datafile && !file->is_cfs) && + (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || + arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) || + /* non-data file can be skipped regardless of backup type */ + !(file->is_datafile && !file->is_cfs)) + { + elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", file->path); + continue; + } } /* Directories were created before */ From c3c05e83b680ed66d33d38070e089c68b5f7bbee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:58:01 +0300 Subject: [PATCH 0692/2107] README: add to limitions notice about incremental chains and timelines --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fed987db3..a8f36f577 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. * Remote mode is in beta stage. +* Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. ## Installation and Setup ### Windows Installation From 6a64e5640e3ae1eaf46eea5d33418b40d67d416c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 11:58:28 +0300 Subject: [PATCH 0693/2107] tests: added validate.ValidateTest.test_validation_after_backup --- tests/validate.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index fc3a01d46..b2d2e0ce0 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3365,6 +3365,44 @@ def test_corrupt_pg_control_via_resetxlog(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validation_after_backup(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, options=['--stream']) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + gdb.continue_execution_until_exit() + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # validate empty backup list # page from future during validate # page from future during backup From 2fd345773894a53398382b4266fb612c7a4090a1 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 23 May 2019 12:25:55 +0300 Subject: [PATCH 0694/2107] Fix help: --remote-proto default value is ssh --- src/help.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/help.c b/src/help.c index b52483d63..a69b40c86 100644 --- a/src/help.c +++ b/src/help.c @@ -313,7 +313,7 @@ help_backup(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); @@ -407,7 +407,7 @@ help_restore(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); @@ -689,7 +689,7 @@ help_set_config(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); @@ -736,7 +736,7 @@ help_add_instance(void) printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); @@ -786,7 +786,7 @@ help_archive_push(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); @@ -815,7 +815,7 @@ help_archive_get(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: none)\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); printf(_(" --remote-host=hostname remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); From 3b9d6c4d3adcad17f13f27b3a63b3901639c797a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 15:23:03 +0300 Subject: [PATCH 0695/2107] tests: added remote.RemoteTest.test_remote_sanity --- tests/remote.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/remote.py b/tests/remote.py index 50697b3e5..064ee219f 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -12,7 +12,7 @@ class RemoteTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_remote_1(self): + def test_remote_sanity(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -21,26 +21,24 @@ def test_remote_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) -# self.set_archiving(backup_dir, 'node', node, remote=True) node.slow_start() - self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--remote-proto=ssh', - '--remote-host=localhost']) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + try: + self.backup_node( + backup_dir, 'node', + node, options=['--remote-proto=ssh', '--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remote-host option is missing." + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "Insert correct error", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) From 420a527a65a92194e895aa5fe75837033c2f53b2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 May 2019 20:28:46 +0300 Subject: [PATCH 0696/2107] tests: added checkdb.CheckdbTest.test_checkdb_sigint_handling --- tests/checkdb.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/checkdb.py b/tests/checkdb.py index 038fb7cff..6ea671a03 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -442,3 +442,58 @@ def test_checkdb_block_validation_sanity(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_checkdb_sigint_handling(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + # truncate log_file + #with open(node.pg_log_file, 'w') as f: + # f.truncate() + # f.close() + + # FULL backup + gdb = self.checkdb_node( + backup_dir, 'node', gdb=True, + options=[ + '-d', 'postgres', '-j', '4', + '--skip-block-validation', + '--amcheck', '-p', str(node.port)]) + + gdb.set_breakpoint('amcheck_one_index') + gdb.run_until_break() + + gdb.continue_execution_until_break(4) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGINT') + gdb.continue_execution_until_exit() + + with open(node.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('could not receive data from client', output) + self.assertNotIn('could not send data to client', output) + self.assertNotIn('connection to client lost', output) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 64d0443057c190d8f85a93a2ce52294b1c4111a6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 May 2019 00:40:42 +0300 Subject: [PATCH 0697/2107] tests: add property pg_config_version --- tests/backup.py | 8 +++----- tests/checkdb.py | 7 +------ tests/delta.py | 2 -- tests/helpers/ptrack_helpers.py | 5 +++++ tests/restore.py | 18 +++++++++++------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 1317829d4..1f76b2963 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1050,6 +1050,9 @@ def test_backup_concurrent_drop_table(self): # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): """""" + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1067,9 +1070,6 @@ def test_pg_11_adjusted_wal_segment_size(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') - node.pgbench_init(scale=5) # FULL STREAM backup @@ -1119,8 +1119,6 @@ def test_pg_11_adjusted_wal_segment_size(self): '--delete-wal', '--retention-redundancy=1']) - print(output) - # validate self.validate_pb(backup_dir) diff --git a/tests/checkdb.py b/tests/checkdb.py index 6ea671a03..0a8054f54 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -466,11 +466,6 @@ def test_checkdb_sigint_handling(self): "postgres", "create extension amcheck_next") - # truncate log_file - #with open(node.pg_log_file, 'w') as f: - # f.truncate() - # f.close() - # FULL backup gdb = self.checkdb_node( backup_dir, 'node', gdb=True, @@ -482,7 +477,7 @@ def test_checkdb_sigint_handling(self): gdb.set_breakpoint('amcheck_one_index') gdb.run_until_break() - gdb.continue_execution_until_break(4) + gdb.continue_execution_until_break(10) gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') diff --git a/tests/delta.py b/tests/delta.py index 2c030fc03..8083c57c2 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -928,8 +928,6 @@ def test_alter_database_set_tablespace_delta(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', - 'checkpoint_timeout': '30s', 'autovacuum': 'off' } ) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 1ed69f498..86f6e27ce 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -274,6 +274,11 @@ def __init__(self, *args, **kwargs): if 'PGPROBACKUP_SSH_REMOTE' in self.test_env: self.remote = True + @property + def pg_config_version(self): + return self.version_to_num( + testgres.get_pg_config()['VERSION'].split(" ")[1]) + # if 'PGPROBACKUP_SSH_HOST' in self.test_env: # self.remote_host = self.test_env['PGPROBACKUP_SSH_HOST'] # else diff --git a/tests/restore.py b/tests/restore.py index 28a6c61a2..00f5b5a21 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2159,6 +2159,9 @@ def test_pg_11_group_access(self): """ test group access for PG >= 11 """ + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -2172,9 +2175,6 @@ def test_pg_11_group_access(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') - # take FULL backup self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -2200,6 +2200,9 @@ def test_pg_10_waldir(self): """ test group access for PG >= 11 """ + if self.pg_config_version < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL >= 10 for this test') + fname = self.id().split('.')[3] wal_dir = os.path.join( os.path.join(self.tmp_path, module_name, fname), 'wal_dir') @@ -2216,9 +2219,6 @@ def test_pg_10_waldir(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') - # take FULL backup self.backup_node( backup_dir, 'node', node, options=['--stream']) @@ -2335,7 +2335,11 @@ def test_lost_non_data_file(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'Insert correct error message', e.message, + 'is not found', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Data files restoring failed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 84c6fd4101d8769a381865cc9dad41637fdc052b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 May 2019 15:07:23 +0300 Subject: [PATCH 0698/2107] README: bump current version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8f36f577..9c0c2c8ce 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Current release -[2.1.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.1) +[2.1.2](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.2) ## Documentation From db10096f0f8a5b7e7abafdf4265a9eb1505dcf48 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 May 2019 15:12:15 +0300 Subject: [PATCH 0699/2107] Issue 74: copy_file and backup_data_file() always treat missing source file as non-error condition --- src/backup.c | 5 +++-- src/data.c | 30 ++++++++++++++++++++---------- src/merge.c | 9 +++++---- src/pg_probackup.h | 5 +++-- src/restore.c | 4 ++-- 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/backup.c b/src/backup.c index d47e7a1a7..2fe82bc03 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2466,7 +2466,8 @@ backup_files(void *arg) arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, - instance_config.compress_level)) + instance_config.compress_level, + true)) { /* disappeared file not to be confused with 'not changed' */ if (file->write_size != FILE_NOT_FOUND) @@ -2508,7 +2509,7 @@ backup_files(void *arg) else dst = arguments->to_root; if (skip || - !copy_file(FIO_DB_HOST, dst, FIO_BACKUP_HOST, file)) + !copy_file(FIO_DB_HOST, dst, FIO_BACKUP_HOST, file, true)) { /* disappeared file not to be confused with 'not changed' */ if (file->write_size != FILE_NOT_FOUND) diff --git a/src/data.c b/src/data.c index bcc92bd88..f45effd07 100644 --- a/src/data.c +++ b/src/data.c @@ -521,7 +521,7 @@ bool backup_data_file(backup_files_arg* arguments, const char *to_path, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel) + CompressAlg calg, int clevel, bool missing_ok) { FILE *in; FILE *out; @@ -567,9 +567,14 @@ backup_data_file(backup_files_arg* arguments, */ if (errno == ENOENT) { - elog(LOG, "File \"%s\" is not found", file->path); - file->write_size = FILE_NOT_FOUND; - return false; + if (missing_ok) + { + elog(LOG, "File \"%s\" is not found", file->path); + file->write_size = FILE_NOT_FOUND; + return false; + } + else + elog(ERROR, "File \"%s\" is not found", file->path); } elog(ERROR, "cannot open file \"%s\": %s", @@ -754,7 +759,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, break; /* - * We need to truncate result file if data file in a incremental backup + * We need to truncate result file if data file in an incremental backup * less than data file in a full backup. We know it thanks to n_blocks. * * It may be equal to -1, then we don't want to truncate the result @@ -939,7 +944,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, */ bool copy_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file) + fio_location to_location, pgFile *file, bool missing_ok) { char to_path[MAXPGPATH]; FILE *in; @@ -963,12 +968,17 @@ copy_file(fio_location from_location, const char *to_root, FIN_FILE_CRC32(true, crc); file->crc = crc; - /* maybe deleted, it's not error */ + /* maybe deleted, it's not error in case of backup */ if (errno == ENOENT) { - elog(LOG, "File \"%s\" is not found", file->path); - file->write_size = FILE_NOT_FOUND; - return false; + if (missing_ok) + { + elog(LOG, "File \"%s\" is not found", file->path); + file->write_size = FILE_NOT_FOUND; + return false; + } + else + elog(ERROR, "File \"%s\" is not found", file->path); } elog(ERROR, "cannot open source file \"%s\": %s", file->path, diff --git a/src/merge.c b/src/merge.c index 3ac73cf77..07cc54fde 100644 --- a/src/merge.c +++ b/src/merge.c @@ -489,7 +489,7 @@ merge_files(void *arg) * of DELTA backup we should consider n_blocks to truncate the target * backup. */ - if (file->write_size == BYTES_INVALID && file->n_blocks == -1) + if (file->write_size == BYTES_INVALID && file->n_blocks == BLOCKNUM_INVALID) { elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", file->path); @@ -605,7 +605,8 @@ merge_files(void *arg) to_backup->start_lsn, to_backup->backup_mode, to_backup->compress_alg, - to_backup->compress_level); + to_backup->compress_level, + false); file->path = prev_path; @@ -645,12 +646,12 @@ merge_files(void *arg) argument->from_external); makeExternalDirPathByNum(to_root, argument->to_external_prefix, new_dir_num); - copy_file(FIO_LOCAL_HOST, to_root, FIO_LOCAL_HOST, file); + copy_file(FIO_LOCAL_HOST, to_root, FIO_LOCAL_HOST, file, false); } else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); else - copy_file(FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); + copy_file(FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file, false); /* * We need to save compression algorithm type of the target backup to be diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4527a33e1..446658ceb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -629,13 +629,14 @@ extern bool backup_data_file(backup_files_arg* arguments, const char *to_path, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel); + CompressAlg calg, int clevel, + bool missing_ok); extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); extern bool copy_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file); + fio_location to_location, pgFile *file, bool missing_ok); extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); diff --git a/src/restore.c b/src/restore.c index 0e1fde486..2964e4aa2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -742,7 +742,7 @@ restore_files(void *arg) if (backup_contains_external(external_path, arguments->dest_external_dirs)) copy_file(FIO_BACKUP_HOST, - external_path, FIO_DB_HOST, file); + external_path, FIO_DB_HOST, file, false); } else if (strcmp(file->name, "pg_control") == 0) copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, @@ -751,7 +751,7 @@ restore_files(void *arg) else copy_file(FIO_BACKUP_HOST, instance_config.pgdata, FIO_DB_HOST, - file); + file, false); /* print size of restored file */ if (file->write_size != BYTES_INVALID) From ead629bfa7a051453522f64bf84a933320ea1b72 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 27 May 2019 11:27:43 +0300 Subject: [PATCH 0700/2107] Issue 67: permission mask is not preserved by restore --- src/dir.c | 150 ++++++++++++++++++++++++++++++++++++++++++++- src/merge.c | 2 +- src/pg_probackup.h | 9 ++- src/restore.c | 2 +- 4 files changed, 159 insertions(+), 4 deletions(-) diff --git a/src/dir.c b/src/dir.c index b2afa3f69..f8637947c 100644 --- a/src/dir.c +++ b/src/dir.c @@ -340,6 +340,16 @@ pgFileComparePath(const void *f1, const void *f2) return strcmp(f1p->path, f2p->path); } +/* Compare two pgFile with their name in ascending order of ASCII code. */ +int +pgFileCompareName(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + pgFile *f2p = *(pgFile **)f2; + + return strcmp(f1p->name, f2p->name); +} + /* * Compare two pgFile with their path and external_dir_num * in ascending order of ASCII code. @@ -1041,6 +1051,142 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) opt_path_map(opt, arg, &external_remap_list, "external directory"); } +/* + * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise + * an error if target directories exist. + * + * If **extract_tablespaces** is true then try to extract tablespace data + * directories into their initial path using tablespace_map file. + * + * Enforce permissions from backup_content.control. The only + * problem now is with PGDATA itself, we must preserve PGDATA permissions + * somewhere. + * + * TODO: symlink handling. If user located symlink in PG_TBLSPC_DIR, it will + * be restored as directory. + */ +void +create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, + bool extract_tablespaces, fio_location location) +{ + int i; + parray *links = NULL; + mode_t pg_tablespace_mode; + char to_path[MAXPGPATH]; + + /* Ugly: get PG_TBLSPC_DIR pemission mask */ + for (i = 0; i < parray_num(dest_files); i++) + { + pgFile *file = (pgFile *) parray_get(dest_files, i); + + if (!S_ISDIR(file->mode)) + continue; + + if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0) + { + if (file->external_dir_num == 0) + { + pg_tablespace_mode = file->mode; + break; + } + } + } + + /* get tablespace map */ + if (extract_tablespaces) + { + links = parray_new(); + read_tablespace_map(links, backup_dir); + /* Sort links by a link name */ + parray_qsort(links, pgFileCompareName); + } + + /* Fun part is that backup_content.control is from beginning + * of a backup, and tablespace_map is from the end + * of a backup. + * If we trust tablspace_map, we would have to we create first + * tablespaces from it, then the start creating directories and files + * from backup_content. + * The problem if that backup_content could contain files from + * deleted tablespaces and so would have to + * check every file and directory if it comes from tablespace, + * not presented in tablespace_map and skip it restoring if it + * is not. + * Trusting backup_content.control is safest way, there is no risk + * of not restoring something. + */ + + elog(LOG, "Restore directories and symlinks..."); + + /* create directories */ + for (i = 0; i < parray_num(dest_files); i++) + { + char parent_dir[MAXPGPATH]; + pgFile *dir = (pgFile *) parray_get(dest_files, i); + + if (!S_ISDIR(dir->mode)) + continue; + + /* skip external directory content */ + if (dir->external_dir_num != 0) + continue; + + /* tablespace_map exists */ + if (links) + { + /* get parent dir of rel_path */ + strncpy(parent_dir, dir->rel_path, MAXPGPATH); + get_parent_directory(parent_dir); + + /* check if directory is actually link to tablespace */ + if (strcmp(parent_dir, PG_TBLSPC_DIR) == 0) + { + /* this directory located in pg_tblspc + * check it against tablespace map + */ + pgFile **link = (pgFile **) parray_bsearch(links, dir, pgFileCompareName); + + /* got match */ + if (link) + { + const char *linked_path = get_tablespace_mapping((*link)->linked); + + if (!is_absolute_path(linked_path)) + elog(ERROR, "Tablespace directory is not an absolute path: %s\n", + linked_path); + + join_path_components(to_path, data_dir, dir->rel_path); + + elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + linked_path, to_path); + + /* create tablespace directory */ + fio_mkdir(linked_path, pg_tablespace_mode, location); + + /* create link to linked_path */ + if (fio_symlink(linked_path, to_path, location) < 0) + elog(ERROR, "Could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + continue; + } + } + } + + /* This is not symlink, create directory */ + elog(INFO, "Create directory \"%s\"", dir->rel_path); + + join_path_components(to_path, data_dir, dir->rel_path); + fio_mkdir(to_path, dir->mode, location); + } + + if (extract_tablespaces) + { + parray_walk(links, pgFileFree); + parray_free(links); + } +} + /* * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise * an error if target directories exist. @@ -1049,7 +1195,7 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) * directories into their initial path using tablespace_map file. */ void -create_data_directories(const char *data_dir, const char *backup_dir, +create_data_directories_manual(const char *data_dir, const char *backup_dir, bool extract_tablespaces, fio_location location) { parray *dirs, @@ -1234,6 +1380,8 @@ read_tablespace_map(parray *files, const char *backup_dir) file->path = pgut_malloc(strlen(link_name) + 1); strcpy(file->path, link_name); + file->name = file->path; + file->linked = pgut_malloc(strlen(path) + 1); strcpy(file->linked, path); diff --git a/src/merge.c b/src/merge.c index 3ac73cf77..43103865a 100644 --- a/src/merge.c +++ b/src/merge.c @@ -255,7 +255,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) write_backup_status(to_backup, BACKUP_STATUS_MERGING); write_backup_status(from_backup, BACKUP_STATUS_MERGING); - create_data_directories(to_database_path, from_backup_path, false, FIO_BACKUP_HOST); + create_data_directories(files, to_database_path, from_backup_path, false, FIO_BACKUP_HOST); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4527a33e1..9dc6d1156 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -577,11 +577,17 @@ extern const char* deparse_compress_alg(int alg); extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root, int external_dir_num, fio_location location); -extern void create_data_directories(const char *data_dir, +extern void create_data_directories_manual(const char *data_dir, const char *backup_dir, bool extract_tablespaces, fio_location location); +extern void create_data_directories(parray *dest_files, + const char *data_dir, + const char *backup_dir, + bool extract_tablespaces, + fio_location location); + extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); @@ -614,6 +620,7 @@ extern void pgFileDelete(pgFile *file); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read, fio_location location); +extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); diff --git a/src/restore.c b/src/restore.c index 0e1fde486..613e80358 100644 --- a/src/restore.c +++ b/src/restore.c @@ -442,7 +442,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ pgBackupGetPath(dest_backup, dest_backup_path, lengthof(dest_backup_path), NULL); - create_data_directories(instance_config.pgdata, dest_backup_path, true, + create_data_directories(dest_files, instance_config.pgdata, dest_backup_path, true, FIO_DB_HOST); /* From b09140af4d08041214a31b6f090dff9bda8eaa5b Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 27 May 2019 16:05:00 +0300 Subject: [PATCH 0701/2107] remove unused function: create_data_directories_manual() --- src/dir.c | 152 --------------------------------------------- src/pg_probackup.h | 5 -- 2 files changed, 157 deletions(-) diff --git a/src/dir.c b/src/dir.c index f8637947c..6872415fd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1187,158 +1187,6 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba } } -/* - * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise - * an error if target directories exist. - * - * If **extract_tablespaces** is true then try to extract tablespace data - * directories into their initial path using tablespace_map file. - */ -void -create_data_directories_manual(const char *data_dir, const char *backup_dir, - bool extract_tablespaces, fio_location location) -{ - parray *dirs, - *links = NULL; - size_t i; - char backup_database_dir[MAXPGPATH], - to_path[MAXPGPATH]; - dirs = parray_new(); - if (extract_tablespaces) - { - links = parray_new(); - read_tablespace_map(links, backup_dir); - /* Sort links by a link name*/ - parray_qsort(links, pgFileComparePath); - } - - join_path_components(backup_database_dir, backup_dir, DATABASE_DIR); - list_data_directories(dirs, backup_database_dir, "", false, - FIO_BACKUP_HOST); - - elog(LOG, "Restore directories and symlinks..."); - - for (i = 0; i < parray_num(dirs); i++) - { - pgFile *dir = (pgFile *) parray_get(dirs, i); - - Assert(S_ISDIR(dir->mode)); - - /* Try to create symlink and linked directory if necessary */ - if (extract_tablespaces && - path_is_prefix_of_path(PG_TBLSPC_DIR, dir->rel_path)) - { - char *link_ptr = GetRelativePath(dir->rel_path, PG_TBLSPC_DIR), - *link_sep, - *tmp_ptr; - char link_name[MAXPGPATH]; - pgFile **link; - - /* Extract link name from relative path */ - link_sep = first_dir_separator(link_ptr); - if (link_sep != NULL) - { - int len = link_sep - link_ptr; - strncpy(link_name, link_ptr, len); - link_name[len] = '\0'; - } - else - goto create_directory; - - tmp_ptr = dir->path; - dir->path = link_name; - /* Search only by symlink name without path */ - link = (pgFile **) parray_bsearch(links, dir, pgFileComparePath); - dir->path = tmp_ptr; - - if (link) - { - const char *linked_path = get_tablespace_mapping((*link)->linked); - const char *dir_created; - - if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - linked_path); - - /* Check if linked directory was created earlier */ - dir_created = get_tablespace_created(link_name); - if (dir_created) - { - /* - * If symlink and linked directory were created do not - * create it second time. - */ - if (strcmp(dir_created, linked_path) == 0) - { - /* - * Create rest of directories. - * First check is there any directory name after - * separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - else - continue; - } - else - elog(ERROR, "tablespace directory \"%s\" of page backup does not " - "match with previous created tablespace directory \"%s\" of symlink \"%s\"", - linked_path, dir_created, link_name); - } - - if (link_sep) - elog(VERBOSE, "create directory \"%s\" and symbolic link \"%.*s\"", - linked_path, - (int) (link_sep - dir->rel_path), dir->rel_path); - else - elog(VERBOSE, "create directory \"%s\" and symbolic link \"%s\"", - linked_path, dir->rel_path); - - /* Firstly, create linked directory */ - fio_mkdir(linked_path, DIR_PERMISSION, location); - - join_path_components(to_path, data_dir, PG_TBLSPC_DIR); - /* Create pg_tblspc directory just in case */ - fio_mkdir(to_path, DIR_PERMISSION, location); - - /* Secondly, create link */ - join_path_components(to_path, to_path, link_name); - if (fio_symlink(linked_path, to_path, location) < 0) - elog(ERROR, "could not create symbolic link \"%s\": %s", - to_path, strerror(errno)); - - /* Save linked directory */ - set_tablespace_created(link_name, linked_path); - - /* - * Create rest of directories. - * First check is there any directory name after separator. - */ - if (link_sep != NULL && *(link_sep + 1) != '\0') - goto create_directory; - - continue; - } - } - -create_directory: - elog(VERBOSE, "create directory \"%s\"", dir->rel_path); - - /* This is not symlink, create directory */ - join_path_components(to_path, data_dir, dir->rel_path); - fio_mkdir(to_path, DIR_PERMISSION, location); - } - - if (extract_tablespaces) - { - parray_walk(links, pgFileFree); - parray_free(links); - } - - parray_walk(dirs, pgFileFree); - parray_free(dirs); -} - /* * Read names of symbolik names of tablespaces with links to directories from * tablespace_map or tablespace_map.txt. diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 9dc6d1156..3b8d3ce3a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -577,11 +577,6 @@ extern const char* deparse_compress_alg(int alg); extern void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root, int external_dir_num, fio_location location); -extern void create_data_directories_manual(const char *data_dir, - const char *backup_dir, - bool extract_tablespaces, - fio_location location); - extern void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, From 64d5d32fffc131726f8a179a1312ff8ba74d9eac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 27 May 2019 16:57:46 +0300 Subject: [PATCH 0702/2107] improve comments in create_data_directories() --- src/dir.c | 171 ++++++++---------------------------------------------- 1 file changed, 25 insertions(+), 146 deletions(-) diff --git a/src/dir.c b/src/dir.c index 6872415fd..7d84856b3 100644 --- a/src/dir.c +++ b/src/dir.c @@ -124,16 +124,11 @@ static char dir_check_file(pgFile *file); static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, bool omit_symlink, parray *black_list, int external_dir_num, fio_location location); - -static void list_data_directories(parray *files, const char *root, - const char *rel_path, bool exclude, - fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; -static TablespaceCreatedList tablespace_created_dirs = {NULL, NULL}; /* Extra directories mapping */ static TablespaceList external_remap_list = {NULL, NULL}; @@ -840,108 +835,6 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, fio_closedir(dir); } -/* - * List data directories excluding directories from pgdata_exclude_dir array. - * - * We exclude first level of directories and on the first level we check all - * files and directories. - */ -static void -list_data_directories(parray *files, const char *root, const char *rel_path, - bool exclude, fio_location location) -{ - char full_path[MAXPGPATH]; - DIR *dir; - struct dirent *dent; - int prev_errno; - bool has_child_dirs = false; - - join_path_components(full_path, root, rel_path); - - /* open directory and list contents */ - dir = fio_opendir(full_path, location); - if (dir == NULL) - elog(ERROR, "Cannot open directory \"%s\": %s", full_path, - strerror(errno)); - - errno = 0; - while ((dent = fio_readdir(dir))) - { - char child[MAXPGPATH]; - bool skip = false; - struct stat st; - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - /* Make full child path */ - join_path_components(child, full_path, dent->d_name); - - if (fio_stat(child, &st, false, location) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - /* Check for exclude for the first level of listing */ - if (exclude && rel_path[0] == '\0') - { - int i; - - for (i = 0; pgdata_exclude_dir[i]; i++) - { - if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0) - { - skip = true; - break; - } - } - } - if (skip) - continue; - - has_child_dirs = true; - /* Make relative child path */ - join_path_components(child, rel_path, dent->d_name); - list_data_directories(files, root, child, exclude, location); - } - - /* List only full and last directories */ - if (rel_path[0] != '\0' && !has_child_dirs) - parray_append(files, - pgFileNew(full_path, rel_path, false, 0, location)); - - prev_errno = errno; - fio_closedir(dir); - - if (prev_errno && prev_errno != ENOENT) - elog(ERROR, "Cannot read directory \"%s\": %s", - full_path, strerror(prev_errno)); -} - -/* - * Save create directory path into memory. We can use it in next page restore to - * not raise the error "restore tablespace destination is not empty" in - * create_data_directories(). - */ -static void -set_tablespace_created(const char *link, const char *dir) -{ - TablespaceCreatedListCell *cell = pgut_new(TablespaceCreatedListCell); - - strcpy(cell->link_name, link); - strcpy(cell->linked_dir, dir); - cell->next = NULL; - - if (tablespace_created_dirs.tail) - tablespace_created_dirs.tail->next = cell; - else - tablespace_created_dirs.head = cell; - tablespace_created_dirs.tail = cell; -} - /* * Retrieve tablespace path, either relocated or original depending on whether * -T was passed or not. @@ -960,21 +853,6 @@ get_tablespace_mapping(const char *dir) return dir; } -/* - * Is directory was created when symlink was created in restore_directories(). - */ -static const char * -get_tablespace_created(const char *link) -{ - TablespaceCreatedListCell *cell; - - for (cell = tablespace_created_dirs.head; cell; cell = cell->next) - if (strcmp(link, cell->link_name) == 0) - return cell->linked_dir; - - return NULL; -} - /* * Split argument into old_dir and new_dir and append to mapping * list. @@ -1052,15 +930,15 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) } /* - * Create backup directories from **backup_dir** to **data_dir**. Doesn't raise - * an error if target directories exist. + * Create directories from **dest_files** in **data_dir**. * * If **extract_tablespaces** is true then try to extract tablespace data * directories into their initial path using tablespace_map file. + * Use **backup_dir** for tablespace_map extracting. * * Enforce permissions from backup_content.control. The only - * problem now is with PGDATA itself, we must preserve PGDATA permissions - * somewhere. + * problem now is with PGDATA itself. + * TODO: we must preserve PGDATA permissions somewhere. Is it actually a problem? * * TODO: symlink handling. If user located symlink in PG_TBLSPC_DIR, it will * be restored as directory. @@ -1071,10 +949,12 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba { int i; parray *links = NULL; - mode_t pg_tablespace_mode; + mode_t pg_tablespace_mode = 0; char to_path[MAXPGPATH]; - /* Ugly: get PG_TBLSPC_DIR pemission mask */ + /* Ugly: get PG_TBLSPC_DIR pemission mask. + * We will use it to set permissions for tablespace directories. + */ for (i = 0; i < parray_num(dest_files); i++) { pgFile *file = (pgFile *) parray_get(dest_files, i); @@ -1082,16 +962,22 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba if (!S_ISDIR(file->mode)) continue; + /* skip external directory content */ + if (file->external_dir_num != 0) + continue; + + /* look for 'pg_tblspc' directory */ if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0) { - if (file->external_dir_num == 0) - { - pg_tablespace_mode = file->mode; - break; - } + pg_tablespace_mode = file->mode; + break; } } + /* sanity */ + if (!pg_tablespace_mode) + pg_tablespace_mode = DIR_PERMISSION; + /* get tablespace map */ if (extract_tablespaces) { @@ -1101,19 +987,12 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba parray_qsort(links, pgFileCompareName); } - /* Fun part is that backup_content.control is from beginning - * of a backup, and tablespace_map is from the end - * of a backup. - * If we trust tablspace_map, we would have to we create first - * tablespaces from it, then the start creating directories and files - * from backup_content. - * The problem if that backup_content could contain files from - * deleted tablespaces and so would have to - * check every file and directory if it comes from tablespace, - * not presented in tablespace_map and skip it restoring if it - * is not. - * Trusting backup_content.control is safest way, there is no risk - * of not restoring something. + /* + * We iterate over dest_files and for every directory with parent 'pg_tblspc' + * we must lookup this directory name in tablespace map. + * If we got a match, we treat this directory as tablespace. + * It means that we create directory specified in tablespace_map and + * original directory created as symlink to it. */ elog(LOG, "Restore directories and symlinks..."); From c5947043953c2be0897ec43c7de025f97da39886 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 27 May 2019 18:00:59 +0300 Subject: [PATCH 0703/2107] [Issue #62] Replace pgwin32_safestat with fio_safestat --- src/utils/file.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/utils/file.c b/src/utils/file.c index 07a646d45..34b2b18f4 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -51,6 +51,41 @@ static bool fio_is_remote_fd(int fd) } #ifdef WIN32 + +#undef stat + +/* + * The stat() function in win32 is not guaranteed to update the st_size + * field when run. So we define our own version that uses the Win32 API + * to update this field. + */ +static int +fio_safestat(const char *path, struct stat *buf) +{ + int r; + WIN32_FILE_ATTRIBUTE_DATA attr; + + r = stat(path, buf); + if (r < 0) + return r; + + if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) + { + errno = ENOENT; + return -1; + } + + /* + * XXX no support for large files here, but we don't do that in general on + * Win32 yet. + */ + buf->st_size = attr.nFileSizeLow; + + return 0; +} + +#define stat(x) fio_safestat(x) + static ssize_t pread(int fd, void* buf, size_t size, off_t off) { off_t rc = lseek(fd, off, SEEK_SET); From ca02ecade092730cb520e93e2b157be17fd753dc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 27 May 2019 18:22:25 +0300 Subject: [PATCH 0704/2107] limit tablespace permissions kludge to only PG11 --- src/dir.c | 62 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/dir.c b/src/dir.c index 7d84856b3..426d16963 100644 --- a/src/dir.c +++ b/src/dir.c @@ -939,6 +939,7 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) * Enforce permissions from backup_content.control. The only * problem now is with PGDATA itself. * TODO: we must preserve PGDATA permissions somewhere. Is it actually a problem? + * Shouldn`t starting postgres force correct permissions on PGDATA? * * TODO: symlink handling. If user located symlink in PG_TBLSPC_DIR, it will * be restored as directory. @@ -949,35 +950,9 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba { int i; parray *links = NULL; - mode_t pg_tablespace_mode = 0; + mode_t pg_tablespace_mode = DIR_PERMISSION; char to_path[MAXPGPATH]; - /* Ugly: get PG_TBLSPC_DIR pemission mask. - * We will use it to set permissions for tablespace directories. - */ - for (i = 0; i < parray_num(dest_files); i++) - { - pgFile *file = (pgFile *) parray_get(dest_files, i); - - if (!S_ISDIR(file->mode)) - continue; - - /* skip external directory content */ - if (file->external_dir_num != 0) - continue; - - /* look for 'pg_tblspc' directory */ - if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0) - { - pg_tablespace_mode = file->mode; - break; - } - } - - /* sanity */ - if (!pg_tablespace_mode) - pg_tablespace_mode = DIR_PERMISSION; - /* get tablespace map */ if (extract_tablespaces) { @@ -987,6 +962,39 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba parray_qsort(links, pgFileCompareName); } + /* + * We have no idea about tablespace permission + * For PG < 11 we can just force default permissions. + */ +#if PG_VERSION_NUM >= 110000 + if (links) + { + /* For PG>=11 we use temp kludge: trust permissions on 'pg_tblspc' + * and force them on every tablespace. + * TODO: remove kludge and ask data_directory_mode + * at the start of backup. + */ + for (i = 0; i < parray_num(dest_files); i++) + { + pgFile *file = (pgFile *) parray_get(dest_files, i); + + if (!S_ISDIR(file->mode)) + continue; + + /* skip external directory content */ + if (file->external_dir_num != 0) + continue; + + /* look for 'pg_tblspc' directory */ + if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0) + { + pg_tablespace_mode = file->mode; + break; + } + } + } +#endif + /* * We iterate over dest_files and for every directory with parent 'pg_tblspc' * we must lookup this directory name in tablespace map. From d436270b8e5ca29d299ab79a4e0a6a5f313e565e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 11:11:41 +0300 Subject: [PATCH 0705/2107] tests: minor fixes --- tests/merge.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 63f2e59ca..6e52d9407 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -5,6 +5,7 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import shutil from datetime import datetime, timedelta +import time module_name = "merge" @@ -1704,6 +1705,9 @@ def test_failed_merge_after_delete(self): page_id = self.backup_node( backup_dir, 'node', node, backup_type='page') + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + gdb = self.merge_backup( backup_dir, 'node', page_id, gdb=True, options=['--log-level-console=verbose']) @@ -1730,10 +1734,14 @@ def test_failed_merge_after_delete(self): backup_dir, 'backups', 'node', full_id, 'database', 'base', dboid) - self.assertFalse( - os.path.isdir(db_path), - 'Directory {0} should not exist'.format( - db_path, full_id)) + self.merge_backup( + backup_dir, 'node', page_id_2, + options=['--log-level-console=verbose']) + + #self.assertFalse( + # os.path.isdir(db_path), + # 'Directory {0} should not exist'.format( + # db_path, full_id)) self.del_test_dir(module_name, fname) @@ -1795,7 +1803,7 @@ def test_failed_merge_after_delete_1(self): # backup half-merged self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) self.assertEqual( full_id, self.show_pb(backup_dir, 'node')[0]['id']) From 8514420c3fd6b843c8f98a5c7e9fc03ca4e2129e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 11:22:58 +0300 Subject: [PATCH 0706/2107] tests: minor changes --- tests/merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 6e52d9407..d574bf923 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1802,8 +1802,8 @@ def test_failed_merge_after_delete_1(self): gdb._execute('signal SIGKILL') # backup half-merged - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + #self.assertEqual( + # 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) self.assertEqual( full_id, self.show_pb(backup_dir, 'node')[0]['id']) From 88fed2b4244e65347dc4a7f9440b0cf21511d254 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 12:41:03 +0300 Subject: [PATCH 0707/2107] tests: add suit of basic tests --- tests/Readme.md | 6 +++- tests/__init__.py | 5 +++ tests/archive.py | 9 +----- tests/backup.py | 13 +++----- tests/checkdb.py | 2 +- tests/compatibility.py | 5 --- tests/compression.py | 6 +--- tests/delta.py | 22 +++---------- tests/external.py | 2 +- tests/false_positive.py | 6 +--- tests/merge.py | 2 +- tests/page.py | 9 +----- tests/ptrack.py | 68 +++++++++-------------------------------- tests/replica.py | 2 +- tests/restore.py | 9 ++---- tests/retention.py | 4 +-- tests/validate.py | 38 ++++++++--------------- 17 files changed, 59 insertions(+), 149 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 61d000fcc..2cfb35f4a 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -24,7 +24,11 @@ Specify path to pg_probackup binary file. By default tests use Remote backup depends on key authentithication to local machine via ssh as current user. - PGPROBACKUP_SSH_REMOTE=ON + export PGPROBACKUP_SSH_REMOTE=ON + +Run suit of basic simple tests: + export PG_PROBACKUP_TEST_BASIC=ON + Usage: pip install testgres diff --git a/tests/__init__.py b/tests/__init__.py index 495d4bdbb..a0a552dca 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,5 @@ import unittest +import os from . import init, merge, option, show, compatibility, \ backup, delete, delta, restore, validate, \ @@ -10,6 +11,10 @@ def load_tests(loader, tests, pattern): suite = unittest.TestSuite() + + if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON': + loader.testMethodPrefix = 'test_basic' + # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) diff --git a/tests/archive.py b/tests/archive.py index 4f6a6e210..465151d63 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -25,7 +25,6 @@ def test_pgpro434_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) @@ -83,7 +82,6 @@ def test_pgpro434_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) @@ -232,7 +230,6 @@ def test_pgpro434_3(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) @@ -287,7 +284,6 @@ def test_arhive_push_file_exists(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) @@ -363,7 +359,6 @@ def test_arhive_push_file_exists_overwrite(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) @@ -639,7 +634,7 @@ def test_master_and_replica_parallel_archiving(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_master_and_replica_concurrent_archiving(self): + def test_basic_master_and_replica_concurrent_archiving(self): """ make node 'master 'with archiving, take archive backup and turn it into replica, @@ -736,7 +731,6 @@ def test_archive_pg_receivexlog(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) @@ -810,7 +804,6 @@ def test_archive_pg_receivexlog_compression_pg10(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) self.init_pb(backup_dir) diff --git a/tests/backup.py b/tests/backup.py index 1f76b2963..e3e58d9c7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -263,8 +263,7 @@ def test_ptrack_threads_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -291,8 +290,7 @@ def test_page_corruption_heal_via_ptrack_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -348,8 +346,7 @@ def test_page_corruption_heal_via_ptrack_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -521,7 +518,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_tablespace_handling(self): + def test_basic_tablespace_handling(self): """ make node, take full backup, check that restore with tablespace mapping will end with error, take page backup, @@ -968,7 +965,7 @@ def test_persistent_slot_for_stream_backup(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_temp_slot_for_stream_backup(self): + def test_basic_temp_slot_for_stream_backup(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/checkdb.py b/tests/checkdb.py index 0a8054f54..8bab89b9f 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -212,7 +212,7 @@ def test_checkdb_amcheck_only_sanity(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_checkdb_amcheck_only_sanity_1(self): + def test_basic_checkdb_amcheck_only_sanity(self): """""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/compatibility.py b/tests/compatibility.py index b4d66af37..6830ce431 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -20,7 +20,6 @@ def test_backward_compatibility_page(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'} ) self.init_pb(backup_dir, old_binary=True) @@ -123,7 +122,6 @@ def test_backward_compatibility_delta(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'} ) self.init_pb(backup_dir, old_binary=True) @@ -226,7 +224,6 @@ def test_backward_compatibility_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'} ) self.init_pb(backup_dir, old_binary=True) @@ -329,7 +326,6 @@ def test_backward_compatibility_compression(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) @@ -494,7 +490,6 @@ def test_backward_compatibility_merge(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) diff --git a/tests/compression.py b/tests/compression.py index fd86d15a3..ef2ecf5ee 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -12,7 +12,7 @@ class CompressionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_compression_stream_zlib(self): + def test_basic_compression_stream_zlib(self): """ make archive node, make full and page stream backups, check data correctness in restored instance @@ -25,7 +25,6 @@ def test_compression_stream_zlib(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} ) @@ -245,7 +244,6 @@ def test_compression_stream_pglz(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} ) @@ -355,7 +353,6 @@ def test_compression_archive_pglz(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} ) @@ -465,7 +462,6 @@ def test_compression_wrong_algorithm(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on'} ) diff --git a/tests/delta.py b/tests/delta.py index 8083c57c2..c4343daf6 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -13,7 +13,7 @@ class DeltaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") - def test_delta_vacuum_truncate_1(self): + def test_basic_delta_vacuum_truncate(self): """ make node, create table, take full backup, delete last 3 pages, vacuum relation, @@ -27,7 +27,6 @@ def test_delta_vacuum_truncate_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off'}) @@ -92,7 +91,7 @@ def test_delta_vacuum_truncate_1(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_delta_vacuum_truncate_2(self): + def test_delta_vacuum_truncate_1(self): """ make node, create table, take full backup, delete last 3 pages, vacuum relation, @@ -106,7 +105,6 @@ def test_delta_vacuum_truncate_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' } @@ -182,7 +180,7 @@ def test_delta_vacuum_truncate_2(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_delta_vacuum_truncate_3(self): + def test_delta_vacuum_truncate_2(self): """ make node, create table, take full backup, delete last 3 pages, vacuum relation, @@ -196,7 +194,6 @@ def test_delta_vacuum_truncate_3(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' } @@ -265,7 +262,6 @@ def test_delta_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s' } ) @@ -424,7 +420,6 @@ def test_delta_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', @@ -506,7 +501,6 @@ def test_delta_vacuum_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s' } ) @@ -598,7 +592,6 @@ def test_create_db(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'max_wal_senders': '2', 'checkpoint_timeout': '5min', 'autovacuum': 'off' } @@ -728,7 +721,6 @@ def test_exists_in_previous_backup(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'max_wal_senders': '2', 'checkpoint_timeout': '5min', 'autovacuum': 'off' } @@ -835,7 +827,6 @@ def test_alter_table_set_tablespace_delta(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' } @@ -1019,7 +1010,6 @@ def test_delta_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' } @@ -1099,8 +1089,7 @@ def test_delta_corruption_heal_via_ptrack_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1158,8 +1147,7 @@ def test_page_corruption_heal_via_ptrack_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/external.py b/tests/external.py index df5695261..4673dec92 100644 --- a/tests/external.py +++ b/tests/external.py @@ -13,7 +13,7 @@ class ExternalTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_external_simple(self): + def test_basic_external(self): """ make node, create external directory, take backup with external directory, restore backup, check that diff --git a/tests/false_positive.py b/tests/false_positive.py index 558283a7c..d33309f4a 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -20,9 +20,7 @@ def test_validate_wal_lost_segment(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -123,7 +121,6 @@ def test_ptrack_concurrent_get_and_clear_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } @@ -205,7 +202,6 @@ def test_ptrack_concurrent_get_and_clear_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } diff --git a/tests/merge.py b/tests/merge.py index d574bf923..e9920ccd5 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -12,7 +12,7 @@ class MergeTest(ProbackupTest, unittest.TestCase): - def test_merge_full_page(self): + def test_basic_merge_full_page(self): """ Test MERGE command, it merges FULL backup with target PAGE backups """ diff --git a/tests/page.py b/tests/page.py index 62f7e0546..7b7a32364 100644 --- a/tests/page.py +++ b/tests/page.py @@ -13,7 +13,7 @@ class PageBackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") - def test_page_vacuum_truncate(self): + def test_basic_page_vacuum_truncate(self): """ make node, create table, take full backup, delete last 3 pages, vacuum relation, @@ -27,7 +27,6 @@ def test_page_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'autovacuum': 'off' } @@ -119,7 +118,6 @@ def test_page_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) @@ -207,7 +205,6 @@ def test_page_archive(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s'} ) @@ -301,7 +298,6 @@ def test_page_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', @@ -383,7 +379,6 @@ def test_page_delete(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' } @@ -462,7 +457,6 @@ def test_page_delete_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'autovacuum': 'off' } @@ -1030,7 +1024,6 @@ def test_page_create_db(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'max_wal_senders': '2', 'checkpoint_timeout': '5min', 'autovacuum': 'off' } diff --git a/tests/ptrack.py b/tests/ptrack.py index 76d7d3ba8..3387a8b4d 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -24,7 +24,6 @@ def test_ptrack_enable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s' } ) @@ -72,7 +71,6 @@ def test_ptrack_disable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on' } @@ -132,7 +130,6 @@ def test_ptrack_uncommited_xact(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } @@ -193,7 +190,6 @@ def test_ptrack_vacuum_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } @@ -284,7 +280,6 @@ def test_ptrack_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -369,7 +364,6 @@ def test_ptrack_simple(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } @@ -444,7 +438,6 @@ def test_ptrack_get_block(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '300s', 'ptrack_enable': 'on' } @@ -515,7 +508,6 @@ def test_ptrack_stream(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -609,7 +601,6 @@ def test_ptrack_archive(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -715,7 +706,6 @@ def test_ptrack_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'} @@ -797,7 +787,6 @@ def test_page_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'} @@ -868,7 +857,6 @@ def test_full_pgpro417(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' } @@ -945,7 +933,6 @@ def test_create_db(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'max_wal_senders': '2', 'checkpoint_timeout': '5min', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -1070,7 +1057,6 @@ def test_create_db_on_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -1169,7 +1155,6 @@ def test_alter_table_set_tablespace_ptrack(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off' @@ -1266,7 +1251,6 @@ def test_alter_database_set_tablespace_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'} @@ -1336,7 +1320,6 @@ def test_drop_tablespace(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'} @@ -1419,7 +1402,6 @@ def test_ptrack_alter_tablespace(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', 'autovacuum': 'off'}) @@ -1531,7 +1513,6 @@ def test_ptrack_multiple_segments(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'ptrack_enable': 'on', 'fsync': 'off', 'autovacuum': 'off', 'full_page_writes': 'off' @@ -1641,7 +1622,6 @@ def test_atexit_fail(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'max_connections': '15'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1693,8 +1673,8 @@ def test_ptrack_clean(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1805,7 +1785,6 @@ def test_ptrack_clean_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'archive_timeout': '30s'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1940,8 +1919,7 @@ def test_ptrack_cluster_on_btree(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2020,8 +1998,7 @@ def test_ptrack_cluster_on_gist(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2097,8 +2074,7 @@ def test_ptrack_cluster_on_btree_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2196,8 +2172,7 @@ def test_ptrack_cluster_on_gist_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2302,7 +2277,6 @@ def test_ptrack_empty(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2381,8 +2355,7 @@ def test_ptrack_empty_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2475,8 +2448,7 @@ def test_ptrack_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2644,8 +2616,7 @@ def test_ptrack_vacuum(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2731,7 +2702,6 @@ def test_ptrack_vacuum_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'checkpoint_timeout': '30'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2839,8 +2809,7 @@ def test_ptrack_vacuum_bits_frozen(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2918,8 +2887,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3022,8 +2990,7 @@ def test_ptrack_vacuum_bits_visibility(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3102,8 +3069,7 @@ def test_ptrack_vacuum_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3183,7 +3149,6 @@ def test_ptrack_vacuum_full_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'autovacuum': 'off', 'archive_timeout': '30s'} ) @@ -3291,8 +3256,7 @@ def test_ptrack_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3372,7 +3336,6 @@ def test_ptrack_vacuum_truncate_replica(self): initdb_params=['--data-checksums'], pg_options={ 'ptrack_enable': 'on', - 'max_wal_senders': '2', 'checkpoint_timeout': '30'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3471,8 +3434,7 @@ def test_ptrack_recovery(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', - 'max_wal_senders': '2'}) + 'ptrack_enable': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/replica.py b/tests/replica.py index 116dbdd98..1d21a58fa 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -278,7 +278,7 @@ def test_replica_archive_page_backup(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_make_replica_via_restore(self): + def test_basic_make_replica_via_restore(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica diff --git a/tests/restore.py b/tests/restore.py index 00f5b5a21..5a86b0bb1 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1301,7 +1301,6 @@ def test_zags_block_corrupt(self): conn.execute( "insert into tbl select i from generate_series(0,100) as i") - node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored'), initdb_params=['--data-checksums']) @@ -1439,9 +1438,7 @@ def test_restore_chain(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1529,9 +1526,7 @@ def test_restore_chain_with_corrupted_backup(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/retention.py b/tests/retention.py index 21ee76962..cdab447bc 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -739,7 +739,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_window_merge_multiple_descendants(self): + def test_basic_window_merge_multiple_descendants(self): """ PAGEb3 | PAGEa3 @@ -938,8 +938,6 @@ def test_window_merge_multiple_descendants(self): '--retention-window=1', '--expired', '--merge-expired', '--log-level-console=log']) - print(output) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) # Merging chain A diff --git a/tests/validate.py b/tests/validate.py index b2d2e0ce0..c6dc2b746 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -15,7 +15,7 @@ class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure - def test_validate_nullified_heap_page_backup(self): + def test_basic_validate_nullified_heap_page_backup(self): """ make node with nullified heap block """ @@ -216,7 +216,7 @@ def test_validate_wal_unreal_values(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_validate_corrupted_intermediate_backup(self): + def test_basic_validate_corrupted_intermediate_backup(self): """ make archive node, take FULL, PAGE1, PAGE2 backups, corrupt file in PAGE1 backup, @@ -1621,8 +1621,7 @@ def test_pgpro702_688(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1663,8 +1662,7 @@ def test_pgpro688(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1694,8 +1692,7 @@ def test_pgpro561(self): node1 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node1'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1808,7 +1805,6 @@ def test_validate_corrupted_full(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_senders': '2', 'checkpoint_timeout': '30'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1931,8 +1927,7 @@ def test_validate_corrupted_full_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2043,8 +2038,7 @@ def test_validate_corrupted_full_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2396,8 +2390,7 @@ def test_validate_corrupted_full_missing(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2697,8 +2690,7 @@ def test_validate_specific_backup_with_missing_backup(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2841,8 +2833,7 @@ def test_validate_specific_backup_with_missing_backup_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2963,8 +2954,7 @@ def test_validate_with_missing_backup_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3153,8 +3143,7 @@ def test_validate_with_missing_backup_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3301,8 +3290,7 @@ def test_corrupt_pg_control_via_resetxlog(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_senders': '2'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From d38b5e6ece64a0bac09544c823f3f9c3dfcbf844 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 28 May 2019 14:09:52 +0300 Subject: [PATCH 0708/2107] Removed libpgfeutils.lib from Windows project --- gen_probackup_project.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 0fe27fd74..6de352e86 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -196,8 +196,10 @@ sub build_pgprobackup $probackup->AddIncludeDir("$currpath/src"); $probackup->AddIncludeDir("$currpath/src/utils"); - $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + # $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + $probackup->AddReference($libpq, $libpgcommon, $libpgport); $probackup->AddLibrary('ws2_32.lib'); + $probackup->AddLibrary('advapi32.lib'); $probackup->Save(); return $solution->{vcver}; From 1b0eb6869496b92aa817012cef09069ee5134256 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 17:50:03 +0300 Subject: [PATCH 0709/2107] tests: fix pgpro2068 --- tests/pgpro2068.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 74655209f..fa8e17d1e 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -23,12 +23,12 @@ def test_minrecpoint_on_replica(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '60min', + # 'checkpoint_timeout': '60min', 'checkpoint_completion_target': '0.9', 'bgwriter_delay': '10ms', 'bgwriter_lru_maxpages': '2000', 'bgwriter_lru_multiplier': '4.0', - 'max_wal_size': '100GB'}) + 'max_wal_size': '256MB'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -75,36 +75,42 @@ def test_minrecpoint_on_replica(self): pgbench.wait() pgbench.stdout.close() - # generate some more data and leave it in background pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "30"]) + options=["-c", "4", "-j 4", "-T", "100"]) # get pids of background workers startup_pid = replica.safe_psql( 'postgres', - "select pid from pg_stat_activity where backend_type = 'startup'").rstrip() + "select pid from pg_stat_activity " + "where backend_type = 'startup'").rstrip() checkpointer_pid = replica.safe_psql( 'postgres', - "select pid from pg_stat_activity where backend_type = 'checkpointer'").rstrip() + "select pid from pg_stat_activity " + "where backend_type = 'checkpointer'").rstrip() bgwriter_pid = replica.safe_psql( 'postgres', - "select pid from pg_stat_activity where backend_type = 'background writer'").rstrip() + "select pid from pg_stat_activity " + "where backend_type = 'background writer'").rstrip() # wait for shared buffer on replica to be filled with dirty data sleep(10) # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) + gdb_checkpointer._execute('handle SIGINT noprint nostop pass') + gdb_checkpointer._execute('handle SIGUSR1 noprint nostop pass') gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') gdb_checkpointer.continue_execution_until_break() # break recovery on UpdateControlFile gdb_recovery = self.gdb_attach(startup_pid) + gdb_recovery._execute('handle SIGINT noprint nostop pass') + gdb_recovery._execute('handle SIGUSR1 noprint nostop pass') gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') gdb_recovery.continue_execution_until_break() gdb_recovery.set_breakpoint('UpdateControlFile') From b2fdd73999aeb52adaa9afdbcfc515250ecca028 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 17:59:17 +0300 Subject: [PATCH 0710/2107] [Issue #65] Improvement of merge crash safety, previously FULL backup would gain OK status before changing of start_time. If crash happens here, then in rare case of FULL backup having multiple children it could lead to data corruption when restoring one of such children --- src/merge.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 2cb9ab361..015a490d7 100644 --- a/src/merge.c +++ b/src/merge.c @@ -328,8 +328,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Update to_backup metadata. + * We cannot set backup status to OK just yet, + * because it still has old start_time. */ - to_backup->status = BACKUP_STATUS_OK; StrNCpy(to_backup->program_version, PROGRAM_VERSION, sizeof(to_backup->program_version)); to_backup->parent_backup = INVALID_BACKUP_ID; @@ -421,7 +422,12 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Merging finished, now we can safely update ID of the destination backup. + * TODO: for this critical section we must save incremental backup start_tome + * to FULL backup meta, so even if crash happens after incremental backup removal + * but before full backup obtaining new start_time we could safely continue + * this failed backup. */ + to_backup->status = BACKUP_STATUS_OK; to_backup->start_time = from_backup->start_time; write_backup(to_backup); From 284613bb2f0a06b56c5dec15f0b92b5833ba0575 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 19:51:19 +0300 Subject: [PATCH 0711/2107] tests: added tests.backup.BackupTest.test_drop_table --- tests/backup.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e3e58d9c7..91465d162 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1246,3 +1246,38 @@ def test_sigquit_handling(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_drop_table(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + connect_1 = node.connect("postgres") + connect_1.execute( + "create table t_heap as select i" + " as id from generate_series(0,100) i") + connect_1.commit() + + connect_2 = node.connect("postgres") + connect_2.execute("SELECT * FROM t_heap") + connect_2.commit() + + # DROP table + connect_2.execute("DROP TABLE t_heap") + connect_2.commit() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 4d5ed225417cec268bad11d3039fab4cef8ea230 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 22:10:21 +0300 Subject: [PATCH 0712/2107] fix stat() macro --- src/utils/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 34b2b18f4..ec9350370 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -84,7 +84,7 @@ fio_safestat(const char *path, struct stat *buf) return 0; } -#define stat(x) fio_safestat(x) +#define stat(x, y) fio_safestat(x, y) static ssize_t pread(int fd, void* buf, size_t size, off_t off) { From 64550a6675a75f7cd09d5d40bbf093a012db0c31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 May 2019 22:37:12 +0300 Subject: [PATCH 0713/2107] tests: added backup.BackupTest.test_basic_missing_dir_permissions and backup.BackupTest.test_basic_missing_file_permissions --- tests/backup.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 91465d162..ff9ff41d3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1281,3 +1281,87 @@ def test_drop_table(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_missing_file_permissions(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pg_class')").rstrip() + + full_path = os.path.join(node.data_dir, relative_path) + + os.chmod(full_path, 000) + + try: + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: cannot open file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + os.chmod(full_path, 700) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_missing_dir_permissions(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + full_path = os.path.join(node.data_dir, 'pg_twophase') + + os.chmod(full_path, 000) + + try: + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Cannot open directory', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + os.chmod(full_path, 700) + + # Clean after yourself + self.del_test_dir(module_name, fname) From fcc1c4b93988ced46f14a210afe51ea3938eaaa1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 May 2019 12:09:53 +0300 Subject: [PATCH 0714/2107] tests: fix checkdb tests --- tests/checkdb.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 8bab89b9f..4db231a18 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -88,7 +88,7 @@ def test_checkdb_amcheck_only_sanity(self): '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '-d', 'postgres','-p', str(node.port)]) + '-d', 'postgres', '-p', str(node.port)]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -202,7 +202,8 @@ def test_checkdb_amcheck_only_sanity(self): 'ERROR: Checkdb --amcheck failed', log_file_content) self.assertIn( - "WARNING: Thread [1]. Amcheck failed for index: 'public.t_heap_id_idx':", + "WARNING: Thread [1]. Amcheck failed in database 'postgres' " + "for index: 'public.t_heap_id_idx':", log_file_content) self.assertIn( 'ERROR: could not open relation with OID', @@ -333,11 +334,13 @@ def test_basic_checkdb_amcheck_only_sanity(self): with open(log_file_path) as f: log_file_content = f.read() self.assertIn( - "WARNING: Thread [1]. Amcheck failed for index: 'public.pgbench_accounts_pkey':", + "WARNING: Thread [1]. Amcheck failed in database 'db1' " + "for index: 'public.pgbench_accounts_pkey':", log_file_content) self.assertIn( - "WARNING: Thread [1]. Amcheck failed for index: 'public.some_index':", + "WARNING: Thread [1]. Amcheck failed in database 'db2' " + "for index: 'public.some_index':", log_file_content) self.assertIn( From 0fa0ec2d8c1135ca4f66357d9df0bd7a2d26dadf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 May 2019 18:48:57 +0300 Subject: [PATCH 0715/2107] Readme: update instruction about installing debug packages --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 9c0c2c8ce..0a2166bbf 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update apt-get install pg-probackup-{11,10,9.6,9.5} +apt-get install pg-probackup-{11,10,9.6,9.5}-dbg #DEB-SRC Packages echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ @@ -64,14 +65,17 @@ apt-get source pg-probackup-{11,10,9.6,9.5} #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm yum install pg_probackup-{11,10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5}-debuginfo #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm yum install pg_probackup-{11,10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5}-debuginfo #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm yum install pg_probackup-{11,10,9.6,9.5} +yum install pg_probackup-{11,10,9.6,9.5}-debuginfo #SRPM Packages yumdownloader --source pg_probackup-{11,10,9.6,9.5} From 2d7e60f9b15e851ed661531c2e7e8dc4311167e8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 May 2019 18:49:38 +0300 Subject: [PATCH 0716/2107] tests: minor fixes --- tests/merge.py | 54 ++++++++++++++++++++++++++++------------------ tests/pgpro2068.py | 2 +- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index e9920ccd5..cb7e2083c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -176,7 +176,7 @@ def test_merge_compressed_backups_1(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"], + set_replication=True, initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' } @@ -253,7 +253,7 @@ def test_merge_compressed_and_uncompressed_backups(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"], + set_replication=True, initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' } @@ -329,7 +329,7 @@ def test_merge_compressed_and_uncompressed_backups_1(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"], + set_replication=True, initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' } @@ -406,7 +406,7 @@ def test_merge_compressed_and_uncompressed_backups_2(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"], + set_replication=True, initdb_params=["--data-checksums"], pg_options={ 'autovacuum': 'off' } @@ -1734,14 +1734,22 @@ def test_failed_merge_after_delete(self): backup_dir, 'backups', 'node', full_id, 'database', 'base', dboid) - self.merge_backup( - backup_dir, 'node', page_id_2, - options=['--log-level-console=verbose']) - - #self.assertFalse( - # os.path.isdir(db_path), - # 'Directory {0} should not exist'.format( - # db_path, full_id)) + try: + self.merge_backup( + backup_dir, 'node', page_id_2, + options=['--log-level-console=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of missing parent.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Parent full backup for the given " + "backup {0} was not found".format( + page_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.del_test_dir(module_name, fname) @@ -1798,22 +1806,26 @@ def test_failed_merge_after_delete_1(self): gdb.set_breakpoint('pgFileDelete') gdb.continue_execution_until_break(30) - gdb._execute('signal SIGKILL') - # backup half-merged - #self.assertEqual( - # 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) - self.assertEqual( full_id, self.show_pb(backup_dir, 'node')[0]['id']) # restore node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + try: + self.restore_node(backup_dir, 'node', node) + self.assertEqual( + 1, 0, + "Expecting Error because of orphan status.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} is orphan".format(page_1), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) self.del_test_dir(module_name, fname) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index fa8e17d1e..6004483ca 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -26,7 +26,7 @@ def test_minrecpoint_on_replica(self): # 'checkpoint_timeout': '60min', 'checkpoint_completion_target': '0.9', 'bgwriter_delay': '10ms', - 'bgwriter_lru_maxpages': '2000', + 'bgwriter_lru_maxpages': '1000', 'bgwriter_lru_multiplier': '4.0', 'max_wal_size': '256MB'}) From 2ac21b541dd54059a61a10c46af8f844956bf2e0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 May 2019 19:06:21 +0300 Subject: [PATCH 0717/2107] Release 2.1.3 --- README.md | 2 +- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0a2166bbf..dc461d6bf 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Current release -[2.1.2](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.2) +[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) ## Documentation diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2ffaa7fa2..47d42af74 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -196,8 +196,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.2" -#define AGENT_PROTOCOL_VERSION 20102 +#define PROGRAM_VERSION "2.1.3" +#define AGENT_PROTOCOL_VERSION 20103 /* * An instance configuration. It can be stored in a configuration file or passed diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 18458de4e..ac690b66b 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.2 \ No newline at end of file +pg_probackup 2.1.3 \ No newline at end of file From a9bbff9fa4830be99946725a4fe80a66146462d6 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Fri, 31 May 2019 12:43:53 +0300 Subject: [PATCH 0718/2107] Wait for previous segment in case of invalid stop_lsn --- src/backup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backup.c b/src/backup.c index 2fe82bc03..b0f669301 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1989,6 +1989,7 @@ pg_stop_backup(pgBackup *backup) else xlog_path = arclog_path; + wait_wal_lsn(stop_backup_lsn, false, true); stop_backup_lsn = get_last_wal_lsn(xlog_path, backup->start_lsn, stop_backup_lsn, backup->tli, true, instance_config.xlog_seg_size); From 30126c6eb5ec6756285c051dd5ff670a8aa670a3 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 31 May 2019 18:15:43 +0300 Subject: [PATCH 0719/2107] Refactoring of checkdb. Move code to a separate file, improve error messages --- Makefile | 2 +- src/backup.c | 931 +++++++++------------------------------------ src/checkdb.c | 709 ++++++++++++++++++++++++++++++++++ src/configure.c | 16 +- src/data.c | 31 +- src/pg_probackup.c | 24 +- src/pg_probackup.h | 66 ++-- src/utils/pgut.c | 12 + src/utils/pgut.h | 1 + tests/checkdb.py | 14 +- 10 files changed, 987 insertions(+), 819 deletions(-) create mode 100644 src/checkdb.c diff --git a/Makefile b/Makefile index 687fe269c..499255f8b 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PROGRAM = pg_probackup OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ src/utils/parray.o src/utils/pgut.o src/utils/thread.o src/utils/remote.o src/utils/file.o -OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \ +OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ src/validate.o diff --git a/src/backup.c b/src/backup.c index b0f669301..d6cd988d8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -49,8 +49,6 @@ const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; -/* list of indexes for use in checkdb --amcheck */ -static parray *index_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -68,6 +66,9 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; + + XLogRecPtr startpos; + TimeLineID starttli; } StreamThreadArg; static pthread_t stream_thread; @@ -76,11 +77,6 @@ static StreamThreadArg stream_thread_arg = {"", NULL, 1}; static int is_ptrack_enable = false; bool is_ptrack_support = false; bool exclusive_backup = false; -bool heapallindexed_is_supported = false; - -/* Backup connections */ -static PGconn *backup_conn = NULL; -static PGconn *master_conn = NULL; /* PostgreSQL server version from "backup_conn" */ static int server_version = 0; @@ -95,69 +91,67 @@ static bool pg_stop_backup_is_sent = false; * Backup routines */ static void backup_cleanup(bool fatal, void *userdata); -static void backup_disconnect(bool fatal, void *userdata); - -static void pgdata_basic_setup(bool amcheck_only); static void *backup_files(void *arg); -static void do_backup_instance(void); - -static void do_block_validation(void); -static void do_amcheck(void); -static void *check_files(void *arg); -static void *check_indexes(void *arg); -static parray* get_index_list(PGresult* res_db, int db_number, - bool first_db_with_amcheck, PGconn* db_conn); -static bool amcheck_one_index(backup_files_arg *arguments, - pg_indexEntry *ind); +static void do_backup_instance(PGconn *backup_conn); -static void pg_start_backup(const char *label, bool smooth, pgBackup *backup); +static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, + PGconn *backup_conn, PGconn *master_conn); static void pg_switch_wal(PGconn *conn); -static void pg_stop_backup(pgBackup *backup); -static int checkpoint_timeout(void); +static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn); +static int checkpoint_timeout(PGconn *backup_conn); //static void backup_list_file(parray *files, const char *root, ) -static void parse_backup_filelist_filenames(parray *files, const char *root); static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment); -static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup); -static void make_pagemap_from_ptrack(parray *files); +static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup, PGconn *backup_conn); +static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn); static void *StreamLog(void *arg); -static void check_external_for_tablespaces(parray *external_list); +static void check_external_for_tablespaces(parray *external_list, + PGconn *backup_conn); /* Ptrack functions */ -static void pg_ptrack_clear(void); -static bool pg_ptrack_support(void); -static bool pg_ptrack_enable(void); -static bool pg_checksum_enable(void); -static bool pg_is_in_recovery(void); -static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid); +static void pg_ptrack_clear(PGconn *backup_conn); +static bool pg_ptrack_support(PGconn *backup_conn); +static bool pg_ptrack_enable(PGconn *backup_conn); +static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, + PGconn *backup_conn); static char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid, - size_t *result_size); -static XLogRecPtr get_last_ptrack_lsn(void); + size_t *result_size, + PGconn *backup_conn); +static XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn); /* Check functions */ -static void check_server_version(void); -static void check_system_identifiers(void); -static void confirm_block_size(const char *name, int blcksz); +static bool pg_checksum_enable(PGconn *conn); +static bool pg_is_in_recovery(PGconn *conn); +static void check_server_version(PGconn *conn); +static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); -#define disconnect_and_exit(code) \ - { \ - if (conn != NULL) PQfinish(conn); \ - exit(code); \ +static void +backup_stopbackup_callback(bool fatal, void *userdata) +{ + PGconn *pg_startbackup_conn = (PGconn *) userdata; + /* + * If backup is in progress, notify stop of backup to PostgreSQL + */ + if (backup_in_progress) + { + elog(WARNING, "backup in progress, stop backup"); + pg_stop_backup(NULL, pg_startbackup_conn); /* don't care stop_lsn on error case */ } +} /* * Take a backup of a single postgresql instance. * Move files from 'pgdata' to a subdirectory in 'backup_path'. */ static void -do_backup_instance(void) +do_backup_instance(PGconn *backup_conn) { int i; char database_path[MAXPGPATH]; @@ -177,13 +171,15 @@ do_backup_instance(void) parray *external_dirs = NULL; pgFile *pg_control = NULL; + PGconn *master_conn = NULL; + PGconn *pg_startbackup_conn = NULL; elog(LOG, "Database backup start"); if(current.external_dir_str) { external_dirs = make_external_directory_list(current.external_dir_str, false); - check_external_for_tablespaces(external_dirs); + check_external_for_tablespaces(external_dirs, backup_conn); } /* Initialize size summary */ @@ -228,7 +224,7 @@ do_backup_instance(void) */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(); + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn); if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) { @@ -242,13 +238,27 @@ do_backup_instance(void) /* Clear ptrack files for FULL and PAGE backup */ if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) - pg_ptrack_clear(); + pg_ptrack_clear(backup_conn); /* notify start of backup to PostgreSQL server */ time2iso(label, lengthof(label), current.start_time); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); - pg_start_backup(label, smooth_checkpoint, ¤t); + + /* Create connection to master server needed to call pg_start_backup */ + if (current.from_replica && exclusive_backup) + { + master_conn = pgut_connect(instance_config.master_conn_opt.pghost, + instance_config.master_conn_opt.pgport, + instance_config.master_conn_opt.pgdatabase, + instance_config.master_conn_opt.pguser); + pg_startbackup_conn = master_conn; + } + else + pg_startbackup_conn = backup_conn; + + pg_start_backup(label, smooth_checkpoint, ¤t, + backup_conn, pg_startbackup_conn); /* For incremental backup check that start_lsn is not from the past */ if (current.backup_mode != BACKUP_MODE_FULL && @@ -270,6 +280,10 @@ do_backup_instance(void) /* start stream replication */ if (stream_wal) { + /* How long we should wait for streaming end after pg_stop_backup */ + stream_stop_timeout = checkpoint_timeout(backup_conn); + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); @@ -278,10 +292,10 @@ do_backup_instance(void) /* * Connect in replication mode to the server. */ - stream_thread_arg.conn = pgut_connect_replication(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); + stream_thread_arg.conn = pgut_connect_replication(instance_config.conn_opt.pghost, + instance_config.conn_opt.pgport, + instance_config.conn_opt.pgdatabase, + instance_config.conn_opt.pguser); if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) { @@ -307,6 +321,9 @@ do_backup_instance(void) /* By default there are some error */ stream_thread_arg.ret = 1; + /* we must use startpos as start_lsn from start_backup */ + stream_thread_arg.startpos = current.start_lsn; + stream_thread_arg.starttli = current.tli; thread_interrupted = false; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); @@ -346,13 +363,13 @@ do_backup_instance(void) * 1 - create 'base' * 2 - create 'base/1' * - * Sorted array is used at least in parse_backup_filelist_filenames(), + * Sorted array is used at least in parse_filelist_filenames(), * extractPageMap(), make_pagemap_from_ptrack(). */ parray_qsort(backup_files_list, pgFileComparePath); /* Extract information about files in backup_list parsing their names:*/ - parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); + parse_filelist_filenames(backup_files_list, instance_config.pgdata); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -381,7 +398,7 @@ do_backup_instance(void) /* * Build the page map from ptrack information. */ - make_pagemap_from_ptrack(backup_files_list); + make_pagemap_from_ptrack(backup_files_list, backup_conn); } /* @@ -443,8 +460,8 @@ do_backup_instance(void) arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; - arg->backup_conn = NULL; - arg->cancel_conn = NULL; + arg->conn_arg.conn = NULL; + arg->conn_arg.cancel_conn = NULL; /* By default there are some error */ arg->ret = 1; } @@ -514,9 +531,8 @@ do_backup_instance(void) } } - /* Notify end of backup */ - pg_stop_backup(¤t); + pg_stop_backup(¤t, pg_startbackup_conn); if (current.from_replica && !exclusive_backup) set_min_recovery_point(pg_control, database_path, current.stop_lsn); @@ -589,251 +605,6 @@ do_backup_instance(void) backup_files_list = NULL; } -/* collect list of files and run threads to check files in the instance */ -static void -do_block_validation(void) -{ - int i; - char database_path[MAXPGPATH]; - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - backup_files_arg *threads_args; - bool check_isok = true; - - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - /* initialize file list */ - backup_files_list = parray_new(); - - /* list files with the logical path. omit $PGDATA */ - dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, 0, FIO_DB_HOST); - - /* - * Sort pathname ascending. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - */ - parray_qsort(backup_files_list, pgFileComparePath); - /* Extract information about files in backup_list parsing their names:*/ - parse_backup_filelist_filenames(backup_files_list, instance_config.pgdata); - - /* setup threads */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - pg_atomic_clear_flag(&file->lock); - } - - /* Sort by size for load balancing */ - parray_qsort(backup_files_list, pgFileCompareSize); - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); - - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - arg->from_root = instance_config.pgdata; - arg->to_root = database_path; - arg->files_list = backup_files_list; - arg->index_list = NULL; - arg->prev_filelist = NULL; - arg->prev_start_lsn = InvalidXLogRecPtr; - arg->backup_conn = NULL; - arg->cancel_conn = NULL; - /* By default there are some error */ - arg->ret = 1; - } - - /* TODO write better info message */ - elog(INFO, "Start checking data files"); - - /* Run threads */ - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - elog(VERBOSE, "Start thread num: %i", i); - - pthread_create(&threads[i], NULL, check_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret > 0) - check_isok = false; - } - - /* cleanup */ - if (backup_files_list) - { - parray_walk(backup_files_list, pgFileFree); - parray_free(backup_files_list); - backup_files_list = NULL; - } - - /* TODO write better info message */ - if (check_isok) - elog(INFO, "Data files are valid"); - else - elog(ERROR, "Checkdb failed"); -} - -/* - * Entry point of checkdb --amcheck. - * - * Connect to all databases in the cluster - * and get list of persistent indexes, - * then run parallel threads to perform bt_index_check() - * for all indexes from the list. - * - * If amcheck extension is not installed in the database, - * skip this database and report it via warning message. - */ -static void -do_amcheck(void) -{ - int i; - char database_path[MAXPGPATH]; - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - backup_files_arg *threads_args; - bool check_isok = true; - PGresult *res_db; - int n_databases = 0; - bool first_db_with_amcheck = true; - PGconn *db_conn = NULL; - bool db_skipped = false; - - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - - res_db = pgut_execute(backup_conn, - "SELECT datname, oid, dattablespace " - "FROM pg_database " - "WHERE datname NOT IN ('template0', 'template1')", - 0, NULL); - - n_databases = PQntuples(res_db); - - elog(INFO, "Start amchecking PostgreSQL instance"); - - /* For each database check indexes. In parallel. */ - for(i = 0; i < n_databases; i++) - { - int j; - - if (index_list != NULL) - { - free(index_list); - index_list = NULL; - } - - index_list = get_index_list(res_db, i, - first_db_with_amcheck, db_conn); - - if (index_list == NULL) - { - if (db_conn) - pgut_disconnect(db_conn); - db_skipped = true; - continue; - } - - first_db_with_amcheck = false; - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); - - for (j = 0; j < num_threads; j++) - { - backup_files_arg *arg = &(threads_args[j]); - - arg->from_root = instance_config.pgdata; - arg->to_root = database_path; - arg->files_list = NULL; - arg->index_list = index_list; - arg->prev_filelist = NULL; - arg->prev_start_lsn = InvalidXLogRecPtr; - arg->backup_conn = NULL; - arg->cancel_conn = NULL; - arg->thread_num = j + 1; - /* By default there are some error */ - arg->ret = 1; - } - - /* Run threads */ - for (j = 0; j < num_threads; j++) - { - backup_files_arg *arg = &(threads_args[j]); - elog(VERBOSE, "Start thread num: %i", j); - pthread_create(&threads[j], NULL, check_indexes, arg); - } - - /* Wait threads */ - for (j = 0; j < num_threads; j++) - { - pthread_join(threads[j], NULL); - if (threads_args[j].ret > 0) - check_isok = false; - } - - /* cleanup */ - pgut_disconnect(db_conn); - - if (interrupted) - break; - } - - /* Inform user about amcheck results */ - if (!check_isok || interrupted) - elog(ERROR, "Checkdb --amcheck failed"); - - if (db_skipped) - elog(ERROR, "Some databases were not amchecked"); - - elog(INFO, "Checkdb --amcheck executed successfully"); - - /* We cannot state that all indexes are ok - * without checking indexes in all databases - */ - if (check_isok && !interrupted && !db_skipped) - elog(INFO, "Indexes are valid"); - - /* cleanup */ - PQclear(res_db); -} - -/* Entry point of pg_probackup CHECKDB subcommand. */ -void -do_checkdb(bool need_amcheck) -{ - bool amcheck_only = false; - - if (skip_block_validation && !need_amcheck) - elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); - - if (skip_block_validation && need_amcheck) - amcheck_only = true; - - pgdata_basic_setup(amcheck_only); - - if (!skip_block_validation) - do_block_validation(); - - if (need_amcheck) - do_amcheck(); -} - /* * Common code for CHECKDB and BACKUP commands. * Ensure that we're able to connect to the instance @@ -843,44 +614,38 @@ do_checkdb(bool need_amcheck) * check remote PostgreSQL instance. * Also checking system ID in this case serves no purpose, because * all work is done by server. + * + * Returns established connection */ -static void -pgdata_basic_setup(bool amcheck_only) +PGconn * +pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) { - /* PGDATA is always required unless running checkdb in amcheck only mode */ - if (!instance_config.pgdata && !amcheck_only) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + PGconn *cur_conn; /* Create connection for PostgreSQL */ - backup_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); - pgut_atexit_push(backup_disconnect, NULL); + cur_conn = pgut_connect(conn_opt.pghost, conn_opt.pgport, + conn_opt.pgdatabase, + conn_opt.pguser); - current.primary_conninfo = pgut_get_conninfo_string(backup_conn); + current.primary_conninfo = pgut_get_conninfo_string(cur_conn); /* Confirm data block size and xlog block size are compatible */ - confirm_block_size("block_size", BLCKSZ); - confirm_block_size("wal_block_size", XLOG_BLCKSZ); + confirm_block_size(cur_conn, "block_size", BLCKSZ); + confirm_block_size(cur_conn, "wal_block_size", XLOG_BLCKSZ); + nodeInfo->block_size = BLCKSZ; + nodeInfo->wal_block_size = XLOG_BLCKSZ; - current.from_replica = pg_is_in_recovery(); + current.from_replica = pg_is_in_recovery(cur_conn); /* Confirm that this server version is supported */ - check_server_version(); + check_server_version(cur_conn); - if (pg_checksum_enable()) + if (pg_checksum_enable(cur_conn)) current.checksum_version = 1; else current.checksum_version = 0; - /* - * Ensure that backup directory was initialized for the same PostgreSQL - * instance we opened connection to. And that target backup database PGDATA - * belogns to the same instance. - */ - if (!amcheck_only) - check_system_identifiers(); + nodeInfo->checksum_version = current.checksum_version; if (current.checksum_version) elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " @@ -892,6 +657,11 @@ pgdata_basic_setup(bool amcheck_only) StrNCpy(current.server_version, server_version_str, sizeof(current.server_version)); + + StrNCpy(nodeInfo->server_version, server_version_str, + sizeof(nodeInfo->server_version)); + + return cur_conn; } /* @@ -900,11 +670,23 @@ pgdata_basic_setup(bool amcheck_only) int do_backup(time_t start_time, bool no_validate) { + PGconn *backup_conn = NULL; + + if (!instance_config.pgdata) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); /* * setup backup_conn, do some compatibility checks and * fill basic info about instance */ - pgdata_basic_setup(false); + backup_conn = pgdata_basic_setup(instance_config.conn_opt, + &(current.nodeInfo)); + /* + * Ensure that backup directory was initialized for the same PostgreSQL + * instance we opened connection to. And that target backup database PGDATA + * belogns to the same instance. + */ + check_system_identifiers(backup_conn, instance_config.pgdata); /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 @@ -917,10 +699,10 @@ do_backup(time_t start_time, bool no_validate) current.stream = stream_wal; - is_ptrack_support = pg_ptrack_support(); + is_ptrack_support = pg_ptrack_support(backup_conn); if (is_ptrack_support) { - is_ptrack_enable = pg_ptrack_enable(); + is_ptrack_enable = pg_ptrack_enable(backup_conn); } if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) @@ -935,18 +717,10 @@ do_backup(time_t start_time, bool no_validate) } if (current.from_replica && exclusive_backup) - { /* Check master connection options */ - if (instance_config.master_host == NULL) + if (instance_config.master_conn_opt.pghost == NULL) elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); - /* Create connection to master server */ - master_conn = pgut_connect(instance_config.master_host, - instance_config.master_port, - instance_config.master_db, - instance_config.master_user); - } - /* Start backup. Update backup status. */ current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; @@ -974,7 +748,7 @@ do_backup(time_t start_time, bool no_validate) pgut_atexit_push(backup_cleanup, NULL); /* backup data */ - do_backup_instance(); + do_backup_instance(backup_conn); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -1016,12 +790,12 @@ do_backup(time_t start_time, bool no_validate) * Confirm that this server version is supported */ static void -check_server_version(void) +check_server_version(PGconn *conn) { PGresult *res; /* confirm server version */ - server_version = PQserverVersion(backup_conn); + server_version = PQserverVersion(conn); if (server_version == 0) elog(ERROR, "Unknown server version %d", server_version); @@ -1044,7 +818,7 @@ check_server_version(void) "server version is %s, must be %s or higher for backup from replica", server_version_str, "9.6"); - res = pgut_execute_extended(backup_conn, "SELECT pgpro_edition()", + res = pgut_execute_extended(conn, "SELECT pgpro_edition()", 0, NULL, true, true); /* @@ -1088,14 +862,14 @@ check_server_version(void) * belogns to the same instance. * All system identifiers must be equal. */ -static void -check_system_identifiers(void) +void +check_system_identifiers(PGconn *conn, char *pgdata) { uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(instance_config.pgdata); - system_id_conn = get_remote_system_identifier(backup_conn); + system_id_pgdata = get_system_identifier(pgdata); + system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ if (current.backup_mode == BACKUP_MODE_INVALID) @@ -1124,15 +898,15 @@ check_system_identifiers(void) * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. */ static void -confirm_block_size(const char *name, int blcksz) +confirm_block_size(PGconn *conn, const char *name, int blcksz) { PGresult *res; char *endp; int block_size; - res = pgut_execute(backup_conn, "SELECT pg_catalog.current_setting($1)", 1, &name); + res = pgut_execute(conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) - elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(backup_conn)); + elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(conn)); block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); if ((endp && *endp) || block_size != blcksz) @@ -1147,7 +921,8 @@ confirm_block_size(const char *name, int blcksz) * Notify start of backup to PostgreSQL server. */ static void -pg_start_backup(const char *label, bool smooth, pgBackup *backup) +pg_start_backup(const char *label, bool smooth, pgBackup *backup, + PGconn *backup_conn, PGconn *pg_startbackup_conn) { PGresult *res; const char *params[2]; @@ -1158,10 +933,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) params[0] = label; /* For 9.5 replica we call pg_start_backup() on master */ - if (backup->from_replica && exclusive_backup) - conn = master_conn; - else - conn = backup_conn; + conn = pg_startbackup_conn; /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; @@ -1181,6 +953,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; + pgut_atexit_push(backup_stopbackup_callback, pg_startbackup_conn); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1214,7 +987,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup) * wait for start_lsn to be replayed by replica */ if (backup->from_replica && exclusive_backup) - wait_replica_wal_lsn(backup->start_lsn, true); + wait_replica_wal_lsn(backup->start_lsn, true, backup_conn); } /* @@ -1243,7 +1016,7 @@ pg_switch_wal(PGconn *conn) * TODO Maybe we should rather check ptrack_version()? */ static bool -pg_ptrack_support(void) +pg_ptrack_support(PGconn *backup_conn) { PGresult *res_db; @@ -1282,7 +1055,7 @@ pg_ptrack_support(void) /* Check if ptrack is enabled in target instance */ static bool -pg_ptrack_enable(void) +pg_ptrack_enable(PGconn *backup_conn) { PGresult *res_db; @@ -1299,11 +1072,11 @@ pg_ptrack_enable(void) /* Check if ptrack is enabled in target instance */ static bool -pg_checksum_enable(void) +pg_checksum_enable(PGconn *conn) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "SHOW data_checksums", 0, NULL); + res_db = pgut_execute(conn, "SHOW data_checksums", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) { @@ -1316,11 +1089,11 @@ pg_checksum_enable(void) /* Check if target instance is replica */ static bool -pg_is_in_recovery(void) +pg_is_in_recovery(PGconn *conn) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); + res_db = pgut_execute(conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); if (PQgetvalue(res_db, 0, 0)[0] == 't') { @@ -1333,7 +1106,7 @@ pg_is_in_recovery(void) /* Clear ptrack files in all databases of the instance we connected to */ static void -pg_ptrack_clear(void) +pg_ptrack_clear(PGconn *backup_conn) { PGresult *res_db, *res; @@ -1358,9 +1131,10 @@ pg_ptrack_clear(void) dbOid = atoi(PQgetvalue(res_db, i, 1)); tblspcOid = atoi(PQgetvalue(res_db, i, 2)); - tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, dbname, - instance_config.pguser); + instance_config.conn_opt.pguser); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", 0, NULL); PQclear(res); @@ -1380,7 +1154,7 @@ pg_ptrack_clear(void) } static bool -pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) { char *params[2]; char *dbname; @@ -1440,7 +1214,7 @@ pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid) */ static char * pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, - size_t *result_size) + size_t *result_size, PGconn *backup_conn) { PGconn *tmp_conn; PGresult *res_db, @@ -1476,9 +1250,9 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, return NULL; } - tmp_conn = pgut_connect(instance_config.pghost, instance_config.pgport, + tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, dbname, - instance_config.pguser); + instance_config.conn_opt.pguser); sprintf(params[0], "%i", tablespace_oid); sprintf(params[1], "%i", rel_filenode); res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", @@ -1707,7 +1481,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) * Wait for target 'lsn' on replica instance from master. */ static void -wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) +wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup, + PGconn *backup_conn) { uint32 try_count = 0; @@ -1775,7 +1550,7 @@ wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup) * Notify end of backup to PostgreSQL server. */ static void -pg_stop_backup(pgBackup *backup) +pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) { PGconn *conn; PGresult *res; @@ -1803,11 +1578,7 @@ pg_stop_backup(pgBackup *backup) if (!backup_in_progress) elog(ERROR, "backup is not in progress"); - /* For 9.5 replica we call pg_stop_backup() on master */ - if (current.from_replica && exclusive_backup) - conn = master_conn; - else - conn = backup_conn; + conn = pg_startbackup_conn; /* Remove annoying NOTICE messages generated by backend */ res = pgut_execute(conn, "SET client_min_messages = warning;", @@ -1903,6 +1674,9 @@ pg_stop_backup(pgBackup *backup) elog(ERROR, "Failed to send pg_stop_backup query"); } + /* After we have sent pg_stop_backup, we don't need this callback anymore */ + pgut_atexit_pop(backup_stopbackup_callback, pg_startbackup_conn); + /* * Wait for the result of pg_stop_backup(), but no longer than * archive_timeout seconds @@ -2152,7 +1926,7 @@ pg_stop_backup(pgBackup *backup) * Retreive checkpoint_timeout GUC value in seconds. */ static int -checkpoint_timeout(void) +checkpoint_timeout(PGconn *backup_conn) { PGresult *res; const char *val; @@ -2197,174 +1971,6 @@ backup_cleanup(bool fatal, void *userdata) current.status = BACKUP_STATUS_ERROR; write_backup(¤t); } - - /* - * If backup is in progress, notify stop of backup to PostgreSQL - */ - if (backup_in_progress) - { - elog(WARNING, "backup in progress, stop backup"); - pg_stop_backup(NULL); /* don't care stop_lsn on error case */ - } -} - -/* - * Disconnect backup connection during quit pg_probackup. - */ -static void -backup_disconnect(bool fatal, void *userdata) -{ - pgut_disconnect(backup_conn); - if (master_conn) - pgut_disconnect(master_conn); -} - -/* - * Check files in PGDATA. - * Read all files listed in backup_files_list. - * If the file is 'datafile' (regular relation's main fork), read it page by page, - * verify checksum and copy. - */ -static void * -check_files(void *arg) -{ - int i; - backup_files_arg *arguments = (backup_files_arg *) arg; - int n_backup_files_list = 0; - - if (arguments->files_list) - n_backup_files_list = parray_num(arguments->files_list); - - /* check a file */ - for (i = 0; i < n_backup_files_list; i++) - { - int ret; - struct stat buf; - pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - elog(VERBOSE, "Checking file: \"%s\" ", file->path); - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during checkdb"); - - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_backup_files_list, file->path); - - /* stat file to check its current state */ - ret = stat(file->path, &buf); - if (ret == -1) - { - if (errno == ENOENT) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - elog(LOG, "File \"%s\" is not found", file->path); - continue; - } - else - { - elog(ERROR, - "can't stat file to check \"%s\": %s", - file->path, strerror(errno)); - } - } - - /* No need to check directories */ - if (S_ISDIR(buf.st_mode)) - continue; - - if (S_ISREG(buf.st_mode)) - { - /* check only uncompressed by cfs datafiles */ - if (file->is_datafile && !file->is_cfs) - { - char to_path[MAXPGPATH]; - - join_path_components(to_path, arguments->to_root, - file->path + strlen(arguments->from_root) + 1); - - if (!check_data_file(arguments, file)) - arguments->ret = 2; /* corruption found */ - } - } - else - elog(WARNING, "unexpected file type %d", buf.st_mode); - } - - /* Ret values: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - if (arguments->ret == 1) - arguments->ret = 0; - - return NULL; -} - -/* Check indexes with amcheck */ -static void * -check_indexes(void *arg) -{ - int i; - backup_files_arg *arguments = (backup_files_arg *) arg; - int n_indexes = 0; - - if (arguments->index_list) - n_indexes = parray_num(arguments->index_list); - - for (i = 0; i < n_indexes; i++) - { - pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); - - if (!pg_atomic_test_set_flag(&ind->lock)) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Thread [%d]: interrupted during checkdb --amcheck", - arguments->thread_num); - - if (progress) - elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'", - arguments->thread_num, i + 1, n_indexes, - ind->amcheck_nspname, ind->name); - - if (arguments->backup_conn == NULL) - { - - arguments->backup_conn = pgut_connect(instance_config.pghost, - instance_config.pgport, - ind->dbname, - instance_config.pguser); - arguments->cancel_conn = PQgetCancel(arguments->backup_conn); - } - - /* remember that we have a failed check */ - if (!amcheck_one_index(arguments, ind)) - arguments->ret = 2; /* corruption found */ - } - - /* Close connection */ - if (arguments->backup_conn) - pgut_disconnect(arguments->backup_conn); - - /* Ret values: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - if (arguments->ret == 1) - arguments->ret = 0; - - return NULL; } /* @@ -2529,8 +2135,8 @@ backup_files(void *arg) } /* Close connection */ - if (arguments->backup_conn) - pgut_disconnect(arguments->backup_conn); + if (arguments->conn_arg.conn) + pgut_disconnect(arguments->conn_arg.conn); /* Data files transferring is successful */ arguments->ret = 0; @@ -2545,8 +2151,8 @@ backup_files(void *arg) * - set flags for database directories * - set flags for datafiles */ -static void -parse_backup_filelist_filenames(parray *files, const char *root) +void +parse_filelist_filenames(parray *files, const char *root) { size_t i = 0; Oid unlogged_file_reloid = 0; @@ -2731,7 +2337,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) * NOTE we rely on the fact that provided parray is sorted by file->path. */ static void -make_pagemap_from_ptrack(parray *files) +make_pagemap_from_ptrack(parray *files, PGconn *backup_conn) { size_t i; Oid dbOid_with_ptrack_init = 0; @@ -2763,7 +2369,7 @@ make_pagemap_from_ptrack(parray *files) * to avoid any possible specific errors. */ if ((file->tblspcOid == GLOBALTABLESPACE_OID) || - pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid)) + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid, backup_conn)) { dbOid_with_ptrack_init = file->dbOid; tblspcOid_with_ptrack_init = file->tblspcOid; @@ -2789,7 +2395,7 @@ make_pagemap_from_ptrack(parray *files) ptrack_nonparsed_size = 0; ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, - file->relOid, &ptrack_nonparsed_size); + file->relOid, &ptrack_nonparsed_size, backup_conn); } if (ptrack_nonparsed != NULL) @@ -2848,8 +2454,6 @@ make_pagemap_from_ptrack(parray *files) } } elog(LOG, "Pagemap compiled"); -// res = pgut_execute(backup_conn, "SET client_min_messages = warning;", 0, NULL, true); -// PQclear(pgut_execute(backup_conn, "CHECKPOINT;", 0, NULL, true)); } @@ -2865,7 +2469,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during backup"); + elog(ERROR, "Interrupted during backup stop_streaming"); /* we assume that we get called once at the end of each segment */ if (segment_finished) @@ -2893,14 +2497,11 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) } /* pg_stop_backup() was executed, wait for the completion of stream */ - if (stream_stop_timeout == 0) + if (stream_stop_begin == 0) { elog(INFO, "Wait for LSN %X/%X to be streamed", (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); - stream_stop_timeout = checkpoint_timeout(); - stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; - stream_stop_begin = time(NULL); } @@ -2922,20 +2523,12 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) static void * StreamLog(void *arg) { - XLogRecPtr startpos; - TimeLineID starttli; StreamThreadArg *stream_arg = (StreamThreadArg *) arg; - /* - * We must use startpos as start_lsn from start_backup - */ - startpos = current.start_lsn; - starttli = current.tli; - /* * Always start streaming at the beginning of a segment */ - startpos -= startpos % instance_config.xlog_seg_size; + stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; /* Initialize timeout */ stream_stop_timeout = 0; @@ -2959,7 +2552,8 @@ StreamLog(void *arg) * Start the replication */ elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), - (uint32) (startpos >> 32), (uint32) startpos, starttli); + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli); #if PG_VERSION_NUM >= 90600 { @@ -2967,8 +2561,8 @@ StreamLog(void *arg) MemSet(&ctl, 0, sizeof(ctl)); - ctl.startpos = startpos; - ctl.timeline = starttli; + ctl.startpos = stream_arg->startpos; + ctl.timeline = stream_arg->starttli; ctl.sysidentifier = NULL; #if PG_VERSION_NUM >= 100000 @@ -2998,14 +2592,14 @@ StreamLog(void *arg) #endif } #else - if(ReceiveXlogStream(stream_arg->conn, startpos, starttli, NULL, + if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, NULL, (char *) stream_arg->basedir, stop_streaming, standby_message_timeout, NULL, false, false) == false) elog(ERROR, "Problem in receivexlog"); #endif elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), - (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, starttli); + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); stream_arg->ret = 0; PQfinish(stream_arg->conn); @@ -3018,7 +2612,7 @@ StreamLog(void *arg) * Get lsn of the moment when ptrack was enabled the last time. */ static XLogRecPtr -get_last_ptrack_lsn(void) +get_last_ptrack_lsn(PGconn *backup_conn) { PGresult *res; @@ -3039,7 +2633,7 @@ get_last_ptrack_lsn(void) } char * -pg_ptrack_get_block(backup_files_arg *arguments, +pg_ptrack_get_block(ConnectionArgs *arguments, Oid dbOid, Oid tblsOid, Oid relOid, @@ -3064,19 +2658,19 @@ pg_ptrack_get_block(backup_files_arg *arguments, sprintf(params[2], "%i", relOid); sprintf(params[3], "%u", blknum); - if (arguments->backup_conn == NULL) + if (arguments->conn == NULL) { - arguments->backup_conn = pgut_connect(instance_config.pghost, - instance_config.pgport, - instance_config.pgdatabase, - instance_config.pguser); + arguments->conn = pgut_connect(instance_config.conn_opt.pghost, + instance_config.conn_opt.pgport, + instance_config.conn_opt.pgdatabase, + instance_config.conn_opt.pguser); } if (arguments->cancel_conn == NULL) - arguments->cancel_conn = PQgetCancel(arguments->backup_conn); + arguments->cancel_conn = PQgetCancel(arguments->conn); //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - res = pgut_execute_parallel(arguments->backup_conn, + res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", 4, (const char **)params, true, false, false); @@ -3109,9 +2703,8 @@ pg_ptrack_get_block(backup_files_arg *arguments, } static void -check_external_for_tablespaces(parray *external_list) +check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) { - PGconn *conn; PGresult *res; int i = 0; int j = 0; @@ -3120,8 +2713,7 @@ check_external_for_tablespaces(parray *external_list) "FROM pg_catalog.pg_tablespace " "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; - conn = backup_conn; - res = pgut_execute(conn, query, 0, NULL); + res = pgut_execute(backup_conn, query, 0, NULL); /* Check successfull execution of query */ if (!res) @@ -3174,172 +2766,3 @@ check_external_for_tablespaces(parray *external_list) } } } - -/* Get index list for given database */ -static parray* -get_index_list(PGresult *res_db, int db_number, - bool first_db_with_amcheck, PGconn *db_conn) -{ - PGresult *res; - char *nspname = NULL; - int i; - - dbname = PQgetvalue(res_db, db_number, 0); - - db_conn = pgut_connect(instance_config.pghost, instance_config.pgport, - dbname, - instance_config.pguser); - - res = pgut_execute(db_conn, "SELECT " - "extname, nspname, extversion " - "FROM pg_namespace n " - "JOIN pg_extension e " - "ON n.oid=e.extnamespace " - "WHERE e.extname IN ('amcheck', 'amcheck_next') " - "ORDER BY extversion DESC " - "LIMIT 1", - 0, NULL); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - elog(ERROR, "Cannot check if amcheck is installed in database %s: %s", - dbname, PQerrorMessage(db_conn)); - } - - if (PQntuples(res) < 1) - { - elog(WARNING, "Extension 'amcheck' or 'amcheck_next' are not installed in database %s", dbname); - return NULL; - } - - nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); - strcpy(nspname, PQgetvalue(res, 0, 1)); - - /* heapallindexed_is_supported is database specific */ - if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && - strcmp(PQgetvalue(res, 0, 2), "1") != 0) - heapallindexed_is_supported = true; - - elog(INFO, "Amchecking database '%s' using extension '%s' version %s from schema '%s'", - dbname, PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); - - if (!heapallindexed_is_supported && heapallindexed) - elog(WARNING, "Extension '%s' verion %s in schema '%s' do not support 'heapallindexed' option", - PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); - - /* - * In order to avoid duplicates, select global indexes - * (tablespace pg_global with oid 1664) only once. - * - * select only persistent btree indexes. - */ - if (first_db_with_amcheck) - { - - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " - "FROM pg_index idx " - "JOIN pg_class cls ON idx.indexrelid=cls.oid " - "JOIN pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't'", - 0, NULL); - } - else - { - - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " - "FROM pg_index idx " - "JOIN pg_class cls ON idx.indexrelid=cls.oid " - "JOIN pg_am am ON cls.relam=am.oid " - "LEFT JOIN pg_tablespace tbl " - "ON cls.reltablespace=tbl.oid " - "AND tbl.spcname <> 'pg_global' " - "WHERE am.amname='btree' AND cls.relpersistence != 't'", - 0, NULL); - } - - /* add info needed to check indexes into index_list */ - for (i = 0; i < PQntuples(res); i++) - { - pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); - char *name = NULL; - - ind->indexrelid = atoi(PQgetvalue(res, i, 0)); - name = PQgetvalue(res, i, 1); - ind->name = pgut_malloc(strlen(name) + 1); - strcpy(ind->name, name); /* enough buffer size guaranteed */ - - ind->dbname = pgut_malloc(strlen(dbname) + 1); - strcpy(ind->dbname, dbname); - - ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); - strcpy(ind->amcheck_nspname, nspname); - pg_atomic_clear_flag(&ind->lock); - - if (index_list == NULL) - index_list = parray_new(); - - parray_append(index_list, ind); - } - - PQclear(res); - - return index_list; -} - -/* check one index. Return true if everything is ok, false otherwise. */ -static bool -amcheck_one_index(backup_files_arg *arguments, - pg_indexEntry *ind) -{ - PGresult *res; - char *params[2]; - char *query = NULL; - - params[0] = palloc(64); - - /* first argument is index oid */ - sprintf(params[0], "%i", ind->indexrelid); - /* second argument is heapallindexed */ - params[1] = heapallindexed ? "true" : "false"; - - if (interrupted) - elog(ERROR, "Interrupted"); - - if (heapallindexed_is_supported) - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); - sprintf(query, "SELECT %s.bt_index_check($1, $2)", ind->amcheck_nspname); - - res = pgut_execute_parallel(arguments->backup_conn, - arguments->cancel_conn, - query, 2, (const char **)params, true, true, true); - } - else - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); - sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); - - res = pgut_execute_parallel(arguments->backup_conn, - arguments->cancel_conn, - query, 1, (const char **)params, true, true, true); - } - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING, "Thread [%d]. Amcheck failed for index: '%s.%s': %s", - arguments->thread_num, ind->amcheck_nspname, - ind->name, PQresultErrorMessage(res)); - - pfree(params[0]); - PQclear(res); - return false; - } - else - elog(LOG, "Thread [%d]. Amcheck succeeded for index: '%s.%s'", - arguments->thread_num, ind->amcheck_nspname, ind->name); - - pfree(params[0]); - PQclear(res); - return true; -} diff --git a/src/checkdb.c b/src/checkdb.c new file mode 100644 index 000000000..6b736bdc5 --- /dev/null +++ b/src/checkdb.c @@ -0,0 +1,709 @@ +/*------------------------------------------------------------------------- + * + * src/checkdb.c + * pg_probackup checkdb subcommand + * + * It allows to validate all data files located in PGDATA + * via block checksums matching and page header sanity checks. + * Optionally all indexes in all databases in PostgreSQL + * instance can be logically verified using extensions + * amcheck or amcheck_next. + * + * Portions Copyright (c) 2019-2019, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#include +#include +#include + +#include "utils/thread.h" +#include "utils/file.h" + + +typedef struct +{ + /* list of files to validate */ + parray *files_list; + /* if page checksums are enabled in this postgres instance? */ + uint32 checksum_version; + /* + * conn and cancel_conn + * to use in check_data_file + * to connect to postgres if we've failed to validate page + * and want to read it via buffer cache to ensure + */ + ConnectionArgs conn_arg; + /* number of thread for debugging */ + int thread_num; + /* + * Return value from the thread: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ + int ret; +} check_files_arg; + + +typedef struct +{ + /* list of indexes to amcheck */ + parray *index_list; + /* + * credentials to connect to postgres instance + * used for compatibility checks of blocksize, + * server version and so on + */ + ConnectionOptions conn_opt; + /* + * conn and cancel_conn + * to use in threads to connect to databases + */ + ConnectionArgs conn_arg; + /* number of thread for debugging */ + int thread_num; + /* + * Return value from the thread: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ + int ret; +} check_indexes_arg; + +typedef struct pg_indexEntry +{ + Oid indexrelid; + char *name; + bool heapallindexed_is_supported; + /* schema where amcheck extention is located */ + char *amcheck_nspname; + /* lock for synchronization of parallel threads */ + volatile pg_atomic_flag lock; +} pg_indexEntry; + +static void +pg_indexEntry_free(void *index) +{ + pg_indexEntry *index_ptr; + + if (index == NULL) + return; + + index_ptr = (pg_indexEntry *) index; + + if (index_ptr->name) + free(index_ptr->name); + if (index_ptr->amcheck_nspname) + free(index_ptr->amcheck_nspname); + + free(index_ptr); +} + + +static void *check_files(void *arg); +static void do_block_validation(char *pgdata, uint32 checksum_version); + +static void *check_indexes(void *arg); +static parray* get_index_list(const char *dbname, bool first_db_with_amcheck, + PGconn *db_conn); +static bool amcheck_one_index(check_indexes_arg *arguments, + pg_indexEntry *ind); +static void do_amcheck(ConnectionOptions conn_opt, PGconn *conn); + +/* + * Check files in PGDATA. + * Read all files listed in files_list. + * If the file is 'datafile' (regular relation's main fork), read it page by page, + * verify checksum and copy. + */ +static void * +check_files(void *arg) +{ + int i; + check_files_arg *arguments = (check_files_arg *) arg; + int n_files_list = 0; + + if (arguments->files_list) + n_files_list = parray_num(arguments->files_list); + + /* check a file */ + for (i = 0; i < n_files_list; i++) + { + int ret; + struct stat buf; + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + elog(VERBOSE, "Checking file: \"%s\" ", file->path); + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "interrupted during checkdb"); + + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_files_list, file->path); + + /* stat file to check its current state */ + ret = stat(file->path, &buf); + if (ret == -1) + { + if (errno == ENOENT) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + elog(LOG, "File \"%s\" is not found", file->path); + continue; + } + else + { + elog(ERROR, + "can't stat file to check \"%s\": %s", + file->path, strerror(errno)); + } + } + + /* No need to check directories */ + if (S_ISDIR(buf.st_mode)) + continue; + + if (S_ISREG(buf.st_mode)) + { + /* check only uncompressed by cfs datafiles */ + if (file->is_datafile && !file->is_cfs) + { + /* + * TODO deep inside check_data_file + * uses global variables to set connections. + * Need refactoring. + */ + if (!check_data_file(&(arguments->conn_arg), file, + arguments->checksum_version)) + arguments->ret = 2; /* corruption found */ + } + } + else + elog(WARNING, "unexpected file type %d", buf.st_mode); + } + + /* Ret values: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ + if (arguments->ret == 1) + arguments->ret = 0; + + return NULL; +} + +/* collect list of files and run threads to check files in the instance */ +static void +do_block_validation(char *pgdata, uint32 checksum_version) +{ + int i; + /* arrays with meta info for multi threaded check */ + pthread_t *threads; + check_files_arg *threads_args; + bool check_isok = true; + parray *files_list = NULL; + + /* initialize file list */ + files_list = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + dir_list_file(files_list, pgdata, + true, true, false, 0, FIO_DB_HOST); + + /* + * Sort pathname ascending. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + */ + parray_qsort(files_list, pgFileComparePath); + /* Extract information about files in pgdata parsing their names:*/ + parse_filelist_filenames(files_list, pgdata); + + /* setup threads */ + for (i = 0; i < parray_num(files_list); i++) + { + pgFile *file = (pgFile *) parray_get(files_list, i); + pg_atomic_init_flag(&file->lock); + } + + /* Sort by size for load balancing */ + parray_qsort(files_list, pgFileCompareSize); + + /* init thread args with own file lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (check_files_arg *) palloc(sizeof(check_files_arg)*num_threads); + + for (i = 0; i < num_threads; i++) + { + check_files_arg *arg = &(threads_args[i]); + + arg->files_list = files_list; + arg->checksum_version = checksum_version; + + arg->conn_arg.conn = NULL; + arg->conn_arg.cancel_conn = NULL; + + arg->thread_num = i + 1; + /* By default there is some error */ + arg->ret = 1; + } + + elog(INFO, "Start checking data files"); + + /* Run threads */ + for (i = 0; i < num_threads; i++) + { + check_files_arg *arg = &(threads_args[i]); + + elog(VERBOSE, "Start thread num: %i", i); + + pthread_create(&threads[i], NULL, check_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret > 0) + check_isok = false; + } + + /* cleanup */ + if (files_list) + { + parray_walk(files_list, pgFileFree); + parray_free(files_list); + files_list = NULL; + } + + if (check_isok) + elog(INFO, "Data files are valid"); + else + elog(ERROR, "Checkdb failed"); +} + +/* Check indexes with amcheck */ +static void * +check_indexes(void *arg) +{ + int i; + check_indexes_arg *arguments = (check_indexes_arg *) arg; + int n_indexes = 0; + + if (arguments->index_list) + n_indexes = parray_num(arguments->index_list); + + for (i = 0; i < n_indexes; i++) + { + pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); + + if (!pg_atomic_test_set_flag(&ind->lock)) + continue; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Thread [%d]: interrupted during checkdb --amcheck", + arguments->thread_num); + + if (progress) + elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'", + arguments->thread_num, i + 1, n_indexes, + ind->amcheck_nspname, ind->name); + + if (arguments->conn_arg.conn == NULL) + { + + arguments->conn_arg.conn = pgut_connect(arguments->conn_opt.pghost, + arguments->conn_opt.pgport, + arguments->conn_opt.pgdatabase, + arguments->conn_opt.pguser); + arguments->conn_arg.cancel_conn = PQgetCancel(arguments->conn_arg.conn); + } + + /* remember that we have a failed check */ + if (!amcheck_one_index(arguments, ind)) + arguments->ret = 2; /* corruption found */ + } + + /* Close connection. */ + if (arguments->conn_arg.conn) + pgut_disconnect(arguments->conn_arg.conn); + + /* Ret values: + * 0 everything is ok + * 1 thread errored during execution, e.g. interruption (default value) + * 2 corruption is definitely(!) found + */ + if (arguments->ret == 1) + arguments->ret = 0; + + return NULL; +} + +/* Get index list for given database */ +static parray* +get_index_list(const char *dbname, bool first_db_with_amcheck, + PGconn *db_conn) +{ + PGresult *res; + char *nspname = NULL; + int i; + bool heapallindexed_is_supported = false; + parray *index_list = NULL; + + res = pgut_execute(db_conn, "SELECT " + "extname, nspname, extversion " + "FROM pg_namespace n " + "JOIN pg_extension e " + "ON n.oid=e.extnamespace " + "WHERE e.extname IN ('amcheck', 'amcheck_next') " + "ORDER BY extversion DESC " + "LIMIT 1", + 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(ERROR, "Cannot check if amcheck is installed in database %s: %s", + dbname, PQerrorMessage(db_conn)); + } + + if (PQntuples(res) < 1) + { + elog(WARNING, "Extension 'amcheck' or 'amcheck_next' are " + "not installed in database %s", dbname); + return NULL; + } + + nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); + strcpy(nspname, PQgetvalue(res, 0, 1)); + + /* heapallindexed_is_supported is database specific */ + if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && + strcmp(PQgetvalue(res, 0, 2), "1") != 0) + heapallindexed_is_supported = true; + + elog(INFO, "Amchecking database '%s' using extension '%s' " + "version %s from schema '%s'", + dbname, PQgetvalue(res, 0, 0), + PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + + if (!heapallindexed_is_supported && heapallindexed) + elog(WARNING, "Extension '%s' verion %s in schema '%s'" + "do not support 'heapallindexed' option", + PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), + PQgetvalue(res, 0, 1)); + + /* + * In order to avoid duplicates, select global indexes + * (tablespace pg_global with oid 1664) only once. + * + * select only persistent btree indexes. + */ + if (first_db_with_amcheck) + { + + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " + "FROM pg_index idx " + "JOIN pg_class cls ON idx.indexrelid=cls.oid " + "JOIN pg_am am ON cls.relam=am.oid " + "WHERE am.amname='btree' AND cls.relpersistence != 't'", + 0, NULL); + } + else + { + + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " + "FROM pg_index idx " + "JOIN pg_class cls ON idx.indexrelid=cls.oid " + "JOIN pg_am am ON cls.relam=am.oid " + "LEFT JOIN pg_tablespace tbl " + "ON cls.reltablespace=tbl.oid " + "AND tbl.spcname <> 'pg_global' " + "WHERE am.amname='btree' AND cls.relpersistence != 't'", + 0, NULL); + } + + /* add info needed to check indexes into index_list */ + for (i = 0; i < PQntuples(res); i++) + { + pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); + char *name = NULL; + + ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + name = PQgetvalue(res, i, 1); + ind->name = pgut_malloc(strlen(name) + 1); + strcpy(ind->name, name); /* enough buffer size guaranteed */ + + ind->heapallindexed_is_supported = heapallindexed_is_supported; + ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); + strcpy(ind->amcheck_nspname, nspname); + pg_atomic_clear_flag(&ind->lock); + + if (index_list == NULL) + index_list = parray_new(); + + parray_append(index_list, ind); + } + + PQclear(res); + + return index_list; +} + +/* check one index. Return true if everything is ok, false otherwise. */ +static bool +amcheck_one_index(check_indexes_arg *arguments, + pg_indexEntry *ind) +{ + PGresult *res; + char *params[2]; + char *query = NULL; + + params[0] = palloc(64); + + /* first argument is index oid */ + sprintf(params[0], "%i", ind->indexrelid); + /* second argument is heapallindexed */ + params[1] = heapallindexed ? "true" : "false"; + + if (interrupted) + elog(ERROR, "Interrupted"); + + if (ind->heapallindexed_is_supported) + { + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); + sprintf(query, "SELECT %s.bt_index_check($1, $2)", ind->amcheck_nspname); + + res = pgut_execute_parallel(arguments->conn_arg.conn, + arguments->conn_arg.cancel_conn, + query, 2, (const char **)params, true, true, true); + } + else + { + query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); + sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); + + res = pgut_execute_parallel(arguments->conn_arg.conn, + arguments->conn_arg.cancel_conn, + query, 1, (const char **)params, true, true, true); + } + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING, "Thread [%d]. Amcheck failed in database '%s' for index: '%s.%s': %s", + arguments->thread_num, arguments->conn_opt.pgdatabase, + ind->amcheck_nspname, ind->name, PQresultErrorMessage(res)); + + pfree(params[0]); + pfree(query); + PQclear(res); + return false; + } + else + elog(LOG, "Thread [%d]. Amcheck succeeded in database '%s' for index: '%s.%s'", + arguments->thread_num, + arguments->conn_opt.pgdatabase, ind->amcheck_nspname, ind->name); + + pfree(params[0]); + pfree(query); + PQclear(res); + return true; +} + +/* + * Entry point of checkdb --amcheck. + * + * Connect to all databases in the cluster + * and get list of persistent indexes, + * then run parallel threads to perform bt_index_check() + * for all indexes from the list. + * + * If amcheck extension is not installed in the database, + * skip this database and report it via warning message. + */ +static void +do_amcheck(ConnectionOptions conn_opt, PGconn *conn) +{ + int i; + /* arrays with meta info for multi threaded amcheck */ + pthread_t *threads; + check_indexes_arg *threads_args; + bool check_isok = true; + PGresult *res_db; + int n_databases = 0; + bool first_db_with_amcheck = true; + bool db_skipped = false; + + elog(INFO, "Start amchecking PostgreSQL instance"); + + res_db = pgut_execute(conn, + "SELECT datname, oid, dattablespace " + "FROM pg_database " + "WHERE datname NOT IN ('template0', 'template1')", + 0, NULL); + + /* we don't need this connection anymore */ + if (conn) + pgut_disconnect(conn); + + n_databases = PQntuples(res_db); + + /* For each database check indexes. In parallel. */ + for(i = 0; i < n_databases; i++) + { + int j; + const char *dbname; + PGconn *db_conn = NULL; + parray *index_list = NULL; + + dbname = PQgetvalue(res_db, i, 0); + db_conn = pgut_connect(conn_opt.pghost, conn_opt.pgport, + dbname, conn_opt.pguser); + + index_list = get_index_list(dbname, first_db_with_amcheck, + db_conn); + + /* we don't need this connection anymore */ + if (db_conn) + pgut_disconnect(db_conn); + + if (index_list == NULL) + { + db_skipped = true; + continue; + } + + first_db_with_amcheck = false; + + /* init thread args with own index lists */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (check_indexes_arg *) palloc(sizeof(check_indexes_arg)*num_threads); + + for (j = 0; j < num_threads; j++) + { + check_indexes_arg *arg = &(threads_args[j]); + + arg->index_list = index_list; + arg->conn_arg.conn = NULL; + arg->conn_arg.cancel_conn = NULL; + + arg->conn_opt.pghost = conn_opt.pghost; + arg->conn_opt.pgport = conn_opt.pgport; + arg->conn_opt.pgdatabase = dbname; + arg->conn_opt.pguser = conn_opt.pguser; + + arg->thread_num = j + 1; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + for (j = 0; j < num_threads; j++) + { + check_indexes_arg *arg = &(threads_args[j]); + elog(VERBOSE, "Start thread num: %i", j); + pthread_create(&threads[j], NULL, check_indexes, arg); + } + + /* Wait threads */ + for (j = 0; j < num_threads; j++) + { + pthread_join(threads[j], NULL); + if (threads_args[j].ret > 0) + check_isok = false; + } + + if (check_isok) + elog(INFO, "Amcheck succeeded for database '%s'", dbname); + else + elog(WARNING, "Amcheck failed for database %s", dbname); + + parray_walk(index_list, pg_indexEntry_free); + parray_free(index_list); + + if (interrupted) + break; + } + + /* cleanup */ + PQclear(res_db); + + /* Inform user about amcheck results */ + if (interrupted) + elog(ERROR, "checkdb --amcheck is interrupted."); + + if (check_isok) + { + elog(INFO, "checkdb --amcheck finished successfully. " + "All checked indexes are valid."); + + if (db_skipped) + elog(ERROR, "Some databases were not amchecked."); + else + elog(INFO, "All databases were amchecked."); + } + else + elog(ERROR, "checkdb --amcheck finished with failure. " + "Not all checked indexes are valid. %s", + db_skipped?"Some databases were not amchecked.": + "All databases were amchecked."); +} + +/* Entry point of pg_probackup CHECKDB subcommand */ +void +do_checkdb(bool need_amcheck, + ConnectionOptions conn_opt, char *pgdata) +{ + PGNodeInfo nodeInfo; + PGconn *cur_conn; + + if (skip_block_validation && !need_amcheck) + elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); + + if (!skip_block_validation) + { + if (!pgdata) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + + /* get node info */ + cur_conn = pgdata_basic_setup(conn_opt, &nodeInfo); + + /* ensure that conn credentials and pgdata are consistent */ + check_system_identifiers(cur_conn, pgdata); + + /* + * we don't need this connection anymore. + * block validation can last long time, + * so we don't hold the connection open, + * rather open new connection for amcheck + */ + if (cur_conn) + pgut_disconnect(cur_conn); + + do_block_validation(pgdata, nodeInfo.checksum_version); + } + + if (need_amcheck) + { + cur_conn = pgdata_basic_setup(conn_opt, &nodeInfo); + do_amcheck(conn_opt, cur_conn); + } +} diff --git a/src/configure.c b/src/configure.c index 4726503e1..7c581f1e0 100644 --- a/src/configure.c +++ b/src/configure.c @@ -72,43 +72,43 @@ ConfigOption instance_options[] = /* Connection options */ { 's', 'd', "pgdatabase", - &instance_config.pgdatabase, SOURCE_CMD, 0, + &instance_config.conn_opt.pgdatabase, SOURCE_CMD, 0, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance_config.pghost, SOURCE_CMD, 0, + &instance_config.conn_opt.pghost, SOURCE_CMD, 0, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance_config.pgport, SOURCE_CMD, 0, + &instance_config.conn_opt.pgport, SOURCE_CMD, 0, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance_config.pguser, SOURCE_CMD, 0, + &instance_config.conn_opt.pguser, SOURCE_CMD, 0, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance_config.master_db, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, 0, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance_config.master_host, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pghost, SOURCE_CMD, 0, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance_config.master_port, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgport, SOURCE_CMD, 0, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance_config.master_user, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pguser, SOURCE_CMD, 0, OPTION_REPLICA_GROUP, 0, option_get_value }, { diff --git a/src/data.c b/src/data.c index f45effd07..5abc8190b 100644 --- a/src/data.c +++ b/src/data.c @@ -195,7 +195,8 @@ parse_page(Page page, XLogRecPtr *lsn) */ static int read_page_from_file(pgFile *file, BlockNumber blknum, - FILE *in, Page page, XLogRecPtr *page_lsn) + FILE *in, Page page, XLogRecPtr *page_lsn, + uint32 checksum_version) { off_t offset = blknum * BLCKSZ; ssize_t read_len = 0; @@ -251,7 +252,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, } /* Verify checksum */ - if (current.checksum_version) + if (checksum_version) { BlockNumber blkno = file->segno * RELSEG_SIZE + blknum; /* @@ -289,13 +290,14 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * PageIsCorrupted(-4) if the page check mismatch */ static int32 -prepare_page(backup_files_arg *arguments, +prepare_page(ConnectionArgs *arguments, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, FILE *in, BlockNumber *n_skipped, BackupMode backup_mode, Page page, - bool strict) + bool strict, + uint32 checksum_version) { XLogRecPtr page_lsn = 0; int try_again = 100; @@ -316,7 +318,8 @@ prepare_page(backup_files_arg *arguments, { while(!page_is_valid && try_again) { - int result = read_page_from_file(file, blknum, in, page, &page_lsn); + int result = read_page_from_file(file, blknum, in, page, + &page_lsn, checksum_version); try_again--; if (result == 0) @@ -335,7 +338,7 @@ prepare_page(backup_files_arg *arguments, * If ptrack support is available use it to get invalid block * instead of rereading it 99 times */ - //elog(WARNING, "Checksum_Version: %i", current.checksum_version ? 1 : 0); + //elog(WARNING, "Checksum_Version: %i", checksum_version ? 1 : 0); if (result == -1 && is_ptrack_support && strict) { @@ -399,7 +402,7 @@ prepare_page(backup_files_arg *arguments, */ memcpy(page, ptrack_page, BLCKSZ); free(ptrack_page); - if (current.checksum_version) + if (checksum_version) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* get lsn from page, provided by pg_ptrack_get_block() */ @@ -631,9 +634,9 @@ backup_data_file(backup_files_arg* arguments, RetryUsingPtrack: for (blknum = 0; blknum < nblocks; blknum++) { - page_state = prepare_page(arguments, file, prev_backup_start_lsn, + page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true); + backup_mode, curr_page, true, current.checksum_version); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -655,9 +658,9 @@ backup_data_file(backup_files_arg* arguments, iter = datapagemap_iterate(&file->pagemap); while (datapagemap_next(iter, &blknum)) { - page_state = prepare_page(arguments, file, prev_backup_start_lsn, + page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true); + backup_mode, curr_page, true, current.checksum_version); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -1189,8 +1192,8 @@ validate_one_page(Page page, pgFile *file, * also returns true if the file was not found */ bool -check_data_file(backup_files_arg* arguments, - pgFile *file) +check_data_file(ConnectionArgs *arguments, + pgFile *file, uint32 checksum_version) { FILE *in; BlockNumber blknum = 0; @@ -1235,7 +1238,7 @@ check_data_file(backup_files_arg* arguments, { page_state = prepare_page(arguments, file, InvalidXLogRecPtr, blknum, nblocks, in, &n_blocks_skipped, - BACKUP_MODE_FULL, curr_page, false); + BACKUP_MODE_FULL, curr_page, false, checksum_version); if (page_state == PageIsTruncated) break; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5da9afca5..2cf6e0edd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -418,7 +418,10 @@ main(int argc, char *argv[]) elog(ERROR, "-B, --backup-path must be an absolute path"); - /* Option --instance is required for all commands except init and show */ + /* + * Option --instance is required for all commands except + * init, show, checkdb and validate + */ if (instance_name == NULL) { if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && @@ -553,16 +556,16 @@ main(int argc, char *argv[]) elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); } - if (!instance_config.pghost && instance_config.remote.host) - instance_config.pghost = instance_config.remote.host; + if (!instance_config.conn_opt.pghost && instance_config.remote.host) + instance_config.conn_opt.pghost = instance_config.remote.host; /* Setup stream options. They are used in streamutil.c. */ - if (instance_config.pghost != NULL) - dbhost = pstrdup(instance_config.pghost); - if (instance_config.pgport != NULL) - dbport = pstrdup(instance_config.pgport); - if (instance_config.pguser != NULL) - dbuser = pstrdup(instance_config.pguser); + if (instance_config.conn_opt.pghost != NULL) + dbhost = pstrdup(instance_config.conn_opt.pghost); + if (instance_config.conn_opt.pgport != NULL) + dbport = pstrdup(instance_config.conn_opt.pgport); + if (instance_config.conn_opt.pguser != NULL) + dbuser = pstrdup(instance_config.conn_opt.pguser); /* setup exclusion list for file search */ if (!backup_logs) @@ -663,7 +666,8 @@ main(int argc, char *argv[]) do_set_config(false); break; case CHECKDB_CMD: - do_checkdb(need_amcheck); + do_checkdb(need_amcheck, + instance_config.conn_opt, instance_config.pgdata); break; case NO_CMD: /* Should not happen */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 47d42af74..485c9f72e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -148,15 +148,6 @@ typedef struct pgFile * i.e. datafiles without _ptrack */ } pgFile; -typedef struct pg_indexEntry -{ - Oid indexrelid; - char *name; - char *dbname; - char *amcheck_nspname; /* schema where amcheck extention is located */ - volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ -} pg_indexEntry; - /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ @@ -199,6 +190,21 @@ typedef enum ShowFormat #define PROGRAM_VERSION "2.1.3" #define AGENT_PROTOCOL_VERSION 20103 + +typedef struct ConnectionOptions +{ + const char *pgdatabase; + const char *pghost; + const char *pgport; + const char *pguser; +} ConnectionOptions; + +typedef struct ConnectionArgs +{ + PGconn *conn; + PGcancel *cancel_conn; +} ConnectionArgs; + /* * An instance configuration. It can be stored in a configuration file or passed * from command line. @@ -210,15 +216,10 @@ typedef struct InstanceConfig char *pgdata; char *external_dir_str; - const char *pgdatabase; - const char *pghost; - const char *pgport; - const char *pguser; - const char *master_host; - const char *master_port; - const char *master_db; - const char *master_user; + ConnectionOptions conn_opt; + ConnectionOptions master_conn_opt; + uint32 replica_timeout; /* Wait timeout for WAL segment archiving */ @@ -241,6 +242,16 @@ typedef struct InstanceConfig extern ConfigOption instance_options[]; extern InstanceConfig instance_config; +typedef struct PGNodeInfo +{ + uint32 block_size; + uint32 wal_block_size; + uint32 checksum_version; + + char program_version[100]; + char server_version[100]; +} PGNodeInfo; + typedef struct pgBackup pgBackup; /* Information about single backup stored in backup.conf */ @@ -278,10 +289,10 @@ struct pgBackup int compress_level; /* Fields needed for compatibility check */ + PGNodeInfo nodeInfo; uint32 block_size; uint32 wal_block_size; uint32 checksum_version; - char program_version[100]; char server_version[100]; @@ -330,9 +341,7 @@ typedef struct parray *external_dirs; XLogRecPtr prev_start_lsn; - PGconn *backup_conn; - PGcancel *cancel_conn; - parray *index_list; + ConnectionArgs conn_arg; int thread_num; /* @@ -342,6 +351,7 @@ typedef struct int ret; } backup_files_arg; + /* * When copying datafiles to backup we validate and compress them block * by block. Thus special header is required for each data block. @@ -462,13 +472,14 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, bool no_validate); -extern void do_checkdb(bool need_amcheck); +extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, + char *pgdata); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern char *pg_ptrack_get_block(backup_files_arg *arguments, +extern char *pg_ptrack_get_block(ConnectionArgs *arguments, Oid dbOid, Oid tblsOid, Oid relOid, BlockNumber blknum, size_t *result_size); @@ -625,8 +636,7 @@ extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); /* in data.c */ -extern bool check_data_file(backup_files_arg* arguments, - pgFile *file); +extern bool check_data_file(ConnectionArgs* arguments, pgFile* file, uint32 checksum_version); extern bool backup_data_file(backup_files_arg* arguments, const char *to_path, pgFile *file, XLogRecPtr prev_backup_start_lsn, @@ -685,4 +695,10 @@ extern bool parse_page(Page page, XLogRecPtr *lsn); int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); + +extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); +extern void check_system_identifiers(PGconn *conn, char *pgdata); +extern void parse_filelist_filenames(parray *files, const char *root); + + #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index f51980f0c..49eae0486 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -228,7 +228,10 @@ pgut_connect(const char *host, const char *port, dbname, username, password); if (PQstatus(conn) == CONNECTION_OK) + { + pgut_atexit_push(pgut_disconnect_callback, conn); return conn; + } if (conn && PQconnectionNeedsPassword(conn) && prompt_password) { @@ -354,6 +357,7 @@ pgut_disconnect(PGconn *conn) { if (conn) PQfinish(conn); + pgut_atexit_pop(pgut_disconnect_callback, conn); } @@ -781,6 +785,14 @@ struct pgut_atexit_item static pgut_atexit_item *pgut_atexit_stack = NULL; +void +pgut_disconnect_callback(bool fatal, void *userdata) +{ + PGconn *conn = (PGconn *) userdata; + if (conn) + pgut_disconnect(conn); +} + void pgut_atexit_push(pgut_atexit_callback callback, void *userdata) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 588df7f9f..d196aad3d 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -43,6 +43,7 @@ extern PGconn *pgut_connect_replication(const char *host, const char *port, const char *dbname, const char *username); extern void pgut_disconnect(PGconn *conn); +extern void pgut_disconnect_callback(bool fatal, void *userdata); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, const char **params); extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, diff --git a/tests/checkdb.py b/tests/checkdb.py index 4db231a18..704b5ac24 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -75,10 +75,10 @@ def test_checkdb_amcheck_only_sanity(self): '-d', 'postgres', '-p', str(node.port)]) self.assertIn( - 'INFO: Checkdb --amcheck executed successfully', + 'INFO: checkdb --amcheck finished successfully', output) self.assertIn( - 'INFO: Indexes are valid', + 'All checked indexes are valid', output) # logging to file sanity @@ -143,7 +143,7 @@ def test_checkdb_amcheck_only_sanity(self): with open(log_file_path) as f: log_file_content = f.read() self.assertIn( - 'INFO: Checkdb --amcheck executed successfully', + 'INFO: checkdb --amcheck finished successfully', log_file_content) self.assertIn( 'VERBOSE: (query)', @@ -167,7 +167,7 @@ def test_checkdb_amcheck_only_sanity(self): with open(log_file_path) as f: log_file_content = f.read() self.assertIn( - 'INFO: Checkdb --amcheck executed successfully', + 'INFO: checkdb --amcheck finished successfully', log_file_content) self.assertIn( 'VERBOSE: (query)', @@ -199,7 +199,7 @@ def test_checkdb_amcheck_only_sanity(self): with open(log_file_path) as f: log_file_content = f.read() self.assertIn( - 'ERROR: Checkdb --amcheck failed', + 'ERROR: checkdb --amcheck finished with failure', log_file_content) self.assertIn( "WARNING: Thread [1]. Amcheck failed in database 'postgres' " @@ -323,7 +323,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Checkdb --amcheck failed", + "ERROR: checkdb --amcheck finished with failure", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -344,7 +344,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): log_file_content) self.assertIn( - "ERROR: Checkdb --amcheck failed", + "ERROR: checkdb --amcheck finished with failure", log_file_content) # Clean after yourself From b4442c57625286f90d5ec6c0cee02f5de42f8d70 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 1 Jun 2019 12:07:09 +0300 Subject: [PATCH 0720/2107] tests: added external.ExternalTest.test_external_validation --- tests/external.py | 96 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/tests/external.py b/tests/external.py index 4673dec92..9d5c77ee4 100644 --- a/tests/external.py +++ b/tests/external.py @@ -565,7 +565,7 @@ def test_external_backward_compatibility_merge_2(self): """ take backup with old binary without external dirs support take delta backup with new binary and 2 external directories - merge delta backup ajd restore it + merge delta backup and restore it """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -654,7 +654,7 @@ def test_external_backward_compatibility_merge_2(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - # Merge chain chain with new binary + # Merge chain using new binary self.merge_backup(backup_dir, 'node', backup_id=backup_id) # Restore merged backup @@ -663,15 +663,19 @@ def test_external_backward_compatibility_merge_2(self): node_restored.cleanup() - external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + external_dir1_new = self.get_tblspace_path( + node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path( + node_restored, 'external_dir2') self.restore_node( backup_dir, 'node', node_restored, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node_restored.base_dir, exclude_dirs=['logs']) @@ -699,7 +703,7 @@ def test_external_merge(self): node.pgbench_init(scale=3) - # FULL backup with old binary without external dirs support + # take temp FULL backup tmp_id = self.backup_node( backup_dir, 'node', node, options=["-j", "4", "--stream"]) @@ -753,8 +757,10 @@ def test_external_merge(self): backup_dir, 'node', node, options=[ "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -2459,3 +2465,75 @@ def test_smart_restore_externals(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_external_validation(self): + """ + make node, create database, + take full backup with external directory, + corrupt external file in backup, + run validate which should fail + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take temp FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', '-E', "{0}".format(external_dir)]) + + # Corrupt file + file = os.path.join( + backup_dir, 'backups', 'node', full_id, + 'external_directories', 'externaldir1', 'postgresql.auto.conf') + + with open(file, "r+b", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + try: + self.validate_pb(backup_dir) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because file in external dir is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Invalid CRC of backup file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 513bc487c25cf5e8212af69ef01bd5cf04103d8d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 1 Jun 2019 15:00:32 +0300 Subject: [PATCH 0721/2107] tests: remove psycopg2 --- tests/Readme.md | 1 - tests/delta.py | 46 ++++++++++++++------------------- tests/helpers/ptrack_helpers.py | 40 ++++++++++------------------ tests/ptrack.py | 29 ++++++++++----------- 4 files changed, 46 insertions(+), 70 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 2cfb35f4a..c0d9c0248 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -32,7 +32,6 @@ Run suit of basic simple tests: Usage: pip install testgres - pip install psycopg2 export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` diff --git a/tests/delta.py b/tests/delta.py index c4343daf6..a08d62d12 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -5,6 +5,7 @@ from testgres import QueryException import subprocess import time +from threading import Thread module_name = 'delta' @@ -522,48 +523,40 @@ def test_delta_vacuum_full(self): " as id from generate_series(0,1000000) i" ) - # create async connection - conn = self.get_async_connect(port=node.port) + pg_connect = node.connect("postgres", autocommit=True) - self.wait(conn) - - acurs = conn.cursor() - acurs.execute("select pg_backend_pid()") - - self.wait(conn) - pid = acurs.fetchall()[0][0] - print(pid) - - gdb = self.gdb_attach(pid) + gdb = self.gdb_attach(pg_connect.pid) gdb.set_breakpoint('reform_and_rewrite_tuple') gdb.continue_execution_until_running() - acurs.execute("VACUUM FULL t_heap") + process = Thread( + target=pg_connect.execute, args=["VACUUM FULL t_heap"]) + process.start() - if gdb.stopped_in_breakpoint(): - gdb.continue_execution_until_break(20) + while not gdb.stopped_in_breakpoint: + sleep(1) - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream'] - ) + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--stream'] - ) + backup_type='delta', options=['--stream']) + if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + gdb.remove_all_breakpoints() + gdb._execute('detach') + process.join() + old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') self.restore_node( backup_dir, 'node', node_restored, options=["-j", "4", "-T", "{0}={1}".format( - old_tablespace, new_tablespace)] - ) + old_tablespace, new_tablespace)]) # Physical comparison if self.paranoia: @@ -1125,7 +1118,6 @@ def test_delta_corruption_heal_via_ptrack_1(self): backup_type="delta", options=["-j", "4", "--stream", '--log-level-file=verbose']) - # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() @@ -1254,9 +1246,9 @@ def test_delta_nullified_heap_page_backup(self): backup_dir, 'node', node) # Nullify some block in PostgreSQL - file = os.path.join(node.data_dir, file_path).replace("\\","/") + file = os.path.join(node.data_dir, file_path).replace("\\", "/") if os.name == 'nt': - file = file.replace("\\","/") + file = file.replace("\\", "/") with open(file, 'r+b', 0) as f: f.seek(8192) @@ -1281,7 +1273,7 @@ def test_delta_nullified_heap_page_backup(self): content) self.assertNotIn( "Skipping blknum: 1 in file: {0}".format(file), - content) + content) # Restore DELTA backup node_restored = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 86f6e27ce..eb201488f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -9,7 +9,6 @@ import re import getpass import select -import psycopg2 from time import sleep import re import json @@ -1408,31 +1407,6 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): fail = True self.assertFalse(fail, error_message) - def get_async_connect(self, database=None, host=None, port=5432): - if not database: - database = 'postgres' - if not host: - host = '127.0.0.1' - - return psycopg2.connect( - database='postgres', - host='127.0.0.1', - port=port, - async_=True - ) - - def wait(self, connection): - while True: - state = connection.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [connection.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([connection.fileno()], [], []) - else: - raise psycopg2.OperationalError('poll() returned %s' % state) - def gdb_attach(self, pid): return GDBobj([str(pid)], self.verbose, attach=True) @@ -1540,7 +1514,7 @@ def remove_all_breakpoints(self): return raise GdbException( - 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) ) def run_until_break(self): @@ -1632,6 +1606,18 @@ def _execute(self, cmd, running=True): self.proc.stdin.flush() self.proc.stdin.write(cmd + '\n') self.proc.stdin.flush() + sleep(1) + + # look for command we just send + while True: + line = self.proc.stdout.readline() + if self.verbose: + print(repr(line)) + + if cmd not in line: + continue + else: + break while True: line = self.proc.stdout.readline() diff --git a/tests/ptrack.py b/tests/ptrack.py index 3387a8b4d..5d6d567fe 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -7,6 +7,7 @@ import shutil import sys import time +from threading import Thread module_name = 'ptrack' @@ -213,27 +214,21 @@ def test_ptrack_vacuum_full(self): " as id from generate_series(0,1000000) i" ) - # create async connection - conn = self.get_async_connect(port=node.port) + pg_connect = node.connect("postgres", autocommit=True) - self.wait(conn) - - acurs = conn.cursor() - acurs.execute("select pg_backend_pid()") - - self.wait(conn) - pid = acurs.fetchall()[0][0] - print(pid) - - gdb = self.gdb_attach(pid) + gdb = self.gdb_attach(pg_connect.pid) gdb.set_breakpoint('reform_and_rewrite_tuple') gdb.continue_execution_until_running() - acurs.execute("VACUUM FULL t_heap") + process = Thread( + target=pg_connect.execute, args=["VACUUM FULL t_heap"]) + process.start() - if gdb.stopped_in_breakpoint(): - gdb.continue_execution_until_break(20) + while not gdb.stopped_in_breakpoint: + sleep(1) + + gdb.continue_execution_until_break(20) self.backup_node( backup_dir, 'node', node, backup_type='ptrack') @@ -244,6 +239,10 @@ def test_ptrack_vacuum_full(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + gdb.remove_all_breakpoints() + gdb._execute('detach') + process.join() + old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') From bcb3a047578e653931dba087220d1694465e064a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 4 Jun 2019 13:42:09 +0300 Subject: [PATCH 0722/2107] bugfix: enforce permission mask from backup_content.control when copying non-data file --- src/data.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index 5abc8190b..22f0e7b4b 100644 --- a/src/data.c +++ b/src/data.c @@ -955,7 +955,6 @@ copy_file(fio_location from_location, const char *to_root, size_t read_len = 0; int errno_tmp; char buf[BLCKSZ]; - struct stat st; pg_crc32 crc; INIT_FILE_CRC32(true, crc); @@ -999,15 +998,6 @@ copy_file(fio_location from_location, const char *to_root, to_path, strerror(errno_tmp)); } - /* stat source file to change mode of destination file */ - if (fio_ffstat(in, &st) == -1) - { - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot stat \"%s\": %s", file->path, - strerror(errno)); - } - /* copy content and calc CRC */ for (;;) { @@ -1064,7 +1054,7 @@ copy_file(fio_location from_location, const char *to_root, file->crc = crc; /* update file permission */ - if (fio_chmod(to_path, st.st_mode, to_location) == -1) + if (fio_chmod(to_path, file->mode, to_location) == -1) { errno_tmp = errno; fio_fclose(in); From c92e10d52bb2f934a1c16d4bfeaad279c4145db4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 4 Jun 2019 21:11:12 +0300 Subject: [PATCH 0723/2107] tests: added validate.ValidateTest.test_validate_corrupt_tablespace_map --- tests/validate.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index c6dc2b746..4b5f26ea2 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3391,6 +3391,46 @@ def test_validation_after_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_corrupt_tablespace_map(self): + """ + Check that corruption in tablespace_map is detected + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'external_dir') + + node.safe_psql( + 'postgres', + 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'tablespace_map') + + # Corrupt tablespace_map file in FULL backup + with open(tablespace_map, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + # validate empty backup list # page from future during validate # page from future during backup From 2050cc03501bd02e7fa606313a87dec7593f5e8b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 5 Jun 2019 01:42:52 +0300 Subject: [PATCH 0724/2107] tests: fix module pgpro2068 --- tests/pgpro2068.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 6004483ca..0def742a4 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -6,6 +6,7 @@ from time import sleep import shutil import signal +from testgres import ProcessType module_name = '2068' @@ -81,25 +82,14 @@ def test_minrecpoint_on_replica(self): stderr=subprocess.STDOUT, options=["-c", "4", "-j 4", "-T", "100"]) - # get pids of background workers - startup_pid = replica.safe_psql( - 'postgres', - "select pid from pg_stat_activity " - "where backend_type = 'startup'").rstrip() - - checkpointer_pid = replica.safe_psql( - 'postgres', - "select pid from pg_stat_activity " - "where backend_type = 'checkpointer'").rstrip() - - bgwriter_pid = replica.safe_psql( - 'postgres', - "select pid from pg_stat_activity " - "where backend_type = 'background writer'").rstrip() - # wait for shared buffer on replica to be filled with dirty data sleep(10) + # get pids of replica background workers + startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] + checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] + bgwriter_pid = replica.auxiliary_pids[ProcessType.BackgroundWriter][0] + # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) gdb_checkpointer._execute('handle SIGINT noprint nostop pass') @@ -138,7 +128,29 @@ def test_minrecpoint_on_replica(self): 'recovery.conf', "recovery_target_action = 'promote'") replica.slow_start() - script = ''' + if self.get_version(node) < 100000: + script = ''' +DO +$$ +relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") +current_xlog_lsn = plpy.execute("select pg_last_xlog_replay_location() as lsn")[0]['lsn'] +plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) +found_corruption = False +for relation in relations: + pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) + + if pages_from_future.nrows() == 0: + continue + + for page in pages_from_future: + plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) + found_corruption = True +if found_corruption: + plpy.error('Found Corruption') +$$ LANGUAGE plpythonu; +''' + else: + script = ''' DO $$ relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") From 8641332a69339520091e9ae5fb9cbe21fdbcafea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 5 Jun 2019 02:28:01 +0300 Subject: [PATCH 0725/2107] [Issue #79] DB-level partial restore, initial commit --- src/backup.c | 66 +++++++++++++++++- src/data.c | 33 +++++++++ src/dir.c | 135 +++++++++++++++++++++++++++++++++++- src/pg_probackup.c | 71 ++++++++++++++++++- src/pg_probackup.h | 20 +++++- src/restore.c | 167 +++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 479 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index d6cd988d8..72521685b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -111,6 +111,7 @@ static void *StreamLog(void *arg); static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); +static parray *get_database_map(PGconn *pg_startbackup_conn); /* Ptrack functions */ static void pg_ptrack_clear(PGconn *backup_conn); @@ -169,6 +170,7 @@ do_backup_instance(PGconn *backup_conn) parray *prev_backup_filelist = NULL; parray *backup_list = NULL; parray *external_dirs = NULL; + parray *database_map = NULL; pgFile *pg_control = NULL; PGconn *master_conn = NULL; @@ -260,7 +262,10 @@ do_backup_instance(PGconn *backup_conn) pg_start_backup(label, smooth_checkpoint, ¤t, backup_conn, pg_startbackup_conn); - /* For incremental backup check that start_lsn is not from the past */ + /* For incremental backup check that start_lsn is not from the past + * Though it will not save us if PostgreSQL instance is actually + * restored STREAM backup. + */ if (current.backup_mode != BACKUP_MODE_FULL && prev_backup->start_lsn > current.start_lsn) elog(ERROR, "Current START LSN %X/%X is lower than START LSN %X/%X of previous backup %s. " @@ -336,6 +341,9 @@ do_backup_instance(PGconn *backup_conn) dir_list_file(backup_files_list, instance_config.pgdata, true, true, false, 0, FIO_DB_HOST); + /* create database_map used for partial restore */ + database_map = get_database_map(pg_startbackup_conn); + /* * Append to backup list all files and directories * from external directory option @@ -502,6 +510,15 @@ do_backup_instance(PGconn *backup_conn) } } + /* write database map to file and add it to control file */ + if (database_map) + { + write_database_map(¤t, database_map, backup_files_list); + /* we don`t need it anymore */ + parray_walk(database_map, db_map_entry_free); + parray_free(database_map); + } + /* clean previous backup file list */ if (prev_backup_filelist) { @@ -1053,6 +1070,53 @@ pg_ptrack_support(PGconn *backup_conn) return true; } +/* + * Create 'datname to Oid' map + */ +parray * +get_database_map(PGconn *conn) +{ + PGresult *res; + parray *database_map = NULL; + int i; + + res = pgut_execute(conn, + "SELECT oid, datname FROM pg_catalog.pg_database " + "WHERE datname NOT IN ('template1', 'template0')", + 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(WARNING, "Failed to get database map: %s", + PQerrorMessage(conn)); + + return NULL; + } + + /* Construct database map */ + for (i = 0; i < PQntuples(res); i++) + { + char *datname = NULL; + db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); + + /* get Oid */ + db_entry->dbOid = atoi(PQgetvalue(res, i, 0)); + + /* get datname */ + datname = PQgetvalue(res, i, 1); + db_entry->datname = pgut_malloc(strlen(datname) + 1); + strcpy(db_entry->datname, datname); + + if (database_map == NULL) + database_map = parray_new(); + + parray_append(database_map, db_entry); + } + + return database_map; +} + /* Check if ptrack is enabled in target instance */ static bool pg_ptrack_enable(PGconn *backup_conn) diff --git a/src/data.c b/src/data.c index 22f0e7b4b..4a63a261d 100644 --- a/src/data.c +++ b/src/data.c @@ -1071,6 +1071,39 @@ copy_file(fio_location from_location, const char *to_root, return true; } +/* + * Create empty file, used for partial restore + */ +bool +create_empty_file(fio_location from_location, const char *to_root, + fio_location to_location, pgFile *file) +{ + char to_path[MAXPGPATH]; + FILE *out; + + /* open backup file for write */ + join_path_components(to_path, to_root, file->rel_path); + out = fio_fopen(to_path, PG_BINARY_W, to_location); + if (out == NULL) + { + elog(ERROR, "cannot open destination file \"%s\": %s", + to_path, strerror(errno)); + } + + /* update file permission */ + if (fio_chmod(to_path, file->mode, to_location) == -1) + { + fio_fclose(out); + elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + strerror(errno)); + } + + if (fio_fclose(out)) + elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + + return true; +} + /* * Validate given page. * diff --git a/src/dir.c b/src/dir.c index 426d16963..60f0ea1c4 100644 --- a/src/dir.c +++ b/src/dir.c @@ -441,6 +441,15 @@ BlackListCompare(const void *str1, const void *str2) return strcmp(*(char **) str1, *(char **) str2); } +void +db_map_entry_free(void *map) +{ + db_map_entry *m = (db_map_entry *) map; + + free(m->datname); + free(map); +} + /* * List files, symbolic links and directories in the directory "root" and add * pgFile objects to "files". We add "root" to "files" if add_root is true. @@ -1299,10 +1308,12 @@ print_file_list(FILE *out, const parray *files, const char *root, fio_fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"", + "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " + "\"dbOid\":\"%u\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, - deparse_compress_alg(file->compress_alg), file->external_dir_num); + deparse_compress_alg(file->compress_alg), file->external_dir_num, + file->dbOid); if (file->is_datafile) fio_fprintf(out, ",\"segno\":\"%d\"", file->segno); @@ -1491,7 +1502,8 @@ dir_read_file_list(const char *root, const char *external_prefix, external_dir_num, crc, segno, - n_blocks; + n_blocks, + dbOid; /* used for partial restore */ pgFile *file; get_control_value(buf, "path", path, NULL, true); @@ -1502,6 +1514,7 @@ dir_read_file_list(const char *root, const char *external_prefix, get_control_value(buf, "crc", NULL, &crc, true); get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); + get_control_value(buf, "dbOid", NULL, &dbOid, false); if (external_dir_num && external_prefix) { @@ -1524,6 +1537,7 @@ dir_read_file_list(const char *root, const char *external_prefix, file->crc = (pg_crc32) crc; file->compress_alg = parse_compress_alg(compress_alg_string); file->external_dir_num = external_dir_num; + file->dbOid = dbOid ? dbOid : 0; /* * Optional fields @@ -1689,3 +1703,118 @@ backup_contains_external(const char *dir, parray *dirs_list) search_result = parray_bsearch(dirs_list, dir, BlackListCompare); return search_result != NULL; } + +/* + * Print database_map + */ +void +print_database_map(FILE *out, parray *database_map) +{ + int i; + + for (i = 0; i < parray_num(database_map); i++) + { + db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, i); + + fio_fprintf(out, "{\"dbOid\":\"%u\", \"datname\":\"%s\"}\n", + db_entry->dbOid, db_entry->datname); + } + +} + +/* + * Create file 'database_map' and add its meta to backup_content.control + * NULL check for database_map must be done by the caller. + */ +void +write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_list) +{ + FILE *fp; + pgFile *file; + char path[MAXPGPATH]; + char database_map_path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(database_map_path, path, DATABASE_MAP); + + fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); + if (fp == NULL) + elog(ERROR, "Cannot open file list \"%s\": %s", path, + strerror(errno)); + + print_database_map(fp, database_map); + if (fio_fflush(fp) || fio_fclose(fp)) + { + fio_unlink(database_map_path, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write file list \"%s\": %s", + database_map_path, strerror(errno)); + } + + /* */ + file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, + FIO_BACKUP_HOST); + file->crc = pgFileGetCRC(file->path, true, false, + &file->read_size, FIO_BACKUP_HOST); + file->write_size = file->read_size; + free(file->path); + file->path = strdup(DATABASE_MAP); + parray_append(backup_files_list, file); +} + +/* + * read database map, return NULL if database_map in empty or missing + */ +parray * +read_database_map(pgBackup *backup) +{ + FILE *fp; + parray *database_map; + char buf[MAXPGPATH]; + char path[MAXPGPATH]; + char database_map_path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(database_map_path, path, DATABASE_MAP); + + fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); + if (fp == NULL) + { + /* It is ok for database_map to be missing */ + if (errno == ENOENT) + { + elog(VERBOSE, "Failed to locate file \"%s\"", database_map_path); + return NULL; + } + else + elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); + } + + database_map = parray_new(); + + while (fgets(buf, lengthof(buf), fp)) + { + char datname[MAXPGPATH]; + int64 dbOid; + + db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); + + get_control_value(buf, "dbOid", NULL, &dbOid, true); + get_control_value(buf, "datname", datname, NULL, true); + + db_entry->dbOid = dbOid; + db_entry->datname = pgut_strdup(datname); + + parray_append(database_map, db_entry); + } + + fio_close_stream(fp); + + /* Return NULL if file is empty */ + if (parray_num(database_map) == 0) + { + parray_free(database_map); + return NULL; + } + + return database_map; +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2cf6e0edd..5876e12b1 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -96,6 +96,10 @@ bool no_validate = false; bool skip_block_validation = false; bool skip_external_dirs = false; +/* array for datnames, provided via db-include and db-exclude */ +static parray *datname_exclude_list = NULL; +static parray *datname_include_list = NULL; + /* checkdb options */ bool need_amcheck = false; bool heapallindexed = false; @@ -133,6 +137,9 @@ static void opt_show_format(ConfigOption *opt, const char *arg); static void compress_init(void); +static void opt_datname_exclude_list(ConfigOption *opt, const char *arg); +static void opt_datname_include_list(ConfigOption *opt, const char *arg); + /* * Short name should be non-printable ASCII character. */ @@ -171,6 +178,8 @@ static ConfigOption cmd_options[] = { 'b', 143, "no-validate", &no_validate, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, + { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, + { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -631,15 +640,37 @@ main(int argc, char *argv[]) return do_backup(start_time, no_validate); } case RESTORE_CMD: - return do_restore_or_validate(current.backup_id, - recovery_target_options, - true); + { + parray *datname_list = NULL; + /* true for 'include', false for 'exclude' */ + bool partial_restore_type = false; + + if (datname_exclude_list && datname_include_list) + elog(ERROR, "You cannot specify '--db-include' and '--db-exclude' together"); + + /* At this point we are sure that user requested partial restore */ + if (datname_exclude_list) + datname_list = datname_exclude_list; + + if (datname_include_list) + { + partial_restore_type = true; + datname_list = datname_include_list; + } + return do_restore_or_validate(current.backup_id, + recovery_target_options, + true, + datname_list, + partial_restore_type); + } case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0) return do_validate_all(); else return do_restore_or_validate(current.backup_id, recovery_target_options, + false, + NULL, false); case SHOW_CMD: return do_show(current.backup_id); @@ -741,3 +772,37 @@ compress_init(void) elog(ERROR, "Multithread backup does not support pglz compression"); } } + +/* Construct array of datnames, provided by user via db-exclude option */ +void +opt_datname_exclude_list(ConfigOption *opt, const char *arg) +{ + char *dbname = NULL; + + if (!datname_exclude_list) + datname_exclude_list = parray_new(); + + dbname = pgut_malloc(strlen(arg) + 1); + + /* add sanity for database name */ + strcpy(dbname, arg); + + parray_append(datname_exclude_list, dbname); +} + +/* Construct array of datnames, provided by user via db-include option */ +void +opt_datname_include_list(ConfigOption *opt, const char *arg) +{ + char *dbname = NULL; + + if (!datname_include_list) + datname_include_list = parray_new(); + + dbname = pgut_malloc(strlen(arg) + 1); + + /* add sanity for database name */ + strcpy(dbname, arg); + + parray_append(datname_include_list, dbname); +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 485c9f72e..fbf9d0c59 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -60,6 +60,7 @@ extern const char *PROGRAM_EMAIL; #define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" #define EXTERNAL_DIR "external_directories/externaldir" +#define DATABASE_MAP "database_map" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 @@ -83,6 +84,12 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +typedef struct db_map_entry +{ + Oid dbOid; + char *datname; +} db_map_entry; + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, @@ -486,7 +493,9 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments, /* in restore.c */ extern int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - bool is_restore); + bool is_restore, + parray * datname_list, + bool partial_restore_type); extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); @@ -500,6 +509,8 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( extern void do_merge(time_t backup_id); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); +extern parray *read_database_map(pgBackup *backup); + /* in init.c */ extern int do_init(void); extern int do_add_instance(void); @@ -601,6 +612,11 @@ extern void check_tablespace_mapping(pgBackup *backup); extern void check_external_dir_mapping(pgBackup *backup); extern char *get_external_remap(char *current_dir); +extern void print_database_map(FILE *out, parray *database_list); +extern void write_database_map(pgBackup *backup, parray *database_list, + parray *backup_file_list); +extern void db_map_entry_free(void *map); + extern void print_file_list(FILE *out, const parray *files, const char *root, const char *external_prefix, parray *external_list); extern parray *dir_read_file_list(const char *root, const char *external_prefix, @@ -649,6 +665,8 @@ extern void restore_data_file(const char *to_path, uint32 backup_version); extern bool copy_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file, bool missing_ok); +extern bool create_empty_file(fio_location from_location, const char *to_root, + fio_location to_location, pgFile *file); extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); diff --git a/src/restore.c b/src/restore.c index 6c434cc65..d513808f9 100644 --- a/src/restore.c +++ b/src/restore.c @@ -25,6 +25,7 @@ typedef struct char *external_prefix; parray *dest_external_dirs; parray *dest_files; + parray *dbOid_exclude_list; /* * Return value from the thread. @@ -34,19 +35,24 @@ typedef struct } restore_files_arg; static void restore_backup(pgBackup *backup, parray *dest_external_dirs, - parray *dest_files); + parray *dest_files, parray *dbOid_exclude_list); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); +static parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, + bool partial_restore_type); + +static int pgCompareOid(const void *f1, const void *f2); + /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - bool is_restore) + bool is_restore, parray *datname_list, bool partial_restore_type) { int i = 0; int j = 0; @@ -58,6 +64,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, pgBackup *corrupted_backup = NULL; char *action = is_restore ? "Restore":"Validate"; parray *parent_chain = NULL; + parray *dbOid_exclude_list = NULL; if (is_restore) { @@ -428,6 +435,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup_path[MAXPGPATH]; int i; + /* + * Get a list of dbOid`s to skip if user requested the partial restore. + * It is important that we do this after(!) validation so + * database_map can be trusted. + */ + dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, datname_list, + partial_restore_type); + /* * Preparations for actual restoring. */ @@ -481,7 +496,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (rt->no_validate && !lock_backup(backup)) elog(ERROR, "Cannot lock backup directory"); - restore_backup(backup, dest_external_dirs, dest_files); + restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list); } if (dest_external_dirs != NULL) @@ -508,7 +523,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore one backup. */ void -restore_backup(pgBackup *backup, parray *dest_external_dirs, parray *dest_files) +restore_backup(pgBackup *backup, parray *dest_external_dirs, + parray *dest_files, parray *dbOid_exclude_list) { char timestamp[100]; char database_path[MAXPGPATH]; @@ -615,6 +631,7 @@ restore_backup(pgBackup *backup, parray *dest_external_dirs, parray *dest_files) arg->external_prefix = external_prefix; arg->dest_external_dirs = dest_external_dirs; arg->dest_files = dest_files; + arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -707,6 +724,13 @@ restore_files(void *arg) continue; } + /* Do not restore database_map file */ + if (path_is_prefix_of_path(DATABASE_MAP, file->rel_path)) + { + elog(VERBOSE, "Skip database_map"); + continue; + } + /* Do no restore external directory file if a user doesn't want */ if (skip_external_dirs && file->external_dir_num > 0) continue; @@ -716,14 +740,31 @@ restore_files(void *arg) pgFileCompareRelPathWithExternal) == NULL) continue; + /* only files from pgdata can be skipped by partial restore */ + if (arguments->dbOid_exclude_list && !file->external_dir_num) + { + /* exclude map is not empty */ + if (parray_bsearch(arguments->dbOid_exclude_list, + &file->dbOid, pgCompareOid)) + { + /* got a match, destination file will truncated */ + create_empty_file(FIO_BACKUP_HOST, + instance_config.pgdata, FIO_DB_HOST, file); + + elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", file->rel_path); + continue; + } + } + /* * restore the file. * We treat datafiles separately, cause they were backed up block by * block and have BackupPageHeader meta information, so we cannot just * copy the file from backup. */ - elog(VERBOSE, "Restoring file %s, is_datafile %i, is_cfs %i", + elog(VERBOSE, "Restoring file \"%s\", is_datafile %i, is_cfs %i", file->path, file->is_datafile?1:0, file->is_cfs?1:0); + if (file->is_datafile && !file->is_cfs) { char to_path[MAXPGPATH]; @@ -1118,3 +1159,119 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } + +/* Return dbOid array of databases that should not be restored + * Regardless of what options user used, db-include or db-exclude, + * we convert it into exclude_list. + * Return NULL if partial restore is not requested. + */ +parray * +get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_restore_type) +{ + int i; + int j; + parray *database_map = NULL; + parray * dbOid_exclude_list = NULL; + bool found_match = false; + + /* partial restore was not requested */ + if (!datname_list) + return NULL; + + database_map = read_database_map(backup); + + /* partial restore requested but database_map is missing */ + if (datname_list && !database_map) + elog(ERROR, "database_map is missing in backup %s", base36enc(backup->start_time)); + + /* So we have db-include list and database list for it. + * We must form up a list of databases to exclude + */ + if (partial_restore_type) + { + /* For 'include' find dbOid of every datname NOT specified by user */ + for (i = 0; i < parray_num(datname_list); i++) + { + char *datname = (char *) parray_get(datname_list, i); + + found_match = false; + for (j = 0; j < parray_num(database_map); j++) + { + db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); + + /* got a match */ + if (strcmp(db_entry->datname, datname) == 0) + { + found_match = true; + /* for db-include we must exclude db_entry from database_map */ + parray_remove(database_map, j); + j--; + } + } + /* If specified datname is not found in database_map, error out */ + if (!found_match) + elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", + datname, base36enc(backup->start_time)); + } + + /* At this moment only databases to exclude are left in the map */ + for (j = 0; j < parray_num(database_map); j++) + { + db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); + + if (!dbOid_exclude_list) + dbOid_exclude_list = parray_new(); + parray_append(dbOid_exclude_list, &db_entry->dbOid); + } + } + else + { + /* For exclude job is easier, find dbOid for every specified datname */ + for (i = 0; i < parray_num(datname_list); i++) + { + char *datname = (char *) parray_get(datname_list, i); + + found_match = false; + for (j = 0; j < parray_num(database_map); j++) + { + db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); + + /* got a match */ + if (strcmp(db_entry->datname, datname) == 0) + { + found_match = true; + /* for db-exclude we must add dbOid to exclude list */ + if (!dbOid_exclude_list) + dbOid_exclude_list = parray_new(); + parray_append(dbOid_exclude_list, &db_entry->dbOid); + } + } + /* If specified datname is not found in database_map, error out */ + if (!found_match) + elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", + datname, base36enc(backup->start_time)); + } + } + + /* extra sanity, we must be totaly sure that list is not empty */ + if (parray_num(dbOid_exclude_list) < 1) + { + parray_free(dbOid_exclude_list); + return NULL; + } + + if (dbOid_exclude_list) + parray_qsort(dbOid_exclude_list, pgCompareOid); + + return dbOid_exclude_list; +} + +/* Compare two Oid */ +int +pgCompareOid(const void *f1, const void *f2) +{ + Oid *f1p = *(Oid **)f1; + Oid *f2p = *(Oid **)f2; + + return (*(Oid*)f1p - *(Oid*)f2p); +} \ No newline at end of file From 581432759322ef2dc08561a5eb29f12f99730312 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 5 Jun 2019 12:49:05 +0300 Subject: [PATCH 0726/2107] tests: fix backup.BackupTest.test_sigint_handling --- tests/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index ff9ff41d3..bfcb1a0bb 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1159,7 +1159,7 @@ def test_sigint_handling(self): gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') - gdb.continue_execution_until_exit() + # gdb.continue_execution_until_exit() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] From 29bbcf9b822051eef73dbed8536fbe15720f0819 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 5 Jun 2019 16:41:36 +0300 Subject: [PATCH 0727/2107] tests: minor fixes --- tests/backup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index bfcb1a0bb..e55e6512c 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1159,7 +1159,7 @@ def test_sigint_handling(self): gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') - # gdb.continue_execution_until_exit() + gdb.continue_execution_until_error() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] @@ -1197,7 +1197,7 @@ def test_sigterm_handling(self): gdb.remove_all_breakpoints() gdb._execute('signal SIGTERM') - gdb.continue_execution_until_exit() + gdb.continue_execution_until_error() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] @@ -1235,7 +1235,7 @@ def test_sigquit_handling(self): gdb.remove_all_breakpoints() gdb._execute('signal SIGQUIT') - gdb.continue_execution_until_exit() + gdb.continue_execution_until_error() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] From e66d96ac169fd31351acb6a1ae18c296931e454d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Jun 2019 19:17:01 +0300 Subject: [PATCH 0728/2107] Readme: minor fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc461d6bf..6e040d69b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Extended logging settings * Custom commands to simplify WAL log archiving * External to PGDATA directories, such as directories with config files and scripts, can be included in backup -* Remote backup, restore, archive-push and archive-get operations via ssh (beta) +* Remote backup, restore, add-instance, archive-push and archive-get operations via ssh (beta) * Checking running PostgreSQL instance for the sights of corruption in read-only mode via `checkdb` command. To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. From 0bd2dc0e50797c8bf762924de2ddd8ab4337f0bf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Jun 2019 03:12:40 +0300 Subject: [PATCH 0729/2107] [Issue #79] Minor changes and improvement of comments --- src/dir.c | 10 +++++++--- src/restore.c | 35 ++++++++++++++++------------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/dir.c b/src/dir.c index 60f0ea1c4..b175e37be 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1779,11 +1779,15 @@ read_database_map(pgBackup *backup) fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); if (fp == NULL) { - /* It is ok for database_map to be missing */ + /* It is NOT ok for database_map to be missing at this point + * But at this point we are sure that partial restore is requested, + * so we error here if database_map is missing. + * It`s a job of the caller to error if database_map is not empty. + */ if (errno == ENOENT) { - elog(VERBOSE, "Failed to locate file \"%s\"", database_map_path); - return NULL; + elog(ERROR, "Backup %s has missing database_map: \"%s\"", + base36enc(backup->start_time), database_map_path); } else elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); diff --git a/src/restore.c b/src/restore.c index d513808f9..fb93d62bc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -439,8 +439,12 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Get a list of dbOid`s to skip if user requested the partial restore. * It is important that we do this after(!) validation so * database_map can be trusted. + * NOTE: database_map could be missing for legal reasons, e.g. missing + * permissions on pg_database during `backup` and, as long as user do not request + * partial restore, it`s OK */ - dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, datname_list, + if (datname_list) + dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, datname_list, partial_restore_type); /* @@ -1163,7 +1167,6 @@ parseRecoveryTargetOptions(const char *target_time, /* Return dbOid array of databases that should not be restored * Regardless of what options user used, db-include or db-exclude, * we convert it into exclude_list. - * Return NULL if partial restore is not requested. */ parray * get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_restore_type) @@ -1172,17 +1175,13 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest int j; parray *database_map = NULL; parray * dbOid_exclude_list = NULL; - bool found_match = false; - - /* partial restore was not requested */ - if (!datname_list) - return NULL; + /* get database_map from file */ database_map = read_database_map(backup); /* partial restore requested but database_map is missing */ - if (datname_list && !database_map) - elog(ERROR, "database_map is missing in backup %s", base36enc(backup->start_time)); + if (!database_map) + elog(ERROR, "Backup %s has empty database_map", base36enc(backup->start_time)); /* So we have db-include list and database list for it. * We must form up a list of databases to exclude @@ -1192,9 +1191,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest /* For 'include' find dbOid of every datname NOT specified by user */ for (i = 0; i < parray_num(datname_list); i++) { + bool found_match = false; char *datname = (char *) parray_get(datname_list, i); - found_match = false; for (j = 0; j < parray_num(database_map); j++) { db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); @@ -1229,9 +1228,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest /* For exclude job is easier, find dbOid for every specified datname */ for (i = 0; i < parray_num(datname_list); i++) { + bool found_match = false; char *datname = (char *) parray_get(datname_list, i); - found_match = false; for (j = 0; j < parray_num(database_map); j++) { db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); @@ -1253,15 +1252,13 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest } } - /* extra sanity, we must be totaly sure that list is not empty */ - if (parray_num(dbOid_exclude_list) < 1) - { - parray_free(dbOid_exclude_list); - return NULL; - } + /* extra sanity, we must be totally sure that list is not empty */ + if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) + elog(ERROR, "Failed to find a match for partial restore in database_map of backup %s", + base36enc(backup->start_time)); - if (dbOid_exclude_list) - parray_qsort(dbOid_exclude_list, pgCompareOid); + /* sort dbOid array in ASC order */ + parray_qsort(dbOid_exclude_list, pgCompareOid); return dbOid_exclude_list; } From a4df3b417879a59eb7016c118d771e7682b5f144 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Jun 2019 03:18:49 +0300 Subject: [PATCH 0730/2107] [Issue #79] added tests for partial restore --- tests/helpers/ptrack_helpers.py | 18 +- tests/restore.py | 420 ++++++++++++++++++++++++++++++++ 2 files changed, 432 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index eb201488f..450f336fd 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -130,7 +130,7 @@ def slow_start(self, replica=False): self.start() while True: try: - if self.safe_psql('postgres', query) == 't\n': + if self.safe_psql('template1', query) == 't\n': break except testgres.QueryException as e: if 'database system is starting up' in e[0]: @@ -296,8 +296,6 @@ def pg_config_version(self): # print('PGPROBACKUP_SSH_USER is not set') # exit(1) - - def make_simple_node( self, base_dir=None, @@ -342,6 +340,10 @@ def make_simple_node( 'postgresql.auto.conf', 'max_wal_senders = 10') + # set major version + with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: + node.major_version = f.read().rstrip() + return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): @@ -584,6 +586,13 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B): return filelist_diff + # used for partial restore + def truncate_every_file_in_dir(self, path): + for file in os.listdir(path): + print(file) + with open(os.path.join(path, file), "w") as f: + f.close() + def check_ptrack_recovery(self, idx_dict): size = idx_dict['size'] for PageNum in range(size): @@ -1309,8 +1318,6 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): os.path.join(restored_pgdata['pgdata'], directory), restored_pgdata['dirs'][directory]['mode']) - - for directory in original_pgdata['dirs']: if directory not in restored_pgdata['dirs']: fail = True @@ -1318,7 +1325,6 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): error_message += ' in restored PGDATA: {0}\n'.format( os.path.join(restored_pgdata['pgdata'], directory)) - for file in restored_pgdata['files']: # File is present in RESTORED PGDATA # but not present in ORIGINAL diff --git a/tests/restore.py b/tests/restore.py index 5a86b0bb1..acb2b009f 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -8,6 +8,8 @@ from datetime import datetime, timedelta import hashlib import shutil +import json +from testgres import QueryException module_name = 'restore' @@ -2340,3 +2342,421 @@ def test_lost_non_data_file(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_restore_specific_database_proof_of_concept(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'shared_buffers': '512MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=200) + + exit(1) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '100', '-c8', '-j2', '--no-vacuum']) + + pgbench.wait() + pgbench.stdout.close() + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_restore_exclude(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-include='db1'", + "--db-exclude='db2'"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node_restored_1) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + db1_path = os.path.join( + node_restored_1.data_dir, 'base', db_list['db1']) + db5_path = os.path.join( + node_restored_1.data_dir, 'base', db_list['db5']) + + self.truncate_every_file_in_dir(db1_path) + self.truncate_every_file_in_dir(db5_path) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_2')) + node_restored_2.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + node_restored_2.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'postgres', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_restore_exclude_tablespace(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + cat_version = node.get_control_data()["Catalog version number"] + version_specific_dir = 'PG_' + node.major_version + '_' + cat_version + + # PG_10_201707211 + # pg_tblspc/33172/PG_9.5_201510051/16386/ + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "-T", "{0}={1}".format( + node_tablespace, node1_tablespace)]) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + # truncate every db + for db in db_list: + # with exception below + if db in ['db1', 'db5']: + self.truncate_every_file_in_dir( + os.path.join( + node_restored_1.data_dir, 'pg_tblspc', + tbl_oid, version_specific_dir, db_list[db])) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_2')) + node_restored_2.cleanup() + node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + node_restored_2.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'postgres', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_restore_include(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-include='db1'", + "--db-exclude='db2'"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node_restored_1) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + # truncate every db + for db in db_list: + # with exception below + if db in ['template0', 'template1', 'postgres', 'db1', 'db5']: + continue + self.truncate_every_file_in_dir( + os.path.join( + node_restored_1.data_dir, 'base', db_list[db])) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_2')) + node_restored_2.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-include=db1", + "--db-include=db5", + "--db-include=postgres"]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + node_restored_2.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'db1', + 'select 1') + + node_restored_2.safe_psql( + 'db5', + 'select 1') + + node_restored_2.safe_psql( + 'template1', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db2', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db10', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# partial restore +# 0. basic test for db-exclude and db-include + +# 1. old backup without support of partial restore +# 2. FULL backup do not support partial restore, but incremental do +# 3. database_map is missing for legal reasons, e.g. no permissions for pg_database +# 4. database_map is empty for illegal reason +# 5. database_map contain garbage From 836382f372cae65a857d048b3dad7ac4fd51388a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 7 Jun 2019 13:50:56 +0300 Subject: [PATCH 0731/2107] [Issue #79] minor fixes --- src/restore.c | 62 ++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/restore.c b/src/restore.c index fb93d62bc..ddba953c7 100644 --- a/src/restore.c +++ b/src/restore.c @@ -436,7 +436,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, int i; /* - * Get a list of dbOid`s to skip if user requested the partial restore. + * Get a list of dbOids to skip if user requested the partial restore. * It is important that we do this after(!) validation so * database_map can be trusted. * NOTE: database_map could be missing for legal reasons, e.g. missing @@ -692,11 +692,32 @@ restore_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during restore database"); + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + if (progress) elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->files), file->rel_path); + /* only files from pgdata can be skipped by partial restore */ + if (arguments->dbOid_exclude_list && + file->external_dir_num == 0) + { + /* exclude map is not empty */ + if (parray_bsearch(arguments->dbOid_exclude_list, + &file->dbOid, pgCompareOid)) + { + /* got a match, destination file will truncated */ + create_empty_file(FIO_BACKUP_HOST, + instance_config.pgdata, FIO_DB_HOST, file); + + elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", file->rel_path); + continue; + } + } + /* * For PAGE and PTRACK backups skip datafiles which haven't changed * since previous backup and thus were not backed up. @@ -717,10 +738,6 @@ restore_files(void *arg) } } - /* Directories were created before */ - if (S_ISDIR(file->mode)) - continue; - /* Do not restore tablespace_map file */ if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) { @@ -729,7 +746,8 @@ restore_files(void *arg) } /* Do not restore database_map file */ - if (path_is_prefix_of_path(DATABASE_MAP, file->rel_path)) + if ((file->external_dir_num == 0) && + strcmp(DATABASE_MAP, file->rel_path) == 0) { elog(VERBOSE, "Skip database_map"); continue; @@ -744,22 +762,6 @@ restore_files(void *arg) pgFileCompareRelPathWithExternal) == NULL) continue; - /* only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && !file->external_dir_num) - { - /* exclude map is not empty */ - if (parray_bsearch(arguments->dbOid_exclude_list, - &file->dbOid, pgCompareOid)) - { - /* got a match, destination file will truncated */ - create_empty_file(FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, file); - - elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", file->rel_path); - continue; - } - } - /* * restore the file. * We treat datafiles separately, cause they were backed up block by @@ -1164,9 +1166,9 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } -/* Return dbOid array of databases that should not be restored - * Regardless of what options user used, db-include or db-exclude, - * we convert it into exclude_list. +/* Return array of dbOids of databases that should not be restored + * Regardless of what option user used, db-include or db-exclude, + * we always convert it into exclude_list. */ parray * get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_restore_type) @@ -1174,17 +1176,17 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest int i; int j; parray *database_map = NULL; - parray * dbOid_exclude_list = NULL; + parray *dbOid_exclude_list = NULL; /* get database_map from file */ database_map = read_database_map(backup); /* partial restore requested but database_map is missing */ if (!database_map) - elog(ERROR, "Backup %s has empty database_map", base36enc(backup->start_time)); + elog(ERROR, "Backup %s has empty or mangled database_map", base36enc(backup->start_time)); - /* So we have db-include list and database list for it. - * We must form up a list of databases to exclude + /* So we have list of datnames and database_map for it. + * We must construct a list of dbOids to exclude. */ if (partial_restore_type) { @@ -1263,7 +1265,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_rest return dbOid_exclude_list; } -/* Compare two Oid */ +/* Compare two Oids */ int pgCompareOid(const void *f1, const void *f2) { From 66a1c643d569c24cc2197591b8cd54ce1e150036 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 7 Jun 2019 22:22:26 +0300 Subject: [PATCH 0732/2107] update backup_content.control during backup; implement buffered write of backup_content.control --- src/backup.c | 19 ++++++++++++ src/catalog.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++--- src/dir.c | 46 ---------------------------- 3 files changed, 98 insertions(+), 51 deletions(-) diff --git a/src/backup.c b/src/backup.c index d6cd988d8..8a03e325f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -462,6 +462,7 @@ do_backup_instance(PGconn *backup_conn) arg->prev_start_lsn = prev_backup_start_lsn; arg->conn_arg.conn = NULL; arg->conn_arg.cancel_conn = NULL; + arg->thread_num = i+1; /* By default there are some error */ arg->ret = 1; } @@ -1987,6 +1988,9 @@ backup_files(void *arg) int i; backup_files_arg *arguments = (backup_files_arg *) arg; int n_backup_files_list = parray_num(arguments->files_list); + static time_t prev_time; + + prev_time = current.start_time; /* backup a file */ for (i = 0; i < n_backup_files_list; i++) @@ -1995,6 +1999,21 @@ backup_files(void *arg) struct stat buf; pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + + if (arguments->thread_num == 1) + { + /* update every 10 seconds */ + if ((difftime(time(NULL), prev_time)) > 10) + { + prev_time = time(NULL); + elog(INFO, "write_backup_filelist N=%ld, starttime %ld, time %ld", + parray_num(backup_files_list), current.start_time, prev_time); + + write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, + NULL, arguments->external_dirs); + } + } + if (!pg_atomic_test_set_flag(&file->lock)) continue; elog(VERBOSE, "Copying file: \"%s\" ", file->path); diff --git a/src/catalog.c b/src/catalog.c index 43ebdf791..043f231d9 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -632,22 +632,96 @@ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, const char *external_prefix, parray *external_list) { - FILE *fp; + FILE *out; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; int errno_temp; + size_t i = 0; + #define BUFFERSZ BLCKSZ*500 + char buf[BUFFERSZ]; + size_t write_len = 0; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - fp = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) + out = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); + if (out == NULL) elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, strerror(errno)); - print_file_list(fp, files, root, external_prefix, external_list); + /* print each file in the list */ + while(i < parray_num(files)) + { + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; + char line[BLCKSZ]; + int len = 0; + + /* omit root directory portion */ + if (root && strstr(path, root) == path) + path = GetRelativePath(path, root); + else if (file->external_dir_num && !external_prefix) + { + Assert(external_list); + path = GetRelativePath(path, parray_get(external_list, + file->external_dir_num - 1)); + } - if (fio_fflush(fp) || fio_fclose(fp)) + len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"is_cfs\":\"%u\", \"crc\":\"%u\", " + "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"", + path, file->write_size, file->mode, + file->is_datafile ? 1 : 0, + file->is_cfs ? 1 : 0, + file->crc, + deparse_compress_alg(file->compress_alg), + file->external_dir_num); + + if (file->is_datafile) + len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); + + if (file->linked) + len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); + + if (file->n_blocks != BLOCKNUM_INVALID) + len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); + + len += sprintf(line+len, "}\n"); + + if (write_len + len <= BUFFERSZ) + { + memcpy(buf+write_len, line, len); + write_len += len; + } + else + { + /* write buffer to file */ + if (fio_fwrite(out, buf, write_len) != write_len) + { + errno_temp = errno; + fio_unlink(path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write file list \"%s\": %s", + path_temp, strerror(errno)); + } + /* reset write_len */ + write_len = 0; + } + + i++; + } + + /* write what is left in the buffer to file */ + if (write_len > 0) + if (fio_fwrite(out, buf, write_len) != write_len) + { + errno_temp = errno; + fio_unlink(path_temp, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write file list \"%s\": %s", + path_temp, strerror(errno)); + } + + if (fio_fflush(out) || fio_fclose(out)) { errno_temp = errno; fio_unlink(path_temp, FIO_BACKUP_HOST); diff --git a/src/dir.c b/src/dir.c index 426d16963..50dd3d044 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1271,52 +1271,6 @@ get_external_remap(char *current_dir) return current_dir; } -/* - * Print backup content list. - */ -void -print_file_list(FILE *out, const parray *files, const char *root, - const char *external_prefix, parray *external_list) -{ - size_t i; - - /* print each file in the list */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - char *path = file->path; - - /* omit root directory portion */ - if (root && strstr(path, root) == path) - path = GetRelativePath(path, root); - else if (file->external_dir_num && !external_prefix) - { - Assert(external_list); - path = GetRelativePath(path, parray_get(external_list, - file->external_dir_num - 1)); - } - - fio_fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " - "\"mode\":\"%u\", \"is_datafile\":\"%u\", " - "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"", - path, file->write_size, file->mode, - file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, - deparse_compress_alg(file->compress_alg), file->external_dir_num); - - if (file->is_datafile) - fio_fprintf(out, ",\"segno\":\"%d\"", file->segno); - - if (file->linked) - fio_fprintf(out, ",\"linked\":\"%s\"", file->linked); - - if (file->n_blocks != BLOCKNUM_INVALID) - fio_fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); - - fio_fprintf(out, "}\n"); - } -} - /* Parsing states for get_control_value() */ #define CONTROL_WAIT_NAME 1 #define CONTROL_INNAME 2 From dd2d85d11ff78175d9dbeab9c194021dd64e6e4c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Jun 2019 20:56:01 +0300 Subject: [PATCH 0733/2107] tests: added truncate_every_file_in_dir --- tests/helpers/ptrack_helpers.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index eb201488f..7bd31a898 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -584,6 +584,12 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B): return filelist_diff + # used for partial restore + def truncate_every_file_in_dir(self, path): + for file in os.listdir(path): + with open(os.path.join(path, file), "w") as f: + f.close() + def check_ptrack_recovery(self, idx_dict): size = idx_dict['size'] for PageNum in range(size): @@ -734,20 +740,24 @@ def clean_pb(self, backup_dir): def backup_node( self, backup_dir, instance, node, data_dir=False, - backup_type='full', options=[], asynchronous=False, gdb=False, + backup_type='full', datname=False, options=[], + asynchronous=False, gdb=False, old_binary=False, return_id=True ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') exit(1) + if not datname: + datname = 'postgres' + cmd_list = [ 'backup', '-B', backup_dir, '--instance={0}'.format(instance), # "-D", pgdata, '-p', '%i' % node.port, - '-d', 'postgres' + '-d', datname ] if data_dir: From 06805ea4094d2b380c220a177b164371fc6b1650 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Jun 2019 21:03:05 +0300 Subject: [PATCH 0734/2107] [Issue 79] be more paranoid about missing or mangled database_map --- src/backup.c | 13 +++++++++++-- src/dir.c | 13 +++---------- src/restore.c | 48 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/backup.c b/src/backup.c index 72521685b..f350ece8b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1072,6 +1072,7 @@ pg_ptrack_support(PGconn *backup_conn) /* * Create 'datname to Oid' map + * Return NULL if failed to construct database_map */ parray * get_database_map(PGconn *conn) @@ -1080,10 +1081,10 @@ get_database_map(PGconn *conn) parray *database_map = NULL; int i; - res = pgut_execute(conn, + res = pgut_execute_extended(conn, "SELECT oid, datname FROM pg_catalog.pg_database " "WHERE datname NOT IN ('template1', 'template0')", - 0, NULL); + 0, NULL, true, true); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -1114,6 +1115,14 @@ get_database_map(PGconn *conn) parray_append(database_map, db_entry); } + /* extra paranoia */ + if (database_map && (parray_num(database_map) == 0)) + { + parray_free(database_map); + elog(WARNING, "Failed to get database map"); + return NULL; + } + return database_map; } diff --git a/src/dir.c b/src/dir.c index b175e37be..f1fefbfc0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1779,18 +1779,11 @@ read_database_map(pgBackup *backup) fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); if (fp == NULL) { - /* It is NOT ok for database_map to be missing at this point - * But at this point we are sure that partial restore is requested, - * so we error here if database_map is missing. + /* It is NOT ok for database_map to be missing at this point, so + * we should error here. * It`s a job of the caller to error if database_map is not empty. */ - if (errno == ENOENT) - { - elog(ERROR, "Backup %s has missing database_map: \"%s\"", - base36enc(backup->start_time), database_map_path); - } - else - elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); + elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); } database_map = parray_new(); diff --git a/src/restore.c b/src/restore.c index ddba953c7..253e13d2e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -42,8 +42,8 @@ static void create_recovery_conf(time_t backup_id, static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); -static parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, - bool partial_restore_type); +static parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, + parray *datname_list, bool partial_restore_type); static int pgCompareOid(const void *f1, const void *f2); @@ -435,6 +435,15 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup_path[MAXPGPATH]; int i; + /* + * Preparations for actual restoring. + */ + pgBackupGetPath(dest_backup, control_file, lengthof(control_file), + DATABASE_FILE_LIST); + dest_files = dir_read_file_list(NULL, NULL, control_file, + FIO_BACKUP_HOST); + parray_qsort(dest_files, pgFileCompareRelPathWithExternal); + /* * Get a list of dbOids to skip if user requested the partial restore. * It is important that we do this after(!) validation so @@ -444,18 +453,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * partial restore, it`s OK */ if (datname_list) - dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, datname_list, + dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, datname_list, partial_restore_type); - /* - * Preparations for actual restoring. - */ - pgBackupGetPath(dest_backup, control_file, lengthof(control_file), - DATABASE_FILE_LIST); - dest_files = dir_read_file_list(NULL, NULL, control_file, - FIO_BACKUP_HOST); - parray_qsort(dest_files, pgFileCompareRelPathWithExternal); - /* * Restore dest_backup internal directories. */ @@ -1171,19 +1171,39 @@ parseRecoveryTargetOptions(const char *target_time, * we always convert it into exclude_list. */ parray * -get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, bool partial_restore_type) +get_dbOid_exclude_list(pgBackup *backup, parray *files, + parray *datname_list, bool partial_restore_type) { int i; int j; parray *database_map = NULL; parray *dbOid_exclude_list = NULL; + bool found_database_map = false; + + /* make sure that database_map is in backup_content.control */ + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + if ((file->external_dir_num == 0) && + strcmp(DATABASE_MAP, file->rel_path) == 0) + { + found_database_map = true; + break; + } + } + + if (!found_database_map) + elog(ERROR, "Backup %s has missing database_map, partial restore is impossible.", + base36enc(backup->start_time)); /* get database_map from file */ database_map = read_database_map(backup); /* partial restore requested but database_map is missing */ if (!database_map) - elog(ERROR, "Backup %s has empty or mangled database_map", base36enc(backup->start_time)); + elog(ERROR, "Backup %s has empty or mangled database_map, partial restore is impossible.", + base36enc(backup->start_time)); /* So we have list of datnames and database_map for it. * We must construct a list of dbOids to exclude. From e58cf5b776dd13c9a0e22ef509864c3d86d8d223 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 10 Jun 2019 21:03:34 +0300 Subject: [PATCH 0735/2107] tests: more tests for partial restore --- tests/restore.py | 448 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 437 insertions(+), 11 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index acb2b009f..a71fc7488 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2429,8 +2429,8 @@ def test_partial_restore_exclude(self): self.restore_node( backup_dir, 'node', node_restored_1, options=[ - "--db-include='db1'", - "--db-exclude='db2'"]) + "--db-include=db1", + "--db-exclude=db2"]) self.assertEqual( 1, 0, "Expecting Error because of 'db-exclude' and 'db-include'.\n " @@ -2670,8 +2670,8 @@ def test_partial_restore_include(self): self.restore_node( backup_dir, 'node', node_restored_1, options=[ - "--db-include='db1'", - "--db-exclude='db2'"]) + "--db-include=db1", + "--db-exclude=db2"]) self.assertEqual( 1, 0, "Expecting Error because of 'db-exclude' and 'db-include'.\n " @@ -2753,10 +2753,436 @@ def test_partial_restore_include(self): # Clean after yourself self.del_test_dir(module_name, fname) -# partial restore -# 0. basic test for db-exclude and db-include + -# 1. old backup without support of partial restore -# 2. FULL backup do not support partial restore, but incremental do -# 3. database_map is missing for legal reasons, e.g. no permissions for pg_database -# 4. database_map is empty for illegal reason -# 5. database_map contain garbage + def test_partial_restore_backward_compatibility_1(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with old binary, without partial restore support + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored, options=[ + "--db-exclude=db5"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup do not support partial restore.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has missing database_map".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # incremental backup with partial restore support + for i in range(11, 15, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # get db list + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + db_list_splitted = db_list_raw.splitlines() + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # get etalon + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db5'])) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db14'])) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + # get new node + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-exclude=db5", + "--db-exclude=db14"]) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + self.compare_pgdata(pgdata_restored, pgdata_restored_1) + + def test_partial_restore_backward_compatibility_merge(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with old binary, without partial restore support + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored, options=[ + "--db-exclude=db5"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup do not support partial restore.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has missing database_map".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # incremental backup with partial restore support + for i in range(11, 15, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # get db list + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + db_list_splitted = db_list_raw.splitlines() + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # get etalon + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db5'])) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db14'])) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + # get new node + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + # merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-exclude=db5", + "--db-exclude=db14"]) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + self.compare_pgdata(pgdata_restored, pgdata_restored_1) + + def test_missing_database_map(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + node.safe_psql( + "postgres", + "CREATE DATABASE backupdb") + + if self.get_version(node) > self.version_to_num('10.0'): + # bootstrap for 10/11 + node.safe_psql( + "backupdb", + "REVOKE ALL on SCHEMA public from public; " + "REVOKE ALL on SCHEMA pg_catalog from public; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE postgres to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + # we use it for partial restore and checkdb + # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; " + # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;" + ) + else: + # bootstrap for 9.5/9.6 + node.safe_psql( + "backupdb", + "REVOKE ALL on SCHEMA public from public; " + "REVOKE ALL on SCHEMA pg_catalog from public; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE postgres to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + # we use it for ptrack + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + # we use it for partial restore and checkdb + # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; " + # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;" + ) + + # FULL backup without database_map + backup_id = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', "-U", "backup"]) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + # backup has missing database_map and that is legal + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db5", "--db-exclude=db9"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has missing database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has missing database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # check that simple restore is still possible + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_empty_and_mangled_database_map(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with database_map + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + # truncate database_map + path = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'database_map') + with open(path, "w") as f: + f.close() + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has empty or mangled database_map, " + "partial restore is impossible".format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has empty or mangled database_map, " + "partial restore is impossible".format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # mangle database_map + with open(path, "w") as f: + f.write("42") + f.close() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: field "dbOid" is not found in the line 42 of ' + 'the file backup_content.control', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # check that simple restore is still possible + self.restore_node( + backup_dir, 'node', node_restored, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) From ae19a99893f5d0be3dada508db2aecc0a214c9ae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 11 Jun 2019 09:55:05 +0300 Subject: [PATCH 0736/2107] [Issue #79] minor fixes --- src/restore.c | 4 ++-- tests/restore.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index 253e13d2e..5013dd97c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1276,7 +1276,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, /* extra sanity, we must be totally sure that list is not empty */ if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) - elog(ERROR, "Failed to find a match for partial restore in database_map of backup %s", + elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", base36enc(backup->start_time)); /* sort dbOid array in ASC order */ @@ -1293,4 +1293,4 @@ pgCompareOid(const void *f1, const void *f2) Oid *f2p = *(Oid **)f2; return (*(Oid*)f1p - *(Oid*)f2p); -} \ No newline at end of file +} diff --git a/tests/restore.py b/tests/restore.py index a71fc7488..a1cfb2286 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3180,6 +3180,22 @@ def test_empty_and_mangled_database_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: field "dbOid" is not found in the line 42 of ' + 'the file backup_content.control', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + # check that simple restore is still possible self.restore_node( backup_dir, 'node', node_restored, options=['--no-validate']) From 7ef294501a70fd6bd91a64eb3b469975137fe96c Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Jun 2019 12:58:50 +0300 Subject: [PATCH 0737/2107] write only completed files to backup_content_control of running backup --- src/backup.c | 12 ++++++------ src/catalog.c | 11 ++++++----- src/merge.c | 3 +-- src/pg_probackup.h | 6 ++++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index 8a03e325f..035e0ba54 100644 --- a/src/backup.c +++ b/src/backup.c @@ -575,7 +575,7 @@ do_backup_instance(PGconn *backup_conn) /* Print the list of files to backup catalog */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, - NULL, external_dirs); + external_dirs); /* clean external directories list */ if (external_dirs) @@ -1999,18 +1999,17 @@ backup_files(void *arg) struct stat buf; pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - if (arguments->thread_num == 1) { /* update every 10 seconds */ if ((difftime(time(NULL), prev_time)) > 10) { prev_time = time(NULL); - elog(INFO, "write_backup_filelist N=%ld, starttime %ld, time %ld", - parray_num(backup_files_list), current.start_time, prev_time); + elog(INFO, "write_backup_filelist N=%d, starttime %ld, time %ld", + n_backup_files_list, current.start_time, prev_time); - write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, - NULL, arguments->external_dirs); + write_backup_filelist(¤t, arguments->files_list, instance_config.pgdata, + arguments->external_dirs); } } @@ -2146,6 +2145,7 @@ backup_files(void *arg) } } + file->backuped = true; elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", file->path, file->write_size); } diff --git a/src/catalog.c b/src/catalog.c index 043f231d9..260fc1b8e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -630,7 +630,7 @@ write_backup(pgBackup *backup) */ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, - const char *external_prefix, parray *external_list) + parray *external_list) { FILE *out; char path[MAXPGPATH]; @@ -657,12 +657,15 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char line[BLCKSZ]; int len = 0; + i++; + if (!file->backuped) + continue; + /* omit root directory portion */ if (root && strstr(path, root) == path) path = GetRelativePath(path, root); - else if (file->external_dir_num && !external_prefix) + else if (file->external_dir_num && external_list) { - Assert(external_list); path = GetRelativePath(path, parray_get(external_list, file->external_dir_num - 1)); } @@ -707,8 +710,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* reset write_len */ write_len = 0; } - - i++; } /* write what is left in the buffer to file */ diff --git a/src/merge.c b/src/merge.c index 015a490d7..a8d63aa79 100644 --- a/src/merge.c +++ b/src/merge.c @@ -368,8 +368,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) else to_backup->wal_bytes = BYTES_INVALID; - write_backup_filelist(to_backup, files, from_database_path, - from_external_prefix, NULL); + write_backup_filelist(to_backup, files, from_database_path, NULL); write_backup(to_backup); delete_source_backup: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 485c9f72e..954aa7d22 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -146,6 +146,9 @@ typedef struct pgFile datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, * i.e. datafiles without _ptrack */ + + /* state during bakup */ + bool backuped; /* is file already completely copied into destination backup? */ } pgFile; /* Special values of datapagemap_t bitmapsize */ @@ -555,8 +558,7 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, const char *external_prefix, - parray *external_list); + const char *root, parray *external_list); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); From 096a12b9bcf146247a9aebb119eaba896f03c143 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Jun 2019 14:29:10 +0300 Subject: [PATCH 0738/2107] add size and duration fields to backup.control. update it during backup every 10 seconds --- src/catalog.c | 45 +++++++++++++++++++++++++++++++++++++++++---- src/pg_probackup.h | 4 ++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 260fc1b8e..e034c5d75 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -91,6 +91,31 @@ write_backup_status(pgBackup *backup, BackupStatus status) pgBackupFree(tmp); } +/* update some fields of backup control file */ +void +write_backup_control_on_the_fly(pgBackup *backup) +{ + pgBackup *tmp; + + tmp = read_backup(backup->start_time); + if (!tmp) + { + /* + * Silently exit the function, since read_backup already logged the + * warning message. + */ + return; + } + + tmp->status = backup->status; + tmp->size_on_disk = backup->size_on_disk; + backup->duration = difftime(time(NULL), backup->start_time); + tmp->duration = backup->duration; + write_backup(tmp); + + pgBackupFree(tmp); +} + /* * Create exclusive lockfile in the backup's directory. */ @@ -585,6 +610,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* print external directories list */ if (backup->external_dir_str) fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); + + fio_fprintf(out, "size-on-disk = " INT64_FORMAT "\n", backup->size_on_disk); + fio_fprintf(out, "duration = " INT64_FORMAT "\n", backup->duration); } /* @@ -640,6 +668,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, #define BUFFERSZ BLCKSZ*500 char buf[BUFFERSZ]; size_t write_len = 0; + int64 backup_size_on_disk = BYTES_INVALID; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); @@ -661,14 +690,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (!file->backuped) continue; - /* omit root directory portion */ - if (root && strstr(path, root) == path) - path = GetRelativePath(path, root); - else if (file->external_dir_num && external_list) + backup_size_on_disk += file->write_size; + if (file->external_dir_num && external_list) { path = GetRelativePath(path, parray_get(external_list, file->external_dir_num - 1)); } + else + path = file->rel_path; len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " @@ -737,6 +766,9 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", path_temp, path, strerror(errno_temp)); } + + backup->size_on_disk = backup_size_on_disk; + write_backup_control_on_the_fly(backup); } /* @@ -784,6 +816,8 @@ readBackupControlFile(const char *path) {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, + {'I', 0, "size-on-disk", &backup->size_on_disk, SOURCE_FILE_STRICT}, + {'I', 0, "duration", &backup->duration, SOURCE_FILE_STRICT}, {0} }; @@ -1015,6 +1049,9 @@ pgBackupInit(pgBackup *backup) backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; backup->external_dir_str = NULL; + + backup->size_on_disk = BYTES_INVALID; + backup->duration = (time_t) 0; } /* free pgBackup object */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 954aa7d22..1bd5f2d1d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -310,6 +310,9 @@ struct pgBackup * in the format suitable for recovery.conf */ char *external_dir_str; /* List of external directories, * separated by ':' */ + + int64 size_on_disk; /* updated based on backup_content.control */ + int64 duration; /* TODO write better comment. time(NULL)-start_time */ }; /* Recovery target for restore and validate subcommands */ @@ -547,6 +550,7 @@ extern int do_validate_all(void); extern pgBackup *read_backup(time_t timestamp); extern void write_backup(pgBackup *backup); extern void write_backup_status(pgBackup *backup, BackupStatus status); +extern void write_backup_control_on_the_fly(pgBackup *backup); extern bool lock_backup(pgBackup *backup); extern const char *pgBackupGetBackupMode(pgBackup *backup); From 6f2d3f5cc5a5c67c0cd571227459fbb5e9b734e0 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Jun 2019 18:04:02 +0300 Subject: [PATCH 0739/2107] use data_bytes field instead of introducing new one --- src/backup.c | 23 +++++------------------ src/catalog.c | 22 +++++++++++++++------- src/pg_probackup.h | 1 - 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/backup.c b/src/backup.c index 035e0ba54..c01c029cc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -182,9 +182,6 @@ do_backup_instance(PGconn *backup_conn) check_external_for_tablespaces(external_dirs, backup_conn); } - /* Initialize size summary */ - current.data_bytes = 0; - /* Obtain current timeline */ current.tli = get_current_timeline(false); @@ -432,6 +429,7 @@ do_backup_instance(PGconn *backup_conn) } else join_path_components(dirpath, database_path, dir_name); + file->backuped = true; fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } @@ -581,19 +579,6 @@ do_backup_instance(PGconn *backup_conn) if (external_dirs) free_dir_list(external_dirs); - /* Compute summary of size of regular files in the backup */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - if (S_ISDIR(file->mode)) - current.data_bytes += 4096; - - /* Count the amount of the data actually copied */ - if (S_ISREG(file->mode)) - current.data_bytes += file->write_size; - } - /* Cleanup */ if (backup_list) { @@ -1805,11 +1790,12 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) */ if (backup_files_list) { - file = pgFileNew(backup_label, backup_label, true, 0, + file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0, FIO_BACKUP_HOST); file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; + file->backuped = true; free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); @@ -1851,13 +1837,14 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) if (backup_files_list) { - file = pgFileNew(tablespace_map, tablespace_map, true, 0, + file = pgFileNew(tablespace_map, PG_TABLESPACE_MAP_FILE, true, 0, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) { file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; + file->backuped = true; } free(file->path); file->path = strdup(PG_TABLESPACE_MAP_FILE); diff --git a/src/catalog.c b/src/catalog.c index e034c5d75..31598f3cc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -108,7 +108,7 @@ write_backup_control_on_the_fly(pgBackup *backup) } tmp->status = backup->status; - tmp->size_on_disk = backup->size_on_disk; + tmp->data_bytes = backup->data_bytes; backup->duration = difftime(time(NULL), backup->start_time); tmp->duration = backup->duration; write_backup(tmp); @@ -611,7 +611,6 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->external_dir_str) fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); - fio_fprintf(out, "size-on-disk = " INT64_FORMAT "\n", backup->size_on_disk); fio_fprintf(out, "duration = " INT64_FORMAT "\n", backup->duration); } @@ -668,7 +667,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, #define BUFFERSZ BLCKSZ*500 char buf[BUFFERSZ]; size_t write_len = 0; - int64 backup_size_on_disk = BYTES_INVALID; + int64 backup_size_on_disk = 0; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); @@ -687,10 +686,20 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int len = 0; i++; + if (!file->backuped) + { + elog(WARNING, "file not backuped %s", file->rel_path); continue; + } + + if (S_ISDIR(file->mode)) + backup_size_on_disk += 4096; + + /* Count the amount of the data actually copied */ + if (S_ISREG(file->mode)) + backup_size_on_disk += file->write_size; - backup_size_on_disk += file->write_size; if (file->external_dir_num && external_list) { path = GetRelativePath(path, parray_get(external_list, @@ -767,7 +776,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, path_temp, path, strerror(errno_temp)); } - backup->size_on_disk = backup_size_on_disk; + /* use extra variable to avoid reset of previous data_bytes value in case of error */ + backup->data_bytes = backup_size_on_disk; write_backup_control_on_the_fly(backup); } @@ -816,7 +826,6 @@ readBackupControlFile(const char *path) {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, - {'I', 0, "size-on-disk", &backup->size_on_disk, SOURCE_FILE_STRICT}, {'I', 0, "duration", &backup->duration, SOURCE_FILE_STRICT}, {0} }; @@ -1050,7 +1059,6 @@ pgBackupInit(pgBackup *backup) backup->server_version[0] = '\0'; backup->external_dir_str = NULL; - backup->size_on_disk = BYTES_INVALID; backup->duration = (time_t) 0; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1bd5f2d1d..4b5549b11 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -311,7 +311,6 @@ struct pgBackup char *external_dir_str; /* List of external directories, * separated by ':' */ - int64 size_on_disk; /* updated based on backup_content.control */ int64 duration; /* TODO write better comment. time(NULL)-start_time */ }; From 8ea9bcac8765865020d6bddf8289566b027d01fd Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Jun 2019 18:34:10 +0300 Subject: [PATCH 0740/2107] remove duration variable as unrelated to this feature, code cleanup. --- src/backup.c | 2 -- src/catalog.c | 37 +++---------------------------------- src/pg_probackup.h | 4 +--- 3 files changed, 4 insertions(+), 39 deletions(-) diff --git a/src/backup.c b/src/backup.c index c01c029cc..5587beb31 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1992,8 +1992,6 @@ backup_files(void *arg) if ((difftime(time(NULL), prev_time)) > 10) { prev_time = time(NULL); - elog(INFO, "write_backup_filelist N=%d, starttime %ld, time %ld", - n_backup_files_list, current.start_time, prev_time); write_backup_filelist(¤t, arguments->files_list, instance_config.pgdata, arguments->external_dirs); diff --git a/src/catalog.c b/src/catalog.c index 31598f3cc..d3ad24830 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -91,31 +91,6 @@ write_backup_status(pgBackup *backup, BackupStatus status) pgBackupFree(tmp); } -/* update some fields of backup control file */ -void -write_backup_control_on_the_fly(pgBackup *backup) -{ - pgBackup *tmp; - - tmp = read_backup(backup->start_time); - if (!tmp) - { - /* - * Silently exit the function, since read_backup already logged the - * warning message. - */ - return; - } - - tmp->status = backup->status; - tmp->data_bytes = backup->data_bytes; - backup->duration = difftime(time(NULL), backup->start_time); - tmp->duration = backup->duration; - write_backup(tmp); - - pgBackupFree(tmp); -} - /* * Create exclusive lockfile in the backup's directory. */ @@ -610,8 +585,6 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* print external directories list */ if (backup->external_dir_str) fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); - - fio_fprintf(out, "duration = " INT64_FORMAT "\n", backup->duration); } /* @@ -687,11 +660,10 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, i++; + /* if file is not backuped yet, do not add it to the list */ + /* TODO check that we correctly handle files which disappeared during backups */ if (!file->backuped) - { - elog(WARNING, "file not backuped %s", file->rel_path); continue; - } if (S_ISDIR(file->mode)) backup_size_on_disk += 4096; @@ -778,7 +750,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; - write_backup_control_on_the_fly(backup); + write_backup(backup); } /* @@ -826,7 +798,6 @@ readBackupControlFile(const char *path) {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, - {'I', 0, "duration", &backup->duration, SOURCE_FILE_STRICT}, {0} }; @@ -1058,8 +1029,6 @@ pgBackupInit(pgBackup *backup) backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; backup->external_dir_str = NULL; - - backup->duration = (time_t) 0; } /* free pgBackup object */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4b5549b11..528b068d1 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -310,8 +310,6 @@ struct pgBackup * in the format suitable for recovery.conf */ char *external_dir_str; /* List of external directories, * separated by ':' */ - - int64 duration; /* TODO write better comment. time(NULL)-start_time */ }; /* Recovery target for restore and validate subcommands */ @@ -549,7 +547,7 @@ extern int do_validate_all(void); extern pgBackup *read_backup(time_t timestamp); extern void write_backup(pgBackup *backup); extern void write_backup_status(pgBackup *backup, BackupStatus status); -extern void write_backup_control_on_the_fly(pgBackup *backup); +extern void write_backup_data_bytes(pgBackup *backup); extern bool lock_backup(pgBackup *backup); extern const char *pgBackupGetBackupMode(pgBackup *backup); From 52a5487025b5b14600a90724a808e30077e0e1de Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 11 Jun 2019 18:53:29 +0300 Subject: [PATCH 0741/2107] code cleanup --- src/backup.c | 6 +++++- src/catalog.c | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5587beb31..0787be8e4 100644 --- a/src/backup.c +++ b/src/backup.c @@ -574,6 +574,8 @@ do_backup_instance(PGconn *backup_conn) /* Print the list of files to backup catalog */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, external_dirs); + /* update backup control file to update size info */ + write_backup(¤t); /* clean external directories list */ if (external_dirs) @@ -1988,13 +1990,15 @@ backup_files(void *arg) if (arguments->thread_num == 1) { - /* update every 10 seconds */ + /* update backup_content.control every 10 seconds */ if ((difftime(time(NULL), prev_time)) > 10) { prev_time = time(NULL); write_backup_filelist(¤t, arguments->files_list, instance_config.pgdata, arguments->external_dirs); + /* update backup control file to update size info */ + write_backup(¤t); } } diff --git a/src/catalog.c b/src/catalog.c index d3ad24830..e1ff0168c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -750,7 +750,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; - write_backup(backup); } /* From bc0952a95980e49424b7495f476d79871ffb85b7 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 13 Jun 2019 17:32:43 +0300 Subject: [PATCH 0742/2107] add Documentaion.md --- Documentation.md | 939 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 939 insertions(+) create mode 100644 Documentation.md diff --git a/Documentation.md b/Documentation.md new file mode 100644 index 000000000..8c2f125e1 --- /dev/null +++ b/Documentation.md @@ -0,0 +1,939 @@ +# pg_probackup +--- +**pg_probackup** is a utility to manage backup and recovery of Postgres Pro database clusters. It is designed to perform periodic backups of the Postgres Pro instance that enable you to restore the server in case of a failure. pg_probackup supports Postgres Pro 9.5 or higher. + +### Synopsis + +`pg_probackup init -B backupdir` +`pg_probackup add-instance -B backupdir -D datadir --instance instance_name` +`pg_probackup del-instance -B backupdir --instance instance_name` +`pg_probackup set-config -B backupdir --instance instance_name [option...]` +`pg_probackup show-config -B backupdir --instance instance_name [--format=format]` +`pg_probackup backup -B backupdir --instance instance_name -b backup_mode [option...]` +`pg_probackup merge -B backupdir --instance instance_name -i backup_id [option...]` +`pg_probackup restore -B backupdir --instance instance_name [option...]` +`pg_probackup validate -B backupdir [option...]` +`pg_probackup show -B backupdir [option...]` +`pg_probackup delete -B backupdir --instance instance_name { -i backup_id | --wal | --expired }` +`pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` +`pg_probackup archive-get -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f` +`pg_probackup checkdb -B backupdir --instance instance_name -D datadir [option...]` +`pg_probackup version` +`pg_probackup help [command]` + +* [Installation and Setup](#installation-and-setup) +* [Command-Line Reference](#command-line-reference) +* [Usage](#usage) + +### Overview + +As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: + +- Choosing between full and page-level incremental backups to speed up backup and recovery +- Implementing a single backup strategy for multi-server Postgres Pro clusters +- Automatic data consistency checks and on-demand backup validation without actual data recovery +- Managing backups in accordance with retention policy +- Running backup, restore, and validation processes on multiple parallel threads +- Storing backup data in a compressed state to save disk space +- Taking backups from a standby server to avoid extra load on the master server +- Extended logging settings +- Custom commands to simplify WAL log archiving +- Backing up files and directories located outside of Postgres Pro data directory, such as configuration or log files + +To manage backup data, pg_probackup creates a *backup catalog*. This directory stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. + +Using pg_probackup, you can take full or incremental backups: +- Full backups contain all the data files required to restore the database cluster from scratch. +- Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. pg_probackup supports the following modes of incremental backups: + - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Continuous archiving is not necessary for this mode to operate. Note that this mode can impose read-only I/O pressure equal to a full backup. + - PTRACK backup. In this mode, Postgres Pro tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +Regardless of the chosen backup type, all backups taken with pg_probackup support the following archiving strategies: +- Autonomous backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. +- Archive backups rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). + +### Limitations + +pg_probackup currently has the following limitations: +- Creating backups from a remote server is currently not supported on Windows systems. +- The server from which the backup was taken and the restored server must be compatible by the block_size and wal_block_size parameters and have the same major release number. + +### Installation and Setup + +The pg_probackup package is provided as part of the Postgres Pro distribution. Once you have pg_probackup installed, complete the following setup: + +- Initialize the backup catalog. +- Add a new backup instance to the backup catalog. +- Configure the database cluster to enable pg_probackup backups. + +##### Initializing the Backup Catalog +pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. + +To initialize the backup catalog, run the following command: + +`pg_probackup init -B backupdir` + +where backupdir is the backup catalog. If the backupdir already exists, it must be empty. Otherwise, pg_probackup returns an error. + +pg_probackup creates the backupdir backup catalog, with the following subdirectories: +- *wal/* — directory for WAL files. +- *backups/* — directory for backup files. + +Once the backup catalog is initialized, you can add a new backup instance. + +##### Adding a New Backup Instance +pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. + +To add a new backup instance, run the following command: + +`pg_probackup add-instance -B backupdir -D datadir --instance instance_name [--external-dirs=external_directory_path] [remote_backup_options]` + +where: +- *datadir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. +- *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. +- The optional *--external-dirs* parameter provides the path to one or more directories to include into the backup that are located outside of the data directory. To specify several external directories, separate their paths by a colon. + +pg_probackup creates the instance_name subdirectories under the *backups/* and *wal/* directories of the backup catalog. The *backups/instance_name* directory contains the *pg_probackup.conf* configuration file that controls backup and restore settings for this backup instance. If you run this command with the optional *--external-dirs* parameter, this setting is added to *pg_probackup.conf*, so the specified external directories will be backed up each time you create a backup of this instance. For details on how to fine-tune pg_probackup configuration, see the section called “Configuring pg_probackup”. + +The backup catalog must belong to the file system of the database server. The user launching pg_probackup must have full access to the contents of the backup catalog. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. + +Since pg_probackup uses a regular PostgreSQL connection and the replication protocol, pg_probackup commands require connection options. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the *set-config* command. For details, see the section called “Configuring pg_probackup”. + +##### Configuring the Database Cluster + +Although pg_probackup can be used by a superuser, it is recommended to create a separate user or role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. + +To enable backups, the following rights are required: + +```CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; +GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION txid_current() TO backup; +GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; +``` + +Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. + +Depending on whether you are going to use autonomous or archive backup strategies, Postgres Pro cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the section called “PTRACK Backup” and the section called “Backup from Standby”. + +##### Setting up Autonomous Backups + +To set up the cluster for autonomous backups, complete the following steps: +- Grant the REPLICATION privilege to the backup role: + `ALTER ROLE backup WITH REPLICATION;` +- In the pg_hba.conf file, allow replication on behalf of the backup role. +- Modify the postgresql.conf configuration file of the Postgres Pro server, as follows: + - Make sure the max_wal_senders parameter is set high enough to leave at least one session available for the backup process. + - Set the *wal_level* parameter to be *replica* or higher. + +If you are going to take PAGE backups, you also have to configure WAL archiving as explained in the section called “Setting up Archive Backups”. + +Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server. If you are going to take backups from standby or use PTRACK backups, you must also complete additional setup, as explained in the section called “Backup from Standby” and the section called “PTRACK Backup”, respectively. + +##### Setting up Archive Backups +To set up the cluster for archive backups, complete the following steps: +- Configure the following parameters in postgresql.conf to enable continuous archiving on the Postgres Pro server: + - Make sure the wal_level parameter is set to replica or higher. + - Set the archive_mode parameter. If you are configuring backups on master, archive_mode must be set to on. To perform archiving on standby, set this parameter to always. + - Set the archive_command variable, as follows: + ```archive_command = 'pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f'``` + where backupdir and instance_name refer to the already initialized backup catalog instance for this database cluster. + +Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server. If you are going to take backups from standby or use PTRACK backups, you must also complete additional setup, as explained in the section called “Backup from Standby” and the section called “PTRACK Backup”, respectively. + +##### Backup from Standby + +For Postgres Pro 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: + +- On the standby server, allow replication connections: + - Set the max_wal_senders and hot_standby parameters in postgresql.conf. + - Configure host-based authentication in pg_hba.conf. +- On the master server, enable full_page_writes in postgresql.conf. + + + +>NOTE: Archive backup from the standby server has the following limitations: +- If the standby is promoted to the master during archive backup, the backup fails. +- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable full_page_writes on the master, and not to use a tool like pg_compresslog as archive_command to remove full-page writes from WAL files. + +##### PTRACK Backup + +If you are going to use PTRACK backups, complete the following additional steps: +- In postgresql.conf, set ptrack_enable to on. +- Grant the rights to execute ptrack functions to the backup role: + ``` + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; + ``` + The backup role must have access to all the databases of the cluster. + +### Command-Line Reference +##### Commands + +This section describes pg_probackup commands. Some commands require mandatory options and can take additional options. For detailed descriptions, see the section called “Options”. + +**init** + + pg_probackup init -B backupdir +Initializes the backupdir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backupdir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. + +**add-instance** + + pg_probackup add-instance -B backupdir -D datadir --instance instance_name [--external-dirs=external_directory_path] +Initializes a new backup instance inside the backup catalog backupdir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified datadir data directory. For details, see the section called “Adding a New Backup Instance”. + +**del-instance** + + pg_probackup del-instance -B backupdir --instance instance_name +Deletes all backup and WAL files associated with the specified instance. + +**set-config** + + pg_probackup set-config -B backupdir --instance instance_name + [--log-level-console=log_level] [--log-level-file=log_level] [--log-filename=log_filename] + [--error-log-filename=error_log_filename] [--log-directory=log_directory] + [--log-rotation-size=log_rotation_size] [--log-rotation-age=log_rotation_age] + [--retention-redundancy=redundancy][--retention-window=window] + [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] + [-d dbname] [-h host] [-p port] [-U username] + [--archive-timeout=timeout] [--external-dirs=external_directory_path] + [remote_backup_options] +Adds the specified connection, retention, logging or replica, and compression, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. + +**show-config** + pg_probackup show-config -B backupdir --instance instance_name [--format=plain|json] +Displays the contents of the pg_probackup.conf configuration file located in the backupdir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. +To edit pg_probackup.conf, use the set-config command. +It is not allowed to edit pg_probackup.conf directly. + +**backup** + + pg_probackup backup -B backupdir -b backup_mode --instance instance_name + [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] [--external-dirs=external_directory_path] + [--delete-expired] [--merge-expired] [--delete-wal] [--no-validate] [--skip-block-validation] + [--retention-redundancy=redundancy] [--retention-window=window] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--archive-timeout=timeout] + [--compress] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] + [-j num_threads][--progress] + [logging_options] + [remote_backup_options] +Creates a backup copy of the Postgres Pro instance. The backup_mode option specifies the backup mode to use. For details, see the section called “Creating a Backup”. + +**merge** + + pg_probackup merge -B backupdir --instance instance_name -i backup_id + [-j num_threads][--progress] + [logging_options] + [remote_backup_options] + +Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. For details, see the section called “Merging Backups”. + +**restore** + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [-i backup_id] [{--recovery-target=immediate|latest | --recovery-target-time=time | --recovery-target-xid=xid | --recovery-target-lsn=lsn | --recovery-target-name=recovery_target_name} [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] [-T OLDDIR=NEWDIR] + [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] + [--recovery-target-action=pause|promote|shutdown] + [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + [-j num_threads] [--progress] + [logging_options] + [remote_backup_options] + +Restores the Postgres Pro instance from a backup copy located in the backupdir backup catalog. If you specify a recovery target option, pg_probackup restores the database cluster up to the corresponding recovery target. Otherwise, the most recent backup is used. + +**validate** + + pg_probackup validate -B backupdir + [--instance instance_name] + [-i backup_id] [{--recovery-target-time=time | --recovery-target-xid=xid | --recovery-target-lsn=lsn | --recovery-target-name=recovery_target_name } [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] [--skip-block-validation] + [-j num_threads] [--progress] + +Verifies that all the files required to restore the cluster are present and not corrupted. If you specify the instance_name without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the instance_name with a recovery target option or a backup_id, pg_probackup checks whether it is possible to restore the cluster using these options. If instance_name is not specified, pg_probackup validates all backups available in the backup catalog. + +**show** + + pg_probackup show -B backupdir + [--instance instance_name [-i backup_id]] [--format=plain|json] + +Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. + +**delete** + + pg_probackup delete -B backupdir --instance instance_name + [--wal] {-i backup_id | --expired [--merge-expired] | --merge-expired} [--dry-run] + +Deletes backup or WAL files of the specified backup instance from the backupdir backup catalog: +- The wal option removes the WAL files that are no longer required to restore the cluster from any of the existing backups. +- The -i option removes the specified backup copy. +- The expired option removes the backups that are expired according to the current retention policy. If used together with merge-expired, this option takes effect only after the merge is performed. + +- The merge-expired option merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + +- The dry-run option displays the current status of all the available backups, without deleting or merging expired backups, if any. + +**archive-push** + + pg_probackup archive-push -B backupdir --instance instance_name + --wal-file-path %p --wal-file-name %f' + [--compress][--compress-algorithm=compression_algorithm][--compress-level=compression_level] [--overwrite] + [remote_backup_options] + +Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in Postgres Pro logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. + +You can set archive-push as archive_command in postgresql.conf to perform archive backups. + +**archive-get** + + pg_probackup archive-get -B backupdir --instance instance_name + --wal-file-path %p --wal-file-name %f' + [remote backup options] + +Moves WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as restore_command in recovery.conf when restoring backups using a WAL archive. You do not need to set it manually. + +**checkdb** + + pg_probackup checkdb -D datadir [-B backupdir] [--instance instance_name] + [--amcheck [--heapallindexed] [--skip-block-validation]] + [--progress] [-j num_threads] + +Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified Postgres Pro instance using the amcheck extension. + +**version** + + pg_probackup version + +Prints pg_probackup version. + +**help** + + pg_probackup help [command] + +Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. + +##### Options +This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. For details, see the section called “Configuring pg_probackup”. + +If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. + +**Common Options** +``` + -B directory + --backup-path=directory + BACKUP_PATH +``` +Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. + + -D directory + --pgdata=directory + PGDATA +Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the init command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. + + -i backup_id + -backup-id=backup_id +Specifies the unique identifier of the backup. + + --skip-block-validation + +Disables block-level checksum verification to speed up validation. If this option is used with backup, restore, and validate commands, only file-level checksums will be verified. When used with the checkdb command run with the --amcheck option, --skip-block-validation completely disables validation of data files. + + -j num_threads + --threads=num_threads +Sets the number of parallel threads for backup, recovery, and backup validation processes. + + --progress +Shows the progress of operations. + +**Backup Options** + +The following options can be used together with the backup command. + + -b mode + --backup-mode=mode + +Specifies the backup mode to use. Possible values are: +- FULL — creates a full backup that contains all the data files of the cluster to be restored. +- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. +- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. +- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. + +For details, see the section called “Creating a Backup”. + + -C + --smooth-checkpoint + SMOOTH_CHECKPOINT +Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. + + --stream +Makes an autonomous backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. + + -S slot_name + --slot=slot_name +Specifies the replication slot for WAL streaming. This option can only be used together with the --stream option. + + --temp-slot +Creates a temporary physical replication slot for streaming WAL from the backed up Postgres Pro instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. + + --backup-pg-log +Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. + + -E external_directory_path + --external-dirs=external_directory_path + +Includes the specified directory into the backup. This option is useful to back up configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon. + + --archive-timeout=wait_time +Sets the timeout for WAL segment archiving, in seconds. By default, pg_probackup waits 300 seconds. + + --delete-expired +After a backup copy is successfully created, deletes backups that are expired according to the current retention policy. You can also clean up the expired backups by running the delete command with the expired option. If used together with merge-expired, this option takes effect after the merge is performed. For details, see the section called “Configuring Backup Retention Policy”. + + --merge-expired +After a backup copy is successfully created, merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + + --delete-wal +After a backup copy is successfully created, removes redundant WAL files in accordance with the current retention policy. You can also clean up the expired WAL files by running the delete command with the wal option. For details, see the section called “Configuring Backup Retention Policy”. + +**Restore Options** + + --recovery-target-action=pause|promote|shutdown + Default: pause +Specifies the action the server should take when the recovery target is reached, similar to the recovery_target_action option in the recovery.conf configuration file. + + + -R | --restore-as-replica +Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. + + -T OLDDIR=NEWDIR + --tablespace-mapping=OLDDIR=NEWDIR + +Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. + + --external-mapping=OLDDIR=NEWDIR +Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. + + --skip-external-dirs +Skip external directories included into the backup with the --external-dirs option. The contents of these directories will not be restored. + + --recovery-target-timeline=timeline +Specifies a particular timeline to restore the cluster into. By default, the timeline of the specified backup is used. + + --no-validate +Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running backup or restore operations. + +**Recovery Target Options** +If continuous WAL archiving is configured, you can use one of these options together with restore or validate commands to specify the moment up to which the database cluster must be restored. + + --recovery-target=immediate|latest +Defines when to stop the recovery: +- The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the -i option is omitted. +- The latest value continues the recovery until all WAL segments available in the archive are applied. + + + --recovery-target-lsn=lsn +Specifies the LSN of the write-ahead log location up to which recovery will proceed. + + --recovery-target-name=recovery_target_name +Specifies a named savepoint up to which to restore the cluster data. + + --recovery-target-time=time +Specifies the timestamp up to which recovery will proceed. + + --recovery-target-xid=xid +Specifies the transaction ID up to which recovery will proceed. + + --recovery-target-inclusive=boolean +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with recovery-target-name, recovery-target-time, recovery-target-lsn, or recovery-target-xid options. The default value is taken from the recovery_target_inclusive variable. + +**Delete Options** + + --wal +Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. + + --expired +Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. For details, see the section called “Configuring Backup Retention Policy”. + + --merge-expired +Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + + --dry-run +Displays the current status of all the available backups, without deleting or merging expired backups, if any. + +**Retention Options** +For details on configuring retention policy, see the section called “Configuring Backup Retention Policy”. + + --retention-redundancy=redundancy + Default: 0 +Specifies the number of full backup copies to keep in the data directory. Must be a positive integer. The zero value disables this setting. + + --retention-window=window + Default: 0 +Number of days of recoverability. The zero value disables this setting. + +**Logging Options** + + --log-level-console=log_level + Default: info +Controls which message levels are sent to the console log. Valid values are verbose, log, info, notice, warning, error, and off. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The off level disables console logging. + + --log-level-file=log_level + Default: off +Controls which message levels are sent to a log file. Valid values are verbose, log, info, notice, warning, error, and off. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The off level disables file logging. + + --log-filename=log_filename + Default: pg_probackup.log +Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in log_filename. For example, if you specify the pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. +This option takes effect if file logging is enabled by the log-level-file option. + + --error-log-filename=error_log_filename + Default: none +Defines the filenames of log files for error messages. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying file names, as explained in log_filename. +If error-log-filename is not set, pg_probackup writes all error messages to stderr. + + --log-directory=log_directory + Default: $BACKUP_PATH/log/ +Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. + + --log-rotation-size=log_rotation_size + Default: 0 +Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). + + --log-rotation-age=log_rotation_age + Default: 0 +Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). + +**Connection Options** + + -d dbname + --dbname=dbname + PGDATABASE +Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. + + -h host + --host=host + PGHOST + Default: local socket +Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. + + -p port + --port=port + PGPORT + Default: 5432 +Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. + + -U username + --username=username + PGUSER +User name to connect as. + + -w + --no-password + Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a .pgpass file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. + + -W + --password +Forces a password prompt. + +**Compression Options** + + --compress +Enables compression for data files. You can specify the compression algorithm and level using the --compress-algorithm and --compress-level options, respectively. If you omit these options, --compress uses zlib compression algorithm with compression level 1. +By default, compression is disabled. + + --compress-algorithm=compression_algorithm +Defines the algorithm to use for compressing data files. Possible values are zlib, pglz, and none. If set to zlib or pglz, this option enables compression, regardless of whether the --compress option is specified. By default, compression is disabled. +For the archive-push command, the pglz compression algorithm is not supported. + + --compress-level=compression_level + Default: 1 +Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can only be used together with --compress or --compress-algorithm options. + +**Replica Options** +This section describes the options related to taking a backup from standby. +>NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. + + --master-db=dbname + Default: postgres, the default Postgres Pro database. +Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the set-config command. + + --master-host=host +Deprecated. Specifies the host name of the system on which the master server is running. + + --master-port=port + Default: 5432, the Postgres Pro default port. +Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. + + --master-user=username + Default: postgres, the Postgres Pro default user name. +Deprecated. User name to connect as. + + --replica-timeout=timeout + Default: 300 sec +Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the set-config command. + +**Archiving Options** + + --wal-file-path=wal_file_path %p +Provides the path to the WAL file in archive_command and restore_command used by pg_probackup. The %p variable is required for correct processing. + + --wal-file-name=wal_file_name %f +Provides the name of the WAL file in archive_command and restore_command used by pg_probackup. The %f variable is required for correct processing. + + --overwrite +Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. + +**checkdb Options** + + --amcheck +Performs logical verification of indexes for the specified Postgres Pro instance if no corruption was found while checking data files. Optionally, you can skip validation of data files by specifying --skip-block-validation. You must have the amcheck extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. + + --heapallindexed +Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option starting from Postgres Pro 11. + +**Remote Backup Options** +This section describes the options related to running backup and restore operations remotely via SSH. These options can be used with add-instance, set-config, backup, restore, archive-push, and archive-get commands. + + --remote-proto +Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: +- ssh enables the remote backup mode via SSH. +- none explicitly disables the remote backup mode. + +You can omit this option if the --remote-host option is specified. + + --remote-host +Specifies the remote host IP address or hostname to connect to. + + --remote-port + Default: 22 +Specifies the remote host port to connect to. + + --remote-user +Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. + + --remote-path +Specifies pg_probackup installation directory on the remote system. + + --ssh-options +Specifies a string of SSH command-line options. + +### Usage + +- [Creating a Backup](#creating-a-backup) +- [Validating a Backup](#vaklidating-a-backup) +- [Restoring a Cluster](#restoting-a-cluster) +- [Using pg_probackup in the Remote Backup Mode](#using-pg_probackup-in-the-remote-backup-mode) +- [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) +- [Configuring pg_probackup](#configuring-pg_probackup) +- [Managing the Backup Catalog](#managing-the-backup-Catalog) + +##### Creating a Backup +To create a backup, run the following command: + + pg_probackup backup -B backupdir --instance instance_name -b backup_mode +where backup_mode can take one of the following values: +- FULL — creates a full backup that contains all the data files of the cluster to be restored. +- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. +- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. +- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. + +When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. + +If you have configured PTRACK backups, pg_probackup clears PTRACK bitmap of the relation being processed each time a full or an incremental backup is taken. Thus, the next incremental PTRACK backup contains only the pages that have changed since the previous backup. If a backup failed or was interrupted, some relations can already have their PTRACK forks cleared, so the next incremental backup will be incomplete. The same is true if ptrack_enable was turned off for some time. In this case, you must take a full backup before the next incremental PTRACK backup. + +To make a backup autonomous, add the --stream option to the above command. For example, to create a full autonomous backup, run: + + pg_probackup backup -B backupdir --instance instance_name -b FULL --stream --temp-slot + +The optional --temp-slot parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. + +Autonomous backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. To restore a cluster from an incremental autonomous backup, pg_probackup still requires the full backup and all the incremental backups it depends on. + +Even if you are using continuous archiving, autonomous backups can still be useful in the following cases: + + Autonomous backups can be restored on the server that has no file access to WAL archive. + + Autonomous backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. + +To back up a directory located outside of the data directory, use the optional --external-dirs parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include /etc/dir1/ and /etc/dir2/ directories into the full backup of your node instance that will be stored under the node_backup directory, run: + + pg_probackup backup -B node_backup --instance node -b FULL --external-dirs=/etc/dir1:/etc/dir2 + +pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the set-config command with the --external-dirs option. + +##### Validating Backups +When checksums are enabled for the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page. This guarantees that the backup is free of corrupted pages. Note that pg_probackup reads database files from disk and under heavy write load during backup it can show false positive checksum failures because of partial writes. + +Even if page checksums are disabled, pg_probackup calculates checksums for each file in a backup. Checksums are checked immediately after backup is taken and right before restore, to detect possible backup corruptions. If you would like to skip backup validation, you can specify the --no-validate option when running backup and restore commands. + +To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the validate command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. + +For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: + + pg_probackup validate -B backupdir --instance instance_name --recovery-target-xid=xid + +If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time and transaction ID up to which the recovery is possible. + +##### Restoring a Cluster +To restore the database cluster from a backup, run the restore command with at least the following options: + + pg_probackup restore -B backupdir --instance instance_name -i backup_id + +where: +- backupdir is the backup catalog that stores all backup files and meta information. +- instance_name is the backup instance for the cluster to be restored. +- backup_id specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. + +If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the --tablespace-mapping option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. + +When using the --tablespace-mapping option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: + + pg_probackup restore -B backupdir --instance instance_name -D datadir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir + +Once the restore command is complete, start the database service. If you are restoring an autonomous backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For archive backups, Postgres Pro replays all archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the recovery_target option with the restore command. Note that using the recovery-target=latest value with autonomous backups is only possible if the WAL archive is available at least starting from the time the autonomous backup was taken. + +>NOTE: By default, the restore command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the --no-validate option to skip validation and speed up the recovery. + +**Performing Point-in-Time (PITR) Recovery** +If you have enabled continuous WAL archiving before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using recovery target options with the restore command instead of the -i option shown above. pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the recovery process. + +- To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: + + pg_probackup restore -B backupdir --instance instance_name --recovery-target-time='2017-05-18 14:18:11' + +- To restore the cluster state up to a specific transaction ID, use the recovery-target-xid option: + + pg_probackup restore -B backupdir --instance instance_name --recovery-target-xid=687 +- If you know the exact LSN up to which you need to restore the data, use recovery-target-lsn: + + pg_probackup restore -B backupdir --instance instance_name --recovery-target-lsn=16/B374D848 + +By default, the recovery_target_inclusive parameter defines whether the recovery target is included into the backup. You can explicitly include or exclude the recovery target by adding the --recovery-target-inclusive=boolean option to the commands listed above. + +##### Using pg_probackup in the Remote Backup Mode + +pg_probackup supports the remote backup mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while Postgres Pro instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. + +The typical workflow is as follows: + + - On your local system, configure pg_probackup as explained in the section called “Installation and Setup”. For the add-instance and set-config commands, make sure to specify remote backup options that point to the remote server with the Postgres Pro instance. + +- If you would like to take archive backups, configure continuous WAL archiving on the remote system as explained in the section called “Setting up Archive Backups”. For the archive-push and archive-get commands, you must specify the remote backup options that point to your local system. + +- Run backup or restore commands with remote backup options on your local system. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. + +>NOTE: The remote backup mode is currently unavailable for Windows systems. + +##### Running pg_probackup on Parallel Threads + +Backup, recovery, and validation processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). + +Parallel execution is controlled by the -j/--threads command line option. For example, to create a backup using four parallel threads, run: + + pg_probackup backup -B backupdir --instance instance_name -b FULL -j 4 + +>NOTE: Parallel recovery applies only to copying data from the backup catalog to the data directory of the cluster. When Postgres Pro server is started, WAL records need to be replayed, and this cannot be done in parallel. + +##### Configuring pg_probackup + +Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the backups/instance_name directory to fine-tune pg_probackup configuration. + +Initially, pg_probackup.conf contains the following settings: +- PGDATA — the path to the data directory of the cluster to back up. +- system-identifier — the unique identifier of the Postgres Pro instance. + +Additionally, you can define connection, retention, logging, and replica settings using the set-config command: + + pg_probackup set-config -B backupdir --instance instance_name --external-dirs=external_directory_path [connection_options] [retention_options] [logging_options] [replica_options] + +To view the current settings, run the following command: + + pg_probackup show-config -B backupdir --instance instance_name + +You can override the settings defined in pg_probackup.conf when running the backup command. + +**Specifying Connection Settings** + +If you define connection settings in the pg_probackup.conf configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. + +If nothing is given, the default values are taken. pg_probackup tries to use local connection and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. + +**Configuring Backup Retention Policy** + +By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. + +To configure retention policy, set one or more of the following variables in the pg_probackup.conf file: +- retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. +- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. + +If both retention-redundancy and retention-window options are set, pg_probackup keeps backup copies that satisfy both conditions. For example, if you set retention-redundancy=2 and retention-window=7, pg_probackup cleans up the backup directory to keep only two full backup copies if at least one of them is older than seven days. + +To clean up the backup catalog in accordance with retention policy, run: + + pg_probackup delete -B backupdir --instance instance_name --expired + +pg_probackup deletes all backup copies that do not conform to the defined retention policy. + +If you would like to also remove the WAL files that are no longer required for any of the backups, add the --wal option: + + pg_probackup delete -B backupdir --instance instance_name --expired --wal + +Alternatively, you can use the --delete-expired and --delete-wal options together with the backup command to remove the outdated backup copies once the new backup is created. + +Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the --merge-expired option when running backup or delete commands. + +Suppose you have backed up the node instance in the node-backup directory, with the retention-window option is set to 7, and you have the following backups available on April 10, 2019: + +``` +BACKUP INSTANCE 'node' +=========================================================================================================================================== + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +=========================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 PTRACK STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK + node 10 P7XDHU 2019-04-02 05:27:59+03 PTRACK STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK + node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK +``` + +Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDHU and P7XDQV that are still required, so if you run the delete command with the --expired option, only the P7XDFT full backup will be removed. With the --merge-expired option, the P7XDJA backup is merged with the underlying P7XDHB and P7XDHU backups and becomes a full one, so there is no need to keep these expired backups anymore: + + pg_probackup delete -B node-backup --instance node --expired --merge-expired + pg_probackup show -B node-backup + +``` +BACKUP INSTANCE 'node' +============================================================================================================================================ + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +============================================================================================================================================ + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK +``` + +Note that the Time field for the merged backup displays the time required for the merge. + +##### Managing the Backup Catalog + +With pg_probackup, you can manage backups from the command line: + +- View available backups +- Merge backups +- Delete backups +- Viewing Backup Information + +To view the list of existing backups, run the command: + + pg_probackup show -B backupdir + +pg_probackup displays the list of all the available backups. For example: + +``` +BACKUP INSTANCE 'node' +============================================================================================================================================ + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +============================================================================================================================================ + node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK + node 10 P7XDHU 2018-04-29 05:27:59+03 PTRACK STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1 / 0 11s 39MB 0/F000028 0/F000198 OK + node 10 P7XDFT 2018-04-29 05:26:25+03 PTRACK STREAM 1 / 0 11s 40MB 0/D000028 0/D000198 OK +``` + +For each backup, the following information is provided: +- Instance — the instance name. +- Version — Postgres Pro version. +- ID — the backup identifier. +- Recovery time — the earliest moment for which you can restore the state of the database cluster. +- Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. +- WAL — the way of WAL log handling. Possible values: STREAM for autonomous backups and ARCHIVE for archive backups. +- Current/Parent TLI — current and parent timelines of the database cluster. +- Time — the time it took to perform the backup. +- Data — the size of the data files in this backup. This value does not include the size of WAL files. +- Start LSN — WAL log sequence number corresponding to the start of the backup process. +- Stop LSN — WAL log sequence number corresponding to the end of the backup process. +- Status — backup status. Possible values: + - OK — the backup is complete and valid. + - CORRUPT — some of the backup files are corrupted. + - DONE — the backup is complete, but was not validated. + - ERROR — the backup was aborted because of an unexpected error. + - RUNNING — the backup is in progress. + - MERGING — the backup is being merged. + - ORPHAN — the backup is invalid because one of its parent backups is corrupt. + - DELETING — the backup files are being deleted. + +You can restore the cluster from the backup only if the backup status is OK. + +To get more detailed information about the backup, run the show with the backup ID: + + pg_probackup show -B backupdir --instance instance_name -i backup_id + +The sample output is as follows: +>#Configuration +backup-mode = FULL +stream = false +#Compatibility +block-size = 8192 +wal-block-size = 8192 +checksum-version = 0 +#Result backup info +timelineid = 1 +start-lsn = 0/04000028 +stop-lsn = 0/040000f8 +start-time = '2017-05-16 12:57:29' +end-time = '2017-05-16 12:57:31' +recovery-xid = 597 +recovery-time = '2017-05-16 12:57:31' +data-bytes = 22288792 +status = OK + +**Merging Backups** + +As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: + + pg_probackup merge -B backupdir --instance instance_name -i backup_id + +This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes. + +Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the show command with the backup ID. If the merge is still in progress, the backup status is displayed as MERGING. You can restart the merge if it is interrupted. +Deleting Backups + +To delete a backup that is no longer required, run the following command: + + pg_probackup delete -B backupdir --instance instance_name -i backup_id + +This command will delete the backup with the specified backup_id, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. +In this case, the next PTRACK backup will be incomplete as some changes since the last retained backup will be lost. Either a full backup or an incremental PAGE backup (if all the necessary WAL files are still present in the archive) must be taken then. + +To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the --wal option: + + pg_probackup delete -B backupdir --instance instance_name --wal + +To delete backups that are expired according to the current retention policy, use the --expired option: + + pg_probackup delete -B backupdir --instance instance_name --expired + +Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the --merge-expired option when running this command: + + pg_probackup delete -B backupdir --instance instance_name --expired --merge-expired + +In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. + +Before merging or deleting backups, you can run the delete command with the dry-run option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. + +### Authors +Postgres Professional, Moscow, Russia. +### Credits +pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file From 4cbd8fbb40b92329ab5d751642a9eadaedcc8bf7 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 13 Jun 2019 17:34:15 +0300 Subject: [PATCH 0743/2107] Update Documentation.md --- Documentation.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Documentation.md b/Documentation.md index 8c2f125e1..6a8157192 100644 --- a/Documentation.md +++ b/Documentation.md @@ -5,20 +5,35 @@ ### Synopsis `pg_probackup init -B backupdir` + `pg_probackup add-instance -B backupdir -D datadir --instance instance_name` + `pg_probackup del-instance -B backupdir --instance instance_name` + `pg_probackup set-config -B backupdir --instance instance_name [option...]` + `pg_probackup show-config -B backupdir --instance instance_name [--format=format]` + `pg_probackup backup -B backupdir --instance instance_name -b backup_mode [option...]` + `pg_probackup merge -B backupdir --instance instance_name -i backup_id [option...]` + `pg_probackup restore -B backupdir --instance instance_name [option...]` + `pg_probackup validate -B backupdir [option...]` + `pg_probackup show -B backupdir [option...]` + `pg_probackup delete -B backupdir --instance instance_name { -i backup_id | --wal | --expired }` + `pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` + `pg_probackup archive-get -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f` + `pg_probackup checkdb -B backupdir --instance instance_name -D datadir [option...]` + `pg_probackup version` + `pg_probackup help [command]` * [Installation and Setup](#installation-and-setup) From a1f2f1f85aa02ab1a6270f556af86d7be4827e20 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 13 Jun 2019 17:49:02 +0300 Subject: [PATCH 0744/2107] Update Documentation.md --- Documentation.md | 88 ++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6a8157192..be3a4d64d 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1,6 +1,12 @@ # pg_probackup --- -**pg_probackup** is a utility to manage backup and recovery of Postgres Pro database clusters. It is designed to perform periodic backups of the Postgres Pro instance that enable you to restore the server in case of a failure. pg_probackup supports Postgres Pro 9.5 or higher. +**pg_probackup** is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. + +* [Synopsis](#synopsis) +* [Overview](#overview) +* [Installation and Setup](#installation-and-setup) +* [Command-Line Reference](#command-line-reference) +* [Usage](#usage) ### Synopsis @@ -36,16 +42,12 @@ `pg_probackup help [command]` -* [Installation and Setup](#installation-and-setup) -* [Command-Line Reference](#command-line-reference) -* [Usage](#usage) - ### Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: - Choosing between full and page-level incremental backups to speed up backup and recovery -- Implementing a single backup strategy for multi-server Postgres Pro clusters +- Implementing a single backup strategy for multi-server PostgreSQL clusters - Automatic data consistency checks and on-demand backup validation without actual data recovery - Managing backups in accordance with retention policy - Running backup, restore, and validation processes on multiple parallel threads @@ -53,7 +55,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Taking backups from a standby server to avoid extra load on the master server - Extended logging settings - Custom commands to simplify WAL log archiving -- Backing up files and directories located outside of Postgres Pro data directory, such as configuration or log files +- Backing up files and directories located outside of PostgreSQL data directory, such as configuration or log files To manage backup data, pg_probackup creates a *backup catalog*. This directory stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -62,7 +64,7 @@ Using pg_probackup, you can take full or incremental backups: - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. pg_probackup supports the following modes of incremental backups: - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Continuous archiving is not necessary for this mode to operate. Note that this mode can impose read-only I/O pressure equal to a full backup. - - PTRACK backup. In this mode, Postgres Pro tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. Regardless of the chosen backup type, all backups taken with pg_probackup support the following archiving strategies: - Autonomous backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. @@ -76,7 +78,7 @@ pg_probackup currently has the following limitations: ### Installation and Setup -The pg_probackup package is provided as part of the Postgres Pro distribution. Once you have pg_probackup installed, complete the following setup: +Once you have pg_probackup installed, complete the following setup: - Initialize the backup catalog. - Add a new backup instance to the backup catalog. @@ -87,13 +89,13 @@ pg_probackup stores all WAL and backup files in the corresponding subdirectories To initialize the backup catalog, run the following command: -`pg_probackup init -B backupdir` + pg_probackup init -B backupdir where backupdir is the backup catalog. If the backupdir already exists, it must be empty. Otherwise, pg_probackup returns an error. pg_probackup creates the backupdir backup catalog, with the following subdirectories: - *wal/* — directory for WAL files. -- *backups/* — directory for backup files. +- *backups/* — directory for backup files. Once the backup catalog is initialized, you can add a new backup instance. @@ -102,7 +104,8 @@ pg_probackup can store backups for multiple database clusters in a single backup To add a new backup instance, run the following command: -`pg_probackup add-instance -B backupdir -D datadir --instance instance_name [--external-dirs=external_directory_path] [remote_backup_options]` + pg_probackup add-instance -B backupdir -D datadir --instance instance_name + [--external-dirs=external_directory_path] [remote_backup_options] where: - *datadir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. @@ -137,15 +140,16 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are going to use autonomous or archive backup strategies, Postgres Pro cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the section called “PTRACK Backup” and the section called “Backup from Standby”. +Depending on whether you are going to use autonomous or archive backup strategies, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the section called “PTRACK Backup” and the section called “Backup from Standby”. ##### Setting up Autonomous Backups To set up the cluster for autonomous backups, complete the following steps: - Grant the REPLICATION privilege to the backup role: - `ALTER ROLE backup WITH REPLICATION;` + + ALTER ROLE backup WITH REPLICATION; - In the pg_hba.conf file, allow replication on behalf of the backup role. -- Modify the postgresql.conf configuration file of the Postgres Pro server, as follows: +- Modify the postgresql.conf configuration file of the PostgreSQL server, as follows: - Make sure the max_wal_senders parameter is set high enough to leave at least one session available for the backup process. - Set the *wal_level* parameter to be *replica* or higher. @@ -155,18 +159,18 @@ Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups ##### Setting up Archive Backups To set up the cluster for archive backups, complete the following steps: -- Configure the following parameters in postgresql.conf to enable continuous archiving on the Postgres Pro server: +- Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - Make sure the wal_level parameter is set to replica or higher. - Set the archive_mode parameter. If you are configuring backups on master, archive_mode must be set to on. To perform archiving on standby, set this parameter to always. - Set the archive_command variable, as follows: - ```archive_command = 'pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f'``` + archive_command = 'pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f where backupdir and instance_name refer to the already initialized backup catalog instance for this database cluster. Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server. If you are going to take backups from standby or use PTRACK backups, you must also complete additional setup, as explained in the section called “Backup from Standby” and the section called “PTRACK Backup”, respectively. ##### Backup from Standby -For Postgres Pro 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: +For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: - On the standby server, allow replication connections: - Set the max_wal_senders and hot_standby parameters in postgresql.conf. @@ -184,11 +188,10 @@ For Postgres Pro 9.6 or higher, pg_probackup can take backups from a standby ser If you are going to use PTRACK backups, complete the following additional steps: - In postgresql.conf, set ptrack_enable to on. - Grant the rights to execute ptrack functions to the backup role: - ``` - GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; - GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; - ``` - The backup role must have access to all the databases of the cluster. + + GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; + GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; + The *backup* role must have access to all the databases of the cluster. ### Command-Line Reference ##### Commands @@ -224,6 +227,7 @@ Deletes all backup and WAL files associated with the specified instance. Adds the specified connection, retention, logging or replica, and compression, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. **show-config** + pg_probackup show-config -B backupdir --instance instance_name [--format=plain|json] Displays the contents of the pg_probackup.conf configuration file located in the backupdir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. To edit pg_probackup.conf, use the set-config command. @@ -242,7 +246,7 @@ It is not allowed to edit pg_probackup.conf directly. [-j num_threads][--progress] [logging_options] [remote_backup_options] -Creates a backup copy of the Postgres Pro instance. The backup_mode option specifies the backup mode to use. For details, see the section called “Creating a Backup”. +Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. For details, see the section called “Creating a Backup”. **merge** @@ -266,7 +270,7 @@ Merges the specified incremental backup to its parent full backup, together with [logging_options] [remote_backup_options] -Restores the Postgres Pro instance from a backup copy located in the backupdir backup catalog. If you specify a recovery target option, pg_probackup restores the database cluster up to the corresponding recovery target. Otherwise, the most recent backup is used. +Restores the PostgreSQL instance from a backup copy located in the backupdir backup catalog. If you specify a recovery target option, pg_probackup restores the database cluster up to the corresponding recovery target. Otherwise, the most recent backup is used. **validate** @@ -306,7 +310,7 @@ Deletes backup or WAL files of the specified backup instance from the backupdir [--compress][--compress-algorithm=compression_algorithm][--compress-level=compression_level] [--overwrite] [remote_backup_options] -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in Postgres Pro logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. +Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. You can set archive-push as archive_command in postgresql.conf to perform archive backups. @@ -324,7 +328,7 @@ Moves WAL files from the corresponding subdirectory of the backup catalog to the [--amcheck [--heapallindexed] [--skip-block-validation]] [--progress] [-j num_threads] -Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified Postgres Pro instance using the amcheck extension. +Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified PostgreSQL instance using the amcheck extension. **version** @@ -399,7 +403,7 @@ Makes an autonomous backup that includes all the necessary WAL files by streamin Specifies the replication slot for WAL streaming. This option can only be used together with the --stream option. --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up Postgres Pro instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. --backup-pg-log Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. @@ -580,18 +584,18 @@ This section describes the options related to taking a backup from standby. >NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. --master-db=dbname - Default: postgres, the default Postgres Pro database. + Default: postgres, the default PostgreSQL database. Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the set-config command. --master-host=host Deprecated. Specifies the host name of the system on which the master server is running. --master-port=port - Default: 5432, the Postgres Pro default port. + Default: 5432, the PostgreSQL default port. Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. --master-user=username - Default: postgres, the Postgres Pro default user name. + Default: postgres, the PostgreSQL default user name. Deprecated. User name to connect as. --replica-timeout=timeout @@ -612,10 +616,10 @@ Overwrites archived WAL file. Use this option together with the archive-push com **checkdb Options** --amcheck -Performs logical verification of indexes for the specified Postgres Pro instance if no corruption was found while checking data files. Optionally, you can skip validation of data files by specifying --skip-block-validation. You must have the amcheck extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. +Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. Optionally, you can skip validation of data files by specifying --skip-block-validation. You must have the amcheck extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option starting from Postgres Pro 11. +Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option starting from PostgreSQL 11. **Remote Backup Options** This section describes the options related to running backup and restore operations remotely via SSH. These options can be used with add-instance, set-config, backup, restore, archive-push, and archive-get commands. @@ -716,7 +720,7 @@ When using the --tablespace-mapping option, you must provide absolute paths to t pg_probackup restore -B backupdir --instance instance_name -D datadir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir -Once the restore command is complete, start the database service. If you are restoring an autonomous backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For archive backups, Postgres Pro replays all archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the recovery_target option with the restore command. Note that using the recovery-target=latest value with autonomous backups is only possible if the WAL archive is available at least starting from the time the autonomous backup was taken. +Once the restore command is complete, start the database service. If you are restoring an autonomous backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For archive backups, PostgreSQL replays all archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the recovery_target option with the restore command. Note that using the recovery-target=latest value with autonomous backups is only possible if the WAL archive is available at least starting from the time the autonomous backup was taken. >NOTE: By default, the restore command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the --no-validate option to skip validation and speed up the recovery. @@ -738,11 +742,11 @@ By default, the recovery_target_inclusive parameter defines whether the recovery ##### Using pg_probackup in the Remote Backup Mode -pg_probackup supports the remote backup mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while Postgres Pro instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. +pg_probackup supports the remote backup mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. The typical workflow is as follows: - - On your local system, configure pg_probackup as explained in the section called “Installation and Setup”. For the add-instance and set-config commands, make sure to specify remote backup options that point to the remote server with the Postgres Pro instance. + - On your local system, configure pg_probackup as explained in the section called “Installation and Setup”. For the add-instance and set-config commands, make sure to specify remote backup options that point to the remote server with the PostgreSQL instance. - If you would like to take archive backups, configure continuous WAL archiving on the remote system as explained in the section called “Setting up Archive Backups”. For the archive-push and archive-get commands, you must specify the remote backup options that point to your local system. @@ -758,7 +762,7 @@ Parallel execution is controlled by the -j/--threads command line option. For ex pg_probackup backup -B backupdir --instance instance_name -b FULL -j 4 ->NOTE: Parallel recovery applies only to copying data from the backup catalog to the data directory of the cluster. When Postgres Pro server is started, WAL records need to be replayed, and this cannot be done in parallel. +>NOTE: Parallel recovery applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. ##### Configuring pg_probackup @@ -766,7 +770,7 @@ Once the backup catalog is initialized and a new backup instance is added, you c Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. -- system-identifier — the unique identifier of the Postgres Pro instance. +- system-identifier — the unique identifier of the PostgreSQL instance. Additionally, you can define connection, retention, logging, and replica settings using the set-config command: @@ -869,7 +873,7 @@ BACKUP INSTANCE 'node' For each backup, the following information is provided: - Instance — the instance name. -- Version — Postgres Pro version. +- Version — PostgreSQL version. - ID — the backup identifier. - Recovery time — the earliest moment for which you can restore the state of the database cluster. - Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. @@ -896,7 +900,8 @@ To get more detailed information about the backup, run the show with the backup pg_probackup show -B backupdir --instance instance_name -i backup_id The sample output is as follows: ->#Configuration +``` +#Configuration backup-mode = FULL stream = false #Compatibility @@ -913,6 +918,7 @@ recovery-xid = 597 recovery-time = '2017-05-16 12:57:31' data-bytes = 22288792 status = OK +``` **Merging Backups** @@ -949,6 +955,6 @@ In this case, pg_probackup searches for the oldest incremental backup that satis Before merging or deleting backups, you can run the delete command with the dry-run option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. ### Authors -Postgres Professional, Moscow, Russia. +PostgreSQLfessional, Moscow, Russia. ### Credits pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file From 5a739fa6e284748a76b4506049a69a161e7a23b4 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 13 Jun 2019 17:57:39 +0300 Subject: [PATCH 0745/2107] Update Documentation.md --- Documentation.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index be3a4d64d..a1517ec3a 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1,7 +1,9 @@ # pg_probackup ---- + **pg_probackup** is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. +Current version - 2.1.3 + * [Synopsis](#synopsis) * [Overview](#overview) * [Installation and Setup](#installation-and-setup) From 6e0c213029bfa10741839f4c86fb363ef321c459 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 14 Jun 2019 01:24:22 +0300 Subject: [PATCH 0746/2107] change elevel of "Create directory" message to VERBOSE --- src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index 426d16963..dc77e2510 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1061,7 +1061,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba } /* This is not symlink, create directory */ - elog(INFO, "Create directory \"%s\"", dir->rel_path); + elog(VERBOSE, "Create directory \"%s\"", dir->rel_path); join_path_components(to_path, data_dir, dir->rel_path); fio_mkdir(to_path, dir->mode, location); From 19c30a89978e9d478f3f6e17886d8673fcc9bbb9 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 14 Jun 2019 11:58:33 +0300 Subject: [PATCH 0747/2107] code cleanup: remove unneeded backuped variable --- src/backup.c | 11 ++++++----- src/catalog.c | 7 +------ src/pg_probackup.h | 3 --- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0787be8e4..b568586eb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -429,7 +429,6 @@ do_backup_instance(PGconn *backup_conn) } else join_path_components(dirpath, database_path, dir_name); - file->backuped = true; fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } @@ -443,6 +442,11 @@ do_backup_instance(PGconn *backup_conn) if (prev_backup_filelist) parray_qsort(prev_backup_filelist, pgFileComparePathWithExternal); + /* write initial backup_content.control file and update backup.control */ + write_backup_filelist(¤t, backup_files_list, + instance_config.pgdata, external_dirs); + write_backup(¤t); + /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -1797,7 +1801,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; - file->backuped = true; free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); @@ -1846,7 +1849,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; - file->backuped = true; } free(file->path); file->path = strdup(PG_TABLESPACE_MAP_FILE); @@ -1995,7 +1997,7 @@ backup_files(void *arg) { prev_time = time(NULL); - write_backup_filelist(¤t, arguments->files_list, instance_config.pgdata, + write_backup_filelist(¤t, arguments->files_list, arguments->from_root, arguments->external_dirs); /* update backup control file to update size info */ write_backup(¤t); @@ -2134,7 +2136,6 @@ backup_files(void *arg) } } - file->backuped = true; elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", file->path, file->write_size); } diff --git a/src/catalog.c b/src/catalog.c index e1ff0168c..8c318b6ce 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -660,16 +660,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, i++; - /* if file is not backuped yet, do not add it to the list */ - /* TODO check that we correctly handle files which disappeared during backups */ - if (!file->backuped) - continue; - if (S_ISDIR(file->mode)) backup_size_on_disk += 4096; /* Count the amount of the data actually copied */ - if (S_ISREG(file->mode)) + if (S_ISREG(file->mode) && file->write_size > 0) backup_size_on_disk += file->write_size; if (file->external_dir_num && external_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 528b068d1..53ae515e3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -146,9 +146,6 @@ typedef struct pgFile datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, * i.e. datafiles without _ptrack */ - - /* state during bakup */ - bool backuped; /* is file already completely copied into destination backup? */ } pgFile; /* Special values of datapagemap_t bitmapsize */ From d28ea728c4731e4f8177ecbf85a3da3c3b8f1f9a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 14 Jun 2019 13:12:34 +0300 Subject: [PATCH 0748/2107] fix missing streamed WAL segments --- src/catalog.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 8c318b6ce..a9bed1e06 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -654,7 +654,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, while(i < parray_num(files)) { pgFile *file = (pgFile *) parray_get(files, i); - char *path = file->path; + char *path = file->path; /* for streamed WAL files */ char line[BLCKSZ]; int len = 0; @@ -667,13 +667,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (S_ISREG(file->mode) && file->write_size > 0) backup_size_on_disk += file->write_size; - if (file->external_dir_num && external_list) - { - path = GetRelativePath(path, parray_get(external_list, - file->external_dir_num - 1)); - } - else - path = file->rel_path; + /* for files from PGDATA and external files use rel_path + * streamed WAL files has rel_path relative not to "database/" + * but to "database/pg_wal", so for them use path. + */ + if ((root && strstr(path, root) == path) || + (file->external_dir_num && external_list)) + path = file->rel_path; len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " From 1590e2d259d20db1be53d4458b734800088239e5 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Tue, 15 Jan 2019 14:05:40 +0300 Subject: [PATCH 0749/2107] fix show-config in json format: add quotes for all values --- src/configure.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configure.c b/src/configure.c index 7c581f1e0..f9d92fc1f 100644 --- a/src/configure.c +++ b/src/configure.c @@ -450,6 +450,6 @@ show_configure_json(ConfigOption *opt) return; json_add_value(&show_buf, opt->lname, value, json_level, - opt->type == 's' || opt->flags & OPTION_UNIT); + true); pfree(value); } From ad582b45d3bba85afa4ca206b3c3fa6923dad473 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Jun 2019 18:54:28 +0300 Subject: [PATCH 0750/2107] tests: fix permission issue in external module --- tests/external.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/external.py b/tests/external.py index 9d5c77ee4..1e383c709 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1742,6 +1742,7 @@ def test_external_dir_contain_symlink_on_file(self): # create symlink to directory in external directory src_file = os.path.join(symlinked_dir, 'postgresql.conf') os.mkdir(external_dir) + os.chmod(external_dir, 0700) os.symlink(src_file, file_in_external_dir) # FULL backup with external directories @@ -2119,6 +2120,7 @@ def test_restore_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) + os.chmod(external_dir, 0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2184,6 +2186,7 @@ def test_merge_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) + os.chmod(external_dir, 0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2252,10 +2255,12 @@ def test_restore_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) + os.chmod(external_dir_1, 0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) + os.chmod(external_dir_2, 0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() @@ -2332,10 +2337,12 @@ def test_merge_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) + os.chmod(external_dir_1, 0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) + os.chmod(external_dir_2, 0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() From 93e8a995d80f1832c1f08396a27a4191155d2218 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Jun 2019 19:27:44 +0300 Subject: [PATCH 0751/2107] minor improvement: be more paranoid about illegal values in file->size during merge and validation --- src/merge.c | 42 ++++++++++++++++++++++++++---------------- src/validate.c | 37 ++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/src/merge.c b/src/merge.c index a8d63aa79..e53011381 100644 --- a/src/merge.c +++ b/src/merge.c @@ -468,9 +468,6 @@ merge_files(void *arg) char from_file_path[MAXPGPATH]; char *prev_file_path; - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - /* check for interrupt */ if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during merging backups"); @@ -479,6 +476,9 @@ merge_files(void *arg) if (S_ISDIR(file->mode)) continue; + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); @@ -491,20 +491,28 @@ merge_files(void *arg) /* * Skip files which haven't changed since previous backup. But in case - * of DELTA backup we should consider n_blocks to truncate the target - * backup. + * of DELTA backup we must truncate the target file to n_blocks. + * Unless it is a non data file, in this case truncation is not needed. */ - if (file->write_size == BYTES_INVALID && file->n_blocks == BLOCKNUM_INVALID) + if (file->write_size == BYTES_INVALID) { - elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", - file->path); - - /* - * If the file wasn't changed in PAGE backup, retreive its - * write_size from previous FULL backup. - */ - if (to_file) + /* sanity */ + if (!to_file) + elog(ERROR, "The file \"%s\" is missing in FULL backup %s", + file->rel_path, base36enc(to_backup->start_time)); + + /* for not changed files of all types in PAGE and PTRACK */ + if (from_backup->backup_mode != BACKUP_MODE_DIFF_DELTA || + /* and not changed non-data files in DELTA */ + (!file->is_datafile || file->is_cfs)) { + elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", + file->path); + + /* + * If the file wasn't changed, retreive its + * write_size and compression algorihtm from previous FULL backup. + */ file->compress_alg = to_file->compress_alg; file->write_size = to_file->write_size; @@ -516,9 +524,9 @@ merge_files(void *arg) /* Otherwise just get it from the previous file */ else file->crc = to_file->crc; - } - continue; + continue; + } } /* We need to make full path, file object has relative path */ @@ -667,6 +675,8 @@ merge_files(void *arg) if (file->write_size != BYTES_INVALID) elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); + else + elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", BYTES_INVALID); /* Restore relative path */ file->path = prev_file_path; diff --git a/src/validate.c b/src/validate.c index 0185a7dbb..d8d59199b 100644 --- a/src/validate.c +++ b/src/validate.c @@ -24,11 +24,12 @@ static bool skipped_due_to_lock = false; typedef struct { const char *base_path; - parray *files; + parray *files; bool corrupted; - XLogRecPtr stop_lsn; + XLogRecPtr stop_lsn; uint32 checksum_version; uint32 backup_version; + BackupMode backup_mode; /* * Return value from the thread. @@ -125,6 +126,7 @@ pgBackupValidate(pgBackup *backup) arg->base_path = base_path; arg->files = files; arg->corrupted = false; + arg->backup_mode = backup->backup_mode; arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; arg->backup_version = parse_program_version(backup->program_version); @@ -184,33 +186,46 @@ pgBackupValidateFiles(void *arg) struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during validate"); /* Validate only regular files */ if (!S_ISREG(file->mode)) continue; - /* - * Skip files which has no data, because they - * haven't changed between backups. - */ - if (file->write_size == BYTES_INVALID) - continue; /* * Currently we don't compute checksums for * cfs_compressed data files, so skip them. + * TODO: investigate */ if (file->is_cfs) continue; + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, num_files, file->path); + /* + * Skip files which has no data, because they + * haven't changed between backups. + */ + if (file->write_size == BYTES_INVALID) + { + if (arguments->backup_mode == BACKUP_MODE_FULL) + { + /* It is illegal for file in FULL backup to have BYTES_INVALID */ + elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.", + file->path); + arguments->corrupted = true; + break; + } + else + continue; + } + if (stat(file->path, &st) == -1) { if (errno == ENOENT) From 5c751cbac7507f961540e4807030cc476e370cdd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Jun 2019 20:51:10 +0300 Subject: [PATCH 0752/2107] tests: fix tests.validate.ValidateTest.test_validate_corrupt_tablespace_map --- tests/validate.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/validate.py b/tests/validate.py index 4b5f26ea2..0e9fd76d6 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3429,7 +3429,19 @@ def test_validate_corrupt_tablespace_map(self): f.flush() f.close - self.validate_pb(backup_dir, 'node', backup_id=backup_id) + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Invalid CRC of backup file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) # validate empty backup list # page from future during validate From e495a9a76d5aad67f8b0ba2649a39a60c3727524 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Jun 2019 20:57:08 +0300 Subject: [PATCH 0753/2107] minor improvement: parent TLI in FULL backups now always equal to zero --- src/show.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/show.c b/src/show.c index 7fe163f93..681848f2a 100644 --- a/src/show.c +++ b/src/show.c @@ -380,7 +380,8 @@ show_instance_plain(parray *backup_list, bool show_name) /* Current/Parent TLI */ snprintf(row->tli, lengthof(row->tli), "%u / %u", - backup->tli, get_parent_tli(backup->tli)); + backup->tli, + backup->backup_mode == BACKUP_MODE_FULL ? 0 : get_parent_tli(backup->tli)); widths[cur] = Max(widths[cur], strlen(row->tli)); cur++; @@ -570,7 +571,7 @@ show_instance_json(parray *backup_list) json_add_value(buf, "from-replica", backup->from_replica ? "true" : "false", json_level, - false); + true); json_add_key(buf, "block-size", json_level); appendPQExpBuffer(buf, "%u", backup->block_size); @@ -590,7 +591,12 @@ show_instance_json(parray *backup_list) appendPQExpBuffer(buf, "%d", backup->tli); json_add_key(buf, "parent-tli", json_level); - parent_tli = get_parent_tli(backup->tli); + + /* Only incremental backup can have Parent TLI */ + if (backup->backup_mode == BACKUP_MODE_FULL) + parent_tli = 0; + else + parent_tli = get_parent_tli(backup->tli); appendPQExpBuffer(buf, "%u", parent_tli); snprintf(lsn, lengthof(lsn), "%X/%X", From 49163b381082d734e6002688bd14b8c1b7c5420c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 19 Jun 2019 12:31:41 +0300 Subject: [PATCH 0754/2107] [Issue #83] data corruption as a result of failed merge --- src/merge.c | 61 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/merge.c b/src/merge.c index e53011381..20969781b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -160,6 +160,7 @@ do_merge(time_t backup_id) /* * Merge two backups data files using threads. + * - to_backup - FULL, from_backup - incremental. * - move instance files from from_backup to to_backup * - remove unnecessary directories and files from to_backup * - update metadata of from_backup, it becames FULL backup @@ -355,9 +356,12 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) + { to_backup->data_bytes += 4096; + continue; + } /* Count the amount of the data actually copied */ - else if (S_ISREG(file->mode)) + if (file->write_size > 0) to_backup->data_bytes += file->write_size; } /* compute size of wal files of this backup stored in the archive */ @@ -381,7 +385,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Delete files which are not in from_backup file list. */ - parray_qsort(files, pgFileComparePathDesc); + parray_qsort(files, pgFileComparePathWithExternalDesc); for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); @@ -394,7 +398,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) continue; } - if (parray_bsearch(files, file, pgFileComparePathDesc) == NULL) + if (parray_bsearch(files, file, pgFileComparePathWithExternalDesc) == NULL) { char to_file_path[MAXPGPATH]; char *prev_path; @@ -507,7 +511,7 @@ merge_files(void *arg) (!file->is_datafile || file->is_cfs)) { elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", - file->path); + file->rel_path); /* * If the file wasn't changed, retreive its @@ -529,6 +533,10 @@ merge_files(void *arg) } } + /* TODO optimization: file from incremental backup has size 0, then + * just truncate the file from FULL backup + */ + /* We need to make full path, file object has relative path */ if (file->external_dir_num) { @@ -560,46 +568,51 @@ merge_files(void *arg) if (to_backup->compress_alg == PGLZ_COMPRESS || to_backup->compress_alg == ZLIB_COMPRESS) { + char merge_to_file_path[MAXPGPATH]; char tmp_file_path[MAXPGPATH]; char *prev_path; + snprintf(merge_to_file_path, MAXPGPATH, "%s_merge", to_file_path); snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_file_path); /* Start the magic */ /* * Merge files: - * - if target file exists restore and decompress it to the temp - * path + * - if to_file in FULL backup exists, restore and decompress it to to_file_merge * - decompress source file if necessary and merge it with the - * target decompressed file - * - compress result file + * target decompressed file in to_file_merge. + * - compress result file to to_file_tmp + * - rename to_file_tmp to to_file */ /* - * We need to decompress target file if it exists. + * We need to decompress target file in FULL backup if it exists. */ if (to_file) { elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", - tmp_file_path); + merge_to_file_path); /* - * file->path points to the file in from_root directory. But we - * need the file in directory to_root. + * file->path is relative, to_file_path - is absolute. + * Substitute them. */ prev_path = to_file->path; to_file->path = to_file_path; /* Decompress target file into temporary one */ - restore_data_file(tmp_file_path, to_file, false, false, + restore_data_file(merge_to_file_path, to_file, false, false, parse_program_version(to_backup->program_version)); to_file->path = prev_path; } else elog(VERBOSE, "Restore source file into the temporary path \"%s\"", - tmp_file_path); + merge_to_file_path); + + /* TODO: Optimize merge of new files */ + /* Merge source file with target file */ - restore_data_file(tmp_file_path, file, + restore_data_file(merge_to_file_path, file, from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, false, parse_program_version(from_backup->program_version)); @@ -609,12 +622,12 @@ merge_files(void *arg) /* Again we need to change path */ prev_path = file->path; - file->path = tmp_file_path; + file->path = merge_to_file_path; /* backup_data_file() requires file size to calculate nblocks */ file->size = pgFileSize(file->path); /* Now we can compress the file */ backup_data_file(NULL, /* We shouldn't need 'arguments' here */ - to_file_path, file, + tmp_file_path, file, to_backup->start_lsn, to_backup->backup_mode, to_backup->compress_alg, @@ -623,10 +636,15 @@ merge_files(void *arg) file->path = prev_path; - /* We can remove temporary file now */ - if (unlink(tmp_file_path)) + /* rename temp file */ + if (rename(tmp_file_path, to_file_path) == -1) + elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", + file->path, tmp_file_path, strerror(errno)); + + /* We can remove temporary file */ + if (unlink(merge_to_file_path)) elog(ERROR, "Could not remove temporary file \"%s\": %s", - tmp_file_path, strerror(errno)); + merge_to_file_path, strerror(errno)); } /* * Otherwise merging algorithm is simpler. @@ -676,7 +694,8 @@ merge_files(void *arg) elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); else - elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", BYTES_INVALID); + elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", + file->path, BYTES_INVALID); /* Restore relative path */ file->path = prev_file_path; From 2e20bd28fe4dc2fabf757fb71621ebda79c8408a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 19 Jun 2019 15:31:52 +0300 Subject: [PATCH 0755/2107] minor fix: wrong order of arguments in elog() messages about unexpected file checksumm and file size --- src/data.c | 2 +- src/validate.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index 22f0e7b4b..d243294eb 100644 --- a/src/data.c +++ b/src/data.c @@ -1400,7 +1400,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", - file->path, file->crc, crc); + file->path, crc, file->crc); is_valid = false; } diff --git a/src/validate.c b/src/validate.c index d8d59199b..9a7af09dc 100644 --- a/src/validate.c +++ b/src/validate.c @@ -240,7 +240,7 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file->path, file->write_size, (unsigned long) st.st_size); + file->path, (unsigned long) st.st_size, file->write_size); arguments->corrupted = true; break; } @@ -276,7 +276,7 @@ pgBackupValidateFiles(void *arg) if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, file->crc, crc); + file->path, crc, file->crc); arguments->corrupted = true; } } From 04d396dc7c92637c9c68f4ce10857efd3a2d505c Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 19 Jun 2019 17:37:35 +0300 Subject: [PATCH 0756/2107] Fix error message during files merging --- src/merge.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index e53011381..2514019dd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -676,7 +676,8 @@ merge_files(void *arg) elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", file->path, file->write_size); else - elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", BYTES_INVALID); + elog(ERROR, "Merge of file \"%s\" failed. Invalid size: " INT64_FORMAT " bytes", + file->path, file->write_size); /* Restore relative path */ file->path = prev_file_path; From 68d6b3b3f06a602c378610e15c812a393630590d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 19 Jun 2019 19:43:43 +0300 Subject: [PATCH 0757/2107] Correctly handle spaces in path for remote backup --- src/utils/remote.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 2f75e9d33..bf0bac109 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -76,6 +76,10 @@ void launch_ssh(char* argv[]) } #endif +static bool needs_quotes(char const* path) +{ + return strchr(path, ' ') != NULL; +} bool launch_agent(void) { @@ -137,14 +141,25 @@ bool launch_agent(void) probackup = sep + 1; } } - snprintf(cmd, sizeof(cmd), "%s\\%s agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + if (needs_quotes(instance_config.remote.path) || needs_quotes(pg_probackup)) + snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent %s", + instance_config.remote.path, probackup, PROGRAM_VERSION); + else + snprintf(cmd, sizeof(cmd), "%s\\%s agent %s", + instance_config.remote.path, probackup, PROGRAM_VERSION); #else - snprintf(cmd, sizeof(cmd), "%s/%s agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + if (needs_quotes(instance_config.remote.path) || needs_quotes(pg_probackup)) + snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent %s", + instance_config.remote.path, probackup, PROGRAM_VERSION); + else + snprintf(cmd, sizeof(cmd), "%s/%s agent %s", + instance_config.remote.path, probackup, PROGRAM_VERSION); #endif } else { - snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); + if (needs_quotes(pg_probackup)) + snprintf(cmd, sizeof(cmd), "\"%s\" agent %s", pg_probackup, PROGRAM_VERSION); + else + snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); } #ifdef WIN32 From 66ad28201e394b054c7cb4a9b99645672c2dab50 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 06:54:11 +0300 Subject: [PATCH 0758/2107] tests: for ".partial" WAL file --- tests/archive.py | 191 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 465151d63..7b9d2fa8e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -25,8 +25,8 @@ def test_pgpro434_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} - ) + 'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -411,6 +411,193 @@ def test_arhive_push_file_exists_overwrite(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_arhive_push_partial_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = filename + '.gz' + '.partial' + file = os.path.join(wals_dir, filename) + else: + filename = filename + '.partial' + file = os.path.join(wals_dir, filename) + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + self.switch_wal_segment(node) + sleep(15) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertIn( + 'Reusing stale destination temporary WAL file', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_push_partial_file_exists(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.safe_psql( + "postgres", + "create table t1()") + self.switch_wal_segment(node) + + node.safe_psql( + "postgres", + "create table t2()") + + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = filename_orig + '.gz' + '.partial' + file = os.path.join(wals_dir, filename) + else: + filename = filename_orig + '.partial' + file = os.path.join(wals_dir, filename) + + with open(file, 'a') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + self.switch_wal_segment(node) + sleep(15) + + # check that segment is archived + if self.archive_compress: + filename_orig = filename_orig + '.gz' + + file = os.path.join(wals_dir, filename_orig) + + self.assertTrue(os.path.isfile(file)) + + # log_file = os.path.join(node.logs_dir, 'postgresql.log') + # with open(log_file, 'r') as f: + # log_content = f.read() + # self.assertIn( + # 'Reusing stale destination temporary WAL file', + # log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_push_partial_file_exists_not_stale(self): + """Archive-push if file exists""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.safe_psql( + "postgres", + "create table t1()") + self.switch_wal_segment(node) + + node.safe_psql( + "postgres", + "create table t2()") + + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = filename_orig + '.gz' + '.partial' + file = os.path.join(wals_dir, filename) + else: + filename = filename_orig + '.partial' + file = os.path.join(wals_dir, filename) + + with open(file, 'a') as f: + f.write(b"blahblah") + f.flush() + f.close() + + self.switch_wal_segment(node) + sleep(4) + + with open(file, 'a') as f: + f.write(b"blahblahblahblah") + f.flush() + f.close() + + sleep(10) + + # check that segment is NOT archived + if self.archive_compress: + filename_orig = filename_orig + '.gz' + + file = os.path.join(wals_dir, filename_orig) + + self.assertFalse(os.path.isfile(file)) + + # log_file = os.path.join(node.logs_dir, 'postgresql.log') + # with open(log_file, 'r') as f: + # log_content = f.read() + # self.assertIn( + # 'is not stale', + # log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): From e163c4504c8255111f3c0f8d7db31b5b31dd1e77 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Thu, 20 Jun 2019 11:48:06 +0300 Subject: [PATCH 0759/2107] Rename dubious pg_probackup variable into PROGRAM_NAME_FULL --- src/pg_probackup.c | 8 ++++---- src/pg_probackup.h | 4 ++-- src/utils/remote.c | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2cf6e0edd..b32f72af6 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -20,7 +20,9 @@ #include "utils/thread.h" #include -const char *PROGRAM_NAME = NULL; +const char *PROGRAM_NAME = NULL; /* PROGRAM_NAME_FULL without .exe suffix + * if any */ +const char *PROGRAM_NAME_FULL = NULL; const char *PROGRAM_FULL_PATH = NULL; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; @@ -45,8 +47,6 @@ typedef enum ProbackupSubcmd } ProbackupSubcmd; -char *pg_probackup; /* Program name (argv[0]) */ - /* directory options */ char *backup_path = NULL; /* @@ -235,7 +235,7 @@ main(int argc, char *argv[]) struct stat stat_buf; int rc; - pg_probackup = argv[0]; + PROGRAM_NAME_FULL = argv[0]; /* Initialize current backup */ pgBackupInit(¤t); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 53ae515e3..420028291 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -36,6 +36,7 @@ /* pgut client variables and full path */ extern const char *PROGRAM_NAME; +extern const char *PROGRAM_NAME_FULL; extern const char *PROGRAM_FULL_PATH; extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; @@ -412,7 +413,6 @@ typedef struct BackupPageHeader #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) /* directory options */ -extern char *pg_probackup; extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; @@ -472,7 +472,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, bool no_validate); -extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, +extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); diff --git a/src/utils/remote.c b/src/utils/remote.c index bf0bac109..681d8fb87 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -91,7 +91,7 @@ bool launch_agent(void) ssh_argc = 0; #ifdef WIN32 - ssh_argv[ssh_argc++] = pg_probackup; + ssh_argv[ssh_argc++] = PROGRAM_NAME_FULL; ssh_argv[ssh_argc++] = "ssh"; ssh_argc += 2; /* reserve space for pipe descriptors */ #endif @@ -129,7 +129,7 @@ bool launch_agent(void) if (instance_config.remote.path) { - char const* probackup = pg_probackup; + char const* probackup = PROGRAM_NAME_FULL; char* sep = strrchr(probackup, '/'); if (sep != NULL) { probackup = sep + 1; @@ -141,14 +141,14 @@ bool launch_agent(void) probackup = sep + 1; } } - if (needs_quotes(instance_config.remote.path) || needs_quotes(pg_probackup)) + if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent %s", instance_config.remote.path, probackup, PROGRAM_VERSION); else snprintf(cmd, sizeof(cmd), "%s\\%s agent %s", instance_config.remote.path, probackup, PROGRAM_VERSION); #else - if (needs_quotes(instance_config.remote.path) || needs_quotes(pg_probackup)) + if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent %s", instance_config.remote.path, probackup, PROGRAM_VERSION); else @@ -156,10 +156,10 @@ bool launch_agent(void) instance_config.remote.path, probackup, PROGRAM_VERSION); #endif } else { - if (needs_quotes(pg_probackup)) - snprintf(cmd, sizeof(cmd), "\"%s\" agent %s", pg_probackup, PROGRAM_VERSION); + if (needs_quotes(PROGRAM_NAME_FULL)) + snprintf(cmd, sizeof(cmd), "\"%s\" agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION); else - snprintf(cmd, sizeof(cmd), "%s agent %s", pg_probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "%s agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION); } #ifdef WIN32 From f6ec367700805f33b997da64c55f1a9aaea485b7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 16:38:29 +0300 Subject: [PATCH 0760/2107] tests: fixes for ".partial" tests --- tests/archive.py | 74 ++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 59 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 7b9d2fa8e..81d84b2c8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -411,61 +411,9 @@ def test_arhive_push_file_exists_overwrite(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_arhive_push_partial_file_exists(self): - """Archive-push if file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") - - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = filename + '.gz' + '.partial' - file = os.path.join(wals_dir, filename) - else: - filename = filename + '.partial' - file = os.path.join(wals_dir, filename) - - with open(file, 'a') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close() - - self.switch_wal_segment(node) - sleep(15) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertIn( - 'Reusing stale destination temporary WAL file', - log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): - """Archive-push if file exists""" + """Archive-push if stale .partial file exists""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -479,20 +427,23 @@ def test_archive_push_partial_file_exists(self): node.slow_start() + # this backup is needed only for validation to xid + self.backup_node(backup_dir, 'node', node) + node.safe_psql( "postgres", - "create table t1()") - self.switch_wal_segment(node) + "create table t1(a int)") - node.safe_psql( + xid = node.safe_psql( "postgres", - "create table t2()") + "INSERT INTO t1 VALUES (1) RETURNING (xmin)").rstrip() filename_orig = node.safe_psql( "postgres", "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + # form up path to next .partial WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: filename = filename_orig + '.gz' + '.partial' @@ -502,7 +453,7 @@ def test_archive_push_partial_file_exists(self): file = os.path.join(wals_dir, filename) with open(file, 'a') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.write(b"blahblah") f.flush() f.close() @@ -517,6 +468,10 @@ def test_archive_push_partial_file_exists(self): self.assertTrue(os.path.isfile(file)) + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-xid={0}'.format(xid)]) + # log_file = os.path.join(node.logs_dir, 'postgresql.log') # with open(log_file, 'r') as f: # log_content = f.read() @@ -529,7 +484,7 @@ def test_archive_push_partial_file_exists(self): # @unittest.skip("skip") def test_archive_push_partial_file_exists_not_stale(self): - """Archive-push if file exists""" + """Archive-push if .partial file exists and it is not stale""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -557,6 +512,7 @@ def test_archive_push_partial_file_exists_not_stale(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + # form up path to next .partial WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: filename = filename_orig + '.gz' + '.partial' From fd30ae28b3ed66125f9aa4585c4898b55b502305 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 16:41:04 +0300 Subject: [PATCH 0761/2107] [Issue #90] Remove stale .partial WAL files --- src/archive.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/src/archive.c b/src/archive.c index 5a1053cd1..0360416f1 100644 --- a/src/archive.c +++ b/src/archive.c @@ -142,6 +142,11 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, const char *to_path_p; char to_path_temp[MAXPGPATH]; int errno_temp; + /* partial handling */ + int partial_timeout = 0; + int partial_size = 0; + struct stat st; + bool partial_exists = false; #ifdef HAVE_LIBZ char gz_to_path[MAXPGPATH]; @@ -180,8 +185,11 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + { + partial_exists = true; + elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); + } } else #endif @@ -190,8 +198,65 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + { + partial_exists = true; + elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); + } + } + + /* sleep a second, check if .partial file size is changing, if not, then goto p1 + * Algorihtm is not pretty however we do not expect conflict for '.partial' file + * to be frequent occurrence. + * The main goal is to protect against failed archive-push which left behind + * orphan '.partial' file. + */ + if (partial_exists) + { + while (1) + { + /* exit from loop */ + if (partial_timeout > 10) + { + /* For 10 second the file didn`t changed its size, so consider it stale and reuse it */ + elog(WARNING, "Reusing stale destination temporary WAL file \"%s\"", to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + +#ifdef HAVE_LIBZ + if (is_compress) + { + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); + if (gz_out == NULL) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + else +#endif + { + out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + break; + } + + if (fio_stat(to_path_temp, &st, false, FIO_BACKUP_HOST) < 0) + /* It is ok if partial is gone, we can safely error out */ + elog(ERROR, "Cannot stat destination temporary WAL file \"%s\": %s", to_path_temp, + strerror(errno)); + + /* first round */ + if (!partial_timeout) + partial_size = st.st_size; + + /* file size is changing */ + if (st.st_size > partial_size) + elog(ERROR, "Destination temporary WAL file \"%s\" is not stale", to_path_temp); + + sleep(1); + partial_timeout++; + } } /* copy content */ From 6a3581ceec9471841ce0b3d01f10511d0f934ab7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 18:19:38 +0300 Subject: [PATCH 0762/2107] minor fix: report correct remote mode if remote-host is not NULL and remote-proto is "none" --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b32f72af6..fbfe4a108 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -621,7 +621,7 @@ main(int argc, char *argv[]) elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", instance_config.remote.host ? "true" : "false"); + stream_wal ? "true" : "false", IsSshProtocol() ? "true" : "false"); /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) From e28e34c6e8cc7a8c75648d170826cfa495be9ed1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 18:19:38 +0300 Subject: [PATCH 0763/2107] minor fix: report correct remote mode if remote-host is not NULL and remote-proto is "none" --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b32f72af6..fbfe4a108 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -621,7 +621,7 @@ main(int argc, char *argv[]) elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", instance_config.remote.host ? "true" : "false"); + stream_wal ? "true" : "false", IsSshProtocol() ? "true" : "false"); /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) From 835b5e41850925d3281a50da66da4bd5cc9595f1 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 20 Jun 2019 18:39:05 +0300 Subject: [PATCH 0764/2107] Fix returning errno in remote backup --- src/utils/file.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index ec9350370..568f66b5a 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1148,8 +1148,11 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_PAGE); - if (hdr.arg < 0) /* read error */ - return hdr.arg; + if ((int)hdr.arg < 0) /* read error */ + { + errno = -(int)hdr.arg; + return -1; + } blknum = hdr.arg; if (hdr.size == 0) /* end of segment */ @@ -1205,7 +1208,7 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) { hdr.arg = -errno; hdr.size = 0; - Assert(hdr.arg < 0); + Assert((int)hdr.arg < 0); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } else From b1413d5d609eb7ee2a6fe6588729e1c7688eb669 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 18:57:47 +0300 Subject: [PATCH 0765/2107] bugfix: during backup do not close file if file size is not the mulptiple of BLCKSZ --- src/data.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/data.c b/src/data.c index d243294eb..2a80294cc 100644 --- a/src/data.c +++ b/src/data.c @@ -585,10 +585,7 @@ backup_data_file(backup_files_arg* arguments, } if (file->size % BLCKSZ != 0) - { - fio_fclose(in); elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size); - } /* * Compute expected number of blocks in the file. From 9d2a7a7ca1c993ef9ae0bcacf29388165bf6dea6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Jun 2019 20:49:42 +0300 Subject: [PATCH 0766/2107] update Documentation --- Documentation.md | 721 ++++++++++++++++++++++++++--------------------- 1 file changed, 399 insertions(+), 322 deletions(-) diff --git a/Documentation.md b/Documentation.md index a1517ec3a..67909d676 100644 --- a/Documentation.md +++ b/Documentation.md @@ -9,36 +9,37 @@ Current version - 2.1.3 * [Installation and Setup](#installation-and-setup) * [Command-Line Reference](#command-line-reference) * [Usage](#usage) +* [Quick Start](#quick_start) ### Synopsis -`pg_probackup init -B backupdir` +`pg_probackup init -B backup_dir` -`pg_probackup add-instance -B backupdir -D datadir --instance instance_name` +`pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name` -`pg_probackup del-instance -B backupdir --instance instance_name` +`pg_probackup del-instance -B backup_dir --instance instance_name` -`pg_probackup set-config -B backupdir --instance instance_name [option...]` +`pg_probackup set-config -B backup_dir --instance instance_name [option...]` -`pg_probackup show-config -B backupdir --instance instance_name [--format=format]` +`pg_probackup show-config -B backup_dir --instance instance_name [--format=format]` -`pg_probackup backup -B backupdir --instance instance_name -b backup_mode [option...]` +`pg_probackup show -B backup_dir [option...]` -`pg_probackup merge -B backupdir --instance instance_name -i backup_id [option...]` +`pg_probackup backup -B backup_dir --instance instance_name -b backup_mode [option...]` -`pg_probackup restore -B backupdir --instance instance_name [option...]` +`pg_probackup restore -B backup_dir --instance instance_name [option...]` -`pg_probackup validate -B backupdir [option...]` +`pg_probackup validate -B backup_dir [option...]` -`pg_probackup show -B backupdir [option...]` +`pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` -`pg_probackup delete -B backupdir --instance instance_name { -i backup_id | --wal | --expired }` +`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --wal | --expired | --merge-expired }` -`pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` +`pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` -`pg_probackup archive-get -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f` +`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f` -`pg_probackup checkdb -B backupdir --instance instance_name -D datadir [option...]` +`pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` `pg_probackup version` @@ -48,29 +49,31 @@ Current version - 2.1.3 As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -- Choosing between full and page-level incremental backups to speed up backup and recovery -- Implementing a single backup strategy for multi-server PostgreSQL clusters -- Automatic data consistency checks and on-demand backup validation without actual data recovery -- Managing backups in accordance with retention policy -- Running backup, restore, and validation processes on multiple parallel threads -- Storing backup data in a compressed state to save disk space -- Taking backups from a standby server to avoid extra load on the master server -- Extended logging settings -- Custom commands to simplify WAL log archiving -- Backing up files and directories located outside of PostgreSQL data directory, such as configuration or log files +- Incremental modes: page-level incremental backup with three different approaches of acquiring changed pages allows to choose incremental mode which suits your needs best or mix them in accordance with data flow +- Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery +- Verification: PostgreSQL instance on-demand verification via dedicated `checkdb` command +- Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods - `delete expired` and/or `merge expired` +- Parallelization: Running backup, restore, merge, checkdb and validation processes on multiple parallel threads +- Compression: Storing backup data in a compressed state to save disk space +- Deduplication: Saving disk space by not backpuping the not changed non-data files ('_vm', '_fsm', etc) +- Remote operations: Backup PostgreSQL instance located on remote machine or restore backup on it +- Backup from replica: Avoid extra load on the master server by taking backups from a standby +- External directories: Add to backup content of directories located outside of PostgreSQL data directory, such as configs, logs or pg_dump files +- Extended logging settings: -To manage backup data, pg_probackup creates a *backup catalog*. This directory stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. +To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. Using pg_probackup, you can take full or incremental backups: -- Full backups contain all the data files required to restore the database cluster from scratch. +- FULL backups contain all the data files required to restore the database cluster from scratch. - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. pg_probackup supports the following modes of incremental backups: + - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. - - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Continuous archiving is not necessary for this mode to operate. Note that this mode can impose read-only I/O pressure equal to a full backup. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -Regardless of the chosen backup type, all backups taken with pg_probackup support the following archiving strategies: -- Autonomous backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. -- Archive backups rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery). +Regardless of the chosen backup type, all backups taken with pg_probackup support the following WAL delivery modes: +- ARCHIVE backups rely on [continuous archiving](#setting-up-archive-backups). This is a default WAL delivery mode. + +- STREAM backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of continuous archiving been set up or not, the required WAL segments are included into the backup. ### Limitations @@ -91,11 +94,11 @@ pg_probackup stores all WAL and backup files in the corresponding subdirectories To initialize the backup catalog, run the following command: - pg_probackup init -B backupdir + pg_probackup init -B backup_dir -where backupdir is the backup catalog. If the backupdir already exists, it must be empty. Otherwise, pg_probackup returns an error. +where backup_dir is the path to *backup catalog*. If the backup_dir already exists, it must be empty. Otherwise, pg_probackup returns an error. -pg_probackup creates the backupdir backup catalog, with the following subdirectories: +pg_probackup creates the backup_dir backup catalog, with the following subdirectories: - *wal/* — directory for WAL files. - *backups/* — directory for backup files. @@ -106,19 +109,17 @@ pg_probackup can store backups for multiple database clusters in a single backup To add a new backup instance, run the following command: - pg_probackup add-instance -B backupdir -D datadir --instance instance_name - [--external-dirs=external_directory_path] [remote_backup_options] + pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name + [remote_backup_options] where: -- *datadir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. +- *data_dir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. - *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional *--external-dirs* parameter provides the path to one or more directories to include into the backup that are located outside of the data directory. To specify several external directories, separate their paths by a colon. +- The optional [remote_backup_options](#remote-backup-options) should be used if *data_dir* is located on remote machine. -pg_probackup creates the instance_name subdirectories under the *backups/* and *wal/* directories of the backup catalog. The *backups/instance_name* directory contains the *pg_probackup.conf* configuration file that controls backup and restore settings for this backup instance. If you run this command with the optional *--external-dirs* parameter, this setting is added to *pg_probackup.conf*, so the specified external directories will be backed up each time you create a backup of this instance. For details on how to fine-tune pg_probackup configuration, see the section called “Configuring pg_probackup”. +pg_probackup creates the *instance_name* subdirectories under the *backups/* and *wal/* directories of the backup catalog. The *backups/instance_name* directory contains the *pg_probackup.conf* configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_backup_options](#remote-backup-options), used parameters will be added to *pg_probackup.conf*. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). -The backup catalog must belong to the file system of the database server. The user launching pg_probackup must have full access to the contents of the backup catalog. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. - -Since pg_probackup uses a regular PostgreSQL connection and the replication protocol, pg_probackup commands require connection options. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the *set-config* command. For details, see the section called “Configuring pg_probackup”. +The user launching pg_probackup must have full access to **backup_dir** directory and at least read-only access to **data_dir** directory. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. ##### Configuring the Database Cluster @@ -142,53 +143,58 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are going to use autonomous or archive backup strategies, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the section called “PTRACK Backup” and the section called “Backup from Standby”. +Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up ARCHIVE Backups](#setting-up-archive-backups), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section called [Backup from Standby](#backup-from-standby). -##### Setting up Autonomous Backups +###### Setting up STREAM Backups -To set up the cluster for autonomous backups, complete the following steps: +To set up the cluster for STREAM backups, complete the following steps: - Grant the REPLICATION privilege to the backup role: ALTER ROLE backup WITH REPLICATION; + - In the pg_hba.conf file, allow replication on behalf of the backup role. - Modify the postgresql.conf configuration file of the PostgreSQL server, as follows: - - Make sure the max_wal_senders parameter is set high enough to leave at least one session available for the backup process. - - Set the *wal_level* parameter to be *replica* or higher. + - Make sure the parameter `max_wal_senders` is set high enough to leave at least one session available for the backup process. + - Set the parameter `wal_level` to be higher than `minimal`. -If you are going to take PAGE backups, you also have to configure WAL archiving as explained in the section called “Setting up Archive Backups”. +If you are going to take PAGE backups in STREAM mode, you also have to configure WAL archiving as explained in the section called “Setting up Archive Backups”. -Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server. If you are going to take backups from standby or use PTRACK backups, you must also complete additional setup, as explained in the section called “Backup from Standby” and the section called “PTRACK Backup”, respectively. +Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server in STREAM mode. -##### Setting up Archive Backups +###### Setting up ARCHIVE Backups To set up the cluster for archive backups, complete the following steps: - Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - - Make sure the wal_level parameter is set to replica or higher. - - Set the archive_mode parameter. If you are configuring backups on master, archive_mode must be set to on. To perform archiving on standby, set this parameter to always. + - Make sure the `wal_level` parameter is higher than 'minimal'. + - Set the `archive_mode` parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the archive_command variable, as follows: - archive_command = 'pg_probackup archive-push -B backupdir --instance instance_name --wal-file-path %p --wal-file-name %f - where backupdir and instance_name refer to the already initialized backup catalog instance for this database cluster. + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_backup_options] --wal-file-path %p --wal-file-name %f + where `backup_dir` and `instance_name` refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_backup_options](#remote-backup-options) should be used to archive WAL on remote machine. -Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server. If you are going to take backups from standby or use PTRACK backups, you must also complete additional setup, as explained in the section called “Backup from Standby” and the section called “PTRACK Backup”, respectively. +Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server in ARCHIVE mode. -##### Backup from Standby + +>NOTE: Instead of using `archive_mode` and `archive_command` parameters you may opt for [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to backup_dir/wal/instance_name directory, also WAL compression done be pg_receivewal is supported by pg_probackup. Only using pg_receivewal `zero data loss` archive strategy can be achieved. + +###### Backup from Standby For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: -- On the standby server, allow replication connections: - - Set the max_wal_senders and hot_standby parameters in postgresql.conf. - - Configure host-based authentication in pg_hba.conf. -- On the master server, enable full_page_writes in postgresql.conf. +- On the replica server, set the parameter `hot_standby` to `on`. +- On the master server, set the parameter `full_page_writes` to `on`. +- To perform STREAM backup on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) +- To perform ARCHIVE backup on standby, complete all steps in section [Setting up ARCHIVE Backups](#setting-up-archive-backups) +Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the standby server. ->NOTE: Archive backup from the standby server has the following limitations: -- If the standby is promoted to the master during archive backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable full_page_writes on the master, and not to use a tool like pg_compresslog as archive_command to remove full-page writes from WAL files. +>NOTE: Backup from the standby server has the following limitations: +- If the standby is promoted to the master during backup, the backup fails. +- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable full_page_writes on the master, and not to use a tool like pg_compresslog as archive_command to remove full-page writes from WAL files. -##### PTRACK Backup +###### Setting up PTRACK Backups If you are going to use PTRACK backups, complete the following additional steps: -- In postgresql.conf, set ptrack_enable to on. +- Set the parameter `ptrack_enable` to `on`. - Grant the rights to execute ptrack functions to the backup role: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; @@ -198,139 +204,145 @@ If you are going to use PTRACK backups, complete the following additional steps: ### Command-Line Reference ##### Commands -This section describes pg_probackup commands. Some commands require mandatory options and can take additional options. For detailed descriptions, see the section called “Options”. +This section describes pg_probackup commands. Some commands require mandatory options and can take additional options. Optional parameters encased in square brackets: "[]". For detailed descriptions, see the section called [Options](#options). **init** - pg_probackup init -B backupdir -Initializes the backupdir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backupdir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. + pg_probackup init -B backup_dir [--help] +Initializes the backup_dir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backup_dir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. **add-instance** - pg_probackup add-instance -B backupdir -D datadir --instance instance_name [--external-dirs=external_directory_path] -Initializes a new backup instance inside the backup catalog backupdir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified datadir data directory. For details, see the section called “Adding a New Backup Instance”. + pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name + [--help] [--external-dirs=external_directory_path] +Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section called [Adding a New Backup Instance](#adding-a-new-backup-instance). **del-instance** - pg_probackup del-instance -B backupdir --instance instance_name + pg_probackup del-instance -B backup_dir --instance instance_name + [--help] Deletes all backup and WAL files associated with the specified instance. **set-config** - pg_probackup set-config -B backupdir --instance instance_name - [--log-level-console=log_level] [--log-level-file=log_level] [--log-filename=log_filename] - [--error-log-filename=error_log_filename] [--log-directory=log_directory] - [--log-rotation-size=log_rotation_size] [--log-rotation-age=log_rotation_age] + pg_probackup set-config -B backup_dir --instance instance_name + [--help] [--pgdata=pgdata-path] [--retention-redundancy=redundancy][--retention-window=window] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] [-d dbname] [-h host] [-p port] [-U username] [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [remote_backup_options] + [remote_options] + [logging_options] Adds the specified connection, retention, logging or replica, and compression, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. **show-config** - pg_probackup show-config -B backupdir --instance instance_name [--format=plain|json] -Displays the contents of the pg_probackup.conf configuration file located in the backupdir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. + pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] +Displays the contents of the pg_probackup.conf configuration file located in the backup_dir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. To edit pg_probackup.conf, use the set-config command. It is not allowed to edit pg_probackup.conf directly. **backup** - pg_probackup backup -B backupdir -b backup_mode --instance instance_name - [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] [--external-dirs=external_directory_path] - [--delete-expired] [--merge-expired] [--delete-wal] [--no-validate] [--skip-block-validation] - [--retention-redundancy=redundancy] [--retention-window=window] - [-d dbname] [-h host] [-p port] [-U username] + pg_probackup backup -B backup_dir -b backup_mode --instance instance_name + [--help] [-j num_threads] [--progress] + [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] + [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] - [--archive-timeout=timeout] - [--compress] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] - [-j num_threads][--progress] + [--archive-timeout=timeout] [--external-dirs=external_directory_path] + [connection_options] + [compression_options] + [remote_options] + [retention_options] [logging_options] - [remote_backup_options] -Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. For details, see the section called “Creating a Backup”. +Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. For details, see the section called [Creating a Backup](#creating-a-backup). **merge** - pg_probackup merge -B backupdir --instance instance_name -i backup_id - [-j num_threads][--progress] + pg_probackup merge -B backup_dir --instance instance_name -i backup_id + [--help] [-j num_threads][--progress] [logging_options] - [remote_backup_options] -Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. For details, see the section called “Merging Backups”. +Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. For details, see the section called [Merging Backups](#merging-backups). **restore** - pg_probackup restore -B backupdir --instance instance_name - [-D datadir] - [-i backup_id] [{--recovery-target=immediate|latest | --recovery-target-time=time | --recovery-target-xid=xid | --recovery-target-lsn=lsn | --recovery-target-name=recovery_target_name} [--recovery-target-inclusive=boolean]] - [--recovery-target-timeline=timeline] [-T OLDDIR=NEWDIR] - [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] - [--recovery-target-action=pause|promote|shutdown] - [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + pg_probackup restore -B backup_dir --instance instance_name + [--help] [-D data_dir] [-i backup_id] [-j num_threads] [--progress] + [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] + [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + [recovery_options] [logging_options] - [remote_backup_options] + [remote_options] -Restores the PostgreSQL instance from a backup copy located in the backupdir backup catalog. If you specify a recovery target option, pg_probackup restores the database cluster up to the corresponding recovery target. Otherwise, the most recent backup is used. +Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the most suitable backup and restores it to the corresponding recovery target. Otherwise, the most recent backup is used. **validate** - pg_probackup validate -B backupdir - [--instance instance_name] - [-i backup_id] [{--recovery-target-time=time | --recovery-target-xid=xid | --recovery-target-lsn=lsn | --recovery-target-name=recovery_target_name } [--recovery-target-inclusive=boolean]] - [--recovery-target-timeline=timeline] [--skip-block-validation] + pg_probackup validate -B backup_dir + [--help] [--instance instance_name] [-i backup_id] [-j num_threads] [--progress] + [--skip-block-validation] + [recovery_options] + [logging_options] -Verifies that all the files required to restore the cluster are present and not corrupted. If you specify the instance_name without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the instance_name with a recovery target option or a backup_id, pg_probackup checks whether it is possible to restore the cluster using these options. If instance_name is not specified, pg_probackup validates all backups available in the backup catalog. +Verifies that all the files required to restore the cluster are present and not corrupted. If instance_name is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the instance_name without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the instance_name with a recovery target option and/or a backup_id, pg_probackup checks whether it is possible to restore the cluster using these options. **show** - pg_probackup show -B backupdir - [--instance instance_name [-i backup_id]] [--format=plain|json] + pg_probackup show -B backup_dir + [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. **delete** - pg_probackup delete -B backupdir --instance instance_name - [--wal] {-i backup_id | --expired [--merge-expired] | --merge-expired} [--dry-run] - -Deletes backup or WAL files of the specified backup instance from the backupdir backup catalog: -- The wal option removes the WAL files that are no longer required to restore the cluster from any of the existing backups. -- The -i option removes the specified backup copy. -- The expired option removes the backups that are expired according to the current retention policy. If used together with merge-expired, this option takes effect only after the merge is performed. - -- The merge-expired option merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + pg_probackup delete -B backup_dir --instance instance_name + [--help] [-j num_threads] [--progress] + [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} + [--dry-run] -- The dry-run option displays the current status of all the available backups, without deleting or merging expired backups, if any. +Deletes backup or WAL files of the specified backup instance from the backup_dir backup catalog: +- The `--delete-wal` option removes the WAL files that are no longer required to restore the cluster from any of the existing backups. +- The `-i` option removes the specified backup copy. +- The `--delete-expired` option removes the backups that are expired according to the current retention policy. If used together with `--merge-expired`, this option takes effect only after the merge is performed. +- The `--merge-expired` option merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. +- The `--dry-run` option displays the current retention status of all the available backups, without deleting or merging expired backups, if any. **archive-push** - pg_probackup archive-push -B backupdir --instance instance_name - --wal-file-path %p --wal-file-name %f' - [--compress][--compress-algorithm=compression_algorithm][--compress-level=compression_level] [--overwrite] - [remote_backup_options] + pg_probackup archive-push -B backup_dir --instance instance_name + --wal-file-path %p --wal-file-name %f + [--help] [--compress] [--compress-algorithm=compression_algorithm] + [--compress-level=compression_level] [--overwrite] + [remote_options] + [logging_options] Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. +Copying always done to temp file with `.partial` suffix or, if compression is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving. Copied to archive WAL segments are synced to disk. You can set archive-push as archive_command in postgresql.conf to perform archive backups. **archive-get** - pg_probackup archive-get -B backupdir --instance instance_name - --wal-file-path %p --wal-file-name %f' - [remote backup options] + pg_probackup archive-get -B backup_dir --instance instance_name + --wal-file-path %p --wal-file-name %f + [--help] + [remote_options] + [logging_options] Moves WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as restore_command in recovery.conf when restoring backups using a WAL archive. You do not need to set it manually. **checkdb** - pg_probackup checkdb -D datadir [-B backupdir] [--instance instance_name] + pg_probackup checkdb [-D data_dir] [-B backup_dir] [--instance instance_name] + [--help] [--progress] [-j num_threads] [--amcheck [--heapallindexed] [--skip-block-validation]] - [--progress] [-j num_threads] + [connection_options] + [logging_options] -Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified PostgreSQL instance using the amcheck extension. +Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified PostgreSQL instance using the amcheck extension or the `amcheck_next` extension. **version** @@ -349,7 +361,8 @@ This section describes all command-line options for pg_probackup commands. If th If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. -**Common Options** +###### Common Options +The list of general options. ``` -B directory --backup-path=directory @@ -364,22 +377,21 @@ Specifies the absolute path to the data directory of the database cluster. This -i backup_id -backup-id=backup_id -Specifies the unique identifier of the backup. - - --skip-block-validation - -Disables block-level checksum verification to speed up validation. If this option is used with backup, restore, and validate commands, only file-level checksums will be verified. When used with the checkdb command run with the --amcheck option, --skip-block-validation completely disables validation of data files. +Specifies the unique identifier of the backup. -j num_threads --threads=num_threads -Sets the number of parallel threads for backup, recovery, and backup validation processes. +Sets the number of parallel threads for backup, recovery, and backup validation processes. --progress -Shows the progress of operations. +Shows the progress of operations. + + --help +Shows detailed information about the options that can be used with this command. -**Backup Options** +###### Backup Options -The following options can be used together with the backup command. +The following options can be used together with the `backup` command. Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options) and [Logging Options](#logging-options) can be used. -b mode --backup-mode=mode @@ -405,7 +417,7 @@ Makes an autonomous backup that includes all the necessary WAL files by streamin Specifies the replication slot for WAL streaming. This option can only be used together with the --stream option. --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S --slot`. --backup-pg-log Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. @@ -413,26 +425,19 @@ Includes the log directory into the backup. This directory usually contains log -E external_directory_path --external-dirs=external_directory_path -Includes the specified directory into the backup. This option is useful to back up configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon. +Includes the specified directory into the backup. This option is useful to back up configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. --archive-timeout=wait_time -Sets the timeout for WAL segment archiving, in seconds. By default, pg_probackup waits 300 seconds. - - --delete-expired -After a backup copy is successfully created, deletes backups that are expired according to the current retention policy. You can also clean up the expired backups by running the delete command with the expired option. If used together with merge-expired, this option takes effect after the merge is performed. For details, see the section called “Configuring Backup Retention Policy”. - - --merge-expired -After a backup copy is successfully created, merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. +Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. - --delete-wal -After a backup copy is successfully created, removes redundant WAL files in accordance with the current retention policy. You can also clean up the expired WAL files by running the delete command with the wal option. For details, see the section called “Configuring Backup Retention Policy”. - -**Restore Options** + --skip-block-validation +Disables block-level checksum verification to speed up backup. - --recovery-target-action=pause|promote|shutdown - Default: pause -Specifies the action the server should take when the recovery target is reached, similar to the recovery_target_action option in the recovery.conf configuration file. + --no-validate +Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. +###### Restore Options +The following options can be used together with the `restore` command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. -R | --restore-as-replica Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. @@ -446,25 +451,39 @@ Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. --skip-external-dirs -Skip external directories included into the backup with the --external-dirs option. The contents of these directories will not be restored. +Skip external directories included into the backup with the --external-dirs option. The contents of these directories will not be restored. - --recovery-target-timeline=timeline -Specifies a particular timeline to restore the cluster into. By default, the timeline of the specified backup is used. + --skip-block-validation +Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. --no-validate -Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running backup or restore operations. +Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. -**Recovery Target Options** -If continuous WAL archiving is configured, you can use one of these options together with restore or validate commands to specify the moment up to which the database cluster must be restored. +###### Checkdb Options +The following options can be used together with the `checkdb` command. Additionally [Connection Options](#connection-options) and [Logging Options](#logging-options) can be used. + + --amcheck +Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. + + --skip-block-validation +Skip validation of data files. + + --heapallindexed +Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. + +###### Recovery Target Options +If continuous WAL archiving is configured, you can use one of these options together with `restore` or `validate` commands to specify the moment up to which the database cluster must be restored. --recovery-target=immediate|latest Defines when to stop the recovery: - The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the -i option is omitted. -- The latest value continues the recovery until all WAL segments available in the archive are applied. +- The latest value continues the recovery until all WAL segments available in the archive are applied. + --recovery-target-timeline=timeline +Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. --recovery-target-lsn=lsn -Specifies the LSN of the write-ahead log location up to which recovery will proceed. +Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring database cluster of major version 10 or higher. --recovery-target-name=recovery_target_name Specifies a named savepoint up to which to restore the cluster data. @@ -473,27 +492,17 @@ Specifies a named savepoint up to which to restore the cluster data. Specifies the timestamp up to which recovery will proceed. --recovery-target-xid=xid -Specifies the transaction ID up to which recovery will proceed. +Specifies the transaction ID up to which recovery will proceed. --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with recovery-target-name, recovery-target-time, recovery-target-lsn, or recovery-target-xid options. The default value is taken from the recovery_target_inclusive variable. - -**Delete Options** +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with recovery-target-name, recovery-target-time, recovery-target-lsn, or recovery-target-xid options. The default value is taken from the `recovery_target_inclusive` variable. - --wal -Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. - - --expired -Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. For details, see the section called “Configuring Backup Retention Policy”. - - --merge-expired -Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. - - --dry-run -Displays the current status of all the available backups, without deleting or merging expired backups, if any. + --recovery-target-action=pause|promote|shutdown + Default: pause +Specifies the action the server should take when the recovery target is reached, similar to the `recovery_target_action` option in the recovery.conf configuration file. -**Retention Options** -For details on configuring retention policy, see the section called “Configuring Backup Retention Policy”. +###### Retention Options +You can use these options together with `backup` or `delete` commands. For details on configuring retention policy, see the section called [Configuring Backup Retention Policy](#configuring-backup-retention-policy). --retention-redundancy=redundancy Default: 0 @@ -501,9 +510,22 @@ Specifies the number of full backup copies to keep in the data directory. Must b --retention-window=window Default: 0 -Number of days of recoverability. The zero value disables this setting. +Number of days of recoverability. Must be a positive integer. The zero value disables this setting. + + --delete-wal +Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. + + --delete-expired +Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. + + --merge-expired +Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + + --dry-run +Displays the current status of all the available backups, without deleting or merging expired backups, if any. -**Logging Options** +###### Logging Options +You can use these options with any command. --log-level-console=log_level Default: info @@ -524,7 +546,7 @@ Defines the filenames of log files for error messages. The filenames are treated If error-log-filename is not set, pg_probackup writes all error messages to stderr. --log-directory=log_directory - Default: $BACKUP_PATH/log/ + Default: $BACKUP_PATH/log/ Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. --log-rotation-size=log_rotation_size @@ -535,7 +557,8 @@ Maximum size of an individual log file. If this value is reached, the log file i Default: 0 Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). -**Connection Options** +###### Connection Options +You can use these options together with `backup` and `checkdb` commands. -d dbname --dbname=dbname @@ -567,44 +590,23 @@ User name to connect as. --password Forces a password prompt. -**Compression Options** - - --compress -Enables compression for data files. You can specify the compression algorithm and level using the --compress-algorithm and --compress-level options, respectively. If you omit these options, --compress uses zlib compression algorithm with compression level 1. -By default, compression is disabled. +###### Compression Options +You can use these options together with `backup` and `archive-push` commands. --compress-algorithm=compression_algorithm -Defines the algorithm to use for compressing data files. Possible values are zlib, pglz, and none. If set to zlib or pglz, this option enables compression, regardless of whether the --compress option is specified. By default, compression is disabled. -For the archive-push command, the pglz compression algorithm is not supported. + Default: none +Defines the algorithm to use for compressing data files. Possible values are zlib, pglz, and none. If set to zlib or pglz, this option enables compression. By default, compression is disabled. +For the `archive-push` command, the pglz compression algorithm is not supported. --compress-level=compression_level - Default: 1 -Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can only be used together with --compress or --compress-algorithm options. - -**Replica Options** -This section describes the options related to taking a backup from standby. ->NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. - - --master-db=dbname - Default: postgres, the default PostgreSQL database. -Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the set-config command. - - --master-host=host -Deprecated. Specifies the host name of the system on which the master server is running. - - --master-port=port - Default: 5432, the PostgreSQL default port. -Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. - - --master-user=username - Default: postgres, the PostgreSQL default user name. -Deprecated. User name to connect as. + Default: 1 +Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with --compress-algorithm option. - --replica-timeout=timeout - Default: 300 sec -Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the set-config command. + --compress +Alias for --compress-algorithm=zlib and --compress-level=1. -**Archiving Options** +###### Archiving Options +These options can be used with `archive-push` and `archive-get` commands. --wal-file-path=wal_file_path %p Provides the path to the WAL file in archive_command and restore_command used by pg_probackup. The %p variable is required for correct processing. @@ -613,23 +615,15 @@ Provides the path to the WAL file in archive_command and restore_command used by Provides the name of the WAL file in archive_command and restore_command used by pg_probackup. The %f variable is required for correct processing. --overwrite -Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. - -**checkdb Options** +Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. - --amcheck -Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. Optionally, you can skip validation of data files by specifying --skip-block-validation. You must have the amcheck extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. - - --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option starting from PostgreSQL 11. - -**Remote Backup Options** -This section describes the options related to running backup and restore operations remotely via SSH. These options can be used with add-instance, set-config, backup, restore, archive-push, and archive-get commands. +###### Remote Mode Options +This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with `add-instance`, `set-config`, `backup`, `restore`, `archive-push` and `archive-get` commands. For details on configuring remote operation mode, see the section called [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: -- ssh enables the remote backup mode via SSH. -- none explicitly disables the remote backup mode. +- ssh enables the remote backup mode via SSH. This is the Default value. +- none explicitly disables the remote mode. You can omit this option if the --remote-host option is specified. @@ -641,28 +635,55 @@ Specifies the remote host IP address or hostname to connect to. Specifies the remote host port to connect to. --remote-user + Default: current user Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. --remote-path -Specifies pg_probackup installation directory on the remote system. +Specifies pg_probackup installation directory on the remote system. --ssh-options -Specifies a string of SSH command-line options. +Specifies a string of SSH command-line options. + +###### Replica Options +This section describes the options related to taking a backup from standby. +>NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. + + --master-db=dbname + Default: postgres, the default PostgreSQL database. +Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the set-config command. + + --master-host=host +Deprecated. Specifies the host name of the system on which the master server is running. + + --master-port=port + Default: 5432, the PostgreSQL default port. +Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. + + --master-user=username + Default: postgres, the PostgreSQL default user name. +Deprecated. User name to connect as. + + --replica-timeout=timeout + Default: 300 sec +Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the set-config command. ### Usage - [Creating a Backup](#creating-a-backup) - [Validating a Backup](#vaklidating-a-backup) - [Restoring a Cluster](#restoting-a-cluster) -- [Using pg_probackup in the Remote Backup Mode](#using-pg_probackup-in-the-remote-backup-mode) +- [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - [Configuring pg_probackup](#configuring-pg_probackup) - [Managing the Backup Catalog](#managing-the-backup-Catalog) +- [Configuring Backup Retention Policy](#configuring-backup-retention-policy) +- [Merging Backups](#merging-backups) +- [Deleting Backups](#deleting-backups) ##### Creating a Backup To create a backup, run the following command: - pg_probackup backup -B backupdir --instance instance_name -b backup_mode + pg_probackup backup -B backup_dir --instance instance_name -b backup_mode where backup_mode can take one of the following values: - FULL — creates a full backup that contains all the data files of the cluster to be restored. - DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. @@ -675,7 +696,7 @@ If you have configured PTRACK backups, pg_probackup clears PTRACK bitmap of the To make a backup autonomous, add the --stream option to the above command. For example, to create a full autonomous backup, run: - pg_probackup backup -B backupdir --instance instance_name -b FULL --stream --temp-slot + pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot The optional --temp-slot parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. @@ -702,17 +723,17 @@ To ensure that all the required backup files are present and can be used to rest For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: - pg_probackup validate -B backupdir --instance instance_name --recovery-target-xid=xid + pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=xid If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time and transaction ID up to which the recovery is possible. ##### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: - pg_probackup restore -B backupdir --instance instance_name -i backup_id + pg_probackup restore -B backup_dir --instance instance_name -i backup_id where: -- backupdir is the backup catalog that stores all backup files and meta information. +- backup_dir is the backup catalog that stores all backup files and meta information. - instance_name is the backup instance for the cluster to be restored. - backup_id specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. @@ -720,7 +741,7 @@ If the cluster to restore contains tablespaces, pg_probackup restores them to th When using the --tablespace-mapping option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: - pg_probackup restore -B backupdir --instance instance_name -D datadir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir + pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir Once the restore command is complete, start the database service. If you are restoring an autonomous backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For archive backups, PostgreSQL replays all archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the recovery_target option with the restore command. Note that using the recovery-target=latest value with autonomous backups is only possible if the WAL archive is available at least starting from the time the autonomous backup was taken. @@ -731,18 +752,18 @@ If you have enabled continuous WAL archiving before taking backups, you can rest - To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: - pg_probackup restore -B backupdir --instance instance_name --recovery-target-time='2017-05-18 14:18:11' + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11' - To restore the cluster state up to a specific transaction ID, use the recovery-target-xid option: - pg_probackup restore -B backupdir --instance instance_name --recovery-target-xid=687 + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 - If you know the exact LSN up to which you need to restore the data, use recovery-target-lsn: - pg_probackup restore -B backupdir --instance instance_name --recovery-target-lsn=16/B374D848 + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 By default, the recovery_target_inclusive parameter defines whether the recovery target is included into the backup. You can explicitly include or exclude the recovery target by adding the --recovery-target-inclusive=boolean option to the commands listed above. -##### Using pg_probackup in the Remote Backup Mode +##### Using pg_probackup in the Remote Mode pg_probackup supports the remote backup mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. @@ -762,7 +783,7 @@ Backup, recovery, and validation processes can be executed on several parallel t Parallel execution is controlled by the -j/--threads command line option. For example, to create a backup using four parallel threads, run: - pg_probackup backup -B backupdir --instance instance_name -b FULL -j 4 + pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 >NOTE: Parallel recovery applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. @@ -770,81 +791,27 @@ Parallel execution is controlled by the -j/--threads command line option. For ex Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the backups/instance_name directory to fine-tune pg_probackup configuration. +Since pg_probackup uses a regular PostgreSQL connection and, for STREAM backups, the replication protocol, pg_probackup `backup` and `checkdb` commands require connection options. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the *set-config* command. + Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. - system-identifier — the unique identifier of the PostgreSQL instance. Additionally, you can define connection, retention, logging, and replica settings using the set-config command: - pg_probackup set-config -B backupdir --instance instance_name --external-dirs=external_directory_path [connection_options] [retention_options] [logging_options] [replica_options] + pg_probackup set-config -B backup_dir --instance instance_name --external-dirs=external_directory_path [connection_options] [retention_options] [logging_options] [replica_options] To view the current settings, run the following command: - pg_probackup show-config -B backupdir --instance instance_name + pg_probackup show-config -B backup_dir --instance instance_name -You can override the settings defined in pg_probackup.conf when running the backup command. +You can override the settings defined in pg_probackup.conf when running the backup command via corresponding environment variables and/or command line options. **Specifying Connection Settings** If you define connection settings in the pg_probackup.conf configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. -If nothing is given, the default values are taken. pg_probackup tries to use local connection and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. - -**Configuring Backup Retention Policy** - -By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. - -To configure retention policy, set one or more of the following variables in the pg_probackup.conf file: -- retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. -- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. - -If both retention-redundancy and retention-window options are set, pg_probackup keeps backup copies that satisfy both conditions. For example, if you set retention-redundancy=2 and retention-window=7, pg_probackup cleans up the backup directory to keep only two full backup copies if at least one of them is older than seven days. - -To clean up the backup catalog in accordance with retention policy, run: - - pg_probackup delete -B backupdir --instance instance_name --expired - -pg_probackup deletes all backup copies that do not conform to the defined retention policy. - -If you would like to also remove the WAL files that are no longer required for any of the backups, add the --wal option: - - pg_probackup delete -B backupdir --instance instance_name --expired --wal - -Alternatively, you can use the --delete-expired and --delete-wal options together with the backup command to remove the outdated backup copies once the new backup is created. - -Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the --merge-expired option when running backup or delete commands. - -Suppose you have backed up the node instance in the node-backup directory, with the retention-window option is set to 7, and you have the following backups available on April 10, 2019: - -``` -BACKUP INSTANCE 'node' -=========================================================================================================================================== - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -=========================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 PTRACK STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK - node 10 P7XDHU 2019-04-02 05:27:59+03 PTRACK STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK -``` - -Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDHU and P7XDQV that are still required, so if you run the delete command with the --expired option, only the P7XDFT full backup will be removed. With the --merge-expired option, the P7XDJA backup is merged with the underlying P7XDHB and P7XDHU backups and becomes a full one, so there is no need to keep these expired backups anymore: - - pg_probackup delete -B node-backup --instance node --expired --merge-expired - pg_probackup show -B node-backup - -``` -BACKUP INSTANCE 'node' -============================================================================================================================================ - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -============================================================================================================================================ - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK -``` - -Note that the Time field for the merged backup displays the time required for the merge. +If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix socket(localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. ##### Managing the Backup Catalog @@ -855,9 +822,9 @@ With pg_probackup, you can manage backups from the command line: - Delete backups - Viewing Backup Information -To view the list of existing backups, run the command: +To view the list of existing backups for every instance, run the command: - pg_probackup show -B backupdir + pg_probackup show -B backup_dir pg_probackup displays the list of all the available backups. For example: @@ -875,41 +842,48 @@ BACKUP INSTANCE 'node' For each backup, the following information is provided: - Instance — the instance name. -- Version — PostgreSQL version. +- Version — PostgreSQL major version. - ID — the backup identifier. - Recovery time — the earliest moment for which you can restore the state of the database cluster. - Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. - WAL — the way of WAL log handling. Possible values: STREAM for autonomous backups and ARCHIVE for archive backups. -- Current/Parent TLI — current and parent timelines of the database cluster. +- Current/Parent TLI — timeline identifiers of current backup and its parent. - Time — the time it took to perform the backup. - Data — the size of the data files in this backup. This value does not include the size of WAL files. - Start LSN — WAL log sequence number corresponding to the start of the backup process. - Stop LSN — WAL log sequence number corresponding to the end of the backup process. - Status — backup status. Possible values: - OK — the backup is complete and valid. - - CORRUPT — some of the backup files are corrupted. - - DONE — the backup is complete, but was not validated. - - ERROR — the backup was aborted because of an unexpected error. - RUNNING — the backup is in progress. + - DONE — the backup is complete, but was not validated. - MERGING — the backup is being merged. - - ORPHAN — the backup is invalid because one of its parent backups is corrupt. - DELETING — the backup files are being deleted. + - CORRUPT — some of the backup files are corrupted. + - ERROR — the backup was aborted because of an unexpected error. + - ORPHAN — the backup is invalid because one of its parent backups is corrupt or missing. -You can restore the cluster from the backup only if the backup status is OK. +You can restore the cluster from the backup only if the backup status is OK or DONE. To get more detailed information about the backup, run the show with the backup ID: - pg_probackup show -B backupdir --instance instance_name -i backup_id + pg_probackup show -B backup_dir --instance instance_name -i backup_id The sample output is as follows: ``` #Configuration backup-mode = FULL stream = false +compress-alg = zlib +compress-level = 1 +from-replica = false + #Compatibility block-size = 8192 wal-block-size = 8192 -checksum-version = 0 +checksum-version = 1 +program-version = 2.1.3 +server-version = 10 + #Result backup info timelineid = 1 start-lsn = 0/04000028 @@ -919,38 +893,141 @@ end-time = '2017-05-16 12:57:31' recovery-xid = 597 recovery-time = '2017-05-16 12:57:31' data-bytes = 22288792 +wal-bytes = 16777216 status = OK +parent-backup-id = 'PT8XFX' +primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' +``` + +To get more detailed information about the backup in json format, run the show with the backup ID: + + pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id + +The sample output is as follows: +``` +[ + { + "instance": "node", + "backups": [ + { + "id": "PT91HZ", + "parent-backup-id": "PT8XFX", + "backup-mode": "DELTA", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": false, + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.3", + "server-version": "10", + "current-tli": 16, + "parent-tli": 2, + "start-lsn": "0/8000028", + "stop-lsn": "0/8000160", + "start-time": "2019-06-17 18:25:11+03", + "end-time": "2019-06-17 18:25:16+03", + "recovery-xid": 0, + "recovery-time": "2019-06-17 18:25:15+03", + "data-bytes": 106733, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } +] ``` -**Merging Backups** +##### Configuring Backup Retention Policy + +By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. + +To configure retention policy, set one or more of the following variables in the pg_probackup.conf file: +- retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. +- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. + +If both retention-redundancy and retention-window options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set retention-redundancy=2 and retention-window=7, pg_probackup cleans up the backup directory to keep only two full backup copies and all backups that are newer than seven days. + +To clean up the backup catalog in accordance with retention policy, run: + + pg_probackup delete -B backup_dir --instance instance_name --delete-expired + +pg_probackup deletes all backup copies that do not conform to the defined retention policy. + +If you would like to also remove the WAL files that are no longer required for any of the backups, add the --delete-wal option: + + pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal + +>NOTE: Alternatively, you can use the --delete-expired, --merge-expired and --delete-wal options together with the `backup` command to remove and merge the outdated backup copies once the new backup is created. + +Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the --merge-expired option when running `backup` or `delete` commands. + +Suppose you have backed up the node instance in the node-backup directory, with the retention-window option is set to 7, and you have the following backups available on April 10, 2019: + +``` +BACKUP INSTANCE 'node' +=========================================================================================================================================== + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +=========================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK + ---------------------------------------retention window------------------------------------------------------------- + node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK + node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK +``` + +Even though P7XDHB and P7XDHU backups are outside the retention window, P7XDHB and P7XDHU cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the --delete-expired option, only the P7XDFT full backup will be removed. +With the --merge-expired option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: + + pg_probackup delete -B node-backup --instance node --delete-expired --merge-expired + pg_probackup show -B node-backup + +``` +BACKUP INSTANCE 'node' +============================================================================================================================================ + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +============================================================================================================================================ + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK +``` + +>Note: The Time field for the merged backup displays the time required for the merge. + +##### Merging Backups As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: - pg_probackup merge -B backupdir --instance instance_name -i backup_id + pg_probackup merge -B backup_dir --instance instance_name -i backup_id This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes. -Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the show command with the backup ID. If the merge is still in progress, the backup status is displayed as MERGING. You can restart the merge if it is interrupted. -Deleting Backups +Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the show command with the backup ID. If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. + +##### Deleting Backups To delete a backup that is no longer required, run the following command: - pg_probackup delete -B backupdir --instance instance_name -i backup_id + pg_probackup delete -B backup_dir --instance instance_name -i backup_id This command will delete the backup with the specified backup_id, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. In this case, the next PTRACK backup will be incomplete as some changes since the last retained backup will be lost. Either a full backup or an incremental PAGE backup (if all the necessary WAL files are still present in the archive) must be taken then. To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the --wal option: - pg_probackup delete -B backupdir --instance instance_name --wal + pg_probackup delete -B backup_dir --instance instance_name --delete-wal -To delete backups that are expired according to the current retention policy, use the --expired option: +To delete backups that are expired according to the current retention policy, use the --delete-expired option: - pg_probackup delete -B backupdir --instance instance_name --expired + pg_probackup delete -B backup_dir --instance instance_name --delete-expired Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the --merge-expired option when running this command: - pg_probackup delete -B backupdir --instance instance_name --expired --merge-expired + pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. @@ -959,4 +1036,4 @@ Before merging or deleting backups, you can run the delete command with the dry- ### Authors PostgreSQLfessional, Moscow, Russia. ### Credits -pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. \ No newline at end of file +pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. From 842f7bcc7bb782e4fcd6ca422869419f2ef1b7fb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 21 Jun 2019 15:13:32 +0300 Subject: [PATCH 0767/2107] add "--help" option to all commands in help_pg_probackup() --- src/help.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/help.c b/src/help.c index a69b40c86..e74facf39 100644 --- a/src/help.c +++ b/src/help.c @@ -97,9 +97,11 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); + printf(_(" [--help]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); @@ -126,6 +128,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); @@ -143,6 +146,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); @@ -151,22 +155,27 @@ help_pg_probackup(void) printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--help]\n")); printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); printf(_(" [--heapallindexed]\n")); + printf(_(" [--help]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); printf(_(" [--format=format]\n")); + printf(_(" [--help]\n")); printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n")); printf(_(" [--dry-run]\n")); + printf(_(" [--help]\n")); printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--progress] [-j num-threads]\n")); + printf(_(" [--help]\n")); printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); @@ -174,9 +183,11 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); printf(_(" --instance=instance_name\n")); + printf(_(" [--help]\n")); printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); @@ -188,6 +199,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); @@ -195,6 +207,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); if ((PROGRAM_URL || PROGRAM_EMAIL)) { From 5c09b69d2222da47bad3ff29346223b5d75d0872 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 21 Jun 2019 17:46:53 +0300 Subject: [PATCH 0768/2107] minor improvement: use unsigned instead of float for backup time window --- src/delete.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/delete.c b/src/delete.c index 05aa73d8a..2b546c82e 100644 --- a/src/delete.c +++ b/src/delete.c @@ -209,7 +209,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg time_t days_threshold = 0; /* For fancy reporting */ - float actual_window = 0; + uint32 actual_window = 0; /* Get current time */ current_time = time(NULL); @@ -252,7 +252,9 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg cur_full_backup_num++; } - /* Check if backup in needed by retention policy */ + /* Check if backup in needed by retention policy + * TODO: consider that ERROR backup most likely to have recovery_time == 0 + */ if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) && (instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num))) { @@ -324,7 +326,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg } /* Message about retention state of backups - * TODO: Float is ugly, rewrite somehow. + * TODO: message is ugly, rewrite it to something like show table in stdout. */ cur_full_backup_num = 1; @@ -340,9 +342,9 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg if (backup->recovery_time == 0) actual_window = 0; else - actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24); + actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24); - elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %.2fd/%ud. %s", + elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", base36enc(backup->start_time), pgBackupGetBackupMode(backup), status2str(backup->status), From 253701f622034109a26a5cb9d26ada5a31972cff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 05:05:01 +0300 Subject: [PATCH 0769/2107] update Documentation --- Documentation.md | 407 +++++++++++++++++++++++++++-------------------- 1 file changed, 235 insertions(+), 172 deletions(-) diff --git a/Documentation.md b/Documentation.md index 67909d676..9ffd82199 100644 --- a/Documentation.md +++ b/Documentation.md @@ -4,15 +4,74 @@ Current version - 2.1.3 -* [Synopsis](#synopsis) -* [Overview](#overview) -* [Installation and Setup](#installation-and-setup) -* [Command-Line Reference](#command-line-reference) -* [Usage](#usage) -* [Quick Start](#quick_start) +1. [Synopsis](#synopsis) +2. [Overview](#overview) + 2.1. [Limitations](#limitations) + +3. [Installation and Setup](#installation-and-setup) + 3.1. [Initializing the Backup Catalog](#initializing-the-backup-catalog) + 3.2. [Adding a New Backup Instance](#adding-a-new-backup-instance) + 3.3. [Configuring the Database Cluster](#configuring-the-database-cluster) + 3.4. [Setting up STREAM Backups](#setting-up-stream-backups) + 3.5. [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) + 3.6. [Setting up backup from Standby](#backup-from-standby) + 3.7. [Setting up PTRACK Backups](#setting-up-ptrack-backups) + +4. [Command-Line Reference](#command-line-reference) + 4.1 [Commands](#commands) + 4.1.1 [version](#version) + 4.1.2 [help](#help) + 4.1.3 [add-instance](#add-instance) + 4.1.4 [del-instance](#del-instance) + 4.1.5 [set-config](#set-config) + 4.1.6 [show-config](#show-config) + 4.1.7 [show](#show) + 4.1.8 [backup](#backup) + 4.1.9 [restore](#restore) + 4.1.10 [validate](#validate) + 4.1.11 [merge](#merge) + 4.1.12 [delete](#delete) + 4.1.13 [archive-push](#archive-push) + 4.1.14 [archive-get](#archive-get) + + 4.2 [Options](#options) + 4.2.1 [Common Options](#common-options) + 4.2.2 [Backup Options](#backup-options) + 4.2.3 [Restore Options](#restore-options) + 4.2.4 [Checkdb Options](#checkdb-options) + 4.2.5 [Recovery Target Options](#recovery-target-options) + 4.2.6 [Retention Options](#retention-options) + 4.2.7 [Logging Options](#logging-options) + 4.2.8 [Connection Options](#connection-options) + 4.2.9 [Compression Options](#compression-ptions) + 4.2.10 [Archiving Options](#archiving-options) + 4.2.11 [Remote Mode Options](#remote-mode-options) + 4.2.12 [Compression Options](#compression-ptions) + 4.2.13 [Replica Options](#replica-options) + +5. [Usage](#usage) + 5.1 [Creating a Backup](#creating-a-backup) + 5.2 [Validating a Backup](#vaklidating-a-backup) + 5.3 [Restoring a Cluster](#restoring-a-cluster) + 5.4 [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) + 5.5 [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) + 5.6 [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) + 5.7 [Configuring pg_probackup](#configuring-pg_probackup) + 5.8 [Managing the Backup Catalog](#managing-the-backup-Catalog) + 5.9 [Configuring Backup Retention Policy](#configuring-backup-retention-policy) + 5.10 [Merging Backups](#merging-backups) + 5.11 [Deleting Backups](#deleting-backups) + +6. [Authors](#authors) +7. [Credits](#credits) + ### Synopsis +`pg_probackup version` + +`pg_probackup help [command]` + `pg_probackup init -B backup_dir` `pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name` @@ -33,7 +92,7 @@ Current version - 2.1.3 `pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` -`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --wal | --expired | --merge-expired }` +`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired }` `pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` @@ -41,45 +100,40 @@ Current version - 2.1.3 `pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` -`pg_probackup version` - -`pg_probackup help [command]` - ### Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -- Incremental modes: page-level incremental backup with three different approaches of acquiring changed pages allows to choose incremental mode which suits your needs best or mix them in accordance with data flow +- Incremental backup: page-level incremental backup of three different types allows you to devise the backup strategy in accordance with your data flow - Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery -- Verification: PostgreSQL instance on-demand verification via dedicated `checkdb` command -- Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods - `delete expired` and/or `merge expired` -- Parallelization: Running backup, restore, merge, checkdb and validation processes on multiple parallel threads +- Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb` +- Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` +- Parallelization: Running backup, restore, merge, verificaton and validation processes on multiple parallel threads - Compression: Storing backup data in a compressed state to save disk space -- Deduplication: Saving disk space by not backpuping the not changed non-data files ('_vm', '_fsm', etc) +- Deduplication: Saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) - Remote operations: Backup PostgreSQL instance located on remote machine or restore backup on it - Backup from replica: Avoid extra load on the master server by taking backups from a standby -- External directories: Add to backup content of directories located outside of PostgreSQL data directory, such as configs, logs or pg_dump files -- Extended logging settings: +- External directories: Add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. +To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single **backup catalog**. Using pg_probackup, you can take full or incremental backups: -- FULL backups contain all the data files required to restore the database cluster from scratch. +- FULL backups contain all the data files required to restore the database cluster. - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. pg_probackup supports the following modes of incremental backups: - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. + - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -Regardless of the chosen backup type, all backups taken with pg_probackup support the following WAL delivery modes: -- ARCHIVE backups rely on [continuous archiving](#setting-up-archive-backups). This is a default WAL delivery mode. +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen **backup type** (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following **WAL delivery methods**: +- ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. -- STREAM backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of continuous archiving been set up or not, the required WAL segments are included into the backup. +- STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. -### Limitations +#### Limitations pg_probackup currently has the following limitations: - Creating backups from a remote server is currently not supported on Windows systems. -- The server from which the backup was taken and the restored server must be compatible by the block_size and wal_block_size parameters and have the same major release number. +- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. ### Installation and Setup @@ -89,41 +143,41 @@ Once you have pg_probackup installed, complete the following setup: - Add a new backup instance to the backup catalog. - Configure the database cluster to enable pg_probackup backups. -##### Initializing the Backup Catalog +#### Initializing the Backup Catalog pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. To initialize the backup catalog, run the following command: pg_probackup init -B backup_dir -where backup_dir is the path to *backup catalog*. If the backup_dir already exists, it must be empty. Otherwise, pg_probackup returns an error. +where **backup_dir** is the path to backup catalog. If the backup_dir already exists, it must be empty. Otherwise, pg_probackup returns an error. pg_probackup creates the backup_dir backup catalog, with the following subdirectories: -- *wal/* — directory for WAL files. -- *backups/* — directory for backup files. +- **wal/** — directory for WAL files. +- **backups/** — directory for backup files. Once the backup catalog is initialized, you can add a new backup instance. -##### Adding a New Backup Instance +#### Adding a New Backup Instance pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. To add a new backup instance, run the following command: pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name - [remote_backup_options] + [remote_options] where: -- *data_dir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. -- *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional [remote_backup_options](#remote-backup-options) should be used if *data_dir* is located on remote machine. +- **data_dir** is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. +- **instance_name** is the name of the subdirectories that will store WAL and backup files for this cluster. +- The optional [remote_options](#remote-options) should be used if **data_dir** is located on remote machine. -pg_probackup creates the *instance_name* subdirectories under the *backups/* and *wal/* directories of the backup catalog. The *backups/instance_name* directory contains the *pg_probackup.conf* configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_backup_options](#remote-backup-options), used parameters will be added to *pg_probackup.conf*. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). +pg_probackup creates the **instance_name** subdirectories under the **backups/* and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). The user launching pg_probackup must have full access to **backup_dir** directory and at least read-only access to **data_dir** directory. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. -##### Configuring the Database Cluster +#### Configuring the Database Cluster -Although pg_probackup can be used by a superuser, it is recommended to create a separate user or role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. +Although pg_probackup can be used by a superuser, it is recommended to create a separate user or role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the **backup** role is used as an example. To enable backups, the following rights are required: @@ -143,9 +197,9 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up ARCHIVE Backups](#setting-up-archive-backups), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section called [Backup from Standby](#backup-from-standby). +Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section called [Backup from Standby](#backup-from-standby). -###### Setting up STREAM Backups +#### Setting up STREAM Backups To set up the cluster for STREAM backups, complete the following steps: - Grant the REPLICATION privilege to the backup role: @@ -157,41 +211,42 @@ To set up the cluster for STREAM backups, complete the following steps: - Make sure the parameter `max_wal_senders` is set high enough to leave at least one session available for the backup process. - Set the parameter `wal_level` to be higher than `minimal`. -If you are going to take PAGE backups in STREAM mode, you also have to configure WAL archiving as explained in the section called “Setting up Archive Backups”. +If you are going to take PAGE backups in STREAM mode, you also have to configure WAL archiving as explained in the section called [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). + +Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in STREAM mode. -Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server in STREAM mode. +>NOTE: Even if [continuous WAL archiving](#setting-up-continuous-wal-archiving) is set up, you may still take STREAM backups. It may be useful in some rare cases, such as backup from stale standby. -###### Setting up ARCHIVE Backups -To set up the cluster for archive backups, complete the following steps: +#### Setting up continuous WAL archiving +ARCHIVE backups require [continious WAL archiving](#https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - Make sure the `wal_level` parameter is higher than 'minimal'. - Set the `archive_mode` parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. - - Set the archive_command variable, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_backup_options] --wal-file-path %p --wal-file-name %f - where `backup_dir` and `instance_name` refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_backup_options](#remote-backup-options) should be used to archive WAL on remote machine. + - Set the `archive_command` parameter, as follows: + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_options] --wal-file-path %p --wal-file-name %f + Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-options) should be used to archive WAL to the remote machine. -Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the master server in ARCHIVE mode. +Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of using `archive_mode` and `archive_command` parameters you may opt for [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to backup_dir/wal/instance_name directory, also WAL compression done be pg_receivewal is supported by pg_probackup. Only using pg_receivewal `zero data loss` archive strategy can be achieved. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to 'backup_dir/wal/instance_name' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using `pg_receivewal`. -###### Backup from Standby +#### Backup from Standby For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: -- On the replica server, set the parameter `hot_standby` to `on`. +- On the standby server, set the parameter `hot_standby` to `on`. - On the master server, set the parameter `full_page_writes` to `on`. - To perform STREAM backup on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) -- To perform ARCHIVE backup on standby, complete all steps in section [Setting up ARCHIVE Backups](#setting-up-archive-backups) - -Once these steps are complete, you can start taking FULL, PAGE, or DELTA backups from the standby server. +- To perform ARCHIVE backup on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) +Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK Backups backups of appropriate WAL delivery method, ARCHIVE or STREAM, from the standby server. >NOTE: Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. - All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable full_page_writes on the master, and not to use a tool like pg_compresslog as archive_command to remove full-page writes from WAL files. -###### Setting up PTRACK Backups +#### Setting up PTRACK Backups If you are going to use PTRACK backups, complete the following additional steps: - Set the parameter `ptrack_enable` to `on`. @@ -199,31 +254,46 @@ If you are going to use PTRACK backups, complete the following additional steps: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; - The *backup* role must have access to all the databases of the cluster. + The *backup* role must have access to all the databases of the cluster. ### Command-Line Reference -##### Commands +#### Commands -This section describes pg_probackup commands. Some commands require mandatory options and can take additional options. Optional parameters encased in square brackets: "[]". For detailed descriptions, see the section called [Options](#options). +This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). -**init** +#### version + + pg_probackup version + +Prints pg_probackup version. + +#### help + + pg_probackup help [command] + +Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. + +#### init pg_probackup init -B backup_dir [--help] + Initializes the backup_dir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backup_dir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. -**add-instance** +#### add-instance pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] + Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section called [Adding a New Backup Instance](#adding-a-new-backup-instance). -**del-instance** +#### del-instance pg_probackup del-instance -B backup_dir --instance instance_name [--help] -Deletes all backup and WAL files associated with the specified instance. -**set-config** +Deletes all backups and WAL files associated with the specified instance. + +#### set-config pg_probackup set-config -B backup_dir --instance instance_name [--help] [--pgdata=pgdata-path] @@ -233,16 +303,25 @@ Deletes all backup and WAL files associated with the specified instance. [--archive-timeout=timeout] [--external-dirs=external_directory_path] [remote_options] [logging_options] + Adds the specified connection, retention, logging or replica, and compression, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. -**show-config** +#### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] + Displays the contents of the pg_probackup.conf configuration file located in the backup_dir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. To edit pg_probackup.conf, use the set-config command. -It is not allowed to edit pg_probackup.conf directly. +It is **not recommended** to edit pg_probackup.conf directly. -**backup** +#### show + + pg_probackup show -B backup_dir + [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] + +Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. + +#### backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name [--help] [-j num_threads] [--progress] @@ -255,17 +334,11 @@ It is not allowed to edit pg_probackup.conf directly. [remote_options] [retention_options] [logging_options] -Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. For details, see the section called [Creating a Backup](#creating-a-backup). -**merge** +Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. +For details, see the section [Creating a Backup](#creating-a-backup). - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - [--help] [-j num_threads][--progress] - [logging_options] - -Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. For details, see the section called [Merging Backups](#merging-backups). - -**restore** +#### restore pg_probackup restore -B backup_dir --instance instance_name [--help] [-D data_dir] [-i backup_id] @@ -276,9 +349,10 @@ Merges the specified incremental backup to its parent full backup, together with [logging_options] [remote_options] -Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the most suitable backup and restores it to the corresponding recovery target. Otherwise, the most recent backup is used. +Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. +For details, see the section [Restoring a Cluster](#restoring-a-cluster). -**validate** +#### validate pg_probackup validate -B backup_dir [--help] [--instance instance_name] [-i backup_id] @@ -287,30 +361,29 @@ Restores the PostgreSQL instance from a backup copy located in the backup_dir ba [recovery_options] [logging_options] -Verifies that all the files required to restore the cluster are present and not corrupted. If instance_name is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the instance_name without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the instance_name with a recovery target option and/or a backup_id, pg_probackup checks whether it is possible to restore the cluster using these options. +Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a recovery target option and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. -**show** +#### merge - pg_probackup show -B backup_dir - [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] + pg_probackup merge -B backup_dir --instance instance_name -i backup_id + [--help] [-j num_threads][--progress] + [logging_options] -Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. +Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. +For details, see the section [Merging Backups](#merging-backups). -**delete** +#### delete pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} [--dry-run] + [logging_options] -Deletes backup or WAL files of the specified backup instance from the backup_dir backup catalog: -- The `--delete-wal` option removes the WAL files that are no longer required to restore the cluster from any of the existing backups. -- The `-i` option removes the specified backup copy. -- The `--delete-expired` option removes the backups that are expired according to the current retention policy. If used together with `--merge-expired`, this option takes effect only after the merge is performed. -- The `--merge-expired` option merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. -- The `--dry-run` option displays the current retention status of all the available backups, without deleting or merging expired backups, if any. +Deletes backup with specified **backip_id** or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. +For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-otions) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). -**archive-push** +#### archive-push pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f @@ -320,11 +393,13 @@ Deletes backup or WAL files of the specified backup instance from the backup_dir [logging_options] Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. -Copying always done to temp file with `.partial` suffix or, if compression is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving. Copied to archive WAL segments are synced to disk. +Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. +Copied to archive WAL segments are synced to disk. -You can set archive-push as archive_command in postgresql.conf to perform archive backups. +You can use archive-push in `archive_command` PostgreSQL parameter to set up continous WAl archiving. +For details, see section [Archiving Options](#archiving-options) -**archive-get** +#### archive-get pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f @@ -332,42 +407,20 @@ You can set archive-push as archive_command in postgresql.conf to perform archiv [remote_options] [logging_options] -Moves WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as restore_command in recovery.conf when restoring backups using a WAL archive. You do not need to set it manually. - -**checkdb** - - pg_probackup checkdb [-D data_dir] [-B backup_dir] [--instance instance_name] - [--help] [--progress] [-j num_threads] - [--amcheck [--heapallindexed] [--skip-block-validation]] - [connection_options] - [logging_options] - -Validates all data files located in the specified data directory by performing block-level checksum verification and page header sanity checks. If run with the --amcheck option, this command also performs logical verification of all indexes in the specified PostgreSQL instance using the amcheck extension or the `amcheck_next` extension. - -**version** - - pg_probackup version - -Prints pg_probackup version. - -**help** +Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as `restore_command` in **recovery.conf** when restoring backups using a WAL archive. You do not need to set it manually. - pg_probackup help [command] - -Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. +#### Options -##### Options This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. For details, see the section called “Configuring pg_probackup”. If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. ###### Common Options The list of general options. -``` + -B directory --backup-path=directory BACKUP_PATH -``` Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. -D directory @@ -381,7 +434,7 @@ Specifies the unique identifier of the backup. -j num_threads --threads=num_threads -Sets the number of parallel threads for backup, recovery, and backup validation processes. +Sets the number of parallel threads for backup, restore, merge, validation and verification processes. --progress Shows the progress of operations. @@ -410,7 +463,7 @@ For details, see the section called “Creating a Backup”. Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. --stream -Makes an autonomous backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. +Makes an STREAM backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. -S slot_name --slot=slot_name @@ -671,7 +724,8 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By - [Creating a Backup](#creating-a-backup) - [Validating a Backup](#vaklidating-a-backup) -- [Restoring a Cluster](#restoting-a-cluster) +- [Restoring a Cluster](#restoring-a-cluster) +- [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) - [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - [Configuring pg_probackup](#configuring-pg_probackup) @@ -680,11 +734,12 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By - [Merging Backups](#merging-backups) - [Deleting Backups](#deleting-backups) -##### Creating a Backup +#### Creating a Backup To create a backup, run the following command: - pg_probackup backup -B backup_dir --instance instance_name -b backup_mode -where backup_mode can take one of the following values: + pg_probackup backup -B backup_dir --instance instance_name -b **backup_mode** + +Where **backup_mode** can take one of the following values: - FULL — creates a full backup that contains all the data files of the cluster to be restored. - DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. @@ -692,96 +747,104 @@ where backup_mode can take one of the following values: When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. -If you have configured PTRACK backups, pg_probackup clears PTRACK bitmap of the relation being processed each time a full or an incremental backup is taken. Thus, the next incremental PTRACK backup contains only the pages that have changed since the previous backup. If a backup failed or was interrupted, some relations can already have their PTRACK forks cleared, so the next incremental backup will be incomplete. The same is true if ptrack_enable was turned off for some time. In this case, you must take a full backup before the next incremental PTRACK backup. +If [data checksums](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. +Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. +Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. -To make a backup autonomous, add the --stream option to the above command. For example, to create a full autonomous backup, run: +Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". + +##### STREAM mode + +To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot -The optional --temp-slot parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. +The optional `--temp-slot` parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. -Autonomous backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. To restore a cluster from an incremental autonomous backup, pg_probackup still requires the full backup and all the incremental backups it depends on. +STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. To restore a cluster from an incremental STREAM backup, pg_probackup still requires the full backup and all the incremental backups it depends on. -Even if you are using continuous archiving, autonomous backups can still be useful in the following cases: +Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: - Autonomous backups can be restored on the server that has no file access to WAL archive. + 1. STREAM backups can be restored on the server that has no file access to WAL archive. + 2. STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. + 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](#https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. - Autonomous backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. +##### External directories -To back up a directory located outside of the data directory, use the optional --external-dirs parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include /etc/dir1/ and /etc/dir2/ directories into the full backup of your node instance that will be stored under the node_backup directory, run: +To back up a directory located outside of the data directory, use the optional **--external-dirs** parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your node instance that will be stored under the node_backup directory, run: - pg_probackup backup -B node_backup --instance node -b FULL --external-dirs=/etc/dir1:/etc/dir2 + pg_probackup backup -B backup_dir --instance node -b FULL --external-dirs=/etc/dir1:/etc/dir2 -pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the set-config command with the --external-dirs option. +pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the **--external-dirs** option. -##### Validating Backups -When checksums are enabled for the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page. This guarantees that the backup is free of corrupted pages. Note that pg_probackup reads database files from disk and under heavy write load during backup it can show false positive checksum failures because of partial writes. +#### Validating Backups -Even if page checksums are disabled, pg_probackup calculates checksums for each file in a backup. Checksums are checked immediately after backup is taken and right before restore, to detect possible backup corruptions. If you would like to skip backup validation, you can specify the --no-validate option when running backup and restore commands. +pg_probackup calculates checksums for each file in a backup during `backup` process. The process of checking checksumms of backup data files is called **backup validation**. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. If you would like to skip backup validation, you can specify the `--no-validate` option when running `backup` and `restore` commands. -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the validate command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. +To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the `validate` command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=xid -If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time and transaction ID up to which the recovery is possible. +If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. + +#### Restoring a Cluster -##### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: pg_probackup restore -B backup_dir --instance instance_name -i backup_id -where: -- backup_dir is the backup catalog that stores all backup files and meta information. -- instance_name is the backup instance for the cluster to be restored. -- backup_id specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. +Where: +- **backup_dir** is the backup catalog that stores all backup files and meta information. +- **instance_name** is the backup instance for the cluster to be restored. +- **backup_id** specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. -If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the --tablespace-mapping option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. +If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. -When using the --tablespace-mapping option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: +When using the `--tablespace-mapping` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir -Once the restore command is complete, start the database service. If you are restoring an autonomous backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For archive backups, PostgreSQL replays all archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the recovery_target option with the restore command. Note that using the recovery-target=latest value with autonomous backups is only possible if the WAL archive is available at least starting from the time the autonomous backup was taken. +Once the restore command is complete, start the database service. If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. ->NOTE: By default, the restore command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the --no-validate option to skip validation and speed up the recovery. +>NOTE: By default, the `restore` command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. -**Performing Point-in-Time (PITR) Recovery** -If you have enabled continuous WAL archiving before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using recovery target options with the restore command instead of the -i option shown above. pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the recovery process. +#### Performing Point-in-Time (PITR) Recovery + +If you have enabled [continuous WAL archiving](#) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the `restore` command instead of the -i option shown above. pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process. - To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11' + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' - To restore the cluster state up to a specific transaction ID, use the recovery-target-xid option: pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 -- If you know the exact LSN up to which you need to restore the data, use recovery-target-lsn: - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 +- To restore the cluster state up to a specific LSN, use recovery-target-lsn: -By default, the recovery_target_inclusive parameter defines whether the recovery target is included into the backup. You can explicitly include or exclude the recovery target by adding the --recovery-target-inclusive=boolean option to the commands listed above. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 ##### Using pg_probackup in the Remote Mode -pg_probackup supports the remote backup mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. +pg_probackup supports the remote mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. The typical workflow is as follows: - - On your local system, configure pg_probackup as explained in the section called “Installation and Setup”. For the add-instance and set-config commands, make sure to specify remote backup options that point to the remote server with the PostgreSQL instance. + - On your local system, configure pg_probackup as explained in the section called [Installation and Setup](#installation-and-setup). For the `add-instance` and `set-config` commands, make sure to specify [remote options](#remote-options) that point to the remote server with the PostgreSQL instance. -- If you would like to take archive backups, configure continuous WAL archiving on the remote system as explained in the section called “Setting up Archive Backups”. For the archive-push and archive-get commands, you must specify the remote backup options that point to your local system. +- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section called [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the `archive-push` and `archive-get` commands, you must specify the [remote options](#remote-options) that point to host with **backup catalog**. -- Run backup or restore commands with remote backup options on your local system. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +- Run `backup` or `restore` commands with remote options on system with **backup catalog**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. >NOTE: The remote backup mode is currently unavailable for Windows systems. ##### Running pg_probackup on Parallel Threads -Backup, recovery, and validation processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). +Backup, recovery, merge, checkdb and validate processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). -Parallel execution is controlled by the -j/--threads command line option. For example, to create a backup using four parallel threads, run: +Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 @@ -791,15 +854,15 @@ Parallel execution is controlled by the -j/--threads command line option. For ex Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the backups/instance_name directory to fine-tune pg_probackup configuration. -Since pg_probackup uses a regular PostgreSQL connection and, for STREAM backups, the replication protocol, pg_probackup `backup` and `checkdb` commands require connection options. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the *set-config* command. +For example, `backup` and `checkdb` commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the `set-config` command. Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. - system-identifier — the unique identifier of the PostgreSQL instance. -Additionally, you can define connection, retention, logging, and replica settings using the set-config command: +Additionally, you can define remote, retention, logging and compression settings using the `set-config` command: - pg_probackup set-config -B backup_dir --instance instance_name --external-dirs=external_directory_path [connection_options] [retention_options] [logging_options] [replica_options] + pg_probackup set-config -B backup_dir --instance instance_name --external-dirs=external_directory_path [remote_options] [connection_options] [retention_options] [logging_options] To view the current settings, run the following command: @@ -807,7 +870,7 @@ To view the current settings, run the following command: You can override the settings defined in pg_probackup.conf when running the backup command via corresponding environment variables and/or command line options. -**Specifying Connection Settings** +###### Specifying Connection Settings If you define connection settings in the pg_probackup.conf configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. @@ -980,8 +1043,8 @@ BACKUP INSTANCE 'node' node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK ``` -Even though P7XDHB and P7XDHU backups are outside the retention window, P7XDHB and P7XDHU cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the --delete-expired option, only the P7XDFT full backup will be removed. -With the --merge-expired option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: +Even though P7XDHB and P7XDHU backups are outside the retention window, P7XDHB and P7XDHU cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the `--delete-expired` option, only the P7XDFT full backup will be removed. +With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: pg_probackup delete -B node-backup --instance node --delete-expired --merge-expired pg_probackup show -B node-backup @@ -1014,26 +1077,26 @@ To delete a backup that is no longer required, run the following command: pg_probackup delete -B backup_dir --instance instance_name -i backup_id -This command will delete the backup with the specified backup_id, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. -In this case, the next PTRACK backup will be incomplete as some changes since the last retained backup will be lost. Either a full backup or an incremental PAGE backup (if all the necessary WAL files are still present in the archive) must be taken then. +This command will delete the backup with the specified **backup_id**, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. -To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the --wal option: +To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` option: pg_probackup delete -B backup_dir --instance instance_name --delete-wal -To delete backups that are expired according to the current retention policy, use the --delete-expired option: +To delete backups that are expired according to the current retention policy, use the `--delete-expired` option: pg_probackup delete -B backup_dir --instance instance_name --delete-expired -Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the --merge-expired option when running this command: +Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the `--merge-expired` option when running this command: pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. -Before merging or deleting backups, you can run the delete command with the dry-run option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. +Before merging or deleting backups, you can run the delete command with the `--dry-run` option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. ### Authors PostgreSQLfessional, Moscow, Russia. + ### Credits pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. From 107050800b6f649311d4d3db596845789545e0f7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 12:32:49 +0300 Subject: [PATCH 0770/2107] update Documentation --- Documentation.md | 173 ++++++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 86 deletions(-) diff --git a/Documentation.md b/Documentation.md index 9ffd82199..ee4853a2d 100644 --- a/Documentation.md +++ b/Documentation.md @@ -6,61 +6,61 @@ Current version - 2.1.3 1. [Synopsis](#synopsis) 2. [Overview](#overview) - 2.1. [Limitations](#limitations) + * [Limitations](#limitations) 3. [Installation and Setup](#installation-and-setup) - 3.1. [Initializing the Backup Catalog](#initializing-the-backup-catalog) - 3.2. [Adding a New Backup Instance](#adding-a-new-backup-instance) - 3.3. [Configuring the Database Cluster](#configuring-the-database-cluster) - 3.4. [Setting up STREAM Backups](#setting-up-stream-backups) - 3.5. [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) - 3.6. [Setting up backup from Standby](#backup-from-standby) - 3.7. [Setting up PTRACK Backups](#setting-up-ptrack-backups) + * [Initializing the Backup Catalog](#initializing-the-backup-catalog) + * [Adding a New Backup Instance](#adding-a-new-backup-instance) + * [Configuring the Database Cluster](#configuring-the-database-cluster) + * [Setting up STREAM Backups](#setting-up-stream-backups) + * [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) + * [Setting up backup from Standby](#backup-from-standby) + * [Setting up PTRACK Backups](#setting-up-ptrack-backups) 4. [Command-Line Reference](#command-line-reference) - 4.1 [Commands](#commands) - 4.1.1 [version](#version) - 4.1.2 [help](#help) - 4.1.3 [add-instance](#add-instance) - 4.1.4 [del-instance](#del-instance) - 4.1.5 [set-config](#set-config) - 4.1.6 [show-config](#show-config) - 4.1.7 [show](#show) - 4.1.8 [backup](#backup) - 4.1.9 [restore](#restore) - 4.1.10 [validate](#validate) - 4.1.11 [merge](#merge) - 4.1.12 [delete](#delete) - 4.1.13 [archive-push](#archive-push) - 4.1.14 [archive-get](#archive-get) - - 4.2 [Options](#options) - 4.2.1 [Common Options](#common-options) - 4.2.2 [Backup Options](#backup-options) - 4.2.3 [Restore Options](#restore-options) - 4.2.4 [Checkdb Options](#checkdb-options) - 4.2.5 [Recovery Target Options](#recovery-target-options) - 4.2.6 [Retention Options](#retention-options) - 4.2.7 [Logging Options](#logging-options) - 4.2.8 [Connection Options](#connection-options) - 4.2.9 [Compression Options](#compression-ptions) - 4.2.10 [Archiving Options](#archiving-options) - 4.2.11 [Remote Mode Options](#remote-mode-options) - 4.2.12 [Compression Options](#compression-ptions) - 4.2.13 [Replica Options](#replica-options) + * [Commands](#commands) + * [version](#version) + * [help](#help) + * [add-instance](#add-instance) + * [del-instance](#del-instance) + * [set-config](#set-config) + * [show-config](#show-config) + * [show](#show) + * [backup](#backup) + * [restore](#restore) + * [validate](#validate) + * [merge](#merge) + * [delete](#delete) + * [archive-push](#archive-push) + * [archive-get](#archive-get) + + * [Options](#options) + * [Common Options](#common-options) + * [Backup Options](#backup-options) + * [Restore Options](#restore-options) + * [Checkdb Options](#checkdb-options) + * [Recovery Target Options](#recovery-target-options) + * [Retention Options](#retention-options) + * [Logging Options](#logging-options) + * [Connection Options](#connection-options) + * [Compression Options](#compression-ptions) + * [Archiving Options](#archiving-options) + * [Remote Mode Options](#remote-mode-options) + * [Compression Options](#compression-ptions) + * [Replica Options](#replica-options) 5. [Usage](#usage) - 5.1 [Creating a Backup](#creating-a-backup) - 5.2 [Validating a Backup](#vaklidating-a-backup) - 5.3 [Restoring a Cluster](#restoring-a-cluster) - 5.4 [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) - 5.5 [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - 5.6 [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - 5.7 [Configuring pg_probackup](#configuring-pg_probackup) - 5.8 [Managing the Backup Catalog](#managing-the-backup-Catalog) - 5.9 [Configuring Backup Retention Policy](#configuring-backup-retention-policy) - 5.10 [Merging Backups](#merging-backups) - 5.11 [Deleting Backups](#deleting-backups) + * [Creating a Backup](#creating-a-backup) + * [Validating a Backup](#vaklidating-a-backup) + * [Restoring a Cluster](#restoring-a-cluster) + * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) + * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) + * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) + * [Configuring pg_probackup](#configuring-pg_probackup) + * [Managing the Backup Catalog](#managing-the-backup-Catalog) + * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) + * [Merging Backups](#merging-backups) + * [Deleting Backups](#deleting-backups) 6. [Authors](#authors) 7. [Credits](#credits) @@ -261,39 +261,39 @@ If you are going to use PTRACK backups, complete the following additional steps: This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). -#### version +##### version pg_probackup version Prints pg_probackup version. -#### help +##### help pg_probackup help [command] Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. -#### init +##### init pg_probackup init -B backup_dir [--help] Initializes the backup_dir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backup_dir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. -#### add-instance +##### add-instance pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section called [Adding a New Backup Instance](#adding-a-new-backup-instance). -#### del-instance +##### del-instance pg_probackup del-instance -B backup_dir --instance instance_name [--help] -Deletes all backups and WAL files associated with the specified instance. +Deletes all backups and WAL files associated with the specified instance. -#### set-config +##### set-config pg_probackup set-config -B backup_dir --instance instance_name [--help] [--pgdata=pgdata-path] @@ -304,9 +304,9 @@ Deletes all backups and WAL files associated with the specified instance. [remote_options] [logging_options] -Adds the specified connection, retention, logging or replica, and compression, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. +Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. -#### show-config +##### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] @@ -314,14 +314,14 @@ Displays the contents of the pg_probackup.conf configuration file located in the To edit pg_probackup.conf, use the set-config command. It is **not recommended** to edit pg_probackup.conf directly. -#### show +##### show pg_probackup show -B backup_dir [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. -#### backup +##### backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name [--help] [-j num_threads] [--progress] @@ -338,7 +338,7 @@ Shows the contents of the backup catalog. If instance_name and backup_id are spe Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. For details, see the section [Creating a Backup](#creating-a-backup). -#### restore +##### restore pg_probackup restore -B backup_dir --instance instance_name [--help] [-D data_dir] [-i backup_id] @@ -352,7 +352,7 @@ For details, see the section [Creating a Backup](#creating-a-backup). Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. For details, see the section [Restoring a Cluster](#restoring-a-cluster). -#### validate +##### validate pg_probackup validate -B backup_dir [--help] [--instance instance_name] [-i backup_id] @@ -363,7 +363,7 @@ For details, see the section [Restoring a Cluster](#restoring-a-cluster). Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a recovery target option and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. -#### merge +##### merge pg_probackup merge -B backup_dir --instance instance_name -i backup_id [--help] [-j num_threads][--progress] @@ -372,7 +372,7 @@ Verifies that all the files required to restore the cluster are present and not Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. For details, see the section [Merging Backups](#merging-backups). -#### delete +##### delete pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] @@ -383,7 +383,7 @@ For details, see the section [Merging Backups](#merging-backups). Deletes backup with specified **backip_id** or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-otions) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). -#### archive-push +##### archive-push pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f @@ -392,14 +392,14 @@ For details, see the sections [Deleting Backups](#deleting-backups), [Retention [remote_options] [logging_options] -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by instance_name, system-identifier, and PGDATA. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. +Copies WAL files into the corresponding subdirectory of the backup catalog, and validates the backup instance by instance_name, system-identifier. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. -You can use archive-push in `archive_command` PostgreSQL parameter to set up continous WAl archiving. +You can use archive-push in `archive_command` PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). For details, see section [Archiving Options](#archiving-options) -#### archive-get +##### archive-get pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f @@ -415,7 +415,7 @@ This section describes all command-line options for pg_probackup commands. If th If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. -###### Common Options +##### Common Options The list of general options. -B directory @@ -442,7 +442,7 @@ Shows the progress of operations. --help Shows detailed information about the options that can be used with this command. -###### Backup Options +##### Backup Options The following options can be used together with the `backup` command. Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options) and [Logging Options](#logging-options) can be used. @@ -489,7 +489,7 @@ Disables block-level checksum verification to speed up backup. --no-validate Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. -###### Restore Options +##### Restore Options The following options can be used together with the `restore` command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. -R | --restore-as-replica @@ -524,8 +524,8 @@ Skip validation of data files. --heapallindexed Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. -###### Recovery Target Options -If continuous WAL archiving is configured, you can use one of these options together with `restore` or `validate` commands to specify the moment up to which the database cluster must be restored. +##### Recovery Target Options +If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with `restore` or `validate` commands to specify the moment up to which the database cluster must be restored. --recovery-target=immediate|latest Defines when to stop the recovery: @@ -554,7 +554,7 @@ Specifies whether to stop just after the specified recovery target (true), or ju Default: pause Specifies the action the server should take when the recovery target is reached, similar to the `recovery_target_action` option in the recovery.conf configuration file. -###### Retention Options +##### Retention Options You can use these options together with `backup` or `delete` commands. For details on configuring retention policy, see the section called [Configuring Backup Retention Policy](#configuring-backup-retention-policy). --retention-redundancy=redundancy @@ -577,7 +577,7 @@ Merges the oldest incremental backup that satisfies the requirements of retentio --dry-run Displays the current status of all the available backups, without deleting or merging expired backups, if any. -###### Logging Options +##### Logging Options You can use these options with any command. --log-level-console=log_level @@ -610,7 +610,7 @@ Maximum size of an individual log file. If this value is reached, the log file i Default: 0 Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). -###### Connection Options +##### Connection Options You can use these options together with `backup` and `checkdb` commands. -d dbname @@ -643,7 +643,7 @@ User name to connect as. --password Forces a password prompt. -###### Compression Options +##### Compression Options You can use these options together with `backup` and `archive-push` commands. --compress-algorithm=compression_algorithm @@ -658,7 +658,7 @@ Defines compression level (0 through 9, 0 being no compression and 9 being best --compress Alias for --compress-algorithm=zlib and --compress-level=1. -###### Archiving Options +##### Archiving Options These options can be used with `archive-push` and `archive-get` commands. --wal-file-path=wal_file_path %p @@ -670,7 +670,7 @@ Provides the name of the WAL file in archive_command and restore_command used by --overwrite Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. -###### Remote Mode Options +##### Remote Mode Options This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with `add-instance`, `set-config`, `backup`, `restore`, `archive-push` and `archive-get` commands. For details on configuring remote operation mode, see the section called [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). --remote-proto @@ -697,7 +697,7 @@ Specifies pg_probackup installation directory on the remote system. --ssh-options Specifies a string of SSH command-line options. -###### Replica Options +##### Replica Options This section describes the options related to taking a backup from standby. >NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. @@ -812,7 +812,7 @@ Once the restore command is complete, start the database service. If you are res #### Performing Point-in-Time (PITR) Recovery -If you have enabled [continuous WAL archiving](#) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the `restore` command instead of the -i option shown above. pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process. +If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the `restore` command. If `-i/--backup-id` option is omitted pg_probackup automatically chooses the backup that is the closest to the specified **recovery target** and starts the restore process, otherwise pg_probackup will try to restore **backup_id** to the specified **recovery target**. - To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: @@ -828,7 +828,7 @@ If you have enabled [continuous WAL archiving](#) before taking backups, you can ##### Using pg_probackup in the Remote Mode -pg_probackup supports the remote mode that allows to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. +pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. The typical workflow is as follows: @@ -842,7 +842,7 @@ The typical workflow is as follows: ##### Running pg_probackup on Parallel Threads -Backup, recovery, merge, checkdb and validate processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). +Backup, recovery, merge, delete, checkdb and validate processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: @@ -872,17 +872,18 @@ You can override the settings defined in pg_probackup.conf when running the back ###### Specifying Connection Settings -If you define connection settings in the pg_probackup.conf configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. +If you define connection settings in the **pg_probackup.conf** configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. -If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix socket(localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. +If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. ##### Managing the Backup Catalog With pg_probackup, you can manage backups from the command line: - View available backups +- Validate backups - Merge backups -- Delete backups +- Delete backups - Viewing Backup Information To view the list of existing backups for every instance, run the command: From b4c41303bc32d57e03820bb6e22c8ca4607dc8f1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 13:11:10 +0300 Subject: [PATCH 0771/2107] update Documentation --- Documentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index ee4853a2d..6beb75684 100644 --- a/Documentation.md +++ b/Documentation.md @@ -133,7 +133,7 @@ pg_probackup can take only physical online backups, and online backups require W pg_probackup currently has the following limitations: - Creating backups from a remote server is currently not supported on Windows systems. -- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. +- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. ### Installation and Setup @@ -218,7 +218,7 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK >NOTE: Even if [continuous WAL archiving](#setting-up-continuous-wal-archiving) is set up, you may still take STREAM backups. It may be useful in some rare cases, such as backup from stale standby. #### Setting up continuous WAL archiving -ARCHIVE backups require [continious WAL archiving](#https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: +ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - Make sure the `wal_level` parameter is higher than 'minimal'. - Set the `archive_mode` parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. @@ -747,7 +747,7 @@ Where **backup_mode** can take one of the following values: When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. -If [data checksums](#https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. +If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. @@ -767,7 +767,7 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin 1. STREAM backups can be restored on the server that has no file access to WAL archive. 2. STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. - 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](#https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. + 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. ##### External directories From 7be04839d9b7a939b0a980b0eab0ed4f8d8aabd5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 16:46:30 +0300 Subject: [PATCH 0772/2107] update Documentation --- Documentation.md | 60 +++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6beb75684..37ce75e9d 100644 --- a/Documentation.md +++ b/Documentation.md @@ -104,7 +104,7 @@ Current version - 2.1.3 As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -- Incremental backup: page-level incremental backup of three different types allows you to devise the backup strategy in accordance with your data flow +- Incremental backup: page-level incremental backup of three different modes allows you to devise the backup strategy in accordance with your data flow - Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery - Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb` - Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` @@ -114,6 +114,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Remote operations: Backup PostgreSQL instance located on remote machine or restore backup on it - Backup from replica: Avoid extra load on the master server by taking backups from a standby - External directories: Add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files +- Backup Catalog: get list of backups and all corresponding metadata about them in `plain` or `json` formats To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single **backup catalog**. @@ -124,7 +125,7 @@ Using pg_probackup, you can take full or incremental backups: - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen **backup type** (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following **WAL delivery methods**: +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen **backup mode** (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following **WAL delivery methods**: - ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. - STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. @@ -163,21 +164,20 @@ pg_probackup can store backups for multiple database clusters in a single backup To add a new backup instance, run the following command: - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name - [remote_options] + pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] where: - **data_dir** is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. - **instance_name** is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional [remote_options](#remote-options) should be used if **data_dir** is located on remote machine. +- The optional parameters [remote_options](#remote-options) should be used if **data_dir** is located on remote machine. -pg_probackup creates the **instance_name** subdirectories under the **backups/* and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). +pg_probackup creates the **instance_name** subdirectories under the **backups/** and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). The user launching pg_probackup must have full access to **backup_dir** directory and at least read-only access to **data_dir** directory. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. #### Configuring the Database Cluster -Although pg_probackup can be used by a superuser, it is recommended to create a separate user or role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the **backup** role is used as an example. +Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the **backup** role is used as an example. To enable backups, the following rights are required: @@ -197,7 +197,7 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section called [Backup from Standby](#backup-from-standby). +Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section [Backup from Standby](#backup-from-standby). #### Setting up STREAM Backups @@ -211,7 +211,7 @@ To set up the cluster for STREAM backups, complete the following steps: - Make sure the parameter `max_wal_senders` is set high enough to leave at least one session available for the backup process. - Set the parameter `wal_level` to be higher than `minimal`. -If you are going to take PAGE backups in STREAM mode, you also have to configure WAL archiving as explained in the section called [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). +Even if you are planning to take PAGE backups in STREAM mode, you have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in STREAM mode. @@ -220,10 +220,12 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK #### Setting up continuous WAL archiving ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - - Make sure the `wal_level` parameter is higher than 'minimal'. - - Set the `archive_mode` parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. - - Set the `archive_command` parameter, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_options] --wal-file-path %p --wal-file-name %f + - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. + - Set the [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. + - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: +``` + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_options] --wal-file-path %p --wal-file-name %f +``` Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-options) should be used to archive WAL to the remote machine. Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. @@ -235,16 +237,16 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: -- On the standby server, set the parameter `hot_standby` to `on`. -- On the master server, set the parameter `full_page_writes` to `on`. +- On the standby server, set the parameter [hot_standby](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-HOT-STANDBY) to `on`. +- On the master server, set the parameter [full_page_writes](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-FULL-PAGE-WRITES) to `on`. - To perform STREAM backup on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) - To perform ARCHIVE backup on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) -Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK Backups backups of appropriate WAL delivery method, ARCHIVE or STREAM, from the standby server. +Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups of appropriate WAL delivery method: ARCHIVE or STREAM, from the standby server. >NOTE: Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable full_page_writes on the master, and not to use a tool like pg_compresslog as archive_command to remove full-page writes from WAL files. +- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as `archive_command` to remove full-page writes from WAL files. #### Setting up PTRACK Backups @@ -254,7 +256,7 @@ If you are going to use PTRACK backups, complete the following additional steps: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; - The *backup* role must have access to all the databases of the cluster. +- The *backup* role must have access to all the databases of the cluster. ### Command-Line Reference #### Commands @@ -284,7 +286,7 @@ Initializes the backup_dir backup catalog that will store backup copies, WAL arc pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] -Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section called [Adding a New Backup Instance](#adding-a-new-backup-instance). +Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). ##### del-instance @@ -335,7 +337,7 @@ Shows the contents of the backup catalog. If instance_name and backup_id are spe [retention_options] [logging_options] -Creates a backup copy of the PostgreSQL instance. The backup_mode option specifies the backup mode to use. +Creates a backup copy of the PostgreSQL instance. The **backup_mode** option specifies the backup mode to use. For details, see the section [Creating a Backup](#creating-a-backup). ##### restore @@ -411,7 +413,7 @@ Copies WAL files from the corresponding subdirectory of the backup catalog to th #### Options -This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. For details, see the section called “Configuring pg_probackup”. +This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. For details, see the section “Configuring pg_probackup”. If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. @@ -455,7 +457,7 @@ Specifies the backup mode to use. Possible values are: - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. - PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. -For details, see the section called “Creating a Backup”. +For details, see the section “Creating a Backup”. -C --smooth-checkpoint @@ -555,7 +557,7 @@ Specifies whether to stop just after the specified recovery target (true), or ju Specifies the action the server should take when the recovery target is reached, similar to the `recovery_target_action` option in the recovery.conf configuration file. ##### Retention Options -You can use these options together with `backup` or `delete` commands. For details on configuring retention policy, see the section called [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +You can use these options together with `backup` or `delete` commands. For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). --retention-redundancy=redundancy Default: 0 @@ -671,7 +673,7 @@ Provides the name of the WAL file in archive_command and restore_command used by Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. ##### Remote Mode Options -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with `add-instance`, `set-config`, `backup`, `restore`, `archive-push` and `archive-get` commands. For details on configuring remote operation mode, see the section called [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). +This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with `add-instance`, `set-config`, `backup`, `restore`, `archive-push` and `archive-get` commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: @@ -737,7 +739,7 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By #### Creating a Backup To create a backup, run the following command: - pg_probackup backup -B backup_dir --instance instance_name -b **backup_mode** + pg_probackup backup -B backup_dir --instance instance_name -b backup_mode Where **backup_mode** can take one of the following values: - FULL — creates a full backup that contains all the data files of the cluster to be restored. @@ -753,11 +755,11 @@ Page is considered corrupted if checksumm comparison failed more than 100 times, Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". -##### STREAM mode +##### STREAM To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: - pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot + pg_probackup backup -B backup_dir --instance instance_name -b FULL **--stream** --temp-slot The optional `--temp-slot` parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. @@ -832,9 +834,9 @@ pg_probackup supports the remote mode that allows to perform `backup` and `resto The typical workflow is as follows: - - On your local system, configure pg_probackup as explained in the section called [Installation and Setup](#installation-and-setup). For the `add-instance` and `set-config` commands, make sure to specify [remote options](#remote-options) that point to the remote server with the PostgreSQL instance. + - On your local system, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the `add-instance` and `set-config` commands, make sure to specify [remote options](#remote-options) that point to the remote server with the PostgreSQL instance. -- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section called [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the `archive-push` and `archive-get` commands, you must specify the [remote options](#remote-options) that point to host with **backup catalog**. +- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the `archive-push` and `archive-get` commands, you must specify the [remote options](#remote-options) that point to host with **backup catalog**. - Run `backup` or `restore` commands with remote options on system with **backup catalog**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. From 60fcc3b097e5bfd7263b4c123dcfec319e14444d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 20:34:45 +0300 Subject: [PATCH 0773/2107] update Documentation --- Documentation.md | 68 ++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/Documentation.md b/Documentation.md index 37ce75e9d..69704a40f 100644 --- a/Documentation.md +++ b/Documentation.md @@ -21,6 +21,7 @@ Current version - 2.1.3 * [Commands](#commands) * [version](#version) * [help](#help) + * [init](#init) * [add-instance](#add-instance) * [del-instance](#del-instance) * [set-config](#set-config) @@ -43,17 +44,16 @@ Current version - 2.1.3 * [Retention Options](#retention-options) * [Logging Options](#logging-options) * [Connection Options](#connection-options) - * [Compression Options](#compression-ptions) + * [Compression Options](#compression-options) * [Archiving Options](#archiving-options) * [Remote Mode Options](#remote-mode-options) - * [Compression Options](#compression-ptions) * [Replica Options](#replica-options) 5. [Usage](#usage) * [Creating a Backup](#creating-a-backup) - * [Validating a Backup](#vaklidating-a-backup) + * [Validating a Backup](#validating-a-backup) * [Restoring a Cluster](#restoring-a-cluster) - * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) + * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) * [Configuring pg_probackup](#configuring-pg_probackup) @@ -307,14 +307,15 @@ Deletes all backups and WAL files associated with the specified instance. [logging_options] Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. +It is **not recommended** to edit pg_probackup.conf manually. ##### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] -Displays the contents of the pg_probackup.conf configuration file located in the backup_dir/backups/instance_name directory. You can specify the --format=json option to return the result in the JSON format. By default, configuration settings are shown as plain text. -To edit pg_probackup.conf, use the set-config command. -It is **not recommended** to edit pg_probackup.conf directly. +Displays the contents of the pg_probackup.conf configuration file located in the **backup_dir/backups/instance_name** directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. +To edit pg_probackup.conf, use the [set-config](#set-config) command. +It is **not recommended** to edit pg_probackup.conf manually. ##### show @@ -338,7 +339,7 @@ Shows the contents of the backup catalog. If instance_name and backup_id are spe [logging_options] Creates a backup copy of the PostgreSQL instance. The **backup_mode** option specifies the backup mode to use. -For details, see the section [Creating a Backup](#creating-a-backup). +For details, see the sections [Backup Options](#backup-options) and [Creating a Backup](#creating-a-backup). ##### restore @@ -352,7 +353,7 @@ For details, see the section [Creating a Backup](#creating-a-backup). [remote_options] Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. -For details, see the section [Restoring a Cluster](#restoring-a-cluster). +For details, see the sections [Restore Options](#backup-options) and [Restoring a Cluster](#restoring-a-cluster). ##### validate @@ -363,7 +364,8 @@ For details, see the section [Restoring a Cluster](#restoring-a-cluster). [recovery_options] [logging_options] -Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a recovery target option and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. +Verifies that all the files required to restore the cluster are present and not corrupted. If **instance_name** is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the **instance_name** without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the **instance_name** with a [recovery target option](#recovery-target-options) and/or a **backup_id**, pg_probackup checks whether it is possible to restore the cluster using these options. +For details, see the section [Validating a Backup](#validating-a-backup). ##### merge @@ -409,11 +411,12 @@ For details, see section [Archiving Options](#archiving-options) [remote_options] [logging_options] -Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as `restore_command` in **recovery.conf** when restoring backups using a WAL archive. You do not need to set it manually. +Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in **recovery.conf** when restoring backups using a WAL archive. You do not need to set it manually. #### Options -This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. For details, see the section “Configuring pg_probackup”. +This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. +For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. @@ -423,12 +426,12 @@ The list of general options. -B directory --backup-path=directory BACKUP_PATH -Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. +Specifies the absolute path to the **backup catalog**. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. -D directory --pgdata=directory PGDATA -Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the init command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. +Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the [add-instance](#add-instance) command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. -i backup_id -backup-id=backup_id @@ -446,7 +449,7 @@ Shows detailed information about the options that can be used with this command. ##### Backup Options -The following options can be used together with the `backup` command. Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options) and [Logging Options](#logging-options) can be used. +The following options can be used together with the [backup](#backup) command. Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -b mode --backup-mode=mode @@ -457,7 +460,7 @@ Specifies the backup mode to use. Possible values are: - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. - PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. -For details, see the section “Creating a Backup”. +For details, see the section [Creating a Backup](#creating-a-backup). -C --smooth-checkpoint @@ -469,10 +472,10 @@ Makes an STREAM backup that includes all the necessary WAL files by streaming th -S slot_name --slot=slot_name -Specifies the replication slot for WAL streaming. This option can only be used together with the --stream option. +Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` option. --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S --slot`. +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S / --slot`. --backup-pg-log Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. @@ -480,7 +483,7 @@ Includes the log directory into the backup. This directory usually contains log -E external_directory_path --external-dirs=external_directory_path -Includes the specified directory into the backup. This option is useful to back up configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. +Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. --archive-timeout=wait_time Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. @@ -492,7 +495,7 @@ Disables block-level checksum verification to speed up backup. Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. ##### Restore Options -The following options can be used together with the `restore` command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. +The following options can be used together with the [restore](#restore) command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -R | --restore-as-replica Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. @@ -705,7 +708,7 @@ This section describes the options related to taking a backup from standby. --master-db=dbname Default: postgres, the default PostgreSQL database. -Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the set-config command. +Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the [set-config](#set-config) command. --master-host=host Deprecated. Specifies the host name of the system on which the master server is running. @@ -720,18 +723,18 @@ Deprecated. User name to connect as. --replica-timeout=timeout Default: 300 sec -Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the set-config command. +Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. ### Usage - [Creating a Backup](#creating-a-backup) - [Validating a Backup](#vaklidating-a-backup) - [Restoring a Cluster](#restoring-a-cluster) -- [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-(pitr)-recovery) +- [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) - [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - [Configuring pg_probackup](#configuring-pg_probackup) -- [Managing the Backup Catalog](#managing-the-backup-Catalog) +- [Managing the Backup Catalog](#managing-the-backup-catalog) - [Configuring Backup Retention Policy](#configuring-backup-retention-policy) - [Merging Backups](#merging-backups) - [Deleting Backups](#deleting-backups) @@ -844,27 +847,30 @@ The typical workflow is as follows: ##### Running pg_probackup on Parallel Threads -Backup, recovery, merge, delete, checkdb and validate processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). +[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and validate[#validate] processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 ->NOTE: Parallel recovery applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. +>NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. ##### Configuring pg_probackup -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the backups/instance_name directory to fine-tune pg_probackup configuration. +Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the **backup_dir/backups/instance_name** directory to fine-tune pg_probackup configuration. -For example, `backup` and `checkdb` commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the `set-config` command. +For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. + +> Note: It is **not recommended** to edit pg_probackup.conf manually. Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. -- system-identifier — the unique identifier of the PostgreSQL instance. +- system-identifier — the unique identifier of the PostgreSQL instance. -Additionally, you can define remote, retention, logging and compression settings using the `set-config` command: +Additionally, you can define [remote](#remote-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: - pg_probackup set-config -B backup_dir --instance instance_name --external-dirs=external_directory_path [remote_options] [connection_options] [retention_options] [logging_options] + pg_probackup set-config -B backup_dir --instance instance_name + [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] To view the current settings, run the following command: From 637724920dde17a5e9016c9a632d4cc5def1addc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 22:55:05 +0300 Subject: [PATCH 0774/2107] add section "Verifying a Cluster" to Documentation --- Documentation.md | 103 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/Documentation.md b/Documentation.md index 69704a40f..9fb1cb3b3 100644 --- a/Documentation.md +++ b/Documentation.md @@ -29,6 +29,7 @@ Current version - 2.1.3 * [show](#show) * [backup](#backup) * [restore](#restore) + * [checkdb](#checkdb) * [validate](#validate) * [merge](#merge) * [delete](#delete) @@ -50,6 +51,7 @@ Current version - 2.1.3 * [Replica Options](#replica-options) 5. [Usage](#usage) + * [Verifying a Cluster](#verifying-a-cluster) * [Creating a Backup](#creating-a-backup) * [Validating a Backup](#validating-a-backup) * [Restoring a Cluster](#restoring-a-cluster) @@ -169,9 +171,9 @@ To add a new backup instance, run the following command: where: - **data_dir** is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. - **instance_name** is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional parameters [remote_options](#remote-options) should be used if **data_dir** is located on remote machine. +- The optional parameters [remote_options](#remote-mode-options) should be used if **data_dir** is located on remote machine. -pg_probackup creates the **instance_name** subdirectories under the **backups/** and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). +pg_probackup creates the **instance_name** subdirectories under the **backups/** and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). The user launching pg_probackup must have full access to **backup_dir** directory and at least read-only access to **data_dir** directory. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. @@ -197,7 +199,7 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections called [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and the section [Backup from Standby](#backup-from-standby). +Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and [Setting up Backup from Standby](#backup-from-standby). #### Setting up STREAM Backups @@ -208,30 +210,27 @@ To set up the cluster for STREAM backups, complete the following steps: - In the pg_hba.conf file, allow replication on behalf of the backup role. - Modify the postgresql.conf configuration file of the PostgreSQL server, as follows: - - Make sure the parameter `max_wal_senders` is set high enough to leave at least one session available for the backup process. - - Set the parameter `wal_level` to be higher than `minimal`. + - Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. + - Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. -Even if you are planning to take PAGE backups in STREAM mode, you have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). +If you are planning to take PAGE backups in STREAM mode, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in STREAM mode. ->NOTE: Even if [continuous WAL archiving](#setting-up-continuous-wal-archiving) is set up, you may still take STREAM backups. It may be useful in some rare cases, such as backup from stale standby. - #### Setting up continuous WAL archiving ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. - - Set the [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) parameter. If you are configuring backups on master, `archive_mode` must be set to `on`. To perform archiving on standby, set this parameter to `always`. + - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: -``` - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name [remote_options] --wal-file-path %p --wal-file-name %f -``` - Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-options) should be used to archive WAL to the remote machine. + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options] + + Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to 'backup_dir/wal/instance_name' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using `pg_receivewal`. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to **backup_dir/wal/instance_name** directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using `pg_receivewal`. #### Backup from Standby @@ -244,7 +243,7 @@ For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby serve Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups of appropriate WAL delivery method: ARCHIVE or STREAM, from the standby server. ->NOTE: Backup from the standby server has the following limitations: +Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. - All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as `archive_command` to remove full-page writes from WAL files. @@ -279,7 +278,8 @@ Displays the synopsis of pg_probackup commands. If one of the pg_probackup comma pg_probackup init -B backup_dir [--help] -Initializes the backup_dir backup catalog that will store backup copies, WAL archive, and meta information for the backed up database clusters. If the specified backup_dir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. +Initializes the backup catalog in **backup_dir** that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified **backup_dir** already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. +For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). ##### add-instance @@ -355,6 +355,18 @@ For details, see the sections [Backup Options](#backup-options) and [Creating a Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. For details, see the sections [Restore Options](#backup-options) and [Restoring a Cluster](#restoring-a-cluster). +##### checkdb + + pg_probackup checkdb + [-B backup_dir] [--instance instance_name] [-D data_dir] + [--help] [-j num_threads] [--progress] + [--skip-block-validation] [--amcheck] [--heapallindexed] + [logging_options] + [connection_options] + +Verifyes the PostgreSQL database cluster correctness by detecting physical and logical corruption. +For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying a Cluster](#verifying-a-cluster). + ##### validate pg_probackup validate -B backup_dir @@ -518,23 +530,23 @@ Disables block-level checksum verification to speed up validation. During automa Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. ###### Checkdb Options -The following options can be used together with the `checkdb` command. Additionally [Connection Options](#connection-options) and [Logging Options](#logging-options) can be used. +The following options can be used together with the [checkdb](#checkdb) command. For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster) --amcheck Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. --skip-block-validation -Skip validation of data files. +Skip validation of data files. Can be used only with `--amcheck` option, so only logical verification of indexes is performed. --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the --amcheck option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. +Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the `--amcheck` option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. ##### Recovery Target Options -If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with `restore` or `validate` commands to specify the moment up to which the database cluster must be restored. +If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored. --recovery-target=immediate|latest Defines when to stop the recovery: -- The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the -i option is omitted. +- The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. - The latest value continues the recovery until all WAL segments available in the archive are applied. --recovery-target-timeline=timeline @@ -553,14 +565,14 @@ Specifies the timestamp up to which recovery will proceed. Specifies the transaction ID up to which recovery will proceed. --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with recovery-target-name, recovery-target-time, recovery-target-lsn, or recovery-target-xid options. The default value is taken from the `recovery_target_inclusive` variable. +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn`, or `--recovery-target-xid` options. The default value is taken from the [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. --recovery-target-action=pause|promote|shutdown Default: pause -Specifies the action the server should take when the recovery target is reached, similar to the `recovery_target_action` option in the recovery.conf configuration file. +Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. ##### Retention Options -You can use these options together with `backup` or `delete` commands. For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +You can use these options together with [backup](#backup) and [delete](#delete) commands. For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). --retention-redundancy=redundancy Default: 0 @@ -616,7 +628,7 @@ Maximum size of an individual log file. If this value is reached, the log file i Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). ##### Connection Options -You can use these options together with `backup` and `checkdb` commands. +You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. -d dbname --dbname=dbname @@ -649,7 +661,7 @@ User name to connect as. Forces a password prompt. ##### Compression Options -You can use these options together with `backup` and `archive-push` commands. +You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. --compress-algorithm=compression_algorithm Default: none @@ -676,7 +688,7 @@ Provides the name of the WAL file in archive_command and restore_command used by Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. ##### Remote Mode Options -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with `add-instance`, `set-config`, `backup`, `restore`, `archive-push` and `archive-get` commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). +This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: @@ -728,6 +740,7 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By ### Usage - [Creating a Backup](#creating-a-backup) +- [Verifying a Cluster](#verifying-a-cluster) - [Validating a Backup](#vaklidating-a-backup) - [Restoring a Cluster](#restoring-a-cluster) - [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) @@ -752,6 +765,7 @@ Where **backup_mode** can take one of the following values: When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. +##### Page validation If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. @@ -782,11 +796,36 @@ To back up a directory located outside of the data directory, use the optional * pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the **--external-dirs** option. +#### Verifying a Cluster +To verify that PostgreSQL database cluster is free of corruption, run the following command: + + pg_probackup checkdb [-B backup_dir] [-D data_dir] + +This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: +- `checkdb` is read-only unlike `backup` +- if corrupted page is detected `checkdb` is not aborted, but carry on, until every page in the cluster is validated +- `checkdb` do not strictly require **the backup catalog**, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). + +If **backup_dir** is omitted, then [connection options](#connection-options) and **data_dir** must be provided via environment variables or command-line options. + +Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files. +Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a limited solution to this problems. + +If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` option when running [checkdb](#checkdb) command: + + pg_probackup checkdb -D data_dir --amcheck + +Physical verification can be skipped if `--skip-block-validation` option is used. For logical only verification **backup_dir** and **data_dir** are optional, only [connection options](#connection-options) are mandatory: + + pg_probackup checkdb --amcheck --skip-block-validation + +Logical verification can be done more thoroughly with option `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. + #### Validating Backups pg_probackup calculates checksums for each file in a backup during `backup` process. The process of checking checksumms of backup data files is called **backup validation**. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. If you would like to skip backup validation, you can specify the `--no-validate` option when running `backup` and `restore` commands. -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the `validate` command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. +To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: @@ -837,11 +876,11 @@ pg_probackup supports the remote mode that allows to perform `backup` and `resto The typical workflow is as follows: - - On your local system, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the `add-instance` and `set-config` commands, make sure to specify [remote options](#remote-options) that point to the remote server with the PostgreSQL instance. + - On your local system, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the remote server with the PostgreSQL instance. -- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the `archive-push` and `archive-get` commands, you must specify the [remote options](#remote-options) that point to host with **backup catalog**. +- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to host with **backup catalog**. -- Run `backup` or `restore` commands with remote options on system with **backup catalog**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +- Run [backup](#backup) or [restore](#restore) commands with remote options on system with **backup catalog**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. >NOTE: The remote backup mode is currently unavailable for Windows systems. @@ -867,7 +906,7 @@ Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. - system-identifier — the unique identifier of the PostgreSQL instance. -Additionally, you can define [remote](#remote-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: +Additionally, you can define [remote](#remote-mode-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: pg_probackup set-config -B backup_dir --instance instance_name [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] From da00777bdf1781e2413e8ac4b2b6e9fb10032e68 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 23:05:08 +0300 Subject: [PATCH 0775/2107] minor fixes to Documentation --- Documentation.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Documentation.md b/Documentation.md index 9fb1cb3b3..e550500df 100644 --- a/Documentation.md +++ b/Documentation.md @@ -51,8 +51,8 @@ Current version - 2.1.3 * [Replica Options](#replica-options) 5. [Usage](#usage) - * [Verifying a Cluster](#verifying-a-cluster) * [Creating a Backup](#creating-a-backup) + * [Verifying a Cluster](#verifying-a-cluster) * [Validating a Backup](#validating-a-backup) * [Restoring a Cluster](#restoring-a-cluster) * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) @@ -245,7 +245,7 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as `archive_command` to remove full-page writes from WAL files. +- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. #### Setting up PTRACK Backups @@ -412,7 +412,7 @@ Copies WAL files into the corresponding subdirectory of the backup catalog, and Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. -You can use archive-push in `archive_command` PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). +You can use archive-push in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). For details, see section [Archiving Options](#archiving-options) ##### archive-get @@ -679,13 +679,13 @@ Alias for --compress-algorithm=zlib and --compress-level=1. These options can be used with `archive-push` and `archive-get` commands. --wal-file-path=wal_file_path %p -Provides the path to the WAL file in archive_command and restore_command used by pg_probackup. The %p variable is required for correct processing. +Provides the path to the WAL file in `archive_command` and `restore_command` used by pg_probackup. The %p variable is required for correct processing. --wal-file-name=wal_file_name %f -Provides the name of the WAL file in archive_command and restore_command used by pg_probackup. The %f variable is required for correct processing. +Provides the name of the WAL file in `archive_command` and `restore_command` used by pg_probackup. The %f variable is required for correct processing. --overwrite -Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the --overwrite option. +Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` option. ##### Remote Mode Options This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). @@ -802,8 +802,8 @@ To verify that PostgreSQL database cluster is free of corruption, run the follow pg_probackup checkdb [-B backup_dir] [-D data_dir] This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: -- `checkdb` is read-only unlike `backup` -- if corrupted page is detected `checkdb` is not aborted, but carry on, until every page in the cluster is validated +- `checkdb` is read-only +- if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated - `checkdb` do not strictly require **the backup catalog**, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). If **backup_dir** is omitted, then [connection options](#connection-options) and **data_dir** must be provided via environment variables or command-line options. @@ -852,11 +852,11 @@ When using the `--tablespace-mapping` option, you must provide absolute paths to Once the restore command is complete, start the database service. If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. ->NOTE: By default, the `restore` command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. +>NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. #### Performing Point-in-Time (PITR) Recovery -If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the `restore` command. If `-i/--backup-id` option is omitted pg_probackup automatically chooses the backup that is the closest to the specified **recovery target** and starts the restore process, otherwise pg_probackup will try to restore **backup_id** to the specified **recovery target**. +If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified **recovery target** and starts the restore process, otherwise pg_probackup will try to restore **backup_id** to the specified **recovery target**. - To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: From 53b0fbaea90433d060f0c5591572d03ccd395cb0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 23:09:09 +0300 Subject: [PATCH 0776/2107] minor fixes to Documentation --- Documentation.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index e550500df..cc5b54d73 100644 --- a/Documentation.md +++ b/Documentation.md @@ -223,7 +223,8 @@ ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/do - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options] + + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options] Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. @@ -753,6 +754,7 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By - [Deleting Backups](#deleting-backups) #### Creating a Backup + To create a backup, run the following command: pg_probackup backup -B backup_dir --instance instance_name -b backup_mode @@ -772,8 +774,7 @@ Page is considered corrupted if checksumm comparison failed more than 100 times, Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". -##### STREAM - +##### STREAM mode To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL **--stream** --temp-slot @@ -789,7 +790,6 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. ##### External directories - To back up a directory located outside of the data directory, use the optional **--external-dirs** parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your node instance that will be stored under the node_backup directory, run: pg_probackup backup -B backup_dir --instance node -b FULL --external-dirs=/etc/dir1:/etc/dir2 @@ -797,6 +797,7 @@ To back up a directory located outside of the data directory, use the optional * pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the **--external-dirs** option. #### Verifying a Cluster + To verify that PostgreSQL database cluster is free of corruption, run the following command: pg_probackup checkdb [-B backup_dir] [-D data_dir] From dc754c61abd4ae3449a0612a3da572e97be8c88d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Jun 2019 23:39:47 +0300 Subject: [PATCH 0777/2107] minor fixes to Documentation --- Documentation.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Documentation.md b/Documentation.md index cc5b54d73..9c7d141d7 100644 --- a/Documentation.md +++ b/Documentation.md @@ -224,14 +224,14 @@ ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/do - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options] + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to **backup_dir/wal/instance_name** directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using `pg_receivewal`. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to **backup_dir/wal/instance_name** directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. #### Backup from Standby @@ -323,7 +323,7 @@ It is **not recommended** to edit pg_probackup.conf manually. pg_probackup show -B backup_dir [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] -Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed information about this backup. You can specify the --format=json option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. +Shows the contents of the backup catalog. If **instance_name** and **backup_id** are specified, shows detailed information about this backup. You can specify the **--format=json** option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. ##### backup @@ -353,8 +353,8 @@ For details, see the sections [Backup Options](#backup-options) and [Creating a [logging_options] [remote_options] -Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. -For details, see the sections [Restore Options](#backup-options) and [Restoring a Cluster](#restoring-a-cluster). +Restores the PostgreSQL instance from a backup copy located in the **backup_dir** backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. +For details, see the sections [Restore Options](#restore-options), [Recovery Target Options](#recovery-target-options) and [Restoring a Cluster](#restoring-a-cluster). ##### checkdb @@ -413,7 +413,7 @@ Copies WAL files into the corresponding subdirectory of the backup catalog, and Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. -You can use archive-push in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). +You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). For details, see section [Archiving Options](#archiving-options) ##### archive-get @@ -508,6 +508,7 @@ Disables block-level checksum verification to speed up backup. Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. ##### Restore Options + The following options can be used together with the [restore](#restore) command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -R | --restore-as-replica @@ -522,7 +523,7 @@ Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. --skip-external-dirs -Skip external directories included into the backup with the --external-dirs option. The contents of these directories will not be restored. +Skip external directories included into the backup with the `--external-dirs` option. The contents of these directories will not be restored. --skip-block-validation Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. @@ -543,12 +544,13 @@ Skip validation of data files. Can be used only with `--amcheck` option, so only Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the `--amcheck` option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. ##### Recovery Target Options + If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored. --recovery-target=immediate|latest Defines when to stop the recovery: -- The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. -- The latest value continues the recovery until all WAL segments available in the archive are applied. +The `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. +The `latest` value continues the recovery until all WAL segments available in the archive are applied. --recovery-target-timeline=timeline Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. @@ -566,7 +568,7 @@ Specifies the timestamp up to which recovery will proceed. Specifies the transaction ID up to which recovery will proceed. --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn`, or `--recovery-target-xid` options. The default value is taken from the [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default value is taken from the [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. --recovery-target-action=pause|promote|shutdown Default: pause From c58e537d10ecd9a87dca34da833c310ad9f164b5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 00:03:24 +0300 Subject: [PATCH 0778/2107] minor fixes to Documentation --- Documentation.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Documentation.md b/Documentation.md index 9c7d141d7..b8964d071 100644 --- a/Documentation.md +++ b/Documentation.md @@ -602,20 +602,20 @@ You can use these options with any command. --log-level-console=log_level Default: info -Controls which message levels are sent to the console log. Valid values are verbose, log, info, notice, warning, error, and off. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The off level disables console logging. +Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error`, and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. --log-level-file=log_level Default: off -Controls which message levels are sent to a log file. Valid values are verbose, log, info, notice, warning, error, and off. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The off level disables file logging. +Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error`, and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. --log-filename=log_filename Default: pg_probackup.log -Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in log_filename. For example, if you specify the pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. -This option takes effect if file logging is enabled by the log-level-file option. +Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in **log_filename**. For example, if you specify the pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. +This option takes effect if file logging is enabled by the `log-level-file` option. --error-log-filename=error_log_filename - Default: none -Defines the filenames of log files for error messages. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying file names, as explained in log_filename. + Default: none +Defines the filenames of log files for error messages. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying file names, as explained in **error_log_filename**. If error-log-filename is not set, pg_probackup writes all error messages to stderr. --log-directory=log_directory @@ -627,7 +627,7 @@ Defines the directory in which log files will be created. You must specify the a Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). --log-rotation-age=log_rotation_age - Default: 0 + Default: 0 Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). ##### Connection Options @@ -679,7 +679,7 @@ Defines compression level (0 through 9, 0 being no compression and 9 being best Alias for --compress-algorithm=zlib and --compress-level=1. ##### Archiving Options -These options can be used with `archive-push` and `archive-get` commands. +These options can be used with [archive-push](#archive-push) and [archive-get](#archive-get) commands. --wal-file-path=wal_file_path %p Provides the path to the WAL file in `archive_command` and `restore_command` used by pg_probackup. The %p variable is required for correct processing. @@ -695,10 +695,10 @@ This section describes the options related to running pg_probackup operations re --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: -- ssh enables the remote backup mode via SSH. This is the Default value. -- none explicitly disables the remote mode. +- `ssh` enables the remote backup mode via SSH. This is the Default value. +- `none` explicitly disables the remote mode. -You can omit this option if the --remote-host option is specified. +You can omit this option if the `--remote-host` option is specified. --remote-host Specifies the remote host IP address or hostname to connect to. @@ -924,7 +924,7 @@ You can override the settings defined in pg_probackup.conf when running the back If you define connection settings in the **pg_probackup.conf** configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. -If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. +If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. ##### Managing the Backup Catalog From 4e54fce52266336442e45891bd722aceb3f01275 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 00:15:12 +0300 Subject: [PATCH 0779/2107] minor fixes to Documentation --- Documentation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index b8964d071..6477be469 100644 --- a/Documentation.md +++ b/Documentation.md @@ -532,7 +532,7 @@ Disables block-level checksum verification to speed up validation. During automa Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. ###### Checkdb Options -The following options can be used together with the [checkdb](#checkdb) command. For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster) +The following options can be used together with the [checkdb](#checkdb) command. For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster). --amcheck Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. @@ -549,8 +549,8 @@ If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configure --recovery-target=immediate|latest Defines when to stop the recovery: -The `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. -The `latest` value continues the recovery until all WAL segments available in the archive are applied. +- `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. +- `latest` value continues the recovery until all WAL segments available in the archive are applied. --recovery-target-timeline=timeline Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. From 1c06ec1f103737c58aa7c85e8b6a15a9702ef69e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 00:16:03 +0300 Subject: [PATCH 0780/2107] minor fixes to Documentation --- Documentation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation.md b/Documentation.md index 6477be469..1a27172f2 100644 --- a/Documentation.md +++ b/Documentation.md @@ -552,6 +552,7 @@ Defines when to stop the recovery: - `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. - `latest` value continues the recovery until all WAL segments available in the archive are applied. + --recovery-target-timeline=timeline Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. From 827562e0c1edf3c936b3ec8df1962c3525f039fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 00:30:00 +0300 Subject: [PATCH 0781/2107] minor fixes to Documentation --- Documentation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 1a27172f2..1e5d588fd 100644 --- a/Documentation.md +++ b/Documentation.md @@ -552,6 +552,7 @@ Defines when to stop the recovery: - `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. - `latest` value continues the recovery until all WAL segments available in the archive are applied. +Default value of `--recovery-target` depends on **WAL delivery method** of restored backup: for STREAM backup it is `immediate`, for ARCHIVE - `latest`. --recovery-target-timeline=timeline Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. @@ -699,7 +700,7 @@ Specifies the protocol to use for remote operations. Currently only the SSH prot - `ssh` enables the remote backup mode via SSH. This is the Default value. - `none` explicitly disables the remote mode. -You can omit this option if the `--remote-host` option is specified. +You can omit this option if the `--remote-host` option is specified. --remote-host Specifies the remote host IP address or hostname to connect to. From 648ccab43b77159849e4e03eee73b217d8c96124 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 00:39:44 +0300 Subject: [PATCH 0782/2107] update to Documentation --- Documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 1e5d588fd..c491535fd 100644 --- a/Documentation.md +++ b/Documentation.md @@ -781,7 +781,7 @@ Redardless of data checksums been enabled or not, pg_probackup always check page ##### STREAM mode To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: - pg_probackup backup -B backup_dir --instance instance_name -b FULL **--stream** --temp-slot + pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot The optional `--temp-slot` parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. @@ -891,7 +891,7 @@ The typical workflow is as follows: ##### Running pg_probackup on Parallel Threads -[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and validate[#validate] processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). +[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: From dd85a421dc483ff2bb0590beac11e8fab0b6c260 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 01:41:05 +0300 Subject: [PATCH 0783/2107] update Documentation --- Documentation.md | 85 ++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/Documentation.md b/Documentation.md index c491535fd..79c82c8f2 100644 --- a/Documentation.md +++ b/Documentation.md @@ -90,6 +90,8 @@ Current version - 2.1.3 `pg_probackup restore -B backup_dir --instance instance_name [option...]` +`pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` + `pg_probackup validate -B backup_dir [option...]` `pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` @@ -100,29 +102,27 @@ Current version - 2.1.3 `pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f` -`pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` - ### Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -- Incremental backup: page-level incremental backup of three different modes allows you to devise the backup strategy in accordance with your data flow +- Incremental backup: page-level incremental backup allows you save to disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow - Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery - Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb` - Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` -- Parallelization: Running backup, restore, merge, verificaton and validation processes on multiple parallel threads +- Parallelization: Running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads - Compression: Storing backup data in a compressed state to save disk space - Deduplication: Saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) - Remote operations: Backup PostgreSQL instance located on remote machine or restore backup on it - Backup from replica: Avoid extra load on the master server by taking backups from a standby - External directories: Add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -- Backup Catalog: get list of backups and all corresponding metadata about them in `plain` or `json` formats +- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single **backup catalog**. Using pg_probackup, you can take full or incremental backups: - FULL backups contain all the data files required to restore the database cluster. -- Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. pg_probackup supports the following modes of incremental backups: +- Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. @@ -153,7 +153,7 @@ To initialize the backup catalog, run the following command: pg_probackup init -B backup_dir -where **backup_dir** is the path to backup catalog. If the backup_dir already exists, it must be empty. Otherwise, pg_probackup returns an error. +where **backup_dir** is the path to backup catalog. If the **backup_dir** already exists, it must be empty. Otherwise, pg_probackup returns an error. pg_probackup creates the backup_dir backup catalog, with the following subdirectories: - **wal/** — directory for WAL files. @@ -219,18 +219,16 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK #### Setting up continuous WAL archiving ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: -- Configure the following parameters in postgresql.conf to enable continuous archiving on the PostgreSQL server: - - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. - - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: +- Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. +- If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. +- Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' - Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. + Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. - >NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to **backup_dir/wal/instance_name** directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. #### Backup from Standby @@ -252,11 +250,11 @@ Backup from the standby server has the following limitations: If you are going to use PTRACK backups, complete the following additional steps: - Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute ptrack functions to the backup role: +- Grant the rights to execute ptrack functions to the **backup** role: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; -- The *backup* role must have access to all the databases of the cluster. +- The **backup** role must have access to all the databases of the cluster. ### Command-Line Reference #### Commands @@ -304,8 +302,7 @@ Deletes all backups and WAL files associated with the specified instance. [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] [-d dbname] [-h host] [-p port] [-U username] [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [remote_options] - [logging_options] + [remote_options] [logging_options] Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. It is **not recommended** to edit pg_probackup.conf manually. @@ -316,7 +313,6 @@ It is **not recommended** to edit pg_probackup.conf manually. Displays the contents of the pg_probackup.conf configuration file located in the **backup_dir/backups/instance_name** directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. To edit pg_probackup.conf, use the [set-config](#set-config) command. -It is **not recommended** to edit pg_probackup.conf manually. ##### show @@ -333,11 +329,8 @@ Shows the contents of the backup catalog. If **instance_name** and **backup_id** [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [connection_options] - [compression_options] - [remote_options] - [retention_options] - [logging_options] + [connection_options] [compression_options] [remote_options] + [retention_options] [logging_options] Creates a backup copy of the PostgreSQL instance. The **backup_mode** option specifies the backup mode to use. For details, see the sections [Backup Options](#backup-options) and [Creating a Backup](#creating-a-backup). @@ -349,9 +342,7 @@ For details, see the sections [Backup Options](#backup-options) and [Creating a [-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] - [recovery_options] - [logging_options] - [remote_options] + [recovery_options] [logging_options] [remote_options] Restores the PostgreSQL instance from a backup copy located in the **backup_dir** backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. For details, see the sections [Restore Options](#restore-options), [Recovery Target Options](#recovery-target-options) and [Restoring a Cluster](#restoring-a-cluster). @@ -362,8 +353,7 @@ For details, see the sections [Restore Options](#restore-options), [Recovery Tar [-B backup_dir] [--instance instance_name] [-D data_dir] [--help] [-j num_threads] [--progress] [--skip-block-validation] [--amcheck] [--heapallindexed] - [logging_options] - [connection_options] + [connection_options] [logging_options] Verifyes the PostgreSQL database cluster correctness by detecting physical and logical corruption. For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying a Cluster](#verifying-a-cluster). @@ -374,8 +364,7 @@ For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying [--help] [--instance instance_name] [-i backup_id] [-j num_threads] [--progress] [--skip-block-validation] - [recovery_options] - [logging_options] + [recovery_options] [logging_options] Verifies that all the files required to restore the cluster are present and not corrupted. If **instance_name** is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the **instance_name** without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the **instance_name** with a [recovery target option](#recovery-target-options) and/or a **backup_id**, pg_probackup checks whether it is possible to restore the cluster using these options. For details, see the section [Validating a Backup](#validating-a-backup). @@ -406,8 +395,7 @@ For details, see the sections [Deleting Backups](#deleting-backups), [Retention --wal-file-path %p --wal-file-name %f [--help] [--compress] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] [--overwrite] - [remote_options] - [logging_options] + [remote_options] [logging_options] Copies WAL files into the corresponding subdirectory of the backup catalog, and validates the backup instance by instance_name, system-identifier. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. @@ -418,11 +406,8 @@ For details, see section [Archiving Options](#archiving-options) ##### archive-get - pg_probackup archive-get -B backup_dir --instance instance_name - --wal-file-path %p --wal-file-name %f - [--help] - [remote_options] - [logging_options] + pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f + [--help] [remote_options] [logging_options] Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in **recovery.conf** when restoring backups using a WAL archive. You do not need to set it manually. @@ -545,7 +530,7 @@ Checks that all heap tuples that should be indexed are actually indexed. You can ##### Recovery Target Options -If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored. +If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. --recovery-target=immediate|latest Defines when to stop the recovery: @@ -804,17 +789,17 @@ pg_probackup creates a separate subdirectory in the backup directory for each ex To verify that PostgreSQL database cluster is free of corruption, run the following command: - pg_probackup checkdb [-B backup_dir] [-D data_dir] + pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: - `checkdb` is read-only - if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated - `checkdb` do not strictly require **the backup catalog**, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). -If **backup_dir** is omitted, then [connection options](#connection-options) and **data_dir** must be provided via environment variables or command-line options. +If **backup_dir** and **instance_name** are omitted, then [connection options](#connection-options) and **data_dir** must be provided via environment variables or command-line options. -Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files. -Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a limited solution to this problems. +Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. +Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` option when running [checkdb](#checkdb) command: @@ -822,7 +807,7 @@ If you would like, in addition to physical verification, to verify all indexes i Physical verification can be skipped if `--skip-block-validation` option is used. For logical only verification **backup_dir** and **data_dir** are optional, only [connection options](#connection-options) are mandatory: - pg_probackup checkdb --amcheck --skip-block-validation + pg_probackup checkdb --amcheck --skip-block-validation [connection options] Logical verification can be done more thoroughly with option `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. @@ -849,9 +834,9 @@ Where: - **instance_name** is the backup instance for the cluster to be restored. - **backup_id** specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. -If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. +If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. -When using the `--tablespace-mapping` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: +When using the `--tablespace-mapping/-T` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir @@ -875,6 +860,14 @@ If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiv pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 +- To restore the cluster state up to a specific named restore point, use recovery-target-name: + + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' + +- To restore the cluster to the latest state available in archive, use recovery-target option: + + pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' + ##### Using pg_probackup in the Remote Mode pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. From 223b4e5026ed6e2987bf4187fd9854953a10c032 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 01:45:28 +0300 Subject: [PATCH 0784/2107] Readme: add link to Documentation.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e040d69b..45eb4ef6a 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Documentation -Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). +Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). ## Licence From 16d6924f4719cdcaf253d3f2a340ed81b4814cc7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 13:45:05 +0300 Subject: [PATCH 0785/2107] Documentation: add section "Setting up Cluster Verification" --- Documentation.md | 62 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/Documentation.md b/Documentation.md index 79c82c8f2..646384e30 100644 --- a/Documentation.md +++ b/Documentation.md @@ -13,8 +13,9 @@ Current version - 2.1.3 * [Adding a New Backup Instance](#adding-a-new-backup-instance) * [Configuring the Database Cluster](#configuring-the-database-cluster) * [Setting up STREAM Backups](#setting-up-stream-backups) - * [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) - * [Setting up backup from Standby](#backup-from-standby) + * [Setting up Continuous WAL Archiving](#setting-up-continuous-wal-archiving) + * [Setting up Backup from Standby](#backup-from-standby) + * [Setting up Cluster Verification](#setting-up-cluster-verification) * [Setting up PTRACK Backups](#setting-up-ptrack-backups) 4. [Command-Line Reference](#command-line-reference) @@ -219,7 +220,7 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK #### Setting up continuous WAL archiving ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: -- Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than 'minimal'. +- Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: @@ -246,11 +247,29 @@ Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. - All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. +#### Setting up Cluster Verification + +Logical verification of database cluster requires the following additional setup. Role **backup** is used as an example: +- Install extension `amcheck` or `amcheck_next` in every database of the cluster: + + CREATE EXTENSION amcheck; + +- To perform logical verification the following rights are requiared: +``` +GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; +``` + #### Setting up PTRACK Backups If you are going to use PTRACK backups, complete the following additional steps: - Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute ptrack functions to the **backup** role: +- Grant the rights to execute ptrack functions to the **backup** role: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; @@ -397,7 +416,8 @@ For details, see the sections [Deleting Backups](#deleting-backups), [Retention [--compress-level=compression_level] [--overwrite] [remote_options] [logging_options] -Copies WAL files into the corresponding subdirectory of the backup catalog, and validates the backup instance by instance_name, system-identifier. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the --overwrite option. +Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by **instance_name** and **system-identifier**. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. +If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` option. Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. @@ -473,14 +493,13 @@ Makes an STREAM backup that includes all the necessary WAL files by streaming th Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` option. --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the --stream option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S / --slot`. +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the `--stream` option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S / --slot`. --backup-pg-log Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. -E external_directory_path --external-dirs=external_directory_path - Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. --archive-timeout=wait_time @@ -589,21 +608,23 @@ You can use these options with any command. --log-level-console=log_level Default: info -Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error`, and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. +Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. + +>NOTE: all console log messages are going to stderr, so output from [show](#show) and [show-config](#show-config) commands do not mingle with log messages. --log-level-file=log_level Default: off -Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error`, and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. +Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. --log-filename=log_filename - Default: pg_probackup.log + Default: pg_probackup.log Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in **log_filename**. For example, if you specify the pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. This option takes effect if file logging is enabled by the `log-level-file` option. --error-log-filename=error_log_filename Default: none -Defines the filenames of log files for error messages. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying file names, as explained in **error_log_filename**. -If error-log-filename is not set, pg_probackup writes all error messages to stderr. +Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in **error_log_filename**. For example, if you specify the error-pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. +This option is useful for troubleshooting and monitoring. --log-directory=log_directory Default: $BACKUP_PATH/log/ @@ -663,7 +684,7 @@ For the `archive-push` command, the pglz compression algorithm is not supported. Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with --compress-algorithm option. --compress -Alias for --compress-algorithm=zlib and --compress-level=1. +Alias for `--compress-algorithm=zlib` and `--compress-level=1`. ##### Archiving Options These options can be used with [archive-push](#archive-push) and [archive-get](#archive-get) commands. @@ -779,9 +800,9 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. ##### External directories -To back up a directory located outside of the data directory, use the optional **--external-dirs** parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your node instance that will be stored under the node_backup directory, run: +To back up a directory located outside of the data directory, use the optional **--external-dirs** parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your **instance_name** instance that will be stored under the **backup_dir** directory, run: - pg_probackup backup -B backup_dir --instance node -b FULL --external-dirs=/etc/dir1:/etc/dir2 + pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the **--external-dirs** option. @@ -898,7 +919,7 @@ Once the backup catalog is initialized and a new backup instance is added, you c For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. -> Note: It is **not recommended** to edit pg_probackup.conf manually. +>NOTE: It is **not recommended** to edit pg_probackup.conf manually. Initially, pg_probackup.conf contains the following settings: - PGDATA — the path to the data directory of the cluster to back up. @@ -942,11 +963,10 @@ BACKUP INSTANCE 'node' ============================================================================================================================================ Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status ============================================================================================================================================ - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PTRACK STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 1 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 1 21s 32MB 0/13000028 0/13000198 OK + node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1 / 1 31s 33MB 0/11000028 0/110001D0 OK node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1 / 0 11s 39MB 0/F000028 0/F000198 OK - node 10 P7XDFT 2018-04-29 05:26:25+03 PTRACK STREAM 1 / 0 11s 40MB 0/D000028 0/D000198 OK ``` For each backup, the following information is provided: @@ -1105,7 +1125,7 @@ BACKUP INSTANCE 'node' node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK ``` ->Note: The Time field for the merged backup displays the time required for the merge. +>NOTE: The Time field for the merged backup displays the time required for the merge. ##### Merging Backups From 341b612520c4dc7c7d624de04c9bb0fb1ad9b271 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Jun 2019 20:08:03 +0300 Subject: [PATCH 0786/2107] tests: added tests.backup.BackupTest.test_backup_with_least_privileges_role --- tests/backup.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e55e6512c..7b8ac453d 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1365,3 +1365,113 @@ def test_basic_missing_dir_permissions(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_with_least_privileges_role(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + # pg_options={'ptrack_enable': 'on'}, + initdb_params=['--data-checksums'], + + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + node.safe_psql( + 'backupdb', + "REVOKE TEMPORARY ON DATABASE backupdb FROM PUBLIC;" + "REVOKE ALL on SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL on SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL on SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + # for partial restore, checkdb and ptrack + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + # for exclusive backup for PG 9.5 and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + # ptrack functions + # for fname in [ + # 'oideq(oid, oid)', + # 'ptrack_version()', + # 'pg_ptrack_clear()', + # 'pg_ptrack_control_lsn()', + # 'pg_ptrack_get_and_clear_db(oid, oid)', + # 'pg_ptrack_get_and_clear(oid, oid)', + # 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: + # try: + # node.safe_psql( + # "backupdb", + # "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " + # "TO backup".format(fname)) + # except: + # pass + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', '-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['-U', 'backup']) + + # PAGE + self.backup_node( + backup_dir, 'node', node, backup_type='page', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='page', datname='backupdb', + options=['--stream', '-U', 'backup']) + + # DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # PTRACK + # self.backup_node( + # backup_dir, 'node', node, backup_type='ptrack', + # datname='backupdb', options=['-U', 'backup']) + # self.backup_node( + # backup_dir, 'node', node, backup_type='ptrack', + # datname='backupdb', options=['--stream', '-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 8ad309aae9d09916419b6e9e587e4c67d75fd6f6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 00:43:48 +0300 Subject: [PATCH 0787/2107] Documentation: minor fixes --- Documentation.md | 164 +++++++++++++++++++++++++++++++---------------- 1 file changed, 109 insertions(+), 55 deletions(-) diff --git a/Documentation.md b/Documentation.md index 646384e30..b7639d67c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -119,23 +119,25 @@ As compared to other backup solutions, pg_probackup offers the following benefit - External directories: Add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files - Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats -To manage backup data, pg_probackup creates a **backup catalog**. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single **backup catalog**. +To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. Using pg_probackup, you can take full or incremental backups: + - FULL backups contain all the data files required to restore the database cluster. - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen **backup mode** (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following **WAL delivery methods**: -- ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen `backup mode` (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following `WAL delivery methods`: +- ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. - STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. #### Limitations pg_probackup currently has the following limitations: + - Creating backups from a remote server is currently not supported on Windows systems. - The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. @@ -154,11 +156,12 @@ To initialize the backup catalog, run the following command: pg_probackup init -B backup_dir -where **backup_dir** is the path to backup catalog. If the **backup_dir** already exists, it must be empty. Otherwise, pg_probackup returns an error. +Where *backup_dir* is the path to backup catalog. If the *backup_dir* already exists, it must be empty. Otherwise, pg_probackup returns an error. pg_probackup creates the backup_dir backup catalog, with the following subdirectories: -- **wal/** — directory for WAL files. -- **backups/** — directory for backup files. + +- wal/ — directory for WAL files. +- backups/ — directory for backup files. Once the backup catalog is initialized, you can add a new backup instance. @@ -169,18 +172,21 @@ To add a new backup instance, run the following command: pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] -where: -- **data_dir** is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. -- **instance_name** is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional parameters [remote_options](#remote-mode-options) should be used if **data_dir** is located on remote machine. +Where: + +- *data_dir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. +- *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. +- The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. -pg_probackup creates the **instance_name** subdirectories under the **backups/** and **wal/** directories of the backup catalog. The **backups/instance_name** directory contains the **pg_probackup.conf** configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to **pg_probackup.conf**. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). +pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/instance_name' directory contains the 'pg_probackup.conf' configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. -The user launching pg_probackup must have full access to **backup_dir** directory and at least read-only access to **data_dir** directory. If you specify the path to the backup catalog in the *BACKUP_PATH* environment variable, you can omit the corresponding option when running pg_probackup commands. +For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). + +The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. #### Configuring the Database Cluster -Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the **backup** role is used as an example. +Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. To enable backups, the following rights are required: @@ -205,14 +211,14 @@ Depending on whether you are plan to take STREAM and/or ARCHIVE backups, Postgre #### Setting up STREAM Backups To set up the cluster for STREAM backups, complete the following steps: + - Grant the REPLICATION privilege to the backup role: ALTER ROLE backup WITH REPLICATION; -- In the pg_hba.conf file, allow replication on behalf of the backup role. -- Modify the postgresql.conf configuration file of the PostgreSQL server, as follows: - - Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. - - Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. +- In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow replication on behalf of the *backup* role. +- Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. +- Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. If you are planning to take PAGE backups in STREAM mode, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). @@ -220,17 +226,18 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK #### Setting up continuous WAL archiving ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: + - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. - If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' - Where **backup_dir** and **instance_name** refer to the already initialized **backup catalog** instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. +Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to **backup_dir/wal/instance_name** directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to 'backup_dir/wal/instance_name' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. #### Backup from Standby @@ -244,17 +251,20 @@ For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby serve Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups of appropriate WAL delivery method: ARCHIVE or STREAM, from the standby server. Backup from the standby server has the following limitations: + - If the standby is promoted to the master during backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like **pg_compresslog** as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. +- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like pg_compresslog as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. #### Setting up Cluster Verification -Logical verification of database cluster requires the following additional setup. Role **backup** is used as an example: +Logical verification of database cluster requires the following additional setup. Role *backup* is used as an example: + - Install extension `amcheck` or `amcheck_next` in every database of the cluster: CREATE EXTENSION amcheck; - To perform logical verification the following rights are requiared: + ``` GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; @@ -268,12 +278,13 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; #### Setting up PTRACK Backups If you are going to use PTRACK backups, complete the following additional steps: + - Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute ptrack functions to the **backup** role: +- Grant the rights to execute ptrack functions to the *backup* role: GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; -- The **backup** role must have access to all the databases of the cluster. +- The *backup* role must have access to all the databases of the cluster. ### Command-Line Reference #### Commands @@ -296,7 +307,8 @@ Displays the synopsis of pg_probackup commands. If one of the pg_probackup comma pg_probackup init -B backup_dir [--help] -Initializes the backup catalog in **backup_dir** that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified **backup_dir** already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. +Initializes the backup catalog in *backup_dir* that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified *backup_dir* already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. + For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). ##### add-instance @@ -304,7 +316,9 @@ For details, see the secion [Initializing the Backup Catalog](#initializing-the- pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] -Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). +Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. + +For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). ##### del-instance @@ -324,13 +338,15 @@ Deletes all backups and WAL files associated with the specified instance. [remote_options] [logging_options] Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. + It is **not recommended** to edit pg_probackup.conf manually. ##### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] -Displays the contents of the pg_probackup.conf configuration file located in the **backup_dir/backups/instance_name** directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. +Displays the contents of the pg_probackup.conf configuration file located in the 'backup_dir/backups/instance_name' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. + To edit pg_probackup.conf, use the [set-config](#set-config) command. ##### show @@ -338,7 +354,9 @@ To edit pg_probackup.conf, use the [set-config](#set-config) command. pg_probackup show -B backup_dir [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] -Shows the contents of the backup catalog. If **instance_name** and **backup_id** are specified, shows detailed information about this backup. You can specify the **--format=json** option to return the result in the JSON format. By default, the contents of the backup catalog is shown as plain text. +Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. + +By default, the contents of the backup catalog is shown as plain text. ##### backup @@ -351,7 +369,8 @@ Shows the contents of the backup catalog. If **instance_name** and **backup_id** [connection_options] [compression_options] [remote_options] [retention_options] [logging_options] -Creates a backup copy of the PostgreSQL instance. The **backup_mode** option specifies the backup mode to use. +Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. + For details, see the sections [Backup Options](#backup-options) and [Creating a Backup](#creating-a-backup). ##### restore @@ -363,7 +382,8 @@ For details, see the sections [Backup Options](#backup-options) and [Creating a [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [recovery_options] [logging_options] [remote_options] -Restores the PostgreSQL instance from a backup copy located in the **backup_dir** backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. +Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. + For details, see the sections [Restore Options](#restore-options), [Recovery Target Options](#recovery-target-options) and [Restoring a Cluster](#restoring-a-cluster). ##### checkdb @@ -385,7 +405,8 @@ For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying [--skip-block-validation] [recovery_options] [logging_options] -Verifies that all the files required to restore the cluster are present and not corrupted. If **instance_name** is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the **instance_name** without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the **instance_name** with a [recovery target option](#recovery-target-options) and/or a **backup_id**, pg_probackup checks whether it is possible to restore the cluster using these options. +Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target option](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. + For details, see the section [Validating a Backup](#validating-a-backup). ##### merge @@ -395,6 +416,7 @@ For details, see the section [Validating a Backup](#validating-a-backup). [logging_options] Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. + For details, see the section [Merging Backups](#merging-backups). ##### delete @@ -405,7 +427,8 @@ For details, see the section [Merging Backups](#merging-backups). [--dry-run] [logging_options] -Deletes backup with specified **backip_id** or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. +Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. + For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-otions) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). ##### archive-push @@ -416,12 +439,15 @@ For details, see the sections [Deleting Backups](#deleting-backups), [Retention [--compress-level=compression_level] [--overwrite] [remote_options] [logging_options] -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by **instance_name** and **system-identifier**. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. +Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by *instance_name* and *system-identifier*. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. + If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` option. + Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). + For details, see section [Archiving Options](#archiving-options) ##### archive-get @@ -429,7 +455,7 @@ For details, see section [Archiving Options](#archiving-options) pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [--help] [remote_options] [logging_options] -Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in **recovery.conf** when restoring backups using a WAL archive. You do not need to set it manually. +Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. #### Options @@ -444,7 +470,7 @@ The list of general options. -B directory --backup-path=directory BACKUP_PATH -Specifies the absolute path to the **backup catalog**. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. +Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. -D directory --pgdata=directory @@ -473,6 +499,7 @@ The following options can be used together with the [backup](#backup) command. A --backup-mode=mode Specifies the backup mode to use. Possible values are: + - FULL — creates a full backup that contains all the data files of the cluster to be restored. - DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. @@ -553,10 +580,11 @@ If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configure --recovery-target=immediate|latest Defines when to stop the recovery: + - `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. - `latest` value continues the recovery until all WAL segments available in the archive are applied. -Default value of `--recovery-target` depends on **WAL delivery method** of restored backup: for STREAM backup it is `immediate`, for ARCHIVE - `latest`. +Default value of `--recovery-target` depends on WAL delivery method of restored backup, `immediate` for STREAM backup and `latest` for ARCHIVE. --recovery-target-timeline=timeline Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. @@ -618,12 +646,18 @@ Controls which message levels are sent to a log file. Valid values are `verbose` --log-filename=log_filename Default: pg_probackup.log -Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in **log_filename**. For example, if you specify the pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. +Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in *log_filename*. + +For example, if you specify the 'pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. + This option takes effect if file logging is enabled by the `log-level-file` option. --error-log-filename=error_log_filename Default: none -Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in **error_log_filename**. For example, if you specify the error-pg_probackup-%u.log pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. +Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in *error_log_filename*. + +For example, if you specify the 'error-pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. + This option is useful for troubleshooting and monitoring. --log-directory=log_directory @@ -703,6 +737,7 @@ This section describes the options related to running pg_probackup operations re --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: + - `ssh` enables the remote backup mode via SSH. This is the Default value. - `none` explicitly disables the remote mode. @@ -769,7 +804,8 @@ To create a backup, run the following command: pg_probackup backup -B backup_dir --instance instance_name -b backup_mode -Where **backup_mode** can take one of the following values: +Where *backup_mode* can take one of the following values: + - FULL — creates a full backup that contains all the data files of the cluster to be restored. - DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. @@ -780,6 +816,7 @@ When restoring a cluster from an incremental backup, pg_probackup relies on the ##### Page validation If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. + Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". @@ -800,11 +837,15 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. ##### External directories -To back up a directory located outside of the data directory, use the optional **--external-dirs** parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your **instance_name** instance that will be stored under the **backup_dir** directory, run: +To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. + +For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 -pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the **--external-dirs** option. +pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. + +To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the `--external-dirs` option. #### Verifying a Cluster @@ -813,11 +854,12 @@ To verify that PostgreSQL database cluster is free of corruption, run the follow pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: + - `checkdb` is read-only - if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated -- `checkdb` do not strictly require **the backup catalog**, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). +- `checkdb` do not strictly require the backup catalog, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). -If **backup_dir** and **instance_name** are omitted, then [connection options](#connection-options) and **data_dir** must be provided via environment variables or command-line options. +If *backup_dir* and *instance_name* are omitted, then [connection options](#connection-options) and *data_dir* must be provided via environment variables or command-line options. Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. @@ -826,7 +868,7 @@ If you would like, in addition to physical verification, to verify all indexes i pg_probackup checkdb -D data_dir --amcheck -Physical verification can be skipped if `--skip-block-validation` option is used. For logical only verification **backup_dir** and **data_dir** are optional, only [connection options](#connection-options) are mandatory: +Physical verification can be skipped if `--skip-block-validation` option is used. For logical only verification *backup_dir* and *data_dir* are optional, only [connection options](#connection-options) are mandatory: pg_probackup checkdb --amcheck --skip-block-validation [connection options] @@ -834,7 +876,9 @@ Logical verification can be done more thoroughly with option `--heapallindexed` #### Validating Backups -pg_probackup calculates checksums for each file in a backup during `backup` process. The process of checking checksumms of backup data files is called **backup validation**. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. If you would like to skip backup validation, you can specify the `--no-validate` option when running `backup` and `restore` commands. +pg_probackup calculates checksums for each file in a backup during `backup` process. The process of checking checksumms of backup data files is called `backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. + +If you would like to skip backup validation, you can specify the `--no-validate` option when running `backup` and `restore` commands. To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. @@ -851,9 +895,10 @@ To restore the database cluster from a backup, run the restore command with at l pg_probackup restore -B backup_dir --instance instance_name -i backup_id Where: -- **backup_dir** is the backup catalog that stores all backup files and meta information. -- **instance_name** is the backup instance for the cluster to be restored. -- **backup_id** specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. + +- *backup_dir* is the backup catalog that stores all backup files and meta information. +- *instance_name* is the backup instance for the cluster to be restored. +- *backup_id* specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. @@ -867,7 +912,9 @@ Once the restore command is complete, start the database service. If you are res #### Performing Point-in-Time (PITR) Recovery -If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified **recovery target** and starts the restore process, otherwise pg_probackup will try to restore **backup_id** to the specified **recovery target**. +If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. + +If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore *backup_id* to the specified recovery target. - To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: @@ -897,9 +944,9 @@ The typical workflow is as follows: - On your local system, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the remote server with the PostgreSQL instance. -- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to host with **backup catalog**. +- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to host with backup catalog. -- Run [backup](#backup) or [restore](#restore) commands with remote options on system with **backup catalog**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +- Run [backup](#backup) or [restore](#restore) commands with remote options on system with backup catalog. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. >NOTE: The remote backup mode is currently unavailable for Windows systems. @@ -915,13 +962,14 @@ Parallel execution is controlled by the `-j/--threads` command line option. For ##### Configuring pg_probackup -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the **backup_dir/backups/instance_name** directory to fine-tune pg_probackup configuration. +Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the 'backup_dir/backups/instance_name' directory to fine-tune pg_probackup configuration. For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. >NOTE: It is **not recommended** to edit pg_probackup.conf manually. Initially, pg_probackup.conf contains the following settings: + - PGDATA — the path to the data directory of the cluster to back up. - system-identifier — the unique identifier of the PostgreSQL instance. @@ -938,7 +986,7 @@ You can override the settings defined in pg_probackup.conf when running the back ###### Specifying Connection Settings -If you define connection settings in the **pg_probackup.conf** configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. +If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. @@ -970,6 +1018,7 @@ BACKUP INSTANCE 'node' ``` For each backup, the following information is provided: + - Instance — the instance name. - Version — PostgreSQL major version. - ID — the backup identifier. @@ -982,6 +1031,7 @@ For each backup, the following information is provided: - Start LSN — WAL log sequence number corresponding to the start of the backup process. - Stop LSN — WAL log sequence number corresponding to the end of the backup process. - Status — backup status. Possible values: + - OK — the backup is complete and valid. - RUNNING — the backup is in progress. - DONE — the backup is complete, but was not validated. @@ -998,6 +1048,7 @@ To get more detailed information about the backup, run the show with the backup pg_probackup show -B backup_dir --instance instance_name -i backup_id The sample output is as follows: + ``` #Configuration backup-mode = FULL @@ -1033,6 +1084,7 @@ To get more detailed information about the backup in json format, run the show w pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id The sample output is as follows: + ``` [ { @@ -1074,6 +1126,7 @@ The sample output is as follows: By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. To configure retention policy, set one or more of the following variables in the pg_probackup.conf file: + - retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. - retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. @@ -1085,13 +1138,13 @@ To clean up the backup catalog in accordance with retention policy, run: pg_probackup deletes all backup copies that do not conform to the defined retention policy. -If you would like to also remove the WAL files that are no longer required for any of the backups, add the --delete-wal option: +If you would like to also remove the WAL files that are no longer required for any of the backups, add the `--delete-wal` option: pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal >NOTE: Alternatively, you can use the --delete-expired, --merge-expired and --delete-wal options together with the `backup` command to remove and merge the outdated backup copies once the new backup is created. -Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the --merge-expired option when running `backup` or `delete` commands. +Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` option when running [backup](#backup) or [delete](#delete) commands. Suppose you have backed up the node instance in the node-backup directory, with the retention-window option is set to 7, and you have the following backups available on April 10, 2019: @@ -1110,6 +1163,7 @@ BACKUP INSTANCE 'node' ``` Even though P7XDHB and P7XDHU backups are outside the retention window, P7XDHB and P7XDHU cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the `--delete-expired` option, only the P7XDFT full backup will be removed. + With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: pg_probackup delete -B node-backup --instance node --delete-expired --merge-expired @@ -1143,7 +1197,7 @@ To delete a backup that is no longer required, run the following command: pg_probackup delete -B backup_dir --instance instance_name -i backup_id -This command will delete the backup with the specified **backup_id**, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. +This command will delete the backup with the specified *backup_id*, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` option: From 8de99afbb0dbe2fc28154c89ff4d7abc965e93ec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 00:52:03 +0300 Subject: [PATCH 0788/2107] Documentation: style minor fixes --- Documentation.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation.md b/Documentation.md index b7639d67c..bb8bb1bdb 100644 --- a/Documentation.md +++ b/Documentation.md @@ -178,7 +178,7 @@ Where: - *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. - The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. -pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/instance_name' directory contains the 'pg_probackup.conf' configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. +pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). @@ -237,7 +237,7 @@ Where *backup_dir* and *instance_name* refer to the already initialized backup c Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to 'backup_dir/wal/instance_name' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. #### Backup from Standby @@ -345,7 +345,7 @@ It is **not recommended** to edit pg_probackup.conf manually. pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] -Displays the contents of the pg_probackup.conf configuration file located in the 'backup_dir/backups/instance_name' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. +Displays the contents of the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. To edit pg_probackup.conf, use the [set-config](#set-config) command. @@ -962,7 +962,7 @@ Parallel execution is controlled by the `-j/--threads` command line option. For ##### Configuring pg_probackup -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the 'backup_dir/backups/instance_name' directory to fine-tune pg_probackup configuration. +Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. @@ -986,7 +986,7 @@ You can override the settings defined in pg_probackup.conf when running the back ###### Specifying Connection Settings -If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. +If you define connection settings in the *pg_probackup.conf* configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. @@ -1125,7 +1125,7 @@ The sample output is as follows: By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. -To configure retention policy, set one or more of the following variables in the pg_probackup.conf file: +To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): - retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. - retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. From 7ff166539dd864764bdba246844c2f527d7ae530 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 01:04:54 +0300 Subject: [PATCH 0789/2107] Documentation: style minor fixes --- Documentation.md | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/Documentation.md b/Documentation.md index bb8bb1bdb..470085bbe 100644 --- a/Documentation.md +++ b/Documentation.md @@ -178,7 +178,7 @@ Where: - *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. - The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. -pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls backup and restore settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. +pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls pg_probackup settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). @@ -316,7 +316,7 @@ For details, see the secion [Initializing the Backup Catalog](#initializing-the- pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] -Initializes a new backup instance inside the backup catalog backup_dir and generates the pg_probackup.conf configuration file that controls backup and restore settings for the cluster with the specified data_dir data directory. +Initializes a new backup instance inside the backup catalog *backup_dir* and generates the pg_probackup.conf configuration file that controls pg_probackup settings for the cluster with the specified *data_dir* data directory. For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). @@ -395,6 +395,7 @@ For details, see the sections [Restore Options](#restore-options), [Recovery Tar [connection_options] [logging_options] Verifyes the PostgreSQL database cluster correctness by detecting physical and logical corruption. + For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying a Cluster](#verifying-a-cluster). ##### validate @@ -493,7 +494,9 @@ Shows detailed information about the options that can be used with this command. ##### Backup Options -The following options can be used together with the [backup](#backup) command. Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. +The following options can be used together with the [backup](#backup) command. + +Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -b mode --backup-mode=mode @@ -540,7 +543,9 @@ Skips automatic validation after successfull backup. You can use this option if ##### Restore Options -The following options can be used together with the [restore](#restore) command. Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. +The following options can be used together with the [restore](#restore) command. + +Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -R | --restore-as-replica Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. @@ -563,7 +568,9 @@ Disables block-level checksum verification to speed up validation. During automa Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. ###### Checkdb Options -The following options can be used together with the [checkdb](#checkdb) command. For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster). +The following options can be used together with the [checkdb](#checkdb) command. + +For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster). --amcheck Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. @@ -609,7 +616,9 @@ Specifies whether to stop just after the specified recovery target (true), or ju Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. ##### Retention Options -You can use these options together with [backup](#backup) and [delete](#delete) commands. For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +You can use these options together with [backup](#backup) and [delete](#delete) commands. + +For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). --retention-redundancy=redundancy Default: 0 @@ -733,7 +742,9 @@ Provides the name of the WAL file in `archive_command` and `restore_command` use Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` option. ##### Remote Mode Options -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). +This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. + +For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). --remote-proto Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: @@ -1142,7 +1153,7 @@ If you would like to also remove the WAL files that are no longer required for a pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal ->NOTE: Alternatively, you can use the --delete-expired, --merge-expired and --delete-wal options together with the `backup` command to remove and merge the outdated backup copies once the new backup is created. +>NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired` and `--delete-wal` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` option when running [backup](#backup) or [delete](#delete) commands. @@ -1162,7 +1173,7 @@ BACKUP INSTANCE 'node' node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK ``` -Even though P7XDHB and P7XDHU backups are outside the retention window, P7XDHB and P7XDHU cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the `--delete-expired` option, only the P7XDFT full backup will be removed. +Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the `--delete-expired` option, only the P7XDFT full backup will be removed. With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: From e2e12bf15d3a81361a2734543deb95bffe961d9e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 01:06:58 +0300 Subject: [PATCH 0790/2107] Documentation: style minor fixes --- Documentation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 470085bbe..af53de27e 100644 --- a/Documentation.md +++ b/Documentation.md @@ -190,7 +190,8 @@ Although pg_probackup can be used by a superuser, it is recommended to create a To enable backups, the following rights are required: -```CREATE ROLE backup WITH LOGIN; +``` +CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; From 73e2c861caff6831d62a482cf0570bf32305e237 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 01:51:06 +0300 Subject: [PATCH 0791/2107] Documentation: final fixes --- Documentation.md | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Documentation.md b/Documentation.md index af53de27e..68e300e9c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -709,7 +709,7 @@ User name to connect as. -w --no-password - Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a .pgpass file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. + Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. -W --password @@ -720,7 +720,7 @@ You can use these options together with [backup](#backup) and [archive-push](#ar --compress-algorithm=compression_algorithm Default: none -Defines the algorithm to use for compressing data files. Possible values are zlib, pglz, and none. If set to zlib or pglz, this option enables compression. By default, compression is disabled. +Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. For the `archive-push` command, the pglz compression algorithm is not supported. --compress-level=compression_level @@ -731,13 +731,13 @@ Defines compression level (0 through 9, 0 being no compression and 9 being best Alias for `--compress-algorithm=zlib` and `--compress-level=1`. ##### Archiving Options -These options can be used with [archive-push](#archive-push) and [archive-get](#archive-get) commands. +These options can be used with [archive-push](#archive-push) and [archive-get](#archive-get) commands in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) and [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) settings. --wal-file-path=wal_file_path %p -Provides the path to the WAL file in `archive_command` and `restore_command` used by pg_probackup. The %p variable is required for correct processing. +Provides the path to the WAL file in `archive_command` and `restore_command`. The %p variable is required for correct processing. --wal-file-name=wal_file_name %f -Provides the name of the WAL file in `archive_command` and `restore_command` used by pg_probackup. The %f variable is required for correct processing. +Provides the name of the WAL file in `archive_command` and `restore_command`. The %f variable is required for correct processing. --overwrite Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` option. @@ -774,6 +774,7 @@ Specifies a string of SSH command-line options. ##### Replica Options This section describes the options related to taking a backup from standby. + >NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. --master-db=dbname @@ -844,9 +845,9 @@ STREAM backups include all the WAL segments required to restore the cluster to a Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: - 1. STREAM backups can be restored on the server that has no file access to WAL archive. - 2. STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. - 3. Creating backup from standby of a server that generates small amount of WAL traffic and using [archive_timeout](https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) is not an option. +- STREAM backups can be restored on the server that has no file access to WAL archive. +- STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. +- Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. ##### External directories To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. @@ -857,7 +858,7 @@ For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. -To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the `set-config` command with the `--external-dirs` option. +To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. #### Verifying a Cluster @@ -888,11 +889,11 @@ Logical verification can be done more thoroughly with option `--heapallindexed` #### Validating Backups -pg_probackup calculates checksums for each file in a backup during `backup` process. The process of checking checksumms of backup data files is called `backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. +pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. -If you would like to skip backup validation, you can specify the `--no-validate` option when running `backup` and `restore` commands. +If you would like to skip backup validation, you can specify the `--no-validate` option when running [backup](#backup) and [restore](#restore) commands. -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact recovery target options you are going to use for recovery. If you omit all the parameters, all backups are validated. +To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact recovery target options you are going to use for recovery. For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: @@ -900,6 +901,10 @@ For example, to check that you can restore the database cluster from a backup co If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. +If you specify *backup_id* via `-i/--backup-id` option, then only backup copy with specified backup ID will be validated. If *backup_id* belong to incremental backup, then all its parents starting from FULL backup will be validated. + +If you omit all the parameters, all backups are validated. + #### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: @@ -918,7 +923,9 @@ When using the `--tablespace-mapping/-T` option, you must provide absolute paths pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir -Once the restore command is complete, start the database service. If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. +Once the restore command is complete, start the database service. + +If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. >NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. @@ -994,11 +1001,11 @@ To view the current settings, run the following command: pg_probackup show-config -B backup_dir --instance instance_name -You can override the settings defined in pg_probackup.conf when running the backup command via corresponding environment variables and/or command line options. +You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. ###### Specifying Connection Settings -If you define connection settings in the *pg_probackup.conf* configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. +If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. From 989d7bc77bc26eb4f5ed92b110558b91658062ad Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 13:21:02 +0300 Subject: [PATCH 0792/2107] Documentation: minor fix --- Documentation.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation.md b/Documentation.md index 68e300e9c..8d9535ca9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -524,7 +524,7 @@ Makes an STREAM backup that includes all the necessary WAL files by streaming th Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` option. --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the `--stream` option. Default slot name is `pg_probackup_slot`, which can be changed via option `-S / --slot`. +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the `--stream` option. Default slot name is `pg_probackup_slot`, which can be changed via option `--slot/-S`. --backup-pg-log Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. @@ -610,7 +610,7 @@ Specifies the timestamp up to which recovery will proceed. Specifies the transaction ID up to which recovery will proceed. --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default value is taken from the [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default depends on [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. --recovery-target-action=pause|promote|shutdown Default: pause @@ -721,17 +721,17 @@ You can use these options together with [backup](#backup) and [archive-push](#ar --compress-algorithm=compression_algorithm Default: none Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. -For the `archive-push` command, the pglz compression algorithm is not supported. +For the [archive-push](#archive-push) command, the pglz compression algorithm is not supported. --compress-level=compression_level Default: 1 -Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with --compress-algorithm option. +Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with `--compress-algorithm` option. --compress Alias for `--compress-algorithm=zlib` and `--compress-level=1`. ##### Archiving Options -These options can be used with [archive-push](#archive-push) and [archive-get](#archive-get) commands in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) and [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) settings. +These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. --wal-file-path=wal_file_path %p Provides the path to the WAL file in `archive_command` and `restore_command`. The %p variable is required for correct processing. From f2a17b6ce0eaadb2888769bd1ea061a820a69385 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 17:59:13 +0300 Subject: [PATCH 0793/2107] Documentation: fix headings --- Documentation.md | 143 ++++++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 65 deletions(-) diff --git a/Documentation.md b/Documentation.md index 8d9535ca9..0274dd9b9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1,6 +1,10 @@ +% pg_probackup(1) +% Postgres Professional +% June 2019 + # pg_probackup -**pg_probackup** is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. +pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. Current version - 2.1.3 @@ -69,7 +73,7 @@ Current version - 2.1.3 7. [Credits](#credits) -### Synopsis +## Synopsis `pg_probackup version` @@ -103,7 +107,7 @@ Current version - 2.1.3 `pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f` -### Overview +## Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: @@ -134,14 +138,14 @@ pg_probackup can take only physical online backups, and online backups require W - ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. - STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. -#### Limitations +### Limitations pg_probackup currently has the following limitations: - Creating backups from a remote server is currently not supported on Windows systems. - The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. -### Installation and Setup +## Installation and Setup Once you have pg_probackup installed, complete the following setup: @@ -149,7 +153,7 @@ Once you have pg_probackup installed, complete the following setup: - Add a new backup instance to the backup catalog. - Configure the database cluster to enable pg_probackup backups. -#### Initializing the Backup Catalog +### Initializing the Backup Catalog pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. To initialize the backup catalog, run the following command: @@ -165,7 +169,7 @@ pg_probackup creates the backup_dir backup catalog, with the following subdirect Once the backup catalog is initialized, you can add a new backup instance. -#### Adding a New Backup Instance +### Adding a New Backup Instance pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. To add a new backup instance, run the following command: @@ -184,7 +188,7 @@ For details on how to fine-tune pg_probackup configuration, see the section [Con The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. -#### Configuring the Database Cluster +### Configuring the Database Cluster Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. @@ -207,9 +211,11 @@ GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and [Setting up Backup from Standby](#backup-from-standby). +Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. + +For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and [Setting up Backup from Standby](#backup-from-standby). -#### Setting up STREAM Backups +### Setting up STREAM Backups To set up the cluster for STREAM backups, complete the following steps: @@ -225,7 +231,8 @@ If you are planning to take PAGE backups in STREAM mode, you still have to confi Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in STREAM mode. -#### Setting up continuous WAL archiving +### Setting up continuous WAL archiving + ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. @@ -240,7 +247,7 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK >NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. -#### Backup from Standby +### Backup from Standby For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: @@ -256,7 +263,7 @@ Backup from the standby server has the following limitations: - If the standby is promoted to the master during backup, the backup fails. - All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like pg_compresslog as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. -#### Setting up Cluster Verification +### Setting up Cluster Verification Logical verification of database cluster requires the following additional setup. Role *backup* is used as an example: @@ -276,7 +283,7 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; ``` -#### Setting up PTRACK Backups +### Setting up PTRACK Backups If you are going to use PTRACK backups, complete the following additional steps: @@ -287,24 +294,24 @@ If you are going to use PTRACK backups, complete the following additional steps: GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; - The *backup* role must have access to all the databases of the cluster. -### Command-Line Reference -#### Commands +## Command-Line Reference +### Commands This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). -##### version +#### version pg_probackup version Prints pg_probackup version. -##### help +#### help pg_probackup help [command] Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. -##### init +#### init pg_probackup init -B backup_dir [--help] @@ -312,7 +319,7 @@ Initializes the backup catalog in *backup_dir* that will store backup copies, WA For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). -##### add-instance +#### add-instance pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] [--external-dirs=external_directory_path] @@ -321,14 +328,14 @@ Initializes a new backup instance inside the backup catalog *backup_dir* and gen For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). -##### del-instance +#### del-instance pg_probackup del-instance -B backup_dir --instance instance_name [--help] Deletes all backups and WAL files associated with the specified instance. -##### set-config +#### set-config pg_probackup set-config -B backup_dir --instance instance_name [--help] [--pgdata=pgdata-path] @@ -342,7 +349,7 @@ Adds the specified connection, compression, retention, logging and external dire It is **not recommended** to edit pg_probackup.conf manually. -##### show-config +#### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] @@ -350,7 +357,7 @@ Displays the contents of the pg_probackup.conf configuration file located in the To edit pg_probackup.conf, use the [set-config](#set-config) command. -##### show +#### show pg_probackup show -B backup_dir [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] @@ -359,7 +366,7 @@ Shows the contents of the backup catalog. If *instance_name* and *backup_id* are By default, the contents of the backup catalog is shown as plain text. -##### backup +#### backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name [--help] [-j num_threads] [--progress] @@ -374,7 +381,7 @@ Creates a backup copy of the PostgreSQL instance. The *backup_mode* option speci For details, see the sections [Backup Options](#backup-options) and [Creating a Backup](#creating-a-backup). -##### restore +#### restore pg_probackup restore -B backup_dir --instance instance_name [--help] [-D data_dir] [-i backup_id] @@ -387,7 +394,7 @@ Restores the PostgreSQL instance from a backup copy located in the *backup_dir* For details, see the sections [Restore Options](#restore-options), [Recovery Target Options](#recovery-target-options) and [Restoring a Cluster](#restoring-a-cluster). -##### checkdb +#### checkdb pg_probackup checkdb [-B backup_dir] [--instance instance_name] [-D data_dir] @@ -399,7 +406,7 @@ Verifyes the PostgreSQL database cluster correctness by detecting physical and l For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying a Cluster](#verifying-a-cluster). -##### validate +#### validate pg_probackup validate -B backup_dir [--help] [--instance instance_name] [-i backup_id] @@ -411,7 +418,7 @@ Verifies that all the files required to restore the cluster are present and not For details, see the section [Validating a Backup](#validating-a-backup). -##### merge +#### merge pg_probackup merge -B backup_dir --instance instance_name -i backup_id [--help] [-j num_threads][--progress] @@ -421,7 +428,7 @@ Merges the specified incremental backup to its parent full backup, together with For details, see the section [Merging Backups](#merging-backups). -##### delete +#### delete pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] @@ -433,7 +440,7 @@ Deletes backup with specified *backip_id* or launches the retention purge of bac For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-otions) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). -##### archive-push +#### archive-push pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f @@ -452,21 +459,22 @@ You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/ For details, see section [Archiving Options](#archiving-options) -##### archive-get +#### archive-get pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [--help] [remote_options] [logging_options] Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. -#### Options +### Options This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. + For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. -##### Common Options +#### Common Options The list of general options. -B directory @@ -493,7 +501,7 @@ Shows the progress of operations. --help Shows detailed information about the options that can be used with this command. -##### Backup Options +#### Backup Options The following options can be used together with the [backup](#backup) command. @@ -542,7 +550,7 @@ Disables block-level checksum verification to speed up backup. --no-validate Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. -##### Restore Options +#### Restore Options The following options can be used together with the [restore](#restore) command. @@ -568,7 +576,8 @@ Disables block-level checksum verification to speed up validation. During automa --no-validate Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. -###### Checkdb Options +#### Checkdb Options + The following options can be used together with the [checkdb](#checkdb) command. For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster). @@ -582,7 +591,7 @@ Skip validation of data files. Can be used only with `--amcheck` option, so only --heapallindexed Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the `--amcheck` option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. -##### Recovery Target Options +#### Recovery Target Options If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. @@ -616,7 +625,7 @@ Specifies whether to stop just after the specified recovery target (true), or ju Default: pause Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. -##### Retention Options +#### Retention Options You can use these options together with [backup](#backup) and [delete](#delete) commands. For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). @@ -641,7 +650,7 @@ Merges the oldest incremental backup that satisfies the requirements of retentio --dry-run Displays the current status of all the available backups, without deleting or merging expired backups, if any. -##### Logging Options +#### Logging Options You can use these options with any command. --log-level-console=log_level @@ -682,7 +691,7 @@ Maximum size of an individual log file. If this value is reached, the log file i Default: 0 Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). -##### Connection Options +#### Connection Options You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. -d dbname @@ -715,7 +724,8 @@ User name to connect as. --password Forces a password prompt. -##### Compression Options +#### Compression Options + You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. --compress-algorithm=compression_algorithm @@ -730,7 +740,8 @@ Defines compression level (0 through 9, 0 being no compression and 9 being best --compress Alias for `--compress-algorithm=zlib` and `--compress-level=1`. -##### Archiving Options +#### Archiving Options + These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. --wal-file-path=wal_file_path %p @@ -742,7 +753,8 @@ Provides the name of the WAL file in `archive_command` and `restore_command`. Th --overwrite Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` option. -##### Remote Mode Options +#### Remote Mode Options + This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). @@ -772,7 +784,8 @@ Specifies pg_probackup installation directory on the remote system. --ssh-options Specifies a string of SSH command-line options. -##### Replica Options +#### Replica Options + This section describes the options related to taking a backup from standby. >NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. @@ -796,7 +809,7 @@ Deprecated. User name to connect as. Default: 300 sec Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. -### Usage +## Usage - [Creating a Backup](#creating-a-backup) - [Verifying a Cluster](#verifying-a-cluster) @@ -811,7 +824,7 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By - [Merging Backups](#merging-backups) - [Deleting Backups](#deleting-backups) -#### Creating a Backup +### Creating a Backup To create a backup, run the following command: @@ -826,7 +839,7 @@ Where *backup_mode* can take one of the following values: When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. -##### Page validation +#### Page validation If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. @@ -834,7 +847,7 @@ Page is considered corrupted if checksumm comparison failed more than 100 times, Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". -##### STREAM mode +#### STREAM mode To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot @@ -849,7 +862,7 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin - STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. - Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. -##### External directories +#### External directories To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: @@ -860,7 +873,7 @@ pg_probackup creates a separate subdirectory in the backup directory for each ex To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. -#### Verifying a Cluster +### Verifying a Cluster To verify that PostgreSQL database cluster is free of corruption, run the following command: @@ -887,7 +900,7 @@ Physical verification can be skipped if `--skip-block-validation` option is used Logical verification can be done more thoroughly with option `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. -#### Validating Backups +### Validating Backups pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. @@ -905,7 +918,7 @@ If you specify *backup_id* via `-i/--backup-id` option, then only backup copy wi If you omit all the parameters, all backups are validated. -#### Restoring a Cluster +### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: @@ -929,7 +942,7 @@ If you are restoring an STREAM backup, the restore is complete at once, with the >NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. -#### Performing Point-in-Time (PITR) Recovery +### Performing Point-in-Time (PITR) Recovery If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. @@ -955,7 +968,7 @@ If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the ba pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' -##### Using pg_probackup in the Remote Mode +### Using pg_probackup in the Remote Mode pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. @@ -969,7 +982,7 @@ The typical workflow is as follows: >NOTE: The remote backup mode is currently unavailable for Windows systems. -##### Running pg_probackup on Parallel Threads +### Running pg_probackup on Parallel Threads [Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). @@ -979,7 +992,7 @@ Parallel execution is controlled by the `-j/--threads` command line option. For >NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. -##### Configuring pg_probackup +### Configuring pg_probackup Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. @@ -1003,13 +1016,13 @@ To view the current settings, run the following command: You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. -###### Specifying Connection Settings +### Specifying Connection Settings If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. -##### Managing the Backup Catalog +### Managing the Backup Catalog With pg_probackup, you can manage backups from the command line: @@ -1140,7 +1153,7 @@ The sample output is as follows: ] ``` -##### Configuring Backup Retention Policy +### Configuring Backup Retention Policy By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. @@ -1200,7 +1213,7 @@ BACKUP INSTANCE 'node' >NOTE: The Time field for the merged backup displays the time required for the merge. -##### Merging Backups +### Merging Backups As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: @@ -1210,7 +1223,7 @@ This command merges the specified incremental backup to its parent full backup, Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the show command with the backup ID. If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. -##### Deleting Backups +### Deleting Backups To delete a backup that is no longer required, run the following command: @@ -1234,8 +1247,8 @@ In this case, pg_probackup searches for the oldest incremental backup that satis Before merging or deleting backups, you can run the delete command with the `--dry-run` option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. -### Authors -PostgreSQLfessional, Moscow, Russia. +## Authors +Postgres Professional, Moscow, Russia. -### Credits +## Credits pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. From 3ce65d2eb721ad572e54b6246b4207ac9809bce8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Jun 2019 18:00:45 +0300 Subject: [PATCH 0794/2107] Documentation: quick fix --- Documentation.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index 0274dd9b9..5108fd2db 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1,7 +1,3 @@ -% pg_probackup(1) -% Postgres Professional -% June 2019 - # pg_probackup pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. From 54737fb8f4c9193f6076e6d7a2447216fa57f51c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Jun 2019 01:41:21 +0300 Subject: [PATCH 0795/2107] tests: added retention.RetentionTest.test_retention_redundancy_overlapping_chains --- tests/retention.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/retention.py b/tests/retention.py index cdab447bc..f5cb9b9db 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2,6 +2,7 @@ import unittest from datetime import datetime, timedelta from .helpers.ptrack_helpers import ProbackupTest +from time import sleep module_name = 'retention' @@ -1233,4 +1234,46 @@ def test_window_error_backups(self): backup_dir, 'node', node, backup_type='page') # Change FULLb backup status to ERROR - # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + def test_retention_redundancy_overlapping_chains(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Make backups to be keeped + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + sleep(1) + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Purge backups + log = self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 4217bc2c22551e170115a838a4bac2e4685c1c22 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Jun 2019 13:09:42 +0300 Subject: [PATCH 0796/2107] [Issue #92] Overlapping chains and retention redundancy --- src/catalog.c | 37 +++++++++-- src/delete.c | 31 ++++++++- tests/delete.py | 127 ++++++++++++++++++++++++++++-------- tests/retention.py | 156 ++++++++++++++++++++++++++++----------------- 4 files changed, 258 insertions(+), 93 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index a9bed1e06..e68a4d07b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -441,22 +441,47 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) } /* - * Find the last completed backup on given timeline + * Find the latest valid child of latest valid FULL backup on given timeline */ pgBackup * catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) { int i; - pgBackup *backup = NULL; + pgBackup *full_backup = NULL; /* backup_list is sorted in order of descending ID */ for (i = 0; i < parray_num(backup_list); i++) { - backup = (pgBackup *) parray_get(backup_list, (size_t) i); + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if ((backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) && backup->tli == tli) - return backup; + if ((backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) && backup->tli == tli) + { + full_backup = backup; + break; + } + } + + /* Failed to find valid FULL backup to fulfill ancestor role */ + if (!full_backup) + return NULL; + + /* FULL backup is found, lets find his latest child */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + + if (is_parent(full_backup->start_time, backup, true)) + { + + /* only valid descendants are acceptable */ + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) + { + return backup; + } + } } return NULL; diff --git a/src/delete.c b/src/delete.c index 2b546c82e..0ead9af53 100644 --- a/src/delete.c +++ b/src/delete.c @@ -175,6 +175,8 @@ int do_retention(void) if (delete_wal && !dry_run) do_retention_wal(); + /* TODO: consider dry-run flag */ + if (!backup_merged) elog(INFO, "There are no backups to merge by retention policy"); @@ -203,6 +205,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg int i; time_t current_time; + parray *redundancy_full_backup_list = NULL; + /* For retention calculation */ uint32 n_full_backups = 0; int cur_full_backup_num = 0; @@ -228,8 +232,19 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg backup->status == BACKUP_STATUS_DONE)) { n_full_backups++; + + if (n_full_backups <= instance_config.retention_redundancy) + { + if (!redundancy_full_backup_list) + redundancy_full_backup_list = parray_new(); + + parray_append(redundancy_full_backup_list, backup); + } } } + /* Sort list of full backups to keep */ + if (redundancy_full_backup_list) + parray_qsort(redundancy_full_backup_list, pgBackupCompareIdDesc); } if (instance_config.retention_window > 0) @@ -242,8 +257,20 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) { + bool redundancy_keep = false; pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); + /* check if backups FULL parent is in redundancy list */ + if (redundancy_full_backup_list) + { + pgBackup *full_backup = find_parent_full_backup(backup); + + if (full_backup && parray_bsearch(redundancy_full_backup_list, + full_backup, + pgBackupCompareIdDesc)) + redundancy_keep = true; + } + /* Remember the serial number of latest valid FULL backup */ if (backup->backup_mode == BACKUP_MODE_FULL && (backup->status == BACKUP_STATUS_OK || @@ -256,7 +283,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg * TODO: consider that ERROR backup most likely to have recovery_time == 0 */ if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) && - (instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num))) +// (instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num))) + (instance_config.retention_redundancy == 0 || !redundancy_keep)) { /* This backup is not guarded by retention * @@ -344,6 +372,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg else actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24); + /* TODO: add ancestor(chain full backup) ID */ elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", base36enc(backup->start_time), pgBackupGetBackupMode(backup), diff --git a/tests/delete.py b/tests/delete.py index 5ad4c2508..a9d9eb167 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -89,7 +89,62 @@ def test_del_instance_archive(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_archive_mix_compress_and_non_compressed_segments(self): - """stub""" + """delete full backups""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + initdb_params=['--data-checksums'], + pg_options={'wal_level': 'replica'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving( + backup_dir, 'node', node, compress=False) + node.slow_start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=10) + + # Restart archiving with compression + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.restart() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + # Clean after yourself + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_delete_increment_page(self): @@ -358,11 +413,12 @@ def test_delete_interleaved_incremental_chains(self): backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) - # Change FULL B backup status to ERROR + # Change FULLb to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # FULLb ERROR # FULLa OK + # Take PAGEa1 backup page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -370,15 +426,17 @@ def test_delete_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb ERROR # FULLa OK - # Change FULL B backup status to OK + + # Change FULLb to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa1 backup status to ERROR + # Change PAGEa1 to ERROR self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') # PAGEa1 ERROR # FULLb OK # FULLa OK + page_id_b1 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -386,41 +444,49 @@ def test_delete_interleaved_incremental_chains(self): # PAGEa1 ERROR # FULLb OK # FULLa OK + # Now we start to play with first generation of PAGE backups - # Change PAGEb1 status to ERROR + # Change PAGEb1 and FULLb status to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # Change PAGEa1 status to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK + page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEa2 status to ERROR + + # Change PAGEa2 and FULla to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - # Change PAGEb1 status to OK + # Change PAGEb1 and FULlb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR + page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change PAGEa2 status to OK + # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # PAGEb2 OK # PAGEa2 OK @@ -478,16 +544,15 @@ def test_delete_multiple_descendants(self): # Take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - # Change FULLb backup status to ERROR + # Change FULLb to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change FULLb backup status to OK + # Change FULLb to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # Change PAGEa1 backup status to ERROR @@ -505,15 +570,16 @@ def test_delete_multiple_descendants(self): # FULLb OK # FULLa OK - # Change PAGEa1 backup status to OK + # Change PAGEa1 to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - # Change PAGEb1 backup status to ERROR + # Change PAGEb1 and FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a2 = self.backup_node( @@ -522,20 +588,22 @@ def test_delete_multiple_descendants(self): # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEb1 backup status to OK + # Change PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa2 backup status to ERROR + # Change PAGEa2 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -545,17 +613,21 @@ def test_delete_multiple_descendants(self): # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR - # Change PAGEb2 and PAGEb1 status to ERROR + # Change PAGEb2, PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change FULLa to OK + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb2 ERROR # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a3 = self.backup_node( @@ -566,14 +638,15 @@ def test_delete_multiple_descendants(self): # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK # Change PAGEa3 status to ERROR self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - # Change PAGEb2 status to OK + # Change PAGEb2 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -587,7 +660,7 @@ def test_delete_multiple_descendants(self): # FULLb OK # FULLa OK - # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + # Change PAGEa3, PAGEa2 and PAGEb1 to OK self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') diff --git a/tests/retention.py b/tests/retention.py index f5cb9b9db..bfb76cd23 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -247,11 +247,12 @@ def test_window_expire_interleaved_incremental_chains(self): backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) - # Change FULL B backup status to ERROR + # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # FULLb ERROR # FULLa OK + # Take PAGEa1 backup page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -259,57 +260,69 @@ def test_window_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb ERROR # FULLa OK - # Change FULL B backup status to OK + + # Change FULLb backup status to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa1 backup status to ERROR + # Change PAGEa1 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') # PAGEa1 ERROR # FULLb OK - # FULLa OK + # FULLa ERROR + page_id_b1 = self.backup_node( backup_dir, 'node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR # FULLb OK - # FULLa OK + # FULLa ERROR + # Now we start to play with first generation of PAGE backups - # Change PAGEb1 status to ERROR + # Change PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - # Change PAGEa1 status to OK + # Change PAGEa1 and FULLa to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK + page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEa2 status to ERROR + + # Change PAGEa2 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - # Change PAGEb1 status to OK + # Change PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR + page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change PAGEa2 status to OK + # Change PAGEa2 and FULla to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb2 OK # PAGEa2 OK @@ -321,14 +334,12 @@ def test_window_expire_interleaved_incremental_chains(self): # Purge backups backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): - if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) self.delete_expired( backup_dir, 'node', @@ -372,32 +383,38 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb ERROR # FULLa OK - # Change FULL B backup status to OK + + # Change FULLb backup status to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa1 backup status to ERROR + # Change PAGEa1 and FULLa backup status to ERROR self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') # PAGEa1 ERROR # FULLb OK - # FULLa OK + # FULLa ERROR + page_id_b1 = self.backup_node( backup_dir, 'node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR # FULLb OK - # FULLa OK + # FULLa ERROR + # Now we start to play with first generation of PAGE backups # Change PAGEb1 status to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # Change PAGEa1 status to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -405,24 +422,28 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEa2 status to ERROR + + # Change PAGEa2 and FULLa status to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - # Change PAGEb1 status to OK + # Change PAGEb1 and FULLb status to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change PAGEa2 status to OK + # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb2 OK # PAGEa2 OK @@ -461,11 +482,12 @@ def test_window_merge_interleaved_incremental_chains(self): backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) - # Change FULL B backup status to ERROR + # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # FULLb ERROR # FULLa OK + # Take PAGEa1 backup page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -473,7 +495,8 @@ def test_window_merge_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb ERROR # FULLa OK - # Change FULL B backup status to OK + + # Change FULLb to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # Change PAGEa1 backup status to ERROR @@ -482,6 +505,7 @@ def test_window_merge_interleaved_incremental_chains(self): # PAGEa1 ERROR # FULLb OK # FULLa OK + page_id_b1 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -489,41 +513,49 @@ def test_window_merge_interleaved_incremental_chains(self): # PAGEa1 ERROR # FULLb OK # FULLa OK + # Now we start to play with first generation of PAGE backups - # Change PAGEb1 status to ERROR + # Change PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - # Change PAGEa1 status to OK + # Change PAGEa1 to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK + page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEa2 status to ERROR + + # Change PAGEa2 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - # Change PAGEb1 status to OK + # Change PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR + page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change PAGEa2 status to OK + # Change PAGEa2 and FULLa to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb2 OK # PAGEa2 OK @@ -535,14 +567,12 @@ def test_window_merge_interleaved_incremental_chains(self): # Purge backups backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): - if backup in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) output = self.delete_expired( backup_dir, 'node', @@ -808,12 +838,13 @@ def test_basic_window_merge_multiple_descendants(self): # Change PAGEa1 backup status to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - # Change PAGEb1 backup status to ERROR + # Change PAGEb1 and FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a2 = self.backup_node( @@ -825,20 +856,22 @@ def test_basic_window_merge_multiple_descendants(self): # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEb1 backup status to OK + # Change PAGEb1 and FULLb backup status to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa2 backup status to ERROR + # Change PAGEa2 and FULLa backup status to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -851,17 +884,21 @@ def test_basic_window_merge_multiple_descendants(self): # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR # Change PAGEb2 and PAGEb1 status to ERROR self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + # and FULL stuff + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + # PAGEb2 ERROR # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a3 = self.backup_node( @@ -874,7 +911,7 @@ def test_basic_window_merge_multiple_descendants(self): # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK # Change PAGEa3 status to ERROR @@ -882,6 +919,7 @@ def test_basic_window_merge_multiple_descendants(self): # Change PAGEb2 status to OK self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1203,7 +1241,7 @@ def test_window_chains_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_window_error_backups(self): """ PAGE ERROR From a5eb8db914c25f1f95c2346ad7534b87590e1890 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Jun 2019 19:43:03 +0300 Subject: [PATCH 0797/2107] [Issue #92] Refactoring of catalog_get_last_data_backup(). First we look for latest valid FULL backup, he is the chain ancestor. Then latest valid child of chain ancestor becomes the parent of current backup. --- src/backup.c | 2 +- src/catalog.c | 58 ++++++++++++-- src/pg_probackup.h | 3 +- tests/backup.py | 192 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 246 insertions(+), 9 deletions(-) diff --git a/src/backup.c b/src/backup.c index b568586eb..be3f38193 100644 --- a/src/backup.c +++ b/src/backup.c @@ -198,7 +198,7 @@ do_backup_instance(PGconn *backup_conn) /* get list of backups already taken */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); - prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) elog(ERROR, "Valid backup on current timeline is not found. " "Create new FULL backup before an incremental one."); diff --git a/src/catalog.c b/src/catalog.c index e68a4d07b..aac3527cc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -444,10 +444,12 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) * Find the latest valid child of latest valid FULL backup on given timeline */ pgBackup * -catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) +catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time) { int i; pgBackup *full_backup = NULL; + pgBackup *tmp_backup = NULL; + char *invalid_backup_id; /* backup_list is sorted in order of descending ID */ for (i = 0; i < parray_num(backup_list); i++) @@ -465,23 +467,65 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli) /* Failed to find valid FULL backup to fulfill ancestor role */ if (!full_backup) + { + elog(WARNING, "Failed to find a valid backup chain"); return NULL; + } + + elog(INFO, "Latest valid FULL backup: %s", + base36enc(full_backup->start_time)); /* FULL backup is found, lets find his latest child */ for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (is_parent(full_backup->start_time, backup, true)) + /* only valid descendants are acceptable */ + if ((backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) { - - /* only valid descendants are acceptable */ - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) + switch (scan_parent_chain(backup, &tmp_backup)) { - return backup; + /* broken chain */ + case 0: + invalid_backup_id = base36enc_dup(tmp_backup->parent_backup); + + elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", + base36enc(backup->start_time), invalid_backup_id); + pg_free(invalid_backup_id); + continue; + + /* chain is intact, but at least one parent is invalid */ + case 1: + invalid_backup_id = base36enc_dup(tmp_backup->start_time); + + elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", + base36enc(backup->start_time), invalid_backup_id); + pg_free(invalid_backup_id); + continue; + + /* chain is ok */ + case 2 : + /* Yes, we could call is_parent() earlier, after choosing the ancestor, + * but this way we have an opportunity to report about all possible + * anomalies. + */ + if (is_parent(full_backup->start_time, backup, true)) + { + elog(INFO, "Parent backup: %s", + base36enc(backup->start_time)); + return backup; + } } } + /* skip yourself */ + else if (backup->start_time == current_start_time) + continue; + else + { + elog(WARNING, "Backup %s has status: %s. Cannot be a parent.", + base36enc(backup->start_time), status2str(backup->status)); + } } return NULL; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 420028291..b90a668a5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -553,7 +553,8 @@ extern parray *catalog_get_backup_list(time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, - TimeLineID tli); + TimeLineID tli, + time_t current_start_time); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list); diff --git a/tests/backup.py b/tests/backup.py index 7b8ac453d..1cf3d9395 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2,6 +2,7 @@ import os from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import shutil module_name = 'backup' @@ -1475,3 +1476,194 @@ def test_backup_with_least_privileges_role(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_choosing(self): + """ + PAGE3 <- RUNNING(parent should be FULL) + PAGE2 <- OK + PAGE1 <- CORRUPT + FULL + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGE1 to ERROR + self.change_backup_status(backup_dir, 'node', page1_id, 'ERROR') + + # PAGE3 + page3_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_file_content = f.read() + + self.assertIn( + "WARNING: Backup {0} has invalid parent: {1}. " + "Cannot be a parent".format(page2_id, page1_id), + log_file_content) + + self.assertIn( + "WARNING: Backup {0} has status: ERROR. " + "Cannot be a parent".format(page1_id), + log_file_content) + + self.assertIn( + "Parent backup: {0}".format(full_id), + log_file_content) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + full_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_choosing_1(self): + """ + PAGE3 <- RUNNING(parent should be FULL) + PAGE2 <- OK + PAGE1 <- (missing) + FULL + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Delete PAGE1 + shutil.rmtree( + os.path.join(backup_dir, 'backups', 'node', page1_id)) + + # PAGE3 + page3_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_file_content = f.read() + + self.assertIn( + "WARNING: Backup {0} has missing parent: {1}. " + "Cannot be a parent".format(page2_id, page1_id), + log_file_content) + + self.assertIn( + "Parent backup: {0}".format(full_id), + log_file_content) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + full_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_choosing_2(self): + """ + PAGE3 <- RUNNING(backup should fail) + PAGE2 <- OK + PAGE1 <- OK + FULL <- (missing) + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Delete FULL + shutil.rmtree( + os.path.join(backup_dir, 'backups', 'node', full_id)) + + # PAGE3 + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Failed to find a valid backup chain', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'ERROR: Valid backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb( + backup_dir, 'node')[2]['status'], + 'ERROR') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 4a3ac209e0ae414d122b7932db9420d58e592742 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 28 Jun 2019 00:41:14 +0300 Subject: [PATCH 0798/2107] tests: added backup.BackupTest.test_backup_detect_corruption --- tests/backup.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 7b8ac453d..379d56dd3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -423,6 +423,60 @@ def test_page_corruption_heal_via_ptrack_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_backup_detect_corruption(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Insert correct message', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_tablespace_in_pgdata_pgpro_1376(self): """PGPRO-1376 """ From eb2a397a0122af79e6f4f92535a6e9b058aee2c0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 28 Jun 2019 01:16:10 +0300 Subject: [PATCH 0799/2107] tests: minor fixes for corruption detection --- src/data.c | 2 +- tests/backup.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 2a80294cc..3536895d0 100644 --- a/src/data.c +++ b/src/data.c @@ -356,7 +356,7 @@ prepare_page(ConnectionArgs *arguments, ((strict && !is_ptrack_support) || !strict)) { /* show this message for checkdb or backup without ptrack support */ - elog(WARNING, "CORRUPTION in file %s, block %u", + elog(WARNING, "Corruption detected in file \"%s\", block %u", file->path, blknum); } diff --git a/tests/backup.py b/tests/backup.py index 379d56dd3..297b034c4 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -469,7 +469,12 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'Insert correct message', + 'WARNING: Corruption detected in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Data file corruption', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 5f840ba436df95f3154398acee2303e5f8a58265 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 28 Jun 2019 10:09:03 +0300 Subject: [PATCH 0800/2107] Fix passing error in fio_send_pages --- src/utils/file.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 568f66b5a..71a62713d 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1150,8 +1150,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, if ((int)hdr.arg < 0) /* read error */ { - errno = -(int)hdr.arg; - return -1; + return (int)hdr.arg; } blknum = hdr.arg; From 79f940ff759a184e485b5095b01170d739507057 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 28 Jun 2019 12:47:13 +0300 Subject: [PATCH 0801/2107] checkdb: don`t close descriptor if file size is not aligned BLCKSZ --- src/data.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/data.c b/src/data.c index 3536895d0..0304f10bc 100644 --- a/src/data.c +++ b/src/data.c @@ -585,7 +585,7 @@ backup_data_file(backup_files_arg* arguments, } if (file->size % BLCKSZ != 0) - elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size); + elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size); /* * Compute expected number of blocks in the file. @@ -622,7 +622,7 @@ backup_data_file(backup_files_arg* arguments, if (rc == PAGE_CHECKSUM_MISMATCH && is_ptrack_support) goto RetryUsingPtrack; if (rc < 0) - elog(ERROR, "Failed to read file %s: %s", + elog(ERROR, "Failed to read file \"%s\": %s", file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); n_blocks_read = rc; } @@ -1209,10 +1209,7 @@ check_data_file(ConnectionArgs *arguments, } if (file->size % BLCKSZ != 0) - { - fclose(in); - elog(WARNING, "File: %s, invalid file size %zu", file->path, file->size); - } + elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size); /* * Compute expected number of blocks in the file. From 6caf77ebbe80bb48517d85e1f14b1542d2d84c43 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 29 Jun 2019 03:19:31 +0300 Subject: [PATCH 0802/2107] tests: minor fixes --- tests/retention.py | 58 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index bfb76cd23..1407abd9b 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -798,12 +798,12 @@ def test_basic_window_merge_multiple_descendants(self): # Take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() backup_id_b = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -811,13 +811,13 @@ def test_basic_window_merge_multiple_descendants(self): page_id_a1 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() - # Change FULLb backup status to OK + # Change FULLb to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa1 backup status to ERROR + # Change PAGEa1 to ERROR self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') # PAGEa1 ERROR @@ -832,13 +832,13 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb OK # FULLa OK - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() - # Change PAGEa1 backup status to OK + # Change PAGEa1 to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - # Change PAGEb1 and FULLb backup status to ERROR + # Change PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -850,8 +850,8 @@ def test_basic_window_merge_multiple_descendants(self): page_id_a2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() # PAGEa2 OK # PAGEb1 ERROR @@ -859,11 +859,11 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - # Change PAGEb1 and FULLb backup status to OK + # Change PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa2 and FULLa backup status to ERROR + # Change PAGEa2 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') @@ -876,8 +876,8 @@ def test_basic_window_merge_multiple_descendants(self): page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() # PAGEb2 OK # PAGEa2 ERROR @@ -886,7 +886,7 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb OK # FULLa ERROR - # Change PAGEb2 and PAGEb1 status to ERROR + # Change PAGEb2 and PAGEb1 to ERROR self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') @@ -903,8 +903,8 @@ def test_basic_window_merge_multiple_descendants(self): page_id_a3 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() # PAGEa3 OK # PAGEb2 ERROR @@ -914,11 +914,12 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - # Change PAGEa3 status to ERROR + # Change PAGEa3 to ERROR self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - # Change PAGEb2 status to OK + # Change PAGEb2, PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') page_id_b3 = self.backup_node( @@ -928,7 +929,7 @@ def test_basic_window_merge_multiple_descendants(self): # PAGEa3 ERROR # PAGEb2 OK # PAGEa2 ERROR - # PAGEb1 ERROR + # PAGEb1 OK # PAGEa1 OK # FULLb OK # FULLa OK @@ -949,16 +950,15 @@ def test_basic_window_merge_multiple_descendants(self): # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + self.show_pb( + backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], page_id_a1) self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + self.show_pb( + backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], page_id_a1) - print("Backups {0} and {1} are children of {2}".format( - page_id_a3, page_id_a2, page_id_a1)) - # Purge backups backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): From 361af245df8fbecdcaadb0bfc13e983e5dcd5fd2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Jul 2019 15:15:42 +0300 Subject: [PATCH 0803/2107] tests: remove compatibility.CompatibilityTest.test_backup_concurrent_drop_table --- tests/compatibility.py | 46 ------------------------------------------ 1 file changed, 46 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 6830ce431..527b4d95e 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -532,49 +532,3 @@ def test_backward_compatibility_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_backup_concurrent_drop_table(self): - """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--compress', '--log-level-file=VERBOSE'], - gdb=True, old_binary=True) - - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - - node.safe_psql( - 'postgres', - 'DROP TABLE pgbench_accounts') - - # do checkpoint to guarantee filenode removal - node.safe_psql( - 'postgres', - 'CHECKPOINT') - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - # show_backup = self.show_pb(backup_dir, 'node')[0] - # self.assertEqual(show_backup['status'], "OK") - - # validate with fresh binary, it MUST be successful - self.validate_pb(backup_dir) - - # Clean after yourself - self.del_test_dir(module_name, fname) From 265837b106914e036b6237a1795d7576a0559e4d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Jul 2019 16:24:57 +0300 Subject: [PATCH 0804/2107] tests: minor fixes --- tests/checkdb.py | 4 ++-- tests/merge.py | 43 ++++++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 704b5ac24..1f9a464a0 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -434,12 +434,12 @@ def test_checkdb_block_validation_sanity(self): repr(e.message), self.cmd)) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 1".format( + 'WARNING: Corruption detected in file "{0}", block 1'.format( os.path.normpath(heap_full_path)), e.message) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 5".format( + 'WARNING: Corruption detected in file "{0}", block 5'.format( os.path.normpath(heap_full_path)), e.message) diff --git a/tests/merge.py b/tests/merge.py index cb7e2083c..8c328ff7c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1941,15 +1941,16 @@ def test_merge_multiple_descendants(self): # FULLb OK # FULLa OK - # Change PAGEa1 backup status to OK + # Change PAGEa1 to OK self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - # Change PAGEb1 backup status to ERROR + # Change PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a2 = self.backup_node( @@ -1958,20 +1959,22 @@ def test_merge_multiple_descendants(self): # PAGEa2 OK # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEb1 backup status to OK + # Change PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - # Change PAGEa2 backup status to ERROR + # Change PAGEa2 and FULL to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') # PAGEa2 ERROR # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1981,17 +1984,21 @@ def test_merge_multiple_descendants(self): # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR - # Change PAGEb2 and PAGEb1 status to ERROR + # Change PAGEb2, PAGEb1 and FULLb to ERROR self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change FULLa to OK + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb2 ERROR # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK page_id_a3 = self.backup_node( @@ -2002,14 +2009,16 @@ def test_merge_multiple_descendants(self): # PAGEa2 ERROR # PAGEb1 ERROR # PAGEa1 OK - # FULLb OK + # FULLb ERROR # FULLa OK - # Change PAGEa3 status to ERROR + # Change PAGEa3 and FULLa to ERROR self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - # Change PAGEb2 status to OK + # Change PAGEb2, PAGEb1 and FULLb to OK self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -2018,15 +2027,15 @@ def test_merge_multiple_descendants(self): # PAGEa3 ERROR # PAGEb2 OK # PAGEa2 ERROR - # PAGEb1 ERROR + # PAGEb1 OK # PAGEa1 OK # FULLb OK - # FULLa OK + # FULLa ERROR - # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + # Change PAGEa3, PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') # PAGEb3 OK # PAGEa3 OK From 3eed6718b002ac435fda03c6e06897b69983f603 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 1 Jul 2019 22:35:10 +0300 Subject: [PATCH 0805/2107] Add fio_dsconnect function --- src/utils/file.c | 40 +++++++++++++++++++++++++++++++++++----- src/utils/file.h | 1 + 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 71a62713d..74f603756 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -333,6 +333,20 @@ int fio_open(char const* path, int mode, fio_location location) return fd; } + +/* Close ssh session */ +void +fio_disconnect(void) +{ + if (fio_stdin) + { + SYS_CHECK(close(fio_stdin)); + SYS_CHECK(close(fio_stdout)); + fio_stdin = 0; + fio_stdout = 0; + } +} + /* Open stdio file */ FILE* fio_fopen(char const* path, char const* mode, fio_location location) { @@ -340,14 +354,30 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) if (fio_is_remote(location)) { - int flags = O_RDWR|O_CREAT; + int flags = 0; int fd; if (strcmp(mode, PG_BINARY_W) == 0) { - flags |= O_TRUNC|PG_BINARY; - } else if (strncmp(mode, PG_BINARY_R, strlen(PG_BINARY_R)) == 0) { - flags |= PG_BINARY; + flags = O_TRUNC|PG_BINARY|O_RDWR|O_CREAT; + } else if (strcmp(mode, "w") == 0) { + flags = O_TRUNC|O_RDWR|O_CREAT; + } else if (strcmp(mode, PG_BINARY_R) == 0) { + flags = O_RDONLY|PG_BINARY; + } else if (strcmp(mode, "r") == 0) { + flags = O_RDONLY; + } else if (strcmp(mode, PG_BINARY_R "+") == 0) { + /* stdio fopen("rb+") actually doesn't create unexisted file, but probackup frequently + * needs to open existed file or create new one if not exists. + * In stdio it can be done using two fopen calls: fopen("r+") and if failed then fopen("w"). + * But to eliminate extra call which especially critical in case of remote connection + * we change r+ semantic to create file if not exists. + */ + flags = O_RDWR|O_CREAT|PG_BINARY; + } else if (strcmp(mode, "r+") == 0) { /* see comment above */ + flags |= O_RDWR|O_CREAT; } else if (strcmp(mode, "a") == 0) { - flags |= O_APPEND; + flags |= O_CREAT|O_RDWR|O_APPEND; + } else { + Assert(false); } fd = fio_open(path, flags, location); if (fd >= 0) diff --git a/src/utils/file.h b/src/utils/file.h index f168d6cbe..bb6101015 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -90,6 +90,7 @@ extern int fio_seek(int fd, off_t offs); extern int fio_fstat(int fd, struct stat* st); extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); +extern void fio_disconnect(void); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, fio_location location); From d57b65fa87b057f74780d214e885de8094e78002 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 2 Jul 2019 16:49:38 +0300 Subject: [PATCH 0806/2107] Wait for ssh process completion in fio_disconnect --- src/pg_probackup.h | 1 + src/utils/file.c | 1 + src/utils/remote.c | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 420028291..36e23db44 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -577,6 +577,7 @@ extern bool in_backup_list(parray *backup_list, pgBackup *target_backup); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); extern bool launch_agent(void); extern void launch_ssh(char* argv[]); +extern void wait_ssh(void); #define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS #define COMPRESS_LEVEL_DEFAULT 1 diff --git a/src/utils/file.c b/src/utils/file.c index 74f603756..d9f669a31 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -344,6 +344,7 @@ fio_disconnect(void) SYS_CHECK(close(fio_stdout)); fio_stdin = 0; fio_stdout = 0; + wait_ssh(); } } diff --git a/src/utils/remote.c b/src/utils/remote.c index 681d8fb87..6535bf293 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -53,6 +53,7 @@ static int split_options(int argc, char* argv[], int max_options, char* options) } static int child_pid; + #if 0 static void kill_child(void) { @@ -60,6 +61,14 @@ static void kill_child(void) } #endif + +void wait_ssh(void) +{ + int status; + waitpid(child_pid, &status, 0); + elog(LOG, "SSH process %d is terminated with status %d", child_pid, status); +} + #ifdef WIN32 void launch_ssh(char* argv[]) { From 3b7fb5d4ede03be8e8d590963795563b372eb295 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Jul 2019 20:02:31 +0300 Subject: [PATCH 0807/2107] minor bugfix: close ssh connection when it is not needed --- src/backup.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backup.c b/src/backup.c index b568586eb..3259e1a5e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -534,6 +534,9 @@ do_backup_instance(PGconn *backup_conn) } } + /* ssh connection to longer needed */ + fio_disconnect(); + /* Notify end of backup */ pg_stop_backup(¤t, pg_startbackup_conn); @@ -680,6 +683,9 @@ do_backup(time_t start_time, bool no_validate) */ check_system_identifiers(backup_conn, instance_config.pgdata); + /* ssh connection to longer needed */ + fio_disconnect(); + /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) @@ -2143,6 +2149,9 @@ backup_files(void *arg) elog(WARNING, "unexpected file type %d", buf.st_mode); } + /* ssh connection to longer needed */ + fio_disconnect(); + /* Close connection */ if (arguments->conn_arg.conn) pgut_disconnect(arguments->conn_arg.conn); From da8bf6de2970d97287ee41a7e59131a834890ef1 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 3 Jul 2019 00:06:00 +0300 Subject: [PATCH 0808/2107] Make child_pid variable thread local --- src/utils/remote.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 6535bf293..a9482f363 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -5,6 +5,12 @@ #include #include +#ifdef WIN32 +#define __thread __declspec(thread) +#else +#include +#endif + #include "pg_probackup.h" #include "file.h" @@ -52,7 +58,7 @@ static int split_options(int argc, char* argv[], int max_options, char* options) return argc; } -static int child_pid; +static __thread int child_pid; #if 0 static void kill_child(void) From f87fd1b2ac8880a3a474ef263b168a0b5f94d9a9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 17:57:11 +0300 Subject: [PATCH 0809/2107] tests: added replica.ReplicaTest.test_replica_promote --- tests/replica.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/replica.py b/tests/replica.py index 1d21a58fa..92a41a6af 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -437,3 +437,91 @@ def test_take_backup_from_delayed_replica(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_promote(self): + """ + start backup from replica, during backup promote replica + check that backup is failed + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s', + 'max_wal_size': '32MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_replica( + master, replica, + replica_name='replica', synchronous=True) + + replica.slow_start(replica=True) + + master.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + self.wait_until_replica_catch_with_master(master, replica) + + # start backup from replica + gdb = self.backup_node( + backup_dir, 'replica', replica, gdb=True, + options=['--log-level-file=verbose']) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(20) + + replica.promote() + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + backup_id = self.show_pb( + backup_dir, 'replica')[0]["id"] + + # read log file content + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + f.close + + self.assertIn( + 'ERROR: the standby was promoted during online backup', + log_content) + + self.assertIn( + 'WARNING: Backup {0} is running, ' + 'setting its status to ERROR'.format(backup_id), + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 61282253dbab442715a4a9a3c5639dc80f8aa0a1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 17:58:36 +0300 Subject: [PATCH 0810/2107] tests: minor fix for basic tests suit --- tests/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index a0a552dca..e86bfabf6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,8 +12,9 @@ def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON': - loader.testMethodPrefix = 'test_basic' + if 'PG_PROBACKUP_TEST_BASIC' in os.environ: + if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON': + loader.testMethodPrefix = 'test_basic' # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) From c3d89c2552f71927edc658d9ec57e6372aaab8a1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 18:00:15 +0300 Subject: [PATCH 0811/2107] tests: minor fixes in module "archive" --- tests/archive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 81d84b2c8..497d8bb0b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -452,22 +452,23 @@ def test_archive_push_partial_file_exists(self): filename = filename_orig + '.partial' file = os.path.join(wals_dir, filename) + # emulate stale .partial file with open(file, 'a') as f: f.write(b"blahblah") f.flush() f.close() self.switch_wal_segment(node) - sleep(15) + sleep(20) # check that segment is archived if self.archive_compress: filename_orig = filename_orig + '.gz' file = os.path.join(wals_dir, filename_orig) - self.assertTrue(os.path.isfile(file)) + # successful validate means that archive-push reused stale wal segment self.validate_pb( backup_dir, 'node', options=['--recovery-target-xid={0}'.format(xid)]) From b231d845f32e98b58c7a2d7e56323c8791cbe160 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 18:06:47 +0300 Subject: [PATCH 0812/2107] tests: added backup.BackupTest.test_backup_truncate_misaligned --- tests/backup.py | 86 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 297b034c4..6be9677bb 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -468,16 +468,74 @@ def test_backup_detect_corruption(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'WARNING: Corruption detected in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Data file corruption', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.remote: + self.assertTrue( + "ERROR: Failed to read file" in e.message and + "data file checksum mismatch" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'WARNING: Corruption detected in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Data file corruption', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_truncate_misaligned(self): + """ + make node, truncate file to size not even to BLCKSIZE, + take backup + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100000) i") + + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + output = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"], return_id=False) + + self.assertIn("WARNING: File", output) + self.assertIn("invalid file size", output) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1449,16 +1507,16 @@ def test_backup_with_least_privileges_role(self): node.safe_psql( 'backupdb', - "REVOKE TEMPORARY ON DATABASE backupdb FROM PUBLIC;" - "REVOKE ALL on SCHEMA public from PUBLIC; " + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL on SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL on SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " From e1880c57c6e2d120f3174099ee23841f7f7e98da Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 18:09:00 +0300 Subject: [PATCH 0813/2107] minor bugfix: close ssh connection when it is not needed anymore --- src/backup.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index 3259e1a5e..c19483e0e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -344,6 +344,9 @@ do_backup_instance(PGconn *backup_conn) dir_list_file(backup_files_list, parray_get(external_dirs, i), false, true, false, i+1, FIO_DB_HOST); + /* close ssh session in main thread */ + fio_disconnect(); + /* Sanity check for backup_files_list, thank you, Windows: * https://github.com/postgrespro/pg_probackup/issues/48 */ @@ -512,6 +515,9 @@ do_backup_instance(PGconn *backup_conn) parray_free(prev_backup_filelist); } + /* Notify end of backup */ + pg_stop_backup(¤t, pg_startbackup_conn); + /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. */ @@ -532,16 +538,16 @@ do_backup_instance(PGconn *backup_conn) break; } } - } - - /* ssh connection to longer needed */ - fio_disconnect(); - /* Notify end of backup */ - pg_stop_backup(¤t, pg_startbackup_conn); + if (!pg_control) + elog(ERROR, "Failed to find file \"%s\" in backup filelist.", + pg_control_path); - if (current.from_replica && !exclusive_backup) set_min_recovery_point(pg_control, database_path, current.stop_lsn); + } + + /* close ssh session in main thread */ + fio_disconnect(); /* Add archived xlog files into the list of files of this backup */ if (stream_wal) @@ -683,9 +689,6 @@ do_backup(time_t start_time, bool no_validate) */ check_system_identifiers(backup_conn, instance_config.pgdata); - /* ssh connection to longer needed */ - fio_disconnect(); - /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) From 0e99b6fb966efff4e91a94cd8c365d8312beed70 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jul 2019 18:17:50 +0300 Subject: [PATCH 0814/2107] Documentation: added example for "--ssh-options" setting --- Documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 5108fd2db..f3828312c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -107,7 +107,7 @@ Current version - 2.1.3 As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -- Incremental backup: page-level incremental backup allows you save to disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow +- Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow - Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery - Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb` - Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` @@ -778,7 +778,7 @@ Specifies remote host user for SSH connection. If you omit this option, the curr Specifies pg_probackup installation directory on the remote system. --ssh-options -Specifies a string of SSH command-line options. +Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found here: (https://linux.die.net/man/5/ssh_config)[https://linux.die.net/man/5/ssh_config] #### Replica Options From 3873318eec4bf817a782d9ff879370d0a1ccdb5e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 3 Jul 2019 23:22:48 +0300 Subject: [PATCH 0815/2107] Decompress restored file by remote agent --- src/data.c | 77 +++++++++++++++++----------------------------- src/pg_probackup.h | 2 ++ src/utils/file.c | 45 +++++++++++++++++++++++++++ src/utils/file.h | 4 ++- 4 files changed, 79 insertions(+), 49 deletions(-) diff --git a/src/data.c b/src/data.c index 0304f10bc..950fe4009 100644 --- a/src/data.c +++ b/src/data.c @@ -89,7 +89,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -static int32 +int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, const char **errormsg) { @@ -719,6 +719,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, BlockNumber blknum = 0, truncate_from = 0; bool need_truncate = false; + size_t rc; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) @@ -750,9 +751,9 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, { off_t write_pos; size_t read_len; - DataPage compressed_page; /* used as read buffer */ DataPage page; - int32 uncompressed_size = 0; + int32 compressed_size; + const char *errormsg = NULL; /* File didn`t changed. Nothig to copy */ if (file->write_size == BYTES_INVALID) @@ -789,7 +790,9 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno_tmp)); } - if (header.block == 0 && header.compressed_size == 0) + compressed_size = header.compressed_size; + + if (header.block == 0 && compressed_size) { elog(VERBOSE, "Skip empty block of \"%s\"", file->path); continue; @@ -801,7 +804,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum = header.block; - if (header.compressed_size == PageIsTruncated) + if (compressed_size == PageIsTruncated) { /* * Backup contains information that this block was truncated. @@ -812,39 +815,14 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, break; } - Assert(header.compressed_size <= BLCKSZ); + Assert(compressed_size <= BLCKSZ); /* read a page from file */ - read_len = fread(compressed_page.data, 1, - MAXALIGN(header.compressed_size), in); - if (read_len != MAXALIGN(header.compressed_size)) + read_len = fread(page.data, 1, + MAXALIGN(compressed_size), in); + if (read_len != MAXALIGN(compressed_size)) elog(ERROR, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, file->path, read_len, header.compressed_size); - - /* - * if page size is smaller than BLCKSZ, decompress the page. - * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. - * we have to check, whether it is compressed or not using - * page_may_be_compressed() function. - */ - if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, - backup_version)) - { - const char *errormsg = NULL; - - uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, - file->compress_alg, &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, file->path, errormsg); - - if (uncompressed_size != BLCKSZ) - elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - file->path, uncompressed_size); - } + blknum, file->path, read_len, compressed_size); write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : blknum * BLCKSZ; @@ -865,21 +843,24 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno)); } - /* if we uncompressed the page - write page.data, - * if page wasn't compressed - - * write what we've read - compressed_page.data + /* + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. */ - if (uncompressed_size == BLCKSZ) - { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); - } - else + rc = (compressed_size != BLCKSZ || page_may_be_compressed(page.data, file->compress_alg, backup_version)) + ? fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg, &errormsg) + : fio_fwrite(out, page.data, compressed_size); + + if (rc != compressed_size) { - if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + if (errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); + + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); } } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 36e23db44..8a02fbb77 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -700,6 +700,8 @@ int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); extern void check_system_identifiers(PGconn *conn, char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); +extern int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, const char **errormsg); #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/file.c b/src/utils/file.c index d9f669a31..a8b72dfb5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -566,6 +566,35 @@ size_t fio_fwrite(FILE* f, void const* buf, size_t size) : fwrite(buf, 1, size, f); } +/* Write data to stdio file */ +size_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char** errmsg) +{ + uint32 decompressed_size; + char decompressed_page[BLCKSZ]; + + if (fio_is_remote_file(f)) + { + fio_header hdr; + + hdr.cop = FIO_WRITE_COMPRESSED; + hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; + hdr.size = size; + hdr.arg = compress_alg; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + + return size; + } + decompressed_size = do_decompress(decompressed_page, + BLCKSZ, + buf, + size, + (CompressAlg)compress_alg, errmsg); + return decompressed_size != BLCKSZ + ? 0 : fwrite(decompressed_page, 1, decompressed_size, f); +} + /* Write data to the file */ ssize_t fio_write(int fd, void const* buf, size_t size) { @@ -1381,6 +1410,22 @@ void fio_communicate(int in, int out) case FIO_WRITE: /* Write to the current position in file */ IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); break; + case FIO_WRITE_COMPRESSED: /* Write to the current position in file */ + { + char decompressed_page[BLCKSZ]; + char const* errmsg = NULL; + int32 decompressed_size = do_decompress(decompressed_page, BLCKSZ, + buf, + hdr.size, + (CompressAlg)hdr.arg, &errmsg); + if (errmsg != NULL || decompressed_size != BLCKSZ) + { + fprintf(stderr, "Failed to decompress block: %s", errmsg ? errmsg: "unknown error"); + exit(EXIT_FAILURE); + } + IO_CHECK(fio_write_all(fd[hdr.handle], decompressed_page, BLCKSZ), BLCKSZ); + } + break; case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { buf_size = hdr.arg; diff --git a/src/utils/file.h b/src/utils/file.h index bb6101015..1d19ec00b 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -33,7 +33,8 @@ typedef enum FIO_READDIR, FIO_CLOSEDIR, FIO_SEND_PAGES, - FIO_PAGE + FIO_PAGE, + FIO_WRITE_COMPRESSED, } fio_operations; typedef enum @@ -69,6 +70,7 @@ extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); +extern size_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char** errmsg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); From 74b25e4383e86ec5656a0a8ae8e3ea96f63742d8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Jul 2019 01:33:59 +0300 Subject: [PATCH 0816/2107] Revert "Decompress restored file by remote agent" This reverts commit 3873318eec4bf817a782d9ff879370d0a1ccdb5e. --- src/data.c | 77 +++++++++++++++++++++++++++++----------------- src/pg_probackup.h | 2 -- src/utils/file.c | 45 --------------------------- src/utils/file.h | 4 +-- 4 files changed, 49 insertions(+), 79 deletions(-) diff --git a/src/data.c b/src/data.c index 950fe4009..0304f10bc 100644 --- a/src/data.c +++ b/src/data.c @@ -89,7 +89,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -int32 +static int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, const char **errormsg) { @@ -719,7 +719,6 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, BlockNumber blknum = 0, truncate_from = 0; bool need_truncate = false; - size_t rc; /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ if (file->write_size != BYTES_INVALID) @@ -751,9 +750,9 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, { off_t write_pos; size_t read_len; + DataPage compressed_page; /* used as read buffer */ DataPage page; - int32 compressed_size; - const char *errormsg = NULL; + int32 uncompressed_size = 0; /* File didn`t changed. Nothig to copy */ if (file->write_size == BYTES_INVALID) @@ -790,9 +789,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno_tmp)); } - compressed_size = header.compressed_size; - - if (header.block == 0 && compressed_size) + if (header.block == 0 && header.compressed_size == 0) { elog(VERBOSE, "Skip empty block of \"%s\"", file->path); continue; @@ -804,7 +801,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum = header.block; - if (compressed_size == PageIsTruncated) + if (header.compressed_size == PageIsTruncated) { /* * Backup contains information that this block was truncated. @@ -815,14 +812,39 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, break; } - Assert(compressed_size <= BLCKSZ); + Assert(header.compressed_size <= BLCKSZ); /* read a page from file */ - read_len = fread(page.data, 1, - MAXALIGN(compressed_size), in); - if (read_len != MAXALIGN(compressed_size)) + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + if (read_len != MAXALIGN(header.compressed_size)) elog(ERROR, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, file->path, read_len, compressed_size); + blknum, file->path, read_len, header.compressed_size); + + /* + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. + */ + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) + { + const char *errormsg = NULL; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + header.compressed_size, + file->compress_alg, &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, file->path, errormsg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + file->path, uncompressed_size); + } write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : blknum * BLCKSZ; @@ -843,24 +865,21 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, blknum, file->path, strerror(errno)); } - /* - * if page size is smaller than BLCKSZ, decompress the page. - * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. - * we have to check, whether it is compressed or not using - * page_may_be_compressed() function. + /* if we uncompressed the page - write page.data, + * if page wasn't compressed - + * write what we've read - compressed_page.data */ - rc = (compressed_size != BLCKSZ || page_may_be_compressed(page.data, file->compress_alg, backup_version)) - ? fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg, &errormsg) - : fio_fwrite(out, page.data, compressed_size); - - if (rc != compressed_size) + if (uncompressed_size == BLCKSZ) { - if (errormsg != NULL) - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, file->path, errormsg); - - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + } + else + { + if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); } } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8a02fbb77..36e23db44 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -700,8 +700,6 @@ int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); extern void check_system_identifiers(PGconn *conn, char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); -extern int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, const char **errormsg); #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/file.c b/src/utils/file.c index a8b72dfb5..d9f669a31 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -566,35 +566,6 @@ size_t fio_fwrite(FILE* f, void const* buf, size_t size) : fwrite(buf, 1, size, f); } -/* Write data to stdio file */ -size_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char** errmsg) -{ - uint32 decompressed_size; - char decompressed_page[BLCKSZ]; - - if (fio_is_remote_file(f)) - { - fio_header hdr; - - hdr.cop = FIO_WRITE_COMPRESSED; - hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; - hdr.size = size; - hdr.arg = compress_alg; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - - return size; - } - decompressed_size = do_decompress(decompressed_page, - BLCKSZ, - buf, - size, - (CompressAlg)compress_alg, errmsg); - return decompressed_size != BLCKSZ - ? 0 : fwrite(decompressed_page, 1, decompressed_size, f); -} - /* Write data to the file */ ssize_t fio_write(int fd, void const* buf, size_t size) { @@ -1410,22 +1381,6 @@ void fio_communicate(int in, int out) case FIO_WRITE: /* Write to the current position in file */ IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); break; - case FIO_WRITE_COMPRESSED: /* Write to the current position in file */ - { - char decompressed_page[BLCKSZ]; - char const* errmsg = NULL; - int32 decompressed_size = do_decompress(decompressed_page, BLCKSZ, - buf, - hdr.size, - (CompressAlg)hdr.arg, &errmsg); - if (errmsg != NULL || decompressed_size != BLCKSZ) - { - fprintf(stderr, "Failed to decompress block: %s", errmsg ? errmsg: "unknown error"); - exit(EXIT_FAILURE); - } - IO_CHECK(fio_write_all(fd[hdr.handle], decompressed_page, BLCKSZ), BLCKSZ); - } - break; case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { buf_size = hdr.arg; diff --git a/src/utils/file.h b/src/utils/file.h index 1d19ec00b..bb6101015 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -33,8 +33,7 @@ typedef enum FIO_READDIR, FIO_CLOSEDIR, FIO_SEND_PAGES, - FIO_PAGE, - FIO_WRITE_COMPRESSED, + FIO_PAGE } fio_operations; typedef enum @@ -70,7 +69,6 @@ extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern size_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char** errmsg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); From 3b97c4f4cd8c583e5751908755c311009e139c03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Jul 2019 01:44:14 +0300 Subject: [PATCH 0817/2107] minor bugfix: del-instance was failing due to presense of .history files in archive directory --- src/delete.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/delete.c b/src/delete.c index 2b546c82e..5dbdbee17 100644 --- a/src/delete.c +++ b/src/delete.c @@ -803,10 +803,13 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, int do_delete_instance(void) { - parray *backup_list; - int i; + parray *backup_list; + parray *xlog_files_list; + int i; + int rc; char instance_config_path[MAXPGPATH]; + /* Delete all backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -823,23 +826,40 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - delete_walfiles(InvalidXLogRecPtr, 0, instance_config.xlog_seg_size); + xlog_files_list = parray_new(); + dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + + for (i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *wal_file = (pgFile *) parray_get(xlog_files_list, i); + if (S_ISREG(wal_file->mode)) + { + rc = unlink(wal_file->path); + if (rc != 0) + elog(WARNING, "Failed to remove file \"%s\": %s", + wal_file->path, strerror(errno)); + } + } + + /* Cleanup */ + parray_walk(xlog_files_list, pgFileFree); + parray_free(xlog_files_list); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); if (remove(instance_config_path)) { - elog(ERROR, "can't remove \"%s\": %s", instance_config_path, + elog(ERROR, "Can't remove \"%s\": %s", instance_config_path, strerror(errno)); } /* Delete instance root directories */ if (rmdir(backup_instance_path) != 0) - elog(ERROR, "can't remove \"%s\": %s", backup_instance_path, + elog(ERROR, "Can't remove \"%s\": %s", backup_instance_path, strerror(errno)); if (rmdir(arclog_path) != 0) - elog(ERROR, "can't remove \"%s\": %s", arclog_path, + elog(ERROR, "Can't remove \"%s\": %s", arclog_path, strerror(errno)); elog(INFO, "Instance '%s' successfully deleted", instance_name); From 57da2e5b3884632480e383dff3fe1f7e958c8217 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Jul 2019 01:46:45 +0300 Subject: [PATCH 0818/2107] rename "omit_symlink" flag to follow_symlink --- src/dir.c | 20 ++++++++++---------- src/pg_probackup.h | 4 ++-- src/utils/file.c | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dir.c b/src/dir.c index 53653a8e0..f05359d5e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -122,7 +122,7 @@ static int BlackListCompare(const void *str1, const void *str2); static char dir_check_file(pgFile *file); static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, + bool follow_symlink, parray *black_list, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -159,14 +159,14 @@ dir_create_dir(const char *dir, mode_t mode) } pgFile * -pgFileNew(const char *path, const char *rel_path, bool omit_symlink, +pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location) { struct stat st; pgFile *file; /* stat the file */ - if (fio_stat(path, &st, omit_symlink, location) < 0) + if (fio_stat(path, &st, follow_symlink, location) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -445,11 +445,11 @@ BlackListCompare(const void *str1, const void *str2) * List files, symbolic links and directories in the directory "root" and add * pgFile objects to "files". We add "root" to "files" if add_root is true. * - * When omit_symlink is true, symbolic link is ignored and only file or + * When follow_symlink is true, symbolic link is ignored and only file or * directory linked to will be listed. */ void -dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, +dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, int external_dir_num, fio_location location) { pgFile *file; @@ -490,7 +490,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, parray_qsort(black_list, BlackListCompare); } - file = pgFileNew(root, "", omit_symlink, external_dir_num, location); + file = pgFileNew(root, "", follow_symlink, external_dir_num, location); if (file == NULL) { /* For external directory this is not ok */ @@ -512,7 +512,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, if (add_root) parray_append(files, file); - dir_list_file_internal(files, file, exclude, omit_symlink, black_list, + dir_list_file_internal(files, file, exclude, follow_symlink, black_list, external_dir_num, location); if (!add_root) @@ -731,7 +731,7 @@ dir_check_file(pgFile *file) */ static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool omit_symlink, parray *black_list, + bool follow_symlink, parray *black_list, int external_dir_num, fio_location location) { DIR *dir; @@ -764,7 +764,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, join_path_components(child, parent->path, dent->d_name); join_path_components(rel_child, parent->rel_path, dent->d_name); - file = pgFileNew(child, rel_child, omit_symlink, external_dir_num, + file = pgFileNew(child, rel_child, follow_symlink, external_dir_num, location); if (file == NULL) continue; @@ -821,7 +821,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, * recursively. */ if (S_ISDIR(file->mode)) - dir_list_file_internal(files, file, exclude, omit_symlink, + dir_list_file_internal(files, file, exclude, follow_symlink, black_list, external_dir_num, location); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 36e23db44..000b9acb2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -587,7 +587,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool omit_symlink, bool add_root, int external_dir_num, fio_location location); + bool follow_symlink, bool add_root, int external_dir_num, fio_location location); extern void create_data_directories(parray *dest_files, const char *data_dir, @@ -620,7 +620,7 @@ extern bool fileExists(const char *path, fio_location location); extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, const char *rel_path, - bool omit_symlink, int external_dir_num, + bool follow_symlink, int external_dir_num, fio_location location); extern pgFile *pgFileInit(const char *path, const char *rel_path); extern void pgFileDelete(pgFile *file); diff --git a/src/utils/file.c b/src/utils/file.c index d9f669a31..5a2f1d7a8 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -663,7 +663,7 @@ int fio_fstat(int fd, struct stat* st) } /* Get information about file */ -int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location) +int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) { if (fio_is_remote(location)) { @@ -672,7 +672,7 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati hdr.cop = FIO_STAT; hdr.handle = -1; - hdr.arg = follow_symlinks; + hdr.arg = follow_symlink; hdr.size = path_len; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -691,7 +691,7 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_locati } else { - return follow_symlinks ? stat(path, st) : lstat(path, st); + return follow_symlink ? stat(path, st) : lstat(path, st); } } From 9b5f9b8c99496fa9def940331a4445c0c2e38490 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Jul 2019 22:51:07 +0300 Subject: [PATCH 0819/2107] [Issue #90] Refactoring --- src/archive.c | 84 ++++++++++++++++++++++++---------------------- src/pg_probackup.h | 1 + 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/src/archive.c b/src/archive.c index 0360416f1..45badcf02 100644 --- a/src/archive.c +++ b/src/archive.c @@ -143,10 +143,10 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, char to_path_temp[MAXPGPATH]; int errno_temp; /* partial handling */ - int partial_timeout = 0; - int partial_size = 0; struct stat st; - bool partial_exists = false; + int partial_file_timeout = 0; + int partial_file_size = 0; + bool partial_file_exists = false; #ifdef HAVE_LIBZ char gz_to_path[MAXPGPATH]; @@ -186,7 +186,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) { - partial_exists = true; + partial_file_exists = true; elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); } @@ -199,47 +199,28 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) { - partial_exists = true; + partial_file_exists = true; elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); } } - /* sleep a second, check if .partial file size is changing, if not, then goto p1 - * Algorihtm is not pretty however we do not expect conflict for '.partial' file - * to be frequent occurrence. - * The main goal is to protect against failed archive-push which left behind - * orphan '.partial' file. + /* Partial file is already exists, it could have happened due to failed archive-push, + * in this case partial file can be discarded, or due to concurrent archiving. + * + * Our main goal here is to try to handle partial file to prevent stalling of + * continious archiving. + * To ensure that ecncountered partial file is actually a stale "orphaned" file, + * check its size every second. + * If the size has not changed in PARTIAL_WAL_TIMER seconds, we can consider + * the file stale and reuse it. + * If file size is changing, it means that another archiver works at the same + * directory with the same files. Such partial files cannot be reused. */ - if (partial_exists) + if (partial_file_exists) { - while (1) + while (partial_file_timeout < PARTIAL_WAL_TIMER) { - /* exit from loop */ - if (partial_timeout > 10) - { - /* For 10 second the file didn`t changed its size, so consider it stale and reuse it */ - elog(WARNING, "Reusing stale destination temporary WAL file \"%s\"", to_path_temp); - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - -#ifdef HAVE_LIBZ - if (is_compress) - { - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); - if (gz_out == NULL) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); - } - else -#endif - { - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); - } - break; - } if (fio_stat(to_path_temp, &st, false, FIO_BACKUP_HOST) < 0) /* It is ok if partial is gone, we can safely error out */ @@ -247,15 +228,36 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, strerror(errno)); /* first round */ - if (!partial_timeout) - partial_size = st.st_size; + if (!partial_file_timeout) + partial_file_size = st.st_size; /* file size is changing */ - if (st.st_size > partial_size) + if (st.st_size > partial_file_size) elog(ERROR, "Destination temporary WAL file \"%s\" is not stale", to_path_temp); sleep(1); - partial_timeout++; + partial_file_timeout++; + } + + /* Partial segment is considered stale, so reuse it */ + elog(WARNING, "Reusing stale destination temporary WAL file \"%s\"", to_path_temp); + fio_unlink(to_path_temp, FIO_BACKUP_HOST); + +#ifdef HAVE_LIBZ + if (is_compress) + { + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); + if (gz_out == NULL) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); + } + else +#endif + { + out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) + elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", + to_path_temp, strerror(errno)); } } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 000b9acb2..66b814b72 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -63,6 +63,7 @@ extern const char *PROGRAM_EMAIL; #define EXTERNAL_DIR "external_directories/externaldir" /* Timeout defaults */ +#define PARTIAL_WAL_TIMER 60 #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 From 9f8df3faa3ef5bca236e41bc65845dd4783067e8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jul 2019 00:01:03 +0300 Subject: [PATCH 0820/2107] [Issue #90] minor changes --- src/archive.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/archive.c b/src/archive.c index 45badcf02..ff3678340 100644 --- a/src/archive.c +++ b/src/archive.c @@ -144,7 +144,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, int errno_temp; /* partial handling */ struct stat st; - int partial_file_timeout = 0; + int partial_try_count = 0; int partial_file_size = 0; bool partial_file_exists = false; @@ -219,7 +219,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, */ if (partial_file_exists) { - while (partial_file_timeout < PARTIAL_WAL_TIMER) + while (partial_try_count < PARTIAL_WAL_TIMER) { if (fio_stat(to_path_temp, &st, false, FIO_BACKUP_HOST) < 0) @@ -228,7 +228,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, strerror(errno)); /* first round */ - if (!partial_file_timeout) + if (!partial_try_count) partial_file_size = st.st_size; /* file size is changing */ @@ -236,7 +236,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, elog(ERROR, "Destination temporary WAL file \"%s\" is not stale", to_path_temp); sleep(1); - partial_file_timeout++; + partial_try_count++; } /* Partial segment is considered stale, so reuse it */ From 3a0046e6862c0a0a9679f24b0f3c9f1f09756a9c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jul 2019 00:25:36 +0300 Subject: [PATCH 0821/2107] [Issue #90] tests fix --- tests/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 497d8bb0b..6d7dd3884 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -459,7 +459,7 @@ def test_archive_push_partial_file_exists(self): f.close() self.switch_wal_segment(node) - sleep(20) + sleep(70) # check that segment is archived if self.archive_compress: @@ -528,14 +528,14 @@ def test_archive_push_partial_file_exists_not_stale(self): f.close() self.switch_wal_segment(node) - sleep(4) + sleep(30) with open(file, 'a') as f: f.write(b"blahblahblahblah") f.flush() f.close() - sleep(10) + sleep(40) # check that segment is NOT archived if self.archive_compress: From b12b89f3ba7098f0febfb534b99bdb70266eced1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 6 Jul 2019 03:18:03 +0300 Subject: [PATCH 0822/2107] tests: minor fixes for "checkdb" module --- tests/checkdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 704b5ac24..1f9a464a0 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -434,12 +434,12 @@ def test_checkdb_block_validation_sanity(self): repr(e.message), self.cmd)) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 1".format( + 'WARNING: Corruption detected in file "{0}", block 1'.format( os.path.normpath(heap_full_path)), e.message) self.assertIn( - "WARNING: CORRUPTION in file {0}, block 5".format( + 'WARNING: Corruption detected in file "{0}", block 5'.format( os.path.normpath(heap_full_path)), e.message) From 2a78020243ed60311834df6de6b7d3a75dc05a5c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 6 Jul 2019 03:22:27 +0300 Subject: [PATCH 0823/2107] checkdb bugfixes: query to select indexes not from "pg_global" tablespace was incorrect, also index namespace used in "--progress" and WARNING messages was incorrect --- src/checkdb.c | 58 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 6b736bdc5..051c97131 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -79,6 +79,7 @@ typedef struct pg_indexEntry { Oid indexrelid; char *name; + char *namespace; bool heapallindexed_is_supported; /* schema where amcheck extention is located */ char *amcheck_nspname; @@ -98,6 +99,8 @@ pg_indexEntry_free(void *index) if (index_ptr->name) free(index_ptr->name); + if (index_ptr->name) + free(index_ptr->namespace); if (index_ptr->amcheck_nspname) free(index_ptr->amcheck_nspname); @@ -324,7 +327,7 @@ check_indexes(void *arg) if (progress) elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'", arguments->thread_num, i + 1, n_indexes, - ind->amcheck_nspname, ind->name); + ind->namespace, ind->name); if (arguments->conn_arg.conn == NULL) { @@ -362,7 +365,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, PGconn *db_conn) { PGresult *res; - char *nspname = NULL; + char *amcheck_nspname = NULL; int i; bool heapallindexed_is_supported = false; parray *index_list = NULL; @@ -391,8 +394,8 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, return NULL; } - nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); - strcpy(nspname, PQgetvalue(res, 0, 1)); + amcheck_nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); + strcpy(amcheck_nspname, PQgetvalue(res, 0, 1)); /* heapallindexed_is_supported is database specific */ if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && @@ -419,24 +422,28 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, if (first_db_with_amcheck) { - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " - "FROM pg_index idx " - "JOIN pg_class cls ON idx.indexrelid=cls.oid " - "JOIN pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't'", + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname " + "FROM pg_catalog.pg_index idx " + "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " + "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " + "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " + "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "ORDER BY nmspc.nspname DESC", 0, NULL); } else { - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname " - "FROM pg_index idx " - "JOIN pg_class cls ON idx.indexrelid=cls.oid " - "JOIN pg_am am ON cls.relam=am.oid " - "LEFT JOIN pg_tablespace tbl " - "ON cls.reltablespace=tbl.oid " - "AND tbl.spcname <> 'pg_global' " - "WHERE am.amname='btree' AND cls.relpersistence != 't'", + res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname " + "FROM pg_catalog.pg_index idx " + "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " + "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " + "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " + "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " + "(cls.reltablespace IN " + "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " + "OR cls.reltablespace = 0) " + "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -445,15 +452,24 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, { pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); char *name = NULL; + char *namespace = NULL; + /* index oid */ ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + + /* index relname */ name = PQgetvalue(res, i, 1); ind->name = pgut_malloc(strlen(name) + 1); strcpy(ind->name, name); /* enough buffer size guaranteed */ + /* index namespace */ + namespace = PQgetvalue(res, i, 2); + ind->namespace = pgut_malloc(strlen(namespace) + 1); + strcpy(ind->namespace, namespace); /* enough buffer size guaranteed */ + ind->heapallindexed_is_supported = heapallindexed_is_supported; - ind->amcheck_nspname = pgut_malloc(strlen(nspname) + 1); - strcpy(ind->amcheck_nspname, nspname); + ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); + strcpy(ind->amcheck_nspname, amcheck_nspname); pg_atomic_clear_flag(&ind->lock); if (index_list == NULL) @@ -509,7 +525,7 @@ amcheck_one_index(check_indexes_arg *arguments, { elog(WARNING, "Thread [%d]. Amcheck failed in database '%s' for index: '%s.%s': %s", arguments->thread_num, arguments->conn_opt.pgdatabase, - ind->amcheck_nspname, ind->name, PQresultErrorMessage(res)); + ind->namespace, ind->name, PQresultErrorMessage(res)); pfree(params[0]); pfree(query); @@ -519,7 +535,7 @@ amcheck_one_index(check_indexes_arg *arguments, else elog(LOG, "Thread [%d]. Amcheck succeeded in database '%s' for index: '%s.%s'", arguments->thread_num, - arguments->conn_opt.pgdatabase, ind->amcheck_nspname, ind->name); + arguments->conn_opt.pgdatabase, ind->namespace, ind->name); pfree(params[0]); pfree(query); From fd9165257647440ac50e99648822973c3a6fccd7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 6 Jul 2019 04:00:45 +0300 Subject: [PATCH 0824/2107] [Issue #92] Improvements pointed out by review --- src/backup.c | 5 +++-- src/catalog.c | 11 ++++------- src/delete.c | 6 +++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index c4c1d7380..96723f336 100644 --- a/src/backup.c +++ b/src/backup.c @@ -200,8 +200,9 @@ do_backup_instance(PGconn *backup_conn) prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) - elog(ERROR, "Valid backup on current timeline is not found. " - "Create new FULL backup before an incremental one."); + elog(ERROR, "Valid backup on current timeline %X is not found. " + "Create new FULL backup before an incremental one.", + current.tli); pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); diff --git a/src/catalog.c b/src/catalog.c index aac3527cc..4259a4ae6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -467,10 +467,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current /* Failed to find valid FULL backup to fulfill ancestor role */ if (!full_backup) - { - elog(WARNING, "Failed to find a valid backup chain"); return NULL; - } elog(INFO, "Latest valid FULL backup: %s", base36enc(full_backup->start_time)); @@ -480,7 +477,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* only valid descendants are acceptable */ + /* only valid descendants are acceptable for evaluation */ if ((backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE)) { @@ -505,9 +502,9 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current continue; /* chain is ok */ - case 2 : - /* Yes, we could call is_parent() earlier, after choosing the ancestor, - * but this way we have an opportunity to report about all possible + case 2: + /* Yes, we could call is_parent() earlier - after choosing the ancestor, + * but this way we have an opportunity to detect and report all possible * anomalies. */ if (is_parent(full_backup->start_time, backup, true)) diff --git a/src/delete.c b/src/delete.c index 15a62bfeb..1bce734cd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -225,7 +225,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Consider only valid backups for Redundancy */ + /* Consider only valid FULL backups for Redundancy */ if (instance_config.retention_redundancy > 0 && backup->backup_mode == BACKUP_MODE_FULL && (backup->status == BACKUP_STATUS_OK || @@ -233,6 +233,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { n_full_backups++; + /* Add every FULL backup that satisfy Redundancy policy to separate list */ if (n_full_backups <= instance_config.retention_redundancy) { if (!redundancy_full_backup_list) @@ -260,7 +261,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg bool redundancy_keep = false; pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - /* check if backups FULL parent is in redundancy list */ + /* check if backup`s FULL ancestor is in redundancy list */ if (redundancy_full_backup_list) { pgBackup *full_backup = find_parent_full_backup(backup); @@ -283,7 +284,6 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg * TODO: consider that ERROR backup most likely to have recovery_time == 0 */ if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) && -// (instance_config.retention_redundancy <= (n_full_backups - cur_full_backup_num))) (instance_config.retention_redundancy == 0 || !redundancy_keep)) { /* This backup is not guarded by retention From 99938658c6ce109d3ad327c2f5dd80fb2a5ec34b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jul 2019 17:31:42 +0300 Subject: [PATCH 0825/2107] tests: minor fixes --- tests/archive.py | 2 +- tests/backup.py | 6 +++--- tests/checkdb.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 497d8bb0b..1d9a4ab0f 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -413,7 +413,7 @@ def test_arhive_push_file_exists_overwrite(self): # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): - """Archive-push if stale .partial file exists""" + """Archive-push if stale '.partial' file exists""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/backup.py b/tests/backup.py index 2ac50c155..79c9c49e5 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -131,7 +131,7 @@ def test_incremental_backup_without_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Valid backup on current timeline is not found. " + "ERROR: Valid backup on current timeline 1 is not found. " "Create new FULL backup before an incremental one.", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( @@ -149,7 +149,7 @@ def test_incremental_backup_without_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Valid backup on current timeline is not found. " + "ERROR: Valid backup on current timeline 1 is not found. " "Create new FULL backup before an incremental one.", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( @@ -213,7 +213,7 @@ def test_incremental_backup_corrupt_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Valid backup on current timeline is not found. " + "ERROR: Valid backup on current timeline 1 is not found. " "Create new FULL backup before an incremental one.", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( diff --git a/tests/checkdb.py b/tests/checkdb.py index 1f9a464a0..7c0bdda8e 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -484,7 +484,7 @@ def test_checkdb_sigint_handling(self): gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') - gdb.continue_execution_until_exit() + gdb.continue_execution_until_error() with open(node.pg_log_file, 'r') as f: output = f.read() From a7f11405ff9e72eace4976fc6a87c87f15eb56d4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jul 2019 17:33:10 +0300 Subject: [PATCH 0826/2107] correctly open recovery.conf --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 6c434cc65..b52c1dc3f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -789,7 +789,7 @@ create_recovery_conf(time_t backup_id, elog(LOG, "creating recovery.conf"); snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); - fp = fio_fopen(path, "wt", FIO_DB_HOST); + fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); if (fp == NULL) elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, strerror(errno)); From 549d66a29ca6d27bb8ebd4c9cdc2945d4d891054 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jul 2019 17:34:12 +0300 Subject: [PATCH 0827/2107] checkdb: escape database name in amcheck related elog messages --- src/checkdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/checkdb.c b/src/checkdb.c index 051c97131..c19d84ae9 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -649,7 +649,7 @@ do_amcheck(ConnectionOptions conn_opt, PGconn *conn) if (check_isok) elog(INFO, "Amcheck succeeded for database '%s'", dbname); else - elog(WARNING, "Amcheck failed for database %s", dbname); + elog(WARNING, "Amcheck failed for database '%s'", dbname); parray_walk(index_list, pg_indexEntry_free); parray_free(index_list); From c3ebbec9535322a1fb9e29118c03d7099b52371a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 02:40:20 +0300 Subject: [PATCH 0828/2107] correct flag for recovery.conf creation --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index b52c1dc3f..c8efcb396 100644 --- a/src/restore.c +++ b/src/restore.c @@ -789,7 +789,7 @@ create_recovery_conf(time_t backup_id, elog(LOG, "creating recovery.conf"); snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + fp = fio_fopen(path, "w", FIO_DB_HOST); if (fp == NULL) elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, strerror(errno)); From b598c2987c5ba0daefe4d1561e49cc51c80271b2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 02:41:20 +0300 Subject: [PATCH 0829/2107] tests: minor fixes --- tests/archive.py | 64 +++++++++++++++++++++++++++++++++--------------- tests/backup.py | 8 +----- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 1d9a4ab0f..d623aa70d 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -311,17 +311,33 @@ def test_arhive_push_file_exists(self): "from generate_series(0,100500) i") log_file = os.path.join(node.logs_dir, 'postgresql.log') + self.switch_wal_segment(node) + sleep(1) + with open(log_file, 'r') as f: log_content = f.read() - self.assertTrue( - 'LOG: archive command failed with exit code 1' in log_content and - 'DETAIL: The failed archive command was:' in log_content and - 'INFO: pg_probackup archive-push from' in log_content and - 'ERROR: WAL segment ' in log_content and - '{0}" already exists.'.format(filename) in log_content, - 'Expecting error messages about failed archive_command' - ) - self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + self.assertIn( + 'LOG: archive command failed with exit code 1', + log_content) + + self.assertIn( + 'DETAIL: The failed archive command was:', + log_content) + + self.assertIn( + 'INFO: pg_probackup archive-push from', + log_content) + + self.assertIn( + 'ERROR: WAL segment ', + log_content) + + self.assertIn( + 'already exists.', + log_content) + + self.assertNotIn( + 'pg_probackup archive-push completed successfully', log_content) if self.get_version(node) < 100000: wal_src = os.path.join( @@ -342,9 +358,10 @@ def test_arhive_push_file_exists(self): with open(log_file, 'r') as f: log_content = f.read() - self.assertTrue( - 'pg_probackup archive-push completed successfully' in log_content, - 'Expecting messages about successfull execution archive_command') + + self.assertIn( + 'pg_probackup archive-push completed successfully', + log_content) # Clean after yourself self.del_test_dir(module_name, fname) @@ -386,16 +403,23 @@ def test_arhive_push_file_exists_overwrite(self): "from generate_series(0,100500) i") log_file = os.path.join(node.logs_dir, 'postgresql.log') + self.switch_wal_segment(node) + sleep(1) + with open(log_file, 'r') as f: log_content = f.read() - self.assertTrue( - 'LOG: archive command failed with exit code 1' in log_content and - 'DETAIL: The failed archive command was:' in log_content and - 'INFO: pg_probackup archive-push from' in log_content and - '{0}" already exists.'.format(filename) in log_content, - 'Expecting error messages about failed archive_command' - ) - self.assertFalse('pg_probackup archive-push completed successfully' in log_content) + + self.assertIn( + 'LOG: archive command failed with exit code 1', log_content) + self.assertIn( + 'DETAIL: The failed archive command was:', log_content) + self.assertIn( + 'INFO: pg_probackup archive-push from', log_content) + self.assertIn( + '{0}" already exists.'.format(filename), log_content) + + self.assertNotIn( + 'pg_probackup archive-push completed successfully', log_content) self.set_archiving(backup_dir, 'node', node, overwrite=True) node.reload() diff --git a/tests/backup.py b/tests/backup.py index 79c9c49e5..47cf818b3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1765,13 +1765,7 @@ def test_parent_choosing_2(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'WARNING: Failed to find a valid backup chain', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'ERROR: Valid backup on current timeline is not found. ' + 'ERROR: Valid backup on current timeline 1 is not found. ' 'Create new FULL backup before an incremental one.', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( From 433d7bdc6a0f6cabb9db8f6120c78d75e161dade Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 16:02:30 +0300 Subject: [PATCH 0830/2107] tests: fix expected results --- tests/expected/option_help.out | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 539e0d307..2d7ae3b76 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -26,9 +26,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] pg_probackup show-config -B backup-path --instance=instance_name [--format=format] + [--help] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-D pgdata-path] [-C] @@ -55,6 +57,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] pg_probackup restore -B backup-path --instance=instance_name [-D pgdata-path] [-i backup-id] [-j num-threads] @@ -72,6 +75,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] pg_probackup validate -B backup-path [--instance=instance_name] [-i backup-id] [--progress] [-j num-threads] @@ -80,22 +84,27 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target-timeline=timeline] [--recovery-target-name=target-name] [--skip-block-validation] + [--help] pg_probackup checkdb [-B backup-path] [--instance=instance_name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] [--heapallindexed] + [--help] pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] [--format=format] + [--help] pg_probackup delete -B backup-path --instance=instance_name [--wal] [-i backup-id | --expired | --merge-expired] [--dry-run] + [--help] pg_probackup merge -B backup-path --instance=instance_name -i backup-id [--progress] [-j num-threads] + [--help] pg_probackup add-instance -B backup-path -D pgdata-path --instance=instance_name @@ -103,9 +112,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] pg_probackup del-instance -B backup-path --instance=instance_name + [--help] pg_probackup archive-push -B backup-path --instance=instance_name --wal-file-path=wal-file-path @@ -117,6 +128,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] pg_probackup archive-get -B backup-path --instance=instance_name --wal-file-path=wal-file-path @@ -124,6 +136,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--help] Read the website for details. Report bugs to . From 3c05d877087233a555705573be62bae2532528ff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 19:47:27 +0300 Subject: [PATCH 0831/2107] tests: minor fixes for 9.5 version --- tests/archive.py | 28 ++++++++++++++++++++-------- tests/delete.py | 3 +-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index d623aa70d..5f4aa0f8e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -462,10 +462,16 @@ def test_archive_push_partial_file_exists(self): "postgres", "INSERT INTO t1 VALUES (1) RETURNING (xmin)").rstrip() - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + if self.get_version(node) < 100000: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() + else: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() # form up path to next .partial WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') @@ -532,10 +538,16 @@ def test_archive_push_partial_file_exists_not_stale(self): "postgres", "create table t2()") - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + if self.get_version(node) < 100000: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() + else: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() # form up path to next .partial WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') diff --git a/tests/delete.py b/tests/delete.py index a9d9eb167..858d8b2c8 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -93,8 +93,7 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), - initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 29ebf687d4d378edbf58dd2d54be16b36c1d9223 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 19:48:27 +0300 Subject: [PATCH 0832/2107] Documentation: minor fixes --- Documentation.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Documentation.md b/Documentation.md index f3828312c..20dcd6487 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1052,7 +1052,7 @@ For each backup, the following information is provided: - ID — the backup identifier. - Recovery time — the earliest moment for which you can restore the state of the database cluster. - Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. -- WAL — the way of WAL log handling. Possible values: STREAM for autonomous backups and ARCHIVE for archive backups. +- WAL — the WAL delivery method. Possible values: STREAM and ARCHIVE. - Current/Parent TLI — timeline identifiers of current backup and its parent. - Time — the time it took to perform the backup. - Data — the size of the data files in this backup. This value does not include the size of WAL files. @@ -1061,8 +1061,8 @@ For each backup, the following information is provided: - Status — backup status. Possible values: - OK — the backup is complete and valid. - - RUNNING — the backup is in progress. - DONE — the backup is complete, but was not validated. + - RUNNING — the backup is in progress. - MERGING — the backup is being merged. - DELETING — the backup files are being deleted. - CORRUPT — some of the backup files are corrupted. @@ -1156,9 +1156,9 @@ By default, all backup copies created with pg_probackup are stored in the specif To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): - retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. -- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if retention-window=7, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. +- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if `retention-window=7`, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. -If both retention-redundancy and retention-window options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set retention-redundancy=2 and retention-window=7, pg_probackup cleans up the backup directory to keep only two full backup copies and all backups that are newer than seven days. +If both `retention-redundancy` and `retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `retention-redundancy=2` and `retention-window=7`, pg_probackup cleans up the backup directory to keep only two full backup copies and all backups that are newer than seven days. To clean up the backup catalog in accordance with retention policy, run: @@ -1170,11 +1170,11 @@ If you would like to also remove the WAL files that are no longer required for a pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal ->NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired` and `--delete-wal` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. +>NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` option when running [backup](#backup) or [delete](#delete) commands. -Suppose you have backed up the node instance in the node-backup directory, with the retention-window option is set to 7, and you have the following backups available on April 10, 2019: +Suppose you have backed up the node instance in the *backup_dir* directory, with the `retention-window` option is set to 7, and you have the following backups available on April 10, 2019: ``` BACKUP INSTANCE 'node' @@ -1190,7 +1190,7 @@ BACKUP INSTANCE 'node' node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK ``` -Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so if you run the `delete` command with the `--delete-expired` option, only the P7XDFT full backup will be removed. +Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` option, only the P7XDFT full backup will be removed. With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: @@ -1215,9 +1215,13 @@ As you take more and more incremental backups, the total size of the backup cata pg_probackup merge -B backup_dir --instance instance_name -i backup_id -This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes. +This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes, I/O and network traffic in case of [remote](#using-pg_probackup-in-the-remote-mode) backup. + +Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the [show](#show) command with the backup ID: + + pg_probackup show -B backup_dir --instance instance_name -i backup_id -Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the show command with the backup ID. If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. +If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. ### Deleting Backups @@ -1225,7 +1229,7 @@ To delete a backup that is no longer required, run the following command: pg_probackup delete -B backup_dir --instance instance_name -i backup_id -This command will delete the backup with the specified *backup_id*, together with all the incremental backups that followed, if any. This way, you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. +This command will delete the backup with the specified *backup_id*, together with all the incremental backups that descend from *backup_id* if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` option: From 7b95a40b508abbad4678b78f7e7a5038a0f10691 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jul 2019 19:57:08 +0300 Subject: [PATCH 0833/2107] Documentation: another minor fix --- Documentation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 20dcd6487..335af81c8 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1174,7 +1174,7 @@ If you would like to also remove the WAL files that are no longer required for a Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` option when running [backup](#backup) or [delete](#delete) commands. -Suppose you have backed up the node instance in the *backup_dir* directory, with the `retention-window` option is set to 7, and you have the following backups available on April 10, 2019: +Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `retention-window` option is set to 7, and you have the following backups available on April 10, 2019: ``` BACKUP INSTANCE 'node' @@ -1194,8 +1194,8 @@ Even though P7XDHB and P7XDHU backups are outside the retention window, they can With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: - pg_probackup delete -B node-backup --instance node --delete-expired --merge-expired - pg_probackup show -B node-backup + pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired + pg_probackup show -B backup_dir ``` BACKUP INSTANCE 'node' From 6273f80e502be68cb4b6d0a452309d07628da453 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jul 2019 02:26:00 +0300 Subject: [PATCH 0834/2107] minor improvement: use slurpFile instead fetchFile, because fetchFile is relying on pg_read_binary_file(), which is insecure and redundant --- src/util.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/util.c b/src/util.c index 5f9e87f59..93790283d 100644 --- a/src/util.c +++ b/src/util.c @@ -153,7 +153,7 @@ get_current_timeline(bool safe) size_t size; /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, + buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, safe, FIO_DB_HOST); if (safe && buffer == NULL) return 0; @@ -196,7 +196,7 @@ get_checkpoint_location(PGconn *conn) size_t size; ControlFileData ControlFile; - buffer = fetchFile(conn, "global/pg_control", &size); + buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -212,7 +212,7 @@ get_system_identifier(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -246,7 +246,7 @@ get_remote_system_identifier(PGconn *conn) size_t size; ControlFileData ControlFile; - buffer = fetchFile(conn, "global/pg_control", &size); + buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -263,9 +263,7 @@ get_xlog_seg_size(char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_DB_HOST); - if (buffer == NULL) - return 0; + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -283,7 +281,7 @@ get_data_checksum_version(bool safe) size_t size; /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, "global/pg_control", &size, + buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, safe, FIO_DB_HOST); if (buffer == NULL) return 0; @@ -301,9 +299,8 @@ get_pgcontrol_checksum(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, "global/pg_control", &size, false, FIO_BACKUP_HOST); - if (buffer == NULL) - return 0; + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST); + digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -325,9 +322,6 @@ set_min_recovery_point(pgFile *file, const char *backup_path, /* First fetch file content */ buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - if (buffer == NULL) - elog(ERROR, "ERROR"); - digestControlFile(&ControlFile, buffer, size); elog(LOG, "Current minRecPoint %X/%X", From 3d9bb1b076dc5632a10f716f09fc544445bd3d9d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jul 2019 12:31:38 +0300 Subject: [PATCH 0835/2107] minor change of elevel --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 4259a4ae6..9ad797c78 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -469,7 +469,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current if (!full_backup) return NULL; - elog(INFO, "Latest valid FULL backup: %s", + elog(LOG, "Latest valid FULL backup: %s", base36enc(full_backup->start_time)); /* FULL backup is found, lets find his latest child */ From b38692f46490b3d47414b35b2f8d2f33f6eebcb3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jul 2019 18:01:28 +0300 Subject: [PATCH 0836/2107] tests: now ptrack tests by default are disabled, they can be enables via PG_PROBACKUP_PTRACK env variable. Multiple fixes, especially for PG 9.5 --- tests/Readme.md | 3 + tests/__init__.py | 5 +- tests/archive.py | 23 ++--- tests/backup.py | 176 +++++++++++++++++++++----------- tests/checkdb.py | 2 +- tests/compatibility.py | 17 +-- tests/compression.py | 98 +++++++----------- tests/delete.py | 3 + tests/delta.py | 6 ++ tests/exclude.py | 12 +-- tests/false_positive.py | 11 +- tests/helpers/ptrack_helpers.py | 8 +- tests/logging.py | 7 +- tests/merge.py | 28 +++-- tests/restore.py | 65 +++++++----- tests/retention.py | 4 + 16 files changed, 281 insertions(+), 187 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index c0d9c0248..d24d095cd 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -29,6 +29,9 @@ Remote backup depends on key authentithication to local machine via ssh as curre Run suit of basic simple tests: export PG_PROBACKUP_TEST_BASIC=ON +Run ptrack tests: + export PG_PROBACKUP_PTRACK=ON + Usage: pip install testgres diff --git a/tests/__init__.py b/tests/__init__.py index e86bfabf6..7cc96276d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -37,7 +37,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(merge)) suite.addTests(loader.loadTestsFromModule(option)) suite.addTests(loader.loadTestsFromModule(page)) -# suite.addTests(loader.loadTestsFromModule(ptrack)) suite.addTests(loader.loadTestsFromModule(pgpro560)) suite.addTests(loader.loadTestsFromModule(pgpro589)) suite.addTests(loader.loadTestsFromModule(pgpro2068)) @@ -50,6 +49,10 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(validate)) + if 'PG_PROBACKUP_PTRACK' in os.environ: + if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': + suite.addTests(loader.loadTestsFromModule(ptrack)) + return suite # test_pgpro434_2 unexpected success diff --git a/tests/archive.py b/tests/archive.py index 5f4aa0f8e..1ce602210 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -228,10 +228,8 @@ def test_pgpro434_3(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) + initdb_params=['--data-checksums']) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -258,18 +256,19 @@ def test_pgpro434_3(self): log_file = os.path.join(backup_dir, 'log/pg_probackup.log') with open(log_file, 'r') as f: log_content = f.read() - self.assertNotIn( - "ERROR: pg_stop_backup doesn't answer", - log_content, - "pg_stop_backup timeouted") + + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: log_content = f.read() - self.assertNotIn( - 'FailedAssertion', - log_content, - 'PostgreSQL crashed because of a failed assert') + + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/backup.py b/tests/backup.py index 47cf818b3..d18e2d978 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -15,6 +15,9 @@ class BackupTest(ProbackupTest, unittest.TestCase): # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -109,6 +112,9 @@ def test_smooth_checkpoint(self): # @unittest.skip("skip") def test_incremental_backup_without_full(self): """page-level backup without validated full backup""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -168,8 +174,7 @@ def test_incremental_backup_corrupt_full(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -230,6 +235,9 @@ def test_incremental_backup_corrupt_full(self): # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -258,6 +266,9 @@ def test_ptrack_threads(self): # @unittest.skip("skip") def test_ptrack_threads_stream(self): """ptrack multi thread backup mode and stream""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -287,6 +298,9 @@ def test_ptrack_threads_stream(self): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -343,6 +357,9 @@ def test_page_corruption_heal_via_ptrack_1(self): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -988,6 +1005,9 @@ def test_drop_rel_during_backup_page(self): # @unittest.skip("skip") def test_drop_rel_during_backup_ptrack(self): """""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1492,10 +1512,12 @@ def test_backup_with_least_privileges_role(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - # pg_options={'ptrack_enable': 'on'}, initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s'}) - ) + if self.ptrack: + node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1506,58 +1528,87 @@ def test_backup_with_least_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - # for partial restore, checkdb and ptrack - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - # for exclusive backup for PG 9.5 and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - # ptrack functions - # for fname in [ - # 'oideq(oid, oid)', - # 'ptrack_version()', - # 'pg_ptrack_clear()', - # 'pg_ptrack_control_lsn()', - # 'pg_ptrack_get_and_clear_db(oid, oid)', - # 'pg_ptrack_get_and_clear(oid, oid)', - # 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: - # try: - # node.safe_psql( - # "backupdb", - # "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " - # "TO backup".format(fname)) - # except: - # pass + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if self.ptrack: + for fname in [ + 'oideq(oid, oid)', + 'ptrack_version()', + 'pg_ptrack_clear()', + 'pg_ptrack_control_lsn()', + 'pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_ptrack_get_and_clear(oid, oid)', + 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: + # try: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " + "TO backup".format(fname)) + # except: + # pass # FULL backup self.backup_node( @@ -1584,12 +1635,13 @@ def test_backup_with_least_privileges_role(self): datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK - # self.backup_node( - # backup_dir, 'node', node, backup_type='ptrack', - # datname='backupdb', options=['-U', 'backup']) - # self.backup_node( - # backup_dir, 'node', node, backup_type='ptrack', - # datname='backupdb', options=['--stream', '-U', 'backup']) + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/checkdb.py b/tests/checkdb.py index 7c0bdda8e..1d546591b 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -229,7 +229,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): # create two databases node.safe_psql("postgres", "create database db1") try: - node.safe_psql( + node.safe_psql( "db1", "create extension amcheck") except QueryException as e: diff --git a/tests/compatibility.py b/tests/compatibility.py index 527b4d95e..7c7038828 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -217,6 +217,10 @@ def test_backward_compatibility_delta(self): # @unittest.skip("skip") def test_backward_compatibility_ptrack(self): """Description in jira issue PGPRO-434""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -224,8 +228,9 @@ def test_backward_compatibility_ptrack(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off'} - ) + 'autovacuum': 'off', + 'ptrack_enable': 'on'}) + self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -262,7 +267,7 @@ def test_backward_compatibility_ptrack(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Delta BACKUP with old binary + # ptrack BACKUP with old binary pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -272,7 +277,7 @@ def test_backward_compatibility_ptrack(self): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta', + backup_dir, 'node', node, backup_type='ptrack', old_binary=True) if self.paranoia: @@ -287,7 +292,7 @@ def test_backward_compatibility_ptrack(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Delta BACKUP with new binary + # Ptrack BACKUP with new binary pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -297,7 +302,7 @@ def test_backward_compatibility_ptrack(self): pgbench.stdout.close() self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_dir, 'node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/compression.py b/tests/compression.py index ef2ecf5ee..ea288a5e1 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -23,11 +23,7 @@ def test_basic_compression_stream_zlib(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -59,15 +55,15 @@ def test_basic_compression_stream_zlib(self): options=[ '--stream', '--compress-algorithm=zlib']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=zlib']) # Drop Node @@ -105,11 +101,11 @@ def test_basic_compression_stream_zlib(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -117,8 +113,8 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -135,11 +131,7 @@ def test_compression_archive_zlib(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -167,14 +159,14 @@ def test_compression_archive_zlib(self): backup_dir, 'node', node, backup_type='page', options=["--compress-algorithm=zlib"]) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=zlib']) # Drop Node @@ -212,11 +204,11 @@ def test_compression_archive_zlib(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -224,8 +216,8 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -242,11 +234,7 @@ def test_compression_stream_pglz(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -275,7 +263,7 @@ def test_compression_stream_pglz(self): backup_dir, 'node', node, backup_type='page', options=['--stream', '--compress-algorithm=pglz']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " @@ -283,7 +271,7 @@ def test_compression_stream_pglz(self): "from generate_series(512,768) i") ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=pglz']) # Drop Node @@ -321,11 +309,11 @@ def test_compression_stream_pglz(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -333,8 +321,8 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -351,11 +339,7 @@ def test_compression_archive_pglz(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -384,15 +368,15 @@ def test_compression_archive_pglz(self): backup_dir, 'node', node, backup_type='page', options=['--compress-algorithm=pglz']) - # PTRACK BACKUP + # DELTA BACKUP node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(200,300) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=pglz']) # Drop Node @@ -430,11 +414,11 @@ def test_compression_archive_pglz(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Check ptrack backup + # Check delta backup self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + "INFO: Restore of backup {0} completed.".format(delta_backup_id), self.restore_node( - backup_dir, 'node', node, backup_id=ptrack_backup_id, + backup_dir, 'node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", "--recovery-target-action=promote"]), @@ -442,8 +426,8 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - ptrack_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) node.cleanup() # Clean after yourself @@ -460,11 +444,7 @@ def test_compression_wrong_algorithm(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on'} - ) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/delete.py b/tests/delete.py index 858d8b2c8..ce6bbb98d 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -188,6 +188,9 @@ def test_delete_increment_page(self): # @unittest.skip("skip") def test_delete_increment_ptrack(self): """delete increment and all after him""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/delta.py b/tests/delta.py index a08d62d12..6a5a9d736 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1078,6 +1078,9 @@ def test_delta_delete(self): # @unittest.skip("skip") def test_delta_corruption_heal_via_ptrack_1(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1135,6 +1138,9 @@ def test_delta_corruption_heal_via_ptrack_1(self): # @unittest.skip("skip") def test_page_corruption_heal_via_ptrack_2(self): """make node, corrupt some page, check that backup failed""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/exclude.py b/tests/exclude.py index 5e641bada..2e644d7e9 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -20,10 +20,7 @@ def test_exclude_temp_tables(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2', - 'shared_buffers': '1GB', 'fsync': 'off', 'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -102,7 +99,7 @@ def test_exclude_temp_tables(self): def test_exclude_unlogged_tables_1(self): """ make node without archiving, create unlogged table, take full backup, - alter table to unlogged, take ptrack backup, restore ptrack backup, + alter table to unlogged, take delta backup, restore delta backup, check that PGDATA`s are physically the same """ fname = self.id().split('.')[3] @@ -113,8 +110,7 @@ def test_exclude_unlogged_tables_1(self): initdb_params=['--data-checksums'], pg_options={ 'autovacuum': 'off', - "shared_buffers": "10MB", - 'ptrack_enable': 'on'}) + "shared_buffers": "10MB"}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -138,7 +134,7 @@ def test_exclude_unlogged_tables_1(self): node.safe_psql('postgres', "alter table test set logged") self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + backup_dir, 'node', node, backup_type='delta', options=['--stream'] ) diff --git a/tests/false_positive.py b/tests/false_positive.py index d33309f4a..d3b27f877 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -57,8 +57,7 @@ def test_incremental_backup_corrupt_full_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -114,6 +113,10 @@ def test_incremental_backup_corrupt_full_1(self): def test_ptrack_concurrent_get_and_clear_1(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -195,6 +198,10 @@ def test_ptrack_concurrent_get_and_clear_1(self): def test_ptrack_concurrent_get_and_clear_2(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" + + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 7bd31a898..52edc894d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -271,7 +271,13 @@ def __init__(self, *args, **kwargs): self.remote_user = None if 'PGPROBACKUP_SSH_REMOTE' in self.test_env: - self.remote = True + if self.test_env['PGPROBACKUP_SSH_REMOTE'] == 'ON': + self.remote = True + + self.ptrack = False + if 'PG_PROBACKUP_PTRACK' in self.test_env: + if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': + self.ptrack = True @property def pg_config_version(self): diff --git a/tests/logging.py b/tests/logging.py index c905695bd..efde1d0b9 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -142,7 +142,7 @@ def test_truncate_rotation_file(self): log_file_size) self.assertNotIn( - 'WARNING:', + 'WARNING: cannot read creation timestamp from rotation file', output) self.assertTrue(os.path.isfile(rotation_file_path)) @@ -166,7 +166,6 @@ def test_unlink_rotation_file(self): backup_dir, 'node', options=['--log-rotation-age=1d']) - self.backup_node( backup_dir, 'node', node, options=[ @@ -212,7 +211,7 @@ def test_unlink_rotation_file(self): return_id=False) self.assertNotIn( - 'WARNING:', + 'WARNING: missing rotation file:', output) # check that log file wasn`t rotated @@ -291,7 +290,7 @@ def test_garbage_in_rotation_file(self): return_id=False) self.assertNotIn( - 'WARNING:', + 'WARNING: rotation file', output) # check that log file wasn`t rotated diff --git a/tests/merge.py b/tests/merge.py index 8c328ff7c..e539f2816 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -826,6 +826,9 @@ def test_merge_ptrack_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1651,8 +1654,6 @@ def test_losing_file_after_failed_merge(self): os.remove(file_to_remove) # Try to continue failed MERGE - #print(backup_id) - #exit(1) self.merge_backup(backup_dir, "node", backup_id) self.assertEqual( @@ -1875,15 +1876,28 @@ def test_merge_backup_from_future(self): pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) - pgdata_restored = self.pgdata_content(node.data_dir) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that merged backup has the same state as + node_restored.cleanup() + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) + pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself diff --git a/tests/restore.py b/tests/restore.py index 5a86b0bb1..0175d19df 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -492,6 +492,9 @@ def test_restore_to_lsn_not_inclusive(self): # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -540,6 +543,9 @@ def test_restore_full_ptrack_archive(self): # @unittest.skip("skip") def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -595,6 +601,9 @@ def test_restore_ptrack(self): # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -647,6 +656,9 @@ def test_restore_full_ptrack_under_load(self): recovery to latest from full + ptrack backups with loads when ptrack backup do """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -711,6 +723,9 @@ def test_restore_full_under_load_ptrack(self): recovery to latest from full + page backups with loads when full backup do """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -2028,12 +2043,12 @@ def test_restore_target_new_options(self): node.slow_start() - # Restore with recovery target lsn + # Restore with recovery target name node.cleanup() self.restore_node( backup_dir, 'node', node, options=[ - '--recovery-target-lsn={0}'.format(target_lsn), + '--recovery-target-name={0}'.format(target_name), "--recovery-target-action=promote", '--recovery-target-timeline=1', ]) @@ -2042,7 +2057,7 @@ def test_restore_target_new_options(self): recovery_conf_content = f.read() self.assertIn( - "recovery_target_lsn = '{0}'".format(target_lsn), + "recovery_target_name = '{0}'".format(target_name), recovery_conf_content) self.assertIn( @@ -2055,32 +2070,34 @@ def test_restore_target_new_options(self): node.slow_start() - # Restore with recovery target name - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-name={0}'.format(target_name), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) + # Restore with recovery target lsn + if self.get_version(node) >= 100000: - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) - self.assertIn( - "recovery_target_name = '{0}'".format(target_name), - recovery_conf_content) + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) - node.slow_start() + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/retention.py b/tests/retention.py index 1407abd9b..9eb5b0206 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1281,6 +1281,10 @@ def test_retention_redundancy_overlapping_chains(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums']) + if self.get_version(node) < 90600: + self.del_test_dir(module_name, fname) + return unittest.skip('Skipped because ptrack support is disabled') + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 8c5d513a6f4743dd5a0850e580a4ea7f400c9bc8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jul 2019 18:22:20 +0300 Subject: [PATCH 0837/2107] Documentation: minor improvements --- Documentation.md | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/Documentation.md b/Documentation.md index 335af81c8..b9c26b2c2 100644 --- a/Documentation.md +++ b/Documentation.md @@ -184,32 +184,55 @@ For details on how to fine-tune pg_probackup configuration, see the section [Con The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. +>NOTE: For PostgreSQL >= 11 it is recommended to use [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature, so backup can be done by OS user with read-only permissions. + ### Configuring the Database Cluster Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. -To enable backups, the following rights are required: +To perform [backup](#backup), the following permissions are required: +For PostgreSQL 9.5: ``` +BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; -GRANT EXECUTE ON FUNCTION txid_current() TO backup; -GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +COMMIT; +``` + +For PostgreSQL >= 9.6: ``` +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +COMMIT; +``` + +>NOTE: In PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` can be executed only by superuser role. So during backup of PostgreSQL 9.5 pg_probackup will use them only if backup role is superuser, although it is NOT recommended to run backup under superuser. Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. -For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up PTRACK Backups](#setting-up-ptrack-backups) and [Setting up Backup from Standby](#backup-from-standby). +For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#backup-from-standby) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). ### Setting up STREAM Backups From 66618d01423d13cfa0820934c022ddaffe310f27 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 14 Jul 2019 03:07:15 +0300 Subject: [PATCH 0838/2107] minor fix --- src/utils/pgut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 49eae0486..7ef828ec1 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -419,7 +419,7 @@ pgut_execute_parallel(PGconn* conn, } if (!PQconsumeInput(conn)) - elog(ERROR, "query failed: %squery was: %s", + elog(ERROR, "query failed: %s query was: %s", PQerrorMessage(conn), query); /* query is no done */ From 25fd9280c1ebc0128279737b623b7f8446d13e21 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 14 Jul 2019 03:07:52 +0300 Subject: [PATCH 0839/2107] tests: minor fix --- tests/compression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compression.py b/tests/compression.py index ea288a5e1..19ea30757 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -269,8 +269,8 @@ def test_compression_stream_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - ptrack_result = node.execute("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=pglz']) From b41bc03be351a69211b696b977e947d8b73a0938 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 14 Jul 2019 13:46:28 +0300 Subject: [PATCH 0840/2107] tests: fixes for module "pgpro560" --- tests/pgpro560.py | 69 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 7ebb2131d..3aada4319 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -1,6 +1,6 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess @@ -30,19 +30,22 @@ def test_pgpro560_control_file_loss(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - file = os.path.join(node.base_dir,'data', 'global', 'pg_control') + file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') os.remove(file) try: self.backup_node(backup_dir, 'node', node, options=['--stream']) # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because pg_control was deleted.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because pg_control was deleted.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: could not open file' in e.message - and 'pg_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + 'ERROR: could not open file' in e.message and + 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -75,24 +78,50 @@ def test_pgpro560_systemid_mismatch(self): try: self.backup_node(backup_dir, 'node1', node2, options=['--stream']) # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of SYSTEM ID mismatch.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: Backup data directory was initialized for system id' in e.message - and 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + if self.get_version(node1) > 90600: + self.assertTrue( + 'ERROR: Backup data directory was ' + 'initialized for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'ERROR: System identifier mismatch. ' + 'Connected PostgreSQL instance has system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) try: - self.backup_node(backup_dir, 'node1', node2, data_dir=node1.data_dir, options=['--stream']) + self.backup_node( + backup_dir, 'node1', node2, + data_dir=node1.data_dir, options=['--stream']) # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of of SYSTEM ID mismatch.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + self.assertEqual( + 1, 0, + "Expecting Error because of of SYSTEM ID mismatch.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: Backup data directory was initialized for system id' in e.message - and 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + if self.get_version(node1) > 90600: + self.assertTrue( + 'ERROR: Backup data directory was initialized ' + 'for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'ERROR: System identifier mismatch. ' + 'Connected PostgreSQL instance has system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) From 9af4c7be9b1518e63554f6451f57a57bad2d66ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Jul 2019 15:05:27 +0300 Subject: [PATCH 0841/2107] Documentation: minor fixes --- Documentation.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index b9c26b2c2..2ceda0fa6 100644 --- a/Documentation.md +++ b/Documentation.md @@ -209,7 +209,25 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup COMMIT; ``` -For PostgreSQL >= 9.6: +For PostgreSQL 9.6: +``` +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +COMMIT; +``` + +For PostgreSQL >= 10: ``` BEGIN; CREATE ROLE backup WITH LOGIN; @@ -220,6 +238,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; @@ -307,10 +326,10 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; If you are going to use PTRACK backups, complete the following additional steps: - Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute ptrack functions to the *backup* role: +- Grant the rights to execute `ptrack` functions to the *backup* role: - GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; - GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup; + GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; + GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; - The *backup* role must have access to all the databases of the cluster. ## Command-Line Reference From a466eea901719b48c9dc00a30976ea8e7154af72 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Jul 2019 15:11:04 +0300 Subject: [PATCH 0842/2107] minor bugfix: in asynchronous execution of pg_stop_backup() was small risk of ignoring server side error. Reported by Alex Ignatov alexign@ozon.ru --- src/backup.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 96723f336..42408d353 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1689,7 +1689,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) while (1) { - if (!PQconsumeInput(conn) || PQisBusy(conn)) + if (!PQconsumeInput(conn)) + elog(ERROR, "pg_stop backup() failed: %s", + PQerrorMessage(conn)); + + if (PQisBusy(conn)) { pg_stop_backup_timeout++; sleep(1); @@ -1755,6 +1759,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) char *xlog_path, stream_xlog_path[MAXPGPATH]; + elog(WARNING, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + if (stream_wal) { pgBackupGetPath2(backup, stream_xlog_path, @@ -1885,10 +1892,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) char *xlog_path, stream_xlog_path[MAXPGPATH]; - /* Wait for stop_lsn to be received by replica */ - /* XXX Do we need this? */ -// if (current.from_replica) -// wait_replica_wal_lsn(stop_backup_lsn, false); /* * Wait for stop_lsn to be archived or streamed. * We wait for stop_lsn in stream mode just in case. From 0515c2b4dd0387acbab698cd748c6517e76891d4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Jul 2019 19:20:39 +0300 Subject: [PATCH 0843/2107] minor improvement: NodeInfo is now separate variable and not the element of pgBackup structure --- src/backup.c | 247 ++++++++++++++++++--------------------------- src/catalog.c | 16 +++ src/checkdb.c | 3 + src/pg_probackup.c | 9 +- src/pg_probackup.h | 10 +- src/show.c | 2 +- 6 files changed, 128 insertions(+), 159 deletions(-) diff --git a/src/backup.c b/src/backup.c index 42408d353..e71e6426d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -78,10 +78,6 @@ static int is_ptrack_enable = false; bool is_ptrack_support = false; bool exclusive_backup = false; -/* PostgreSQL server version from "backup_conn" */ -static int server_version = 0; -static char server_version_str[100] = ""; - /* Is pg_start_backup() was executed */ static bool backup_in_progress = false; /* Is pg_stop_backup() was sent */ @@ -94,18 +90,17 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn); +static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGconn *backup_conn, PGconn *master_conn); + PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn); static void pg_switch_wal(PGconn *conn); -static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn); +static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static int checkpoint_timeout(PGconn *backup_conn); //static void backup_list_file(parray *files, const char *root, ) static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment); -static void wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup, PGconn *backup_conn); static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn); static void *StreamLog(void *arg); @@ -128,7 +123,8 @@ static XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn); /* Check functions */ static bool pg_checksum_enable(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); -static void check_server_version(PGconn *conn); +static bool pg_is_superuser(PGconn *conn); +static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); @@ -142,7 +138,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) if (backup_in_progress) { elog(WARNING, "backup in progress, stop backup"); - pg_stop_backup(NULL, pg_startbackup_conn); /* don't care stop_lsn on error case */ + pg_stop_backup(NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */ } } @@ -151,7 +147,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) * Move files from 'pgdata' to a subdirectory in 'backup_path'. */ static void -do_backup_instance(PGconn *backup_conn) +do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) { int i; char database_path[MAXPGPATH]; @@ -255,8 +251,7 @@ do_backup_instance(PGconn *backup_conn) else pg_startbackup_conn = backup_conn; - pg_start_backup(label, smooth_checkpoint, ¤t, - backup_conn, pg_startbackup_conn); + pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn, pg_startbackup_conn); /* For incremental backup check that start_lsn is not from the past */ if (current.backup_mode != BACKUP_MODE_FULL && @@ -517,7 +512,7 @@ do_backup_instance(PGconn *backup_conn) } /* Notify end of backup */ - pg_stop_backup(¤t, pg_startbackup_conn); + pg_stop_backup(¤t, pg_startbackup_conn, nodeInfo); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -636,11 +631,12 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) confirm_block_size(cur_conn, "wal_block_size", XLOG_BLCKSZ); nodeInfo->block_size = BLCKSZ; nodeInfo->wal_block_size = XLOG_BLCKSZ; + nodeInfo->is_superuser = pg_is_superuser(cur_conn); current.from_replica = pg_is_in_recovery(cur_conn); /* Confirm that this server version is supported */ - check_server_version(cur_conn); + check_server_version(cur_conn, nodeInfo); if (pg_checksum_enable(cur_conn)) current.checksum_version = 1; @@ -657,11 +653,12 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) "pg_probackup have no way to detect data block corruption without them. " "Reinitialize PGDATA with option '--data-checksums'."); - StrNCpy(current.server_version, server_version_str, - sizeof(current.server_version)); + if (nodeInfo->is_superuser) + elog(WARNING, "Current PostgreSQL role is superuser. " + "It is not recommended to run backup or checkdb as superuser."); - StrNCpy(nodeInfo->server_version, server_version_str, - sizeof(nodeInfo->server_version)); + StrNCpy(current.server_version, nodeInfo->server_version_str, + sizeof(current.server_version)); return cur_conn; } @@ -673,16 +670,24 @@ int do_backup(time_t start_time, bool no_validate) { PGconn *backup_conn = NULL; + PGNodeInfo nodeInfo; + char pretty_data_bytes[20]; + + /* Initialize PGInfonode */ + pgNodeInit(&nodeInfo); if (!instance_config.pgdata) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); + + current.compress_alg = instance_config.compress_alg; + current.compress_level = instance_config.compress_level; + /* * setup backup_conn, do some compatibility checks and * fill basic info about instance */ - backup_conn = pgdata_basic_setup(instance_config.conn_opt, - &(current.nodeInfo)); + backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); /* * Ensure that backup directory was initialized for the same PostgreSQL * instance we opened connection to. And that target backup database PGDATA @@ -690,17 +695,19 @@ do_backup(time_t start_time, bool no_validate) */ check_system_identifiers(backup_conn, instance_config.pgdata); + elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " + "wal-method: %s, remote: %s, replica: %s, compress-algorithm: %s, compress-level: %i", + PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), + current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", + current.from_replica ? "true" : "false", deparse_compress_alg(current.compress_alg), + current.compress_level); + /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) elog(ERROR, "Failed to retreive wal_segment_size"); #endif - current.compress_alg = instance_config.compress_alg; - current.compress_level = instance_config.compress_level; - - current.stream = stream_wal; - is_ptrack_support = pg_ptrack_support(backup_conn); if (is_ptrack_support) { @@ -750,7 +757,7 @@ do_backup(time_t start_time, bool no_validate) pgut_atexit_push(backup_cleanup, NULL); /* backup data */ - do_backup_instance(backup_conn); + do_backup_instance(backup_conn, &nodeInfo); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -766,12 +773,13 @@ do_backup(time_t start_time, bool no_validate) current.status = BACKUP_STATUS_DONE; write_backup(¤t); - //elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "", - // current.data_bytes); - if (!no_validate) pgBackupValidate(¤t); + /* Notify user about backup size */ + pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); + elog(INFO, "Backup %s real size: %s", base36enc(current.start_time), pretty_data_bytes); + if (current.status == BACKUP_STATUS_OK || current.status == BACKUP_STATUS_DONE) elog(INFO, "Backup %s completed", base36enc(current.start_time)); @@ -792,33 +800,33 @@ do_backup(time_t start_time, bool no_validate) * Confirm that this server version is supported */ static void -check_server_version(PGconn *conn) +check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) { PGresult *res; /* confirm server version */ - server_version = PQserverVersion(conn); + nodeInfo->server_version = PQserverVersion(conn); - if (server_version == 0) - elog(ERROR, "Unknown server version %d", server_version); + if (nodeInfo->server_version == 0) + elog(ERROR, "Unknown server version %d", nodeInfo->server_version); - if (server_version < 100000) - sprintf(server_version_str, "%d.%d", - server_version / 10000, - (server_version / 100) % 100); + if (nodeInfo->server_version < 100000) + sprintf(nodeInfo->server_version_str, "%d.%d", + nodeInfo->server_version / 10000, + (nodeInfo->server_version / 100) % 100); else - sprintf(server_version_str, "%d", - server_version / 10000); + sprintf(nodeInfo->server_version_str, "%d", + nodeInfo->server_version / 10000); - if (server_version < 90500) + if (nodeInfo->server_version < 90500) elog(ERROR, "server version is %s, must be %s or higher", - server_version_str, "9.5"); + nodeInfo->server_version_str, "9.5"); - if (current.from_replica && server_version < 90600) + if (current.from_replica && nodeInfo->server_version < 90600) elog(ERROR, "server version is %s, must be %s or higher for backup from replica", - server_version_str, "9.6"); + nodeInfo->server_version_str, "9.6"); res = pgut_execute_extended(conn, "SELECT pgpro_edition()", 0, NULL, true, true); @@ -832,29 +840,29 @@ check_server_version(PGconn *conn) /* It seems we connected to PostgreSQL (not Postgres Pro) */ elog(ERROR, "%s was built with Postgres Pro %s %s, " "but connection is made with PostgreSQL %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, server_version_str); - else if (strcmp(server_version_str, PG_MAJORVERSION) != 0 && + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) elog(ERROR, "%s was built with Postgres Pro %s %s, " "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, - server_version_str, PQgetvalue(res, 0, 0)); + nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); #else if (PQresultStatus(res) != PGRES_FATAL_ERROR) /* It seems we connected to Postgres Pro (not PostgreSQL) */ elog(ERROR, "%s was built with PostgreSQL %s, " "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, - server_version_str, PQgetvalue(res, 0, 0)); - else if (strcmp(server_version_str, PG_MAJORVERSION) != 0) + nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); + else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", - PROGRAM_NAME, PG_MAJORVERSION, server_version_str); + PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str); #endif PQclear(res); /* Do exclusive backup only for PostgreSQL 9.5 */ - exclusive_backup = server_version < 90600 || + exclusive_backup = nodeInfo->server_version < 90600 || current.backup_mode == BACKUP_MODE_DIFF_PTRACK; } @@ -889,6 +897,7 @@ check_system_identifiers(PGconn *conn, char *pgdata) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but connected instance system id is " UINT64_FORMAT, instance_config.system_identifier, system_id_conn); + if (system_id_pgdata != instance_config.system_identifier) elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " "but target backup directory system id is " UINT64_FORMAT, @@ -924,7 +933,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) */ static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGconn *backup_conn, PGconn *pg_startbackup_conn) + PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *pg_startbackup_conn) { PGresult *res; const char *params[2]; @@ -965,11 +974,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, PQclear(res); if (current.backup_mode == BACKUP_MODE_DIFF_PAGE && - (!(backup->from_replica && !exclusive_backup))) + !backup->from_replica && + !(nodeInfo->server_version < 90600 && + !nodeInfo->is_superuser)) /* * Switch to a new WAL segment. It is necessary to get archived WAL * segment, which includes start LSN of current backup. - * Don`t do this for replica backups unless it`s PG 9.5 + * Don`t do this for replica backups and for PG 9.5 if pguser is not superuser + * (because in 9.5 only superuser can switch WAL) */ pg_switch_wal(conn); @@ -984,16 +996,11 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, else if (!stream_wal) /* ...for others wait for previous segment */ wait_wal_lsn(backup->start_lsn, true, true); - - /* In case of backup from replica for PostgreSQL 9.5 - * wait for start_lsn to be replayed by replica - */ - if (backup->from_replica && exclusive_backup) - wait_replica_wal_lsn(backup->start_lsn, true, backup_conn); } /* * Switch to a new WAL segment. It should be called only for master. + * For PG 9.5 it should be called only if pguser is superuser. */ static void pg_switch_wal(PGconn *conn) @@ -1005,9 +1012,9 @@ pg_switch_wal(PGconn *conn) PQclear(res); #if PG_VERSION_NUM >= 100000 - res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_wal()", 0, NULL); + res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL); #else - res = pgut_execute(conn, "SELECT * FROM pg_catalog.pg_switch_xlog()", 0, NULL); + res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_xlog()", 0, NULL); #endif PQclear(res); @@ -1080,13 +1087,13 @@ pg_checksum_enable(PGconn *conn) res_db = pgut_execute(conn, "SHOW data_checksums", 0, NULL); - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + if (strcmp(PQgetvalue(res_db, 0, 0), "on") == 0) { PQclear(res_db); - return false; + return true; } PQclear(res_db); - return true; + return false; } /* Check if target instance is replica */ @@ -1106,6 +1113,24 @@ pg_is_in_recovery(PGconn *conn) return false; } + +/* Check if current PostgreSQL role is superuser */ +static bool +pg_is_superuser(PGconn *conn) +{ + PGresult *res; + + res = pgut_execute(conn, "SELECT pg_catalog.current_setting('is_superuser')", 0, NULL); + + if (strcmp(PQgetvalue(res, 0, 0), "on") == 0) + { + PQclear(res); + return true; + } + PQclear(res); + return false; +} + /* Clear ptrack files in all databases of the instance we connected to */ static void pg_ptrack_clear(PGconn *backup_conn) @@ -1479,80 +1504,12 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) } } -/* - * Wait for target 'lsn' on replica instance from master. - */ -static void -wait_replica_wal_lsn(XLogRecPtr lsn, bool is_start_backup, - PGconn *backup_conn) -{ - uint32 try_count = 0; - - while (true) - { - XLogRecPtr replica_lsn; - - /* - * For lsn from pg_start_backup() we need it to be replayed on replica's - * data. - */ - if (is_start_backup) - { - replica_lsn = get_checkpoint_location(backup_conn); - } - /* - * For lsn from pg_stop_backup() we need it only to be received by - * replica and fsync()'ed on WAL segment. - */ - else - { - PGresult *res; - uint32 lsn_hi; - uint32 lsn_lo; - -#if PG_VERSION_NUM >= 100000 - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_wal_receive_lsn()", - 0, NULL); -#else - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_last_xlog_receive_location()", - 0, NULL); -#endif - - /* Extract LSN from result */ - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - replica_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - PQclear(res); - } - - /* target lsn was replicated */ - if (replica_lsn >= lsn) - break; - - sleep(1); - if (interrupted) - elog(ERROR, "Interrupted during waiting for target LSN"); - try_count++; - - /* Inform user if target lsn is absent in first attempt */ - if (try_count == 1) - elog(INFO, "Wait for target LSN %X/%X to be received by replica", - (uint32) (lsn >> 32), (uint32) lsn); - - if (instance_config.replica_timeout > 0 && - try_count > instance_config.replica_timeout) - elog(ERROR, "Target LSN %X/%X could not be recevied by replica " - "in %d seconds", - (uint32) (lsn >> 32), (uint32) lsn, - instance_config.replica_timeout); - } -} - /* * Notify end of backup to PostgreSQL server. */ static void -pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) +pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, + PGNodeInfo *nodeInfo) { PGconn *conn; PGresult *res; @@ -1588,27 +1545,23 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn) PQclear(res); /* Create restore point - * only if it`s backup from master, or exclusive replica(wich connects to master) + * Only if it is backup from master. + * If PG 9.5, then create restore point only if + * pguser is superuser. */ - if (backup != NULL && (!current.from_replica || (current.from_replica && exclusive_backup))) + if (backup != NULL && !current.from_replica && + !(nodeInfo->server_version < 90600 && + !nodeInfo->is_superuser)) { const char *params[1]; char name[1024]; - if (!current.from_replica) - snprintf(name, lengthof(name), "pg_probackup, backup_id %s", - base36enc(backup->start_time)); - else - snprintf(name, lengthof(name), "pg_probackup, backup_id %s. Replica Backup", - base36enc(backup->start_time)); + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup->start_time)); params[0] = name; res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", 1, params); - /* Extract timeline and LSN from the result */ - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - //restore_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; PQclear(res); } diff --git a/src/catalog.c b/src/catalog.c index 9ad797c78..34e35793d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1053,6 +1053,22 @@ deparse_compress_alg(int alg) return NULL; } +/* + * Fill PGNodeInfo struct with default values. + */ +void +pgNodeInit(PGNodeInfo *node) +{ + node->block_size = 0; + node->wal_block_size = 0; + node->checksum_version = 0; + + node->is_superuser = false; + + node->server_version = 0; + node->server_version_str[0] = '\0'; +} + /* * Fill pgBackup struct with default values. */ diff --git a/src/checkdb.c b/src/checkdb.c index c19d84ae9..dd1145f47 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -690,6 +690,9 @@ do_checkdb(bool need_amcheck, PGNodeInfo nodeInfo; PGconn *cur_conn; + /* Initialize PGInfonode */ + pgNodeInit(&nodeInfo); + if (skip_block_validation && !need_amcheck) elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fbfe4a108..49b4363b0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -612,17 +612,10 @@ main(int argc, char *argv[]) return do_init(); case BACKUP_CMD: { - const char *backup_mode; - time_t start_time; + time_t start_time = time(NULL); - start_time = time(NULL); - backup_mode = deparse_backup_mode(current.backup_mode); current.stream = stream_wal; - elog(INFO, "Backup start, pg_probackup version: %s, backup ID: %s, backup mode: %s, instance: %s, stream: %s, remote %s", - PROGRAM_VERSION, base36enc(start_time), backup_mode, instance_name, - stream_wal ? "true" : "false", IsSshProtocol() ? "true" : "false"); - /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) elog(ERROR, "required parameter not specified: BACKUP_MODE " diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a90508f75..2b7a3f3a8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -248,9 +248,11 @@ typedef struct PGNodeInfo uint32 block_size; uint32 wal_block_size; uint32 checksum_version; + bool is_superuser; + + int server_version; + char server_version_str[100]; - char program_version[100]; - char server_version[100]; } PGNodeInfo; typedef struct pgBackup pgBackup; @@ -290,7 +292,6 @@ struct pgBackup int compress_level; /* Fields needed for compatibility check */ - PGNodeInfo nodeInfo; uint32 block_size; uint32 wal_block_size; uint32 checksum_version; @@ -564,6 +565,7 @@ extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); +extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); extern int pgBackupCompareId(const void *f1, const void *f2); @@ -697,6 +699,8 @@ extern bool parse_page(Page page, XLogRecPtr *lsn); int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); +extern void pretty_size(int64 size, char *buf, size_t len); + extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); extern void check_system_identifiers(PGconn *conn, char *pgdata); diff --git a/src/show.c b/src/show.c index 681848f2a..4ba95ebc8 100644 --- a/src/show.c +++ b/src/show.c @@ -118,7 +118,7 @@ do_show(time_t requested_backup_id) return show_backup(requested_backup_id); } -static void +void pretty_size(int64 size, char *buf, size_t len) { int exp = 0; From 69eed7acb5e9899b31eb9b68f2f940a2cc677de9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Jul 2019 19:24:31 +0300 Subject: [PATCH 0844/2107] tests: minor fixes for "archive" module --- tests/archive.py | 79 ++++++++++++++++++++++++++++++++- tests/backup.py | 55 +++++++++++++++++++---- tests/helpers/ptrack_helpers.py | 2 + tests/replica.py | 3 ++ 4 files changed, 129 insertions(+), 10 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 1ce602210..49ef5553a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -240,7 +240,6 @@ def test_pgpro434_3(self): backup_dir, 'node', node, options=[ "--archive-timeout=60", - "--stream", "--log-level-file=info"], gdb=True) @@ -253,7 +252,83 @@ def test_pgpro434_3(self): gdb.continue_execution_until_exit() - log_file = os.path.join(backup_dir, 'log/pg_probackup.log') + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + "ERROR: Switched WAL segment 000000010000000000000002 " + "could not be archived in 60 seconds", + log_content) + + self.assertIn( + "ERROR: Switched WAL segment 000000010000000000000002 " + "could not be archived in 60 seconds", + log_content) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pgpro434_4(self): + """ + Check pg_stop_backup_timeout, needed backup_timeout + Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + gdb = self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=info"], + gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + node.append_conf( + 'postgresql.auto.conf', "archive_command = 'exit 1'") + node.reload() + + os.environ["PGAPPNAME"] = "foo" + + pid = node.safe_psql( + "postgres", + "SELECT pid " + "FROM pg_stat_activity " + "WHERE application_name = 'pg_probackup'").rstrip() + + os.environ["PGAPPNAME"] = "pg_probackup" + + postgres_gdb = self.gdb_attach(pid) + postgres_gdb.set_breakpoint('do_pg_stop_backup') + postgres_gdb.continue_execution_until_running() + + gdb.continue_execution_until_exit() + # gdb._execute('detach') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') with open(log_file, 'r') as f: log_content = f.read() diff --git a/tests/backup.py b/tests/backup.py index d18e2d978..b55b18989 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1528,6 +1528,7 @@ def test_backup_with_least_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') + # PG 9.5 if self.get_version(node) < 90600: node.safe_psql( 'backupdb', @@ -1559,6 +1560,43 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 else: node.safe_psql( 'backupdb', @@ -1595,17 +1633,18 @@ def test_backup_with_least_privileges_role(self): if self.ptrack: for fname in [ - 'oideq(oid, oid)', - 'ptrack_version()', - 'pg_ptrack_clear()', - 'pg_ptrack_control_lsn()', - 'pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_ptrack_get_and_clear(oid, oid)', - 'pg_ptrack_get_block_2(oid, oid, oid, bigint)']: + 'pg_catalog.oideq(oid, oid)', + 'pg_catalog.ptrack_version()', + 'pg_catalog.pg_ptrack_clear()', + 'pg_catalog.pg_ptrack_control_lsn()', + 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', + 'pg_catalog.pg_stop_backup()']: # try: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.{0} " + "GRANT EXECUTE ON FUNCTION {0} " "TO backup".format(fname)) # except: # pass diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 52edc894d..d3337de55 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -279,6 +279,8 @@ def __init__(self, *args, **kwargs): if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': self.ptrack = True + os.environ["PGAPPNAME"] = "pg_probackup" + @property def pg_config_version(self): return self.version_to_num( diff --git a/tests/replica.py b/tests/replica.py index 92a41a6af..1d38d2659 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -18,6 +18,9 @@ def test_replica_stream_ptrack_backup(self): make node, take full backup, restore it and make replica from it, take full stream backup from replica """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( From 14514a4815ba69aa160c27e58b13403ccba914d2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Jul 2019 19:39:12 +0300 Subject: [PATCH 0845/2107] minor improvement: be more paranoid about system ID mismatch when streaming or reading WAL files. Replace RunIdentifySystem() with IdentifySystem() for logging purposes --- src/backup.c | 98 ++++++++++++++++++++++++++++++++++-------------- src/parsexlog.c | 14 +++++++ src/utils/pgut.c | 3 ++ 3 files changed, 86 insertions(+), 29 deletions(-) diff --git a/src/backup.c b/src/backup.c index e71e6426d..fc6505428 100644 --- a/src/backup.c +++ b/src/backup.c @@ -103,6 +103,7 @@ static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment); static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn); static void *StreamLog(void *arg); +static void IdentifySystem(StreamThreadArg *stream_thread_arg); static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); @@ -289,30 +290,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) instance_config.conn_opt.pgport, instance_config.conn_opt.pgdatabase, instance_config.conn_opt.pguser); + /* sanity */ + IdentifySystem(&stream_thread_arg); - if (!CheckServerVersionForStreaming(stream_thread_arg.conn)) - { - PQfinish(stream_thread_arg.conn); - /* - * Error message already written in CheckServerVersionForStreaming(). - * There's no hope of recovering from a version mismatch, so don't - * retry. - */ - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - /* - * Identify server, obtaining start LSN position and current timeline ID - * at the same time, necessary if not valid data can be found in the - * existing output directory. - */ - if (!RunIdentifySystem(stream_thread_arg.conn, NULL, NULL, NULL, NULL)) - { - PQfinish(stream_thread_arg.conn); - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - /* By default there are some error */ + /* By default there are some error */ stream_thread_arg.ret = 1; /* we must use startpos as start_lsn from start_backup */ stream_thread_arg.startpos = current.start_lsn; @@ -522,7 +503,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) char pg_control_path[MAXPGPATH]; snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", - instance_config.pgdata, "global/pg_control"); + instance_config.pgdata, XLOG_CONTROL_FILE); for (i = 0; i < parray_num(backup_files_list); i++) { @@ -2529,7 +2510,7 @@ StreamLog(void *arg) /* * Start the replication */ - elog(LOG, _("started streaming WAL at %X/%X (timeline %u)"), + elog(LOG, "started streaming WAL at %X/%X (timeline %u)", (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, stream_arg->starttli); @@ -2570,13 +2551,13 @@ StreamLog(void *arg) #endif } #else - if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, NULL, - (char *) stream_arg->basedir, stop_streaming, - standby_message_timeout, NULL, false, false) == false) + if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, + NULL, (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) elog(ERROR, "Problem in receivexlog"); #endif - elog(LOG, _("finished streaming WAL at %X/%X (timeline %u)"), + elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); stream_arg->ret = 0; @@ -2744,3 +2725,62 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) } } } + +/* + * Run IDENTIFY_SYSTEM through a given connection and + * check system identifier and timeline are matching + */ +void +IdentifySystem(StreamThreadArg *stream_thread_arg) +{ + PGresult *res; + + uint64 stream_conn_sysidentifier = 0; + char *stream_conn_sysidentifier_str; + TimeLineID stream_conn_tli = 0; + + if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) + { + PQfinish(stream_thread_arg->conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtain server system identifier and timeline + */ + res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING,"Could not send replication command \"%s\": %s", + "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); + PQfinish(stream_thread_arg->conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); + stream_conn_tli = atoi(PQgetvalue(res, 0, 1)); + + /* Additional sanity, primary for PG 9.5, + * where system id can be obtained only via "IDENTIFY SYSTEM" + */ + if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) + elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); + + if (stream_conn_sysidentifier != instance_config.system_identifier) + elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " + "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", + stream_conn_sysidentifier, instance_config.system_identifier); + + if (stream_conn_tli != current.tli) + elog(ERROR, "Timeline identifier mismatch. " + "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", + stream_conn_tli, current.tli); + + PQclear(res); +} diff --git a/src/parsexlog.c b/src/parsexlog.c index bd8585f5a..0e2ae588c 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -510,9 +510,18 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli, wal_seg_size, false, false, true); + if (xlogreader == NULL) + elog(ERROR, "Out of memory"); + + xlogreader->system_identifier = instance_config.system_identifier; + res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ + if (errormsg) + elog(WARNING, "Could not read WAL record at %X/%X: %s", + (uint32) (target_lsn >> 32), (uint32) (target_lsn), errormsg); + CleanupXLogPageRead(xlogreader); XLogReaderFree(xlogreader); @@ -551,6 +560,11 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, false, false, true); + if (xlogreader == NULL) + elog(ERROR, "Out of memory"); + + xlogreader->system_identifier = instance_config.system_identifier; + /* * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 7ef828ec1..6cb685c7f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -206,6 +206,9 @@ pgut_get_conninfo_string(PGconn *conn) return connstr; } +/* TODO: it is better to use PQconnectdbParams like in psql + * It will allow to set application_name for pg_probackup + */ PGconn * pgut_connect(const char *host, const char *port, const char *dbname, const char *username) From fc385aae7403c49507108c447ef68367e399f758 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 16 Jul 2019 00:56:26 +0300 Subject: [PATCH 0846/2107] Documentation and Readme update about versioning --- Documentation.md | 7 ++++++- README.md | 10 +++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Documentation.md b/Documentation.md index 2ceda0fa6..6f77eeec6 100644 --- a/Documentation.md +++ b/Documentation.md @@ -6,6 +6,7 @@ Current version - 2.1.3 1. [Synopsis](#synopsis) 2. [Overview](#overview) + * [Versioning](#versioning) * [Limitations](#limitations) 3. [Installation and Setup](#installation-and-setup) @@ -134,12 +135,16 @@ pg_probackup can take only physical online backups, and online backups require W - ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. - STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. +### Versioning + +pg_probackup is following [semantic](https://semver.org/) versioning. + ### Limitations pg_probackup currently has the following limitations: - Creating backups from a remote server is currently not supported on Windows systems. -- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. +- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. ## Installation and Setup diff --git a/README.md b/README.md index 45eb4ef6a..d888c11fb 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * Remote mode is in beta stage. * Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. +## Current release + +[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) + ## Installation and Setup ### Windows Installation [Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L) @@ -81,7 +85,7 @@ yum install pg_probackup-{11,10,9.6,9.5}-debuginfo yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` -Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup). +Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). ## Building from source ### Linux @@ -104,10 +108,6 @@ SET PATH=%PATH%;C:\msys64\usr\bin gen_probackup_project.pl C:\path_to_postgresql_source_tree ``` -## Current release - -[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) - ## Documentation Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). From 8fb9f94c6d21cc96ae2fa72bd46d11d0cc4e769c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 16 Jul 2019 01:18:00 +0300 Subject: [PATCH 0847/2107] tests: fix for backup.BackupTest.test_backup_detect_corruption --- tests/backup.py | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index b55b18989..ec8288fd9 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -469,12 +469,16 @@ def test_backup_detect_corruption(self): "postgres", "select pg_relation_filepath('t_heap')").rstrip() + node.stop() + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: f.seek(9000) f.write(b"bla") f.flush() f.close + node.slow_start() + try: self.backup_node( backup_dir, 'node', node, @@ -486,23 +490,33 @@ def test_backup_detect_corruption(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - if self.remote: + if self.ptrack: self.assertTrue( - "ERROR: Failed to read file" in e.message and - "data file checksum mismatch" in e.message, + 'WARNING: page verification failed, ' + 'calculated checksum' in e.message and + 'ERROR: query failed: ERROR: ' + 'invalid page in block 1 of relation' in e.message and + 'ERROR: Data files transferring failed' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) else: - self.assertIn( - 'WARNING: Corruption detected in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Data file corruption', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.remote: + self.assertTrue( + "ERROR: Failed to read file" in e.message and + "data file checksum mismatch" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'WARNING: Corruption detected in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Data file corruption', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) From 221b7759c0926724dc02b969e1ae4e0436089dbb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jul 2019 15:42:47 +0300 Subject: [PATCH 0848/2107] minor improvement: alwys open ssh connection with option 'PasswordAuthentication=no' --- src/utils/remote.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index a9482f363..8a9a86d2b 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -126,11 +126,9 @@ bool launch_agent(void) if (instance_config.remote.ssh_options != NULL) { ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options)); } - if (num_threads > 1) - { - ssh_argv[ssh_argc++] = "-o"; - ssh_argv[ssh_argc++] = "PasswordAuthentication=no"; - } + + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "PasswordAuthentication=no"; ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "Compression=no"; From 884da1cec74100eacf2b0ec2cc7fbdd9875c24e0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jul 2019 15:44:20 +0300 Subject: [PATCH 0849/2107] comments minor changes --- src/backup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index fc6505428..c935b7566 100644 --- a/src/backup.c +++ b/src/backup.c @@ -809,6 +809,7 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) "server version is %s, must be %s or higher for backup from replica", nodeInfo->server_version_str, "9.6"); + /* TODO: search pg_proc for pgpro_edition before calling */ res = pgut_execute_extended(conn, "SELECT pgpro_edition()", 0, NULL, true, true); @@ -1526,9 +1527,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PQclear(res); /* Create restore point - * Only if it is backup from master. - * If PG 9.5, then create restore point only if - * pguser is superuser. + * Only if backup is from master. + * For PG 9.5 create restore point only if pguser is superuser. */ if (backup != NULL && !current.from_replica && !(nodeInfo->server_version < 90600 && From 833f31de7d29fe3e0a7858b1370e828f59c268a3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jul 2019 15:57:58 +0300 Subject: [PATCH 0850/2107] Documentation: update --- Documentation.md | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6f77eeec6..ac8fed087 100644 --- a/Documentation.md +++ b/Documentation.md @@ -5,11 +5,13 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database c Current version - 2.1.3 1. [Synopsis](#synopsis) -2. [Overview](#overview) - * [Versioning](#versioning) +2. [Versioning](#versioning) +3. [Overview](#overview) + * [Backup modes](#backup-modes) + * [WAL methods](#wal-methods) * [Limitations](#limitations) -3. [Installation and Setup](#installation-and-setup) +4. [Installation and Setup](#installation-and-setup) * [Initializing the Backup Catalog](#initializing-the-backup-catalog) * [Adding a New Backup Instance](#adding-a-new-backup-instance) * [Configuring the Database Cluster](#configuring-the-database-cluster) @@ -19,7 +21,7 @@ Current version - 2.1.3 * [Setting up Cluster Verification](#setting-up-cluster-verification) * [Setting up PTRACK Backups](#setting-up-ptrack-backups) -4. [Command-Line Reference](#command-line-reference) +5. [Command-Line Reference](#command-line-reference) * [Commands](#commands) * [version](#version) * [help](#help) @@ -52,7 +54,7 @@ Current version - 2.1.3 * [Remote Mode Options](#remote-mode-options) * [Replica Options](#replica-options) -5. [Usage](#usage) +6. [Usage](#usage) * [Creating a Backup](#creating-a-backup) * [Verifying a Cluster](#verifying-a-cluster) * [Validating a Backup](#validating-a-backup) @@ -66,8 +68,8 @@ Current version - 2.1.3 * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) -6. [Authors](#authors) -7. [Credits](#credits) +7. [Authors](#authors) +8. [Credits](#credits) ## Synopsis @@ -104,6 +106,11 @@ Current version - 2.1.3 `pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f` + +## Versioning + +pg_probackup is following [semantic](https://semver.org/) versioning. + ## Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: @@ -122,23 +129,23 @@ As compared to other backup solutions, pg_probackup offers the following benefit To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. +### Backup Modes + Using pg_probackup, you can take full or incremental backups: - FULL backups contain all the data files required to restore the database cluster. - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. + +### WAL methods -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen `backup mode` (FULL, PAGE, DELTA, etc), all backups taken with pg_probackup must use one of the following `WAL delivery methods`: +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen [backup mode](#backup-modes) (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery methods`: - ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. - STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. -### Versioning - -pg_probackup is following [semantic](https://semver.org/) versioning. - ### Limitations pg_probackup currently has the following limitations: @@ -284,11 +291,13 @@ ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/do archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' -Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. +Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero data loss` archive strategy can be achieved only by using pg_receivewal. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. + +>NOTE: using pg_probackup command [archive-push](#archive-push) for continious archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be gzip and '.gz' suffix is mandatory. ### Backup from Standby @@ -1015,13 +1024,15 @@ If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the ba pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. +Do note that pg_probackup rely on passwordless SSH connection for communication between the hosts. + The typical workflow is as follows: - - On your local system, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the remote server with the PostgreSQL instance. + - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. -- If you would like to take ARCHIVE backups, configure continuous WAL archiving on the remote system as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to host with backup catalog. +- If you would like to take remote backup in [PAGE](#backup-modes) mode, or rely on [ARCHIVE](#wal-methods) WAL delivery method, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. -- Run [backup](#backup) or [restore](#restore) commands with remote options on system with backup catalog. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) on backup host. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. >NOTE: The remote backup mode is currently unavailable for Windows systems. From 837534f1787beb11441ba9b572beb90c5ec02b20 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jul 2019 18:20:48 +0300 Subject: [PATCH 0851/2107] Documentation: added "ARCHIVE mode" section --- Documentation.md | 51 +++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/Documentation.md b/Documentation.md index ac8fed087..28d63a7bb 100644 --- a/Documentation.md +++ b/Documentation.md @@ -7,8 +7,6 @@ Current version - 2.1.3 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) 3. [Overview](#overview) - * [Backup modes](#backup-modes) - * [WAL methods](#wal-methods) * [Limitations](#limitations) 4. [Installation and Setup](#installation-and-setup) @@ -129,9 +127,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. -### Backup Modes - -Using pg_probackup, you can take full or incremental backups: +Using pg_probackup, you can take full or incremental [backups](#creating-a-backup): - FULL backups contain all the data files required to restore the database cluster. - Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: @@ -139,12 +135,10 @@ Using pg_probackup, you can take full or incremental backups: - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -### WAL methods - -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen [backup mode](#backup-modes) (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery methods`: +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen backup mode (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery methods`: -- ARCHIVE. Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. -- STREAM. Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. +- [ARCHIVE](#archive-mode). Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. +- [STREAM](#stream-mode). Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. ### Limitations @@ -889,24 +883,33 @@ Where *backup_mode* can take one of the following values: - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. - PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. -When restoring a cluster from an incremental backup, pg_probackup relies on the previous full backup to restore all the data files first. Thus, you must create at least one full backup before taking incremental ones. +When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `backup chain`. You must create at least one full backup before taking incremental ones. -#### Page validation -If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. -Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. +#### ARCHIVE mode -Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. +ARCHIVE is the default WAl delivery mode. -Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". +For example, to make a FULL backup in ARCHIVE mode, run: + + pg_probackup backup -B backup_dir --instance instance_name -b FULL + +ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. + +During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This costly precations eliminates the risk of silent WAL corruption. #### STREAM mode -To make a STREAM backup, add the `--stream` option to the above command. For example, to create a full STREAM backup, run: + +STREAM is the optional WAl delivery mode. + +For example, to make a FULL backup in STREAM mode, add the `--stream` option to the command from the previous example: pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot -The optional `--temp-slot` parameter ensures that the required segments remain available if the WAL is rotated before the backup is complete. +The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. + +STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. -STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. To restore a cluster from an incremental STREAM backup, pg_probackup still requires the full backup and all the incremental backups it depends on. +During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN in '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This costly precations eliminates the risk of silent WAL corruption. Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: @@ -914,6 +917,14 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin - STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. - Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. +#### Page validation +If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. +Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. + +Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. + +Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". + #### External directories To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. @@ -1030,7 +1041,7 @@ The typical workflow is as follows: - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. -- If you would like to take remote backup in [PAGE](#backup-modes) mode, or rely on [ARCHIVE](#wal-methods) WAL delivery method, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. +- If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery method, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. - Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) on backup host. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. From 8d58c9d700bd344db5448d53f8f2ddf9cf5e5374 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 10:52:18 +0300 Subject: [PATCH 0852/2107] Documentation: update --- Documentation.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 28d63a7bb..babd84c91 100644 --- a/Documentation.md +++ b/Documentation.md @@ -740,6 +740,8 @@ Maximum lifetime of an individual log file. If this value is reached, the log fi #### Connection Options You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. +All libpq environment variables are fully [supported](https://www.postgresql.org/docs/current/libpq-envars.html). + -d dbname --dbname=dbname PGDATABASE @@ -764,7 +766,7 @@ User name to connect as. -w --no-password - Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. + Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file or PGPASSWORD environment variable, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. -W --password @@ -895,7 +897,7 @@ For example, to make a FULL backup in ARCHIVE mode, run: ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. -During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This costly precations eliminates the risk of silent WAL corruption. +During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. #### STREAM mode @@ -909,7 +911,7 @@ The optional `--temp-slot` flag ensures that the required segments remain availa STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. -During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN in '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This costly precations eliminates the risk of silent WAL corruption. +During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN in '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: From 7f0ab09a9a809657af936aac2e1b88ab03c8465d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 10:53:36 +0300 Subject: [PATCH 0853/2107] Documentation: update --- Documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index babd84c91..56dec0fd4 100644 --- a/Documentation.md +++ b/Documentation.md @@ -740,7 +740,7 @@ Maximum lifetime of an individual log file. If this value is reached, the log fi #### Connection Options You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. -All libpq environment variables are fully [supported](https://www.postgresql.org/docs/current/libpq-envars.html). +All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. -d dbname --dbname=dbname From 8fa0bbc3697751253c4f772a4ae1ba5e3f07fb2e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 11:01:01 +0300 Subject: [PATCH 0854/2107] minor improvement of messages for restore --- src/restore.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index c8efcb396..c7c03e208 100644 --- a/src/restore.c +++ b/src/restore.c @@ -160,7 +160,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (!satisfy_recovery_target(current_backup, rt)) { if (target_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "target backup %s does not satisfy restore options", + elog(ERROR, "Requested backup %s does not satisfy restore options", base36enc(target_backup_id)); else /* Try to find another backup that satisfies target options */ @@ -175,8 +175,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } + /* TODO: Show latest possible target */ if (dest_backup == NULL) - elog(ERROR, "Backup satisfying target options is not found."); + { + /* Failed to find target backup */ + if (target_backup_id) + elog(ERROR, "Requested backup %s is not found.", base36enc(target_backup_id)); + else + elog(ERROR, "Backup satisfying target options is not found."); + /* TODO: check if user asked PITR or just restore of latest backup */ + } /* If we already found dest_backup, look for full backup. */ if (dest_backup->backup_mode == BACKUP_MODE_FULL) @@ -954,6 +962,7 @@ read_timeline_history(TimeLineID targetTLI) return result; } +/* TODO: do not ignore timelines. What if requested target located in different timeline? */ bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) { @@ -969,6 +978,7 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) return true; } +/* TODO description */ bool satisfy_timeline(const parray *timelines, const pgBackup *backup) { From 2a87440ca71b58ba779d939bf0ac2e69218b49d8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 12:18:41 +0300 Subject: [PATCH 0855/2107] Documentation: minor update --- Documentation.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 56dec0fd4..55c37edb4 100644 --- a/Documentation.md +++ b/Documentation.md @@ -317,7 +317,7 @@ Logical verification of database cluster requires the following additional setup CREATE EXTENSION amcheck; -- To perform logical verification the following rights are requiared: +- To perform logical verification the following permissions are requiared: ``` GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; @@ -503,7 +503,7 @@ Copied to archive WAL segments are synced to disk. You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). -For details, see section [Archiving Options](#archiving-options) +For details, see sections [Archiving Options](#archiving-options) and [Compression Options](#compression-options). #### archive-get @@ -792,6 +792,8 @@ Alias for `--compress-algorithm=zlib` and `--compress-level=1`. These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. +Additionally [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. + --wal-file-path=wal_file_path %p Provides the path to the WAL file in `archive_command` and `restore_command`. The %p variable is required for correct processing. From bb6d32fd33816be531b45152baa2bbb3efe29e30 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 12:20:39 +0300 Subject: [PATCH 0856/2107] Release 2.1.4 --- Documentation.md | 2 +- README.md | 2 +- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation.md b/Documentation.md index 55c37edb4..98e6b82a9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.1.3 +Current version - 2.1.4 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) diff --git a/README.md b/README.md index d888c11fb..f81ecf49b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) +[2.1.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.4) ## Installation and Setup ### Windows Installation diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2b7a3f3a8..90de4e7fa 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -188,8 +188,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.3" -#define AGENT_PROTOCOL_VERSION 20103 +#define PROGRAM_VERSION "2.1.4" +#define AGENT_PROTOCOL_VERSION 20104 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index ac690b66b..1a5c9cb70 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.3 \ No newline at end of file +pg_probackup 2.1.4 \ No newline at end of file From 028029e899d78c2f7b9f21f511b8cb48afe09771 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 12:39:31 +0300 Subject: [PATCH 0857/2107] Documentation: minor fixes --- Documentation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 98e6b82a9..37c33c9d6 100644 --- a/Documentation.md +++ b/Documentation.md @@ -280,7 +280,7 @@ Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. -- If you are configuring backups on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. +- If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' @@ -891,7 +891,7 @@ When restoring a cluster from an incremental backup, pg_probackup relies on the #### ARCHIVE mode -ARCHIVE is the default WAl delivery mode. +ARCHIVE is the default WAL delivery mode. For example, to make a FULL backup in ARCHIVE mode, run: @@ -903,7 +903,7 @@ During [backup](#backup) pg_probackup ensures that WAL files containing WAL reco #### STREAM mode -STREAM is the optional WAl delivery mode. +STREAM is the optional WAL delivery mode. For example, to make a FULL backup in STREAM mode, add the `--stream` option to the command from the previous example: From ab2d595a024e1bec5e9620e89a135602930f544b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jul 2019 20:05:30 +0300 Subject: [PATCH 0858/2107] tests: fixes --- tests/archive.py | 19 ++++++++++--------- tests/merge.py | 8 +++++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 49ef5553a..67a4429d0 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -256,15 +256,16 @@ def test_pgpro434_3(self): with open(log_file, 'r') as f: log_content = f.read() - self.assertIn( - "ERROR: Switched WAL segment 000000010000000000000002 " - "could not be archived in 60 seconds", - log_content) - - self.assertIn( - "ERROR: Switched WAL segment 000000010000000000000002 " - "could not be archived in 60 seconds", - log_content) + # in PG =< 9.6 pg_stop_backup always wait + if self.get_version(node) < 100000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: Switched WAL segment 000000010000000000000002 " + "could not be archived in 60 seconds", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: diff --git a/tests/merge.py b/tests/merge.py index e539f2816..ebd8b7feb 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1540,8 +1540,8 @@ def test_crash_after_opening_backup_control_2(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('print_file_list') - gdb.continue_execution_until_break() + gdb.set_breakpoint('fio_fwrite') + gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -1631,9 +1631,11 @@ def test_losing_file_after_failed_merge(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('print_file_list') + gdb.set_breakpoint('fio_fwrite') gdb.continue_execution_until_break() + gdb._execute('bt') + gdb._execute('signal SIGKILL') print(self.show_pb( From 2ec247d2cbfc4c3605d4c588ae7ea98d46f43fd2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Jul 2019 06:32:52 +0300 Subject: [PATCH 0859/2107] tests: minor fixes --- tests/__init__.py | 8 ++++---- tests/merge.py | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7cc96276d..e844d29b8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,6 +16,10 @@ def load_tests(loader, tests, pattern): if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON': loader.testMethodPrefix = 'test_basic' + if 'PG_PROBACKUP_PTRACK' in os.environ: + if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': + suite.addTests(loader.loadTestsFromModule(ptrack)) + # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) @@ -49,10 +53,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(validate)) - if 'PG_PROBACKUP_PTRACK' in os.environ: - if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': - suite.addTests(loader.loadTestsFromModule(ptrack)) - return suite # test_pgpro434_2 unexpected success diff --git a/tests/merge.py b/tests/merge.py index ebd8b7feb..064c06289 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1632,9 +1632,7 @@ def test_losing_file_after_failed_merge(self): gdb.run_until_break() gdb.set_breakpoint('fio_fwrite') - gdb.continue_execution_until_break() - - gdb._execute('bt') + gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') From 8d5c578ad8f7401dc83cb301e422ddc102aba276 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jul 2019 14:47:21 +0300 Subject: [PATCH 0860/2107] Documentation: update for ARCHIVE and STREAM sections --- Documentation.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 37c33c9d6..5ba62bccd 100644 --- a/Documentation.md +++ b/Documentation.md @@ -897,7 +897,7 @@ For example, to make a FULL backup in ARCHIVE mode, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL -ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. +Unlike backup in STREAM mode, ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. @@ -911,7 +911,7 @@ For example, to make a FULL backup in STREAM mode, add the `--stream` option to The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. -STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. +Unlike backup in ARCHIVE mode, STREAM backup include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN in '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. @@ -925,11 +925,12 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. -Page is considered corrupted if checksumm comparison failed more than 100 times, is this case backup is aborted. +Page is considered corrupted if checksumm comparison failed more than 100 times, in this case backup is aborted. Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". #### External directories + To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: From f6cd14ee296d5d68e63e464308648a359e3a09cc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 00:14:14 +0300 Subject: [PATCH 0861/2107] Documentation: update section WAL archiving --- Documentation.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Documentation.md b/Documentation.md index 5ba62bccd..12cdb4a71 100644 --- a/Documentation.md +++ b/Documentation.md @@ -261,7 +261,7 @@ For details, see the sections [Setting up STREAM Backups](#setting-up-stream-bac ### Setting up STREAM Backups -To set up the cluster for STREAM backups, complete the following steps: +To set up the cluster for [STREAM](#stream-mode) backups, complete the following steps: - Grant the REPLICATION privilege to the backup role: @@ -273,11 +273,11 @@ To set up the cluster for STREAM backups, complete the following steps: If you are planning to take PAGE backups in STREAM mode, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). -Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in STREAM mode. +Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups with [STREAM](#stream-mode) WAL mode. ### Setting up continuous WAL archiving -ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: +Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time-pitr-recovery) and making backups with [ARCHIVE](#archive-mode) WAL delivery mode require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. - If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. @@ -285,13 +285,15 @@ ARCHIVE backups require [continious WAL archiving](https://www.postgresql.org/do archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' -Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote machine. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). +Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote host. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). -Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups in ARCHIVE mode. +Once these steps are complete, you can start making backups with ARCHIVE WAL mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. +If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to synchronize the value of this setting with pg_probackup option `--archive-timeout`. + +>NOTE: using pg_probackup command [archive-push](#archive-push) for continious archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be `gzip`, and '.gz' suffix in filename is mandatory. ->NOTE: using pg_probackup command [archive-push](#archive-push) for continious archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be gzip and '.gz' suffix is mandatory. +>NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. ### Backup from Standby @@ -302,7 +304,7 @@ For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby serve - To perform STREAM backup on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) - To perform ARCHIVE backup on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) -Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups of appropriate WAL delivery method: ARCHIVE or STREAM, from the standby server. +Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups with appropriate WAL delivery mode: ARCHIVE or STREAM, from the standby server. Backup from the standby server has the following limitations: @@ -922,6 +924,7 @@ Even if you are using [continuous archiving](#setting-up-continuous-wal-archivin - Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. #### Page validation + If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. From 05e1ec1754b94bbceaab40c2b47d34f666bf2a46 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 11:05:59 +0300 Subject: [PATCH 0862/2107] tests: added backup.BackupTest.test_backup_with_less_privileges_role --- tests/backup.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index ec8288fd9..80d5a503b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1883,3 +1883,157 @@ def test_parent_choosing_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_with_less_privileges_role(self): + """ + check permissions correctness from documentation: + https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s'}) + + if self.ptrack: + node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 10 + else: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + + # enable STREAM backup + node.safe_psql( + 'backupdb', + 'ALTER ROLE backup WITH REPLICATION;') + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', '-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['-U', 'backup']) + + # PAGE + self.backup_node( + backup_dir, 'node', node, backup_type='page', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='page', datname='backupdb', + options=['--stream', '-U', 'backup']) + + # DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # Restore as replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'node', replica) + self.set_replica(node, replica) + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + replica.slow_start(replica=True) + + # FULL backup from replica + self.backup_node( + backup_dir, 'replica', replica, + datname='backupdb', options=['--stream', '-U', 'backup']) + + self.backup_node( + backup_dir, 'replica', replica, datname='backupdb', + options=['-U', 'backup', '--log-level-file=verbose']) + + # PAGE + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # DELTA + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 5d3056d98b8f8526375cfa4bd4890a8c423ea516 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 11:15:17 +0300 Subject: [PATCH 0863/2107] tests: update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f81ecf49b..d888c11fb 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.1.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.4) +[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) ## Installation and Setup ### Windows Installation From 64f166f4a0859083b1674d86761ff2ad9ed126e9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 16:20:05 +0300 Subject: [PATCH 0864/2107] Documentation: minor update --- Documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 12cdb4a71..544e936a7 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1051,7 +1051,7 @@ The typical workflow is as follows: - If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery method, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. -- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) on backup host. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. >NOTE: The remote backup mode is currently unavailable for Windows systems. From 44923e7fb745acf6241a10feae2b3f2cff44b6b7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 16:22:02 +0300 Subject: [PATCH 0865/2107] Release 2.1.4 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d888c11fb..f81ecf49b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.1.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.3) +[2.1.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.4) ## Installation and Setup ### Windows Installation From db73f84057d70969514b3cd0daf971f529bdcc74 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 20:10:58 +0300 Subject: [PATCH 0866/2107] [Issue #101] Multiple spelling fixes. Reported by Alexander Lakhin --- README.md | 4 ++-- gen_probackup_project.pl | 2 +- src/backup.c | 16 ++++++++-------- src/catalog.c | 8 ++++---- src/checkdb.c | 4 ++-- src/data.c | 10 +++++----- src/delete.c | 4 ++-- src/dir.c | 4 ++-- src/help.c | 2 +- src/merge.c | 4 +++- src/parsexlog.c | 4 ++-- src/pg_probackup.c | 2 +- src/pg_probackup.h | 4 ++-- src/restore.c | 6 +++--- src/utils/file.c | 12 ++++++------ src/utils/parray.c | 4 ++-- src/utils/pgut.c | 2 +- src/validate.c | 6 +++--- tests/Readme.md | 6 +++--- tests/archive.py | 14 +++++++------- tests/auth_test.py | 4 ++-- tests/backup.py | 6 +++--- tests/checkdb.py | 2 +- tests/compression.py | 4 ++-- tests/helpers/ptrack_helpers.py | 2 +- tests/locking.py | 4 ++-- tests/merge.py | 2 +- tests/page.py | 2 +- tests/pgpro589.py | 2 +- tests/ptrack.py | 4 ++-- tests/restore.py | 2 +- tests/validate.py | 2 +- travis/backup_restore.sh | 2 +- 33 files changed, 79 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index f81ecf49b..fd55b54ed 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). -## Licence +## License -This module available under the [license](LICENSE) similar to [PostgreSQL](https://www.postgresql.org/about/licence/). +This module available under the [license](LICENSE) similar to [PostgreSQL](https://www.postgresql.org/about/license/). ## Feedback diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 6de352e86..1c5a9df26 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -5,7 +5,7 @@ our $currpath; BEGIN { -# path to the pg_pprobackup dir +# path to the pg_probackup dir $currpath = File::Basename::dirname(Cwd::abs_path($0)); use Cwd; use File::Basename; diff --git a/src/backup.c b/src/backup.c index c935b7566..29d6d34e6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -28,7 +28,7 @@ /* * Macro needed to parse ptrack. - * NOTE Keep those values syncronised with definitions in ptrack.h + * NOTE Keep those values synchronized with definitions in ptrack.h */ #define PTRACK_BITS_PER_HEAPBLOCK 1 #define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) @@ -39,7 +39,7 @@ static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; /* * How long we should wait for streaming end in seconds. - * Retreived as checkpoint_timeout + checkpoint_timeout * 0.1 + * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 */ static uint32 stream_stop_timeout = 0; /* Time in which we started to wait for streaming end */ @@ -451,7 +451,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* Run threads */ thread_interrupted = false; - elog(INFO, "Start transfering data files"); + elog(INFO, "Start transferring data files"); for (i = 0; i < num_threads; i++) { backup_files_arg *arg = &(threads_args[i]); @@ -468,7 +468,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) backup_isok = false; } if (backup_isok) - elog(INFO, "Data files are transfered"); + elog(INFO, "Data files are transferred"); else elog(ERROR, "Data files transferring failed"); @@ -686,7 +686,7 @@ do_backup(time_t start_time, bool no_validate) /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) - elog(ERROR, "Failed to retreive wal_segment_size"); + elog(ERROR, "Failed to retrieve wal_segment_size"); #endif is_ptrack_support = pg_ptrack_support(backup_conn); @@ -1346,7 +1346,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) tli = get_current_timeline(false); - /* Compute the name of the WAL file containig requested LSN */ + /* Compute the name of the WAL file containing requested LSN */ GetXLogSegNo(lsn, targetSegNo, instance_config.xlog_seg_size); if (wait_prev_segment) targetSegNo--; @@ -1862,7 +1862,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, } /* - * Retreive checkpoint_timeout GUC value in seconds. + * Retrieve checkpoint_timeout GUC value in seconds. */ static int checkpoint_timeout(PGconn *backup_conn) @@ -2360,7 +2360,7 @@ make_pagemap_from_ptrack(parray *files, PGconn *backup_conn) if (ptrack_nonparsed != NULL) { /* - * pg_ptrack_get_and_clear() returns ptrack with VARHDR cutted out. + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cut out. * Compute the beginning of the ptrack map related to this segment * * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 diff --git a/src/catalog.c b/src/catalog.c index 34e35793d..ba7fa5aee 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -260,7 +260,7 @@ lock_backup(pgBackup *backup) fio_unlink(lock_file, FIO_BACKUP_HOST); errno = save_errno; - elog(ERROR, "Culd not write lock file \"%s\": %s", + elog(ERROR, "Could not write lock file \"%s\": %s", lock_file, strerror(errno)); } @@ -1022,7 +1022,7 @@ parse_compress_alg(const char *arg) len = strlen(arg); if (len == 0) - elog(ERROR, "compress algrorithm is empty"); + elog(ERROR, "compress algorithm is empty"); if (pg_strncasecmp("zlib", arg, len) == 0) return ZLIB_COMPRESS; @@ -1231,7 +1231,7 @@ find_parent_full_backup(pgBackup *current_backup) } /* - * Interate over parent chain and look for any problems. + * Iterate over parent chain and look for any problems. * Return 0 if chain is broken. * result_backup must contain oldest existing backup after missing backup. * we have no way to know if there are multiple missing backups. @@ -1262,7 +1262,7 @@ scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) target_backup = target_backup->parent_backup_link; } - /* Prevous loop will skip FULL backup because his parent_backup_link is NULL */ + /* Previous loop will skip FULL backup because his parent_backup_link is NULL */ if (target_backup->backup_mode == BACKUP_MODE_FULL && (target_backup->status != BACKUP_STATUS_OK && target_backup->status != BACKUP_STATUS_DONE)) diff --git a/src/checkdb.c b/src/checkdb.c index dd1145f47..e878d688f 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -81,7 +81,7 @@ typedef struct pg_indexEntry char *name; char *namespace; bool heapallindexed_is_supported; - /* schema where amcheck extention is located */ + /* schema where amcheck extension is located */ char *amcheck_nspname; /* lock for synchronization of parallel threads */ volatile pg_atomic_flag lock; @@ -408,7 +408,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); if (!heapallindexed_is_supported && heapallindexed) - elog(WARNING, "Extension '%s' verion %s in schema '%s'" + elog(WARNING, "Extension '%s' version %s in schema '%s'" "do not support 'heapallindexed' option", PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); diff --git a/src/data.c b/src/data.c index 0304f10bc..2b9121ede 100644 --- a/src/data.c +++ b/src/data.c @@ -159,7 +159,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) return false; } #endif - /* otherwize let's try to decompress the page */ + /* otherwise let's try to decompress the page */ return true; } return false; @@ -396,7 +396,7 @@ prepare_page(ConnectionArgs *arguments, { /* * We need to copy the page that was successfully - * retreieved from ptrack into our output "page" parameter. + * retrieved from ptrack into our output "page" parameter. * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ @@ -482,7 +482,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, compressed_page, header.compressed_size); write_buffer_size += MAXALIGN(header.compressed_size); } - /* Nonpositive value means that compression failed. Write it as is. */ + /* Non-positive value means that compression failed. Write it as is. */ else { header.compressed_size = BLCKSZ; @@ -754,7 +754,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, DataPage page; int32 uncompressed_size = 0; - /* File didn`t changed. Nothig to copy */ + /* File didn`t changed. Nothing to copy */ if (file->write_size == BYTES_INVALID) break; @@ -887,7 +887,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do * But during DELTA backup we read every file in PGDATA and thus DELTA backup * knows exact size of every file at the time of backup. - * So when restoring file from DELTA backup we, knowning it`s size at + * So when restoring file from DELTA backup we, knowing it`s size at * a time of a backup, can truncate file to this size. */ if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) diff --git a/src/delete.c b/src/delete.c index 1bce734cd..a6b0d92aa 100644 --- a/src/delete.c +++ b/src/delete.c @@ -116,7 +116,7 @@ do_delete(time_t backup_id) * * Invalid backups handled in Oracle style, so invalid backups are ignored * for the purpose of retention fulfillment, - * i.e. CORRUPT full backup do not taken in account when deteremine + * i.e. CORRUPT full backup do not taken in account when determine * which FULL backup should be keeped for redundancy obligation(only valid do), * but if invalid backup is not guarded by retention - it is removed */ @@ -491,7 +491,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l * 2 PAGE1 * 3 FULL * - * Сonsequentially merge incremental backups from PAGE1 to PAGE3 + * Consequentially merge incremental backups from PAGE1 to PAGE3 * into FULL. */ diff --git a/src/dir.c b/src/dir.c index f05359d5e..948ca5aeb 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1075,7 +1075,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba } /* - * Read names of symbolik names of tablespaces with links to directories from + * Read names of symbolic names of tablespaces with links to directories from * tablespace_map or tablespace_map.txt. */ void @@ -1568,7 +1568,7 @@ pgFileSize(const char *path) } /* - * Construct parray containing remmaped external directories paths + * Construct parray containing remapped external directories paths * from string like /path1:/path2 */ parray * diff --git a/src/help.c b/src/help.c index e74facf39..85b1ed858 100644 --- a/src/help.c +++ b/src/help.c @@ -540,7 +540,7 @@ help_show(void) printf(_(" [--format=format]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name show info about specific intstance\n")); + printf(_(" --instance=instance_name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); printf(_(" --format=format show format=PLAIN|JSON\n\n")); } diff --git a/src/merge.c b/src/merge.c index 20969781b..832bd90cb 100644 --- a/src/merge.c +++ b/src/merge.c @@ -270,7 +270,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) false); /* - * Rename external directoties in to_backup (if exists) + * Rename external directories in to_backup (if exists) * according to numeration of external dirs in from_backup. */ if (to_external) @@ -594,6 +594,8 @@ merge_files(void *arg) elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", merge_to_file_path); + // TODO: truncate merge_to_file_path just in case? + /* * file->path is relative, to_file_path - is absolute. * Substitute them. diff --git a/src/parsexlog.c b/src/parsexlog.c index 0e2ae588c..9008b5285 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -228,7 +228,7 @@ static XLogRecPtr wal_target_lsn = InvalidXLogRecPtr; * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the * given timeline. Collect data blocks touched by the WAL records into a page map. * - * Pagemap extracting is processed using threads. Eeach thread reads single WAL + * Pagemap extracting is processed using threads. Each thread reads single WAL * file. */ void @@ -1491,7 +1491,7 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) continue; - /* We only care about the main fork; others are copied in toto */ + /* We only care about the main fork; others are copied as is */ if (forknum != MAIN_FORKNUM) continue; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 49b4363b0..915b4ca8c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -490,7 +490,7 @@ main(int argc, char *argv[]) /* Usually checkdb for file logging requires log_directory * to be specified explicitly, but if backup_dir and instance name are provided, - * checkdb can use the tusual default values or values from config + * checkdb can use the usual default values or values from config */ if (backup_subcmd == CHECKDB_CMD && (instance_config.logger.log_level_file != LOG_OFF && diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 90de4e7fa..78cba094c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -66,7 +66,7 @@ extern const char *PROGRAM_EMAIL; #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 -/* Direcotry/File permission */ +/* Directory/File permission */ #define DIR_PERMISSION (0700) #define FILE_PERMISSION (0600) @@ -264,7 +264,7 @@ struct pgBackup time_t backup_id; /* Identifier of the backup. * Currently it's the same as start_time */ BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ - TimeLineID tli; /* timeline of start and stop baskup lsns */ + TimeLineID tli; /* timeline of start and stop backup lsns */ XLogRecPtr start_lsn; /* backup's starting transaction log location */ XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ time_t start_time; /* since this moment backup has status diff --git a/src/restore.c b/src/restore.c index c7c03e208..c3c279b78 100644 --- a/src/restore.c +++ b/src/restore.c @@ -298,7 +298,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { check_tablespace_mapping(dest_backup); - /* no point in checking external directories if their restore is not resquested */ + /* no point in checking external directories if their restore is not requested */ if (!skip_external_dirs) check_external_dir_mapping(dest_backup); } @@ -377,7 +377,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, rt->target_xid, rt->target_lsn, base_full_backup->tli, instance_config.xlog_seg_size); } - /* Orphinize every OK descendant of corrupted backup */ + /* Orphanize every OK descendant of corrupted backup */ else { char *corrupted_backup_id; @@ -1064,7 +1064,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_lsn(target_lsn, &dummy_lsn)) rt->target_lsn = dummy_lsn; else - elog(ERROR, "Invalid value of --ecovery-target-lsn option %s", + elog(ERROR, "Invalid value of --recovery-target-lsn option %s", target_lsn); } diff --git a/src/utils/file.c b/src/utils/file.c index 5a2f1d7a8..ba0575da3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -37,7 +37,7 @@ typedef struct /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) -/* Use specified file descriptors as stding/stdout for FIO functions */ +/* Use specified file descriptors as stdin/stdout for FIO functions */ void fio_redirect(int in, int out) { fio_stdin = in; @@ -726,7 +726,7 @@ int fio_access(char const* path, int mode, fio_location location) } } -/* Create symbolink link */ +/* Create symbolic link */ int fio_symlink(char const* target, char const* link_path, fio_location location) { if (fio_is_remote(location)) @@ -822,7 +822,7 @@ int fio_mkdir(char const* path, int mode, fio_location location) } } -/* Checnge file mode */ +/* Change file mode */ int fio_chmod(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) @@ -954,7 +954,7 @@ fio_gzread(gzFile f, void *buf, unsigned size) while (1) { - if (gz->strm.avail_in != 0) /* If there is some data in receiver buffer, then decmpress it */ + if (gz->strm.avail_in != 0) /* If there is some data in receiver buffer, then decompress it */ { rc = inflate(&gz->strm, Z_NO_FLUSH); if (rc == Z_STREAM_END) @@ -1021,7 +1021,7 @@ fio_gzwrite(gzFile f, void const* buf, unsigned size) { rc = deflate(&gz->strm, Z_NO_FLUSH); Assert(rc == Z_OK); - gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of bufer */ + gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ } else { @@ -1429,7 +1429,7 @@ void fio_communicate(int in, int out) case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ SYS_CHECK(remove_file_or_dir(buf)); break; - case FIO_MKDIR: /* Create direcory */ + case FIO_MKDIR: /* Create directory */ hdr.size = 0; hdr.arg = dir_create_dir(buf, hdr.arg); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); diff --git a/src/utils/parray.c b/src/utils/parray.c index 54ff9593c..23c227b19 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -15,7 +15,7 @@ /* members of struct parray are hidden from client. */ struct parray { - void **data; /* poiter array, expanded if necessary */ + void **data; /* pointer array, expanded if necessary */ size_t alloced; /* number of elements allocated */ size_t used; /* number of elements in use */ }; @@ -97,7 +97,7 @@ parray_insert(parray *array, size_t index, void *elem) } /* - * Concatinate two parray. + * Concatenate two parray. * parray_concat() appends the copy of the content of src to the end of dest. */ parray * diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6cb685c7f..92a8028a2 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -757,7 +757,7 @@ on_interrupt(void) interrupted = true; /* - * User promts password, call on_cleanup() byhand. Unless we do that we will + * User prompts password, call on_cleanup() byhand. Unless we do that we will * get stuck forever until a user enters a password. */ if (in_password) diff --git a/src/validate.c b/src/validate.c index 9a7af09dc..9ccc5ba8a 100644 --- a/src/validate.c +++ b/src/validate.c @@ -371,7 +371,7 @@ do_validate_all(void) /* TODO: Probably we should have different exit code for every condition * and they combination: * 0 - all backups are valid - * 1 - some backups are corrup + * 1 - some backups are corrupt * 2 - some backups where skipped due to concurrent locks * 3 - some backups are corrupt and some are skipped due to concurrent locks */ @@ -547,7 +547,7 @@ do_validate_instance(void) /* For every OK backup we try to revalidate all his ORPHAN descendants. */ if (current_backup->status == BACKUP_STATUS_OK) { - /* revalidate all ORPHAN descendats + /* revalidate all ORPHAN descendants * be very careful not to miss a missing backup * for every backup we must check that he is descendant of current_backup */ @@ -592,7 +592,7 @@ do_validate_instance(void) skipped_due_to_lock = true; continue; } - /* Revaliate backup files*/ + /* Revalidate backup files*/ pgBackupValidate(backup); if (backup->status == BACKUP_STATUS_OK) diff --git a/tests/Readme.md b/tests/Readme.md index d24d095cd..3adf0c019 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,11 +1,11 @@ [см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` -Note: For now these are works on Linix and "kinda" works on Windows +Note: For now these are works on Linux and "kinda" works on Windows ``` ``` -Windows Note: For tablespaceses tests to work on Windows, you should explicitly(!) grant current user full access to tmp_dirs +Windows Note: For tablespaces tests to work on Windows, you should explicitly(!) grant current user full access to tmp_dirs ``` @@ -23,7 +23,7 @@ Enable compatibility tests: Specify path to pg_probackup binary file. By default tests use /pg_probackup/ export PGPROBACKUPBIN= -Remote backup depends on key authentithication to local machine via ssh as current user. +Remote backup depends on key authentication to local machine via ssh as current user. export PGPROBACKUP_SSH_REMOTE=ON Run suit of basic simple tests: diff --git a/tests/archive.py b/tests/archive.py index 67a4429d0..cf3441960 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -47,7 +47,7 @@ def test_pgpro434_1(self): backup_dir, 'node', node) node.slow_start() - # Recreate backup calagoue + # Recreate backup catalog self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -350,7 +350,7 @@ def test_pgpro434_4(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_arhive_push_file_exists(self): + def test_archive_push_file_exists(self): """Archive-push if file exists""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -359,8 +359,8 @@ def test_arhive_push_file_exists(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} - ) + 'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -442,7 +442,7 @@ def test_arhive_push_file_exists(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_arhive_push_file_exists_overwrite(self): + def test_archive_push_file_exists_overwrite(self): """Archive-push if file exists""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -451,8 +451,8 @@ def test_arhive_push_file_exists_overwrite(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'} - ) + 'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) diff --git a/tests/auth_test.py b/tests/auth_test.py index 8b8378348..d9b5c283e 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -24,9 +24,9 @@ class SimpleAuthTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") - def test_backup_via_unpriviledged_user(self): + def test_backup_via_unprivileged_user(self): """ - Make node, create unpriviledged user, try to + Make node, create unprivileged user, try to run a backups without EXECUTE rights on certain functions """ diff --git a/tests/backup.py b/tests/backup.py index 80d5a503b..81370a173 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -643,7 +643,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): path = os.path.join(root, file) list = list + [path] - # We expect that relfilenode occures only once + # We expect that relfilenode can be encountered only once if len(list) > 1: message = "" for string in list: @@ -2019,7 +2019,7 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'replica', replica, datname='backupdb', options=['-U', 'backup', '--log-level-file=verbose']) - # PAGE + # PAGE backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['-U', 'backup']) @@ -2027,7 +2027,7 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) - # DELTA + # DELTA backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['-U', 'backup']) diff --git a/tests/checkdb.py b/tests/checkdb.py index 1d546591b..64a733ac7 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -330,7 +330,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): # corruption of both indexes in db1 and db2 must be detected # also the that amcheck is not installed in 'postgres' - # musted be logged + # should be logged with open(log_file_path) as f: log_file_content = f.read() self.assertIn( diff --git a/tests/compression.py b/tests/compression.py index 19ea30757..b8788a46c 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -472,9 +472,9 @@ def test_compression_wrong_algorithm(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_uncompressable_pages(self): + def test_incompressible_pages(self): """ - make archive node, create table with uncompressable toast pages, + make archive node, create table with incompressible toast pages, take backup with compression, make sure that page was not compressed, restore backup and check data correctness """ diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d3337de55..a6e151ae8 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1418,7 +1418,7 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): else: error_message += ( - '\nFile dissappearance.\n ' + '\nFile disappearance.\n ' 'File: {0}\n').format( os.path.join(restored_pgdata['pgdata'], file) ) diff --git a/tests/locking.py b/tests/locking.py index 918509c60..54389bde6 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -15,7 +15,7 @@ def test_locking_running_validate_1(self): """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, - concurrect RUNNING backup with pid file and active process is legal + concurrent RUNNING backup with pid file and active process is legal """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -406,7 +406,7 @@ def test_locking_restore_locked_without_validation(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_locking_concurrent_vaidate_and_backup(self): + def test_locking_concurrent_validate_and_backup(self): """ make node, take full backup, launch validate and stop it in the middle, take page backup. diff --git a/tests/merge.py b/tests/merge.py index 064c06289..d17d6b603 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1345,7 +1345,7 @@ def test_continue_failed_merge_3(self): def test_merge_different_compression_algo(self): """ - Check that backups with different compression algorihtms can be merged + Check that backups with different compression algorithms can be merged """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/page.py b/tests/page.py index 7b7a32364..9a717c234 100644 --- a/tests/page.py +++ b/tests/page.py @@ -906,7 +906,7 @@ def test_page_backup_with_alien_wal_segment(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,100000) i;") - # copy lastest wal segment + # copy latest wal segment wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( wals_dir, f)) and not f.endswith('.backup')] diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 3ed398725..122b2793f 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -27,7 +27,7 @@ def test_pgpro589(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - # make erroneus archive_command + # make erroneous archive_command node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") node.slow_start() diff --git a/tests/ptrack.py b/tests/ptrack.py index 5d6d567fe..7025d994b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -122,8 +122,8 @@ def test_ptrack_disable(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_ptrack_uncommited_xact(self): - """make ptrack backup while there is uncommited open transaction""" + def test_ptrack_uncommitted_xact(self): + """make ptrack backup while there is uncommitted open transaction""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/restore.py b/tests/restore.py index 0175d19df..583a99a0a 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -821,7 +821,7 @@ def test_restore_with_tablespace_mapping_1(self): # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because restore destionation is not empty.\n " + "Expecting Error because restore destination is not empty.\n " "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: diff --git a/tests/validate.py b/tests/validate.py index 0e9fd76d6..1c919907f 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -304,7 +304,7 @@ def test_basic_validate_corrupted_intermediate_backup(self): def test_validate_corrupted_intermediate_backups(self): """ make archive node, take FULL, PAGE1, PAGE2 backups, - corrupt file in FULL and PAGE1 backupd, run validate on PAGE1, + corrupt file in FULL and PAGE1 backups, run validate on PAGE1, expect FULL and PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh index 7fe1cfd8f..b3c9df1ed 100644 --- a/travis/backup_restore.sh +++ b/travis/backup_restore.sh @@ -27,7 +27,7 @@ yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-deve make top_srcdir=postgresql-$PGVERSION make install top_srcdir=postgresql-$PGVERSION -# initalize cluster and database +# initialize cluster and database yum install -y postgresql95-server su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" cat < $PGDATA/pg_hba.conf From c0d3ab5456312315a7cfdc158cdf863cbdf36647 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jul 2019 21:52:50 +0300 Subject: [PATCH 0867/2107] [Issue #101] replace "sparse" with "compact" --- src/delete.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/delete.c b/src/delete.c index a6b0d92aa..83be5d522 100644 --- a/src/delete.c +++ b/src/delete.c @@ -347,7 +347,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg backup->parent_backup_link, pgBackupCompareIdDesc)) { - /* make keep list a bit sparse */ + /* make keep list a bit more compact */ parray_append(to_keep_list, backup); continue; } @@ -445,7 +445,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l continue; } - /* FULL backup in purge list, thanks to sparsing of keep_list current backup is + /* FULL backup in purge list, thanks to compacting of keep_list current backup is * final target for merge, but there could be intermediate incremental * backups from purge_list. */ From 19ea1bf5dc5205c65497f4a7f899877deadc592b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 25 Jul 2019 15:11:46 +0300 Subject: [PATCH 0868/2107] Eliminate waitpid for Windows --- src/utils/remote.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils/remote.c b/src/utils/remote.c index 8a9a86d2b..36b1a4188 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -70,9 +70,16 @@ static void kill_child(void) void wait_ssh(void) { +/* + * We need to wait termination of SSH process to eliminate zombies. + * There is no waitpid() function at Windows but there are no zombie processes caused by lack of wait/waitpid. + * So just disable waitpid for Windows. + */ +#ifndef WIN32 int status; waitpid(child_pid, &status, 0); elog(LOG, "SSH process %d is terminated with status %d", child_pid, status); +#endif } #ifdef WIN32 From e876fcfb0f10f236c1f3110528fb4f5d82053ee5 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 25 Jul 2019 18:23:52 +0300 Subject: [PATCH 0869/2107] Fixed script for generate Windows project --- gen_probackup_project.pl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 1c5a9df26..511e7e07c 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -155,7 +155,8 @@ sub build_pgprobackup 'restore.c', 'show.c', 'util.c', - 'validate.c' + 'validate.c', + 'checkdb.c' ); $probackup->AddFiles( "$currpath/src/utils", @@ -166,7 +167,8 @@ sub build_pgprobackup 'logger.c', 'parray.c', 'pgut.c', - 'thread.c' + 'thread.c', + 'remote.c' ); $probackup->AddFile("$pgsrc/src/backend/access/transam/xlogreader.c"); $probackup->AddFile("$pgsrc/src/backend/utils/hash/pg_crc.c"); From b55bfe82becbbac69b934f3d57d673f2d791f580 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 26 Jul 2019 11:05:19 +0300 Subject: [PATCH 0870/2107] Version 2.1.5 --- README.md | 2 +- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fd55b54ed..fae9ea171 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Documentation -Currently the latest documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). +Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). ## License diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 78cba094c..e646d760d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -188,8 +188,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.4" -#define AGENT_PROTOCOL_VERSION 20104 +#define PROGRAM_VERSION "2.1.5" +#define AGENT_PROTOCOL_VERSION 20105 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1a5c9cb70..9c96acc51 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.4 \ No newline at end of file +pg_probackup 2.1.5 \ No newline at end of file From 4e9f0d6d2ef0f6f5fc39fd86763de87a6a3b8ad8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 26 Jul 2019 12:17:28 +0300 Subject: [PATCH 0871/2107] remove obsolete scripts --- doit.cmd | 3 --- doit96.cmd | 1 - 2 files changed, 4 deletions(-) delete mode 100644 doit.cmd delete mode 100644 doit96.cmd diff --git a/doit.cmd b/doit.cmd deleted file mode 100644 index 7830a7ead..000000000 --- a/doit.cmd +++ /dev/null @@ -1,3 +0,0 @@ -CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 -SET PERL5LIB=. -perl gen_probackup_project.pl C:\Shared\Postgresql\myPostgres\11\postgrespro \ No newline at end of file diff --git a/doit96.cmd b/doit96.cmd deleted file mode 100644 index 94d242c99..000000000 --- a/doit96.cmd +++ /dev/null @@ -1 +0,0 @@ -perl win32build96.pl "C:\PgPro96" "C:\PgProject\pg96ee\postgrespro\src" \ No newline at end of file From 3700cc4c5252655d170ccfb46088aab3deb9f372 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 02:51:30 +0300 Subject: [PATCH 0872/2107] Readme: update latest version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fae9ea171..c5749a880 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.1.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.4) +[2.1.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.5) ## Installation and Setup ### Windows Installation From 9b270b0f7059f95a858f8cfea0a48ae62dd4cd85 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 05:07:41 +0300 Subject: [PATCH 0873/2107] Readme: ptrack support --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c5749a880..1cbdab651 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * `Autonomous backups` streams via replication protocol all the WAL files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup. * `Archive backups` rely on continuous archiving. +## ptrack support + `PTRACK` backup support provided via following options: * vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) * Postgres Pro Standard 9.5, 9.6, 10, 11 From 77a0e050ab3413e700b654774879c02567773057 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 05:45:34 +0300 Subject: [PATCH 0874/2107] Documentation: review be Mikhail Kulagin --- Documentation.md | 1432 ++++++++++++++++++++++++---------------------- 1 file changed, 736 insertions(+), 696 deletions(-) diff --git a/Documentation.md b/Documentation.md index 544e936a7..8151dafb3 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.1.4 +Current version - 2.1.5 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) @@ -15,11 +15,30 @@ Current version - 2.1.4 * [Configuring the Database Cluster](#configuring-the-database-cluster) * [Setting up STREAM Backups](#setting-up-stream-backups) * [Setting up Continuous WAL Archiving](#setting-up-continuous-wal-archiving) - * [Setting up Backup from Standby](#backup-from-standby) + * [Setting up Backup from Standby](#setting-up-backup-from-standby) * [Setting up Cluster Verification](#setting-up-cluster-verification) * [Setting up PTRACK Backups](#setting-up-ptrack-backups) + * [Configuring the Remote Mode](#configuring-the-remote-mode) -5. [Command-Line Reference](#command-line-reference) +5. [Usage](#usage) + * [Creating a Backup](#creating-a-backup) + * [ARCHIVE WAL mode](#archive-mode) + * [STREAM WAL mode](#stream-mode) + * [Page validation](#page-validation) + * [External directories](#external-directories) + * [Verifying a Cluster](#verifying-a-cluster) + * [Validating a Backup](#validating-a-backup) + * [Restoring a Cluster](#restoring-a-cluster) + * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) + * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) + * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) + * [Configuring pg_probackup](#configuring-pg_probackup) + * [Managing the Backup Catalog](#managing-the-backup-Catalog) + * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) + * [Merging Backups](#merging-backups) + * [Deleting Backups](#deleting-backups) + +6. [Command-Line Reference](#command-line-reference) * [Commands](#commands) * [version](#version) * [help](#help) @@ -37,12 +56,8 @@ Current version - 2.1.4 * [delete](#delete) * [archive-push](#archive-push) * [archive-get](#archive-get) - * [Options](#options) * [Common Options](#common-options) - * [Backup Options](#backup-options) - * [Restore Options](#restore-options) - * [Checkdb Options](#checkdb-options) * [Recovery Target Options](#recovery-target-options) * [Retention Options](#retention-options) * [Logging Options](#logging-options) @@ -52,20 +67,6 @@ Current version - 2.1.4 * [Remote Mode Options](#remote-mode-options) * [Replica Options](#replica-options) -6. [Usage](#usage) - * [Creating a Backup](#creating-a-backup) - * [Verifying a Cluster](#verifying-a-cluster) - * [Validating a Backup](#validating-a-backup) - * [Restoring a Cluster](#restoring-a-cluster) - * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) - * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - * [Configuring pg_probackup](#configuring-pg_probackup) - * [Managing the Backup Catalog](#managing-the-backup-Catalog) - * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) - * [Merging Backups](#merging-backups) - * [Deleting Backups](#deleting-backups) - 7. [Authors](#authors) 8. [Credits](#credits) @@ -100,29 +101,29 @@ Current version - 2.1.4 `pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired }` -`pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [option...]` +`pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` -`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f` +`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name` ## Versioning -pg_probackup is following [semantic](https://semver.org/) versioning. +pg_probackup is following the [semantic](https://semver.org/) versioning. ## Overview As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: - Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow -- Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery -- Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb` -- Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` -- Parallelization: Running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads -- Compression: Storing backup data in a compressed state to save disk space -- Deduplication: Saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) -- Remote operations: Backup PostgreSQL instance located on remote machine or restore backup on it -- Backup from replica: Avoid extra load on the master server by taking backups from a standby -- External directories: Add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files +- Validation: automatic data consistency checks and on-demand backup validation without actual data recovery +- Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` +- Retention: managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` +- Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads +- Compression: storing backup data in a compressed state to save disk space +- Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) +- Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it +- Backup from replica: avoid extra load on the master server by taking backups from a standby +- External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files - Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -135,17 +136,20 @@ Using pg_probackup, you can take full or incremental [backups](#creating-a-backu - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen backup mode (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery methods`: +pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen backup mode (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery modes`: -- [ARCHIVE](#archive-mode). Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery method. -- [STREAM](#stream-mode). Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. +- [ARCHIVE](#archive-mode). Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery mode. +- [STREAM](#stream-mode). Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. Because of that backups of this WAL mode are called `autonomous` or `standalone`. ### Limitations pg_probackup currently has the following limitations: -- Creating backups from a remote server is currently not supported on Windows systems. -- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. +- Only PostgreSQL of versions 9.5 and newer are supported. +- Currently remode mode of operations is not supported on Windows systems. +- On Unix systems backup of PostgreSQL verions =< 10 is possible only by the same OS user PostgreSQL server is running by. For example, if PostgreSQL server is running by user *postgres*, then backup must be run by user *postgres*. If backup is running in [remote mode](#using-pg_probackup-in-the-remote-mode) using `ssh`, then this limitation apply differently: value for `--remote-user` option should be *postgres*. +- During backup of PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` will be executed only if backup role is superuser. Because of that backup of a cluster with low amount of WAL traffic with non-superuser role may take more time than backup of the same cluster with superuser role. +- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. Also depending on cluster configuration PostgreSQL itself may apply additional restrictions such as CPU architecture platform and libc/libicu versions. ## Installation and Setup @@ -153,9 +157,11 @@ Once you have pg_probackup installed, complete the following setup: - Initialize the backup catalog. - Add a new backup instance to the backup catalog. -- Configure the database cluster to enable pg_probackup backups. +- Configure the database cluster to enable pg_probackup backups. +- Optionally, configure SSH for running pg_probackup operations in remote mode. ### Initializing the Backup Catalog + pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. To initialize the backup catalog, run the following command: @@ -164,6 +170,8 @@ To initialize the backup catalog, run the following command: Where *backup_dir* is the path to backup catalog. If the *backup_dir* already exists, it must be empty. Otherwise, pg_probackup returns an error. +The user launching pg_probackup must have full access to *backup_dir* directory. + pg_probackup creates the backup_dir backup catalog, with the following subdirectories: - wal/ — directory for WAL files. @@ -172,6 +180,7 @@ pg_probackup creates the backup_dir backup catalog, with the following subdirect Once the backup catalog is initialized, you can add a new backup instance. ### Adding a New Backup Instance + pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. To add a new backup instance, run the following command: @@ -184,7 +193,7 @@ Where: - *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. - The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. -pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls pg_probackup settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to 'pg_probackup.conf'. +pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls pg_probackup settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to pg_probackup.conf. For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). @@ -196,7 +205,7 @@ The user launching pg_probackup must have full access to *backup_dir* directory Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. -To perform [backup](#backup), the following permissions are required: +To perform [backup](#backup), the following permissions for role *backup* are required only in database **used for connection** to PostgreSQL server: For PostgreSQL 9.5: ``` @@ -251,13 +260,13 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup COMMIT; ``` ->NOTE: In PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` can be executed only by superuser role. So during backup of PostgreSQL 9.5 pg_probackup will use them only if backup role is superuser, although it is NOT recommended to run backup under superuser. +In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow connection to database cluster on behalf of the *backup* role. -Since pg_probackup needs to read cluster files directly, pg_probackup must be started on behalf of an OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. +Since pg_probackup needs to read cluster files directly, pg_probackup must be started by (in case of remote backup - connected to) OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. -Depending on whether you are plan to take STREAM and/or ARCHIVE backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server or create PTRACK backups, additional setup is required. +Depending on whether you are plan to take [autonomous](#stream-mode) and/or [archive](#archive-mode) backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server, run pg_probackup in remote mode or create PTRACK backups, additional setup is required. -For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#backup-from-standby) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). +For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#setting-up-backup-from-standby), [Configuring the Remote Mode](#configuring-the-remote-mode) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). ### Setting up STREAM Backups @@ -271,38 +280,38 @@ To set up the cluster for [STREAM](#stream-mode) backups, complete the following - Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. - Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. -If you are planning to take PAGE backups in STREAM mode, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). +If you are planning to take PAGE backups in STREAM mode or perform PITR with STREAM backups, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups with [STREAM](#stream-mode) WAL mode. ### Setting up continuous WAL archiving -Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time-pitr-recovery) and making backups with [ARCHIVE](#archive-mode) WAL delivery mode require [continious WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continious archiving in the cluster, complete the following steps: +Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time-pitr-recovery) and making backups with [ARCHIVE](#archive-mode) WAL delivery mode require [continuous WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continuous archiving in the cluster, complete the following steps: - Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. -- If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on`. To perform archiving on standby, set this parameter to `always`. +- If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on` or `always`. To perform archiving on standby, set this parameter to `always`. - Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f [remote_options]' + archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote host. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). Once these steps are complete, you can start making backups with ARCHIVE WAL mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). -If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to synchronize the value of this setting with pg_probackup option `--archive-timeout`. +If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to set the value of this setting slightly lower than pg_probackup parameter `--archive-timeout` (default 5 min), so there should be enough time for rotated segment to be streamed to replica and send to archive before backup is aborted because of `--archive-timeout`. ->NOTE: using pg_probackup command [archive-push](#archive-push) for continious archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be `gzip`, and '.gz' suffix in filename is mandatory. +>NOTE: using pg_probackup command [archive-push](#archive-push) for continuous archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be `gzip`, and '.gz' suffix in filename is mandatory. >NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. -### Backup from Standby +### Setting up Backup from Standby For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: - On the standby server, set the parameter [hot_standby](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-HOT-STANDBY) to `on`. - On the master server, set the parameter [full_page_writes](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-FULL-PAGE-WRITES) to `on`. -- To perform STREAM backup on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) -- To perform ARCHIVE backup on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) +- To perform autonomous backups on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) +- To perform archive backups on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups with appropriate WAL delivery mode: ARCHIVE or STREAM, from the standby server. @@ -315,11 +324,11 @@ Backup from the standby server has the following limitations: Logical verification of database cluster requires the following additional setup. Role *backup* is used as an example: -- Install extension `amcheck` or `amcheck_next` in every database of the cluster: +- Install extension [amcheck](https://www.postgresql.org/docs/current/amcheck.html) or [amcheck_next](https://github.com/petergeoghegan/amcheck) **in every database** of the cluster: CREATE EXTENSION amcheck; -- To perform logical verification the following permissions are requiared: +- To perform logical verification the following permissions are required **in every database** of the cluster: ``` GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; @@ -333,996 +342,1027 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; ### Setting up PTRACK Backups +Backup mode PTACK can be used only on Postgrespro Standart and Postgrespro Enterprise installations or patched vanilla PostgreSQL. Links to ptrack patches can be found [here](https://github.com/postgrespro/pg_probackup#ptrack-support). + If you are going to use PTRACK backups, complete the following additional steps: - Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute `ptrack` functions to the *backup* role: +- Grant the rights to execute `ptrack` functions to the *backup* role **in every database** of the cluster: GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; + - The *backup* role must have access to all the databases of the cluster. -## Command-Line Reference -### Commands +### Configuring the Remote Mode -This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). +pg_probackup supports the remote mode that allows to perform backup, restore and WAL archiving operations remotely. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to backup and/or to restore is located on a remote system. Currently the only supported remote protocol is SSH. -#### version +#### Setup SSH - pg_probackup version +If you are going to use pg_probackup in remote mode via ssh, complete the following steps: -Prints pg_probackup version. +- Install pg_probackup on both systems: `backup_host` and `db_host`. +- For communication between the hosts setup the passwordless SSH connection between *backup* user on `backup_host` and *postgres* user on 'db_host': -#### help + [backup@backup_host] ssh-copy-id postgres@db_host - pg_probackup help [command] +- If you planning to rely on [continuous WAL archiving](#setting-up-continuous-wal-archiving), then setup passwordless SSH connection between *postgres* user on `db_host` and *backup* user on `backup_host`: -Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. + [postgres@db_host] ssh-copy-id backup@backup_host -#### init +Where: - pg_probackup init -B backup_dir [--help] +- *backup_host* is the system with *backup catalog*. +- *db_host* is the system with PostgreSQL cluster. +- *backup* is the OS user on *backup_host* used to run pg_probackup. +- *postgres* is the OS user on *db_host* used to run PostgreSQL cluster. Note, that for PostgreSQL versions >= 11, a more secure approach can used thanks to [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature. -Initializes the backup catalog in *backup_dir* that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified *backup_dir* already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. +pg_probackup in remote mode via `ssh` works as follows: -For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). +- only the following commands can be launched in remote mode: [add-instance](#add-instance), [backup](#backup), [restore](#restore), [archive-push](#archive-push), [archive-get](#archive-get). +- when started in remote mode the main pg_probackup process on local system connects via ssh to remote system and launches there number of agent proccesses equal to specified value of option `-j/--threads`. +- the main pg_probackup process use remote agents to access remote files and transfer data between local and remote systems. +- remote agents are smart and capable of handling some logic on their own to minimize the network traffic and number of round-trips between hosts. +- usually the main proccess is started on *backup_host* and connects to *db_host*, but in case of `archive-push` and `archive-get` commands the main process is started on *db_host* and connects to *backup_host*. +- after completition of data transfer the remote agents are terminated and ssh connections are closed. +- if an error condition is encountered by a remote agent, then all agents are terminated and error details are reported by the main pg_probackup process, which exits with error. +- compression is always done on *db_host*. +- decompression is always done on *backup_host*. -#### add-instance +>NOTE: You can improse [additional restrictions](https://man.openbsd.org/OpenBSD-current/man8/sshd.8#AUTHORIZED_KEYS_FILE_FORMAT) on ssh settings to protect the system in the event of account compromise. - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name - [--help] [--external-dirs=external_directory_path] +## Usage -Initializes a new backup instance inside the backup catalog *backup_dir* and generates the pg_probackup.conf configuration file that controls pg_probackup settings for the cluster with the specified *data_dir* data directory. +### Creating a Backup -For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). +To create a backup, run the following command: -#### del-instance + pg_probackup backup -B backup_dir --instance instance_name -b backup_mode - pg_probackup del-instance -B backup_dir --instance instance_name - [--help] +Where *backup_mode* can take one of the following values: -Deletes all backups and WAL files associated with the specified instance. +- FULL — creates a full backup that contains all the data files of the cluster to be restored. +- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. +- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. +- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. -#### set-config +When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `the backup chain`. You must create at least one full backup before taking incremental ones. - pg_probackup set-config -B backup_dir --instance instance_name - [--help] [--pgdata=pgdata-path] - [--retention-redundancy=redundancy][--retention-window=window] - [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] - [-d dbname] [-h host] [-p port] [-U username] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [remote_options] [logging_options] +#### ARCHIVE mode -Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. +ARCHIVE is the default WAL delivery mode. -It is **not recommended** to edit pg_probackup.conf manually. +For example, to make a FULL backup in ARCHIVE mode, run: -#### show-config + pg_probackup backup -B backup_dir --instance instance_name -b FULL - pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] +Unlike backup in STREAM mode, ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. -Displays the contents of the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. +During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. -To edit pg_probackup.conf, use the [set-config](#set-config) command. +#### STREAM mode -#### show +STREAM is the optional WAL delivery mode. - pg_probackup show -B backup_dir - [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] +For example, to make a FULL backup in STREAM mode, add the `--stream` flag to the command from the previous example: -Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. + pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot -By default, the contents of the backup catalog is shown as plain text. +The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. -#### backup +Unlike backup in ARCHIVE mode, STREAM backup include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - pg_probackup backup -B backup_dir -b backup_mode --instance instance_name - [--help] [-j num_threads] [--progress] - [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] - [--no-validate] [--skip-block-validation] - [-w --no-password] [-W --password] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [connection_options] [compression_options] [remote_options] - [retention_options] [logging_options] +During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN to '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. -Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. +Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: -For details, see the sections [Backup Options](#backup-options) and [Creating a Backup](#creating-a-backup). +- STREAM backups can be restored on the server that has no file access to WAL archive. +- STREAM backups enable you to restore the cluster state at the point in time for which WAL files in archive are no longer available. +- Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. -#### restore +#### Page validation - pg_probackup restore -B backup_dir --instance instance_name - [--help] [-D data_dir] [-i backup_id] - [-j num_threads] [--progress] - [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] - [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] - [recovery_options] [logging_options] [remote_options] +If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. +Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. -Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a recovery target option, pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. +Page is considered corrupted if checksumm comparison failed more than 100 times, in this case backup is aborted. -For details, see the sections [Restore Options](#restore-options), [Recovery Target Options](#recovery-target-options) and [Restoring a Cluster](#restoring-a-cluster). +Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". -#### checkdb +#### External directories - pg_probackup checkdb - [-B backup_dir] [--instance instance_name] [-D data_dir] - [--help] [-j num_threads] [--progress] - [--skip-block-validation] [--amcheck] [--heapallindexed] - [connection_options] [logging_options] +To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. -Verifyes the PostgreSQL database cluster correctness by detecting physical and logical corruption. +For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: -For details, see the sections [Checkdb Options](#checkdb-options) and [Verifying a Cluster](#verifying-a-cluster). + pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 -#### validate +pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. - pg_probackup validate -B backup_dir - [--help] [--instance instance_name] [-i backup_id] - [-j num_threads] [--progress] - [--skip-block-validation] - [recovery_options] [logging_options] +To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. -Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target option](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. +### Verifying a Cluster -For details, see the section [Validating a Backup](#validating-a-backup). +To verify that PostgreSQL database cluster is free of corruption, run the following command: -#### merge + pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - [--help] [-j num_threads][--progress] - [logging_options] +This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: -Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. +- `checkdb` is read-only +- if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated +- `checkdb` do not strictly require *the backup catalog*, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). -For details, see the section [Merging Backups](#merging-backups). +If *backup_dir* and *instance_name* are omitted, then [connection options](#connection-options) and *data_dir* must be provided via environment variables or command-line options. -#### delete +Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. +Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. - pg_probackup delete -B backup_dir --instance instance_name - [--help] [-j num_threads] [--progress] - [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} - [--dry-run] - [logging_options] +If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` flag when running [checkdb](#checkdb) command: -Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. + pg_probackup checkdb -D data_dir --amcheck -For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-otions) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +Physical verification can be skipped if `--skip-block-validation` flag is used. For logical only verification *backup_dir* and *data_dir* are optional, only [connection options](#connection-options) are mandatory: -#### archive-push + pg_probackup checkdb --amcheck --skip-block-validation {connection_options} - pg_probackup archive-push -B backup_dir --instance instance_name - --wal-file-path %p --wal-file-name %f - [--help] [--compress] [--compress-algorithm=compression_algorithm] - [--compress-level=compression_level] [--overwrite] - [remote_options] [logging_options] +Logical verification can be done more thoroughly with flag `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by *instance_name* and *system-identifier*. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. +### Validating Backups -If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` option. +pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `the backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruption. -Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. -Copied to archive WAL segments are synced to disk. +If you would like to skip backup validation, you can specify the `--no-validate` flag when running [backup](#backup) and [restore](#restore) commands. -You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). +To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact [recovery target options](#recovery-target-options) you are going to use for recovery. -For details, see sections [Archiving Options](#archiving-options) and [Compression Options](#compression-options). +For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: -#### archive-get + pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 - pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path %p --wal-file-name %f - [--help] [remote_options] [logging_options] +If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. -Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. +If you specify *backup_id* via `-i/--backup-id` option, then only backup copy with specified backup ID will be validated. If *backup_id* is specified with [recovery target options](#recovery-target-options) then validate will check whether it is possible to restore the specified backup to the specified `recovery target`. -### Options +For example, to check that you can restore the database cluster from a backup copy with *backup_id* up to the specified timestamp, run this command: -This section describes all command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. + pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' -For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). +If *backup_id* belong to incremental backup, then all its parents starting from FULL backup will be validated. -If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. +If you omit all the parameters, all backups are validated. -#### Common Options -The list of general options. +### Restoring a Cluster - -B directory - --backup-path=directory - BACKUP_PATH -Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. +To restore the database cluster from a backup, run the restore command with at least the following options: - -D directory - --pgdata=directory - PGDATA -Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the [add-instance](#add-instance) command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. + pg_probackup restore -B backup_dir --instance instance_name -i backup_id - -i backup_id - -backup-id=backup_id -Specifies the unique identifier of the backup. +Where: - -j num_threads - --threads=num_threads -Sets the number of parallel threads for backup, restore, merge, validation and verification processes. +- *backup_dir* is the backup catalog that stores all backup files and meta information. +- *instance_name* is the backup instance for the cluster to be restored. +- *backup_id* specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. - --progress -Shows the progress of operations. +If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. - --help -Shows detailed information about the options that can be used with this command. +When using the `--tablespace-mapping/-T` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: -#### Backup Options + pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir -The following options can be used together with the [backup](#backup) command. +Once the restore command is complete, start the database service. -Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. +If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. - -b mode - --backup-mode=mode +>NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` flag to skip validation and speed up the recovery. -Specifies the backup mode to use. Possible values are: +### Performing Point-in-Time (PITR) Recovery -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. +If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. -For details, see the section [Creating a Backup](#creating-a-backup). +If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore *backup_id* to the specified recovery target. - -C - --smooth-checkpoint - SMOOTH_CHECKPOINT -Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. +- To restore the cluster state at the exact time, specify the `--recovery-target-time` option, in the timestamp format. For example: - --stream -Makes an STREAM backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' - -S slot_name - --slot=slot_name -Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` option. +- To restore the cluster state up to a specific transaction ID, use the `--recovery-target-xid` option: - --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This option can only be used together with the `--stream` option. Default slot name is `pg_probackup_slot`, which can be changed via option `--slot/-S`. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 - --backup-pg-log -Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. +- To restore the cluster state up to a specific LSN, use `--recovery-target-lsn` option: - -E external_directory_path - --external-dirs=external_directory_path -Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 - --archive-timeout=wait_time -Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. +- To restore the cluster state up to a specific named restore point, use `--recovery-target-name` option: - --skip-block-validation -Disables block-level checksum verification to speed up backup. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' - --no-validate -Skips automatic validation after successfull backup. You can use this option if you validate backups regularly and would like to save time when running backup operations. +- To restore the backup to the latest state available in archive, use `--recovery-target` option with `latest` value: -#### Restore Options + pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' -The following options can be used together with the [restore](#restore) command. +- To restore the cluster to the earliest point of consistency, use `--recovery-target` option with `immediate` value: -Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. + pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' - -R | --restore-as-replica -Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. +### Using pg_probackup in the Remote Mode - -T OLDDIR=NEWDIR - --tablespace-mapping=OLDDIR=NEWDIR +pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. -Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. +Do note that pg_probackup rely on passwordless SSH connection for communication between the hosts. - --external-mapping=OLDDIR=NEWDIR -Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. +The typical workflow is as follows: - --skip-external-dirs -Skip external directories included into the backup with the `--external-dirs` option. The contents of these directories will not be restored. + - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. - --skip-block-validation -Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. +- If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery mode, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. - --no-validate -Skips backup validation. You can use this option if you validate backups regularly and would like to save time when running restore operations. +- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. -#### Checkdb Options +>NOTE: The remote backup mode is currently unavailable for Windows systems. -The following options can be used together with the [checkdb](#checkdb) command. +### Running pg_probackup on Parallel Threads -For details on verifying PostgreSQL database cluster, see section [Verifying a Cluster](#verifying-a-cluster). +[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk and network bandwidth). - --amcheck -Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. +Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: - --skip-block-validation -Skip validation of data files. Can be used only with `--amcheck` option, so only logical verification of indexes is performed. + pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 - --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this option only together with the `--amcheck` option. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. +>NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. -#### Recovery Target Options +### Configuring pg_probackup -If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. +Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. - --recovery-target=immediate|latest -Defines when to stop the recovery: +For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. -- `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. -- `latest` value continues the recovery until all WAL segments available in the archive are applied. +>NOTE: It is **not recommended** to edit pg_probackup.conf manually. -Default value of `--recovery-target` depends on WAL delivery method of restored backup, `immediate` for STREAM backup and `latest` for ARCHIVE. +Initially, pg_probackup.conf contains the following settings: - --recovery-target-timeline=timeline -Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. +- PGDATA — the path to the data directory of the cluster to back up. +- system-identifier — the unique identifier of the PostgreSQL instance. - --recovery-target-lsn=lsn -Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring database cluster of major version 10 or higher. +Additionally, you can define [remote](#remote-mode-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: - --recovery-target-name=recovery_target_name -Specifies a named savepoint up to which to restore the cluster data. + pg_probackup set-config -B backup_dir --instance instance_name + [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] - --recovery-target-time=time -Specifies the timestamp up to which recovery will proceed. +To view the current settings, run the following command: - --recovery-target-xid=xid -Specifies the transaction ID up to which recovery will proceed. + pg_probackup show-config -B backup_dir --instance instance_name - --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default depends on [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. +You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. - --recovery-target-action=pause|promote|shutdown - Default: pause -Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. +### Specifying Connection Settings -#### Retention Options -You can use these options together with [backup](#backup) and [delete](#delete) commands. +If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. -For details on configuring retention policy, see the section [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. - --retention-redundancy=redundancy - Default: 0 -Specifies the number of full backup copies to keep in the data directory. Must be a positive integer. The zero value disables this setting. +### Managing the Backup Catalog - --retention-window=window - Default: 0 -Number of days of recoverability. Must be a positive integer. The zero value disables this setting. +With pg_probackup, you can manage backups from the command line: - --delete-wal -Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. +- View available backups +- Validate backups +- Merge backups +- Delete backups +- Viewing Backup Information - --delete-expired -Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. +To view the list of existing backups for every instance, run the command: - --merge-expired -Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. + pg_probackup show -B backup_dir - --dry-run -Displays the current status of all the available backups, without deleting or merging expired backups, if any. +pg_probackup displays the list of all the available backups. For example: -#### Logging Options -You can use these options with any command. +``` +BACKUP INSTANCE 'node' +============================================================================================================================================ + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +============================================================================================================================================ + node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 1 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 1 21s 32MB 0/13000028 0/13000198 OK + node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1 / 1 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1 / 0 11s 39MB 0/F000028 0/F000198 OK +``` - --log-level-console=log_level - Default: info -Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. +For each backup, the following information is provided: ->NOTE: all console log messages are going to stderr, so output from [show](#show) and [show-config](#show-config) commands do not mingle with log messages. +- Instance — the instance name. +- Version — PostgreSQL major version. +- ID — the backup identifier. +- Recovery time — the earliest moment for which you can restore the state of the database cluster. +- Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. +- WAL — the WAL delivery mode. Possible values: STREAM and ARCHIVE. +- Current/Parent TLI — timeline identifiers of current backup and its parent. +- Time — the time it took to perform the backup. +- Data — the size of the data files in this backup. This value does not include the size of WAL files. +- Start LSN — WAL log sequence number corresponding to the start of the backup process. +- Stop LSN — WAL log sequence number corresponding to the end of the backup process. +- Status — backup status. Possible values: - --log-level-file=log_level - Default: off -Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. + - OK — the backup is complete and valid. + - DONE — the backup is complete, but was not validated. + - RUNNING — the backup is in progress. + - MERGING — the backup is being merged. + - DELETING — the backup files are being deleted. + - CORRUPT — some of the backup files are corrupted. + - ERROR — the backup was aborted because of an unexpected error. + - ORPHAN — the backup is invalid because one of its parent backups is corrupt or missing. - --log-filename=log_filename - Default: pg_probackup.log -Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in *log_filename*. +You can restore the cluster from the backup only if the backup status is OK or DONE. -For example, if you specify the 'pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. +To get more detailed information about the backup, run the show with the backup ID: -This option takes effect if file logging is enabled by the `log-level-file` option. + pg_probackup show -B backup_dir --instance instance_name -i backup_id - --error-log-filename=error_log_filename - Default: none -Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames, as explained in *error_log_filename*. +The sample output is as follows: -For example, if you specify the 'error-pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. +``` +#Configuration +backup-mode = FULL +stream = false +compress-alg = zlib +compress-level = 1 +from-replica = false -This option is useful for troubleshooting and monitoring. +#Compatibility +block-size = 8192 +wal-block-size = 8192 +checksum-version = 1 +program-version = 2.1.3 +server-version = 10 - --log-directory=log_directory - Default: $BACKUP_PATH/log/ -Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. +#Result backup info +timelineid = 1 +start-lsn = 0/04000028 +stop-lsn = 0/040000f8 +start-time = '2017-05-16 12:57:29' +end-time = '2017-05-16 12:57:31' +recovery-xid = 597 +recovery-time = '2017-05-16 12:57:31' +data-bytes = 22288792 +wal-bytes = 16777216 +status = OK +parent-backup-id = 'PT8XFX' +primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' +``` - --log-rotation-size=log_rotation_size - Default: 0 -Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). +To get more detailed information about the backup in json format, run the show with the backup ID: - --log-rotation-age=log_rotation_age - Default: 0 -Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). + pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id -#### Connection Options -You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. +The sample output is as follows: -All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. +``` +[ + { + "instance": "node", + "backups": [ + { + "id": "PT91HZ", + "parent-backup-id": "PT8XFX", + "backup-mode": "DELTA", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": false, + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.3", + "server-version": "10", + "current-tli": 16, + "parent-tli": 2, + "start-lsn": "0/8000028", + "stop-lsn": "0/8000160", + "start-time": "2019-06-17 18:25:11+03", + "end-time": "2019-06-17 18:25:16+03", + "recovery-xid": 0, + "recovery-time": "2019-06-17 18:25:15+03", + "data-bytes": 106733, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } +] +``` - -d dbname - --dbname=dbname - PGDATABASE -Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. +### Configuring Backup Retention Policy - -h host - --host=host - PGHOST - Default: local socket -Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. +By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. - -p port - --port=port - PGPORT - Default: 5432 -Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. +To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): - -U username - --username=username - PGUSER -User name to connect as. + --retention-redundancy=redundancy +Specifies **the number of full backup copies** to keep in the backup catalog. - -w - --no-password - Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file or PGPASSWORD environment variable, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. + --retention-window=window +Defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in **the number of days** from the current moment. For example, if `retention-window=7`, pg_probackup must delete all backup copies that are older than seven days, with all the corresponding WAL files. - -W - --password -Forces a password prompt. +If both `--retention-redundancy` and `--retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `--retention-redundancy=2` and `--retention-window=7`, pg_probackup purges the backup catalog to keep only two full backup copies and all backups that are newer than seven days. -#### Compression Options +To clean up the backup catalog in accordance with retention policy, run: -You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. + pg_probackup delete -B backup_dir --instance instance_name --delete-expired - --compress-algorithm=compression_algorithm - Default: none -Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. -For the [archive-push](#archive-push) command, the pglz compression algorithm is not supported. +pg_probackup deletes all backup copies that do not conform to the defined retention policy. - --compress-level=compression_level - Default: 1 -Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with `--compress-algorithm` option. +If you would like to also remove the WAL files that are no longer required for any of the backups, add the `--delete-wal` flag: - --compress -Alias for `--compress-algorithm=zlib` and `--compress-level=1`. + pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal -#### Archiving Options +>NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. -These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. +Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` flag when running [backup](#backup) or [delete](#delete) commands. -Additionally [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. +Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `--retention-window` option is set to *7*, and you have the following backups available on April 10, 2019: + +``` +BACKUP INSTANCE 'node' +=========================================================================================================================================== + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +=========================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK + ---------------------------------------retention window------------------------------------------------------------- + node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK + node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK + node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK +``` - --wal-file-path=wal_file_path %p -Provides the path to the WAL file in `archive_command` and `restore_command`. The %p variable is required for correct processing. +Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` flag, only the P7XDFT full backup will be removed. - --wal-file-name=wal_file_name %f -Provides the name of the WAL file in `archive_command` and `restore_command`. The %f variable is required for correct processing. +With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: - --overwrite -Overwrites archived WAL file. Use this option together with the archive-push command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` option. + pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired + pg_probackup show -B backup_dir -#### Remote Mode Options +``` +BACKUP INSTANCE 'node' +============================================================================================================================================ + Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status +============================================================================================================================================ + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK +``` -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. +>NOTE: The Time field for the merged backup displays the time required for the merge. -For details on configuring remote operation mode, see the section [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). +### Merging Backups - --remote-proto -Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: +As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: -- `ssh` enables the remote backup mode via SSH. This is the Default value. -- `none` explicitly disables the remote mode. + pg_probackup merge -B backup_dir --instance instance_name -i backup_id -You can omit this option if the `--remote-host` option is specified. +This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes, I/O and network traffic in case of [remote](#using-pg_probackup-in-the-remote-mode) backup. - --remote-host -Specifies the remote host IP address or hostname to connect to. +Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the [show](#show) command with the backup ID: - --remote-port - Default: 22 -Specifies the remote host port to connect to. + pg_probackup show -B backup_dir --instance instance_name -i backup_id - --remote-user - Default: current user -Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. +If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. - --remote-path -Specifies pg_probackup installation directory on the remote system. +### Deleting Backups - --ssh-options -Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found here: (https://linux.die.net/man/5/ssh_config)[https://linux.die.net/man/5/ssh_config] +To delete a backup that is no longer required, run the following command: -#### Replica Options + pg_probackup delete -B backup_dir --instance instance_name -i backup_id -This section describes the options related to taking a backup from standby. +This command will delete the backup with the specified *backup_id*, together with all the incremental backups that descend from *backup_id* if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. ->NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. +To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` flag: - --master-db=dbname - Default: postgres, the default PostgreSQL database. -Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the [set-config](#set-config) command. + pg_probackup delete -B backup_dir --instance instance_name --delete-wal - --master-host=host -Deprecated. Specifies the host name of the system on which the master server is running. +To delete backups that are expired according to the current retention policy, use the `--delete-expired` flag: - --master-port=port - Default: 5432, the PostgreSQL default port. -Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. + pg_probackup delete -B backup_dir --instance instance_name --delete-expired - --master-user=username - Default: postgres, the PostgreSQL default user name. -Deprecated. User name to connect as. +Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the `--merge-expired` flag when running this command: - --replica-timeout=timeout - Default: 300 sec -Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. + pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired + +In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. + +Before merging or deleting backups, you can run the `delete` command with the `--dry-run` flag, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. + +## Command-Line Reference +### Commands + +This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). + +#### version + + pg_probackup version + +Prints pg_probackup version. + +#### help + + pg_probackup help [command] + +Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. + +#### init + + pg_probackup init -B backup_dir [--help] + +Initializes the backup catalog in *backup_dir* that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified *backup_dir* already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. + +For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). -## Usage +#### add-instance -- [Creating a Backup](#creating-a-backup) -- [Verifying a Cluster](#verifying-a-cluster) -- [Validating a Backup](#vaklidating-a-backup) -- [Restoring a Cluster](#restoring-a-cluster) -- [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) -- [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) -- [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) -- [Configuring pg_probackup](#configuring-pg_probackup) -- [Managing the Backup Catalog](#managing-the-backup-catalog) -- [Configuring Backup Retention Policy](#configuring-backup-retention-policy) -- [Merging Backups](#merging-backups) -- [Deleting Backups](#deleting-backups) + pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name + [--help] -### Creating a Backup +Initializes a new backup instance inside the backup catalog *backup_dir* and generates the pg_probackup.conf configuration file that controls pg_probackup settings for the cluster with the specified *data_dir* data directory. -To create a backup, run the following command: +For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). - pg_probackup backup -B backup_dir --instance instance_name -b backup_mode +#### del-instance -Where *backup_mode* can take one of the following values: + pg_probackup del-instance -B backup_dir --instance instance_name + [--help] -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. +Deletes all backups and WAL files associated with the specified instance. -When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `backup chain`. You must create at least one full backup before taking incremental ones. +#### set-config -#### ARCHIVE mode + pg_probackup set-config -B backup_dir --instance instance_name + [--help] [--pgdata=pgdata-path] + [--retention-redundancy=redundancy][--retention-window=window] + [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] + [-d dbname] [-h host] [-p port] [-U username] + [--archive-timeout=timeout] [--external-dirs=external_directory_path] + [remote_options] [logging_options] -ARCHIVE is the default WAL delivery mode. +Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. -For example, to make a FULL backup in ARCHIVE mode, run: +For all available settings, see the [Options](#options) section. - pg_probackup backup -B backup_dir --instance instance_name -b FULL +It is **not recommended** to edit pg_probackup.conf manually. -Unlike backup in STREAM mode, ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. +#### show-config -During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. + pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] -#### STREAM mode +Displays the contents of the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. -STREAM is the optional WAL delivery mode. +To edit pg_probackup.conf, use the [set-config](#set-config) command. -For example, to make a FULL backup in STREAM mode, add the `--stream` option to the command from the previous example: +#### show - pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot + pg_probackup show -B backup_dir + [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] -The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. +Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. -Unlike backup in ARCHIVE mode, STREAM backup include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. +By default, the contents of the backup catalog is shown as plain text. -During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN in '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. +#### backup -Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: + pg_probackup backup -B backup_dir -b backup_mode --instance instance_name + [--help] [-j num_threads] [--progress] + [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] + [--no-validate] [--skip-block-validation] + [-w --no-password] [-W --password] + [--archive-timeout=timeout] [--external-dirs=external_directory_path] + [connection_options] [compression_options] [remote_options] + [retention_options] [logging_options] -- STREAM backups can be restored on the server that has no file access to WAL archive. -- STREAM backups enable you to restore the cluster state at the point in time for which WAL files are no longer available. -- Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. +Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. -#### Page validation + -b mode + --backup-mode=mode -If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. -Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. +Specifies the backup mode to use. Possible values are: -Page is considered corrupted if checksumm comparison failed more than 100 times, in this case backup is aborted. +- FULL — creates a full backup that contains all the data files of the cluster to be restored. +- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. +- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. +- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. -Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". + -C + --smooth-checkpoint +Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. -#### External directories + --stream +Makes an [STREAM](#stream-mode) backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. -To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. + --temp-slot +Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This flag can only be used together with the `--stream` flag. Default slot name is `pg_probackup_slot`, which can be changed via option `--slot/-S`. -For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: + -S slot_name + --slot=slot_name +Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` flag. - pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 + --backup-pg-log +Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. -pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. + -E external_directory_path + --external-dirs=external_directory_path +Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. -To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. + --archive-timeout=wait_time +Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. -### Verifying a Cluster + --skip-block-validation +Disables block-level checksum verification to speed up backup. -To verify that PostgreSQL database cluster is free of corruption, run the following command: + --no-validate +Skips automatic validation after successfull backup. You can use this flag if you validate backups regularly and would like to save time when running backup operations. - pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] +Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: +For details on usage, see the section (#creating-a-backup). -- `checkdb` is read-only -- if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated -- `checkdb` do not strictly require the backup catalog, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). +#### restore -If *backup_dir* and *instance_name* are omitted, then [connection options](#connection-options) and *data_dir* must be provided via environment variables or command-line options. + pg_probackup restore -B backup_dir --instance instance_name + [--help] [-D data_dir] [-i backup_id] + [-j num_threads] [--progress] + [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] + [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + [recovery_options] [logging_options] [remote_options] -Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. -Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. +Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a [recovery target option](#recovery-target-options), pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. -If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` option when running [checkdb](#checkdb) command: + -R | --restore-as-replica +Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. - pg_probackup checkdb -D data_dir --amcheck + -T OLDDIR=NEWDIR + --tablespace-mapping=OLDDIR=NEWDIR -Physical verification can be skipped if `--skip-block-validation` option is used. For logical only verification *backup_dir* and *data_dir* are optional, only [connection options](#connection-options) are mandatory: +Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. - pg_probackup checkdb --amcheck --skip-block-validation [connection options] + --external-mapping=OLDDIR=NEWDIR +Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. -Logical verification can be done more thoroughly with option `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. + --skip-external-dirs +Skip external directories included into the backup with the `--external-dirs` option. The contents of these directories will not be restored. -### Validating Backups + --skip-block-validation +Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. -pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruptions. + --no-validate +Skips backup validation. You can use this flag if you validate backups regularly and would like to save time when running restore operations. -If you would like to skip backup validation, you can specify the `--no-validate` option when running [backup](#backup) and [restore](#restore) commands. +Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact recovery target options you are going to use for recovery. +For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). -For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: +#### checkdb - pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=xid + pg_probackup checkdb + [-B backup_dir] [--instance instance_name] [-D data_dir] + [--help] [-j num_threads] [--progress] + [--skip-block-validation] [--amcheck] [--heapallindexed] + [connection_options] [logging_options] -If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. +Verifies the PostgreSQL database cluster correctness by detecting physical and logical corruption. -If you specify *backup_id* via `-i/--backup-id` option, then only backup copy with specified backup ID will be validated. If *backup_id* belong to incremental backup, then all its parents starting from FULL backup will be validated. + --amcheck +Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. -If you omit all the parameters, all backups are validated. + --skip-block-validation +Skip validation of data files. Can be used only with `--amcheck` flag, so only logical verification of indexes is performed. -### Restoring a Cluster + --heapallindexed +Checks that all heap tuples that should be indexed are actually indexed. You can use this flag only together with the `--amcheck` flag. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. -To restore the database cluster from a backup, run the restore command with at least the following options: +Additionally [Connection Options](#connection-options) and [Logging Options](#logging-options) can be used. - pg_probackup restore -B backup_dir --instance instance_name -i backup_id +For details on usage, see the section [Verifying a Cluster](#verifying-a-cluster). -Where: +#### validate -- *backup_dir* is the backup catalog that stores all backup files and meta information. -- *instance_name* is the backup instance for the cluster to be restored. -- *backup_id* specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. + pg_probackup validate -B backup_dir + [--help] [--instance instance_name] [-i backup_id] + [-j num_threads] [--progress] + [--skip-block-validation] + [recovery_options] [logging_options] -If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. +Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target option](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. -When using the `--tablespace-mapping/-T` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: +For details, see the section [Validating a Backup](#validating-a-backup). - pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir +#### merge -Once the restore command is complete, start the database service. + pg_probackup merge -B backup_dir --instance instance_name -i backup_id + [--help] [-j num_threads] [--progress] + [logging_options] -If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. +Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. ->NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` option to skip validation and speed up the recovery. +For details, see the section [Merging Backups](#merging-backups). -### Performing Point-in-Time (PITR) Recovery +#### delete -If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. + pg_probackup delete -B backup_dir --instance instance_name + [--help] [-j num_threads] [--progress] + [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} + [--dry-run] + [logging_options] -If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore *backup_id* to the specified recovery target. +Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. -- To restore the cluster state at the exact time, specify the recovery-target-time option, in the timestamp format. For example: +For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-options) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' +#### archive-push -- To restore the cluster state up to a specific transaction ID, use the recovery-target-xid option: + pg_probackup archive-push -B backup_dir --instance instance_name + --wal-file-path=wal_file_path --wal-file-name=wal_file_name + [--help] [--compress] [--compress-algorithm=compression_algorithm] + [--compress-level=compression_level] [--overwrite] + [remote_options] [logging_options] - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 +Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by *instance_name* and *system-identifier*. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. -- To restore the cluster state up to a specific LSN, use recovery-target-lsn: +If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` flag. - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 +Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. +Copied to archive WAL segments are synced to disk. -- To restore the cluster state up to a specific named restore point, use recovery-target-name: +You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' +For details, see sections [Archiving Options](#archiving-options) and [Compression Options](#compression-options). -- To restore the cluster to the latest state available in archive, use recovery-target option: +#### archive-get - pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' + pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name + [--help] [remote_options] [logging_options] -### Using pg_probackup in the Remote Mode +Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. -pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. +### Options -Do note that pg_probackup rely on passwordless SSH connection for communication between the hosts. +This section describes command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. -The typical workflow is as follows: +For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). - - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. +If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. -- If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery method, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. +#### Common Options +The list of general options. -- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. + -B directory + --backup-path=directory + BACKUP_PATH +Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. ->NOTE: The remote backup mode is currently unavailable for Windows systems. + -D directory + --pgdata=directory + PGDATA +Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the [add-instance](#add-instance) command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. -### Running pg_probackup on Parallel Threads + -i backup_id + -backup-id=backup_id +Specifies the unique identifier of the backup. -[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network throughput). + -j num_threads + --threads=num_threads +Sets the number of parallel threads for backup, restore, merge, validation and verification processes. -Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: + --progress +Shows the progress of operations. - pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 + --help +Shows detailed information about the options that can be used with this command. ->NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. +#### Recovery Target Options -### Configuring pg_probackup +If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. + --recovery-target=immediate|latest +Defines when to stop the recovery: -For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. +- `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. +- `latest` value continues the recovery until all WAL segments available in the archive are applied. ->NOTE: It is **not recommended** to edit pg_probackup.conf manually. +Default value of `--recovery-target` depends on WAL delivery method of restored backup, `immediate` for STREAM backup and `latest` for ARCHIVE. -Initially, pg_probackup.conf contains the following settings: + --recovery-target-timeline=timeline +Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. -- PGDATA — the path to the data directory of the cluster to back up. -- system-identifier — the unique identifier of the PostgreSQL instance. + --recovery-target-lsn=lsn +Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring database cluster of major version 10 or higher. -Additionally, you can define [remote](#remote-mode-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: + --recovery-target-name=recovery_target_name +Specifies a named savepoint up to which to restore the cluster data. - pg_probackup set-config -B backup_dir --instance instance_name - [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] + --recovery-target-time=time +Specifies the timestamp up to which recovery will proceed. -To view the current settings, run the following command: + --recovery-target-xid=xid +Specifies the transaction ID up to which recovery will proceed. - pg_probackup show-config -B backup_dir --instance instance_name + --recovery-target-inclusive=boolean +Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default depends on [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. -You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. + --recovery-target-action=pause|promote|shutdown + Default: pause +Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. -### Specifying Connection Settings +#### Retention Options -If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. +You can use these options together with [backup](#backup) and [delete](#delete) commands. -If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. +For details on configuring retention policy, see the sections [Configuring Backup Retention Policy](#configuring-backup-retention-policy). -### Managing the Backup Catalog + --retention-redundancy=redundancy + Default: 0 +Specifies the number of full backup copies to keep in the data directory. Must be a positive integer. The zero value disables this setting. -With pg_probackup, you can manage backups from the command line: + --retention-window=window + Default: 0 +Number of days of recoverability. Must be a positive integer. The zero value disables this setting. -- View available backups -- Validate backups -- Merge backups -- Delete backups -- Viewing Backup Information + --delete-wal +Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. -To view the list of existing backups for every instance, run the command: + --delete-expired +Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. - pg_probackup show -B backup_dir + --merge-expired +Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. -pg_probackup displays the list of all the available backups. For example: + --dry-run +Displays the current status of all the available backups, without deleting or merging expired backups, if any. -``` -BACKUP INSTANCE 'node' -============================================================================================================================================ - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -============================================================================================================================================ - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 1 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 1 21s 32MB 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1 / 1 31s 33MB 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1 / 0 11s 39MB 0/F000028 0/F000198 OK -``` +#### Logging Options -For each backup, the following information is provided: +You can use these options with any command. -- Instance — the instance name. -- Version — PostgreSQL major version. -- ID — the backup identifier. -- Recovery time — the earliest moment for which you can restore the state of the database cluster. -- Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. -- WAL — the WAL delivery method. Possible values: STREAM and ARCHIVE. -- Current/Parent TLI — timeline identifiers of current backup and its parent. -- Time — the time it took to perform the backup. -- Data — the size of the data files in this backup. This value does not include the size of WAL files. -- Start LSN — WAL log sequence number corresponding to the start of the backup process. -- Stop LSN — WAL log sequence number corresponding to the end of the backup process. -- Status — backup status. Possible values: + --log-level-console=log_level + Default: info +Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. - - OK — the backup is complete and valid. - - DONE — the backup is complete, but was not validated. - - RUNNING — the backup is in progress. - - MERGING — the backup is being merged. - - DELETING — the backup files are being deleted. - - CORRUPT — some of the backup files are corrupted. - - ERROR — the backup was aborted because of an unexpected error. - - ORPHAN — the backup is invalid because one of its parent backups is corrupt or missing. +>NOTE: all console log messages are going to stderr, so output from [show](#show) and [show-config](#show-config) commands do not mingle with log messages. -You can restore the cluster from the backup only if the backup status is OK or DONE. + --log-level-file=log_level + Default: off +Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. -To get more detailed information about the backup, run the show with the backup ID: + --log-filename=log_filename + Default: pg_probackup.log +Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. - pg_probackup show -B backup_dir --instance instance_name -i backup_id +For example, if you specify the 'pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. -The sample output is as follows: +This option takes effect if file logging is enabled by the `log-level-file` option. -``` -#Configuration -backup-mode = FULL -stream = false -compress-alg = zlib -compress-level = 1 -from-replica = false + --error-log-filename=error_log_filename + Default: none +Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. -#Compatibility -block-size = 8192 -wal-block-size = 8192 -checksum-version = 1 -program-version = 2.1.3 -server-version = 10 +For example, if you specify the 'error-pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. -#Result backup info -timelineid = 1 -start-lsn = 0/04000028 -stop-lsn = 0/040000f8 -start-time = '2017-05-16 12:57:29' -end-time = '2017-05-16 12:57:31' -recovery-xid = 597 -recovery-time = '2017-05-16 12:57:31' -data-bytes = 22288792 -wal-bytes = 16777216 -status = OK -parent-backup-id = 'PT8XFX' -primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' -``` +This option is useful for troubleshooting and monitoring. -To get more detailed information about the backup in json format, run the show with the backup ID: + --log-directory=log_directory + Default: $BACKUP_PATH/log/ +Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. - pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id + --log-rotation-size=log_rotation_size + Default: 0 +Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). -The sample output is as follows: + --log-rotation-age=log_rotation_age + Default: 0 +Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). -``` -[ - { - "instance": "node", - "backups": [ - { - "id": "PT91HZ", - "parent-backup-id": "PT8XFX", - "backup-mode": "DELTA", - "wal": "ARCHIVE", - "compress-alg": "zlib", - "compress-level": 1, - "from-replica": false, - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.3", - "server-version": "10", - "current-tli": 16, - "parent-tli": 2, - "start-lsn": "0/8000028", - "stop-lsn": "0/8000160", - "start-time": "2019-06-17 18:25:11+03", - "end-time": "2019-06-17 18:25:16+03", - "recovery-xid": 0, - "recovery-time": "2019-06-17 18:25:15+03", - "data-bytes": 106733, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } -] -``` +#### Connection Options -### Configuring Backup Retention Policy +You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. -By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. +All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. -To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): + -d dbname + --dbname=dbname + PGDATABASE +Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. -- retention-redundancy — specifies the number of full backup copies to keep in the backup catalog. -- retention-window — defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in the number of days from the current moment. For example, if `retention-window=7`, pg_probackup must keep at least one full backup copy that is older than seven days, with all the corresponding WAL files. + -h host + --host=host + PGHOST + Default: local socket +Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. -If both `retention-redundancy` and `retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `retention-redundancy=2` and `retention-window=7`, pg_probackup cleans up the backup directory to keep only two full backup copies and all backups that are newer than seven days. + -p port + --port=port + PGPORT + Default: 5432 +Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. -To clean up the backup catalog in accordance with retention policy, run: + -U username + --username=username + PGUSER +User name to connect as. - pg_probackup delete -B backup_dir --instance instance_name --delete-expired + -w + --no-password + Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file or PGPASSWORD environment variable, the connection attempt will fail. This flag can be useful in batch jobs and scripts where no user is present to enter a password. -pg_probackup deletes all backup copies that do not conform to the defined retention policy. + -W + --password +Forces a password prompt. -If you would like to also remove the WAL files that are no longer required for any of the backups, add the `--delete-wal` option: +#### Compression Options - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal +You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. ->NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. + --compress-algorithm=compression_algorithm + Default: none +Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. +For the [archive-push](#archive-push) command, the pglz compression algorithm is not supported. -Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` option when running [backup](#backup) or [delete](#delete) commands. + --compress-level=compression_level + Default: 1 +Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with `--compress-algorithm` option. -Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `retention-window` option is set to 7, and you have the following backups available on April 10, 2019: + --compress +Alias for `--compress-algorithm=zlib` and `--compress-level=1`. -``` -BACKUP INSTANCE 'node' -=========================================================================================================================================== - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -=========================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK - ---------------------------------------retention window------------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK -``` +#### Archiving Options -Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` option, only the P7XDFT full backup will be removed. +These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. -With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: +Additionally [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. - pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired - pg_probackup show -B backup_dir + --wal-file-path=wal_file_path +Provides the path to the WAL file in `archive_command` and `restore_command`. The `%p` variable as value for this option is required for correct processing. -``` -BACKUP INSTANCE 'node' -============================================================================================================================================ - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -============================================================================================================================================ - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK -``` + --wal-file-name=wal_file_name +Provides the name of the WAL file in `archive_command` and `restore_command`. The `%f` variable as value is required for correct processing. ->NOTE: The Time field for the merged backup displays the time required for the merge. + --overwrite +Overwrites archived WAL file. Use this flag together with the [archive-push](#archive-push) command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` flag. -### Merging Backups +#### Remote Mode Options -As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: +This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. - pg_probackup merge -B backup_dir --instance instance_name -i backup_id +For details on configuring and usage of remote operation mode, see the sections [Configuring the Remote Mode](#configuring-the-remote-mode) and [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). -This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes, I/O and network traffic in case of [remote](#using-pg_probackup-in-the-remote-mode) backup. + --remote-proto=proto +Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: -Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the [show](#show) command with the backup ID: +- `ssh` enables the remote backup mode via SSH. This is the Default value. +- `none` explicitly disables the remote mode. - pg_probackup show -B backup_dir --instance instance_name -i backup_id +You can omit this option if the `--remote-host` option is specified. -If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. + --remote-host=destination +Specifies the remote host IP address or hostname to connect to. -### Deleting Backups + --remote-port=port + Default: 22 +Specifies the remote host port to connect to. -To delete a backup that is no longer required, run the following command: + --remote-user=username + Default: current user +Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. - pg_probackup delete -B backup_dir --instance instance_name -i backup_id + --remote-path=path +Specifies pg_probackup installation directory on the remote system. -This command will delete the backup with the specified *backup_id*, together with all the incremental backups that descend from *backup_id* if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. + --ssh-options=ssh_options +Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found on [ssh_config manual page](https://man.openbsd.org/ssh_config.5). -To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` option: +#### Replica Options - pg_probackup delete -B backup_dir --instance instance_name --delete-wal +This section describes the options related to taking a backup from standby. -To delete backups that are expired according to the current retention policy, use the `--delete-expired` option: +>NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. - pg_probackup delete -B backup_dir --instance instance_name --delete-expired + --master-db=dbname + Default: postgres, the default PostgreSQL database. +Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the [set-config](#set-config) command. -Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the `--merge-expired` option when running this command: + --master-host=host +Deprecated. Specifies the host name of the system on which the master server is running. - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired + --master-port=port + Default: 5432, the PostgreSQL default port. +Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. -In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. + --master-user=username + Default: postgres, the PostgreSQL default user name. +Deprecated. User name to connect as. -Before merging or deleting backups, you can run the delete command with the `--dry-run` option, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. + --replica-timeout=timeout + Default: 300 sec +Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. ## Authors Postgres Professional, Moscow, Russia. From bfb4e6370ef9b75f4ca4afd73361997af6e996a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 05:52:59 +0300 Subject: [PATCH 0875/2107] Documentation: minor fix --- Documentation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 8151dafb3..d0fb83799 100644 --- a/Documentation.md +++ b/Documentation.md @@ -150,6 +150,7 @@ pg_probackup currently has the following limitations: - On Unix systems backup of PostgreSQL verions =< 10 is possible only by the same OS user PostgreSQL server is running by. For example, if PostgreSQL server is running by user *postgres*, then backup must be run by user *postgres*. If backup is running in [remote mode](#using-pg_probackup-in-the-remote-mode) using `ssh`, then this limitation apply differently: value for `--remote-user` option should be *postgres*. - During backup of PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` will be executed only if backup role is superuser. Because of that backup of a cluster with low amount of WAL traffic with non-superuser role may take more time than backup of the same cluster with superuser role. - The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. Also depending on cluster configuration PostgreSQL itself may apply additional restrictions such as CPU architecture platform and libc/libicu versions. +- Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. ## Installation and Setup @@ -363,7 +364,7 @@ pg_probackup supports the remote mode that allows to perform backup, restore and If you are going to use pg_probackup in remote mode via ssh, complete the following steps: - Install pg_probackup on both systems: `backup_host` and `db_host`. -- For communication between the hosts setup the passwordless SSH connection between *backup* user on `backup_host` and *postgres* user on 'db_host': +- For communication between the hosts setup the passwordless SSH connection between *backup* user on `backup_host` and *postgres* user on `db_host`: [backup@backup_host] ssh-copy-id postgres@db_host From f296a8ad8ed1cf62249708b86df76ad1343bd552 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 05:56:46 +0300 Subject: [PATCH 0876/2107] Readme: update --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cbdab651..d9bff7110 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,8 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Documentation -Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). +Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup) +Slightly outdated documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). ## License From cbef8068249ac6f74f9831aeaabfa8f3bb65806a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jul 2019 05:57:52 +0300 Subject: [PATCH 0877/2107] tests: minor fix --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a6e151ae8..4d9067891 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1037,10 +1037,10 @@ def set_archiving( archive_command = archive_command + '--overwrite ' if os.name == 'posix': - archive_command = archive_command + '--wal-file-path %p --wal-file-name %f' + archive_command = archive_command + '--wal-file-path=%p --wal-file-name=%f' elif os.name == 'nt': - archive_command = archive_command + '--wal-file-path "%p" --wal-file-name "%f"' + archive_command = archive_command + '--wal-file-path="%p" --wal-file-name="%f"' node.append_conf( 'postgresql.auto.conf', From 0309dd7a20d56e73f6ec536d63647846ef2ce1e0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Jul 2019 00:55:53 +0300 Subject: [PATCH 0878/2107] minor bugfix: initialize backup.control before libpq connection, so errors are logged and failed backup gain ERROR status --- src/backup.c | 69 +++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/backup.c b/src/backup.c index 29d6d34e6..26f7b19c2 100644 --- a/src/backup.c +++ b/src/backup.c @@ -661,14 +661,50 @@ do_backup(time_t start_time, bool no_validate) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); + /* Update backup status and other metainfo. */ + current.status = BACKUP_STATUS_RUNNING; + current.start_time = start_time; + StrNCpy(current.program_version, PROGRAM_VERSION, + sizeof(current.program_version)); + current.compress_alg = instance_config.compress_alg; current.compress_level = instance_config.compress_level; + /* Save list of external directories */ + if (instance_config.external_dir_str && + pg_strcasecmp(instance_config.external_dir_str, "none") != 0) + { + current.external_dir_str = instance_config.external_dir_str; + } + + elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " + "wal-method: %s, remote: %s, compress-algorithm: %s, compress-level: %i", + PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), + current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", + deparse_compress_alg(current.compress_alg), current.compress_level); + + /* Create backup directory and BACKUP_CONTROL_FILE */ + if (pgBackupCreateDir(¤t)) + elog(ERROR, "Cannot create backup directory"); + if (!lock_backup(¤t)) + elog(ERROR, "Cannot lock backup %s directory", + base36enc(current.start_time)); + write_backup(¤t); + + /* set the error processing function for the backup process */ + pgut_atexit_push(backup_cleanup, NULL); + + elog(LOG, "Backup destination is initialized"); + /* * setup backup_conn, do some compatibility checks and * fill basic info about instance */ backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); + + if (current.from_replica) + elog(INFO, "Backup %s is going to be taken from standby", base36enc(start_time)); + /* * Ensure that backup directory was initialized for the same PostgreSQL * instance we opened connection to. And that target backup database PGDATA @@ -676,13 +712,6 @@ do_backup(time_t start_time, bool no_validate) */ check_system_identifiers(backup_conn, instance_config.pgdata); - elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " - "wal-method: %s, remote: %s, replica: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), - current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", - current.from_replica ? "true" : "false", deparse_compress_alg(current.compress_alg), - current.compress_level); - /* below perform checks specific for backup command */ #if PG_VERSION_NUM >= 110000 if (!RetrieveWalSegSize(backup_conn)) @@ -711,32 +740,6 @@ do_backup(time_t start_time, bool no_validate) if (instance_config.master_conn_opt.pghost == NULL) elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); - /* Start backup. Update backup status. */ - current.status = BACKUP_STATUS_RUNNING; - current.start_time = start_time; - StrNCpy(current.program_version, PROGRAM_VERSION, - sizeof(current.program_version)); - - /* Save list of external directories */ - if (instance_config.external_dir_str && - pg_strcasecmp(instance_config.external_dir_str, "none") != 0) - { - current.external_dir_str = instance_config.external_dir_str; - } - - /* Create backup directory and BACKUP_CONTROL_FILE */ - if (pgBackupCreateDir(¤t)) - elog(ERROR, "Cannot create backup directory"); - if (!lock_backup(¤t)) - elog(ERROR, "Cannot lock backup %s directory", - base36enc(current.start_time)); - write_backup(¤t); - - elog(LOG, "Backup destination is initialized"); - - /* set the error processing function for the backup process */ - pgut_atexit_push(backup_cleanup, NULL); - /* backup data */ do_backup_instance(backup_conn, &nodeInfo); pgut_atexit_pop(backup_cleanup, NULL); From 961293ccb5fa42d8aa5c1aa5b93cd8141f0fb48a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Jul 2019 23:09:01 +0300 Subject: [PATCH 0879/2107] Documentation: update --- Documentation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index d0fb83799..c9cc79052 100644 --- a/Documentation.md +++ b/Documentation.md @@ -33,7 +33,7 @@ Current version - 2.1.5 * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) * [Configuring pg_probackup](#configuring-pg_probackup) - * [Managing the Backup Catalog](#managing-the-backup-Catalog) + * [Managing the Backup Catalog](#managing-the-backup-catalog) * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -490,7 +490,7 @@ Physical verification can be skipped if `--skip-block-validation` flag is used. Logical verification can be done more thoroughly with flag `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. -### Validating Backups +### Validating a Backup pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `the backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruption. @@ -985,7 +985,7 @@ Skips automatic validation after successfull backup. You can use this flag if yo Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. -For details on usage, see the section (#creating-a-backup). +For details on usage, see the section [Creating a Backup](#creating-a-backup). #### restore From 759aefc0cbff3904d4ddf6ba0d1c662d11020706 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 01:20:24 +0300 Subject: [PATCH 0880/2107] [Issue #103] select oldest backup with valid start_lsn and tli to determine oldest lsn for WAL purge --- src/delete.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/delete.c b/src/delete.c index 83be5d522..a43501449 100644 --- a/src/delete.c +++ b/src/delete.c @@ -622,6 +622,7 @@ do_retention_wal(void) XLogRecPtr oldest_lsn = InvalidXLogRecPtr; TimeLineID oldest_tli = 0; bool backup_list_is_empty = false; + int i; /* Get list of backups. */ backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); @@ -630,14 +631,17 @@ do_retention_wal(void) backup_list_is_empty = true; /* Save LSN and Timeline to remove unnecessary WAL segments */ - if (!backup_list_is_empty) + for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) { - pgBackup *backup = NULL; - /* Get LSN and TLI of oldest alive backup */ - backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1); + pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1); - oldest_tli = backup->tli; - oldest_lsn = backup->start_lsn; + /* Get LSN and TLI of the oldest backup with valid start_lsn and tli */ + if (backup->tli > 0 && !XLogRecPtrIsInvalid(backup->start_lsn)) + { + oldest_tli = backup->tli; + oldest_lsn = backup->start_lsn; + break; + } } /* Be paranoid */ From 2aacd567b299f67e4de5905e54e5b467396db104 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 01:45:15 +0300 Subject: [PATCH 0881/2107] Documentation: fix --- Documentation.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index c9cc79052..934b9e52c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -582,6 +582,12 @@ The typical workflow is as follows: - Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. +For example, to create archive full backup using remote mode via ssh, run: + + pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-host=192.168.0.2 --remote-user=postgres + +For example, to restore backup on remote system using ssh, run: + >NOTE: The remote backup mode is currently unavailable for Windows systems. ### Running pg_probackup on Parallel Threads @@ -953,8 +959,10 @@ Specifies the backup mode to use. Possible values are: - PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. - PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. - -C - --smooth-checkpoint +``` +-C +--smooth-checkpoint +``` Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. --stream From a0e52d537406f7200ca5b717aa190cc2e1a756ec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 01:57:34 +0300 Subject: [PATCH 0882/2107] Documentation: update --- Documentation.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 934b9e52c..3e5e8d0f6 100644 --- a/Documentation.md +++ b/Documentation.md @@ -582,11 +582,13 @@ The typical workflow is as follows: - Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. -For example, to create archive full backup using remote mode via ssh, run: +For example, to create archive full backup using remote mode through ssh connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: - pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-host=192.168.0.2 --remote-user=postgres + pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 -For example, to restore backup on remote system using ssh, run: +For example, to restore latest backup on remote system using remote mode through ssh connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: + + pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 >NOTE: The remote backup mode is currently unavailable for Windows systems. From bdd0459e53750469c7beeeca200f8fe30fcedb03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 12:30:50 +0300 Subject: [PATCH 0883/2107] Documentation: minor fix --- Documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 3e5e8d0f6..4a2fdec6c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -582,11 +582,11 @@ The typical workflow is as follows: - Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. -For example, to create archive full backup using remote mode through ssh connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: +For example, to create archive full backup using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 -For example, to restore latest backup on remote system using remote mode through ssh connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: +For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 From b142cb6cff99a9c08eab9c067bacc62a585d0812 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 12:51:46 +0300 Subject: [PATCH 0884/2107] Documentation: add info about semicolon delimiter for --external-dirs on Windows --- Documentation.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 4a2fdec6c..6a439134f 100644 --- a/Documentation.md +++ b/Documentation.md @@ -453,12 +453,16 @@ Redardless of data checksums been enabled or not, pg_probackup always check page #### External directories -To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons. +To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons, on Windows system paths must be separated by semicolon instead. For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 +For example, to include 'C:\dir1' and 'C:\dir2' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory on Windows system, run: + + pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 + pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. From 46f875b4bdead49690b78b06d32cdaab294af0af Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 30 Jul 2019 13:41:43 +0300 Subject: [PATCH 0885/2107] Documentation: external dirs update --- Documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6a439134f..8a88b3309 100644 --- a/Documentation.md +++ b/Documentation.md @@ -455,11 +455,11 @@ Redardless of data checksums been enabled or not, pg_probackup always check page To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons, on Windows system paths must be separated by semicolon instead. -For example, to include '/etc/dir1/' and '/etc/dir2/' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: +For example, to include `'/etc/dir1/'` and `'/etc/dir2/'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 -For example, to include 'C:\dir1' and 'C:\dir2' directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory on Windows system, run: +For example, to include `'C:\dir1\'` and `'C:\dir2\'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory on Windows system, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 From 7ead1199c276973b136dfbaf8ca96994575e02b7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 31 Jul 2019 17:16:57 +0300 Subject: [PATCH 0886/2107] bugfix: print "version" command output to stdout instead of stderr --- src/pg_probackup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 915b4ca8c..fbb7387aa 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -323,11 +323,11 @@ main(int argc, char *argv[]) || strcmp(argv[1], "-V") == 0) { #ifdef PGPRO_VERSION - fprintf(stderr, "%s %s (Postgres Pro %s %s)\n", + fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", PROGRAM_NAME, PROGRAM_VERSION, PGPRO_VERSION, PGPRO_EDITION); #else - fprintf(stderr, "%s %s (PostgreSQL %s)\n", + fprintf(stdout, "%s %s (PostgreSQL %s)\n", PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); #endif exit(0); From 98f5bbd3cfbaf22bbaf00c9e0586b2304dab6e61 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 1 Aug 2019 22:10:59 +0300 Subject: [PATCH 0887/2107] fix error message about wrong agruments for 'delete' command --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fbb7387aa..839ca9409 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -643,7 +643,7 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify --merge-expired and --backup-id options together"); if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: " - "--expired |--wal |--merge-expired |--delete-invalid |--backup_id"); + "--delete-expired |--delete-wal |--merge-expired |--backup_id"); if (!backup_id_string) return do_retention(); else From 52ed2e165146db4eb10eb6d57877fa9df2204cbf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 1 Aug 2019 22:15:59 +0300 Subject: [PATCH 0888/2107] one more fix for error messages about wrong agruments for 'delete' command --- src/pg_probackup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 839ca9409..1fa7126bb 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -638,12 +638,12 @@ main(int argc, char *argv[]) return do_show(current.backup_id); case DELETE_CMD: if (delete_expired && backup_id_string) - elog(ERROR, "You cannot specify --delete-expired and --backup-id options together"); + elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); if (merge_expired && backup_id_string) - elog(ERROR, "You cannot specify --merge-expired and --backup-id options together"); + elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together"); if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: " - "--delete-expired |--delete-wal |--merge-expired |--backup_id"); + "--delete-expired |--delete-wal |--merge-expired |(-i, --backup-id)"); if (!backup_id_string) return do_retention(); else From 142540705d6088fc56156112befd6f44abb16969 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 1 Aug 2019 23:57:14 +0300 Subject: [PATCH 0889/2107] tests: added retention.RetentionTest.test_window_error_backups_1 and retention.RetentionTest.test_window_error_backups_2 --- tests/retention.py | 146 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index 9eb5b0206..beb4c2884 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1264,16 +1264,156 @@ def test_window_error_backups(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=3) - # Take FULL BACKUPs backup_id_a1 = self.backup_node(backup_dir, 'node', node) - page_id_a2 = self.backup_node( + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', gdb=True) + + page_id_a3 = self.backup_node( backup_dir, 'node', node, backup_type='page') # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_error_backups_1(self): + """ + DELTA + PAGE ERROR + FULL + -------window + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + full_id = self.backup_node(backup_dir, 'node', node) + + # Take PAGE BACKUP + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + page_id = self.show_pb(backup_dir, 'node')[1]['id'] + + # Take DELTA backup + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-window=2', '--delete-expired']) + + # Take FULL BACKUP + full2_id = self.backup_node(backup_dir, 'node', node) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_window_error_backups_2(self): + """ + DELTA + PAGE ERROR + FULL + -------window + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + full_id = self.backup_node(backup_dir, 'node', node) + + # Take PAGE BACKUP + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb._execute('signal SIGTERM') + gdb.continue_execution_until_error() + + page_id = self.show_pb(backup_dir, 'node')[1]['id'] + + # Take DELTA backup + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-window=2', '--delete-expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Clean after yourself + # self.del_test_dir(module_name, fname) + + def test_retention_redundancy_overlapping_chains(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + if self.get_version(node) < 90600: + self.del_test_dir(module_name, fname) + return unittest.skip('Skipped because ptrack support is disabled') + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Make backups to be keeped + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + sleep(1) + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Purge backups + log = self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) + def test_retention_redundancy_overlapping_chains(self): """""" fname = self.id().split('.')[3] From b6e97e47f2ccfb09db649b9a7eff2e1d17123c7e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 2 Aug 2019 11:50:28 +0300 Subject: [PATCH 0890/2107] [Issue #106] fix agressive retention purging of backups with empty recovery-time --- src/delete.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/delete.c b/src/delete.c index 83be5d522..0db3c96bd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -259,6 +259,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { bool redundancy_keep = false; + time_t backup_time = 0; pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); /* check if backup`s FULL ancestor is in redundancy list */ @@ -280,10 +281,16 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg cur_full_backup_num++; } - /* Check if backup in needed by retention policy - * TODO: consider that ERROR backup most likely to have recovery_time == 0 + /* Invalid and running backups most likely to have recovery_time == 0, + * so in this case use start_time instead. */ - if ((days_threshold == 0 || (days_threshold > backup->recovery_time)) && + if (backup->recovery_time) + backup_time = backup->recovery_time; + else + backup_time = backup->start_time; + + /* Check if backup in needed by retention policy */ + if ((days_threshold == 0 || (days_threshold > backup_time)) && (instance_config.retention_redundancy == 0 || !redundancy_keep)) { /* This backup is not guarded by retention From ea2a4c2be3eba654ecb6fdfa5429a865deb4ed43 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 2 Aug 2019 11:59:20 +0300 Subject: [PATCH 0891/2107] bugfix: stream_stop_timeout was always initialized to 0 --- src/backup.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 26f7b19c2..d98f36cb7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2493,7 +2493,6 @@ StreamLog(void *arg) stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; /* Initialize timeout */ - stream_stop_timeout = 0; stream_stop_begin = 0; #if PG_VERSION_NUM >= 100000 From f9a0dcca5c840d1a5e2d4199a911903f1809b4fa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 2 Aug 2019 12:47:58 +0300 Subject: [PATCH 0892/2107] Readme: update --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d9bff7110..714c73014 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,15 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ### Linux Installation ```shell #DEB Ubuntu|Debian Packages -echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list -wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update -apt-get install pg-probackup-{11,10,9.6,9.5} -apt-get install pg-probackup-{11,10,9.6,9.5}-dbg +sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +sudo wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +sudo apt-get install pg-probackup-{11,10,9.6,9.5} +sudo apt-get install pg-probackup-{11,10,9.6,9.5}-dbg #DEB-SRC Packages -echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ +sudo echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list -apt-get source pg-probackup-{11,10,9.6,9.5} +sudo apt-get source pg-probackup-{11,10,9.6,9.5} #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm @@ -112,7 +112,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Documentation -Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup) +Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup). Slightly outdated documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). ## License From 5cb84f268603b21add80ec9a934ca6be6abb100c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 2 Aug 2019 12:49:21 +0300 Subject: [PATCH 0893/2107] Readme: update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 714c73014..57625c5ba 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ```shell #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list -sudo wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update +sudo wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{11,10,9.6,9.5} sudo apt-get install pg-probackup-{11,10,9.6,9.5}-dbg From 23697fff67021545093cbcdeff87d3f071c9e8a8 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 5 Aug 2019 16:22:07 +0300 Subject: [PATCH 0894/2107] Review issue_79. Added several TODO comments --- src/backup.c | 28 ++++++++++++++++++---------- src/data.c | 2 +- src/dir.c | 14 ++++++++------ src/restore.c | 34 +++++++++++++++++++++++----------- 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/backup.c b/src/backup.c index 975874b87..9ba4a3787 100644 --- a/src/backup.c +++ b/src/backup.c @@ -493,15 +493,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) } } - /* write database map to file and add it to control file */ - if (database_map) - { - write_database_map(¤t, database_map, backup_files_list); - /* we don`t need it anymore */ - parray_walk(database_map, db_map_entry_free); - parray_free(database_map); - } - /* clean previous backup file list */ if (prev_backup_filelist) { @@ -578,6 +569,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) parray_free(xlog_files_list); } + /* write database map to file and add it to control file */ + if (database_map) + { + write_database_map(¤t, database_map, backup_files_list); + /* we don`t need it anymore */ + parray_walk(database_map, db_map_entry_free); + parray_free(database_map); + } + /* Print the list of files to backup catalog */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, external_dirs); @@ -1066,7 +1066,7 @@ pg_ptrack_support(PGconn *backup_conn) /* * Create 'datname to Oid' map - * Return NULL if failed to construct database_map + * Return NULL if failed to construct database_map // TODO doesn't look safe. See comment below. */ parray * get_database_map(PGconn *conn) @@ -1075,11 +1075,18 @@ get_database_map(PGconn *conn) parray *database_map = NULL; int i; + /* TODO add a comment why we exclude template0 and template1 from the map */ res = pgut_execute_extended(conn, "SELECT oid, datname FROM pg_catalog.pg_database " "WHERE datname NOT IN ('template1', 'template0')", 0, NULL, true, true); + + /* TODO How is that possible? Shouldn't instance have at least one database? + * How can we distinguish case when instance only has template databases + * and case of query failure? + * Is it ok to ignore the failure? + */ if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -1110,6 +1117,7 @@ get_database_map(PGconn *conn) } /* extra paranoia */ + // TODO This code block has no value. Let's delete it. if (database_map && (parray_num(database_map) == 0)) { parray_free(database_map); diff --git a/src/data.c b/src/data.c index ff53d3163..d3428886e 100644 --- a/src/data.c +++ b/src/data.c @@ -1078,7 +1078,7 @@ create_empty_file(fio_location from_location, const char *to_root, char to_path[MAXPGPATH]; FILE *out; - /* open backup file for write */ + /* open file for write */ join_path_components(to_path, to_root, file->rel_path); out = fio_fopen(to_path, PG_BINARY_W, to_location); if (out == NULL) diff --git a/src/dir.c b/src/dir.c index 6bfa97faa..e103dbc38 100644 --- a/src/dir.c +++ b/src/dir.c @@ -249,6 +249,10 @@ pgFileDelete(pgFile *file) } } +/* + * Read the file to compute its CRC. + * As a handy side effect, we return filesize via bytes_read parameter. + */ pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read, fio_location location) @@ -442,12 +446,12 @@ BlackListCompare(const void *str1, const void *str2) } void -db_map_entry_free(void *map) +db_map_entry_free(void *entry) { - db_map_entry *m = (db_map_entry *) map; + db_map_entry *m = (db_map_entry *) entry; free(m->datname); - free(map); + free(entry); } /* @@ -1702,14 +1706,12 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ database_map_path, strerror(errno)); } - /* */ + /* Add metadata to backup_content.control */ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; - free(file->path); - file->path = strdup(DATABASE_MAP); parray_append(backup_files_list, file); } diff --git a/src/restore.c b/src/restore.c index 61f9afd20..098a6487e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -457,8 +457,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * It is important that we do this after(!) validation so * database_map can be trusted. * NOTE: database_map could be missing for legal reasons, e.g. missing - * permissions on pg_database during `backup` and, as long as user do not request - * partial restore, it`s OK + * permissions on pg_database during `backup` and, as long as user + * do not request partial restore, it`s OK. + * + * If partial restore is requested and database map doesn't exist, + * throw an error. */ if (datname_list) dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, datname_list, @@ -709,19 +712,22 @@ restore_files(void *arg) i + 1, (unsigned long) parray_num(arguments->files), file->rel_path); - /* only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && - file->external_dir_num == 0) + /* Only files from pgdata can be skipped by partial restore */ + if (arguments->dbOid_exclude_list && file->external_dir_num == 0) { - /* exclude map is not empty */ + /* Check if the file belongs to the database we exclude */ if (parray_bsearch(arguments->dbOid_exclude_list, &file->dbOid, pgCompareOid)) { - /* got a match, destination file will truncated */ + /* + * We cannot simply skip the file, because it may lead to + * failure during WAL redo; hence, create empty file. + */ create_empty_file(FIO_BACKUP_HOST, instance_config.pgdata, FIO_DB_HOST, file); - elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", file->rel_path); + elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", + file->rel_path); continue; } } @@ -1176,7 +1182,8 @@ parseRecoveryTargetOptions(const char *target_time, return rt; } -/* Return array of dbOids of databases that should not be restored +/* + * Return array of dbOids of databases that should not be restored * Regardless of what option user used, db-include or db-exclude, * we always convert it into exclude_list. */ @@ -1191,6 +1198,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, bool found_database_map = false; /* make sure that database_map is in backup_content.control */ + // TODO can't we use parray_bsearch here? for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); @@ -1203,6 +1211,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, } } + // TODO rephrase error message if (!found_database_map) elog(ERROR, "Backup %s has missing database_map, partial restore is impossible.", base36enc(backup->start_time)); @@ -1215,7 +1224,8 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, elog(ERROR, "Backup %s has empty or mangled database_map, partial restore is impossible.", base36enc(backup->start_time)); - /* So we have list of datnames and database_map for it. + /* + * So we have a list of datnames and database_map for it. * We must construct a list of dbOids to exclude. */ if (partial_restore_type) @@ -1284,7 +1294,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, } } - /* extra sanity, we must be totally sure that list is not empty */ + /* extra sanity: ensure that list is not empty */ if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", base36enc(backup->start_time)); @@ -1296,6 +1306,8 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, } /* Compare two Oids */ +// TODO is it overflow safe? +// TODO move it to dir.c like other compare functions int pgCompareOid(const void *f1, const void *f2) { From 535e417ba806d78ad6b6055e627a39ef91ca5c80 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 5 Aug 2019 23:35:51 +0300 Subject: [PATCH 0895/2107] tests: added retention.RetentionTest.test_agressive_retention_window_purge --- tests/retention.py | 48 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/tests/retention.py b/tests/retention.py index beb4c2884..6e13b21e1 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1,7 +1,7 @@ import os import unittest from datetime import datetime, timedelta -from .helpers.ptrack_helpers import ProbackupTest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from time import sleep @@ -1459,3 +1459,49 @@ def test_retention_redundancy_overlapping_chains(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_agressive_retention_window_purge(self): + """ + https://github.com/postgrespro/pg_probackup/issues/106 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Make ERROR incremental backup + try: + self.backup_node(backup_dir, 'node', node, backup_type='page') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Valid backup on current timeline 1 is not found. " + "Create new FULL backup before an incremental one.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + page_id = self.show_pb(backup_dir, 'node')[0]['id'] + + sleep(1) + + # Make FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--delete-expired', '--retention-window=1', '--stream']) + + # Check number of backups + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Clean after yourself + self.del_test_dir(module_name, fname) From db901a7adfec1e2655394747406c74f3d6e57ccf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 00:00:34 +0300 Subject: [PATCH 0896/2107] [Issue #103] minor fix --- src/delete.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index a43501449..5a3c37f53 100644 --- a/src/delete.c +++ b/src/delete.c @@ -633,7 +633,7 @@ do_retention_wal(void) /* Save LSN and Timeline to remove unnecessary WAL segments */ for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, parray_num(backup_list) -1); + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); /* Get LSN and TLI of the oldest backup with valid start_lsn and tli */ if (backup->tli > 0 && !XLogRecPtrIsInvalid(backup->start_lsn)) From 8b15753a0487bd7a3893f683bc4add14da233dd1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 00:01:05 +0300 Subject: [PATCH 0897/2107] tests: added retention.RetentionTest.test_wal_purge_victim --- tests/retention.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index 6e13b21e1..3a85ee641 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1460,9 +1460,9 @@ def test_retention_redundancy_overlapping_chains(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_agressive_retention_window_purge(self): + def test_wal_purge_victim(self): """ - https://github.com/postgrespro/pg_probackup/issues/106 + https://github.com/postgrespro/pg_probackup/issues/103 """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -1472,6 +1472,7 @@ def test_agressive_retention_window_purge(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.slow_start() # Make ERROR incremental backup @@ -1496,12 +1497,7 @@ def test_agressive_retention_window_purge(self): sleep(1) # Make FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--delete-expired', '--retention-window=1', '--stream']) - - # Check number of backups - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) # Clean after yourself self.del_test_dir(module_name, fname) From 11c02e9b91a4d36f1dcf07efec7873609efcd6f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 11:58:29 +0300 Subject: [PATCH 0898/2107] [Issue #108] added test validate.ValidateTest.test_validate_target_lsn --- tests/validate.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index 1c919907f..1c2a3a902 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3443,6 +3443,60 @@ def test_validate_corrupt_tablespace_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_target_lsn(self): + """ + Check validation to specific LSN + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + node_restored.append_conf( + "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + + node_restored.slow_start() + + self.switch_wal_segment(node) + + backup_id = self.backup_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir) + + target_lsn = self.show_pb(backup_dir, 'node')[1]['stop-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + self.validate_pb( + backup_dir, 'node', + options=[ + '--recovery-target-timeline=2', + '--recovery-target-lsn={0}'.format(target_lsn)]) + # validate empty backup list # page from future during validate # page from future during backup From 6da63188eb53c8144eda8c0b694389e6a8c45dc5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 12:00:44 +0300 Subject: [PATCH 0899/2107] [Issue #104] allow LSN as recovery target for validation if backup ID is not specified --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1fa7126bb..3591d300a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -628,7 +628,7 @@ main(int argc, char *argv[]) recovery_target_options, true); case VALIDATE_CMD: - if (current.backup_id == 0 && target_time == 0 && target_xid == 0) + if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn) return do_validate_all(); else return do_restore_or_validate(current.backup_id, From 3cc9d81b65bd0c99fa2bcb3f4f75de5e7c7f18bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 17:11:48 +0300 Subject: [PATCH 0900/2107] [Issue #104] consider only OK and DONE backups as candidates for PITR validation if backup ID is not specified --- src/restore.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/restore.c b/src/restore.c index c3c279b78..535c81bee 100644 --- a/src/restore.c +++ b/src/restore.c @@ -91,10 +91,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * [PGPRO-1164] If BACKUP_ID is not provided for restore command, * we must find the first valid(!) backup. + + * If target_backup_id is not provided, we can be sure that + * PITR for restore or validate is requested. + * So we can assume that user is more interested in recovery to specific point + * in time and NOT interested in revalidation of invalid backups. + * So based on that assumptions we should choose only OK and DONE backups + * as candidates for validate and restore. */ - if (is_restore && - target_backup_id == INVALID_BACKUP_ID && + if (target_backup_id == INVALID_BACKUP_ID && (current_backup->status != BACKUP_STATUS_OK && current_backup->status != BACKUP_STATUS_DONE)) { @@ -1033,7 +1039,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_time(target_time, &dummy_time, false)) rt->target_time = dummy_time; else - elog(ERROR, "Invalid value for --recovery-target-time option %s", + elog(ERROR, "Invalid value for '--recovery-target-time' option %s", target_time); } @@ -1051,7 +1057,7 @@ parseRecoveryTargetOptions(const char *target_time, #endif rt->target_xid = dummy_xid; else - elog(ERROR, "Invalid value for --recovery-target-xid option %s", + elog(ERROR, "Invalid value for '--recovery-target-xid' option %s", target_xid); } @@ -1064,7 +1070,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_lsn(target_lsn, &dummy_lsn)) rt->target_lsn = dummy_lsn; else - elog(ERROR, "Invalid value of --recovery-target-lsn option %s", + elog(ERROR, "Invalid value of '--recovery-target-lsn' option %s", target_lsn); } @@ -1074,7 +1080,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_bool(target_inclusive, &dummy_bool)) rt->target_inclusive = dummy_bool; else - elog(ERROR, "Invalid value for --recovery-target-inclusive option %s", + elog(ERROR, "Invalid value for '--recovery-target-inclusive' option %s", target_inclusive); } @@ -1083,7 +1089,7 @@ parseRecoveryTargetOptions(const char *target_time, { if ((strcmp(target_stop, "immediate") != 0) && (strcmp(target_stop, "latest") != 0)) - elog(ERROR, "Invalid value for --recovery-target option %s", + elog(ERROR, "Invalid value for '--recovery-target' option %s", target_stop); recovery_target_specified++; @@ -1103,7 +1109,7 @@ parseRecoveryTargetOptions(const char *target_time, if ((strcmp(target_action, "pause") != 0) && (strcmp(target_action, "promote") != 0) && (strcmp(target_action, "shutdown") != 0)) - elog(ERROR, "Invalid value for --recovery-target-action option %s", + elog(ERROR, "Invalid value for '--recovery-target-action' option %s", target_action); rt->target_action = target_action; From 0b4d09e26223530e8229bd23e64d6cd2b5d1d2c0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 19:09:17 +0300 Subject: [PATCH 0901/2107] [Issue #109] added test validate.ValidateTest.test_recovery_target_backup_victim --- tests/validate.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index 1c2a3a902..fce22c56f 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3497,6 +3497,65 @@ def test_validate_target_lsn(self): '--recovery-target-timeline=2', '--recovery-target-lsn={0}'.format(target_lsn)]) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + target_time = node.safe_psql( + "postgres", + "select now()").rstrip() + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-time={0}'.format(target_time)]) + + # validate empty backup list # page from future during validate # page from future during backup From 736fc414efbb89d6e60bde9ce95bce95d1e2e1c3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 6 Aug 2019 19:21:52 +0300 Subject: [PATCH 0902/2107] tests: do not run some tests on Windows --- tests/backup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 81370a173..f15ae1680 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1437,6 +1437,9 @@ def test_drop_table(self): # @unittest.skip("skip") def test_basic_missing_file_permissions(self): """""" + if os.name == 'nt': + return unittest.skip('Skipped because it is POSIX only test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1481,6 +1484,9 @@ def test_basic_missing_file_permissions(self): # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): """""" + if os.name == 'nt': + return unittest.skip('Skipped because it is POSIX only test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( From 52bf25ccd2be5f32bf9812342c0de73211267a44 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 7 Aug 2019 13:23:17 +0300 Subject: [PATCH 0903/2107] minor refactoring in restore.c to improve code readability --- src/restore.c | 104 +++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 57 deletions(-) diff --git a/src/restore.c b/src/restore.c index 098a6487e..56c30a216 100644 --- a/src/restore.c +++ b/src/restore.c @@ -46,6 +46,50 @@ static parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, parray *datname_list, bool partial_restore_type); static int pgCompareOid(const void *f1, const void *f2); +static void set_orphan_status(parray *backups, pgBackup *parent_backup); + +/* + * Iterate over backup list to find all ancestors of the broken parent_backup + * and update their status to BACKUP_STATUS_ORPHAN + */ +static void +set_orphan_status(parray *backups, pgBackup *parent_backup) +{ + /* chain is intact, but at least one parent is invalid */ + char *parent_backup_id; + int j; + + /* parent_backup_id is a human-readable backup ID */ + parent_backup_id = base36enc_dup(parent_backup->start_time); + + for (j = 0; j < parray_num(backups); j++) + { + + pgBackup *backup = (pgBackup *) parray_get(backups, j); + + if (is_parent(parent_backup->start_time, backup, false)) + { + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) + { + write_backup_status(backup, BACKUP_STATUS_ORPHAN); + + elog(WARNING, + "Backup %s is orphaned because his parent %s has status: %s", + base36enc(backup->start_time), + parent_backup_id, + status2str(parent_backup->status)); + } + else + { + elog(WARNING, "Backup %s has parent %s with status: %s", + base36enc(backup->start_time), parent_backup_id, + status2str(parent_backup->status)); + } + } + } + pg_free(parent_backup_id); +} /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. @@ -208,7 +252,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * and orphinize all his descendants */ char *missing_backup_id; - time_t missing_backup_start_time; + time_t missing_backup_start_time; missing_backup_start_time = tmp_backup->parent_backup; missing_backup_id = base36enc_dup(tmp_backup->parent_backup); @@ -244,38 +288,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, else if (result == 1) { /* chain is intact, but at least one parent is invalid */ - char *parent_backup_id; - - /* parent_backup_id contain human-readable backup ID of oldest invalid backup */ - parent_backup_id = base36enc_dup(tmp_backup->start_time); - - for (j = 0; j < parray_num(backups); j++) - { - - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(tmp_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(backup, BACKUP_STATUS_ORPHAN); - - elog(WARNING, - "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - parent_backup_id, - status2str(tmp_backup->status)); - } - else - { - elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(backup->start_time), parent_backup_id, - status2str(tmp_backup->status)); - } - } - } - pg_free(parent_backup_id); + set_orphan_status(backups, tmp_backup); tmp_backup = find_parent_full_backup(dest_backup); /* sanity */ @@ -386,30 +399,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* Orphanize every OK descendant of corrupted backup */ else - { - char *corrupted_backup_id; - corrupted_backup_id = base36enc_dup(corrupted_backup->start_time); - - for (j = 0; j < parray_num(backups); j++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(corrupted_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(backup, BACKUP_STATUS_ORPHAN); - - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - corrupted_backup_id, - status2str(corrupted_backup->status)); - } - } - } - free(corrupted_backup_id); - } + set_orphan_status(backups, corrupted_backup); } /* From e8b9ca1ba82da86cacabcceb0126370d55a0f4ec Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 7 Aug 2019 16:56:56 +0300 Subject: [PATCH 0904/2107] code cleanup, add comments --- src/backup.c | 39 +++++++++++++++++++-------------------- src/data.c | 2 +- src/dir.c | 24 +++++++++++++++++++++--- src/pg_probackup.c | 5 ++--- src/pg_probackup.h | 1 + src/restore.c | 45 +++++++++++++++++++++------------------------ 6 files changed, 65 insertions(+), 51 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9ba4a3787..146ab8364 100644 --- a/src/backup.c +++ b/src/backup.c @@ -315,7 +315,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) dir_list_file(backup_files_list, instance_config.pgdata, true, true, false, 0, FIO_DB_HOST); - /* create database_map used for partial restore */ + /* + * Get database_map (name to oid) for use in partial restore feature. + * It's possible that we fail and database_map will be NULL. + */ database_map = get_database_map(pg_startbackup_conn); /* @@ -573,7 +576,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) if (database_map) { write_database_map(¤t, database_map, backup_files_list); - /* we don`t need it anymore */ + /* cleanup */ parray_walk(database_map, db_map_entry_free); parray_free(database_map); } @@ -1065,8 +1068,15 @@ pg_ptrack_support(PGconn *backup_conn) } /* - * Create 'datname to Oid' map - * Return NULL if failed to construct database_map // TODO doesn't look safe. See comment below. + * Fill 'datname to Oid' map + * + * This function can fail to get the map for legal reasons, e.g. missing + * permissions on pg_database during `backup`. + * As long as user do not use partial restore feature it`s fine. + * + * To avoid breaking a backward compatibility don't throw an ERROR, + * throw a warning instead of an error and return NULL. + * Caller is responsible for checking the result. */ parray * get_database_map(PGconn *conn) @@ -1075,18 +1085,16 @@ get_database_map(PGconn *conn) parray *database_map = NULL; int i; - /* TODO add a comment why we exclude template0 and template1 from the map */ + /* + * Do not include template0 and template1 to the map + * as default databases that must always be restored. + */ res = pgut_execute_extended(conn, "SELECT oid, datname FROM pg_catalog.pg_database " "WHERE datname NOT IN ('template1', 'template0')", 0, NULL, true, true); - - /* TODO How is that possible? Shouldn't instance have at least one database? - * How can we distinguish case when instance only has template databases - * and case of query failure? - * Is it ok to ignore the failure? - */ + /* Don't error out, simply return NULL. See comment above. */ if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -1116,15 +1124,6 @@ get_database_map(PGconn *conn) parray_append(database_map, db_entry); } - /* extra paranoia */ - // TODO This code block has no value. Let's delete it. - if (database_map && (parray_num(database_map) == 0)) - { - parray_free(database_map); - elog(WARNING, "Failed to get database map"); - return NULL; - } - return database_map; } diff --git a/src/data.c b/src/data.c index d3428886e..82994a179 100644 --- a/src/data.c +++ b/src/data.c @@ -1096,7 +1096,7 @@ create_empty_file(fio_location from_location, const char *to_root, } if (fio_fclose(out)) - elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + elog(ERROR, "cannot close \"%s\": %s", to_path, strerror(errno)); return true; } diff --git a/src/dir.c b/src/dir.c index e103dbc38..54ea87c5f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -445,6 +445,22 @@ BlackListCompare(const void *str1, const void *str2) return strcmp(*(char **) str1, *(char **) str2); } +/* Compare two Oids */ +int +pgCompareOid(const void *f1, const void *f2) +{ + Oid v1 = *(Oid *)f1; + Oid v2 = *(Oid *)f2; + + elog(WARNING, "pgCompareOid %u %u", v1, v2); + if (v1 > v2) + return 1; + else if (v1 < v2) + return -1; + else + return 0;} + + void db_map_entry_free(void *entry) { @@ -1679,7 +1695,7 @@ print_database_map(FILE *out, parray *database_map) } /* - * Create file 'database_map' and add its meta to backup_content.control + * Create file 'database_map' and add its meta to backup_files_list * NULL check for database_map must be done by the caller. */ void @@ -1709,8 +1725,10 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ /* Add metadata to backup_content.control */ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(file->path, true, false, - &file->read_size, FIO_BACKUP_HOST); + pfree(file->path); + file->path = strdup(DATABASE_MAP); + file->crc = pgFileGetCRC(database_map_path, true, false, + &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; parray_append(backup_files_list, file); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 57d90ee75..03a879e53 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -641,7 +641,6 @@ main(int argc, char *argv[]) if (datname_exclude_list && datname_include_list) elog(ERROR, "You cannot specify '--db-include' and '--db-exclude' together"); - /* At this point we are sure that user requested partial restore */ if (datname_exclude_list) datname_list = datname_exclude_list; @@ -777,7 +776,7 @@ opt_datname_exclude_list(ConfigOption *opt, const char *arg) dbname = pgut_malloc(strlen(arg) + 1); - /* add sanity for database name */ + /* TODO add sanity for database name */ strcpy(dbname, arg); parray_append(datname_exclude_list, dbname); @@ -794,7 +793,7 @@ opt_datname_include_list(ConfigOption *opt, const char *arg) dbname = pgut_malloc(strlen(arg) + 1); - /* add sanity for database name */ + /* TODO add sanity for database name */ strcpy(dbname, arg); parray_append(datname_include_list, dbname); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e0ecf30b9..19b3c7f9b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -654,6 +654,7 @@ extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileComparePathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); +extern int pgCompareOid(const void *f1, const void *f2); /* in data.c */ extern bool check_data_file(ConnectionArgs* arguments, pgFile* file, uint32 checksum_version); diff --git a/src/restore.c b/src/restore.c index 56c30a216..fe7be6fab 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,8 +44,6 @@ static void *restore_files(void *arg); static parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, parray *datname_list, bool partial_restore_type); - -static int pgCompareOid(const void *f1, const void *f2); static void set_orphan_status(parray *backups, pgBackup *parent_backup); /* @@ -482,7 +480,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* - * At least restore backups files starting from the parent backup. + * Restore backups files starting from the parent backup. */ for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -1185,27 +1183,38 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, int j; parray *database_map = NULL; parray *dbOid_exclude_list = NULL; - bool found_database_map = false; + pgFile *database_map_file = NULL; + pg_crc32 crc; + char path[MAXPGPATH]; + char database_map_path[MAXPGPATH]; /* make sure that database_map is in backup_content.control */ - // TODO can't we use parray_bsearch here? for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); if ((file->external_dir_num == 0) && - strcmp(DATABASE_MAP, file->rel_path) == 0) + strcmp(DATABASE_MAP, file->name) == 0) { - found_database_map = true; + database_map_file = file; break; } } - // TODO rephrase error message - if (!found_database_map) - elog(ERROR, "Backup %s has missing database_map, partial restore is impossible.", + if (!database_map_file) + elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", base36enc(backup->start_time)); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(database_map_path, path, DATABASE_MAP); + + /* check database_map CRC */ + crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); + + if (crc != database_map_file->crc) + elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", + database_map_file->path, crc, database_map_file->crc); + /* get database_map from file */ database_map = read_database_map(backup); @@ -1215,12 +1224,12 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, base36enc(backup->start_time)); /* - * So we have a list of datnames and database_map for it. + * So we have a list of datnames and a database_map for it. * We must construct a list of dbOids to exclude. */ if (partial_restore_type) { - /* For 'include' find dbOid of every datname NOT specified by user */ + /* For 'include' keep dbOid of every datname NOT specified by user */ for (i = 0; i < parray_num(datname_list); i++) { bool found_match = false; @@ -1294,15 +1303,3 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, return dbOid_exclude_list; } - -/* Compare two Oids */ -// TODO is it overflow safe? -// TODO move it to dir.c like other compare functions -int -pgCompareOid(const void *f1, const void *f2) -{ - Oid *f1p = *(Oid **)f1; - Oid *f2p = *(Oid **)f2; - - return (*(Oid*)f1p - *(Oid*)f2p); -} From 5ae396f7d60badd9b3ee4fb29fbc1848ce840247 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Wed, 7 Aug 2019 17:13:19 +0300 Subject: [PATCH 0905/2107] fix comparator --- src/dir.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/dir.c b/src/dir.c index 54ea87c5f..c122d3a3d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -449,13 +449,12 @@ BlackListCompare(const void *str1, const void *str2) int pgCompareOid(const void *f1, const void *f2) { - Oid v1 = *(Oid *)f1; - Oid v2 = *(Oid *)f2; + Oid *v1 = *(Oid **) f1; + Oid *v2 = *(Oid **) f2; - elog(WARNING, "pgCompareOid %u %u", v1, v2); - if (v1 > v2) + if (*v1 > *v2) return 1; - else if (v1 < v2) + else if (*v1 < *v2) return -1; else return 0;} From f4790b6291711da3f82e2c5c1e19fbcbd47b7dee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 7 Aug 2019 18:39:32 +0300 Subject: [PATCH 0906/2107] write dbOid in write_backup_filelist() --- src/catalog.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index ba7fa5aee..8deb31ee2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -744,13 +744,15 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\"", + "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " + "\"dbOid\":\"%u\"", path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, deparse_compress_alg(file->compress_alg), - file->external_dir_num); + file->external_dir_num, + file->dbOid); if (file->is_datafile) len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); From d92434ff84be6e8f18d8c251d1bbe3ad47e89b11 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Aug 2019 15:15:18 +0300 Subject: [PATCH 0907/2107] Refactoring of restore and validate options. Collect them into pgRestoreParams structure --- src/pg_probackup.c | 55 ++++++++++++++++++++----------------- src/pg_probackup.h | 27 ++++++++++++------- src/restore.c | 67 ++++++++++++++++++++++++---------------------- 3 files changed, 82 insertions(+), 67 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 03a879e53..c4e314084 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -89,6 +89,7 @@ static char *target_name = NULL; static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; +static pgRestoreParams *restore_params = NULL; bool restore_as_replica = false; bool no_validate = false; @@ -598,7 +599,32 @@ main(int argc, char *argv[]) target_inclusive, target_tli, target_lsn, (target_stop != NULL) ? target_stop : (target_immediate) ? "immediate" : NULL, - target_name, target_action, no_validate); + target_name, target_action); + + /* keep all params in one structure */ + restore_params = pgut_new(pgRestoreParams); + restore_params->is_restore = (backup_subcmd == RESTORE_CMD); + restore_params->no_validate = no_validate; + restore_params->restore_as_replica = restore_as_replica; + restore_params->skip_block_validation = skip_block_validation; + restore_params->skip_external_dirs = skip_external_dirs; + + /* handle partial restore parameters */ + if (datname_exclude_list && datname_include_list) + elog(ERROR, "You cannot specify '--db-include' and '--db-exclude' together"); + + if (datname_exclude_list) + { + restore_params->is_include_list = false; + restore_params->partial_db_list = datname_exclude_list; + } + else if (datname_include_list) + { + restore_params->is_include_list = true; + restore_params->partial_db_list = datname_include_list; + } + + } if (num_threads < 1) @@ -633,37 +659,16 @@ main(int argc, char *argv[]) return do_backup(start_time, no_validate); } case RESTORE_CMD: - { - parray *datname_list = NULL; - /* true for 'include', false for 'exclude' */ - bool partial_restore_type = false; - - if (datname_exclude_list && datname_include_list) - elog(ERROR, "You cannot specify '--db-include' and '--db-exclude' together"); - - if (datname_exclude_list) - datname_list = datname_exclude_list; - - if (datname_include_list) - { - partial_restore_type = true; - datname_list = datname_include_list; - } - return do_restore_or_validate(current.backup_id, + return do_restore_or_validate(current.backup_id, recovery_target_options, - true, - datname_list, - partial_restore_type); - } + restore_params); case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0) return do_validate_all(); else return do_restore_or_validate(current.backup_id, recovery_target_options, - false, - NULL, - false); + restore_params); case SHOW_CMD: return do_show(current.backup_id); case DELETE_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 19b3c7f9b..1c49b7024 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -336,9 +336,22 @@ typedef struct pgRecoveryTarget const char *target_stop; const char *target_name; const char *target_action; - bool no_validate; } pgRecoveryTarget; +/* Options needed for restore and validate commands */ +typedef struct pgRestoreParams +{ + bool is_restore; + bool no_validate; + bool restore_as_replica; + bool skip_external_dirs; + bool skip_block_validation; + + /* options for partial restore */ + bool is_include_list; + parray *partial_db_list; +} pgRestoreParams; + typedef struct { const char *from_root; @@ -444,11 +457,6 @@ extern char* remote_agent; extern bool is_ptrack_support; extern bool exclusive_backup; -/* restore options */ -extern bool restore_as_replica; -extern bool skip_block_validation; -extern bool skip_external_dirs; - /* delete options */ extern bool delete_wal; extern bool delete_expired; @@ -467,6 +475,7 @@ extern ShowFormat show_format; /* checkdb options */ extern bool heapallindexed; +extern bool skip_block_validation; /* current settings */ extern pgBackup current; @@ -494,9 +503,7 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments, /* in restore.c */ extern int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - bool is_restore, - parray * datname_list, - bool partial_restore_type); + pgRestoreParams *params); extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); @@ -504,7 +511,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, const char *target_stop, const char *target_name, - const char *target_action, bool no_validate); + const char *target_action); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index fe7be6fab..b6c897c07 100644 --- a/src/restore.c +++ b/src/restore.c @@ -26,6 +26,7 @@ typedef struct parray *dest_external_dirs; parray *dest_files; parray *dbOid_exclude_list; + bool skip_external_dirs; /* * Return value from the thread. @@ -35,10 +36,12 @@ typedef struct } restore_files_arg; static void restore_backup(pgBackup *backup, parray *dest_external_dirs, - parray *dest_files, parray *dbOid_exclude_list); + parray *dest_files, parray *dbOid_exclude_list, + pgRestoreParams *params); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, - pgBackup *backup); + pgBackup *backup, + pgRestoreParams *params); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); @@ -94,7 +97,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) */ int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - bool is_restore, parray *datname_list, bool partial_restore_type) + pgRestoreParams *params) { int i = 0; int j = 0; @@ -104,11 +107,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, pgBackup *dest_backup = NULL; pgBackup *base_full_backup = NULL; pgBackup *corrupted_backup = NULL; - char *action = is_restore ? "Restore":"Validate"; + char *action = params->is_restore ? "Restore":"Validate"; parray *parent_chain = NULL; parray *dbOid_exclude_list = NULL; - if (is_restore) + if (params->is_restore) { if (instance_config.pgdata == NULL) elog(ERROR, @@ -142,7 +145,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * we must find the first valid(!) backup. */ - if (is_restore && + if (params->is_restore && target_backup_id == INVALID_BACKUP_ID && (current_backup->status != BACKUP_STATUS_OK && current_backup->status != BACKUP_STATUS_DONE)) @@ -176,7 +179,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if ((current_backup->status == BACKUP_STATUS_ORPHAN || current_backup->status == BACKUP_STATUS_CORRUPT || current_backup->status == BACKUP_STATUS_RUNNING) - && !rt->no_validate) + && !params->no_validate) elog(WARNING, "Backup %s has status: %s", base36enc(current_backup->start_time), status2str(current_backup->status)); else @@ -312,12 +315,12 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Ensure that directories provided in tablespace mapping are valid * i.e. empty or not exist. */ - if (is_restore) + if (params->is_restore) { check_tablespace_mapping(dest_backup); /* no point in checking external directories if their restore is not requested */ - if (!skip_external_dirs) + if (!params->skip_external_dirs) check_external_dir_mapping(dest_backup); } @@ -341,7 +344,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray_append(parent_chain, base_full_backup); /* for validation or restore with enabled validation */ - if (!is_restore || !rt->no_validate) + if (!params->is_restore || !params->no_validate) { if (dest_backup->backup_mode != BACKUP_MODE_FULL) elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); @@ -356,7 +359,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Do not interrupt, validate the next backup */ if (!lock_backup(tmp_backup)) { - if (is_restore) + if (params->is_restore) elog(ERROR, "Cannot lock backup %s directory", base36enc(tmp_backup->start_time)); else @@ -407,7 +410,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (dest_backup->status == BACKUP_STATUS_OK || dest_backup->status == BACKUP_STATUS_DONE) { - if (rt->no_validate) + if (params->no_validate) elog(WARNING, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); else elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); @@ -423,7 +426,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* We ensured that all backups are valid, now restore if required * TODO: before restore - lock entire parent chain */ - if (is_restore) + if (params->is_restore) { parray *dest_external_dirs = NULL; parray *dest_files; @@ -451,9 +454,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * If partial restore is requested and database map doesn't exist, * throw an error. */ - if (datname_list) - dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, datname_list, - partial_restore_type); + if (params->partial_db_list) + dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, params->partial_db_list, + params->is_include_list); /* * Restore dest_backup internal directories. @@ -466,7 +469,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * Restore dest_backup external directories. */ - if (dest_backup->external_dir_str && !skip_external_dirs) + if (dest_backup->external_dir_str && !params->skip_external_dirs) { dest_external_dirs = make_external_directory_list( dest_backup->external_dir_str, @@ -496,10 +499,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Backup was locked during validation if no-validate wasn't * specified. */ - if (rt->no_validate && !lock_backup(backup)) + if (params->no_validate && !lock_backup(backup)) elog(ERROR, "Cannot lock backup directory"); - restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list); + restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); } if (dest_external_dirs != NULL) @@ -509,7 +512,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray_free(dest_files); /* Create recovery.conf with given recovery target parameters */ - create_recovery_conf(target_backup_id, rt, dest_backup); + create_recovery_conf(target_backup_id, rt, dest_backup, params); } /* cleanup */ @@ -527,7 +530,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ void restore_backup(pgBackup *backup, parray *dest_external_dirs, - parray *dest_files, parray *dbOid_exclude_list) + parray *dest_files, parray *dbOid_exclude_list, + pgRestoreParams *params) { char timestamp[100]; char database_path[MAXPGPATH]; @@ -587,7 +591,7 @@ restore_backup(pgBackup *backup, parray *dest_external_dirs, /* * If the entry was an external directory, create it in the backup. */ - if (!skip_external_dirs && + if (!params->skip_external_dirs && file->external_dir_num && S_ISDIR(file->mode) && /* Do not create unnecessary external directories */ parray_bsearch(dest_files, file, pgFileCompareRelPathWithExternal)) @@ -635,6 +639,7 @@ restore_backup(pgBackup *backup, parray *dest_external_dirs, arg->dest_external_dirs = dest_external_dirs; arg->dest_files = dest_files; arg->dbOid_exclude_list = dbOid_exclude_list; + arg->skip_external_dirs = params->skip_external_dirs; /* By default there are some error */ threads_args[i].ret = 1; @@ -756,7 +761,7 @@ restore_files(void *arg) } /* Do no restore external directory file if a user doesn't want */ - if (skip_external_dirs && file->external_dir_num > 0) + if (arguments->skip_external_dirs && file->external_dir_num > 0) continue; /* Skip unnecessary file */ @@ -818,7 +823,8 @@ restore_files(void *arg) static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, - pgBackup *backup) + pgBackup *backup, + pgRestoreParams *params) { char path[MAXPGPATH]; FILE *fp; @@ -831,7 +837,7 @@ create_recovery_conf(time_t backup_id, (rt->time_string || rt->xid_string || rt->lsn_string) || target_latest; /* No need to generate recovery.conf at all. */ - if (!(need_restore_conf || restore_as_replica)) + if (!(need_restore_conf || params->restore_as_replica)) return; elog(LOG, "----------------------------------------"); @@ -884,7 +890,7 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); } - if (restore_as_replica) + if (params->restore_as_replica) { fio_fprintf(fp, "standby_mode = 'on'\n"); @@ -1049,8 +1055,7 @@ parseRecoveryTargetOptions(const char *target_time, const char *target_lsn, const char *target_stop, const char *target_name, - const char *target_action, - bool no_validate) + const char *target_action) { bool dummy_bool; /* @@ -1131,8 +1136,6 @@ parseRecoveryTargetOptions(const char *target_time, rt->target_stop = target_stop; } - rt->no_validate = no_validate; - if (target_name) { recovery_target_specified++; @@ -1177,7 +1180,7 @@ parseRecoveryTargetOptions(const char *target_time, */ parray * get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, bool partial_restore_type) + parray *datname_list, bool is_include_list) { int i; int j; @@ -1227,7 +1230,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, * So we have a list of datnames and a database_map for it. * We must construct a list of dbOids to exclude. */ - if (partial_restore_type) + if (is_include_list) { /* For 'include' keep dbOid of every datname NOT specified by user */ for (i = 0; i < parray_num(datname_list); i++) From f36d81dc08f5b5eef96308b8c89c270c32e02007 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Aug 2019 16:40:26 +0300 Subject: [PATCH 0908/2107] fix uninitialized var --- src/pg_probackup.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 107ba7e95..2167ff361 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -608,6 +608,7 @@ main(int argc, char *argv[]) restore_params->restore_as_replica = restore_as_replica; restore_params->skip_block_validation = skip_block_validation; restore_params->skip_external_dirs = skip_external_dirs; + restore_params->partial_db_list = NULL; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) @@ -623,8 +624,6 @@ main(int argc, char *argv[]) restore_params->is_include_list = true; restore_params->partial_db_list = datname_include_list; } - - } if (num_threads < 1) From 6179808d234ecf4c4d52b49f2d042eea3869b794 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 8 Aug 2019 17:55:23 +0300 Subject: [PATCH 0909/2107] implement partial validate. add db-include and db-exclude options to validate command --- src/backup.c | 2 +- src/help.c | 7 +++++++ src/merge.c | 6 +++--- src/pg_probackup.h | 4 +++- src/restore.c | 5 +---- src/validate.c | 22 +++++++++++++++++++--- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index 146ab8364..b2ac6bb23 100644 --- a/src/backup.c +++ b/src/backup.c @@ -778,7 +778,7 @@ do_backup(time_t start_time, bool no_validate) write_backup(¤t); if (!no_validate) - pgBackupValidate(¤t); + pgBackupValidate(¤t, NULL); /* Notify user about backup size */ pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); diff --git a/src/help.c b/src/help.c index 85b1ed858..324995d6e 100644 --- a/src/help.c +++ b/src/help.c @@ -359,6 +359,7 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); @@ -396,6 +397,8 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + printf(_(" --db-include dbname restore only specified databases\n")); + printf(_(" --db-exclude dbname do not restore specified databases\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -440,6 +443,7 @@ help_validate(void) printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n\n")); + printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -458,6 +462,9 @@ help_validate(void) printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + printf(_(" --db-include dbname restore only specified databases\n")); + printf(_(" --db-exclude dbname do not restore specified databases\n")); + printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); diff --git a/src/merge.c b/src/merge.c index 832bd90cb..e3358bd9a 100644 --- a/src/merge.c +++ b/src/merge.c @@ -146,7 +146,7 @@ do_merge(time_t backup_id) merge_backups(full_backup, from_backup); } - pgBackupValidate(full_backup); + pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); @@ -198,7 +198,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (to_backup->status == BACKUP_STATUS_OK || to_backup->status == BACKUP_STATUS_DONE) { - pgBackupValidate(to_backup); + pgBackupValidate(to_backup, NULL); if (to_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Interrupt merging"); } @@ -211,7 +211,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) from_backup->status == BACKUP_STATUS_DONE || from_backup->status == BACKUP_STATUS_MERGING || from_backup->status == BACKUP_STATUS_DELETING); - pgBackupValidate(from_backup); + pgBackupValidate(from_backup, NULL); if (from_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Interrupt merging"); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1c49b7024..4a6ae7429 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -512,6 +512,8 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, const char *target_stop, const char *target_name, const char *target_action); +extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, + parray *datname_list, bool partial_restore_type); /* in merge.c */ extern void do_merge(time_t backup_id); @@ -556,7 +558,7 @@ extern void help_pg_probackup(void); extern void help_command(char *command); /* in validate.c */ -extern void pgBackupValidate(pgBackup* backup); +extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); extern int do_validate_all(void); /* in catalog.c */ diff --git a/src/restore.c b/src/restore.c index 12077107f..e81bdbef1 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,9 +44,6 @@ static void create_recovery_conf(time_t backup_id, pgRestoreParams *params); static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); - -static parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, bool partial_restore_type); static void set_orphan_status(parray *backups, pgBackup *parent_backup); /* @@ -376,7 +373,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } - pgBackupValidate(tmp_backup); + pgBackupValidate(tmp_backup, params); /* After pgBackupValidate() only following backup * states are possible: ERROR, RUNNING, CORRUPT and OK. * Validate WAL only for OK, because there is no point diff --git a/src/validate.c b/src/validate.c index 9ccc5ba8a..415b258e9 100644 --- a/src/validate.c +++ b/src/validate.c @@ -30,6 +30,7 @@ typedef struct uint32 checksum_version; uint32 backup_version; BackupMode backup_mode; + parray *dbOid_exclude_list; /* * Return value from the thread. @@ -42,7 +43,7 @@ typedef struct * Validate backup files. */ void -pgBackupValidate(pgBackup *backup) +pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { char base_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; @@ -54,6 +55,7 @@ pgBackupValidate(pgBackup *backup) pthread_t *threads; validate_files_arg *threads_args; int i; + parray *dbOid_exclude_list = NULL; /* Check backup version */ if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) @@ -105,6 +107,10 @@ pgBackupValidate(pgBackup *backup) pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST); + if (params && params->partial_db_list) + dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, + params->is_include_list); + /* setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -130,6 +136,7 @@ pgBackupValidate(pgBackup *backup) arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; arg->backup_version = parse_program_version(backup->program_version); + arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -193,6 +200,15 @@ pgBackupValidateFiles(void *arg) if (!S_ISREG(file->mode)) continue; + /* + * If in partial validate, check if the file belongs to the database + * we exclude. Only files from pgdata can be skipped. + */ + if (arguments->dbOid_exclude_list && file->external_dir_num == 0 + && parray_bsearch(arguments->dbOid_exclude_list, + &file->dbOid, pgCompareOid)) + continue; + /* * Currently we don't compute checksums for * cfs_compressed data files, so skip them. @@ -498,7 +514,7 @@ do_validate_instance(void) continue; } /* Valiate backup files*/ - pgBackupValidate(current_backup); + pgBackupValidate(current_backup, NULL); /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) @@ -593,7 +609,7 @@ do_validate_instance(void) continue; } /* Revalidate backup files*/ - pgBackupValidate(backup); + pgBackupValidate(backup, NULL); if (backup->status == BACKUP_STATUS_OK) { From 6a721aa00425d8fb427cb1991336958e5d66950e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 31 Jul 2019 12:58:46 +0300 Subject: [PATCH 0910/2107] Grab abnd log agent messages to stderr --- src/utils/file.c | 21 ++++++++++++++++++++- src/utils/file.h | 5 +++-- src/utils/remote.c | 9 ++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index ba0575da3..40f848a37 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -20,6 +20,7 @@ static __thread unsigned long fio_fdset = 0; static __thread void* fio_stdin_buffer; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; +static __thread int fio_stderr = 0; fio_location MyLocation; @@ -38,10 +39,28 @@ typedef struct #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) /* Use specified file descriptors as stdin/stdout for FIO functions */ -void fio_redirect(int in, int out) +void fio_redirect(int in, int out, int err) { fio_stdin = in; fio_stdout = out; + fio_stderr = err; +} + +void fio_error(int rc, int size, char const* file, int line) +{ + if (remote_agent) + { + fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", file, line, rc, size, rc >= 0 ? "end of data" : strerror(errno)); + exit(EXIT_FAILURE); + } + else + { + char buf[PRINTF_BUF_SIZE]; + int err_size = read(fio_stderr, buf, sizeof(buf)); + if (err_size > 0) + elog(LOG, "Agent error: %s", buf); + elog(ERROR, "Communication error: %s", rc >= 0 ? "end of data" : strerror(errno)); + } } /* Check if file descriptor is local or remote (created by FIO) */ diff --git a/src/utils/file.h b/src/utils/file.h index bb6101015..70dce4255 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -49,7 +49,7 @@ typedef enum #define PAGE_CHECKSUM_MISMATCH (-256) #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) { if (remote_agent) { fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", __FILE__, __LINE__, _rc, (int)(size), _rc >= 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } else elog(ERROR, "Communication error: %s", _rc >= 0 ? "end of data" : strerror(errno)); } } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) typedef struct { @@ -64,7 +64,7 @@ extern fio_location MyLocation; /* Check if FILE handle is local or remote (created by FIO) */ #define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX) -extern void fio_redirect(int in, int out); +extern void fio_redirect(int in, int out, int err); extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); @@ -77,6 +77,7 @@ extern int fio_fseek(FILE* f, off_t offs); extern int fio_ftruncate(FILE* f, off_t size); extern int fio_fclose(FILE* f); extern int fio_ffstat(FILE* f, struct stat* st); +extern void fio_error(int rc, int size, char const* file, int line); struct pgFile; extern int fio_send_pages(FILE* in, FILE* out, struct pgFile *file, XLogRecPtr horizonLsn, diff --git a/src/utils/remote.c b/src/utils/remote.c index 36b1a4188..574d38091 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -110,6 +110,7 @@ bool launch_agent(void) int ssh_argc; int outfd[2]; int infd[2]; + int errfd[2]; ssh_argc = 0; #ifdef WIN32 @@ -195,20 +196,25 @@ bool launch_agent(void) #else SYS_CHECK(pipe(infd)); SYS_CHECK(pipe(outfd)); + SYS_CHECK(pipe(errfd)); SYS_CHECK(child_pid = fork()); if (child_pid == 0) { /* child */ SYS_CHECK(close(STDIN_FILENO)); SYS_CHECK(close(STDOUT_FILENO)); + SYS_CHECK(close(STDERR_FILENO)); SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); + SYS_CHECK(dup2(errfd[1], STDERR_FILENO)); SYS_CHECK(close(infd[0])); SYS_CHECK(close(infd[1])); SYS_CHECK(close(outfd[0])); SYS_CHECK(close(outfd[1])); + SYS_CHECK(close(errfd[0])); + SYS_CHECK(close(errfd[1])); if (execvp(ssh_argv[0], ssh_argv) < 0) return false; @@ -217,9 +223,10 @@ bool launch_agent(void) elog(LOG, "Spawn agent %d version %s", child_pid, PROGRAM_VERSION); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); + SYS_CHECK(close(errfd[1])); /*atexit(kill_child);*/ - fio_redirect(infd[0], outfd[1]); /* write to stdout */ + fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } return true; } From daca2077350fd0e427db652250f41b5d46791aed Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 1 Aug 2019 11:43:17 +0300 Subject: [PATCH 0911/2107] Report ssh error if available --- src/utils/file.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 40f848a37..1b4989f2c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -58,8 +58,9 @@ void fio_error(int rc, int size, char const* file, int line) char buf[PRINTF_BUF_SIZE]; int err_size = read(fio_stderr, buf, sizeof(buf)); if (err_size > 0) - elog(LOG, "Agent error: %s", buf); - elog(ERROR, "Communication error: %s", rc >= 0 ? "end of data" : strerror(errno)); + elog(ERROR, "Agent error: %s", buf); + else + elog(ERROR, "Communication error: %s", rc >= 0 ? "end of data" : strerror(errno)); } } From cd00806f11351f389e46061d1eef364b3744a4cc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Aug 2019 12:50:38 +0300 Subject: [PATCH 0912/2107] fix elog message about agent version mismatch --- src/pg_probackup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3591d300a..59820f658 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -304,7 +304,7 @@ main(int argc, char *argv[]) uint32 agent_version = parse_program_version(remote_agent); elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, "Agent version %s doesn't match master pg_probackup version %s", - remote_agent, PROGRAM_VERSION); + PROGRAM_VERSION, remote_agent); } fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; From 87f214ef7283cbab2d46577718c6f6a59f3365c9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Aug 2019 13:24:55 +0300 Subject: [PATCH 0913/2107] tests: added validate.ValidateTest.test_recovery_target_lsn_backup_victim --- tests/pgpro560.py | 3 +++ tests/validate.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 3aada4319..de8302f56 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -3,6 +3,7 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess +from time import sleep module_name = 'pgpro560' @@ -98,6 +99,8 @@ def test_pgpro560_systemid_mismatch(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + sleep(1) + try: self.backup_node( backup_dir, 'node1', node2, diff --git a/tests/validate.py b/tests/validate.py index fce22c56f..939f0a1f7 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3499,7 +3499,7 @@ def test_validate_target_lsn(self): # @unittest.expectedFailure # @unittest.skip("skip") - def test_recovery_target_backup_victim(self): + def test_recovery_target_time_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup @@ -3555,6 +3555,61 @@ def test_recovery_target_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_lsn_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-lsn={0}'.format(target_lsn)]) # validate empty backup list # page from future during validate From dc76fb13d53af0be799df8b81853466aabda724a Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 9 Aug 2019 17:13:13 +0300 Subject: [PATCH 0914/2107] add verbose message --- src/help.c | 4 ++-- src/validate.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/help.c b/src/help.c index 324995d6e..cccef04b2 100644 --- a/src/help.c +++ b/src/help.c @@ -462,8 +462,8 @@ help_validate(void) printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - printf(_(" --db-include dbname restore only specified databases\n")); - printf(_(" --db-exclude dbname do not restore specified databases\n")); + printf(_(" --db-include dbname validate only files of specified databases\n")); + printf(_(" --db-exclude dbname do not validate files of specified databases\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/validate.c b/src/validate.c index 415b258e9..bcd3c35fc 100644 --- a/src/validate.c +++ b/src/validate.c @@ -207,7 +207,11 @@ pgBackupValidateFiles(void *arg) if (arguments->dbOid_exclude_list && file->external_dir_num == 0 && parray_bsearch(arguments->dbOid_exclude_list, &file->dbOid, pgCompareOid)) + { + elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"", + file->rel_path); continue; + } /* * Currently we don't compute checksums for From 454eeac8f84b557fadee7e808ea55dad64cfba89 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Aug 2019 20:40:01 +0300 Subject: [PATCH 0915/2107] tests: added validate.ValidateTest.test_partial_validate_exclude and validate.ValidateTest.test_partial_validate_include --- tests/restore.py | 43 ---------- tests/validate.py | 199 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 43 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index ec13d227f..5308faa92 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2360,49 +2360,6 @@ def test_lost_non_data_file(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") - def test_restore_specific_database_proof_of_concept(self): - """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off', - 'shared_buffers': '512MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=200) - - exit(1) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '100', '-c8', '-j2', '--no-vacuum']) - - pgbench.wait() - pgbench.stdout.close() - - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) - - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_exclude(self): """""" fname = self.id().split('.')[3] diff --git a/tests/validate.py b/tests/validate.py index 939f0a1f7..4bed793db 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3443,6 +3443,9 @@ def test_validate_corrupt_tablespace_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_target_lsn(self): @@ -3497,6 +3500,9 @@ def test_validate_target_lsn(self): '--recovery-target-timeline=2', '--recovery-target-lsn={0}'.format(target_lsn)]) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_time_backup_victim(self): @@ -3555,6 +3561,9 @@ def test_recovery_target_time_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_lsn_backup_victim(self): @@ -3611,6 +3620,196 @@ def test_recovery_target_lsn_backup_victim(self): backup_dir, 'node', options=['--recovery-target-lsn={0}'.format(target_lsn)]) + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_validate_empty_and_mangled_database_map(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with database_map + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + # truncate database_map + path = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'database_map') + with open(path, "w") as f: + f.close() + + try: + self.validate_pb( + backup_dir, 'node', + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Backup {0} data files are corrupted".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # mangle database_map + with open(path, "w") as f: + f.write("42") + f.close() + + try: + self.validate_pb( + backup_dir, 'node', + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Backup {0} data files are corrupted".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_validate_exclude(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + output = self.validate_pb( + backup_dir, 'node', + options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "--log-level-console=verbose"]) + + self.assertIn( + "VERBOSE: Skip file validation due to partial restore", output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_partial_validate_include(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + output = self.validate_pb( + backup_dir, 'node', + options=[ + "--db-include=db1", + "--db-include=db5", + "--db-include=postgres", + "--log-level-console=verbose"]) + + self.assertIn( + "VERBOSE: Skip file validation due to partial restore", output) + + output = self.validate_pb( + backup_dir, 'node', + options=[ + "--log-level-console=verbose"]) + + self.assertNotIn( + "VERBOSE: Skip file validation due to partial restore", output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # validate empty backup list # page from future during validate # page from future during backup From 08afc881525636cc2be196e5d71934c9f9163b4c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 9 Aug 2019 21:18:33 +0300 Subject: [PATCH 0916/2107] README: update windows installers link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57625c5ba..129e8c197 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Installation and Setup ### Windows Installation -[Installers download link](https://oc.postgrespro.ru/index.php/s/CGsjXlc5NmhRI0L) +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.5). ### Linux Installation ```shell From 0ae5a66bfbdb260056b67b8f7d2d7a85e1530fde Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 11 Aug 2019 13:54:43 +0300 Subject: [PATCH 0917/2107] partial_restore_type as separate type --- src/pg_probackup.c | 22 +++++++++++++++++++--- src/pg_probackup.h | 12 ++++++++++-- src/restore.c | 12 ++++++------ src/validate.c | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index eb6d40931..4ee959e51 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -609,6 +609,7 @@ main(int argc, char *argv[]) restore_params->skip_block_validation = skip_block_validation; restore_params->skip_external_dirs = skip_external_dirs; restore_params->partial_db_list = NULL; + restore_params->partial_restore_type = NONE; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) @@ -616,16 +617,21 @@ main(int argc, char *argv[]) if (datname_exclude_list) { - restore_params->is_include_list = false; + restore_params->partial_restore_type = EXCLUDE; restore_params->partial_db_list = datname_exclude_list; } else if (datname_include_list) { - restore_params->is_include_list = true; + restore_params->partial_restore_type = INCLUDE; restore_params->partial_db_list = datname_include_list; } } + /* sanity */ + if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) + elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", + command_name); + if (num_threads < 1) num_threads = 1; @@ -663,8 +669,15 @@ main(int argc, char *argv[]) restore_params); case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn) + { + /* sanity */ + if (datname_exclude_list || datname_include_list) + elog(ERROR, "You must specify parameter (-i, --backup-id) for partial validation"); + return do_validate_all(); + } else + /* PITR validation and, optionally, partial validation */ return do_restore_or_validate(current.backup_id, recovery_target_options, restore_params); @@ -797,7 +810,10 @@ opt_datname_include_list(ConfigOption *opt, const char *arg) dbname = pgut_malloc(strlen(arg) + 1); - /* TODO add sanity for database name */ + if (strcmp(dbname, "tempate0") == 0 || + strcmp(dbname, "tempate1") == 0) + elog(ERROR, "Databases 'template0' and 'template1' cannot be used for partial restore or validation"); + strcpy(dbname, arg); parray_append(datname_include_list, dbname); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4a6ae7429..366749993 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -91,6 +91,13 @@ typedef struct db_map_entry char *datname; } db_map_entry; +typedef enum PartialRestoreType +{ + NONE, + INCLUDE, + EXCLUDE, +} PartialRestoreType; + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, @@ -348,7 +355,7 @@ typedef struct pgRestoreParams bool skip_block_validation; /* options for partial restore */ - bool is_include_list; + PartialRestoreType partial_restore_type; parray *partial_db_list; } pgRestoreParams; @@ -512,8 +519,9 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, const char *target_stop, const char *target_name, const char *target_action); + extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, bool partial_restore_type); + parray *datname_list, PartialRestoreType partial_restore_type); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index e81bdbef1..ed401d14e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -459,7 +459,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->partial_db_list) dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, params->partial_db_list, - params->is_include_list); + params->partial_restore_type); /* * Restore dest_backup internal directories. @@ -1183,7 +1183,7 @@ parseRecoveryTargetOptions(const char *target_time, */ parray * get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, bool is_include_list) + parray *datname_list, PartialRestoreType partial_restore_type) { int i; int j; @@ -1233,9 +1233,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, * So we have a list of datnames and a database_map for it. * We must construct a list of dbOids to exclude. */ - if (is_include_list) + if (partial_restore_type == INCLUDE) { - /* For 'include' keep dbOid of every datname NOT specified by user */ + /* For 'include', keep dbOid of every datname NOT specified by user */ for (i = 0; i < parray_num(datname_list); i++) { bool found_match = false; @@ -1270,9 +1270,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, parray_append(dbOid_exclude_list, &db_entry->dbOid); } } - else + else if (partial_restore_type == EXCLUDE) { - /* For exclude job is easier, find dbOid for every specified datname */ + /* For exclude, job is easier - find dbOid for every specified datname */ for (i = 0; i < parray_num(datname_list); i++) { bool found_match = false; diff --git a/src/validate.c b/src/validate.c index bcd3c35fc..6bb8d5afe 100644 --- a/src/validate.c +++ b/src/validate.c @@ -109,7 +109,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) if (params && params->partial_db_list) dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, - params->is_include_list); + params->partial_restore_type); /* setup threads */ for (i = 0; i < parray_num(files); i++) From 57d3e125848c86a8b7f593abb939c40ede15d4b3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 11 Aug 2019 13:54:58 +0300 Subject: [PATCH 0918/2107] tests: fixes --- tests/validate.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index 4bed793db..eb7b3c0f9 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3659,7 +3659,7 @@ def test_partial_validate_empty_and_mangled_database_map(self): try: self.validate_pb( backup_dir, 'node', - options=["--db-include=db1", '--no-validate']) + options=["--db-include=db1"]) self.assertEqual( 1, 0, "Expecting Error because database_map is empty.\n " @@ -3680,7 +3680,7 @@ def test_partial_validate_empty_and_mangled_database_map(self): try: self.validate_pb( backup_dir, 'node', - options=["--db-include=db1", '--no-validate']) + options=["--db-include=db1"]) self.assertEqual( 1, 0, "Expecting Error because database_map is empty.\n " @@ -3735,8 +3735,27 @@ def test_partial_validate_exclude(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "--log-level-console=verbose"]) + self.assertEqual( + 1, 0, + "Expecting Error because of missing backup ID.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You must specify parameter (-i, --backup-id) for partial validation", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + output = self.validate_pb( - backup_dir, 'node', + backup_dir, 'node', backup_id, options=[ "--db-exclude=db1", "--db-exclude=db5", @@ -3789,7 +3808,7 @@ def test_partial_validate_include(self): repr(e.message), self.cmd)) output = self.validate_pb( - backup_dir, 'node', + backup_dir, 'node', backup_id, options=[ "--db-include=db1", "--db-include=db5", @@ -3800,9 +3819,8 @@ def test_partial_validate_include(self): "VERBOSE: Skip file validation due to partial restore", output) output = self.validate_pb( - backup_dir, 'node', - options=[ - "--log-level-console=verbose"]) + backup_dir, 'node', backup_id, + options=["--log-level-console=verbose"]) self.assertNotIn( "VERBOSE: Skip file validation due to partial restore", output) From ecd37a7a343dee6fc4db512670ad5f20838148eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Aug 2019 18:54:56 +0300 Subject: [PATCH 0919/2107] update help --- src/help.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/help.c b/src/help.c index cccef04b2..60f448592 100644 --- a/src/help.c +++ b/src/help.c @@ -143,6 +143,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -155,6 +156,7 @@ help_pg_probackup(void) printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); + printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--help]\n")); printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); @@ -397,6 +399,8 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + + printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); printf(_(" --db-exclude dbname do not restore specified databases\n")); @@ -462,6 +466,7 @@ help_validate(void) printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + printf(_("\n Partial validation options:\n")); printf(_(" --db-include dbname validate only files of specified databases\n")); printf(_(" --db-exclude dbname do not validate files of specified databases\n")); From 4edc414cb98ef31f039a6d68d643677b8547d392 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 12 Aug 2019 19:03:13 +0300 Subject: [PATCH 0920/2107] Documentation: partial restore update --- Documentation.md | 68 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 8a88b3309..708519232 100644 --- a/Documentation.md +++ b/Documentation.md @@ -18,6 +18,7 @@ Current version - 2.1.5 * [Setting up Backup from Standby](#setting-up-backup-from-standby) * [Setting up Cluster Verification](#setting-up-cluster-verification) * [Setting up PTRACK Backups](#setting-up-ptrack-backups) + * [Setting up Partial Restore](#setting-up-partial-restore) * [Configuring the Remote Mode](#configuring-the-remote-mode) 5. [Usage](#usage) @@ -28,7 +29,9 @@ Current version - 2.1.5 * [External directories](#external-directories) * [Verifying a Cluster](#verifying-a-cluster) * [Validating a Backup](#validating-a-backup) + * [Partial Validation](#partial-validation) * [Restoring a Cluster](#restoring-a-cluster) + * [Partial Restore](#partial-restore) * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) @@ -125,6 +128,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Backup from replica: avoid extra load on the master server by taking backups from a standby - External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files - Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats +- Partial Restore: restore the only specified databases or skip the specified databases. To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -518,6 +522,26 @@ If *backup_id* belong to incremental backup, then all its parents starting from If you omit all the parameters, all backups are validated. +#### Partial Validation + +If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can validate or exclude from validation the arbitraty number of specific databases using [partial restore options](#partial-restore-options) with the [validate](#validate) command. + +To validate only one or more databases, run the `validate` command with the following options: + + pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-include=database_name + +The option `--db-include` can be specified multiple times. For example, to validate only databases "db1" and "db2", run the following command: + + pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-include=db1 --db-include=db2 + +To exclude one or more specific databases from validation, run the following options: + + pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-exclude=database_name + +The option `--db-exclude` can be specified multiple times. For example, to exclude the databases "db1" and "db2" from validation, run the following command: + + pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 + ### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: @@ -542,6 +566,30 @@ If you are restoring an STREAM backup, the restore is complete at once, with the >NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` flag to skip validation and speed up the recovery. +#### Partial Restore + +If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can restore or exclude from restore the arbitraty number of specific databases using [partial restore options](#partial-restore-options) with the [restore](#restore) commands. + +To restore only one or more databases, run the restore command with the following options: + + pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name + +The option `--db-include` can be specified multiple times. For example, to restore only databases `db1` and `db2`, run the following command: + + pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 + +To exclude one or more specific databases from restore, run the following options: + + pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name + +The option `--db-exclude` can be specified multiple times. For example, to exclude the databases `db1` and `db2` from restore, run the following command: + + pg_probackup restore -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 + +Partial restore rely on lax behaviour of PostgreSQL recovery process toward truncated files. Files of excluded databases restored as null sized files, allowing recovery to work properly. After successfull starting of PostgreSQL cluster, you must drop excluded databases using `DROP DATABASE` command. + +>NOTE: The databases `template0` and `template1` are always restored. + ### Performing Point-in-Time (PITR) Recovery If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. @@ -1009,6 +1057,7 @@ For details on usage, see the section [Creating a Backup](#creating-a-backup). [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [recovery_options] [logging_options] [remote_options] + [partial_restore_options] Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a [recovery target option](#recovery-target-options), pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. @@ -1032,7 +1081,7 @@ Disables block-level checksum verification to speed up validation. During automa --no-validate Skips backup validation. You can use this flag if you validate backups regularly and would like to save time when running restore operations. -Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. +Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). @@ -1065,9 +1114,12 @@ For details on usage, see the section [Verifying a Cluster](#verifying-a-cluster [--help] [--instance instance_name] [-i backup_id] [-j num_threads] [--progress] [--skip-block-validation] - [recovery_options] [logging_options] + [recovery_target_options] [logging_options] + [partial_restore_options] + +Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target options](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. -Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target option](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. +If you specify the [partial restore options](#partial-restore-options) and a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. For details, see the section [Validating a Backup](#validating-a-backup). @@ -1354,6 +1406,16 @@ Specifies pg_probackup installation directory on the remote system. --ssh-options=ssh_options Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found on [ssh_config manual page](https://man.openbsd.org/ssh_config.5). +#### Partial Restore Options + +This section describes the options related to partial restore of cluster from backup. These options can be used with [restore](#restore) and [validate](#validate) commands. + + --db-exclude=dbname +Specifies database name to exclude from restore. All other databases in the cluster will be restored as usual, including `template0` and `template1`. This option can be specified multiple times for multiple databases. + + --db-include=dbname +Specifies database name to restore from backup. All other databases in the cluster will not be restored, with exception of `template0` and `template1`. This option can be specified multiple times for multiple databases. + #### Replica Options This section describes the options related to taking a backup from standby. From 0ff4ecd54796925bc9c101bdf6d1ee648c37b6d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 13 Aug 2019 11:54:48 +0300 Subject: [PATCH 0921/2107] Revert "Removed libpgfeutils.lib from Windows project" This reverts commit d38b5e6ece64a0bac09544c823f3f9c3dfcbf844. --- gen_probackup_project.pl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 511e7e07c..7845f03ea 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -198,10 +198,8 @@ sub build_pgprobackup $probackup->AddIncludeDir("$currpath/src"); $probackup->AddIncludeDir("$currpath/src/utils"); - # $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); - $probackup->AddReference($libpq, $libpgcommon, $libpgport); + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); $probackup->AddLibrary('ws2_32.lib'); - $probackup->AddLibrary('advapi32.lib'); $probackup->Save(); return $solution->{vcver}; From 1c9013eb0f8d4666f742743fb31cbdbc04de2ac5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 13 Aug 2019 19:23:47 +0300 Subject: [PATCH 0922/2107] Windows: 9.5 build --- gen_probackup_project.pl | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 7845f03ea..d1fc3c3e2 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -61,6 +61,16 @@ BEGIN my $libpq; my @unlink_on_exit; +if (-d "src/fe_utils") +{ + libpgfeutils = 1; +} +else +{ + libpgfeutils = 0; +} + + use lib "src/tools/msvc"; @@ -129,7 +139,10 @@ sub build_pgprobackup $libpq = $solution->AddProject('libpq', 'dll', 'interfaces', 'src/interfaces/libpq'); - $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + if ($libpgfeutils) + { + $libpgfeutils = $solution->AddProject('libpgfeutils', 'lib', 'misc'); + } $libpgcommon = $solution->AddProject('libpgcommon', 'lib', 'misc'); $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); @@ -198,7 +211,14 @@ sub build_pgprobackup $probackup->AddIncludeDir("$currpath/src"); $probackup->AddIncludeDir("$currpath/src/utils"); - $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + if (libpgfeutils) + { + $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); + } + else + { + $probackup->AddReference($libpq, $libpgcommon, $libpgport); + } $probackup->AddLibrary('ws2_32.lib'); $probackup->Save(); From b95a1589a4f26deacfe518f0376d4c3ec9dde70b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 13 Aug 2019 19:40:35 +0300 Subject: [PATCH 0923/2107] Windows: minor fix --- gen_probackup_project.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index d1fc3c3e2..4189b1b07 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -63,11 +63,11 @@ BEGIN if (-d "src/fe_utils") { - libpgfeutils = 1; + $libpgfeutils = 1; } else { - libpgfeutils = 0; + $libpgfeutils = 0; } @@ -211,7 +211,7 @@ sub build_pgprobackup $probackup->AddIncludeDir("$currpath/src"); $probackup->AddIncludeDir("$currpath/src/utils"); - if (libpgfeutils) + if ($libpgfeutils) { $probackup->AddReference($libpq, $libpgfeutils, $libpgcommon, $libpgport); } From aef926ffd0aaeadd169c0c98bcf08040fbf97f9f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 27 Aug 2019 19:24:10 +0300 Subject: [PATCH 0924/2107] check for potential errors when fgets() returns NULL --- src/dir.c | 12 ++++++++++++ src/restore.c | 3 +++ src/utils/configuration.c | 3 +++ 3 files changed, 18 insertions(+) diff --git a/src/dir.c b/src/dir.c index c122d3a3d..2f6d80901 100644 --- a/src/dir.c +++ b/src/dir.c @@ -514,6 +514,9 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink parray_append(black_list, pgut_strdup(black_item)); } + if (ferror(black_list_file)) + elog(ERROR, "Failed to read from file: \"%s\"", path); + fio_close_stream(black_list_file); parray_qsort(black_list, BlackListCompare); } @@ -1154,6 +1157,9 @@ read_tablespace_map(parray *files, const char *backup_dir) parray_append(files, file); } + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", map_path); + fio_close_stream(fp); } @@ -1529,6 +1535,9 @@ dir_read_file_list(const char *root, const char *external_prefix, parray_append(files, file); } + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", file_txt); + fio_close_stream(fp); return files; } @@ -1775,6 +1784,9 @@ read_database_map(pgBackup *backup) parray_append(database_map, db_entry); } + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", database_map_path); + fio_close_stream(fp); /* Return NULL if file is empty */ diff --git a/src/restore.c b/src/restore.c index ed401d14e..73a25929b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -996,6 +996,9 @@ read_timeline_history(TimeLineID targetTLI) /* we ignore the remainder of each line */ } + if (fd && (ferror(fd))) + elog(ERROR, "Failed to read from file: \"%s\"", path); + if (fd) fclose(fd); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index cf5436f29..74fc73898 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -523,6 +523,9 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, } } + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", path); + fio_close_stream(fp); return parsed_options; From dd124b34d9d95936fb18ebb41cb98baa8cb68b07 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 27 Aug 2019 20:01:38 +0300 Subject: [PATCH 0925/2107] [Issue #113] improve test coverage --- tests/merge.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index d17d6b603..ddcf2e850 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -111,8 +111,7 @@ def test_merge_compressed_backups(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"] - ) + initdb_params=["--data-checksums"]) self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) @@ -120,8 +119,7 @@ def test_merge_compressed_backups(self): node.slow_start() # Do full compressed backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib']) + self.backup_node(backup_dir, "node", node, options=['--compress']) show_backup = self.show_pb(backup_dir, "node")[0] self.assertEqual(show_backup["status"], "OK") @@ -137,8 +135,7 @@ def test_merge_compressed_backups(self): # Do compressed page backup self.backup_node( - backup_dir, "node", node, backup_type="page", - options=['--compress-algorithm=zlib']) + backup_dir, "node", node, backup_type="page", options=['--compress']) show_backup = self.show_pb(backup_dir, "node")[1] page_id = show_backup["id"] @@ -146,7 +143,7 @@ def test_merge_compressed_backups(self): self.assertEqual(show_backup["backup-mode"], "PAGE") # Merge all backups - self.merge_backup(backup_dir, "node", page_id) + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) show_backups = self.show_pb(backup_dir, "node") self.assertEqual(len(show_backups), 1) @@ -191,8 +188,7 @@ def test_merge_compressed_backups_1(self): node.pgbench_init(scale=5) # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib', '--stream']) + self.backup_node(backup_dir, "node", node, options=['--compress', '--stream']) show_backup = self.show_pb(backup_dir, "node")[0] self.assertEqual(show_backup["status"], "OK") @@ -204,8 +200,8 @@ def test_merge_compressed_backups_1(self): # Do compressed DELTA backup self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--compress-algorithm=zlib', '--stream']) + backup_dir, "node", node, + backup_type="delta", options=['--compress', '--stream']) # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) @@ -213,8 +209,7 @@ def test_merge_compressed_backups_1(self): # Do compressed PAGE backup self.backup_node( - backup_dir, "node", node, backup_type="page", - options=['--compress-algorithm=zlib']) + backup_dir, "node", node, backup_type="page", options=['--compress']) pgdata = self.pgdata_content(node.data_dir) @@ -225,7 +220,7 @@ def test_merge_compressed_backups_1(self): self.assertEqual(show_backup["backup-mode"], "PAGE") # Merge all backups - self.merge_backup(backup_dir, "node", page_id) + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) show_backups = self.show_pb(backup_dir, "node") self.assertEqual(len(show_backups), 1) @@ -282,15 +277,14 @@ def test_merge_compressed_and_uncompressed_backups(self): # Do compressed DELTA backup self.backup_node( backup_dir, "node", node, backup_type="delta", - options=['--compress-algorithm=zlib', '--stream']) + options=['--compress', '--stream']) # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) pgbench.wait() # Do uncompressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page") + self.backup_node(backup_dir, "node", node, backup_type="page") pgdata = self.pgdata_content(node.data_dir) @@ -301,7 +295,7 @@ def test_merge_compressed_and_uncompressed_backups(self): self.assertEqual(show_backup["backup-mode"], "PAGE") # Merge all backups - self.merge_backup(backup_dir, "node", page_id) + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) show_backups = self.show_pb(backup_dir, "node") self.assertEqual(len(show_backups), 1) From 9c4cdef76a0d91c500254c6727c7ec918c31ad81 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 27 Aug 2019 20:21:02 +0300 Subject: [PATCH 0926/2107] [Issue #113] bugfix: use rel_path attribute insead of path for sorting purposes and bsearch --- src/dir.c | 12 +++++++++++- src/merge.c | 8 ++++---- src/pg_probackup.h | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/dir.c b/src/dir.c index 2f6d80901..8732e7120 100644 --- a/src/dir.c +++ b/src/dir.c @@ -187,7 +187,7 @@ pgFile * pgFileInit(const char *path, const char *rel_path) { pgFile *file; - char *file_name; + char *file_name = NULL; file = (pgFile *) pgut_malloc(sizeof(pgFile)); MemSet(file, 0, sizeof(pgFile)); @@ -414,6 +414,16 @@ pgFileComparePathWithExternalDesc(const void *f1, const void *f2) return -pgFileComparePathWithExternal(f1, f2); } +/* + * Compare two pgFile with their rel_path and external_dir_num + * in descending order of ASCII code. + */ +int +pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2) +{ + return -pgFileCompareRelPathWithExternal(f1, f2); +} + /* Compare two pgFile with their linked directory path. */ int pgFileCompareLinked(const void *f1, const void *f2) diff --git a/src/merge.c b/src/merge.c index e3358bd9a..2d00fa78a 100644 --- a/src/merge.c +++ b/src/merge.c @@ -236,7 +236,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) DATABASE_FILE_LIST); to_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); /* To delete from leaf, sort in reversed order */ - parray_qsort(to_files, pgFileComparePathWithExternalDesc); + parray_qsort(to_files, pgFileCompareRelPathWithExternalDesc); /* * Get list of files which need to be moved. */ @@ -385,7 +385,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) /* * Delete files which are not in from_backup file list. */ - parray_qsort(files, pgFileComparePathWithExternalDesc); + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); for (i = 0; i < parray_num(to_files); i++) { pgFile *file = (pgFile *) parray_get(to_files, i); @@ -398,7 +398,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) continue; } - if (parray_bsearch(files, file, pgFileComparePathWithExternalDesc) == NULL) + if (parray_bsearch(files, file, pgFileCompareRelPathWithExternalDesc) == NULL) { char to_file_path[MAXPGPATH]; char *prev_path; @@ -488,7 +488,7 @@ merge_files(void *arg) i + 1, num_files, file->path); res_file = parray_bsearch(argument->to_files, file, - pgFileComparePathWithExternalDesc); + pgFileCompareRelPathWithExternalDesc); to_file = (res_file) ? *res_file : NULL; join_path_components(to_file_path, argument->to_root, file->path); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 40b8ccb03..690e4d333 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -668,6 +668,7 @@ extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileComparePathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); +extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileComparePathDesc(const void *f1, const void *f2); extern int pgFileComparePathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); From 8f1a45961a02b8a91ba4785bd6c2727533382842 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 28 Aug 2019 13:01:56 +0300 Subject: [PATCH 0927/2107] Issue #112: Set datestyle before pg_stop_backup() --- src/backup.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index b2ac6bb23..e83219072 100644 --- a/src/backup.c +++ b/src/backup.c @@ -612,7 +612,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) * check remote PostgreSQL instance. * Also checking system ID in this case serves no purpose, because * all work is done by server. - * + * * Returns established connection */ PGconn * @@ -1609,6 +1609,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, 0, NULL); PQclear(res); + /* Make proper timestamp format for parse_time() */ + res = pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); + PQclear(res); + /* Create restore point * Only if backup is from master. * For PG 9.5 create restore point only if pguser is superuser. From 2286c4bfe8264575a05f265f7dce02bdd26b4589 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Aug 2019 13:10:09 +0300 Subject: [PATCH 0928/2107] [Issue #112] tests: added time_stamp.CheckTimeStamp.test_server_date_style --- tests/time_stamp.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index adc6cd3e6..72115e1d4 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -52,3 +52,23 @@ def test_start_time_format(self): node.stop() # Clean after yourself self.del_test_dir(module_name, fname) + + def test_server_date_style(self): + """Issue #112""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={"datestyle": "'GERMAN, DMY'"}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From fb35cb83b6e344434e17956c3a03656afc28ee10 Mon Sep 17 00:00:00 2001 From: Arthur Zakirov Date: Wed, 28 Aug 2019 13:24:42 +0300 Subject: [PATCH 0929/2107] Issue #112: Try to not hang out --- src/utils/configuration.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 74fc73898..958025a25 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1171,6 +1171,8 @@ parse_time(const char *value, time_t *result, bool utc_default) /* wrong format */ else if (!IsSpace(*value)) return false; + else + value++; } tmp[len] = '\0'; From 5707a727b50cc9b23a1d8b652c46d1c654a350bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Aug 2019 18:52:22 +0300 Subject: [PATCH 0930/2107] added TODO comment for show --- src/show.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/show.c b/src/show.c index 4ba95ebc8..17592b267 100644 --- a/src/show.c +++ b/src/show.c @@ -284,6 +284,7 @@ show_backup(time_t requested_backup_id) backup = read_backup(requested_backup_id); if (backup == NULL) { + // TODO for 3.0: we should ERROR out here. elog(INFO, "Requested backup \"%s\" is not found.", /* We do not need free base36enc's result, we exit anyway */ base36enc(requested_backup_id)); From 8c570b57f1e2ec4cc0e612b1307ebd2ba556e033 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Aug 2019 20:08:42 +0300 Subject: [PATCH 0931/2107] tests: detect missing backup in case of using "show" with "--backup-id" option --- tests/helpers/ptrack_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index dc998ea1a..f7f8bbc70 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -881,6 +881,10 @@ def show_pb( return backup else: backup_list.append(backup) + + if backup_id is not None: + self.assertTrue(False, "Failed to find backup with ID: {0}".format(backup_id)) + return backup_list else: show_splitted = self.run_pb( @@ -935,6 +939,10 @@ def show_pb( var = var.strip('"') var = var.strip("'") specific_record[name.strip()] = var + + if not specific_record: + self.assertTrue(False, "Failed to find backup with ID: {0}".format(backup_id)) + return specific_record def validate_pb( From 5e618d44c52186634b06e1194fa542051da27751 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Aug 2019 20:12:13 +0300 Subject: [PATCH 0932/2107] tests: added retention.RetentionTest.test_failed_merge_redundancy_retention --- tests/retention.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/retention.py b/tests/retention.py index 3a85ee641..11e4e0333 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1501,3 +1501,99 @@ def test_wal_purge_victim(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_failed_merge_redundancy_retention(self): + """ + Check that retention purge works correctly with MERGING backups + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join( + module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL1 backup + full_id = self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # FULL2 backup + self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # FULL3 backup + self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=2']) + + self.set_config( + backup_dir, 'node', options=['--retention-window=2']) + + # create pair of MERGING backup as a result of failed merge + gdb = self.merge_backup( + backup_dir, 'node', delta_id, gdb=True) + gdb.set_breakpoint('copy_file') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + gdb._execute('signal SIGKILL') + + # "expire" first full backup + backups = os.path.join(backup_dir, 'backups', 'node') + with open( + os.path.join( + backups, full_id, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + # run retention merge + self.delete_expired( + backup_dir, 'node', options=['--delete-expired']) + + self.assertEqual( + 'MERGING', + self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "MERGING"') + + self.assertEqual( + 'MERGING', + self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "MERGING"') + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) + + # Clean after yourself + self.del_test_dir(module_name, fname) From f270d38b0ebbd3fb075bd5d839b2538c0d1599a7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Aug 2019 20:26:36 +0300 Subject: [PATCH 0933/2107] [Issue #114]: allow invalid backups to be included in to_keep_list during retention purge --- src/delete.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/delete.c b/src/delete.c index 387df3c9f..965f14935 100644 --- a/src/delete.c +++ b/src/delete.c @@ -332,10 +332,12 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Do not keep invalid backups by retention */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - continue; + /* Do not keep invalid backups by retention + * Turns out it was not a very good idea - [Issue #114] + */ + //if (backup->status != BACKUP_STATUS_OK && + // backup->status != BACKUP_STATUS_DONE) + // continue; /* only incremental backups should be in keep list */ if (backup->backup_mode == BACKUP_MODE_FULL) From b278a2fbf5d8e87fb9339c176f92b1fefdf0086c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 29 Aug 2019 23:05:53 +0300 Subject: [PATCH 0934/2107] tests: minor fixes --- tests/restore.py | 334 ++++++++++++++++++++++++++++------------------- 1 file changed, 200 insertions(+), 134 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index 5308faa92..0a409dfff 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2930,140 +2930,6 @@ def test_partial_restore_backward_compatibility_merge(self): self.compare_pgdata(pgdata_restored, pgdata_restored_1) - def test_missing_database_map(self): - """ - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - node.safe_psql( - "postgres", - "CREATE DATABASE backupdb") - - if self.get_version(node) > self.version_to_num('10.0'): - # bootstrap for 10/11 - node.safe_psql( - "backupdb", - "REVOKE ALL on SCHEMA public from public; " - "REVOKE ALL on SCHEMA pg_catalog from public; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE postgres to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - # we use it for partial restore and checkdb - # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; " - # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;" - ) - else: - # bootstrap for 9.5/9.6 - node.safe_psql( - "backupdb", - "REVOKE ALL on SCHEMA public from public; " - "REVOKE ALL on SCHEMA pg_catalog from public; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM public; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE postgres to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - # we use it for ptrack - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - # we use it for partial restore and checkdb - # "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; " - # "GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, oid) TO backup;" - ) - - # FULL backup without database_map - backup_id = self.backup_node( - backup_dir, 'node', node, datname='backupdb', - options=['--stream', "-U", "backup"]) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - - # backup has missing database_map and that is legal - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db5", "--db-exclude=db9"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has missing database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has missing database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # check that simple restore is still possible - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - def test_empty_and_mangled_database_map(self): """ """ @@ -3176,3 +3042,203 @@ def test_empty_and_mangled_database_map(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + def test_missing_database_map(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + node.safe_psql( + "postgres", + "CREATE DATABASE backupdb") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if self.ptrack: + for fname in [ + 'pg_catalog.oideq(oid, oid)', + 'pg_catalog.ptrack_version()', + 'pg_catalog.pg_ptrack_clear()', + 'pg_catalog.pg_ptrack_control_lsn()', + 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', + 'pg_catalog.pg_stop_backup()']: + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION {0} " + "TO backup".format(fname)) + + # FULL backup without database_map + backup_id = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', "-U", "backup", '--log-level-file=verbose']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + # backup has missing database_map and that is legal + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db5", "--db-exclude=db9"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # check that simple restore is still possible + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From d8b8b0898ecadb4d24867e13de59c519239e08aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 31 Aug 2019 02:16:45 +0300 Subject: [PATCH 0935/2107] [Issue #79]: remove partial validation --- src/catalog.c | 21 +++++++++++++++++++++ src/pg_probackup.h | 6 ++++-- src/restore.c | 39 ++++++++++++++++++++++++++------------- src/validate.c | 27 ++++++++++++++------------- 4 files changed, 65 insertions(+), 28 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 8deb31ee2..1b3f48a3c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -415,6 +415,27 @@ catalog_get_backup_list(time_t requested_backup_id) return NULL; } +/* + * Create list of backup datafiles. + * If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error. + * If valid backup id is passed only matching backup will be added to the list. + */ +parray * +get_backup_filelist(pgBackup *backup) +{ + parray *files = NULL; + char backup_filelist_path[MAXPGPATH]; + + pgBackupGetPath(backup, backup_filelist_path, lengthof(backup_filelist_path), DATABASE_FILE_LIST); + files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST); + + /* redundant sanity? */ + if (!files) + elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time)); + + return files; +} + /* * Lock list of backups. Function goes in backward direction. */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 690e4d333..aab114a80 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -521,8 +521,10 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_stop, const char *target_name, const char *target_action); -extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, PartialRestoreType partial_restore_type); +extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, + PartialRestoreType partial_restore_type); + +extern parray *get_backup_filelist(pgBackup *backup); /* in merge.c */ extern void do_merge(time_t backup_id); diff --git a/src/restore.c b/src/restore.c index 73a25929b..f4701c79e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -373,7 +373,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } } + /* validate datafiles only */ pgBackupValidate(tmp_backup, params); + /* After pgBackupValidate() only following backup * states are possible: ERROR, RUNNING, CORRUPT and OK. * Validate WAL only for OK, because there is no point @@ -390,6 +392,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* There is no point in wal validation of corrupted backups */ + // TODO: there should be a way for a user to request only(!) WAL validation if (!corrupted_backup) { /* @@ -458,7 +461,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * throw an error. */ if (params->partial_db_list) - dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, params->partial_db_list, + dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list, params->partial_restore_type); /* @@ -1185,19 +1188,22 @@ parseRecoveryTargetOptions(const char *target_time, * we always convert it into exclude_list. */ parray * -get_dbOid_exclude_list(pgBackup *backup, parray *files, - parray *datname_list, PartialRestoreType partial_restore_type) +get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, + PartialRestoreType partial_restore_type) { int i; int j; - parray *database_map = NULL; - parray *dbOid_exclude_list = NULL; - pgFile *database_map_file = NULL; - pg_crc32 crc; +// pg_crc32 crc; + parray *database_map = NULL; + parray *dbOid_exclude_list = NULL; + pgFile *database_map_file = NULL; char path[MAXPGPATH]; char database_map_path[MAXPGPATH]; + parray *files = NULL; + + files = get_backup_filelist(backup); - /* make sure that database_map is in backup_content.control */ + /* look for 'database_map' file in backup_content.control */ for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); @@ -1218,11 +1224,11 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, join_path_components(database_map_path, path, DATABASE_MAP); /* check database_map CRC */ - crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); - - if (crc != database_map_file->crc) - elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - database_map_file->path, crc, database_map_file->crc); +// crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); +// +// if (crc != database_map_file->crc) +// elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", +// database_map_file->path, crc, database_map_file->crc); /* get database_map from file */ database_map = read_database_map(backup); @@ -1307,6 +1313,13 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files, elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", base36enc(backup->start_time)); + /* clean backup filelist */ + if (files) + { + parray_walk(files, pgFileFree); + parray_free(files); + } + /* sort dbOid array in ASC order */ parray_qsort(dbOid_exclude_list, pgCompareOid); diff --git a/src/validate.c b/src/validate.c index 6bb8d5afe..d523f88ef 100644 --- a/src/validate.c +++ b/src/validate.c @@ -41,6 +41,7 @@ typedef struct /* * Validate backup files. + * TODO: partial validation. */ void pgBackupValidate(pgBackup *backup, pgRestoreParams *params) @@ -55,7 +56,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pthread_t *threads; validate_files_arg *threads_args; int i; - parray *dbOid_exclude_list = NULL; +// parray *dbOid_exclude_list = NULL; /* Check backup version */ if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) @@ -107,9 +108,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST); - if (params && params->partial_db_list) - dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, - params->partial_restore_type); +// if (params && params->partial_db_list) +// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, +// params->partial_restore_type); /* setup threads */ for (i = 0; i < parray_num(files); i++) @@ -136,7 +137,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; arg->backup_version = parse_program_version(backup->program_version); - arg->dbOid_exclude_list = dbOid_exclude_list; +// arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -204,14 +205,14 @@ pgBackupValidateFiles(void *arg) * If in partial validate, check if the file belongs to the database * we exclude. Only files from pgdata can be skipped. */ - if (arguments->dbOid_exclude_list && file->external_dir_num == 0 - && parray_bsearch(arguments->dbOid_exclude_list, - &file->dbOid, pgCompareOid)) - { - elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"", - file->rel_path); - continue; - } + //if (arguments->dbOid_exclude_list && file->external_dir_num == 0 + // && parray_bsearch(arguments->dbOid_exclude_list, + // &file->dbOid, pgCompareOid)) + //{ + // elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"", + // file->rel_path); + // continue; + //} /* * Currently we don't compute checksums for From eb9436bc1962a7e1d3f7178f0ffcf7216c3ec278 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 31 Aug 2019 03:28:42 +0300 Subject: [PATCH 0936/2107] [Issue #79]: remove partial validate from documentation --- Documentation.md | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/Documentation.md b/Documentation.md index 708519232..1556e1fbd 100644 --- a/Documentation.md +++ b/Documentation.md @@ -29,7 +29,6 @@ Current version - 2.1.5 * [External directories](#external-directories) * [Verifying a Cluster](#verifying-a-cluster) * [Validating a Backup](#validating-a-backup) - * [Partial Validation](#partial-validation) * [Restoring a Cluster](#restoring-a-cluster) * [Partial Restore](#partial-restore) * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) @@ -522,26 +521,6 @@ If *backup_id* belong to incremental backup, then all its parents starting from If you omit all the parameters, all backups are validated. -#### Partial Validation - -If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can validate or exclude from validation the arbitraty number of specific databases using [partial restore options](#partial-restore-options) with the [validate](#validate) command. - -To validate only one or more databases, run the `validate` command with the following options: - - pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-include=database_name - -The option `--db-include` can be specified multiple times. For example, to validate only databases "db1" and "db2", run the following command: - - pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-include=db1 --db-include=db2 - -To exclude one or more specific databases from validation, run the following options: - - pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-exclude=database_name - -The option `--db-exclude` can be specified multiple times. For example, to exclude the databases "db1" and "db2" from validation, run the following command: - - pg_probackup validate -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 - ### Restoring a Cluster To restore the database cluster from a backup, run the restore command with at least the following options: @@ -1115,12 +1094,9 @@ For details on usage, see the section [Verifying a Cluster](#verifying-a-cluster [-j num_threads] [--progress] [--skip-block-validation] [recovery_target_options] [logging_options] - [partial_restore_options] Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target options](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. -If you specify the [partial restore options](#partial-restore-options) and a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. - For details, see the section [Validating a Backup](#validating-a-backup). #### merge @@ -1408,7 +1384,7 @@ Specifies a string of SSH command-line options. For example, the following optio #### Partial Restore Options -This section describes the options related to partial restore of cluster from backup. These options can be used with [restore](#restore) and [validate](#validate) commands. +This section describes the options related to partial restore of cluster from backup. These options can be used with [restore](#restore) command. --db-exclude=dbname Specifies database name to exclude from restore. All other databases in the cluster will be restored as usual, including `template0` and `template1`. This option can be specified multiple times for multiple databases. From 0f0e961fead1a132e1f7126abaa28d25989ee7c4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 31 Aug 2019 03:30:57 +0300 Subject: [PATCH 0937/2107] [Issue #79]: remove partial validate from help --- src/help.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/help.c b/src/help.c index 60f448592..87a0e3312 100644 --- a/src/help.c +++ b/src/help.c @@ -156,7 +156,6 @@ help_pg_probackup(void) printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n")); - printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--help]\n")); printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); @@ -447,7 +446,6 @@ help_validate(void) printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n\n")); - printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -466,10 +464,6 @@ help_validate(void) printf(_(" the named restore point to which recovery will proceed\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - printf(_("\n Partial validation options:\n")); - printf(_(" --db-include dbname validate only files of specified databases\n")); - printf(_(" --db-exclude dbname do not validate files of specified databases\n")); - printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); From 2aea9cc7606b46a84dee7d7252ea095048e71cb1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 31 Aug 2019 15:48:52 +0300 Subject: [PATCH 0938/2107] print stop_lsn with elevel LOG --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index e83219072..178f08a89 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1808,6 +1808,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); } + elog(LOG, "current.stop_lsn: %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + /* Write backup_label and tablespace_map */ if (!exclusive_backup) { From 18211c5f1866144d786156afa9139a04a5e720c9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 09:30:45 +0300 Subject: [PATCH 0939/2107] [Issue #115] test coverage --- tests/replica.py | 437 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 435 insertions(+), 2 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 1d38d2659..0c3672064 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -4,6 +4,9 @@ from datetime import datetime, timedelta import subprocess import time +from distutils.dir_util import copy_tree +from testgres import ProcessType +from time import sleep module_name = 'replica' @@ -351,8 +354,7 @@ def test_take_backup_from_delayed_replica(self): base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s'}) + pg_options={'archive_timeout': '10s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -528,3 +530,434 @@ def test_replica_promote(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_stop_lsn_null_offset(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + output = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=verbose', + '--no-validate', + '--stream'], + return_id=False) + + self.assertIn( + 'WARNING: Invalid stop_backup_lsn value 0/3000000', + output) + + self.assertIn( + 'WARNING: WAL segment 000000010000000000000003 could not be streamed in 30 seconds', + output) + + self.assertIn( + 'WARNING: Failed to get next WAL record after 0/3000000, looking for previous WAL record', + output) + + self.assertIn( + 'LOG: Looking for LSN 0/3000000 in segment: 000000010000000000000002', + output) + + self.assertIn( + 'INFO: Wait for LSN 0/3000000 in streamed WAL segment', + output) + + self.assertIn( + 'LOG: Record 0/2000160 has endpoint 0/3000000 which is ' + 'equal or greater than requested LSN 0/3000000', + output) + + self.assertIn( + 'LOG: Found prior LSN: 0/2000160', + output) + + self.assertIn( + 'LOG: current.stop_lsn: 0/2000160', + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_stop_lsn_null_offset_next_record(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + # open connection to master + conn = master.connect() + + gdb = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=40', + '--log-level-file=verbose', + '--no-validate', + '--stream'], + gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb.continue_execution_until_running() + + sleep(5) + + conn.execute("create table t1()") + conn.commit() + + while 'RUNNING' in self.show_pb(backup_dir, 'replica')[0]['status']: + sleep(5) + + file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + + with open(file) as f: + log_content = f.read() + + self.assertIn( + 'WARNING: Invalid stop_backup_lsn value 0/3000000', + log_content) + + self.assertIn( + 'LOG: Looking for segment: 000000010000000000000003', + log_content) + + self.assertIn( + 'LOG: First record in WAL segment "000000010000000000000003": 0/3000028', + log_content) + + self.assertIn( + 'LOG: current.stop_lsn: 0/3000028', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_replica_null_offset(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + # take backup from replica + output = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=verbose', + '--no-validate'], + return_id=False) + + self.assertIn( + 'WARNING: Invalid stop_backup_lsn value 0/3000000', + output) + + self.assertIn( + 'WARNING: WAL segment 000000010000000000000003 could not be archived in 30 seconds', + output) + + self.assertIn( + 'WARNING: Failed to get next WAL record after 0/3000000, looking for previous WAL record', + output) + + self.assertIn( + 'LOG: Looking for LSN 0/3000000 in segment: 000000010000000000000002', + output) + + self.assertIn( + 'LOG: Record 0/2000160 has endpoint 0/3000000 which is ' + 'equal or greater than requested LSN 0/3000000', + output) + + self.assertIn( + 'LOG: Found prior LSN: 0/2000160', + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_archive_replica_not_null_offset(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + # take backup from replica + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=verbose', + '--no-validate'], + return_id=False) + + try: + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=verbose', + '--no-validate']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of archive timeout. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'LOG: Looking for LSN 0/3000060 in segment: 000000010000000000000003', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Wait for LSN 0/3000060 in archived WAL segment', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + 'ERROR: WAL segment 000000010000000000000003 could not be archived in 30 seconds', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_toast(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + # open connection to master + output = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=verbose', + '--log-level-file=verbose', + '--no-validate', + '--stream'], + return_id=False) + + pgdata = self.pgdata_content(replica.data_dir) + + self.assertIn( + 'WARNING: Could not read WAL record at', + output) + + self.assertIn( + 'LOG: Found prior LSN:', + output) + + replica.cleanup() + self.restore_node(backup_dir, 'replica', replica) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# TODO: +# null offset STOP LSN and latest record in previous segment is conrecord (manual only) +# archiving from promoted delayed replica From 7c5e799c2194178a4581994cfad95b37148380b1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 10:46:01 +0300 Subject: [PATCH 0940/2107] [Issue #115] look-forward approach of handling the invalid stop_lsn received from replica --- src/backup.c | 193 ++++++++++++++++++++++++++++++--------------- src/data.c | 6 +- src/parsexlog.c | 69 ++++++++++++---- src/pg_probackup.h | 3 + 4 files changed, 191 insertions(+), 80 deletions(-) diff --git a/src/backup.c b/src/backup.c index 178f08a89..2285d371f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -98,9 +98,10 @@ static void pg_switch_wal(PGconn *conn); static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static int checkpoint_timeout(PGconn *backup_conn); -//static void backup_list_file(parray *files, const char *root, ) -static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, - bool wait_prev_segment); +static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, + bool in_prev_segment, bool segment_only, + int timeout_elevel, bool in_stream_dir); + static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn); static void *StreamLog(void *arg); static void IdentifySystem(StreamThreadArg *stream_thread_arg); @@ -506,6 +507,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* Notify end of backup */ pg_stop_backup(¤t, pg_startbackup_conn, nodeInfo); + elog(LOG, "current.stop_lsn: %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. */ @@ -992,7 +996,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) /* In PAGE mode wait for current segment... */ - wait_wal_lsn(backup->start_lsn, true, false); + wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false); /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream @@ -1000,7 +1004,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, */ else if (!stream_wal) /* ...for others wait for previous segment */ - wait_wal_lsn(backup->start_lsn, true, true); + wait_wal_lsn(backup->start_lsn, true, backup->tli, true, true, ERROR, false); } /* @@ -1395,25 +1399,34 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, } /* - * Wait for target 'lsn'. + * Wait for target 'lsn' or WAL segment, containing 'lsn'. * * If current backup started in archive mode wait for 'lsn' to be archived in * archive 'wal' directory with WAL segment file. * If current backup started in stream mode wait for 'lsn' to be streamed in * 'pg_wal' directory. * - * If 'is_start_lsn' is true and backup mode is PAGE then we wait for 'lsn' to - * be archived in archive 'wal' directory regardless stream mode. + * If 'is_start_lsn' is true then issue warning for first-time users. * - * If 'wait_prev_segment' wait for previous segment. + * If 'in_prev_segment' is set, look for LSN in previous segment. + * If 'segment_only' is set, then instead of looking for LSN, look for segment itself. + * If 'in_prev_segment' and 'segment_only' are both set, then wait for previous segment. + * + * Flag 'in_stream_dir' determine whether we looking for wal in 'pg_wal' directory or + * in archive. Do note, that we cannot rely sorely on 'stream_wal' because, for example, + * PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. + + * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than + * ERROR is used, then return InvalidXLogRecPtr. TODO: return something more concrete, for example 1. * * Returns LSN of last valid record if wait_prev_segment is not true, otherwise * returns InvalidXLogRecPtr. */ static XLogRecPtr -wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) +wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, + bool in_prev_segment, bool segment_only, + int timeout_elevel, bool in_stream_dir) { - TimeLineID tli; XLogSegNo targetSegNo; char pg_wal_dir[MAXPGPATH]; char wal_segment_path[MAXPGPATH], @@ -1422,16 +1435,15 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) bool file_exists = false; uint32 try_count = 0, timeout; + char *wal_delivery_str = in_stream_dir ? "streamed":"archived"; #ifdef HAVE_LIBZ char gz_wal_segment_path[MAXPGPATH]; #endif - tli = get_current_timeline(false); - /* Compute the name of the WAL file containing requested LSN */ GetXLogSegNo(lsn, targetSegNo, instance_config.xlog_seg_size); - if (wait_prev_segment) + if (in_prev_segment) targetSegNo--; GetXLogFileName(wal_segment, tli, targetSegNo, instance_config.xlog_seg_size); @@ -1443,8 +1455,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) * * In pg_stop_backup it depends only on stream_wal. */ - if (stream_wal && - (current.backup_mode != BACKUP_MODE_DIFF_PAGE || !is_start_lsn)) + if (in_stream_dir) { pgBackupGetPath2(¤t, pg_wal_dir, lengthof(pg_wal_dir), DATABASE_DIR, PG_XLOG_DIR); @@ -1462,7 +1473,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) else timeout = ARCHIVE_TIMEOUT_DEFAULT; - if (wait_prev_segment) + if (segment_only) elog(LOG, "Looking for segment: %s", wal_segment); else elog(LOG, "Looking for LSN %X/%X in segment: %s", @@ -1496,14 +1507,15 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (file_exists) { /* Do not check LSN for previous WAL segment */ - if (wait_prev_segment) + if (segment_only) return InvalidXLogRecPtr; /* * A WAL segment found. Check LSN on it. */ - if (wal_contains_lsn(wal_segment_dir, lsn, tli, - instance_config.xlog_seg_size)) + if (!XRecOffIsNull(lsn) && + wal_contains_lsn(wal_segment_dir, lsn, tli, + instance_config.xlog_seg_size)) /* Target LSN was found */ { elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); @@ -1514,19 +1526,24 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) * If we failed to get LSN of valid record in a reasonable time, try * to get LSN of last valid record prior to the target LSN. But only * in case of a backup from a replica. + + * There are two cases for this: + * 1. Replica returned readpoint LSN which just do not exists. We want to look + * for previous record in the same(!) WAL segment which endpoint points to this LSN. + * 2. Replica returened endpoint LSN with 0 offset. We want to look + * for previous record which endpoint points greater or equal LSN in previous WAL segment. */ - if (!exclusive_backup && current.from_replica && - (try_count > timeout / 4)) + if (!exclusive_backup && current.from_replica && try_count > timeout / 2) { XLogRecPtr res; - res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, - lsn, tli, false, - instance_config.xlog_seg_size); + res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, lsn, tli, + in_prev_segment, instance_config.xlog_seg_size); + if (!XLogRecPtrIsInvalid(res)) { /* LSN of the prior record was found */ - elog(LOG, "Found prior LSN: %X/%X, it is used as stop LSN", + elog(LOG, "Found prior LSN: %X/%X", (uint32) (res >> 32), (uint32) res); return res; } @@ -1541,12 +1558,13 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) /* Inform user if WAL segment is absent in first attempt */ if (try_count == 1) { - if (wait_prev_segment) - elog(INFO, "Wait for WAL segment %s to be archived", - wal_segment_path); + if (segment_only) + elog(INFO, "Wait for WAL segment %s to be %s", + wal_segment_path, wal_delivery_str); else - elog(INFO, "Wait for LSN %X/%X in archived WAL segment %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_segment_path); + elog(INFO, "Wait for LSN %X/%X in %s WAL segment %s", + (uint32) (lsn >> 32), (uint32) lsn, + wal_delivery_str, wal_segment_path); } if (!stream_wal && is_start_lsn && try_count == 30) @@ -1557,14 +1575,17 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment) if (timeout > 0 && try_count > timeout) { if (file_exists) - elog(ERROR, "WAL segment %s was archived, " + elog(timeout_elevel, "WAL segment %s was %s, " "but target LSN %X/%X could not be archived in %d seconds", - wal_segment, (uint32) (lsn >> 32), (uint32) lsn, timeout); + wal_segment, wal_delivery_str, + (uint32) (lsn >> 32), (uint32) lsn, timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else - elog(ERROR, - "Switched WAL segment %s could not be archived in %d seconds", - wal_segment, timeout); + elog(timeout_elevel, + "WAL segment %s could not be %s in %d seconds", + wal_segment, wal_delivery_str, timeout); + + return InvalidXLogRecPtr; } } } @@ -1591,6 +1612,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, char *val = NULL; char *stop_backup_query = NULL; bool stop_lsn_exists = false; + XLogRecPtr stop_backup_lsn_tmp = InvalidXLogRecPtr; /* * We will use this values if there are no transactions between start_lsn @@ -1771,17 +1793,28 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* Extract timeline and LSN from results of pg_stop_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 2), &lsn_hi, &lsn_lo); /* Calculate LSN */ - stop_backup_lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + stop_backup_lsn_tmp = ((uint64) lsn_hi) << 32 | lsn_lo; - if (!XRecOffIsValid(stop_backup_lsn)) + if (!XRecOffIsValid(stop_backup_lsn_tmp)) { - if (XRecOffIsNull(stop_backup_lsn)) + /* Replica returned STOP LSN with null offset */ + if (XRecOffIsNull(stop_backup_lsn_tmp)) { char *xlog_path, stream_xlog_path[MAXPGPATH]; + XLogSegNo segno = 0; + XLogRecPtr lsn_tmp = InvalidXLogRecPtr; elog(WARNING, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + + /* + * Note: even with gdb it is very hard to produce automated tests for + * contrecord + null_offset STOP_LSN, so emulate it for manual testing. + */ + //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; + //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", + // (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); if (stream_wal) { @@ -1793,24 +1826,62 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, else xlog_path = arclog_path; - wait_wal_lsn(stop_backup_lsn, false, true); - stop_backup_lsn = get_last_wal_lsn(xlog_path, backup->start_lsn, - stop_backup_lsn, backup->tli, - true, instance_config.xlog_seg_size); + GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); + /* - * Do not check existance of LSN again below using - * wait_wal_lsn(). + * Note, that there is no guarantee that corresponding WAL file is even exists. + * Basically replica may return LSN from future and keep staying in present. + * Yeah, it sucks. + * + * So we should try to do the following: + * 1. Wait for current segment and look in it for the LSN >= STOP_LSN. It should + * solve the problem of occasional 0 offset on write-busy system. + * 2. Failing that, look for record in previous segment with endpoint + * equal or greater than 0 offset LSN. It may(!) solve the problem of 0 offset + * on write-idle system. */ + + /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ + wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + false, true, WARNING, stream_wal); + + /* Optimistically try to get the first record in segment with current stop_lsn */ + lsn_tmp = get_first_wal_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size); + + /* Check if returned LSN is satisfying our requirements */ + if (XLogRecPtrIsInvalid(lsn_tmp) || + !XRecOffIsValid(lsn_tmp) || + lsn_tmp < stop_backup_lsn_tmp) + { + /* No luck, falling back to looking up for previous record */ + elog(WARNING, "Failed to get next WAL record after %X/%X, " + "looking for previous WAL record", + (uint32) (stop_backup_lsn_tmp >> 32), + (uint32) (stop_backup_lsn_tmp)); + + /* Despite looking for previous record there is not guarantee of success + * because previous record can be the contrecord. + */ + lsn_tmp = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + true, false, ERROR, stream_wal); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record prior to %X/%X", + (uint32) (stop_backup_lsn_tmp >> 32), + (uint32) (stop_backup_lsn_tmp)); + } + + /* Setting stop_backup_lsn will set stop point for streaming */ + stop_backup_lsn = lsn_tmp; stop_lsn_exists = true; } else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); } - elog(LOG, "current.stop_lsn: %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); - /* Write backup_label and tablespace_map */ if (!exclusive_backup) { @@ -1900,14 +1971,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (tablespace_map_content) PQclear(tablespace_map_content); PQclear(res); - - if (stream_wal) - { - /* Wait for the completion of stream */ - pthread_join(stream_thread, NULL); - if (stream_thread_arg.ret == 1) - elog(ERROR, "WAL streaming failed"); - } } /* Fill in fields if that is the correct end of backup. */ @@ -1918,13 +1981,20 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* * Wait for stop_lsn to be archived or streamed. - * We wait for stop_lsn in stream mode just in case. + * If replica returned non-existent LSN, look for previous record, + * which endpoint >= stop_lsn */ if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(stop_backup_lsn, false, false); + stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + false, false, ERROR, stream_wal); if (stream_wal) { + /* Wait for the completion of stream */ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); + pgBackupGetPath2(backup, stream_xlog_path, lengthof(stream_xlog_path), DATABASE_DIR, PG_XLOG_DIR); @@ -1933,7 +2003,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, else xlog_path = arclog_path; - backup->tli = get_current_timeline(false); backup->stop_lsn = stop_backup_lsn; elog(LOG, "Getting the Recovery Time from WAL"); @@ -1944,7 +2013,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, backup->start_lsn, backup->stop_lsn, &backup->recovery_time, &backup->recovery_xid)) { - elog(LOG, "Failed to find Recovery Time in WAL. Forced to trust current_timestamp"); + elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); backup->recovery_time = recovery_time; backup->recovery_xid = recovery_xid; } diff --git a/src/data.c b/src/data.c index 82994a179..680f38c12 100644 --- a/src/data.c +++ b/src/data.c @@ -209,7 +209,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, /* The block could have been truncated. It is fine. */ if (read_len == 0) { - elog(LOG, "File %s, block %u, file was truncated", + elog(VERBOSE, "File %s, block %u, file was truncated", file->path, blknum); return 0; } @@ -238,7 +238,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, /* Page is zeroed. No need to check header and checksum. */ if (i == BLCKSZ) { - elog(LOG, "File: %s blknum %u, empty page", file->path, blknum); + elog(VERBOSE, "File: %s blknum %u, empty page", file->path, blknum); return 1; } @@ -422,7 +422,7 @@ prepare_page(ConnectionArgs *arguments, page_lsn && page_lsn < prev_backup_start_lsn) { - elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path); + elog(VERBOSE, "Skipping blknum %u in file: %s", blknum, file->path); (*n_skipped)++; return SkipCurrentPage; } diff --git a/src/parsexlog.c b/src/parsexlog.c index 9008b5285..d9437b19d 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -528,6 +528,51 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, return res; } +/* + * Get LSN of first record within the WAL segment with number 'segno'. + * + * Returns LSN which points to end+1 of the last WAL record if seek_prev_segment + * is true. Otherwise returns LSN of the record prior to stop_lsn. + */ +XLogRecPtr +get_first_wal_lsn(const char *archivedir, XLogSegNo segno, + TimeLineID tli, uint32 wal_seg_size) +{ + XLogReaderState *xlogreader; + XLogReaderData reader_data; + XLogRecPtr record = InvalidXLogRecPtr; + XLogRecPtr startpoint; + char wal_segment[MAXFNAMELEN]; + + if (segno <= 1) + elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); + + GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); + + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, false, true); + if (xlogreader == NULL) + elog(ERROR, "Out of memory"); + xlogreader->system_identifier = instance_config.system_identifier; + + /* Set startpoint to 0 in segno */ + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); + + record = XLogFindNextRecord(xlogreader, startpoint); + + if (XLogRecPtrIsInvalid(record)) + record = InvalidXLogRecPtr; + else + elog(LOG, "First record in WAL segment \"%s\": %X/%X", wal_segment, + (uint32) (record >> 32), (uint32) (record)); + + /* cleanup */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return record; +} + /* * Get LSN of last or prior record within the WAL segment with number 'segno'. * If 'start_lsn' @@ -596,7 +641,6 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, { XLogRecord *record; char *errormsg; - XLogSegNo next_segno = 0; if (interrupted) elog(ERROR, "Interrupted during WAL reading"); @@ -619,23 +663,18 @@ get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, PrintXLogCorruptionMsg(&reader_data, ERROR); } - /* continue reading at next record */ - startpoint = InvalidXLogRecPtr; - - GetXLogSegNo(xlogreader->EndRecPtr, next_segno, wal_seg_size); - if (next_segno > segno) - break; - - if (seek_prev_segment) + if (xlogreader->EndRecPtr >= stop_lsn) { - /* end+1 of last record read */ - res = xlogreader->EndRecPtr; - } - else + elog(LOG, "Record %X/%X has endpoint %X/%X which is equal or greater than requested LSN %X/%X", + (uint32) (xlogreader->ReadRecPtr >> 32), (uint32) (xlogreader->ReadRecPtr), + (uint32) (xlogreader->EndRecPtr >> 32), (uint32) (xlogreader->EndRecPtr), + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); res = xlogreader->ReadRecPtr; - - if (xlogreader->EndRecPtr >= stop_lsn) break; + } + + /* continue reading at next record */ + startpoint = InvalidXLogRecPtr; } CleanupXLogPageRead(xlogreader); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index aab114a80..7e1f3ed83 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -715,6 +715,9 @@ extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, uint32 seg_size); +extern XLogRecPtr get_first_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, + TimeLineID tli, uint32 wal_seg_size); + /* in util.c */ extern TimeLineID get_current_timeline(bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); From a42ebbff0c95336c667ddf5a7be4434d1a3300ac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 11:12:17 +0300 Subject: [PATCH 0941/2107] tests: minor fixes --- tests/delta.py | 2 +- tests/exclude.py | 1 - tests/helpers/ptrack_helpers.py | 4 +++- tests/restore.py | 7 +++++-- tests/validate.py | 3 +++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/delta.py b/tests/delta.py index 6a5a9d736..291d12dbe 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1278,7 +1278,7 @@ def test_delta_nullified_heap_page_backup(self): "LOG: File: {0} blknum 1, empty page".format(file), content) self.assertNotIn( - "Skipping blknum: 1 in file: {0}".format(file), + "Skipping blknum 1 in file: {0}".format(file), content) # Restore DELTA backup diff --git a/tests/exclude.py b/tests/exclude.py index 2e644d7e9..0402c978d 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -26,7 +26,6 @@ def test_exclude_temp_tables(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - conn = node.connect() with node.connect("postgres") as conn: conn.execute( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f7f8bbc70..d736a364b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -326,7 +326,9 @@ def make_simple_node( node.append_conf('postgresql.auto.conf', 'max_connections = 100') node.append_conf('postgresql.auto.conf', 'shared_buffers = 10MB') node.append_conf('postgresql.auto.conf', 'fsync = off') - node.append_conf('postgresql.auto.conf', 'wal_level = logical') + + if 'wal_level' not in pg_options: + node.append_conf('postgresql.auto.conf', 'wal_level = logical') node.append_conf('postgresql.auto.conf', 'hot_standby = off') node.append_conf( diff --git a/tests/restore.py b/tests/restore.py index 0a409dfff..bf3e028f7 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2771,7 +2771,8 @@ def test_partial_restore_backward_compatibility_1(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Backup {0} has missing database_map".format(backup_id), + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible".format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -2823,6 +2824,7 @@ def test_partial_restore_backward_compatibility_1(self): node_restored_1, options=[ "--db-exclude=db5", "--db-exclude=db14"]) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) self.compare_pgdata(pgdata_restored, pgdata_restored_1) @@ -2871,7 +2873,8 @@ def test_partial_restore_backward_compatibility_merge(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Backup {0} has missing database_map".format(backup_id), + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/validate.py b/tests/validate.py index eb7b3c0f9..d7372699e 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3623,6 +3623,7 @@ def test_recovery_target_lsn_backup_victim(self): # Clean after yourself self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_partial_validate_empty_and_mangled_database_map(self): """ """ @@ -3696,6 +3697,7 @@ def test_partial_validate_empty_and_mangled_database_map(self): # Clean after yourself self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_partial_validate_exclude(self): """""" fname = self.id().split('.')[3] @@ -3767,6 +3769,7 @@ def test_partial_validate_exclude(self): # Clean after yourself self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_partial_validate_include(self): """ """ From be339bf4066cd500db58e714e97b9827d853b00f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 13:09:13 +0300 Subject: [PATCH 0942/2107] tests: minor fix --- tests/archive.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 3f02e8d51..b24eb15c3 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -578,12 +578,12 @@ def test_archive_push_partial_file_exists(self): backup_dir, 'node', options=['--recovery-target-xid={0}'.format(xid)]) - # log_file = os.path.join(node.logs_dir, 'postgresql.log') - # with open(log_file, 'r') as f: - # log_content = f.read() - # self.assertIn( - # 'Reusing stale destination temporary WAL file', - # log_content) + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertIn( + 'Reusing stale destination temporary WAL file', + log_content) # Clean after yourself self.del_test_dir(module_name, fname) From 8630b0d1a324663e434556d05306af3d425c841e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 13:54:28 +0300 Subject: [PATCH 0943/2107] [Issue #117] test coverage --- tests/replica.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/replica.py b/tests/replica.py index 0c3672064..f767c4eac 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -958,6 +958,73 @@ def test_replica_toast(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_replica_promote_1(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + # set replica True, so archive_mode 'always' is used. + self.set_archiving(backup_dir, 'master', master, replica=True) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + self.wait_until_replica_catch_with_master(master, replica) + + wal_file = os.path.join( + backup_dir, 'wal', 'master', '000000010000000000000004') + + wal_file_partial = os.path.join( + backup_dir, 'wal', 'master', '000000010000000000000004.partial') + + self.assertFalse(os.path.exists(wal_file)) + + replica.promote() + + while not os.path.exists(wal_file_partial): + sleep(1) + + self.switch_wal_segment(master) + + # sleep to be sure, that any partial timeout is expired + sleep(70) + + self.assertTrue( + os.path.exists(wal_file_partial, + "File {0} disappeared".format(wal_file_partial))) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) # archiving from promoted delayed replica From 62acd2de2dd599e19f31941ece5fcac6879e2f92 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 15:41:13 +0300 Subject: [PATCH 0944/2107] [Issue #117] use suffix ".part" instead of ".partial" for WAL copying --- src/archive.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/archive.c b/src/archive.c index ff3678340..4bf90d3c2 100644 --- a/src/archive.c +++ b/src/archive.c @@ -181,7 +181,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ if (is_compress) { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", gz_to_path); + snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", gz_to_path); gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) @@ -194,7 +194,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, else #endif { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path); out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) @@ -416,7 +416,7 @@ get_wal_file(const char *from_path, const char *to_path) #endif /* open backup file for write */ - snprintf(to_path_temp, sizeof(to_path_temp), "%s.partial", to_path); + snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path); out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_DB_HOST); if (out < 0) From d6564054327f5bbc60e269992001df330e7b6c40 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 15:43:32 +0300 Subject: [PATCH 0945/2107] [Issue #117] fix tests --- tests/archive.py | 18 +++++++++--------- tests/page.py | 2 +- tests/replica.py | 2 +- tests/validate.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index b24eb15c3..d22cd915b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -512,7 +512,7 @@ def test_archive_push_file_exists_overwrite(self): # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): - """Archive-push if stale '.partial' file exists""" + """Archive-push if stale '.part' file exists""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -548,16 +548,16 @@ def test_archive_push_partial_file_exists(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - # form up path to next .partial WAL segment + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - filename = filename_orig + '.gz' + '.partial' + filename = filename_orig + '.gz' + '.part' file = os.path.join(wals_dir, filename) else: - filename = filename_orig + '.partial' + filename = filename_orig + '.part' file = os.path.join(wals_dir, filename) - # emulate stale .partial file + # emulate stale .part file with open(file, 'a') as f: f.write(b"blahblah") f.flush() @@ -590,7 +590,7 @@ def test_archive_push_partial_file_exists(self): # @unittest.skip("skip") def test_archive_push_partial_file_exists_not_stale(self): - """Archive-push if .partial file exists and it is not stale""" + """Archive-push if .part file exists and it is not stale""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -624,13 +624,13 @@ def test_archive_push_partial_file_exists_not_stale(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - # form up path to next .partial WAL segment + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - filename = filename_orig + '.gz' + '.partial' + filename = filename_orig + '.gz' + '.part' file = os.path.join(wals_dir, filename) else: - filename = filename_orig + '.partial' + filename = filename_orig + '.part' file = os.path.join(wals_dir, filename) with open(file, 'a') as f: diff --git a/tests/page.py b/tests/page.py index 9a717c234..fafdce3f0 100644 --- a/tests/page.py +++ b/tests/page.py @@ -678,7 +678,7 @@ def test_page_backup_with_lost_wal_segment(self): # delete last wal segment wals_dir = os.path.join(backup_dir, 'wal', 'node') wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] + wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] wals = map(str, wals) file = os.path.join(wals_dir, max(wals)) os.remove(file) diff --git a/tests/replica.py b/tests/replica.py index f767c4eac..c22089471 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1003,7 +1003,7 @@ def test_replica_promote_1(self): backup_dir, 'wal', 'master', '000000010000000000000004') wal_file_partial = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004.partial') + backup_dir, 'wal', 'master', '000000010000000000000004.part') self.assertFalse(os.path.exists(wal_file)) diff --git a/tests/validate.py b/tests/validate.py index d7372699e..519dc6480 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1760,7 +1760,7 @@ def test_pgpro561(self): # wals_dir = os.path.join(backup_dir, 'wal', 'node1') # wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( -# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] # wals = map(str, wals) # print(wals) @@ -1768,7 +1768,7 @@ def test_pgpro561(self): # wals_dir = os.path.join(backup_dir, 'wal', 'node1') # wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( -# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.partial')] +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] # wals = map(str, wals) # print(wals) From 58b2a55384bc568c2d06d599df02a70941e4bf67 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 15:44:17 +0300 Subject: [PATCH 0946/2107] [Issue #117] Documentation update --- Documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 1556e1fbd..33d921db9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1133,7 +1133,7 @@ Copies WAL files into the corresponding subdirectory of the backup catalog and v If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` flag. -Copying is done to temporary file with `.partial` suffix or, if [compression](#compression-options) is used, with `.gz.partial` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. +Copying is done to temporary file with `.part` suffix or, if [compression](#compression-options) is used, with `.gz.part` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. Copied to archive WAL segments are synced to disk. You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). From 3093bb91b74e5190675dc55a4678630aa2e24252 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Sep 2019 15:57:21 +0300 Subject: [PATCH 0947/2107] tests: minor fix --- tests/replica.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index c22089471..d6dc5ff65 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1003,7 +1003,7 @@ def test_replica_promote_1(self): backup_dir, 'wal', 'master', '000000010000000000000004') wal_file_partial = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004.part') + backup_dir, 'wal', 'master', '000000010000000000000004.partial') self.assertFalse(os.path.exists(wal_file)) @@ -1016,10 +1016,14 @@ def test_replica_promote_1(self): # sleep to be sure, that any partial timeout is expired sleep(70) + + self.assertTrue( + os.path.exists(wal_file_partial), + "File {0} disappeared".format(wal_file)) self.assertTrue( - os.path.exists(wal_file_partial, - "File {0} disappeared".format(wal_file_partial))) + os.path.exists(wal_file_partial), + "File {0} disappeared".format(wal_file_partial)) # Clean after yourself self.del_test_dir(module_name, fname) From 0e2336fb6f97763b5fec505b5ad251d5f7ecd74e Mon Sep 17 00:00:00 2001 From: Anastasia Date: Thu, 5 Sep 2019 16:14:50 +0300 Subject: [PATCH 0948/2107] code review for issue_115. add TODO comments --- src/backup.c | 34 +++++++++++++++++++++++++++------- src/parsexlog.c | 4 ++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2285d371f..07cae000f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1506,6 +1506,8 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, if (file_exists) { + //TODO is this comment also relates to current segment_only case + // and should be updated? /* Do not check LSN for previous WAL segment */ if (segment_only) return InvalidXLogRecPtr; @@ -1805,6 +1807,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, XLogSegNo segno = 0; XLogRecPtr lsn_tmp = InvalidXLogRecPtr; + /* + * TODO Let's rephrase that to be less scary for a user. + * Even though the value is invalid, it's expected postgres behaviour + * and we're trying to fix it below. + */ elog(WARNING, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); @@ -1829,27 +1836,36 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); /* - * Note, that there is no guarantee that corresponding WAL file is even exists. - * Basically replica may return LSN from future and keep staying in present. - * Yeah, it sucks. + * Note, that there is no guarantee that corresponding WAL file even exists. + * Replica may return LSN from future and keep staying in present. + * Or it can return LSN with invalid XRecOff. + * + * That's bad, since we want to get real LSN to save it in backup label file + * and to use it in WAL validation. * - * So we should try to do the following: - * 1. Wait for current segment and look in it for the LSN >= STOP_LSN. It should - * solve the problem of occasional 0 offset on write-busy system. + * So we try to do the following: + * 1. Wait for current segment and look in it for the LSN >= STOP_LSN. + * //TODO what is 'current' segment? + * //TODO how long do we wait? is there a timeout? + * It solves the problem of occasional invalid XRecOff + * on write-busy system. * 2. Failing that, look for record in previous segment with endpoint * equal or greater than 0 offset LSN. It may(!) solve the problem of 0 offset * on write-idle system. + * //TODO what if if didn't? + * //TODO what kind of record that refers to? */ /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, false, true, WARNING, stream_wal); - /* Optimistically try to get the first record in segment with current stop_lsn */ + /* Get the first record in segment with current stop_lsn */ lsn_tmp = get_first_wal_lsn(xlog_path, segno, backup->tli, instance_config.xlog_seg_size); /* Check if returned LSN is satisfying our requirements */ + //TODO what requirements? if (XLogRecPtrIsInvalid(lsn_tmp) || !XRecOffIsValid(lsn_tmp) || lsn_tmp < stop_backup_lsn_tmp) @@ -1877,6 +1893,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, stop_backup_lsn = lsn_tmp; stop_lsn_exists = true; } + //TODO is it good to throw an error here? Shouldn't we rather + // save it as is and mark backup as DONE instead of OK + // and check it in validation and restore? else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); @@ -1981,6 +2000,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* * Wait for stop_lsn to be archived or streamed. + * //TODO is the sentence below outdated? * If replica returned non-existent LSN, look for previous record, * which endpoint >= stop_lsn */ diff --git a/src/parsexlog.c b/src/parsexlog.c index d9437b19d..e5265e7c2 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -581,6 +581,10 @@ get_first_wal_lsn(const char *archivedir, XLogSegNo segno, * * Returns LSN which points to end+1 of the last WAL record if seek_prev_segment * is true. Otherwise returns LSN of the record prior to stop_lsn. + * + * TODO Let's think of better function name. + * it's unclear that "last" in "last_wal_lsn" refers to the + * "closest to stop_lsn backward or forward, depending on seek_prev_segment setting". */ XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, From 07f63e473b89283dcc9fa1802caf212e44e32c0e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Sep 2019 11:26:42 +0300 Subject: [PATCH 0949/2107] added some improvements based on review --- src/backup.c | 92 ++++++++++++++++++++++++---------------------------- 1 file changed, 42 insertions(+), 50 deletions(-) diff --git a/src/backup.c b/src/backup.c index 07cae000f..60f3dc1e1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1399,31 +1399,32 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, } /* - * Wait for target 'lsn' or WAL segment, containing 'lsn'. + * Wait for target LSN or WAL segment, containing target LSN. * - * If current backup started in archive mode wait for 'lsn' to be archived in - * archive 'wal' directory with WAL segment file. - * If current backup started in stream mode wait for 'lsn' to be streamed in - * 'pg_wal' directory. + * Depending on value of flag in_stream_dir wait for target LSN to archived or + * streamed in 'archive_dir' or 'pg_wal' directory. * - * If 'is_start_lsn' is true then issue warning for first-time users. + * If flag 'is_start_lsn' is set then issue warning for first-time users. + * If flag 'in_prev_segment' is set, look for LSN in previous segment, + * with EndRecPtr >= Target LSN. It should be used only for solving + * invalid XRecOff problem. + * If flag 'segment_only' is set, then, instead of waiting for LSN, wait for segment, + * containing that LSN. + * If flags 'in_prev_segment' and 'segment_only' are both set, then wait for + * previous segment. * - * If 'in_prev_segment' is set, look for LSN in previous segment. - * If 'segment_only' is set, then instead of looking for LSN, look for segment itself. - * If 'in_prev_segment' and 'segment_only' are both set, then wait for previous segment. + * Flag 'in_stream_dir' determine whether we looking for WAL in 'pg_wal' directory or + * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' because, + * for example, PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. * - * Flag 'in_stream_dir' determine whether we looking for wal in 'pg_wal' directory or - * in archive. Do note, that we cannot rely sorely on 'stream_wal' because, for example, - * PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. - * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than * ERROR is used, then return InvalidXLogRecPtr. TODO: return something more concrete, for example 1. * - * Returns LSN of last valid record if wait_prev_segment is not true, otherwise - * returns InvalidXLogRecPtr. + * Returns target LSN if such is found, failing that returns LSN of record prior to target LSN. + * Returns InvalidXLogRecPtr if 'segment_only' flag is used. */ static XLogRecPtr -wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, +wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir) { @@ -1442,16 +1443,16 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, #endif /* Compute the name of the WAL file containing requested LSN */ - GetXLogSegNo(lsn, targetSegNo, instance_config.xlog_seg_size); + GetXLogSegNo(target_lsn, targetSegNo, instance_config.xlog_seg_size); if (in_prev_segment) targetSegNo--; GetXLogFileName(wal_segment, tli, targetSegNo, instance_config.xlog_seg_size); /* - * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory if it is + * In pg_start_backup we wait for 'target_lsn' in 'pg_wal' directory if it is * stream and non-page backup. Page backup needs archived WAL files, so we - * wait for 'lsn' in archive 'wal' directory for page backups. + * wait for 'target_lsn' in archive 'wal' directory for page backups. * * In pg_stop_backup it depends only on stream_wal. */ @@ -1468,6 +1469,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, wal_segment_dir = arclog_path; } + /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ if (instance_config.archive_timeout > 0) timeout = instance_config.archive_timeout; else @@ -1477,7 +1479,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, elog(LOG, "Looking for segment: %s", wal_segment); else elog(LOG, "Looking for LSN %X/%X in segment: %s", - (uint32) (lsn >> 32), (uint32) lsn, wal_segment); + (uint32) (target_lsn >> 32), (uint32) target_lsn, wal_segment); #ifdef HAVE_LIBZ snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", @@ -1506,26 +1508,24 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, if (file_exists) { - //TODO is this comment also relates to current segment_only case - // and should be updated? - /* Do not check LSN for previous WAL segment */ + /* Do not check for target LSN */ if (segment_only) return InvalidXLogRecPtr; /* - * A WAL segment found. Check LSN on it. + * A WAL segment found. Look for target LSN in it. */ - if (!XRecOffIsNull(lsn) && - wal_contains_lsn(wal_segment_dir, lsn, tli, + if (!XRecOffIsNull(target_lsn) && + wal_contains_lsn(wal_segment_dir, target_lsn, tli, instance_config.xlog_seg_size)) /* Target LSN was found */ { - elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn); - return lsn; + elog(LOG, "Found LSN: %X/%X", (uint32) (target_lsn >> 32), (uint32) target_lsn); + return target_lsn; } /* - * If we failed to get LSN of valid record in a reasonable time, try + * If we failed to get target LSN in a reasonable time, try * to get LSN of last valid record prior to the target LSN. But only * in case of a backup from a replica. @@ -1539,7 +1539,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, { XLogRecPtr res; - res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, lsn, tli, + res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, in_prev_segment, instance_config.xlog_seg_size); if (!XLogRecPtrIsInvalid(res)) @@ -1565,7 +1565,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, wal_segment_path, wal_delivery_str); else elog(INFO, "Wait for LSN %X/%X in %s WAL segment %s", - (uint32) (lsn >> 32), (uint32) lsn, + (uint32) (target_lsn >> 32), (uint32) target_lsn, wal_delivery_str, wal_segment_path); } @@ -1580,7 +1580,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, elog(timeout_elevel, "WAL segment %s was %s, " "but target LSN %X/%X could not be archived in %d seconds", wal_segment, wal_delivery_str, - (uint32) (lsn >> 32), (uint32) lsn, timeout); + (uint32) (target_lsn >> 32), (uint32) target_lsn, timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(timeout_elevel, @@ -1808,11 +1808,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, XLogRecPtr lsn_tmp = InvalidXLogRecPtr; /* - * TODO Let's rephrase that to be less scary for a user. * Even though the value is invalid, it's expected postgres behaviour * and we're trying to fix it below. */ - elog(WARNING, "Invalid stop_backup_lsn value %X/%X", + elog(LOG, "Null offset in stop_backup_lsn value %X/%X, trying to fix", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); /* @@ -1844,15 +1843,12 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * and to use it in WAL validation. * * So we try to do the following: - * 1. Wait for current segment and look in it for the LSN >= STOP_LSN. - * //TODO what is 'current' segment? - * //TODO how long do we wait? is there a timeout? - * It solves the problem of occasional invalid XRecOff - * on write-busy system. + * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and + * look for the first valid record in it. + * It solves the problem of occasional invalid XRecOff on write-busy system. * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than 0 offset LSN. It may(!) solve the problem of 0 offset - * on write-idle system. - * //TODO what if if didn't? + * equal or greater than stop_lsn. It may(!) solve the problem of 0 offset + * on write-idle system. If that fails too, error out. * //TODO what kind of record that refers to? */ @@ -1864,8 +1860,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, lsn_tmp = get_first_wal_lsn(xlog_path, segno, backup->tli, instance_config.xlog_seg_size); - /* Check if returned LSN is satisfying our requirements */ - //TODO what requirements? + /* Check that returned LSN is valid and greater than stop_lsn */ if (XLogRecPtrIsInvalid(lsn_tmp) || !XRecOffIsValid(lsn_tmp) || lsn_tmp < stop_backup_lsn_tmp) @@ -1893,9 +1888,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, stop_backup_lsn = lsn_tmp; stop_lsn_exists = true; } - //TODO is it good to throw an error here? Shouldn't we rather - // save it as is and mark backup as DONE instead of OK - // and check it in validation and restore? + /* Replica returned something very illegal, error out */ else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); @@ -2000,9 +1993,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* * Wait for stop_lsn to be archived or streamed. - * //TODO is the sentence below outdated? - * If replica returned non-existent LSN, look for previous record, - * which endpoint >= stop_lsn + * If replica returned valid STOP_LSN of not actually existing record, + * look for previous record with endpoint >= STOP_LSN. */ if (!stop_lsn_exists) stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, From bad8ce144e6f82f229d36e2ee3cad2f5b69bc7ff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Sep 2019 11:36:05 +0300 Subject: [PATCH 0950/2107] Documentation: minor spelling fix --- Documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 33d921db9..6e10c9b31 100644 --- a/Documentation.md +++ b/Documentation.md @@ -371,7 +371,7 @@ If you are going to use pg_probackup in remote mode via ssh, complete the follow [backup@backup_host] ssh-copy-id postgres@db_host -- If you planning to rely on [continuous WAL archiving](#setting-up-continuous-wal-archiving), then setup passwordless SSH connection between *postgres* user on `db_host` and *backup* user on `backup_host`: +- If you are planning to rely on [continuous WAL archiving](#setting-up-continuous-wal-archiving), then setup passwordless SSH connection between *postgres* user on `db_host` and *backup* user on `backup_host`: [postgres@db_host] ssh-copy-id backup@backup_host From 4dcbfc132b628f5ebe6494d19792d853e61e2f51 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Sep 2019 11:43:57 +0300 Subject: [PATCH 0951/2107] add comment to unlink_lock_atexit() --- src/catalog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/catalog.c b/src/catalog.c index 1b3f48a3c..48e5a7efb 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -24,6 +24,7 @@ static pgBackup *readBackupControlFile(const char *path); static bool exit_hook_registered = false; static parray *lock_files = NULL; +/* Iterate over locked backups and delete locks files */ static void unlink_lock_atexit(void) { From a98ff6154a79f23d71ccc8884e3a8f31184e3e01 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Sep 2019 13:20:42 +0300 Subject: [PATCH 0952/2107] [Issue #115]: some improvements of sanity and comments. Also do not wait for NullOffset LSN when looking for previous record --- src/backup.c | 20 ++++++++++++-------- tests/replica.py | 14 +++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index 60f3dc1e1..481735de7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1528,14 +1528,18 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, * If we failed to get target LSN in a reasonable time, try * to get LSN of last valid record prior to the target LSN. But only * in case of a backup from a replica. - + * Note, that with NullOffset target_lsn we do not wait + * for 'timeout / 2' seconds before going for previous record, + * because such LSN cannot be delivered at all. + * * There are two cases for this: * 1. Replica returned readpoint LSN which just do not exists. We want to look * for previous record in the same(!) WAL segment which endpoint points to this LSN. * 2. Replica returened endpoint LSN with 0 offset. We want to look * for previous record which endpoint points greater or equal LSN in previous WAL segment. */ - if (!exclusive_backup && current.from_replica && try_count > timeout / 2) + if (current.from_replica && + (XRecOffIsNull(target_lsn) || try_count > timeout / 2)) { XLogRecPtr res; @@ -1641,7 +1645,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * Only if backup is from master. * For PG 9.5 create restore point only if pguser is superuser. */ - if (backup != NULL && !current.from_replica && + if (backup != NULL && !backup->from_replica && !(nodeInfo->server_version < 90600 && !nodeInfo->is_superuser)) { @@ -1677,7 +1681,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. */ - if (current.from_replica) + if (backup->from_replica) stop_backup_query = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," @@ -1799,8 +1803,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!XRecOffIsValid(stop_backup_lsn_tmp)) { - /* Replica returned STOP LSN with null offset */ - if (XRecOffIsNull(stop_backup_lsn_tmp)) + /* It is ok for replica to return STOP LSN with null offset */ + if (backup->from_replica && XRecOffIsNull(stop_backup_lsn_tmp)) { char *xlog_path, stream_xlog_path[MAXPGPATH]; @@ -1888,7 +1892,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, stop_backup_lsn = lsn_tmp; stop_lsn_exists = true; } - /* Replica returned something very illegal, error out */ + /* PostgreSQL returned something very illegal as STOP_LSN, error out */ else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); @@ -1898,7 +1902,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!exclusive_backup) { Assert(PQnfields(res) >= 4); - pgBackupGetPath(¤t, path, lengthof(path), DATABASE_DIR); + pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); diff --git a/tests/replica.py b/tests/replica.py index 0c3672064..f03687491 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -586,7 +586,7 @@ def test_replica_stop_lsn_null_offset(self): return_id=False) self.assertIn( - 'WARNING: Invalid stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/3000000', output) self.assertIn( @@ -601,10 +601,6 @@ def test_replica_stop_lsn_null_offset(self): 'LOG: Looking for LSN 0/3000000 in segment: 000000010000000000000002', output) - self.assertIn( - 'INFO: Wait for LSN 0/3000000 in streamed WAL segment', - output) - self.assertIn( 'LOG: Record 0/2000160 has endpoint 0/3000000 which is ' 'equal or greater than requested LSN 0/3000000', @@ -697,7 +693,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content = f.read() self.assertIn( - 'WARNING: Invalid stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/3000000', log_content) self.assertIn( @@ -766,7 +762,7 @@ def test_archive_replica_null_offset(self): return_id=False) self.assertIn( - 'WARNING: Invalid stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/3000000', output) self.assertIn( @@ -790,6 +786,8 @@ def test_archive_replica_null_offset(self): 'LOG: Found prior LSN: 0/2000160', output) + print(output) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -949,6 +947,8 @@ def test_replica_toast(self): 'LOG: Found prior LSN:', output) + print(output) + replica.cleanup() self.restore_node(backup_dir, 'replica', replica) From 64b05d95972d4efc69663150ba483fd17651415c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Sep 2019 14:34:49 +0300 Subject: [PATCH 0953/2107] [Issue #115]: minor improvements --- src/backup.c | 17 ++++++++--------- src/parsexlog.c | 21 ++++++++------------- src/pg_probackup.h | 4 ++-- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/backup.c b/src/backup.c index 481735de7..6441e4982 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1528,14 +1528,14 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, * If we failed to get target LSN in a reasonable time, try * to get LSN of last valid record prior to the target LSN. But only * in case of a backup from a replica. - * Note, that with NullOffset target_lsn we do not wait + * Note, that with NullXRecOff target_lsn we do not wait * for 'timeout / 2' seconds before going for previous record, * because such LSN cannot be delivered at all. * * There are two cases for this: * 1. Replica returned readpoint LSN which just do not exists. We want to look * for previous record in the same(!) WAL segment which endpoint points to this LSN. - * 2. Replica returened endpoint LSN with 0 offset. We want to look + * 2. Replica returened endpoint LSN with NullXRecOff. We want to look * for previous record which endpoint points greater or equal LSN in previous WAL segment. */ if (current.from_replica && @@ -1543,7 +1543,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, { XLogRecPtr res; - res = get_last_wal_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, + res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, in_prev_segment, instance_config.xlog_seg_size); if (!XLogRecPtrIsInvalid(res)) @@ -1803,7 +1803,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!XRecOffIsValid(stop_backup_lsn_tmp)) { - /* It is ok for replica to return STOP LSN with null offset */ + /* It is ok for replica to return STOP LSN with NullXRecOff */ if (backup->from_replica && XRecOffIsNull(stop_backup_lsn_tmp)) { char *xlog_path, @@ -1820,7 +1820,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* * Note: even with gdb it is very hard to produce automated tests for - * contrecord + null_offset STOP_LSN, so emulate it for manual testing. + * contrecord + NullXRecOff, so emulate it for manual testing. */ //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", @@ -1841,7 +1841,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* * Note, that there is no guarantee that corresponding WAL file even exists. * Replica may return LSN from future and keep staying in present. - * Or it can return LSN with invalid XRecOff. + * Or it can return LSN with NullXRecOff. * * That's bad, since we want to get real LSN to save it in backup label file * and to use it in WAL validation. @@ -1851,9 +1851,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * look for the first valid record in it. * It solves the problem of occasional invalid XRecOff on write-busy system. * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of 0 offset + * equal or greater than stop_lsn. It may(!) solve the problem of NullXRecOff * on write-idle system. If that fails too, error out. - * //TODO what kind of record that refers to? */ /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ @@ -1861,7 +1860,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, false, true, WARNING, stream_wal); /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_wal_lsn(xlog_path, segno, backup->tli, + lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, instance_config.xlog_seg_size); /* Check that returned LSN is valid and greater than stop_lsn */ diff --git a/src/parsexlog.c b/src/parsexlog.c index e5265e7c2..37460c6bd 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -529,13 +529,10 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, } /* - * Get LSN of first record within the WAL segment with number 'segno'. - * - * Returns LSN which points to end+1 of the last WAL record if seek_prev_segment - * is true. Otherwise returns LSN of the record prior to stop_lsn. + * Get LSN of a first record within the WAL segment with number 'segno'. */ XLogRecPtr -get_first_wal_lsn(const char *archivedir, XLogSegNo segno, +get_first_record_lsn(const char *archivedir, XLogSegNo segno, TimeLineID tli, uint32 wal_seg_size) { XLogReaderState *xlogreader; @@ -574,20 +571,18 @@ get_first_wal_lsn(const char *archivedir, XLogSegNo segno, } /* - * Get LSN of last or prior record within the WAL segment with number 'segno'. - * If 'start_lsn' - * is in the segment with number 'segno' then start from 'start_lsn', otherwise - * start from offset 0 within the segment. + * Get LSN of a record prior to target_lsn. + * If 'start_lsn' is in the segment with number 'segno' then start from 'start_lsn', + * otherwise start from offset 0 within the segment. * - * Returns LSN which points to end+1 of the last WAL record if seek_prev_segment - * is true. Otherwise returns LSN of the record prior to stop_lsn. + * Returns LSN of a record which EndRecPtr is greater or equal to target_lsn. + * If 'seek_prev_segment' is true, then look for prior record in prior WAL segment. * - * TODO Let's think of better function name. * it's unclear that "last" in "last_wal_lsn" refers to the * "closest to stop_lsn backward or forward, depending on seek_prev_segment setting". */ XLogRecPtr -get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, +get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, uint32 wal_seg_size) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7e1f3ed83..a311fd7ef 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -711,11 +711,11 @@ extern bool read_recovery_info(const char *archivedir, TimeLineID tli, TransactionId *recovery_xid); extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, TimeLineID target_tli, uint32 seg_size); -extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, +extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, uint32 seg_size); -extern XLogRecPtr get_first_wal_lsn(const char *archivedir, XLogRecPtr start_lsn, +extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_lsn, TimeLineID tli, uint32 wal_seg_size); /* in util.c */ From 2413516389f436eee6a71cf3be11b446c680af38 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Sep 2019 02:49:41 +0300 Subject: [PATCH 0954/2107] tests: added set_auto_conf() --- tests/helpers/ptrack_helpers.py | 70 +++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d736a364b..18d2673f6 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1018,46 +1018,76 @@ def set_archiving( self, backup_dir, instance, node, replica=False, overwrite=False, compress=False, old_binary=False): + # parse postgresql.auto.conf + options = {} if replica: - archive_mode = 'always' - node.append_conf('postgresql.auto.conf', 'hot_standby = on') + options['archive_mode'] = 'always' + options['hot_standby'] = 'on' else: - archive_mode = 'on' + options['archive_mode'] = 'on' - node.append_conf( - 'postgresql.auto.conf', - 'archive_mode = {0}'.format(archive_mode) - ) if os.name == 'posix': - archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( self.probackup_path, backup_dir, instance) elif os.name == 'nt': - archive_command = '"{0}" archive-push -B {1} --instance={2} '.format( + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( self.probackup_path.replace("\\","\\\\"), - backup_dir.replace("\\","\\\\"), - instance) + backup_dir.replace("\\","\\\\"), instance) # don`t forget to kill old_binary after remote ssh release if self.remote and not old_binary: - archive_command = archive_command + '--remote-proto=ssh --remote-host=localhost ' + options['archive_command'] += '--remote-proto=ssh ' + options['archive_command'] += '--remote-host=localhost ' if self.archive_compress or compress: - archive_command = archive_command + '--compress ' + options['archive_command'] += '--compress ' if overwrite: - archive_command = archive_command + '--overwrite ' + options['archive_command'] += '--overwrite ' if os.name == 'posix': - archive_command = archive_command + '--wal-file-path=%p --wal-file-name=%f' + options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' elif os.name == 'nt': - archive_command = archive_command + '--wal-file-path="%p" --wal-file-name="%f"' + options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' - node.append_conf( - 'postgresql.auto.conf', - "archive_command = '{0}'".format( - archive_command)) + self.set_auto_conf(node, options) + + def set_auto_conf(self, node, options): + + # parse postgresql.auto.conf + path = os.path.join(node.data_dir, 'postgresql.auto.conf') + + with open(path, 'r') as f: + raw_content = f.read() + + current_options = {} + for line in raw_content.splitlines(): + + # ignore comments + if line.startswith('#'): + continue + + name, var = line.partition('=')[::2] + name = name.strip() + var = var.strip() + var = var.strip('"') + var = var.strip("'") + current_options[name] = var + + for option in options: + current_options[option] = options[option] + + auto_conf = '' + for option in current_options: + auto_conf += "{0} = '{1}'\n".format( + option, current_options[option]) + + with open(path, 'wt') as f: + f.write(auto_conf) + f.flush() + f.close() def set_replica( self, master, replica, From 11ab44fb04d5edf11c2eeca723942ac1268ac910 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 16 Sep 2019 13:47:24 +0300 Subject: [PATCH 0955/2107] [Issue #118]: trust txid_snapshot_xmax() when determining recovery_xid --- src/backup.c | 4 ++-- src/parsexlog.c | 3 +-- src/pg_probackup.h | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6441e4982..298dcd55c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2019,6 +2019,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, xlog_path = arclog_path; backup->stop_lsn = stop_backup_lsn; + backup->recovery_xid = recovery_xid; elog(LOG, "Getting the Recovery Time from WAL"); @@ -2026,11 +2027,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!read_recovery_info(xlog_path, backup->tli, instance_config.xlog_seg_size, backup->start_lsn, backup->stop_lsn, - &backup->recovery_time, &backup->recovery_xid)) + &backup->recovery_time)) { elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); backup->recovery_time = recovery_time; - backup->recovery_xid = recovery_xid; } } } diff --git a/src/parsexlog.c b/src/parsexlog.c index 37460c6bd..921fd8a09 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -425,7 +425,7 @@ validate_wal(pgBackup *backup, const char *archivedir, bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, - time_t *recovery_time, TransactionId *recovery_xid) + time_t *recovery_time) { XLogRecPtr startpoint = stop_lsn; XLogReaderState *xlogreader; @@ -472,7 +472,6 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, if (getRecordTimestamp(xlogreader, &last_time)) { *recovery_time = timestamptz_to_time_t(last_time); - *recovery_xid = XLogRecGetXid(xlogreader); /* Found timestamp in WAL record 'record' */ res = true; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a311fd7ef..332a9aa8b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -707,8 +707,7 @@ extern void validate_wal(pgBackup *backup, const char *archivedir, extern bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, - time_t *recovery_time, - TransactionId *recovery_xid); + time_t *recovery_time); extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, TimeLineID target_tli, uint32 seg_size); extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, From d34a6a3ad8f4b38607d3b31f79729a67a08cc8b3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 17 Sep 2019 17:35:27 +0300 Subject: [PATCH 0956/2107] [Issue #116] WAL archive information --- Documentation.md | 277 +++++++++- src/archive.c | 34 +- src/backup.c | 6 +- src/catalog.c | 107 +++- src/configure.c | 234 +++++++- src/delete.c | 10 +- src/dir.c | 70 +-- src/help.c | 5 +- src/init.c | 24 +- src/merge.c | 6 +- src/parsexlog.c | 2 +- src/pg_probackup.c | 32 +- src/pg_probackup.h | 58 +- src/restore.c | 11 +- src/show.c | 942 +++++++++++++++++++++++++------- src/validate.c | 12 +- tests/archive.py | 350 +++++++++++- tests/backup.py | 19 +- tests/delta.py | 2 +- tests/helpers/ptrack_helpers.py | 33 ++ tests/option.py | 3 +- tests/page.py | 20 +- tests/pgpro2068.py | 2 +- tests/pgpro589.py | 4 +- tests/validate.py | 48 +- 25 files changed, 1945 insertions(+), 366 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6e10c9b31..41ef446bf 100644 --- a/Documentation.md +++ b/Documentation.md @@ -36,6 +36,7 @@ Current version - 2.1.5 * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) * [Configuring pg_probackup](#configuring-pg_probackup) * [Managing the Backup Catalog](#managing-the-backup-catalog) + * [Viewing WAL Archive Information](#viewing-wal-archive-information) * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -126,7 +127,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it - Backup from replica: avoid extra load on the master server by taking backups from a standby - External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats +- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats and view WAL Archive information. - Partial Restore: restore the only specified databases or skip the specified databases. To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -300,7 +301,9 @@ Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time- Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote host. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). -Once these steps are complete, you can start making backups with ARCHIVE WAL mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). +Once these steps are complete, you can start making backups with [ARCHIVE](#archive-mode) WAL-mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). + +Current state of WAL Archive can be obtained via [show](#show) command. For details, see the sections [Viewing WAL Archive information](#viewing-wal-archive-information). If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to set the value of this setting slightly lower than pg_probackup parameter `--archive-timeout` (default 5 min), so there should be enough time for rotated segment to be streamed to replica and send to archive before backup is aborted because of `--archive-timeout`. @@ -408,7 +411,7 @@ Where *backup_mode* can take one of the following values: - FULL — creates a full backup that contains all the data files of the cluster to be restored. - DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. +- PAGE — creates an incremental PAGE backup based on the WAL files that have generated since the previous full or incremental backup was taken. Only changed blocks are readed from data files. - PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `the backup chain`. You must create at least one full backup before taking incremental ones. @@ -668,6 +671,7 @@ If nothing is given, the default values are taken. By default pg_probackup tries With pg_probackup, you can manage backups from the command line: - View available backups +- View available WAL Archive Information - Validate backups - Merge backups - Delete backups @@ -794,6 +798,266 @@ The sample output is as follows: ] ``` +#### Viewing WAL Archive Information + +To view the information about WAL archive for every instance, run the command: + + pg_probackup show -B backup_dir [--instance instance_name] --archive + +pg_probackup displays the list of all the available WAL files grouped by timelines. For example: + +``` +ARCHIVE INSTANCE 'node' +=================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=================================================================================================================== + 5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK + 4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK + 3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK + 2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED + 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK + +``` + +For each backup, the following information is provided: + +- TLI — timeline identifier. +- Parent TLI — identifier of timeline TLI branched off. +- Switchpoint — LSN of the moment when the timeline branched off from "Parent TLI". +- Min Segno — number of the first existing WAL segment belonging to the timeline. +- Max Segno — number of the last existing WAL segment belonging to the timeline. +- N segments — number of WAL segments belonging to the timeline. +- Size — the size files take on disk. +- Zratio - compression ratio calculated as "N segments" * wal_seg_size / "Size". +- N backups — number of backups belonging to the timeline. To get the details about backups, use json format. +- Status — archive status for this exact timeline. Possible values: + - OK — all WAL segments between Min and Max are present. + - DEGRADED — some WAL segments between Min and Max are lost. To get details about lost files, use json format. + +To get more detailed information about the WAL archive in json format, run the command: + + pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json + +The sample output is as follows: + +``` +[ + { + "instance": "replica", + "timelines": [ + { + "tli": 5, + "parent-tli": 1, + "switchpoint": "0/B000000", + "min-segno": "000000000000000B", + "max-segno": "000000000000000C", + "n-segments": 2, + "size": 685320, + "zratio": 48.00, + "closest-backup-id": "PXS92O", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 4, + "parent-tli": 3, + "switchpoint": "0/18000000", + "min-segno": "0000000000000018", + "max-segno": "000000000000001A", + "n-segments": 3, + "size": 648625, + "zratio": 77.00, + "closest-backup-id": "PXS9CE", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 3, + "parent-tli": 2, + "switchpoint": "0/15000000", + "min-segno": "0000000000000015", + "max-segno": "0000000000000017", + "n-segments": 3, + "size": 648911, + "zratio": 77.00, + "closest-backup-id": "PXS9CE", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 2, + "parent-tli": 1, + "switchpoint": "0/B000108", + "min-segno": "000000000000000B", + "max-segno": "0000000000000015", + "n-segments": 5, + "size": 892173, + "zratio": 94.00, + "closest-backup-id": "PXS92O", + "status": "DEGRADED", + "lost-segments": [ + { + "begin-segno": "000000000000000D", + "end-segno": "000000000000000E" + }, + { + "begin-segno": "0000000000000010", + "end-segno": "0000000000000012" + } + ], + "backups": [ + { + "id": "PXS9CE", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 2, + "parent-tli": 0, + "start-lsn": "0/C000028", + "stop-lsn": "0/C000160", + "start-time": "2019-09-13 21:43:26+03", + "end-time": "2019-09-13 21:43:30+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:43:29+03", + "data-bytes": 104674852, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + }, + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "0000000000000001", + "max-segno": "000000000000000A", + "n-segments": 10, + "size": 8774805, + "zratio": 19.00, + "closest-backup-id": "", + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "PXS92O", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "true", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/4000028", + "stop-lsn": "0/6000028", + "start-time": "2019-09-13 21:37:36+03", + "end-time": "2019-09-13 21:38:45+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:30+03", + "data-bytes": 25987319, + "wal-bytes": 50331648, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } + ] + }, + { + "instance": "master", + "timelines": [ + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "0000000000000001", + "max-segno": "000000000000000B", + "n-segments": 11, + "size": 8860892, + "zratio": 20.00, + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "PXS92H", + "parent-backup-id": "PXS92C", + "backup-mode": "PAGE", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/4000028", + "stop-lsn": "0/50000B8", + "start-time": "2019-09-13 21:37:29+03", + "end-time": "2019-09-13 21:37:31+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:30+03", + "data-bytes": 1328461, + "wal-bytes": 33554432, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + }, + { + "id": "PXS92C", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/2000028", + "stop-lsn": "0/2000160", + "start-time": "2019-09-13 21:37:24+03", + "end-time": "2019-09-13 21:37:29+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:28+03", + "data-bytes": 24871902, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } + ] + } +] +``` + +Most fields are consistent with plain format, with some exceptions: + +- size is in bytes. +- 'closest-backup-id' attribute contain ID of valid backup closest to the timeline, located on some of the previous timelines. This backup is the closest starting point to reach the timeline from other timelines by PITR. If such backup do not exists, then string is empty. +- DEGRADED timelines contain 'lost-segments' array with information about intervals of missing segments. In OK timelines 'lost-segments' array is empty. +- 'N backups' attribute is replaced with 'backups' array containing backups belonging to the timeline. If timeline has no backups, then 'backups' array is empty. + ### Configuring Backup Retention Policy By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. @@ -963,12 +1227,15 @@ To edit pg_probackup.conf, use the [set-config](#set-config) command. #### show pg_probackup show -B backup_dir - [--help] [--instance instance_name [-i backup_id]] [--format=plain|json] + [--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] -Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. +Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. If `--archive` option is specified, shows the content of WAL archive of the backup catalog. By default, the contents of the backup catalog is shown as plain text. +For details on usage, see the sections [Managing the Backup Catalog](#managing-the-backup-catalog) and [Viewing WAL Archive Information](#viewing-wal-archive-information). + + #### backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name diff --git a/src/archive.c b/src/archive.c index 4bf90d3c2..a42163227 100644 --- a/src/archive.c +++ b/src/archive.c @@ -13,7 +13,7 @@ #include static void push_wal_file(const char *from_path, const char *to_path, - bool is_compress, bool overwrite); + bool is_compress, bool overwrite, int compress_level); static void get_wal_file(const char *from_path, const char *to_path); #ifdef HAVE_LIBZ static const char *get_gz_error(gzFile gzf, int errnum); @@ -31,11 +31,10 @@ static void copy_file_attributes(const char *from_path, * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. * Where archlog_path is $BACKUP_PATH/wal/system_id. * Currently it just copies wal files to the new location. - * TODO: Planned options: list the arclog content, - * compute and validate checksums. */ int -do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) +do_archive_push(InstanceConfig *instance, + char *wal_file_path, char *wal_file_name, bool overwrite) { char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; @@ -60,33 +59,33 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) /* verify that archive-push --instance parameter is valid */ system_id = get_system_identifier(current_dir); - if (instance_config.pgdata == NULL) + if (instance->pgdata == NULL) elog(ERROR, "cannot read pg_probackup.conf for this instance"); - if(system_id != instance_config.system_identifier) + if(system_id != instance->system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instance_name, instance_config.system_identifier, + wal_file_name, instance->name, instance->system_identifier, system_id); /* Create 'archlog_path' directory. Do nothing if it already exists. */ - fio_mkdir(arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); join_path_components(absolute_wal_file_path, current_dir, wal_file_path); - join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); - if (instance_config.compress_alg == PGLZ_COMPRESS) + if (instance->compress_alg == PGLZ_COMPRESS) elog(ERROR, "pglz compression is not supported"); #ifdef HAVE_LIBZ - if (instance_config.compress_alg == ZLIB_COMPRESS) + if (instance->compress_alg == ZLIB_COMPRESS) is_compress = IsXLogFileName(wal_file_name); #endif push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, - overwrite); + overwrite, instance->compress_level); elog(INFO, "pg_probackup archive-push completed successfully"); return 0; @@ -97,7 +96,8 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite) * Move files from arclog_path to pgdata/wal_file_path. */ int -do_archive_get(char *wal_file_path, char *wal_file_name) +do_archive_get(InstanceConfig *instance, + char *wal_file_path, char *wal_file_name) { char backup_wal_file_path[MAXPGPATH]; char absolute_wal_file_path[MAXPGPATH]; @@ -118,7 +118,7 @@ do_archive_get(char *wal_file_path, char *wal_file_name) elog(ERROR, "getcwd() error"); join_path_components(absolute_wal_file_path, current_dir, wal_file_path); - join_path_components(backup_wal_file_path, arclog_path, wal_file_name); + join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); elog(INFO, "pg_probackup archive-get from %s to %s", backup_wal_file_path, absolute_wal_file_path); @@ -134,7 +134,7 @@ do_archive_get(char *wal_file_path, char *wal_file_name) */ void push_wal_file(const char *from_path, const char *to_path, bool is_compress, - bool overwrite) + bool overwrite, int compress_level) { FILE *in = NULL; int out = -1; @@ -183,7 +183,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, { snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", gz_to_path); - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) { partial_file_exists = true; @@ -246,7 +246,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, #ifdef HAVE_LIBZ if (is_compress) { - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST); + gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (gz_out == NULL) elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", to_path_temp, strerror(errno)); diff --git a/src/backup.c b/src/backup.c index 298dcd55c..e92b07dd7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -196,7 +196,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) char prev_backup_filelist_path[MAXPGPATH]; /* get list of backups already taken */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) @@ -1681,7 +1681,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. */ - if (backup->from_replica) + + /* current is used here because of cleanup */ + if (current.from_replica) stop_backup_query = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," diff --git a/src/catalog.c b/src/catalog.c index 48e5a7efb..9a137ae3c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -53,13 +53,14 @@ unlink_lock_atexit(void) * If no backup matches, return NULL. */ pgBackup * -read_backup(time_t timestamp) +read_backup(const char *instance_name, time_t timestamp) { pgBackup tmp; char conf_path[MAXPGPATH]; tmp.start_time = timestamp; - pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE); + pgBackupGetPathInInstance(instance_name, &tmp, conf_path, + lengthof(conf_path), BACKUP_CONTROL_FILE, NULL); return readBackupControlFile(conf_path); } @@ -71,11 +72,12 @@ read_backup(time_t timestamp) * status. */ void -write_backup_status(pgBackup *backup, BackupStatus status) +write_backup_status(pgBackup *backup, BackupStatus status, + const char *instance_name) { pgBackup *tmp; - tmp = read_backup(backup->start_time); + tmp = read_backup(instance_name, backup->start_time); if (!tmp) { /* @@ -302,6 +304,69 @@ IsDir(const char *dirpath, const char *entry, fio_location location) return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); } +/* + * Create list of instances in given backup catalog. + * + * Returns parray of "InstanceConfig" structures, filled with + * actual config of each instance. + */ +parray * +catalog_get_instance_list(void) +{ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *dent; + parray *instances; + + instances = parray_new(); + + /* open directory and list contents */ + join_path_components(path, backup_path, BACKUPS_DIR); + dir = opendir(path); + if (dir == NULL) + elog(ERROR, "Cannot open directory \"%s\": %s", + path, strerror(errno)); + + while (errno = 0, (dent = readdir(dir)) != NULL) + { + char child[MAXPGPATH]; + struct stat st; + InstanceConfig *instance; + + /* skip entries point current dir or parent dir */ + if (strcmp(dent->d_name, ".") == 0 || + strcmp(dent->d_name, "..") == 0) + continue; + + join_path_components(child, path, dent->d_name); + + if (lstat(child, &st) == -1) + elog(ERROR, "Cannot stat file \"%s\": %s", + child, strerror(errno)); + + if (!S_ISDIR(st.st_mode)) + continue; + + instance = readInstanceConfigFile(dent->d_name); + + parray_append(instances, instance); + } + + /* TODO 3.0: switch to ERROR */ + if (parray_num(instances) == 0) + elog(WARNING, "This backup catalog contains no backup instances. Backup instance can be added via 'add-instance' command."); + + if (errno) + elog(ERROR, "Cannot read directory \"%s\": %s", + path, strerror(errno)); + + if (closedir(dir)) + elog(ERROR, "Cannot close directory \"%s\": %s", + path, strerror(errno)); + + return instances; +} + /* * Create list of backups. * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. @@ -309,12 +374,16 @@ IsDir(const char *dirpath, const char *entry, fio_location location) * If valid backup id is passed only matching backup will be added to the list. */ parray * -catalog_get_backup_list(time_t requested_backup_id) +catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) { DIR *data_dir = NULL; struct dirent *data_ent = NULL; parray *backups = NULL; int i; + char backup_instance_path[MAXPGPATH]; + + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); /* open backup instance backups directory */ data_dir = fio_opendir(backup_instance_path, FIO_BACKUP_HOST); @@ -420,6 +489,7 @@ catalog_get_backup_list(time_t requested_backup_id) * Create list of backup datafiles. * If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error. * If valid backup id is passed only matching backup will be added to the list. + * TODO this function only used once. Is it really needed? */ parray * get_backup_filelist(pgBackup *backup) @@ -1195,6 +1265,33 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, base36enc(backup->start_time), subdir1, subdir2); } +/* + * independent from global variable backup_instance_path + * Still depends from backup_path + */ +void +pgBackupGetPathInInstance(const char *instance_name, + const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2) +{ + char backup_instance_path[MAXPGPATH]; + + sprintf(backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + + /* If "subdir1" is NULL do not check "subdir2" */ + if (!subdir1) + snprintf(path, len, "%s/%s", backup_instance_path, + base36enc(backup->start_time)); + else if (!subdir2) + snprintf(path, len, "%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1); + /* "subdir1" and "subdir2" is not NULL */ + else + snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, + base36enc(backup->start_time), subdir1, subdir2); +} + /* * Check if multiple backups consider target backup to be their direct parent */ diff --git a/src/configure.c b/src/configure.c index f9d92fc1f..5619cfd58 100644 --- a/src/configure.c +++ b/src/configure.c @@ -312,10 +312,12 @@ do_set_config(bool missing_ok) } void -init_config(InstanceConfig *config) +init_config(InstanceConfig *config, const char *instance_name) { MemSet(config, 0, sizeof(InstanceConfig)); + config->name = pgut_strdup(instance_name); + /* * Starting from PostgreSQL 11 WAL segment size may vary. Prior to * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. @@ -342,6 +344,236 @@ init_config(InstanceConfig *config) config->remote.proto = (char*)"ssh"; } +/* + * read instance config from file + */ +InstanceConfig * +readInstanceConfigFile(const char *instance_name) +{ + char path[MAXPGPATH]; + InstanceConfig *instance = pgut_new(InstanceConfig); + char *log_level_console = NULL; + char *log_level_file = NULL; + char *compress_alg = NULL; + int parsed_options; + + ConfigOption instance_options[] = + { + /* Instance options */ + { + 's', 'D', "pgdata", + &instance->pgdata, SOURCE_CMD, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + { + 'U', 200, "system-identifier", + &instance->system_identifier, SOURCE_FILE_STRICT, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + #if PG_VERSION_NUM >= 110000 + { + 'u', 201, "xlog-seg-size", + &instance->xlog_seg_size, SOURCE_FILE_STRICT, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + #endif + { + 's', 'E', "external-dirs", + &instance->external_dir_str, SOURCE_CMD, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + /* Connection options */ + { + 's', 'd', "pgdatabase", + &instance->conn_opt.pgdatabase, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'h', "pghost", + &instance->conn_opt.pghost, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'p', "pgport", + &instance->conn_opt.pgport, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + { + 's', 'U', "pguser", + &instance->conn_opt.pguser, SOURCE_CMD, 0, + OPTION_CONN_GROUP, 0, option_get_value + }, + /* Replica options */ + { + 's', 202, "master-db", + &instance->master_conn_opt.pgdatabase, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 203, "master-host", + &instance->master_conn_opt.pghost, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 204, "master-port", + &instance->master_conn_opt.pgport, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 's', 205, "master-user", + &instance->master_conn_opt.pguser, SOURCE_CMD, 0, + OPTION_REPLICA_GROUP, 0, option_get_value + }, + { + 'u', 206, "replica-timeout", + &instance->replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value + }, + /* Archive options */ + { + 'u', 207, "archive-timeout", + &instance->archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value + }, + /* Logging options */ + { + 's', 208, "log-level-console", + &log_level_console, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 209, "log-level-file", + &log_level_file, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 210, "log-filename", + &instance->logger.log_filename, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 211, "error-log-filename", + &instance->logger.error_log_filename, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 212, "log-directory", + &instance->logger.log_directory, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 'U', 213, "log-rotation-size", + &instance->logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value + }, + { + 'U', 214, "log-rotation-age", + &instance->logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value + }, + /* Retention options */ + { + 'u', 215, "retention-redundancy", + &instance->retention_redundancy, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, + { + 'u', 216, "retention-window", + &instance->retention_window, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, + /* Compression options */ + { + 's', 217, "compress-algorithm", + &compress_alg, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 'u', 218, "compress-level", + &instance->compress_level, SOURCE_CMD, 0, + OPTION_COMPRESS_GROUP, 0, option_get_value + }, + /* Remote backup options */ + { + 's', 219, "remote-proto", + &instance->remote.proto, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 220, "remote-host", + &instance->remote.host, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 221, "remote-port", + &instance->remote.port, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 222, "remote-path", + &instance->remote.path, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 223, "remote-user", + &instance->remote.user, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 224, "ssh-options", + &instance->remote.ssh_options, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { + 's', 225, "ssh-config", + &instance->remote.ssh_config, SOURCE_CMD, 0, + OPTION_REMOTE_GROUP, 0, option_get_value + }, + { 0 } + }; + + + init_config(instance, instance_name); + + sprintf(instance->backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + canonicalize_path(instance->backup_instance_path); + + sprintf(instance->arclog_path, "%s/%s/%s", + backup_path, "wal", instance_name); + canonicalize_path(instance->arclog_path); + + join_path_components(path, instance->backup_instance_path, + BACKUP_CATALOG_CONF_FILE); + + if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) + { + elog(WARNING, "Control file \"%s\" doesn't exist", path); + pfree(instance); + return NULL; + } + + parsed_options = config_read_opt(path, instance_options, WARNING, true, true); + + if (parsed_options == 0) + { + elog(WARNING, "Control file \"%s\" is empty", path); + pfree(instance); + return NULL; + } + + if (log_level_console) + instance->logger.log_level_console = parse_log_level(log_level_console); + + if (log_level_file) + instance->logger.log_level_file = parse_log_level(log_level_file); + + if (compress_alg) + instance->compress_alg = parse_compress_alg(compress_alg); + + return instance; +} + static void assign_log_level_console(ConfigOption *opt, const char *arg) { diff --git a/src/delete.c b/src/delete.c index 965f14935..2a7e68e4b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -37,7 +37,7 @@ do_delete(time_t backup_id) TimeLineID oldest_tli = 0; /* Get complete list of backups */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); delete_list = parray_new(); @@ -133,7 +133,7 @@ int do_retention(void) backup_merged = false; /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -634,7 +634,7 @@ do_retention_wal(void) int i; /* Get list of backups. */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -697,7 +697,7 @@ delete_backup_files(pgBackup *backup) * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which * the error occurs before deleting all backup files. */ - write_backup_status(backup, BACKUP_STATUS_DELETING); + write_backup_status(backup, BACKUP_STATUS_DELETING, instance_name); /* list files to be deleted */ files = parray_new(); @@ -853,7 +853,7 @@ do_delete_instance(void) /* Delete all backups. */ - backup_list = catalog_get_backup_list(INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1); diff --git a/src/dir.c b/src/dir.c index 8732e7120..571e26110 100644 --- a/src/dir.c +++ b/src/dir.c @@ -118,11 +118,11 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; -static int BlackListCompare(const void *str1, const void *str2); +static int pgCompareString(const void *str1, const void *str2); static char dir_check_file(pgFile *file); static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, parray *black_list, + bool follow_symlink, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -450,7 +450,7 @@ pgFileCompareSize(const void *f1, const void *f2) } static int -BlackListCompare(const void *str1, const void *str2) +pgCompareString(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); } @@ -491,45 +491,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink bool add_root, int external_dir_num, fio_location location) { pgFile *file; - parray *black_list = NULL; - char path[MAXPGPATH]; - - join_path_components(path, backup_instance_path, PG_BLACK_LIST); - /* List files with black list */ - if (root && instance_config.pgdata && - strcmp(root, instance_config.pgdata) == 0 && - fileExists(path, FIO_BACKUP_HOST)) - { - FILE *black_list_file = NULL; - char buf[MAXPGPATH * 2]; - char black_item[MAXPGPATH * 2]; - - black_list = parray_new(); - black_list_file = fio_open_stream(path, FIO_BACKUP_HOST); - - if (black_list_file == NULL) - elog(ERROR, "cannot open black_list: %s", strerror(errno)); - - while (fgets(buf, lengthof(buf), black_list_file) != NULL) - { - black_item[0] = '\0'; - join_path_components(black_item, instance_config.pgdata, buf); - - if (black_item[strlen(black_item) - 1] == '\n') - black_item[strlen(black_item) - 1] = '\0'; - - if (black_item[0] == '#' || black_item[0] == '\0') - continue; - - parray_append(black_list, pgut_strdup(black_item)); - } - - if (ferror(black_list_file)) - elog(ERROR, "Failed to read from file: \"%s\"", path); - - fio_close_stream(black_list_file); - parray_qsort(black_list, BlackListCompare); - } file = pgFileNew(root, "", follow_symlink, external_dir_num, location); if (file == NULL) @@ -553,17 +514,11 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink if (add_root) parray_append(files, file); - dir_list_file_internal(files, file, exclude, follow_symlink, black_list, + dir_list_file_internal(files, file, exclude, follow_symlink, external_dir_num, location); if (!add_root) pgFileFree(file); - - if (black_list) - { - parray_walk(black_list, pfree); - parray_free(black_list); - } } #define CHECK_FALSE 0 @@ -772,7 +727,7 @@ dir_check_file(pgFile *file) */ static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, parray *black_list, + bool follow_symlink, int external_dir_num, fio_location location) { DIR *dir; @@ -829,15 +784,6 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, continue; } - /* Skip if the directory is in black_list defined by user */ - if (black_list && parray_bsearch(black_list, file->path, - BlackListCompare)) - { - elog(LOG, "Skip \"%s\": it is in the user's black list", file->path); - pgFileFree(file); - continue; - } - if (exclude) { check_res = dir_check_file(file); @@ -863,7 +809,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, file, exclude, follow_symlink, - black_list, external_dir_num, location); + external_dir_num, location); } if (errno && errno != ENOENT) @@ -1662,7 +1608,7 @@ make_external_directory_list(const char *colon_separated_dirs, bool remap) p = strtok(NULL, EXTERNAL_DIRECTORY_DELIMITER); } pfree(tmp); - parray_qsort(list, BlackListCompare); + parray_qsort(list, pgCompareString); return list; } @@ -1690,7 +1636,7 @@ backup_contains_external(const char *dir, parray *dirs_list) if (!dirs_list) /* There is no external dirs in backup */ return false; - search_result = parray_bsearch(dirs_list, dir, BlackListCompare); + search_result = parray_bsearch(dirs_list, dir, pgCompareString); return search_result != NULL; } diff --git a/src/help.c b/src/help.c index 87a0e3312..3cbba83e9 100644 --- a/src/help.c +++ b/src/help.c @@ -166,7 +166,7 @@ help_pg_probackup(void) printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); - printf(_(" [--format=format]\n")); + printf(_(" [--format=format] [--archive]\n")); printf(_(" [--help]\n")); printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -543,11 +543,12 @@ help_show(void) { printf(_("\n%s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); - printf(_(" [--format=format]\n\n")); + printf(_(" [--format=format] [--archive]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); + printf(_(" --archive show WAL archive\n")); printf(_(" --format=format show format=PLAIN|JSON\n\n")); } diff --git a/src/init.c b/src/init.c index 4e52292aa..bfce95737 100644 --- a/src/init.c +++ b/src/init.c @@ -49,21 +49,21 @@ do_init(void) } int -do_add_instance(void) +do_add_instance(InstanceConfig *instance) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; struct stat st; /* PGDATA is always required */ - if (instance_config.pgdata == NULL) + if (instance->pgdata == NULL) elog(ERROR, "Required parameter not specified: PGDATA " "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - instance_config.system_identifier = get_system_identifier(instance_config.pgdata); + instance->system_identifier = get_system_identifier(instance->pgdata); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ - instance_config.xlog_seg_size = get_xlog_seg_size(instance_config.pgdata); + instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); /* Ensure that all root directories already exist */ if (access(backup_path, F_OK) != 0) @@ -78,18 +78,18 @@ do_add_instance(void) elog(ERROR, "%s directory does not exist.", arclog_path_dir); /* Create directory for data files of this specific instance */ - if (stat(backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "instance '%s' already exists", backup_instance_path); - dir_create_dir(backup_instance_path, DIR_PERMISSION); + if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "instance '%s' already exists", instance->backup_instance_path); + dir_create_dir(instance->backup_instance_path, DIR_PERMISSION); /* * Create directory for wal files of this specific instance. * Existence check is extra paranoid because if we don't have such a * directory in data dir, we shouldn't have it in wal as well. */ - if (stat(arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "arclog_path '%s' already exists", arclog_path); - dir_create_dir(arclog_path, DIR_PERMISSION); + if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + elog(ERROR, "arclog_path '%s' already exists", instance->arclog_path); + dir_create_dir(instance->arclog_path, DIR_PERMISSION); /* * Write initial configuration file. @@ -99,9 +99,9 @@ do_add_instance(void) * We need to manually set options source to save them to the configuration * file. */ - config_set_opt(instance_options, &instance_config.system_identifier, + config_set_opt(instance_options, &instance->system_identifier, SOURCE_FILE); - config_set_opt(instance_options, &instance_config.xlog_seg_size, + config_set_opt(instance_options, &instance->xlog_seg_size, SOURCE_FILE); /* pgdata was set through command line */ do_set_config(true); diff --git a/src/merge.c b/src/merge.c index 2d00fa78a..1cdfde090 100644 --- a/src/merge.c +++ b/src/merge.c @@ -66,7 +66,7 @@ do_merge(time_t backup_id) elog(INFO, "Merge started"); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); /* Find destination backup first */ for (i = 0; i < parray_num(backups); i++) @@ -253,8 +253,8 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) if (from_backup->status == BACKUP_STATUS_DELETING) goto delete_source_backup; - write_backup_status(to_backup, BACKUP_STATUS_MERGING); - write_backup_status(from_backup, BACKUP_STATUS_MERGING); + write_backup_status(to_backup, BACKUP_STATUS_MERGING, instance_name); + write_backup_status(from_backup, BACKUP_STATUS_MERGING, instance_name); create_data_directories(files, to_database_path, from_backup_path, false, FIO_BACKUP_HOST); diff --git a/src/parsexlog.c b/src/parsexlog.c index 921fd8a09..a383dc4b8 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -280,7 +280,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - write_backup_status(backup, BACKUP_STATUS_CORRUPT); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name); elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 4ee959e51..5a463c691 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -126,6 +126,7 @@ static bool file_overwrite = false; /* show options */ ShowFormat show_format = SHOW_PLAIN; +bool show_archive = false; /* current settings */ pgBackup current; @@ -203,6 +204,7 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, + { 'b', 160, "archive", &show_archive, SOURCE_CMD_STRICT }, /* options for backward compatibility */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, @@ -251,7 +253,7 @@ main(int argc, char *argv[]) pgBackupInit(¤t); /* Initialize current instance configuration */ - init_config(&instance_config); + init_config(&instance_config, instance_name); PROGRAM_NAME = get_progname(argv[0]); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); @@ -445,10 +447,28 @@ main(int argc, char *argv[]) */ if ((backup_path != NULL) && instance_name) { + /* + * Fill global variables used to generate pathes inside the instance's + * backup catalog. + * TODO replace global variables with InstanceConfig structure fields + */ sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + /* + * Fill InstanceConfig structure fields used to generate pathes inside + * the instance's backup catalog. + * TODO continue refactoring to use these fields instead of global vars + */ + sprintf(instance_config.backup_instance_path, "%s/%s/%s", + backup_path, BACKUPS_DIR, instance_name); + canonicalize_path(instance_config.backup_instance_path); + + sprintf(instance_config.arclog_path, "%s/%s/%s", + backup_path, "wal", instance_name); + canonicalize_path(instance_config.arclog_path); + /* * Ensure that requested backup instance exists. * for all commands except init, which doesn't take this parameter @@ -641,11 +661,13 @@ main(int argc, char *argv[]) switch (backup_subcmd) { case ARCHIVE_PUSH_CMD: - return do_archive_push(wal_file_path, wal_file_name, file_overwrite); + return do_archive_push(&instance_config, wal_file_path, + wal_file_name, file_overwrite); case ARCHIVE_GET_CMD: - return do_archive_get(wal_file_path, wal_file_name); + return do_archive_get(&instance_config, + wal_file_path, wal_file_name); case ADD_INSTANCE_CMD: - return do_add_instance(); + return do_add_instance(&instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(); case INIT_CMD: @@ -682,7 +704,7 @@ main(int argc, char *argv[]) recovery_target_options, restore_params); case SHOW_CMD: - return do_show(current.backup_id); + return do_show(instance_name, current.backup_id, show_archive); case DELETE_CMD: if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 332a9aa8b..dba3bbb4d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -58,7 +58,6 @@ extern const char *PROGRAM_EMAIL; #define BACKUP_CATALOG_PID "backup.pid" #define DATABASE_FILE_LIST "backup_content.control" #define PG_BACKUP_LABEL_FILE "backup_label" -#define PG_BLACK_LIST "black_list" #define PG_TABLESPACE_MAP_FILE "tablespace_map" #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" @@ -227,6 +226,10 @@ typedef struct ConnectionArgs */ typedef struct InstanceConfig { + char *name; + char arclog_path[MAXPGPATH]; + char backup_instance_path[MAXPGPATH]; + uint64 system_identifier; uint32 xlog_seg_size; @@ -382,6 +385,29 @@ typedef struct } backup_files_arg; +typedef struct timelineInfo timelineInfo; + +/* struct to collect info about timelines in WAL archive */ +struct timelineInfo { + + TimeLineID tli; /* this timeline */ + TimeLineID parent_tli; /* parent timeline. 0 if none */ + timelineInfo *parent_link; /* link to parent timeline */ + XLogRecPtr switchpoint; /* if this timeline has a parent + * switchpoint contains switchpoint LSN, + * otherwise 0 */ + XLogSegNo begin_segno; /* first present segment in this timeline */ + XLogSegNo end_segno; /* last present segment in this timeline */ + int n_xlog_files; /* number of segments (only really existing) + * does not include lost segments */ + size_t size; /* space on disk taken by regular WAL files */ + parray *backups; /* array of pgBackup sturctures with info + * about backups belonging to this timeline */ + parray *lost_segments; /* array of intervals of lost segments */ + pgBackup *closest_backup; /* link to backup, closest to timeline */ +}; + + /* * When copying datafiles to backup we validate and compress them block * by block. Thus special header is required for each data block. @@ -525,6 +551,7 @@ extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); extern parray *get_backup_filelist(pgBackup *backup); +extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI); /* in merge.c */ extern void do_merge(time_t backup_id); @@ -534,21 +561,22 @@ extern parray *read_database_map(pgBackup *backup); /* in init.c */ extern int do_init(void); -extern int do_add_instance(void); +extern int do_add_instance(InstanceConfig *instance); /* in archive.c */ -extern int do_archive_push(char *wal_file_path, char *wal_file_name, - bool overwrite); -extern int do_archive_get(char *wal_file_path, char *wal_file_name); - +extern int do_archive_push(InstanceConfig *instance, char *wal_file_path, + char *wal_file_name, bool overwrite); +extern int do_archive_get(InstanceConfig *instance, char *wal_file_path, + char *wal_file_name); /* in configure.c */ extern void do_show_config(void); extern void do_set_config(bool missing_ok); -extern void init_config(InstanceConfig *config); +extern void init_config(InstanceConfig *config, const char *instance_name); +extern InstanceConfig *readInstanceConfigFile(const char *instance_name); /* in show.c */ -extern int do_show(time_t requested_backup_id); +extern int do_show(const char *instance_name, time_t requested_backup_id, bool show_archive); /* in delete.c */ extern void do_delete(time_t backup_id); @@ -573,15 +601,17 @@ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); extern int do_validate_all(void); /* in catalog.c */ -extern pgBackup *read_backup(time_t timestamp); +extern pgBackup *read_backup(const char *instance_name, time_t timestamp); extern void write_backup(pgBackup *backup); -extern void write_backup_status(pgBackup *backup, BackupStatus status); +extern void write_backup_status(pgBackup *backup, BackupStatus status, + const char *instance_name); extern void write_backup_data_bytes(pgBackup *backup); extern bool lock_backup(pgBackup *backup); extern const char *pgBackupGetBackupMode(pgBackup *backup); -extern parray *catalog_get_backup_list(time_t requested_backup_id); +extern parray *catalog_get_instance_list(void); +extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, @@ -595,6 +625,9 @@ extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); +extern void pgBackupGetPathInInstance(const char *instance_name, + const pgBackup *backup, char *path, size_t len, + const char *subdir1, const char *subdir2); extern int pgBackupCreateDir(pgBackup *backup); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); @@ -621,7 +654,8 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, int external_dir_num, fio_location location); + bool follow_symlink, bool add_root, + int external_dir_num, fio_location location); extern void create_data_directories(parray *dest_files, const char *data_dir, diff --git a/src/restore.c b/src/restore.c index f4701c79e..fb9018880 100644 --- a/src/restore.c +++ b/src/restore.c @@ -42,7 +42,6 @@ static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params); -static parray *read_timeline_history(TimeLineID targetTLI); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); @@ -70,7 +69,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", @@ -125,7 +124,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "%s begin.", action); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); /* Find backup range we should restore or validate. */ while ((i < parray_num(backups)) && !dest_backup) @@ -196,7 +195,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ - timelines = read_timeline_history(rt->target_tli); + timelines = read_timeline_history(arclog_path, rt->target_tli); if (!satisfy_timeline(timelines, current_backup)) { @@ -273,7 +272,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(backup->start_time), missing_backup_id); @@ -920,7 +919,7 @@ create_recovery_conf(time_t backup_id, * based on readTimeLineHistory() in timeline.c */ parray * -read_timeline_history(TimeLineID targetTLI) +read_timeline_history(const char *arclog_path, TimeLineID targetTLI) { parray *result; char path[MAXPGPATH]; diff --git a/src/show.c b/src/show.c index 17592b267..b9a7b5300 100644 --- a/src/show.c +++ b/src/show.c @@ -9,6 +9,7 @@ */ #include "pg_probackup.h" +#include "access/timeline.h" #include #include @@ -16,6 +17,7 @@ #include "utils/json.h" +/* struct to align fields printed in plain format */ typedef struct ShowBackendRow { const char *instance; @@ -32,90 +34,107 @@ typedef struct ShowBackendRow const char *status; } ShowBackendRow; +/* struct to align fields printed in plain format */ +typedef struct ShowArchiveRow +{ + char tli[20]; + char parent_tli[20]; + char switchpoint[20]; + char min_segno[20]; + char max_segno[20]; + char n_segments[20]; + char size[20]; + char zratio[20]; + const char *status; + char n_backups[20]; +} ShowArchiveRow; static void show_instance_start(void); static void show_instance_end(void); -static void show_instance(time_t requested_backup_id, bool show_name); -static int show_backup(time_t requested_backup_id); +static void show_instance(const char *instance_name, time_t requested_backup_id, bool show_name); +static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup); +static int show_backup(const char *instance_name, time_t requested_backup_id); + +static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name); +static void show_instance_json(const char *instance_name, parray *backup_list); + +static void show_instance_archive(InstanceConfig *instance); +static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, + parray *timelines_list, bool show_name); +static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, + parray *tli_list); -static void show_instance_plain(parray *backup_list, bool show_name); -static void show_instance_json(parray *backup_list); +static pgBackup* get_closest_backup(timelineInfo *tlinfo, parray *backup_list); static PQExpBufferData show_buf; static bool first_instance = true; static int32 json_level = 0; +/* + * Entry point of pg_probackup SHOW subcommand. + */ int -do_show(time_t requested_backup_id) +do_show(const char *instance_name, time_t requested_backup_id, bool show_archive) { if (instance_name == NULL && requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use --backup_id option"); + /* + * if instance_name is not specified, + * show information about all instances in this backup catalog + */ if (instance_name == NULL) { - /* Show list of instances */ - char path[MAXPGPATH]; - DIR *dir; - struct dirent *dent; - - /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir = opendir(path); - if (dir == NULL) - elog(ERROR, "Cannot open directory \"%s\": %s", - path, strerror(errno)); + parray *instances = catalog_get_instance_list(); show_instance_start(); - - while (errno = 0, (dent = readdir(dir)) != NULL) + for (int i = 0; i < parray_num(instances); i++) { - char child[MAXPGPATH]; - struct stat st; + InstanceConfig *instance = parray_get(instances, i); + char backup_instance_path[MAXPGPATH]; - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", - child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance->name); - instance_name = dent->d_name; - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - - show_instance(INVALID_BACKUP_ID, true); + if (show_archive) + show_instance_archive(instance); + else + show_instance(instance->name, INVALID_BACKUP_ID, true); } - - if (errno) - elog(ERROR, "Cannot read directory \"%s\": %s", - path, strerror(errno)); - - if (closedir(dir)) - elog(ERROR, "Cannot close directory \"%s\": %s", - path, strerror(errno)); - show_instance_end(); return 0; } - else if (requested_backup_id == INVALID_BACKUP_ID || - show_format == SHOW_JSON) + /* always use */ + else if (show_format == SHOW_JSON || + requested_backup_id == INVALID_BACKUP_ID) { show_instance_start(); - show_instance(requested_backup_id, false); + + if (show_archive) + { + InstanceConfig *instance = readInstanceConfigFile(instance_name); + show_instance_archive(instance); + } + else + show_instance(instance_name, requested_backup_id, false); + show_instance_end(); return 0; } else - return show_backup(requested_backup_id); + { + if (show_archive) + { + InstanceConfig *instance = readInstanceConfigFile(instance_name); + show_instance_archive(instance); + } + else + show_backup(instance_name, requested_backup_id); + + return 0; + } } void @@ -163,63 +182,6 @@ pretty_size(int64 size, char *buf, size_t len) } } -static TimeLineID -get_parent_tli(TimeLineID child_tli) -{ - TimeLineID result = 0; - char path[MAXPGPATH]; - char fline[MAXPGPATH]; - FILE *fd; - - /* Timeline 1 does not have a history file and parent timeline */ - if (child_tli == 1) - return 0; - - /* Search history file in archives */ - snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, - child_tli); - fd = fopen(path, "rt"); - if (fd == NULL) - { - if (errno != ENOENT) - elog(ERROR, "could not open file \"%s\": %s", path, - strerror(errno)); - - /* Did not find history file, do not raise the error */ - return 0; - } - - /* - * Parse the file... - */ - while (fgets(fline, sizeof(fline), fd) != NULL) - { - /* skip leading whitespace and check for # comment */ - char *ptr; - char *endptr; - - for (ptr = fline; *ptr; ptr++) - { - if (!IsSpace(*ptr)) - break; - } - if (*ptr == '\0' || *ptr == '#') - continue; - - /* expect a numeric timeline ID as first field of line */ - result = (TimeLineID) strtoul(ptr, &endptr, 0); - if (endptr == ptr) - elog(ERROR, - "syntax error(timeline ID) in history file: %s", - fline); - } - - fclose(fd); - - /* TLI of the last line is parent TLI */ - return result; -} - /* * Initialize instance visualization. */ @@ -255,16 +217,16 @@ show_instance_end(void) * Show brief meta information about all backups in the backup instance. */ static void -show_instance(time_t requested_backup_id, bool show_name) +show_instance(const char *instance_name, time_t requested_backup_id, bool show_name) { parray *backup_list; - backup_list = catalog_get_backup_list(requested_backup_id); + backup_list = catalog_get_backup_list(instance_name, requested_backup_id); if (show_format == SHOW_PLAIN) - show_instance_plain(backup_list, show_name); + show_instance_plain(instance_name, backup_list, show_name); else if (show_format == SHOW_JSON) - show_instance_json(backup_list); + show_instance_json(instance_name, backup_list); else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -273,15 +235,127 @@ show_instance(time_t requested_backup_id, bool show_name) parray_free(backup_list); } +/* helper routine to print backup info as json object */ +static void +print_backup_json_object(PQExpBuffer buf, pgBackup *backup) +{ + TimeLineID parent_tli; + char timestamp[100] = "----"; + char lsn[20]; + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "id", base36enc(backup->start_time), json_level, + true); + + if (backup->parent_backup != 0) + json_add_value(buf, "parent-backup-id", + base36enc(backup->parent_backup), json_level, true); + + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_level, true); + + json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", + json_level, true); + + json_add_value(buf, "compress-alg", + deparse_compress_alg(backup->compress_alg), json_level, + true); + + json_add_key(buf, "compress-level", json_level); + appendPQExpBuffer(buf, "%d", backup->compress_level); + + json_add_value(buf, "from-replica", + backup->from_replica ? "true" : "false", json_level, + true); + + json_add_key(buf, "block-size", json_level); + appendPQExpBuffer(buf, "%u", backup->block_size); + + json_add_key(buf, "xlog-block-size", json_level); + appendPQExpBuffer(buf, "%u", backup->wal_block_size); + + json_add_key(buf, "checksum-version", json_level); + appendPQExpBuffer(buf, "%u", backup->checksum_version); + + json_add_value(buf, "program-version", backup->program_version, + json_level, true); + json_add_value(buf, "server-version", backup->server_version, + json_level, true); + + json_add_key(buf, "current-tli", json_level); + appendPQExpBuffer(buf, "%d", backup->tli); + + json_add_key(buf, "parent-tli", json_level); + + /* Only incremental backup can have Parent TLI */ + if (backup->backup_mode == BACKUP_MODE_FULL) + parent_tli = 0; + else if (backup->parent_backup_link) + parent_tli = backup->parent_backup_link->tli; + appendPQExpBuffer(buf, "%u", parent_tli); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); + json_add_value(buf, "start-lsn", lsn, json_level, true); + + snprintf(lsn, lengthof(lsn), "%X/%X", + (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); + json_add_value(buf, "stop-lsn", lsn, json_level, true); + + time2iso(timestamp, lengthof(timestamp), backup->start_time); + json_add_value(buf, "start-time", timestamp, json_level, true); + + if (backup->end_time) + { + time2iso(timestamp, lengthof(timestamp), backup->end_time); + json_add_value(buf, "end-time", timestamp, json_level, true); + } + + json_add_key(buf, "recovery-xid", json_level); + appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + + if (backup->recovery_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + json_add_value(buf, "recovery-time", timestamp, json_level, true); + } + + if (backup->data_bytes != BYTES_INVALID) + { + json_add_key(buf, "data-bytes", json_level); + appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + } + + if (backup->wal_bytes != BYTES_INVALID) + { + json_add_key(buf, "wal-bytes", json_level); + appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); + } + + if (backup->primary_conninfo) + json_add_value(buf, "primary_conninfo", backup->primary_conninfo, + json_level, true); + + if (backup->external_dir_str) + json_add_value(buf, "external-dirs", backup->external_dir_str, + json_level, true); + + json_add_value(buf, "status", status2str(backup->status), json_level, + true); + + json_add(buf, JT_END_OBJECT, &json_level); +} + /* * Show detailed meta information about specified backup. */ static int -show_backup(time_t requested_backup_id) +show_backup(const char *instance_name, time_t requested_backup_id) { pgBackup *backup; - backup = read_backup(requested_backup_id); + backup = read_backup(instance_name, requested_backup_id); if (backup == NULL) { // TODO for 3.0: we should ERROR out here. @@ -303,15 +377,11 @@ show_backup(time_t requested_backup_id) return 0; } -/* - * Plain output. - */ - /* * Show instance backups in plain format. */ static void -show_instance_plain(parray *backup_list, bool show_name) +show_instance_plain(const char *instance_name, parray *backup_list, bool show_name) { #define SHOW_FIELDS_COUNT 12 int i; @@ -327,6 +397,7 @@ show_instance_plain(parray *backup_list, bool show_name) uint32 widths_sum = 0; ShowBackendRow *rows; time_t current_time = time(NULL); + TimeLineID parent_tli = 0; for (i = 0; i < SHOW_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -380,9 +451,13 @@ show_instance_plain(parray *backup_list, bool show_name) cur++; /* Current/Parent TLI */ + + if (backup->parent_backup_link != NULL) + parent_tli = backup->parent_backup_link->tli; + snprintf(row->tli, lengthof(row->tli), "%u / %u", backup->tli, - backup->backup_mode == BACKUP_MODE_FULL ? 0 : get_parent_tli(backup->tli)); + backup->backup_mode == BACKUP_MODE_FULL ? 0 : parent_tli); widths[cur] = Max(widths[cur], strlen(row->tli)); cur++; @@ -511,15 +586,11 @@ show_instance_plain(parray *backup_list, bool show_name) pfree(rows); } -/* - * Json output. - */ - /* * Show instance backups in json format. */ static void -show_instance_json(parray *backup_list) +show_instance_json(const char *instance_name, parray *backup_list) { int i; PQExpBuffer buf = &show_buf; @@ -541,118 +612,533 @@ show_instance_json(parray *backup_list) for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = parray_get(backup_list, i); - TimeLineID parent_tli; - char timestamp[100] = "----"; - char lsn[20]; if (i != 0) appendPQExpBufferChar(buf, ','); - json_add(buf, JT_BEGIN_OBJECT, &json_level); + print_backup_json_object(buf, backup); + } + + /* End of backups */ + json_add(buf, JT_END_ARRAY, &json_level); + + /* End of instance object */ + json_add(buf, JT_END_OBJECT, &json_level); + + first_instance = false; +} + +typedef struct xlogInterval +{ + XLogSegNo begin_segno; + XLogSegNo end_segno; +} xlogInterval; + +static timelineInfo * +timelineInfoNew(TimeLineID tli) +{ + timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); + MemSet(tlinfo, 0, sizeof(timelineInfo)); + tlinfo->tli = tli; + tlinfo->switchpoint = InvalidXLogRecPtr; + tlinfo->parent_link = NULL; + return tlinfo; +} + +/* + * show information about WAL archive of the instance + */ +static void +show_instance_archive(InstanceConfig *instance) +{ + parray *xlog_files_list = parray_new(); + parray *timelineinfos; + parray *backups; + timelineInfo *tlinfo; + char arclog_path[MAXPGPATH]; + + /* read all xlog files that belong to this archive */ + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); + dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + parray_qsort(xlog_files_list, pgFileComparePath); + + timelineinfos = parray_new(); + tlinfo = NULL; + + /* walk through files and collect info about timelines */ + for (int i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + TimeLineID tli; + parray *timelines; + + /* regular WAL file */ + if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) + { + int result = 0; + uint32 log, seg; + XLogSegNo segno; + char suffix[MAXPGPATH]; + + result = sscanf(file->name, "%08X%08X%08X.%s", + &tli, &log, &seg, (char *) &suffix); + + if (result < 3) + { + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + continue; + } + + segno = log * instance->xlog_seg_size + seg; + + /* regular WAL file with suffix */ + if (result == 4) + { + /* backup history file. Currently we don't use them */ + if (IsBackupHistoryFileName(file->name)) + { + elog(VERBOSE, "backup history file \"%s\". do nothing", file->name); + continue; + } + /* we only expect compressed wal files with .gz suffix */ + else if (strcmp(suffix, "gz") != 0) + { + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + continue; + } + } + + /* new file belongs to new timeline */ + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + } + /* + * As it is impossible to detect if segments before segno are lost, + * or just do not exist, do not report them as lost. + */ + else if (tlinfo->n_xlog_files != 0) + { + /* check, if segments are consequent */ + XLogSegNo expected_segno = tlinfo->end_segno + 1; + + /* some segments are missing. remember them in lost_segments to report */ + if (segno != expected_segno) + { + xlogInterval *interval = palloc(sizeof(xlogInterval));; + interval->begin_segno = expected_segno; + interval->end_segno = segno - 1; + + if (tlinfo->lost_segments == NULL) + tlinfo->lost_segments = parray_new(); + + parray_append(tlinfo->lost_segments, interval); + } + } + + if (tlinfo->begin_segno == 0) + tlinfo->begin_segno = segno; + + /* this file is the last for this timeline so far */ + tlinfo->end_segno = segno; + /* update counters */ + tlinfo->n_xlog_files++; + tlinfo->size += file->size; + } + /* timeline history file */ + else if (IsTLHistoryFileName(file->name)) + { + TimeLineHistoryEntry *tln; + + sscanf(file->name, "%08X.history", &tli); + timelines = read_timeline_history(arclog_path, tli); + + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + /* + * 1 is the latest timeline in the timelines list. + * 0 - is our timeline, which is of no interest here + */ + tln = (TimeLineHistoryEntry *) parray_get(timelines, 1); + tlinfo->switchpoint = tln->end; + tlinfo->parent_tli = tln->tli; + + /* find parent timeline to link it with this one */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, i); + if (cur->tli == tlinfo->parent_tli) + { + tlinfo->parent_link = cur; + break; + } + } + } + + parray_walk(timelines, pfree); + parray_free(timelines); + } + else + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + } + + /* save information about backups belonging to each timeline */ + backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); + + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *tlinfo = parray_get(timelineinfos, i); + for (int j = 0; j < parray_num(backups); j++) + { + pgBackup *backup = parray_get(backups, j); + if (tlinfo->tli == backup->tli) + { + if (tlinfo->backups == NULL) + tlinfo->backups = parray_new(); + + parray_append(tlinfo->backups, backup); + } + } + } + + /* determine closest backup for every timeline */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *tlinfo = parray_get(timelineinfos, i); + + /* only timeline with switchpoint can possibly have closest backup */ + if XLogRecPtrIsInvalid(tlinfo->switchpoint) + continue; + + /* only timeline with parent timeline can possibly have closest backup */ + if (!tlinfo->parent_link) + continue; + + tlinfo->closest_backup = get_closest_backup(tlinfo, backups); + } + + if (show_format == SHOW_PLAIN) + show_archive_plain(instance->name, instance->xlog_seg_size, timelineinfos, true); + else if (show_format == SHOW_JSON) + show_archive_json(instance->name, instance->xlog_seg_size, timelineinfos); + else + elog(ERROR, "Invalid show format %d", (int) show_format); + + parray_walk(xlog_files_list, pfree); + parray_free(xlog_files_list); +} - json_add_value(buf, "id", base36enc(backup->start_time), json_level, - true); +static void +show_archive_plain(const char *instance_name, uint32 xlog_seg_size, + parray *tli_list, bool show_name) +{ +#define SHOW_ARCHIVE_FIELDS_COUNT 10 + int i; + const char *names[SHOW_ARCHIVE_FIELDS_COUNT] = + { "TLI", "Parent TLI", "Switchpoint", + "Min Segno", "Max Segno", "N segments", "Size", "Zratio", "N backups", "Status"}; + const char *field_formats[SHOW_ARCHIVE_FIELDS_COUNT] = + { " %-*s ", " %-*s ", " %-*s ", " %-*s ", + " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s "}; + uint32 widths[SHOW_ARCHIVE_FIELDS_COUNT]; + uint32 widths_sum = 0; + ShowArchiveRow *rows; + + for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) + widths[i] = strlen(names[i]); + + rows = (ShowArchiveRow *) palloc0(parray_num(tli_list) * + sizeof(ShowArchiveRow)); + + /* + * Fill row values and calculate maximum width of each field. + */ + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + ShowArchiveRow *row = &rows[i]; + int cur = 0; + float zratio = 0; + + /* TLI */ + snprintf(row->tli, lengthof(row->tli), "%u", + tlinfo->tli); + widths[cur] = Max(widths[cur], strlen(row->tli)); + cur++; + + /* Parent TLI */ + snprintf(row->parent_tli, lengthof(row->parent_tli), "%u", + tlinfo->parent_tli); + widths[cur] = Max(widths[cur], strlen(row->parent_tli)); + cur++; + + /* Switchpoint LSN */ + snprintf(row->switchpoint, lengthof(row->switchpoint), "%X/%X", + (uint32) (tlinfo->switchpoint >> 32), + (uint32) tlinfo->switchpoint); + widths[cur] = Max(widths[cur], strlen(row->switchpoint)); + cur++; + + /* Min Segno */ + snprintf(row->min_segno, lengthof(row->min_segno), "%08X%08X", + (uint32) tlinfo->begin_segno / xlog_seg_size, + (uint32) tlinfo->begin_segno % xlog_seg_size); + widths[cur] = Max(widths[cur], strlen(row->min_segno)); + cur++; + + /* Max Segno */ + snprintf(row->max_segno, lengthof(row->max_segno), "%08X%08X", + (uint32) tlinfo->end_segno / xlog_seg_size, + (uint32) tlinfo->end_segno % xlog_seg_size); + widths[cur] = Max(widths[cur], strlen(row->max_segno)); + cur++; + + /* N files */ + snprintf(row->n_segments, lengthof(row->n_segments), "%u", + tlinfo->n_xlog_files); + widths[cur] = Max(widths[cur], strlen(row->n_segments)); + cur++; + + /* Size */ + pretty_size(tlinfo->size, row->size, + lengthof(row->size)); + widths[cur] = Max(widths[cur], strlen(row->size)); + cur++; + + /* Zratio (compression ratio) */ + if (tlinfo->size != 0) + zratio = (float) ((xlog_seg_size*tlinfo->n_xlog_files)/tlinfo->size); + + snprintf(row->zratio, lengthof(row->n_segments), "%.2f", zratio); + widths[cur] = Max(widths[cur], strlen(row->zratio)); + cur++; + + /* N backups */ + snprintf(row->n_backups, lengthof(row->n_backups), "%lu", + tlinfo->backups?parray_num(tlinfo->backups):0); + widths[cur] = Max(widths[cur], strlen(row->n_backups)); + cur++; + + /* Status */ + if (tlinfo->lost_segments == NULL) + row->status = "OK"; + else + row->status = "DEGRADED"; + widths[cur] = Max(widths[cur], strlen(row->status)); + cur++; + } - if (backup->parent_backup != 0) - json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), json_level, true); + for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) + widths_sum += widths[i] + 2 /* two space */; - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), - json_level, true); + if (show_name) + appendPQExpBuffer(&show_buf, "\nARCHIVE INSTANCE '%s'\n", instance_name); - json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", - json_level, true); + /* + * Print header. + */ + for (i = 0; i < widths_sum; i++) + appendPQExpBufferChar(&show_buf, '='); + appendPQExpBufferChar(&show_buf, '\n'); - json_add_value(buf, "compress-alg", - deparse_compress_alg(backup->compress_alg), json_level, - true); + for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) + { + appendPQExpBuffer(&show_buf, field_formats[i], widths[i], names[i]); + } + appendPQExpBufferChar(&show_buf, '\n'); - json_add_key(buf, "compress-level", json_level); - appendPQExpBuffer(buf, "%d", backup->compress_level); + for (i = 0; i < widths_sum; i++) + appendPQExpBufferChar(&show_buf, '='); + appendPQExpBufferChar(&show_buf, '\n'); - json_add_value(buf, "from-replica", - backup->from_replica ? "true" : "false", json_level, - true); + /* + * Print values. + */ + for (i = parray_num(tli_list) - 1; i >= 0; i--) + { + ShowArchiveRow *row = &rows[i]; + int cur = 0; - json_add_key(buf, "block-size", json_level); - appendPQExpBuffer(buf, "%u", backup->block_size); + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->tli); + cur++; - json_add_key(buf, "xlog-block-size", json_level); - appendPQExpBuffer(buf, "%u", backup->wal_block_size); + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->parent_tli); + cur++; - json_add_key(buf, "checksum-version", json_level); - appendPQExpBuffer(buf, "%u", backup->checksum_version); + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->switchpoint); + cur++; - json_add_value(buf, "program-version", backup->program_version, - json_level, true); - json_add_value(buf, "server-version", backup->server_version, - json_level, true); + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->min_segno); + cur++; - json_add_key(buf, "current-tli", json_level); - appendPQExpBuffer(buf, "%d", backup->tli); + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->max_segno); + cur++; + + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->n_segments); + cur++; + + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->size); + cur++; + + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->zratio); + cur++; + + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->n_backups); + cur++; + + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->status); + cur++; + appendPQExpBufferChar(&show_buf, '\n'); + } + + pfree(rows); +} + +static void +show_archive_json(const char *instance_name, uint32 xlog_seg_size, + parray *tli_list) +{ + int i; + PQExpBuffer buf = &show_buf; + + if (!first_instance) + appendPQExpBufferChar(buf, ','); + + /* Begin of instance object */ + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_value(buf, "instance", instance_name, json_level, true); + json_add_key(buf, "timelines", json_level); + + /* + * List timelines. + */ + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + for (i = parray_num(tli_list) - 1; i >= 0; i--) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + char tmp_buf[20]; + float zratio = 0; + + if (i != (parray_num(tli_list) - 1)) + appendPQExpBufferChar(buf, ','); + + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + json_add_key(buf, "tli", json_level); + appendPQExpBuffer(buf, "%u", tlinfo->tli); json_add_key(buf, "parent-tli", json_level); + appendPQExpBuffer(buf, "%u", tlinfo->parent_tli); + + snprintf(tmp_buf, lengthof(tmp_buf), "%X/%X", + (uint32) (tlinfo->switchpoint >> 32), (uint32) tlinfo->switchpoint); + json_add_value(buf, "switchpoint", tmp_buf, json_level, true); - /* Only incremental backup can have Parent TLI */ - if (backup->backup_mode == BACKUP_MODE_FULL) - parent_tli = 0; + snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", + (uint32) tlinfo->begin_segno / xlog_seg_size, + (uint32) tlinfo->begin_segno % xlog_seg_size); + json_add_value(buf, "min-segno", tmp_buf, json_level, true); + + snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", + (uint32) tlinfo->end_segno / xlog_seg_size, + (uint32) tlinfo->end_segno % xlog_seg_size); + json_add_value(buf, "max-segno", tmp_buf, json_level, true); + + json_add_key(buf, "n-segments", json_level); + appendPQExpBuffer(buf, "%d", tlinfo->n_xlog_files); + + json_add_key(buf, "size", json_level); + appendPQExpBuffer(buf, "%lu", tlinfo->size); + + json_add_key(buf, "zratio", json_level); + if (tlinfo->size != 0) + zratio = (float) ((xlog_seg_size*tlinfo->n_xlog_files)/tlinfo->size); + appendPQExpBuffer(buf, "%.2f", zratio); + + if (tlinfo->closest_backup != NULL) + snprintf(tmp_buf, lengthof(tmp_buf), "%s", + base36enc(tlinfo->closest_backup->start_time)); else - parent_tli = get_parent_tli(backup->tli); - appendPQExpBuffer(buf, "%u", parent_tli); + snprintf(tmp_buf, lengthof(tmp_buf), "%s", ""); - snprintf(lsn, lengthof(lsn), "%X/%X", - (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); - json_add_value(buf, "start-lsn", lsn, json_level, true); + json_add_value(buf, "closest-backup-id", tmp_buf, json_level, true); - snprintf(lsn, lengthof(lsn), "%X/%X", - (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - json_add_value(buf, "stop-lsn", lsn, json_level, true); + if (tlinfo->lost_segments == NULL) + json_add_value(buf, "status", "OK", json_level, true); + else + json_add_value(buf, "status", "DEGRADED", json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time); - json_add_value(buf, "start-time", timestamp, json_level, true); + json_add_key(buf, "lost-segments", json_level); - if (backup->end_time) + if (tlinfo->lost_segments != NULL) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); - json_add_value(buf, "end-time", timestamp, json_level, true); - } + json_add(buf, JT_BEGIN_ARRAY, &json_level); - json_add_key(buf, "recovery-xid", json_level); - appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); + for (int j = 0; j < parray_num(tlinfo->lost_segments); j++) + { + xlogInterval *lost_segments = (xlogInterval *) parray_get(tlinfo->lost_segments, j); - if (backup->recovery_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); - json_add_value(buf, "recovery-time", timestamp, json_level, true); - } + if (j != 0) + appendPQExpBufferChar(buf, ','); - if (backup->data_bytes != BYTES_INVALID) - { - json_add_key(buf, "data-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); + json_add(buf, JT_BEGIN_OBJECT, &json_level); + + snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", + (uint32) lost_segments->begin_segno / xlog_seg_size, + (uint32) lost_segments->begin_segno % xlog_seg_size); + json_add_value(buf, "begin-segno", tmp_buf, json_level, true); + + snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", + (uint32) lost_segments->end_segno / xlog_seg_size, + (uint32) lost_segments->end_segno % xlog_seg_size); + json_add_value(buf, "end-segno", tmp_buf, json_level, true); + json_add(buf, JT_END_OBJECT, &json_level); + } + json_add(buf, JT_END_ARRAY, &json_level); } + else + appendPQExpBuffer(buf, "[]"); - if (backup->wal_bytes != BYTES_INVALID) + json_add_key(buf, "backups", json_level); + + if (tlinfo->backups != NULL) { - json_add_key(buf, "wal-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); - } + json_add(buf, JT_BEGIN_ARRAY, &json_level); + for (int j = 0; j < parray_num(tlinfo->backups); j++) + { + pgBackup *backup = parray_get(tlinfo->backups, j); - if (backup->primary_conninfo) - json_add_value(buf, "primary_conninfo", backup->primary_conninfo, - json_level, true); + if (j != 0) + appendPQExpBufferChar(buf, ','); - if (backup->external_dir_str) - json_add_value(buf, "external-dirs", backup->external_dir_str, - json_level, true); + print_backup_json_object(buf, backup); + } - json_add_value(buf, "status", status2str(backup->status), json_level, - true); + json_add(buf, JT_END_ARRAY, &json_level); + } + else + appendPQExpBuffer(buf, "[]"); + /* End of timeline */ json_add(buf, JT_END_OBJECT, &json_level); } - /* End of backups */ + /* End of timelines object */ json_add(buf, JT_END_ARRAY, &json_level); /* End of instance object */ @@ -660,3 +1146,63 @@ show_instance_json(parray *backup_list) first_instance = false; } + +/* + * Iterate over parent timelines of a given timeline and look + * for valid backup closest to given timeline switchpoint. + * + * Returns NULL if such backup is not found. + */ +pgBackup* +get_closest_backup(timelineInfo *tlinfo, parray *backup_list) +{ + pgBackup *closest_backup = NULL; + int i; + + while (tlinfo->parent_link) + { + /* + * Iterate over backups belonging to parent timeline and look + * for candidates. + */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + + /* Backups belonging to timelines other than parent timeline can be safely skipped */ + if (backup->tli != tlinfo->parent_tli) + continue; + + /* Backups in future can be safely skipped */ + if (backup->stop_lsn > tlinfo->switchpoint) + continue; + + /* Only valid backups closest to switchpoint should be considered */ + if (backup->stop_lsn <= tlinfo->switchpoint && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) + { + /* + * We have found first candidate. + * Now we should determine whether it`s closest to switchpoint or nor. + */ + + if (!closest_backup) + closest_backup = backup; + + /* Check if backup is closer to switchpoint than current candidate */ + if (backup->stop_lsn > closest_backup->stop_lsn) + closest_backup = backup; + } + } + + /* Closest backup is found */ + if (closest_backup) + break; + + /* Switch to parent */ + tlinfo = tlinfo->parent_link; + } + + return closest_backup; +} diff --git a/src/validate.c b/src/validate.c index d523f88ef..e13abcd47 100644 --- a/src/validate.c +++ b/src/validate.c @@ -69,7 +69,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", base36enc(backup->start_time), status2str(backup->status)); - write_backup_status(backup, BACKUP_STATUS_ERROR); + write_backup_status(backup, BACKUP_STATUS_ERROR, instance_name); corrupted_backup_found = true; return; } @@ -167,7 +167,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) /* Update backup status */ write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK); + BACKUP_STATUS_OK, instance_name); if (corrupted) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); @@ -426,7 +426,7 @@ do_validate_instance(void) elog(INFO, "Validate backups of the instance '%s'", instance_name); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) @@ -456,7 +456,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(current_backup->start_time), parent_backup_id); @@ -480,7 +480,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(current_backup->start_time), backup_id, status2str(tmp_backup->status)); @@ -553,7 +553,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(backup->start_time), diff --git a/tests/archive.py b/tests/archive.py index d22cd915b..a94170f29 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -7,6 +7,7 @@ import subprocess from sys import exit from time import sleep +from distutils.dir_util import copy_tree module_name = 'archive' @@ -263,8 +264,7 @@ def test_pgpro434_3(self): log_content) else: self.assertIn( - "ERROR: Switched WAL segment 000000010000000000000002 " - "could not be archived in 60 seconds", + "ERROR: WAL segment 000000010000000000000002 could not be archived in 60 seconds", log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') @@ -1115,3 +1115,349 @@ def test_archive_pg_receivexlog_compression_pg10(self): # Clean after yourself pg_receivexlog.kill() self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog(self): + """ + ARCHIVE replica: + + t6 |----------------------- + t5 | |------- + | | + t4 | |-------------- + | | + t3 | |--B1--|/|--B2-|/|-B3--- + | | + t2 |--A1--------A2--- + t1 ---------Y1--Y2-- + + ARCHIVE master: + t1 -Z1--Z2--- + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + + master.slow_start() + + # FULL + master.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'master', master) + + # PAGE + master.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'master', master, backup_type='page') + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + # Check data correctness on replica + replica.slow_start(replica=True) + + # FULL backup replica + Y1 = self.backup_node( + backup_dir, 'replica', replica, + options=['--stream', '--archive-timeout=60s']) + + master.pgbench_init(scale=5) + + # PAGE backup replica + Y2 = self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream', '--archive-timeout=60s']) + + # create timeline t2 + replica.promote() + + # do checkpoint to increment timeline ID in pg_control + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + + # FULL backup replica + A1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=5) + + replica.safe_psql( + 'postgres', + "CREATE TABLE t1 (a text)") + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # DELTA backup replica + A2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='delta') + + # create timeline t3 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + replica.slow_start() + + B1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=2) + + B2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + B3 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + # create timeline t4 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't2 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't3 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,10) i') + + # create timeline t5 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=4', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't4 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + # create timeline t6 + replica.cleanup() + + self.restore_node( + backup_dir, 'replica', replica, backup_id=A1, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + replica.slow_start() + + replica.pgbench_init(scale=2) + + show = self.show_archive(backup_dir, as_text=True) + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + if instance['instance'] == 'master': + master_timelines = instance['timelines'] + + # check that all timelines are ok + for timeline in replica_timelines: + self.assertTrue(timeline['status'], 'OK') + + # check that all timelines are ok + for timeline in master_timelines: + self.assertTrue(timeline['status'], 'OK') + + # create holes in t3 + wals_dir = os.path.join(backup_dir, 'wal', 'replica') + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) + and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') + ] + wals.sort() + + # check that t3 is ok + self.show_archive(backup_dir) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + # check that t3 is not OK + show = self.show_archive(backup_dir) + + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + # sanity + for timeline in replica_timelines: + if timeline['tli'] == 1: + timeline_1 = timeline + continue + + if timeline['tli'] == 2: + timeline_2 = timeline + continue + + if timeline['tli'] == 3: + timeline_3 = timeline + continue + + if timeline['tli'] == 4: + timeline_4 = timeline + continue + + if timeline['tli'] == 5: + timeline_5 = timeline + continue + + if timeline['tli'] == 6: + timeline_6 = timeline + continue + + self.assertEqual(timeline_6['status'], "OK") + self.assertEqual(timeline_5['status'], "OK") + self.assertEqual(timeline_4['status'], "OK") + self.assertEqual(timeline_3['status'], "DEGRADED") + self.assertEqual(timeline_2['status'], "OK") + self.assertEqual(timeline_1['status'], "OK") + + self.assertEqual(len(timeline_3['lost-segments']), 2) + self.assertEqual(timeline_3['lost-segments'][0]['begin-segno'], '0000000000000012') + self.assertEqual(timeline_3['lost-segments'][0]['end-segno'], '0000000000000013') + self.assertEqual(timeline_3['lost-segments'][1]['begin-segno'], '0000000000000017') + self.assertEqual(timeline_3['lost-segments'][1]['end-segno'], '0000000000000017') + + self.assertEqual(len(timeline_6['backups']), 0) + self.assertEqual(len(timeline_5['backups']), 0) + self.assertEqual(len(timeline_4['backups']), 0) + self.assertEqual(len(timeline_3['backups']), 3) + self.assertEqual(len(timeline_2['backups']), 2) + self.assertEqual(len(timeline_1['backups']), 2) + + # check closest backup correctness + self.assertEqual(timeline_6['closest-backup-id'], A1) + self.assertEqual(timeline_5['closest-backup-id'], B2) + self.assertEqual(timeline_4['closest-backup-id'], B2) + self.assertEqual(timeline_3['closest-backup-id'], A1) + self.assertEqual(timeline_2['closest-backup-id'], Y2) + + # check parent tli correctness + self.assertEqual(timeline_6['parent-tli'], 2) + self.assertEqual(timeline_5['parent-tli'], 4) + self.assertEqual(timeline_4['parent-tli'], 3) + self.assertEqual(timeline_3['parent-tli'], 2) + self.assertEqual(timeline_2['parent-tli'], 1) + self.assertEqual(timeline_1['parent-tli'], 0) + + + self.del_test_dir(module_name, fname) + +# important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. +# so write validation code accordingly + +# change wal-seg-size +# +# +#t3 ---------------- +# / +#t2 ---------------- +# / +#t1 -A-------- +# +# + + +#t3 ---------------- +# / +#t2 ---------------- +# / +#t1 -A-------- +# \ No newline at end of file diff --git a/tests/backup.py b/tests/backup.py index f15ae1680..ee7811d40 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3,6 +3,8 @@ from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import shutil +from distutils.dir_util import copy_tree +from testgres import ProcessType module_name = 'backup' @@ -2014,21 +2016,32 @@ def test_backup_with_less_privileges_role(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) + copy_tree( + os.path.join(backup_dir, 'wal', 'node'), + os.path.join(backup_dir, 'wal', 'replica')) + replica.slow_start(replica=True) + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + # FULL backup from replica self.backup_node( backup_dir, 'replica', replica, datname='backupdb', options=['--stream', '-U', 'backup']) + self.switch_wal_segment(node) + self.backup_node( backup_dir, 'replica', replica, datname='backupdb', - options=['-U', 'backup', '--log-level-file=verbose']) + options=['-U', 'backup', '--log-level-file=verbose', '--archive-timeout=100s']) # PAGE backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup']) + datname='backupdb', options=['-U', 'backup', '--archive-timeout=100s']) + self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) @@ -2036,7 +2049,7 @@ def test_backup_with_less_privileges_role(self): # DELTA backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) + datname='backupdb', options=['-U', 'backup', '--archive-timeout=100s']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) diff --git a/tests/delta.py b/tests/delta.py index 291d12dbe..e0f0c9f7e 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1275,7 +1275,7 @@ def test_delta_nullified_heap_page_backup(self): content = f.read() self.assertIn( - "LOG: File: {0} blknum 1, empty page".format(file), + "VERBOSE: File: {0} blknum 1, empty page".format(file), content) self.assertNotIn( "Skipping blknum 1 in file: {0}".format(file), diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 18d2673f6..e8e27b8f0 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -947,6 +947,39 @@ def show_pb( return specific_record + def show_archive( + self, backup_dir, instance=None, options=[], + as_text=False, as_json=True, old_binary=False + ): + + cmd_list = [ + 'show', + '--archive', + '-B', backup_dir, + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run_pb(cmd_list + options, old_binary=old_binary) + + if as_json: + if as_text: + data = self.run_pb(cmd_list + options, old_binary=old_binary) + else: + data = json.loads(self.run_pb(cmd_list + options, old_binary=old_binary)) + return data + else: + show_splitted = self.run_pb( + cmd_list + options, old_binary=old_binary).splitlines() + print(show_splitted) + exit(1) + def validate_pb( self, backup_dir, instance=None, backup_id=None, options=[], old_binary=False, gdb=False diff --git a/tests/option.py b/tests/option.py index 9cf0b8f36..71145e5f0 100644 --- a/tests/option.py +++ b/tests/option.py @@ -100,7 +100,8 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: You must specify at least one of the delete options: --expired |--wal |--merge-expired |--delete-invalid |--backup_id', + 'ERROR: You must specify at least one of the delete options: ' + '--delete-expired |--delete-wal |--merge-expired |(-i, --backup-id)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) diff --git a/tests/page.py b/tests/page.py index fafdce3f0..d064eb744 100644 --- a/tests/page.py +++ b/tests/page.py @@ -696,8 +696,8 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and + 'INFO: Wait for WAL segment' in e.message and + 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -721,8 +721,8 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and + 'INFO: Wait for WAL segment' in e.message and + 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -811,8 +811,8 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and + 'INFO: Wait for WAL segment' in e.message and + 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, @@ -836,8 +836,8 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and + 'INFO: Wait for WAL segment' in e.message and + 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( @@ -933,8 +933,8 @@ def test_page_backup_with_alien_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and + 'INFO: Wait for WAL segment' in e.message and + 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'WAL file is from different database system: WAL file database system identifier is' in e.message and 'pg_control database system identifier is' in e.message and diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 0def742a4..d01e1999d 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -126,7 +126,7 @@ def test_minrecpoint_on_replica(self): 'recovery.conf', "recovery_target = 'immediate'") replica.append_conf( 'recovery.conf', "recovery_target_action = 'promote'") - replica.slow_start() + replica.slow_start(replica=True) if self.get_version(node) < 100000: script = ''' diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 122b2793f..7b2b223e2 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -57,8 +57,8 @@ def test_pgpro589(self): except ProbackupException as e: self.assertTrue( 'INFO: Wait for WAL segment' in e.message and - 'ERROR: Switched WAL segment' in e.message and - 'could not be archived' in e.message, + 'ERROR: WAL segment' in e.message and + 'could not be archived in 10 seconds' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/validate.py b/tests/validate.py index 519dc6480..7855f1ef2 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -13,6 +13,36 @@ class ValidateTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_all_empty_catalog(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup_dir is empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: This backup catalog contains no backup instances', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure def test_basic_validate_nullified_heap_page_backup(self): @@ -843,11 +873,18 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): backup_dir, 'node', node, backup_type='page') # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + target_xid = node.safe_psql( "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i RETURNING (xmin)")[0][0] + "from generate_series(30001, 30001) i RETURNING (xmin)").rstrip() + backup_id_5 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -899,8 +936,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.validate_pb( backup_dir, 'node', options=[ - '-i', backup_id_4, '--xid={0}'.format(target_xid), - "-j", "4"]) + '-i', backup_id_4, '--xid={0}'.format(target_xid), "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of data files corruption.\n " @@ -3599,7 +3635,9 @@ def test_recovery_target_lsn_backup_victim(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,100) i") - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--log-level-console=LOG'], gdb=True) gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -3614,6 +3652,8 @@ def test_recovery_target_lsn_backup_victim(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') + self.switch_wal_segment(node) + target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] self.validate_pb( From 3fa01d32366bc68d8f204b18a76307f18c386453 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 17 Sep 2019 17:37:18 +0300 Subject: [PATCH 0957/2107] tests: minor fix --- tests/expected/option_help.out | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 2d7ae3b76..9b1970ee8 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -72,6 +72,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] + [--db-include | --db-exclude] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] @@ -94,7 +95,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] - [--format=format] + [--format=format] [--archive] [--help] pg_probackup delete -B backup-path --instance=instance_name From b69e29bc0b7c420198773ed82b3900a013a0023e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Sep 2019 09:08:16 +0300 Subject: [PATCH 0958/2107] Improve compatibility between different versions of binary and instance --- src/configure.c | 7 +++++++ src/pg_probackup.c | 12 ++++++++++-- src/show.c | 8 ++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/configure.c b/src/configure.c index 5619cfd58..cbe97e285 100644 --- a/src/configure.c +++ b/src/configure.c @@ -571,7 +571,14 @@ readInstanceConfigFile(const char *instance_name) if (compress_alg) instance->compress_alg = parse_compress_alg(compress_alg); +#if PG_VERSION_NUM >= 110000 + /* If for some reason xlog-seg-size is missing, then set it to 16MB */ + if (!instance->xlog_seg_size) + instance->xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; +#endif + return instance; + } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5a463c691..09912bcb6 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -564,10 +564,18 @@ main(int argc, char *argv[]) #if PG_VERSION_NUM >= 110000 /* Check xlog-seg-size option */ if (instance_name && - backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && + backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != SET_CONFIG_CMD && !IsValidWalSegSize(instance_config.xlog_seg_size)) - elog(ERROR, "Invalid WAL segment size %u", instance_config.xlog_seg_size); + { + /* If we are working with instance of PG<11 using PG11 binary, + * then xlog_seg_size is equal to zero. Manually set it to 16MB. + */ + if (instance_config.xlog_seg_size == 0) + instance_config.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; + else + elog(ERROR, "Invalid WAL segment size %u", instance_config.xlog_seg_size); + } #endif /* Sanity check of --backup-id option */ diff --git a/src/show.c b/src/show.c index b9a7b5300..b75774a2c 100644 --- a/src/show.c +++ b/src/show.c @@ -78,7 +78,11 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive { if (instance_name == NULL && requested_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "You must specify --instance to use --backup_id option"); + elog(ERROR, "You must specify --instance to use (-i, --backup-id) option"); + + if (show_archive && + requested_backup_id != INVALID_BACKUP_ID) + elog(ERROR, "You cannot specify --archive and (-i, --backup-id) options together"); /* * if instance_name is not specified, @@ -105,7 +109,7 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive return 0; } - /* always use */ + /* always use */ else if (show_format == SHOW_JSON || requested_backup_id == INVALID_BACKUP_ID) { From a363405aec34613c19a2b6b2905d2c5aa2c5f59f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Sep 2019 18:33:28 +0300 Subject: [PATCH 0959/2107] tests: minor fixes --- tests/pgpro2068.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index d01e1999d..0e6a8b98d 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -83,7 +83,7 @@ def test_minrecpoint_on_replica(self): options=["-c", "4", "-j 4", "-T", "100"]) # wait for shared buffer on replica to be filled with dirty data - sleep(10) + sleep(20) # get pids of replica background workers startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] @@ -121,11 +121,13 @@ def test_minrecpoint_on_replica(self): except: pass + # MinRecLSN = replica.get_control_data()['Minimum recovery ending location'] + # Promote replica with 'immediate' target action replica.append_conf( 'recovery.conf', "recovery_target = 'immediate'") replica.append_conf( - 'recovery.conf', "recovery_target_action = 'promote'") + 'recovery.conf', "recovery_target_action = 'pause'") replica.slow_start(replica=True) if self.get_version(node) < 100000: @@ -133,7 +135,7 @@ def test_minrecpoint_on_replica(self): DO $$ relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("select pg_last_xlog_replay_location() as lsn")[0]['lsn'] +current_xlog_lsn = plpy.execute("SELECT min_recovery_end_lsn as lsn FROM pg_control_recovery()")[0]['lsn'] plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) found_corruption = False for relation in relations: From 5b38c7308976500073c0fc7fc3c70cd930df1798 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Sep 2019 18:52:13 +0300 Subject: [PATCH 0960/2107] Minor refactoring of WAL timelines constructor --- src/catalog.c | 288 +++++++++++++++++++++++++++++++++++++++++++++ src/help.c | 2 +- src/pg_probackup.c | 3 + src/pg_probackup.h | 8 ++ src/show.c | 250 +-------------------------------------- 5 files changed, 301 insertions(+), 250 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 9a137ae3c..56bc72d70 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -9,6 +9,7 @@ */ #include "pg_probackup.h" +#include "access/timeline.h" #include #include @@ -18,12 +19,25 @@ #include "utils/file.h" #include "utils/configuration.h" +static pgBackup* get_closest_backup(timelineInfo *tlinfo, parray *backup_list); +static pgBackup* get_oldest_backup(timelineInfo *tlinfo, parray *backup_list); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); static bool exit_hook_registered = false; static parray *lock_files = NULL; +static timelineInfo * +timelineInfoNew(TimeLineID tli) +{ + timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); + MemSet(tlinfo, 0, sizeof(timelineInfo)); + tlinfo->tli = tli; + tlinfo->switchpoint = InvalidXLogRecPtr; + tlinfo->parent_link = NULL; + return tlinfo; +} + /* Iterate over locked backups and delete locks files */ static void unlink_lock_atexit(void) @@ -665,6 +679,280 @@ pgBackupCreateDir(pgBackup *backup) return 0; } +/* + * Create list of timelines + */ +parray * +catalog_get_timelines(InstanceConfig *instance) +{ + parray *xlog_files_list = parray_new(); + parray *timelineinfos; + parray *backups; + timelineInfo *tlinfo; + char arclog_path[MAXPGPATH]; + + /* read all xlog files that belong to this archive */ + sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); + dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + parray_qsort(xlog_files_list, pgFileComparePath); + + timelineinfos = parray_new(); + tlinfo = NULL; + + /* walk through files and collect info about timelines */ + for (int i = 0; i < parray_num(xlog_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + TimeLineID tli; + parray *timelines; + + /* regular WAL file */ + if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) + { + int result = 0; + uint32 log, seg; + XLogSegNo segno; + char suffix[MAXPGPATH]; + + result = sscanf(file->name, "%08X%08X%08X.%s", + &tli, &log, &seg, (char *) &suffix); + + if (result < 3) + { + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + continue; + } + + segno = log * instance->xlog_seg_size + seg; + + /* regular WAL file with suffix */ + if (result == 4) + { + /* backup history file. Currently we don't use them */ + if (IsBackupHistoryFileName(file->name)) + { + elog(VERBOSE, "backup history file \"%s\". do nothing", file->name); + continue; + } + /* we only expect compressed wal files with .gz suffix */ + else if (strcmp(suffix, "gz") != 0) + { + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + continue; + } + } + + /* new file belongs to new timeline */ + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + } + /* + * As it is impossible to detect if segments before segno are lost, + * or just do not exist, do not report them as lost. + */ + else if (tlinfo->n_xlog_files != 0) + { + /* check, if segments are consequent */ + XLogSegNo expected_segno = tlinfo->end_segno + 1; + + /* some segments are missing. remember them in lost_segments to report */ + if (segno != expected_segno) + { + xlogInterval *interval = palloc(sizeof(xlogInterval));; + interval->begin_segno = expected_segno; + interval->end_segno = segno - 1; + + if (tlinfo->lost_segments == NULL) + tlinfo->lost_segments = parray_new(); + + parray_append(tlinfo->lost_segments, interval); + } + } + + if (tlinfo->begin_segno == 0) + tlinfo->begin_segno = segno; + + /* this file is the last for this timeline so far */ + tlinfo->end_segno = segno; + /* update counters */ + tlinfo->n_xlog_files++; + tlinfo->size += file->size; + } + /* timeline history file */ + else if (IsTLHistoryFileName(file->name)) + { + TimeLineHistoryEntry *tln; + + sscanf(file->name, "%08X.history", &tli); + timelines = read_timeline_history(arclog_path, tli); + + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + /* + * 1 is the latest timeline in the timelines list. + * 0 - is our timeline, which is of no interest here + */ + tln = (TimeLineHistoryEntry *) parray_get(timelines, 1); + tlinfo->switchpoint = tln->end; + tlinfo->parent_tli = tln->tli; + + /* find parent timeline to link it with this one */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, i); + if (cur->tli == tlinfo->parent_tli) + { + tlinfo->parent_link = cur; + break; + } + } + } + + parray_walk(timelines, pfree); + parray_free(timelines); + } + else + elog(WARNING, "unexpected WAL file name \"%s\"", file->name); + } + + /* save information about backups belonging to each timeline */ + backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); + + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *tlinfo = parray_get(timelineinfos, i); + for (int j = 0; j < parray_num(backups); j++) + { + pgBackup *backup = parray_get(backups, j); + if (tlinfo->tli == backup->tli) + { + if (tlinfo->backups == NULL) + tlinfo->backups = parray_new(); + + parray_append(tlinfo->backups, backup); + } + } + } + + /* determine oldest backup and closest backup for every timeline */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + timelineInfo *tlinfo = parray_get(timelineinfos, i); + + tlinfo->oldest_backup = get_oldest_backup(tlinfo, backups); + tlinfo->closest_backup = get_closest_backup(tlinfo, backups); + } + + parray_walk(xlog_files_list, pfree); + parray_free(xlog_files_list); + + return timelineinfos; +} + +/* + * Iterate over parent timelines of a given timeline and look + * for valid backup closest to given timeline switchpoint. + * + * Returns NULL if such backup is not found. + */ +pgBackup* +get_closest_backup(timelineInfo *tlinfo, parray *backup_list) +{ + pgBackup *closest_backup = NULL; + int i; + + /* Only timeline with switchpoint can possibly have closest backup */ + if (XLogRecPtrIsInvalid(tlinfo->switchpoint)) + return NULL; + + /* Only timeline with parent timeline can possibly have closest backup */ + if (!tlinfo->parent_link) + return NULL; + + while (tlinfo->parent_link) + { + /* + * Iterate over backups belonging to parent timeline and look + * for candidates. + */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + + /* Backups belonging to timelines other than parent timeline can be safely skipped */ + if (backup->tli != tlinfo->parent_tli) + continue; + + /* Backups in future can be safely skipped */ + if (backup->stop_lsn > tlinfo->switchpoint) + continue; + + /* Backups with invalid STOP LSN can be safely skipped */ + if (XLogRecPtrIsInvalid(backup->stop_lsn) || + !XRecOffIsValid(backup->stop_lsn)) + continue; + + /* Only valid backups closest to switchpoint should be considered */ + if (backup->stop_lsn <= tlinfo->switchpoint && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) + { + /* Check if backup is closer to switchpoint than current candidate */ + if (!closest_backup || backup->stop_lsn > closest_backup->stop_lsn) + closest_backup = backup; + } + } + + /* Closest backup is found */ + if (closest_backup) + break; + + /* Switch to parent */ + tlinfo = tlinfo->parent_link; + } + + return closest_backup; +} + +/* + * Iterate over timelines and look for oldest backup on each timeline + * Returns NULL if such backup is not found. + */ +pgBackup* +get_oldest_backup(timelineInfo *tlinfo, parray *backup_list) +{ + pgBackup *oldest_backup = NULL; + int i; + + /* + * Iterate over backups belonging to timeline and look + * for candidates. + */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); + + /* Backups belonging to other timelines can be safely skipped */ + if (backup->tli != tlinfo->tli) + continue; + + /* Backups with invalid START LSN can be safely skipped */ + if (XLogRecPtrIsInvalid(backup->start_lsn) || + !XRecOffIsValid(backup->start_lsn)) + continue; + + /* Check if backup is older than current candidate */ + if (!oldest_backup || backup->start_lsn < oldest_backup->start_lsn) + oldest_backup = backup; + } + + return oldest_backup; +} + /* * Write information about backup.in to stream "out". */ diff --git a/src/help.c b/src/help.c index 3cbba83e9..ffb35a9ed 100644 --- a/src/help.c +++ b/src/help.c @@ -548,7 +548,7 @@ help_show(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); - printf(_(" --archive show WAL archive\n")); + printf(_(" --archive show WAL archive information\n")); printf(_(" --format=format show format=PLAIN|JSON\n\n")); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 09912bcb6..0e1c07e99 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -440,6 +440,9 @@ main(int argc, char *argv[]) backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) elog(ERROR, "required parameter not specified: --instance"); } + else + /* Set instance name */ + instance_config.name = pgut_strdup(instance_name); /* * If --instance option was passed, construct paths for backup data and diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dba3bbb4d..7f02e9bf6 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -405,8 +405,15 @@ struct timelineInfo { * about backups belonging to this timeline */ parray *lost_segments; /* array of intervals of lost segments */ pgBackup *closest_backup; /* link to backup, closest to timeline */ + pgBackup *oldest_backup; /* link to oldest backup on timeline */ }; +typedef struct xlogInterval +{ + XLogSegNo begin_segno; + XLogSegNo end_segno; +} xlogInterval; + /* * When copying datafiles to backup we validate and compress them block @@ -617,6 +624,7 @@ extern void catalog_lock_backup_list(parray *backup_list, int from_idx, extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); +extern parray *catalog_get_timelines(InstanceConfig *instance); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list); diff --git a/src/show.c b/src/show.c index b75774a2c..1e2411a76 100644 --- a/src/show.c +++ b/src/show.c @@ -9,7 +9,6 @@ */ #include "pg_probackup.h" -#include "access/timeline.h" #include #include @@ -64,8 +63,6 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, parray *tli_list); -static pgBackup* get_closest_backup(timelineInfo *tlinfo, parray *backup_list); - static PQExpBufferData show_buf; static bool first_instance = true; static int32 json_level = 0; @@ -632,197 +629,15 @@ show_instance_json(const char *instance_name, parray *backup_list) first_instance = false; } -typedef struct xlogInterval -{ - XLogSegNo begin_segno; - XLogSegNo end_segno; -} xlogInterval; - -static timelineInfo * -timelineInfoNew(TimeLineID tli) -{ - timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); - MemSet(tlinfo, 0, sizeof(timelineInfo)); - tlinfo->tli = tli; - tlinfo->switchpoint = InvalidXLogRecPtr; - tlinfo->parent_link = NULL; - return tlinfo; -} - /* * show information about WAL archive of the instance */ static void show_instance_archive(InstanceConfig *instance) { - parray *xlog_files_list = parray_new(); parray *timelineinfos; - parray *backups; - timelineInfo *tlinfo; - char arclog_path[MAXPGPATH]; - - /* read all xlog files that belong to this archive */ - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); - parray_qsort(xlog_files_list, pgFileComparePath); - - timelineinfos = parray_new(); - tlinfo = NULL; - - /* walk through files and collect info about timelines */ - for (int i = 0; i < parray_num(xlog_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - TimeLineID tli; - parray *timelines; - - /* regular WAL file */ - if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) - { - int result = 0; - uint32 log, seg; - XLogSegNo segno; - char suffix[MAXPGPATH]; - - result = sscanf(file->name, "%08X%08X%08X.%s", - &tli, &log, &seg, (char *) &suffix); - - if (result < 3) - { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - continue; - } - - segno = log * instance->xlog_seg_size + seg; - - /* regular WAL file with suffix */ - if (result == 4) - { - /* backup history file. Currently we don't use them */ - if (IsBackupHistoryFileName(file->name)) - { - elog(VERBOSE, "backup history file \"%s\". do nothing", file->name); - continue; - } - /* we only expect compressed wal files with .gz suffix */ - else if (strcmp(suffix, "gz") != 0) - { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - continue; - } - } - /* new file belongs to new timeline */ - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - } - /* - * As it is impossible to detect if segments before segno are lost, - * or just do not exist, do not report them as lost. - */ - else if (tlinfo->n_xlog_files != 0) - { - /* check, if segments are consequent */ - XLogSegNo expected_segno = tlinfo->end_segno + 1; - - /* some segments are missing. remember them in lost_segments to report */ - if (segno != expected_segno) - { - xlogInterval *interval = palloc(sizeof(xlogInterval));; - interval->begin_segno = expected_segno; - interval->end_segno = segno - 1; - - if (tlinfo->lost_segments == NULL) - tlinfo->lost_segments = parray_new(); - - parray_append(tlinfo->lost_segments, interval); - } - } - - if (tlinfo->begin_segno == 0) - tlinfo->begin_segno = segno; - - /* this file is the last for this timeline so far */ - tlinfo->end_segno = segno; - /* update counters */ - tlinfo->n_xlog_files++; - tlinfo->size += file->size; - } - /* timeline history file */ - else if (IsTLHistoryFileName(file->name)) - { - TimeLineHistoryEntry *tln; - - sscanf(file->name, "%08X.history", &tli); - timelines = read_timeline_history(arclog_path, tli); - - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - /* - * 1 is the latest timeline in the timelines list. - * 0 - is our timeline, which is of no interest here - */ - tln = (TimeLineHistoryEntry *) parray_get(timelines, 1); - tlinfo->switchpoint = tln->end; - tlinfo->parent_tli = tln->tli; - - /* find parent timeline to link it with this one */ - for (int i = 0; i < parray_num(timelineinfos); i++) - { - timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, i); - if (cur->tli == tlinfo->parent_tli) - { - tlinfo->parent_link = cur; - break; - } - } - } - - parray_walk(timelines, pfree); - parray_free(timelines); - } - else - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - } - - /* save information about backups belonging to each timeline */ - backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); - - for (int i = 0; i < parray_num(timelineinfos); i++) - { - timelineInfo *tlinfo = parray_get(timelineinfos, i); - for (int j = 0; j < parray_num(backups); j++) - { - pgBackup *backup = parray_get(backups, j); - if (tlinfo->tli == backup->tli) - { - if (tlinfo->backups == NULL) - tlinfo->backups = parray_new(); - - parray_append(tlinfo->backups, backup); - } - } - } - - /* determine closest backup for every timeline */ - for (int i = 0; i < parray_num(timelineinfos); i++) - { - timelineInfo *tlinfo = parray_get(timelineinfos, i); - - /* only timeline with switchpoint can possibly have closest backup */ - if XLogRecPtrIsInvalid(tlinfo->switchpoint) - continue; - - /* only timeline with parent timeline can possibly have closest backup */ - if (!tlinfo->parent_link) - continue; - - tlinfo->closest_backup = get_closest_backup(tlinfo, backups); - } + timelineinfos = catalog_get_timelines(instance); if (show_format == SHOW_PLAIN) show_archive_plain(instance->name, instance->xlog_seg_size, timelineinfos, true); @@ -830,9 +645,6 @@ show_instance_archive(InstanceConfig *instance) show_archive_json(instance->name, instance->xlog_seg_size, timelineinfos); else elog(ERROR, "Invalid show format %d", (int) show_format); - - parray_walk(xlog_files_list, pfree); - parray_free(xlog_files_list); } static void @@ -1150,63 +962,3 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, first_instance = false; } - -/* - * Iterate over parent timelines of a given timeline and look - * for valid backup closest to given timeline switchpoint. - * - * Returns NULL if such backup is not found. - */ -pgBackup* -get_closest_backup(timelineInfo *tlinfo, parray *backup_list) -{ - pgBackup *closest_backup = NULL; - int i; - - while (tlinfo->parent_link) - { - /* - * Iterate over backups belonging to parent timeline and look - * for candidates. - */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = parray_get(backup_list, i); - - /* Backups belonging to timelines other than parent timeline can be safely skipped */ - if (backup->tli != tlinfo->parent_tli) - continue; - - /* Backups in future can be safely skipped */ - if (backup->stop_lsn > tlinfo->switchpoint) - continue; - - /* Only valid backups closest to switchpoint should be considered */ - if (backup->stop_lsn <= tlinfo->switchpoint && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - { - /* - * We have found first candidate. - * Now we should determine whether it`s closest to switchpoint or nor. - */ - - if (!closest_backup) - closest_backup = backup; - - /* Check if backup is closer to switchpoint than current candidate */ - if (backup->stop_lsn > closest_backup->stop_lsn) - closest_backup = backup; - } - } - - /* Closest backup is found */ - if (closest_backup) - break; - - /* Switch to parent */ - tlinfo = tlinfo->parent_link; - } - - return closest_backup; -} From 32e6407e035e46ee9e46739852a0da1d02d7baf1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 19 Sep 2019 09:51:15 +0300 Subject: [PATCH 0961/2107] make pretty_size() more precise --- src/show.c | 57 ++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/show.c b/src/show.c index 1e2411a76..283936339 100644 --- a/src/show.c +++ b/src/show.c @@ -16,6 +16,8 @@ #include "utils/json.h" +#define half_rounded(x) (((x) + ((x) < 0 ? 0 : 1)) / 2) + /* struct to align fields printed in plain format */ typedef struct ShowBackendRow { @@ -141,7 +143,8 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive void pretty_size(int64 size, char *buf, size_t len) { - int exp = 0; + int64 limit = 10 * 1024; + int64 limit2 = limit * 2 - 1; /* minus means the size is invalid */ if (size < 0) @@ -150,36 +153,30 @@ pretty_size(int64 size, char *buf, size_t len) return; } - /* determine postfix */ - while (size > 9999) - { - ++exp; - size /= 1000; - } - - switch (exp) + if (Abs(size) < limit) + snprintf(buf, len, "%dB", (int) size); + else { - case 0: - snprintf(buf, len, "%dB", (int) size); - break; - case 1: - snprintf(buf, len, "%dkB", (int) size); - break; - case 2: - snprintf(buf, len, "%dMB", (int) size); - break; - case 3: - snprintf(buf, len, "%dGB", (int) size); - break; - case 4: - snprintf(buf, len, "%dTB", (int) size); - break; - case 5: - snprintf(buf, len, "%dPB", (int) size); - break; - default: - strncpy(buf, "***", len); - break; + size >>= 9; + if (Abs(size) < limit2) + snprintf(buf, len, "%dkB", (int) half_rounded(size)); + else + { + size >>= 10; + if (Abs(size) < limit2) + snprintf(buf, len, "%dMB", (int) half_rounded(size)); + else + { + size >>= 10; + if (Abs(size) < limit2) + snprintf(buf, len, "%dGB", (int) half_rounded(size)); + else + { + size >>= 10; + snprintf(buf, len, "%dTB", (int) half_rounded(size)); + } + } + } } } From f6ce9f028767de51ea90f041edd6e22c69a8ed0c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 19 Sep 2019 14:53:09 +0300 Subject: [PATCH 0962/2107] Ignore empty timelines in show archive --- src/show.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/show.c b/src/show.c index 283936339..807fdb624 100644 --- a/src/show.c +++ b/src/show.c @@ -648,6 +648,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *tli_list, bool show_name) { + parray *actual_tli_list = parray_new(); #define SHOW_ARCHIVE_FIELDS_COUNT 10 int i; const char *names[SHOW_ARCHIVE_FIELDS_COUNT] = @@ -663,15 +664,24 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); - rows = (ShowArchiveRow *) palloc0(parray_num(tli_list) * + /* Ignore empty timelines */ + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + + if (tlinfo->n_xlog_files > 0) + parray_append(actual_tli_list, tlinfo); + } + + rows = (ShowArchiveRow *) palloc0(parray_num(actual_tli_list) * sizeof(ShowArchiveRow)); /* * Fill row values and calculate maximum width of each field. */ - for (i = 0; i < parray_num(tli_list); i++) + for (i = 0; i < parray_num(actual_tli_list); i++) { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); ShowArchiveRow *row = &rows[i]; int cur = 0; float zratio = 0; @@ -770,7 +780,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, /* * Print values. */ - for (i = parray_num(tli_list) - 1; i >= 0; i--) + for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) { ShowArchiveRow *row = &rows[i]; int cur = 0; @@ -818,6 +828,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, } pfree(rows); + //TODO: free timelines } static void @@ -826,6 +837,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, { int i; PQExpBuffer buf = &show_buf; + parray *actual_tli_list = parray_new(); if (!first_instance) appendPQExpBufferChar(buf, ','); @@ -836,18 +848,28 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add_value(buf, "instance", instance_name, json_level, true); json_add_key(buf, "timelines", json_level); + /* Ignore empty timelines */ + + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + + if (tlinfo->n_xlog_files > 0) + parray_append(actual_tli_list, tlinfo); + } + /* * List timelines. */ json_add(buf, JT_BEGIN_ARRAY, &json_level); - for (i = parray_num(tli_list) - 1; i >= 0; i--) + for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); char tmp_buf[20]; float zratio = 0; - if (i != (parray_num(tli_list) - 1)) + if (i != (parray_num(actual_tli_list) - 1)) appendPQExpBufferChar(buf, ','); json_add(buf, JT_BEGIN_OBJECT, &json_level); From ec6a627d4d259f32ab581d4b8c16bd13d63a4b31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Sep 2019 10:47:07 +0300 Subject: [PATCH 0963/2107] save files belonging to a timeline into xlog_filelist array in timelineInfo structure --- src/catalog.c | 47 +++++++++++++++++++++++++++++++++++++++++++--- src/pg_probackup.h | 16 ++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 56bc72d70..52e2b759c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -35,6 +35,7 @@ timelineInfoNew(TimeLineID tli) tlinfo->tli = tli; tlinfo->switchpoint = InvalidXLogRecPtr; tlinfo->parent_link = NULL; + tlinfo->xlog_filelist = parray_new(); return tlinfo; } @@ -705,6 +706,7 @@ catalog_get_timelines(InstanceConfig *instance) pgFile *file = (pgFile *) parray_get(xlog_files_list, i); TimeLineID tli; parray *timelines; + xlogFile *wal_file = NULL; /* regular WAL file */ if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) @@ -731,7 +733,39 @@ catalog_get_timelines(InstanceConfig *instance) /* backup history file. Currently we don't use them */ if (IsBackupHistoryFileName(file->name)) { - elog(VERBOSE, "backup history file \"%s\". do nothing", file->name); + elog(VERBOSE, "backup history file \"%s\"", file->name); + + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + } + + /* append file to xlog file list */ + wal_file = palloc(sizeof(xlogFile)); + wal_file->file = *file; + wal_file->segno = segno; + wal_file->type = BACKUP_HISTORY_FILE; + parray_append(tlinfo->xlog_filelist, wal_file); + continue; + } + /* partial WAL segment */ + else if (IsPartialXLogFileName(file->name)) + { + elog(VERBOSE, "partial WAL file \"%s\"", file->name); + + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + } + + /* append file to xlog file list */ + wal_file = palloc(sizeof(xlogFile)); + wal_file->file = *file; + wal_file->segno = segno; + wal_file->type = PARTIAL_SEGMENT; + parray_append(tlinfo->xlog_filelist, wal_file); continue; } /* we only expect compressed wal files with .gz suffix */ @@ -779,6 +813,13 @@ catalog_get_timelines(InstanceConfig *instance) /* update counters */ tlinfo->n_xlog_files++; tlinfo->size += file->size; + + /* append file to xlog file list */ + wal_file = palloc(sizeof(xlogFile)); + wal_file->file = *file; + wal_file->segno = segno; + wal_file->type = SEGMENT; + parray_append(tlinfo->xlog_filelist, wal_file); } /* timeline history file */ else if (IsTLHistoryFileName(file->name)) @@ -847,8 +888,8 @@ catalog_get_timelines(InstanceConfig *instance) tlinfo->closest_backup = get_closest_backup(tlinfo, backups); } - parray_walk(xlog_files_list, pfree); - parray_free(xlog_files_list); + //parray_walk(xlog_files_list, pfree); + //parray_free(xlog_files_list); return timelineinfos; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7f02e9bf6..38c32134c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -403,6 +403,8 @@ struct timelineInfo { size_t size; /* space on disk taken by regular WAL files */ parray *backups; /* array of pgBackup sturctures with info * about backups belonging to this timeline */ + parray *xlog_filelist; /* array of ordinary WAL segments, '.partial' + * and '.backup' files belonging to this timeline */ parray *lost_segments; /* array of intervals of lost segments */ pgBackup *closest_backup; /* link to backup, closest to timeline */ pgBackup *oldest_backup; /* link to oldest backup on timeline */ @@ -414,6 +416,20 @@ typedef struct xlogInterval XLogSegNo end_segno; } xlogInterval; +typedef enum xlogFileType +{ + SEGMENT, + PARTIAL_SEGMENT, + BACKUP_HISTORY_FILE +} xlogFileType; + +typedef struct xlogFile +{ + pgFile file; + XLogSegNo segno; + xlogFileType type; +} xlogFile; + /* * When copying datafiles to backup we validate and compress them block From eb9b2347ebfd8bd8d2ad35e4909068918cf0d077 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Sep 2019 11:02:47 +0300 Subject: [PATCH 0964/2107] [Issue #69] New algorithm of WAL purge, based on timelineInfo infrastructure --- src/delete.c | 250 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 193 insertions(+), 57 deletions(-) diff --git a/src/delete.c b/src/delete.c index 2a7e68e4b..8173e41a3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -15,16 +15,20 @@ #include static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - uint32 xlog_seg_size); + uint32 xlog_seg_size); +static void delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tli, + uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); static void do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list); static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); -static void do_retention_wal(void); +static void do_retention_wal(bool dry_run); +// TODO: more useful messages for dry run. static bool backup_deleted = false; /* At least one backup was deleted */ static bool backup_merged = false; /* At least one merge was enacted */ +static bool wal_deleted = false; /* At least one WAL segments was deleted */ void do_delete(time_t backup_id) @@ -33,8 +37,8 @@ do_delete(time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; - XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TimeLineID oldest_tli = 0; +// XLogRecPtr oldest_lsn = InvalidXLogRecPtr; +// TimeLineID oldest_tli = 0; /* Get complete list of backups */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -86,24 +90,7 @@ do_delete(time_t backup_id) /* Clean WAL segments */ if (delete_wal) - { - Assert(target_backup); - - /* Find oldest LSN, used by backups */ - for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - - if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) - { - oldest_lsn = backup->start_lsn; - oldest_tli = backup->tli; - break; - } - } - - delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); - } + do_retention_wal(false); /* cleanup */ parray_walk(backup_list, pgBackupFree); @@ -172,8 +159,8 @@ int do_retention(void) do_retention_purge(to_keep_list, to_purge_list); /* TODO: some sort of dry run for delete_wal */ - if (delete_wal && !dry_run) - do_retention_wal(); + if (delete_wal) + do_retention_wal(dry_run); /* TODO: consider dry-run flag */ @@ -622,47 +609,44 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) } } -/* Purge WAL */ +/* Purge WAL + * Iterate over timelines + * Look for closest_backup, if exists, goto next timelime + * if not exists, look for oldest backup on timeline + */ static void -do_retention_wal(void) +do_retention_wal(bool dry_run) { - parray *backup_list = NULL; - - XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TimeLineID oldest_tli = 0; - bool backup_list_is_empty = false; + parray *tli_list; int i; - /* Get list of backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + tli_list = catalog_get_timelines(&instance_config); - if (parray_num(backup_list) == 0) - backup_list_is_empty = true; - - /* Save LSN and Timeline to remove unnecessary WAL segments */ - for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) + for (i = 0; i < parray_num(tli_list); i++) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - /* Get LSN and TLI of the oldest backup with valid start_lsn and tli */ - if (backup->tli > 0 && !XLogRecPtrIsInvalid(backup->start_lsn)) - { - oldest_tli = backup->tli; - oldest_lsn = backup->start_lsn; - break; - } - } + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - /* Be paranoid */ - if (!backup_list_is_empty && XLogRecPtrIsInvalid(oldest_lsn)) - elog(ERROR, "Not going to purge WAL because LSN is invalid"); + /* Empty timeline can be safely skipped */ + if (tlinfo->n_xlog_files == 0 && + parray_num(tlinfo->xlog_filelist) == 0) + continue; - /* Purge WAL files */ - delete_walfiles(oldest_lsn, oldest_tli, instance_config.xlog_seg_size); + /* If closest backup is exists, then timeline can be safely skipped */ + if (tlinfo->closest_backup) + continue; - /* Cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); + /* + * Purge all WAL segments before START LSN of oldest backup. + * If there is no backups on timeline, then whole timeline + * can be safely purged. + */ + if (tlinfo->oldest_backup) + delete_walfiles_internal(tlinfo->oldest_backup->start_lsn, + tlinfo, instance_config.xlog_seg_size, dry_run); + else + delete_walfiles_internal(InvalidXLogRecPtr, + tlinfo, instance_config.xlog_seg_size, dry_run); + } } /* @@ -728,6 +712,158 @@ delete_backup_files(pgBackup *backup) return; } +/* Purge WAL archive. + * If 'keep_lsn' is InvalidXLogRecPtr, then whole timeline can be purged + * If 'keep_lsn' is valid LSN, then every lesser segment can be purged. + * If 'dry_run' is set, then don`t actually delete anything. + * + * Case 1: + * archive is not empty, 'keep_lsn' is valid and we can delete something. + * Case 2: + * archive is not empty, 'keep_lsn' is valid and prevening us from deleting anything. + * Case 3: + * archive is not empty, 'keep_lsn' is invalid, drop everyhing in archive. + * Case 4: + * archive is empty, 'keep_lsn' is valid, assume corruption of WAL archive. + * Case 5: + * archive is empty, 'keep_lsn' is invalid, drop backup history files + * and partial WAL segments in archive. + * + * Q: Maybe we should stop treating partial WAL segments as second-class citizens? + */ +static void +delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, + uint32 xlog_seg_size, bool dry_run) +{ + XLogSegNo StartSegNo; /* First segment to delete */ + XLogSegNo EndSegNo = 0; /* Oldest segment to keep */ + int rc; + int i; + int wal_size_logical = 0; + int wal_size_actual = 0; + char wal_pretty_size[20]; + bool purge_all = false; + + /* Timeline is completely empty */ + if (parray_num(tlinfo->xlog_filelist) == 0) + { + elog(INFO, "Timeline %i is empty, nothing to remove", tlinfo->tli); + return; + } + + if (XLogRecPtrIsInvalid(keep_lsn)) + { + /* Drop all segments in timeline */ + elog(INFO, "All files on timeline %i will be removed", tlinfo->tli); + StartSegNo = tlinfo->begin_segno; + EndSegNo = tlinfo->end_segno; + purge_all = true; + } + else + { + /* Drop all segments between begin_segno and segment with keep_lsn (excluding) */ + StartSegNo = tlinfo->begin_segno; + GetXLogSegNo(keep_lsn, EndSegNo, xlog_seg_size); + } + + if (EndSegNo > 0 && EndSegNo > StartSegNo) + elog(INFO, "WAL segments between %08X%08X and %08X%08X on timeline %i will be removed", + (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, + (uint32) EndSegNo / xlog_seg_size, (uint32) EndSegNo % xlog_seg_size, + tlinfo->tli); + + if (EndSegNo > StartSegNo) + /* typical scenario */ + wal_size_logical = (EndSegNo-StartSegNo) * xlog_seg_size; + else if (EndSegNo < StartSegNo) + { + /* It is actually possible for EndSegNo to be less than StartSegNo + * in case of : + * 1. WAL archive corruption. + * 2. There is no actual WAL archive to speak of and + * 'keep_lsn' is coming from STREAM backup. + * + * Assume the worst. + */ + if (StartSegNo > 0 && EndSegNo > 0) + elog(WARNING, "On timeline %i first segment %08X%08X is greater than " + "oldest segment to keep %08X%08X. Possible WAL archive corruption.", + tlinfo->tli, + (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, + (uint32) EndSegNo / xlog_seg_size, (uint32) EndSegNo % xlog_seg_size); + } + else if (EndSegNo == StartSegNo && !purge_all) + { + /* 'Nothing to delete' scenario because of 'keep_lsn' + * with possible exception of partial and backup history files. + */ + elog(INFO, "Nothing to remove on timeline %i", tlinfo->tli); + } + + /* Report the logical size to delete */ + if (wal_size_logical > 0) + { + pretty_size(wal_size_logical, wal_pretty_size, lengthof(wal_pretty_size)); + elog(INFO, "WAL size to remove on timeline %i: %s", + tlinfo->tli, wal_pretty_size); + } + + /* Calculate the actual size to delete */ + for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) + { + xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); + + if (purge_all || wal_file->segno < EndSegNo) + wal_size_actual += wal_file->file.size; + } + + /* Report the actual size to delete */ + if (wal_size_actual > 0) + { + pretty_size(wal_size_actual, wal_pretty_size, lengthof(wal_pretty_size)); + elog(INFO, "Resident data size to free on timeline %i: %s", + tlinfo->tli, wal_pretty_size); + } + + if (dry_run) + return; + + for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) + { + xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); + + if (interrupted) + elog(ERROR, "interrupted during WAL archive purge"); + + /* Any segment equal or greater than EndSegNo must be kept + * unless it`s a 'purge all' scenario. + */ + if (purge_all || wal_file->segno < EndSegNo) + { + /* unlink segment */ + rc = unlink(wal_file->file.path); + if (rc < 0) + { + /* Missing file is not considered as error condition */ + if (errno != ENOENT) + elog(ERROR, "Could not remove file \"%s\": %s", + wal_file->file.path, strerror(errno)); + } + else + { + if (wal_file->type == SEGMENT) + elog(VERBOSE, "Removed WAL segment \"%s\"", wal_file->file.path); + else if (wal_file->type == PARTIAL_SEGMENT) + elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_file->file.path); + else if (wal_file->type == BACKUP_HISTORY_FILE) + elog(VERBOSE, "Removed backup history file \"%s\"", wal_file->file.path); + } + + wal_deleted = true; + } + } +} + /* * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups * was deleted and so oldest_lsn is invalid). @@ -739,7 +875,7 @@ delete_backup_files(pgBackup *backup) */ static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - uint32 xlog_seg_size) + uint32 xlog_seg_size) { XLogSegNo targetSegNo; char oldestSegmentNeeded[MAXFNAMELEN]; From b1975e3ba42cc37d6d7ffb3110bb37de5f44a0f5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Sep 2019 15:51:12 +0300 Subject: [PATCH 0965/2107] tests: added archive.ArchiveTest.test_archive_catalog_1 and archive.ArchiveTest.test_archive_catalog_2 --- tests/archive.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/tests/archive.py b/tests/archive.py index a94170f29..6a0d49a4e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1437,6 +1437,107 @@ def test_archive_catalog(self): self.assertEqual(timeline_2['parent-tli'], 1) self.assertEqual(timeline_1['parent-tli'], 0) + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog_1(self): + """ + double segment - compressed and not + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + original_file = os.path.join(wals_dir, '000000010000000000000001.gz') + tmp_file = os.path.join(wals_dir, '000000010000000000000001') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + os.rename( + os.path.join(wals_dir, '000000010000000000000001'), + os.path.join(wals_dir, '000000010000000000000002')) + + show = self.show_archive(backup_dir) + + for instance in show: + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual(timeline['min-segno'], '0000000000000001') + self.assertEqual(timeline['status'], 'OK') + + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog_2(self): + """ + double segment - compressed and not + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + original_file = os.path.join(wals_dir, '000000010000000000000001.gz') + tmp_file = os.path.join(wals_dir, '000000010000000000000001') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + os.rename( + os.path.join(wals_dir, '000000010000000000000001'), + os.path.join(wals_dir, '000000010000000000000002')) + + os.remove(original_file) + + show = self.show_archive(backup_dir) + + for instance in show: + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual(timeline['min-segno'], '0000000000000002') + self.assertEqual(timeline['status'], 'OK') self.del_test_dir(module_name, fname) From 02840b81ab1e8123c1cf9cd57f9fa7968cb3726b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Sep 2019 15:54:42 +0300 Subject: [PATCH 0966/2107] minor changes --- src/delete.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/delete.c b/src/delete.c index 8173e41a3..0aedacd49 100644 --- a/src/delete.c +++ b/src/delete.c @@ -753,7 +753,7 @@ delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (XLogRecPtrIsInvalid(keep_lsn)) { - /* Drop all segments in timeline */ + /* Drop all files in timeline */ elog(INFO, "All files on timeline %i will be removed", tlinfo->tli); StartSegNo = tlinfo->begin_segno; EndSegNo = tlinfo->end_segno; @@ -769,9 +769,11 @@ delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (EndSegNo > 0 && EndSegNo > StartSegNo) elog(INFO, "WAL segments between %08X%08X and %08X%08X on timeline %i will be removed", (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, - (uint32) EndSegNo / xlog_seg_size, (uint32) EndSegNo % xlog_seg_size, + (uint32) (EndSegNo - 1) / xlog_seg_size, + (uint32) (EndSegNo - 1) % xlog_seg_size, tlinfo->tli); + /* sanity */ if (EndSegNo > StartSegNo) /* typical scenario */ wal_size_logical = (EndSegNo-StartSegNo) * xlog_seg_size; @@ -787,7 +789,7 @@ delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, */ if (StartSegNo > 0 && EndSegNo > 0) elog(WARNING, "On timeline %i first segment %08X%08X is greater than " - "oldest segment to keep %08X%08X. Possible WAL archive corruption.", + "oldest segment to keep %08X%08X. Possible WAL archive corruption!", tlinfo->tli, (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, (uint32) EndSegNo / xlog_seg_size, (uint32) EndSegNo % xlog_seg_size); From 6b8ab83548a9fddfb519b2961d596e77b1a23b3a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Sep 2019 16:04:54 +0300 Subject: [PATCH 0967/2107] cleanup in delete.c --- src/delete.c | 122 ++------------------------------------------------- 1 file changed, 4 insertions(+), 118 deletions(-) diff --git a/src/delete.c b/src/delete.c index 0aedacd49..373f87e1a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,9 +14,7 @@ #include #include -static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - uint32 xlog_seg_size); -static void delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tli, +static void delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); @@ -641,10 +639,10 @@ do_retention_wal(bool dry_run) * can be safely purged. */ if (tlinfo->oldest_backup) - delete_walfiles_internal(tlinfo->oldest_backup->start_lsn, + delete_walfiles(tlinfo->oldest_backup->start_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else - delete_walfiles_internal(InvalidXLogRecPtr, + delete_walfiles(InvalidXLogRecPtr, tlinfo, instance_config.xlog_seg_size, dry_run); } } @@ -732,7 +730,7 @@ delete_backup_files(pgBackup *backup) * Q: Maybe we should stop treating partial WAL segments as second-class citizens? */ static void -delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, +delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, uint32 xlog_seg_size, bool dry_run) { XLogSegNo StartSegNo; /* First segment to delete */ @@ -866,118 +864,6 @@ delete_walfiles_internal(XLogRecPtr keep_lsn, timelineInfo *tlinfo, } } -/* - * Deletes WAL segments up to oldest_lsn or all WAL segments (if all backups - * was deleted and so oldest_lsn is invalid). - * - * oldest_lsn - if valid, function deletes WAL segments, which contain lsn - * older than oldest_lsn. If it is invalid function deletes all WAL segments. - * oldest_tli - is used to construct oldest WAL segment in addition to - * oldest_lsn. - */ -static void -delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli, - uint32 xlog_seg_size) -{ - XLogSegNo targetSegNo; - char oldestSegmentNeeded[MAXFNAMELEN]; - DIR *arcdir; - struct dirent *arcde; - char wal_file[MAXPGPATH]; - char max_wal_file[MAXPGPATH]; - char min_wal_file[MAXPGPATH]; - int rc; - - max_wal_file[0] = '\0'; - min_wal_file[0] = '\0'; - - if (!XLogRecPtrIsInvalid(oldest_lsn)) - { - GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size); - GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo, - xlog_seg_size); - - elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded); - } - else - elog(LOG, "removing all WAL segments"); - - /* - * Now it is time to do the actual work and to remove all the segments - * not needed anymore. - */ - if ((arcdir = opendir(arclog_path)) != NULL) - { - while (errno = 0, (arcde = readdir(arcdir)) != NULL) - { - /* - * We ignore the timeline part of the WAL segment identifiers in - * deciding whether a segment is still needed. This ensures that - * we won't prematurely remove a segment from a parent timeline. - * We could probably be a little more proactive about removing - * segments of non-parent timelines, but that would be a whole lot - * more complicated. - * - * We use the alphanumeric sorting property of the filenames to - * decide which ones are earlier than the exclusiveCleanupFileName - * file. Note that this means files are not removed in the order - * they were originally written, in case this worries you. - * - * We also should not forget that WAL segment can be compressed. - */ - if (IsXLogFileName(arcde->d_name) || - IsPartialXLogFileName(arcde->d_name) || - IsBackupHistoryFileName(arcde->d_name) || - IsCompressedXLogFileName(arcde->d_name)) - { - if (XLogRecPtrIsInvalid(oldest_lsn) || - strncmp(arcde->d_name + 8, oldestSegmentNeeded + 8, 16) < 0) - { - /* - * Use the original file name again now, including any - * extension that might have been chopped off before testing - * the sequence. - */ - snprintf(wal_file, MAXPGPATH, "%s/%s", - arclog_path, arcde->d_name); - - rc = unlink(wal_file); - if (rc != 0) - { - elog(WARNING, "could not remove file \"%s\": %s", - wal_file, strerror(errno)); - break; - } - elog(LOG, "removed WAL segment \"%s\"", wal_file); - - if (max_wal_file[0] == '\0' || - strcmp(max_wal_file + 8, arcde->d_name + 8) < 0) - strcpy(max_wal_file, arcde->d_name); - - if (min_wal_file[0] == '\0' || - strcmp(min_wal_file + 8, arcde->d_name + 8) > 0) - strcpy(min_wal_file, arcde->d_name); - } - } - } - - if (min_wal_file[0] != '\0') - elog(INFO, "removed min WAL segment \"%s\"", min_wal_file); - if (max_wal_file[0] != '\0') - elog(INFO, "removed max WAL segment \"%s\"", max_wal_file); - - if (errno) - elog(WARNING, "could not read archive location \"%s\": %s", - arclog_path, strerror(errno)); - if (closedir(arcdir)) - elog(WARNING, "could not close archive location \"%s\": %s", - arclog_path, strerror(errno)); - } - else - elog(WARNING, "could not open archive location \"%s\": %s", - arclog_path, strerror(errno)); -} - /* Delete all backup files and wal files of given instance. */ int From c9d3c5845adb526fa8cfe9b84c153199dc4072cd Mon Sep 17 00:00:00 2001 From: Anastasia Date: Fri, 20 Sep 2019 16:07:51 +0300 Subject: [PATCH 0968/2107] Fix archive_catalog problem exposed by b1975e3ba. Now it's legal to find in archive two files with equal segno. This can happen if both compressed and non-compessed versions are present. --- src/catalog.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 52e2b759c..2baf49623 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -791,8 +791,15 @@ catalog_get_timelines(InstanceConfig *instance) /* check, if segments are consequent */ XLogSegNo expected_segno = tlinfo->end_segno + 1; - /* some segments are missing. remember them in lost_segments to report */ - if (segno != expected_segno) + /* + * Some segments are missing. remember them in lost_segments to report. + * Normally we expect that segment numbers form an increasing sequence, + * though it's legal to find two files with equal segno in case there + * are both compressed and non-compessed versions. For example + * 000000010000000000000002 and 000000010000000000000002.gz + * + */ + if (segno != expected_segno && segno != tlinfo->end_segno) { xlogInterval *interval = palloc(sizeof(xlogInterval));; interval->begin_segno = expected_segno; From 0e4d9fcec30f010152762298d027dcc02aa1efef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 21 Sep 2019 14:02:46 +0300 Subject: [PATCH 0969/2107] help: add 'Retenion options' to delete section --- src/help.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/help.c b/src/help.c index ffb35a9ed..b1136878f 100644 --- a/src/help.c +++ b/src/help.c @@ -170,7 +170,10 @@ help_pg_probackup(void) printf(_(" [--help]\n")); printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [--wal] [-i backup-id | --expired | --merge-expired]\n")); + printf(_(" [-j num-threads] [--progress]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--delete-wal] [-i backup-id | --delete-expired | --merge-expired]\n")); printf(_(" [--dry-run]\n")); printf(_(" [--help]\n")); @@ -300,7 +303,7 @@ help_backup(void) printf(_(" retention policy after successful backup completion\n")); printf(_(" --merge-expired merge backups expired according to current\n")); printf(_(" retention policy after successful backup completion\n")); - printf(_(" --delete-wal remove redundant archived wal files\n")); + printf(_(" --delete-wal remove redundant wal files in WAL archive\n")); printf(_(" --retention-redundancy=retention-redundancy\n")); printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); @@ -556,18 +559,27 @@ static void help_delete(void) { printf(_("\n%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-i backup-id | --expired | --merge-expired] [--wal]\n")); - printf(_(" [-j num-threads] [--dry-run]\n\n")); + printf(_(" [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n")); + printf(_(" [-j num-threads] [--progress]\n")); + printf(_(" [--retention-redundancy=retention-redundancy]\n")); + printf(_(" [--retention-window=retention-window]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to delete\n")); - printf(_(" --expired delete backups expired according to current\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n")); + + printf(_("\n Retention options:\n")); + printf(_(" --delete-expired delete backups expired according to current\n")); printf(_(" retention policy\n")); printf(_(" --merge-expired merge backups expired according to current\n")); printf(_(" retention policy\n")); - printf(_(" --wal remove unnecessary wal files in WAL ARCHIVE\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --delete-wal remove redundant wal files in WAL archive\n")); + printf(_(" --retention-redundancy=retention-redundancy\n")); + printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); + printf(_(" --retention-window=retention-window\n")); + printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Logging options:\n")); From 8f9861dfdf8837b846f5a7ab0b4f43332f40aba0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 21 Sep 2019 14:50:14 +0300 Subject: [PATCH 0970/2107] Documentation: update 'Configuring Backup Retention Policy' section --- Documentation.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 41ef446bf..9e248424a 100644 --- a/Documentation.md +++ b/Documentation.md @@ -102,11 +102,11 @@ Current version - 2.1.5 `pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` -`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired }` +`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired } [option...]` `pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` -`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name` +`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` ## Versioning @@ -1070,7 +1070,9 @@ Specifies **the number of full backup copies** to keep in the backup catalog. --retention-window=window Defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in **the number of days** from the current moment. For example, if `retention-window=7`, pg_probackup must delete all backup copies that are older than seven days, with all the corresponding WAL files. -If both `--retention-redundancy` and `--retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `--retention-redundancy=2` and `--retention-window=7`, pg_probackup purges the backup catalog to keep only two full backup copies and all backups that are newer than seven days. +If both `--retention-redundancy` and `--retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `--retention-redundancy=2` and `--retention-window=7`, pg_probackup purges the backup catalog to keep only two full backup copies and all backups that are newer than seven days: + + pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 To clean up the backup catalog in accordance with retention policy, run: @@ -1084,6 +1086,10 @@ If you would like to also remove the WAL files that are no longer required for a >NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. +You can set or override the current retention policy by specifying `--retention-redundancy` and `--retention-window` options directly when running `delete` or `backup` commands: + + pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 + Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` flag when running [backup](#backup) or [delete](#delete) commands. Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `--retention-window` option is set to *7*, and you have the following backups available on April 10, 2019: @@ -1380,6 +1386,7 @@ For details, see the section [Merging Backups](#merging-backups). pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] + [--retention-redundancy=redundancy][--retention-window=window] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} [--dry-run] [logging_options] From af50ddfb39ea6f3859dcd921bf20238ac9e84086 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 21 Sep 2019 14:54:41 +0300 Subject: [PATCH 0971/2107] Readme: minor update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 129e8c197..156abe5ee 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup). Slightly outdated documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). +Documentation for current devel can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) ## License From 1f675931752722a2766a5b2e6ac512d9f40514d8 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 23 Sep 2019 13:14:57 +0300 Subject: [PATCH 0972/2107] code cleanup for wal_delete refactoring --- src/catalog.c | 127 ++++++++++++++++++++++---------------------------- src/delete.c | 90 +++++++++++++++++++---------------- 2 files changed, 106 insertions(+), 111 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 2baf49623..e7d4eeb02 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -19,8 +19,8 @@ #include "utils/file.h" #include "utils/configuration.h" -static pgBackup* get_closest_backup(timelineInfo *tlinfo, parray *backup_list); -static pgBackup* get_oldest_backup(timelineInfo *tlinfo, parray *backup_list); +static pgBackup* get_closest_backup(timelineInfo *tlinfo); +static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); @@ -891,75 +891,57 @@ catalog_get_timelines(InstanceConfig *instance) { timelineInfo *tlinfo = parray_get(timelineinfos, i); - tlinfo->oldest_backup = get_oldest_backup(tlinfo, backups); - tlinfo->closest_backup = get_closest_backup(tlinfo, backups); + tlinfo->oldest_backup = get_oldest_backup(tlinfo); + tlinfo->closest_backup = get_closest_backup(tlinfo); } - //parray_walk(xlog_files_list, pfree); - //parray_free(xlog_files_list); - return timelineinfos; } /* - * Iterate over parent timelines of a given timeline and look - * for valid backup closest to given timeline switchpoint. + * Iterate over parent timelines and look for valid backup + * closest to given timeline switchpoint. * - * Returns NULL if such backup is not found. + * If such backup doesn't exist, it means that + * timeline is unreachable. Return NULL. */ pgBackup* -get_closest_backup(timelineInfo *tlinfo, parray *backup_list) +get_closest_backup(timelineInfo *tlinfo) { pgBackup *closest_backup = NULL; int i; - /* Only timeline with switchpoint can possibly have closest backup */ - if (XLogRecPtrIsInvalid(tlinfo->switchpoint)) - return NULL; - - /* Only timeline with parent timeline can possibly have closest backup */ - if (!tlinfo->parent_link) - return NULL; - - while (tlinfo->parent_link) + /* + * Iterate over backups belonging to parent timelines + * and look for candidates. + */ + while (tlinfo->parent_link && !closest_backup) { - /* - * Iterate over backups belonging to parent timeline and look - * for candidates. - */ - for (i = 0; i < parray_num(backup_list); i++) + parray *backup_list = tlinfo->parent_link->backups; + if (backup_list != NULL) { - pgBackup *backup = parray_get(backup_list, i); - - /* Backups belonging to timelines other than parent timeline can be safely skipped */ - if (backup->tli != tlinfo->parent_tli) - continue; - - /* Backups in future can be safely skipped */ - if (backup->stop_lsn > tlinfo->switchpoint) - continue; - - /* Backups with invalid STOP LSN can be safely skipped */ - if (XLogRecPtrIsInvalid(backup->stop_lsn) || - !XRecOffIsValid(backup->stop_lsn)) - continue; - - /* Only valid backups closest to switchpoint should be considered */ - if (backup->stop_lsn <= tlinfo->switchpoint && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) + for (i = 0; i < parray_num(backup_list); i++) { - /* Check if backup is closer to switchpoint than current candidate */ - if (!closest_backup || backup->stop_lsn > closest_backup->stop_lsn) - closest_backup = backup; + pgBackup *backup = parray_get(backup_list, i); + + /* + * Only valid backups made before switchpoint + * should be considered. + */ + if (!XLogRecPtrIsInvalid(backup->stop_lsn) && + !XRecOffIsValid(backup->stop_lsn) && + backup->stop_lsn <= tlinfo->switchpoint && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) + { + /* Check if backup is closer to switchpoint than current candidate */ + if (!closest_backup || backup->stop_lsn > closest_backup->stop_lsn) + closest_backup = backup; + } } } - /* Closest backup is found */ - if (closest_backup) - break; - - /* Switch to parent */ + /* Continue with parent */ tlinfo = tlinfo->parent_link; } @@ -967,35 +949,38 @@ get_closest_backup(timelineInfo *tlinfo, parray *backup_list) } /* - * Iterate over timelines and look for oldest backup on each timeline + * Find oldest backup in given timeline + * to determine what WAL segments belonging to this timeline, + * are not reachable from any backup. + * * Returns NULL if such backup is not found. */ pgBackup* -get_oldest_backup(timelineInfo *tlinfo, parray *backup_list) +get_oldest_backup(timelineInfo *tlinfo) { pgBackup *oldest_backup = NULL; int i; + parray *backup_list = tlinfo->backups; - /* - * Iterate over backups belonging to timeline and look - * for candidates. - */ - for (i = 0; i < parray_num(backup_list); i++) + if (backup_list != NULL) { - pgBackup *backup = parray_get(backup_list, i); - - /* Backups belonging to other timelines can be safely skipped */ - if (backup->tli != tlinfo->tli) - continue; + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = parray_get(backup_list, i); - /* Backups with invalid START LSN can be safely skipped */ - if (XLogRecPtrIsInvalid(backup->start_lsn) || - !XRecOffIsValid(backup->start_lsn)) - continue; + /* Backups with invalid START LSN can be safely skipped */ + if (XLogRecPtrIsInvalid(backup->start_lsn) || + !XRecOffIsValid(backup->start_lsn)) + continue; - /* Check if backup is older than current candidate */ - if (!oldest_backup || backup->start_lsn < oldest_backup->start_lsn) - oldest_backup = backup; + /* + * Check if backup is older than current candidate. + * Here we use start_lsn for comparison, because backup that + * started earlier needs more WAL. + */ + if (!oldest_backup || backup->start_lsn < oldest_backup->start_lsn) + oldest_backup = backup; + } } return oldest_backup; diff --git a/src/delete.c b/src/delete.c index 373f87e1a..9cb14b484 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,7 +14,7 @@ #include #include -static void delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tli, +static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); @@ -607,10 +607,11 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) } } -/* Purge WAL +/* + * Purge WAL * Iterate over timelines - * Look for closest_backup, if exists, goto next timelime - * if not exists, look for oldest backup on timeline + * Look for WAL segment not reachable from existing backups + * and delete them. */ static void do_retention_wal(bool dry_run) @@ -624,25 +625,32 @@ do_retention_wal(bool dry_run) { timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - /* Empty timeline can be safely skipped */ - if (tlinfo->n_xlog_files == 0 && - parray_num(tlinfo->xlog_filelist) == 0) + /* + * Empty timeline (only mentioned in timeline history file) + * has nothing to cleanup. + */ + if (tlinfo->n_xlog_files == 0 && parray_num(tlinfo->xlog_filelist) == 0) continue; - /* If closest backup is exists, then timeline can be safely skipped */ + /* + * If closest backup exists, then timeline is reachable from + * at least one backup and none files should not be removed. + */ if (tlinfo->closest_backup) continue; /* * Purge all WAL segments before START LSN of oldest backup. - * If there is no backups on timeline, then whole timeline + * If timeline doesn't have a backup, then whole timeline * can be safely purged. + * Note, that oldest_backup is not necessarily valid here, + * but still we keep wal for it. */ if (tlinfo->oldest_backup) - delete_walfiles(tlinfo->oldest_backup->start_lsn, + delete_walfiles_in_tli(tlinfo->oldest_backup->start_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else - delete_walfiles(InvalidXLogRecPtr, + delete_walfiles_in_tli(InvalidXLogRecPtr, tlinfo, instance_config.xlog_seg_size, dry_run); } } @@ -710,7 +718,8 @@ delete_backup_files(pgBackup *backup) return; } -/* Purge WAL archive. +/* + * Purge WAL archive. One timeline at a time. * If 'keep_lsn' is InvalidXLogRecPtr, then whole timeline can be purged * If 'keep_lsn' is valid LSN, then every lesser segment can be purged. * If 'dry_run' is set, then don`t actually delete anything. @@ -720,7 +729,8 @@ delete_backup_files(pgBackup *backup) * Case 2: * archive is not empty, 'keep_lsn' is valid and prevening us from deleting anything. * Case 3: - * archive is not empty, 'keep_lsn' is invalid, drop everyhing in archive. + * archive is not empty, 'keep_lsn' is invalid, drop all WAL files in archive, + * belonging to the timeline. * Case 4: * archive is empty, 'keep_lsn' is valid, assume corruption of WAL archive. * Case 5: @@ -730,11 +740,11 @@ delete_backup_files(pgBackup *backup) * Q: Maybe we should stop treating partial WAL segments as second-class citizens? */ static void -delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, +delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, uint32 xlog_seg_size, bool dry_run) { - XLogSegNo StartSegNo; /* First segment to delete */ - XLogSegNo EndSegNo = 0; /* Oldest segment to keep */ + XLogSegNo FirstToDeleteSegNo; + XLogSegNo OldestToKeepSegNo = 0; int rc; int i; int wal_size_logical = 0; @@ -752,32 +762,32 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (XLogRecPtrIsInvalid(keep_lsn)) { /* Drop all files in timeline */ - elog(INFO, "All files on timeline %i will be removed", tlinfo->tli); - StartSegNo = tlinfo->begin_segno; - EndSegNo = tlinfo->end_segno; + elog(INFO, "All files on timeline %i %s be removed", tlinfo->tli, + dry_run?"can":"will"); + FirstToDeleteSegNo = tlinfo->begin_segno; + OldestToKeepSegNo = tlinfo->end_segno; purge_all = true; } else { /* Drop all segments between begin_segno and segment with keep_lsn (excluding) */ - StartSegNo = tlinfo->begin_segno; - GetXLogSegNo(keep_lsn, EndSegNo, xlog_seg_size); + FirstToDeleteSegNo = tlinfo->begin_segno; + GetXLogSegNo(keep_lsn, OldestToKeepSegNo, xlog_seg_size); } - if (EndSegNo > 0 && EndSegNo > StartSegNo) - elog(INFO, "WAL segments between %08X%08X and %08X%08X on timeline %i will be removed", - (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, - (uint32) (EndSegNo - 1) / xlog_seg_size, - (uint32) (EndSegNo - 1) % xlog_seg_size, - tlinfo->tli); + if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo) + elog(INFO, "WAL segments between %08X%08X and %08X%08X on timeline %i %s be removed", + (uint32) FirstToDeleteSegNo / xlog_seg_size, (uint32) FirstToDeleteSegNo % xlog_seg_size, + (uint32) (OldestToKeepSegNo - 1) / xlog_seg_size, + (uint32) (OldestToKeepSegNo - 1) % xlog_seg_size, + tlinfo->tli, dry_run?"can":"will"); /* sanity */ - if (EndSegNo > StartSegNo) - /* typical scenario */ - wal_size_logical = (EndSegNo-StartSegNo) * xlog_seg_size; - else if (EndSegNo < StartSegNo) + if (OldestToKeepSegNo > FirstToDeleteSegNo) + wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; + else if (OldestToKeepSegNo < FirstToDeleteSegNo) { - /* It is actually possible for EndSegNo to be less than StartSegNo + /* It is actually possible for OldestToKeepSegNo to be less than FirstToDeleteSegNo * in case of : * 1. WAL archive corruption. * 2. There is no actual WAL archive to speak of and @@ -785,14 +795,14 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, * * Assume the worst. */ - if (StartSegNo > 0 && EndSegNo > 0) + if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0) elog(WARNING, "On timeline %i first segment %08X%08X is greater than " "oldest segment to keep %08X%08X. Possible WAL archive corruption!", tlinfo->tli, - (uint32) StartSegNo / xlog_seg_size, (uint32) StartSegNo % xlog_seg_size, - (uint32) EndSegNo / xlog_seg_size, (uint32) EndSegNo % xlog_seg_size); + (uint32) FirstToDeleteSegNo / xlog_seg_size, (uint32) FirstToDeleteSegNo % xlog_seg_size, + (uint32) OldestToKeepSegNo / xlog_seg_size, (uint32) OldestToKeepSegNo % xlog_seg_size); } - else if (EndSegNo == StartSegNo && !purge_all) + else if (OldestToKeepSegNo == FirstToDeleteSegNo && !purge_all) { /* 'Nothing to delete' scenario because of 'keep_lsn' * with possible exception of partial and backup history files. @@ -804,7 +814,7 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (wal_size_logical > 0) { pretty_size(wal_size_logical, wal_pretty_size, lengthof(wal_pretty_size)); - elog(INFO, "WAL size to remove on timeline %i: %s", + elog(INFO, "Logical WAL size to remove on timeline %i : %s", tlinfo->tli, wal_pretty_size); } @@ -813,7 +823,7 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); - if (purge_all || wal_file->segno < EndSegNo) + if (purge_all || wal_file->segno < OldestToKeepSegNo) wal_size_actual += wal_file->file.size; } @@ -821,7 +831,7 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (wal_size_actual > 0) { pretty_size(wal_size_actual, wal_pretty_size, lengthof(wal_pretty_size)); - elog(INFO, "Resident data size to free on timeline %i: %s", + elog(INFO, "Resident data size to free on timeline %i : %s", tlinfo->tli, wal_pretty_size); } @@ -838,7 +848,7 @@ delete_walfiles(XLogRecPtr keep_lsn, timelineInfo *tlinfo, /* Any segment equal or greater than EndSegNo must be kept * unless it`s a 'purge all' scenario. */ - if (purge_all || wal_file->segno < EndSegNo) + if (purge_all || wal_file->segno < OldestToKeepSegNo) { /* unlink segment */ rc = unlink(wal_file->file.path); From 1d34fd8c84e590ab4f243671ffdfce7cbe13e5a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 23 Sep 2019 13:57:10 +0300 Subject: [PATCH 0973/2107] minor fixes --- src/catalog.c | 9 +++++---- src/delete.c | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index e7d4eeb02..a902621fc 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -929,7 +929,7 @@ get_closest_backup(timelineInfo *tlinfo) * should be considered. */ if (!XLogRecPtrIsInvalid(backup->stop_lsn) && - !XRecOffIsValid(backup->stop_lsn) && + XRecOffIsValid(backup->stop_lsn) && backup->stop_lsn <= tlinfo->switchpoint && (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE)) @@ -950,10 +950,11 @@ get_closest_backup(timelineInfo *tlinfo) /* * Find oldest backup in given timeline - * to determine what WAL segments belonging to this timeline, - * are not reachable from any backup. + * to determine what WAL segments of this timeline + * are reachable from backups belonging to it. * - * Returns NULL if such backup is not found. + * If such backup doesn't exist, it means that + * there is no backups on this timeline. Return NULL. */ pgBackup* get_oldest_backup(timelineInfo *tlinfo) diff --git a/src/delete.c b/src/delete.c index 9cb14b484..325d321f5 100644 --- a/src/delete.c +++ b/src/delete.c @@ -634,7 +634,7 @@ do_retention_wal(bool dry_run) /* * If closest backup exists, then timeline is reachable from - * at least one backup and none files should not be removed. + * at least one backup and no file should be removed. */ if (tlinfo->closest_backup) continue; From 64d546780e8ade57068ddf778e112f39acf86d6b Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 23 Sep 2019 14:00:24 +0300 Subject: [PATCH 0974/2107] [Issue #58] Handle SIGTERM and SIGQUIT --- src/utils/pgut.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 92a8028a2..cdc90e30f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -674,7 +674,7 @@ on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); old = thread_cancel_conn; - /* be sure handle_sigint doesn't use pointer while freeing */ + /* be sure handle_interrupt doesn't use pointer while freeing */ thread_cancel_conn = NULL; if (old != NULL) @@ -687,7 +687,7 @@ on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) /* Free the old one if we have one */ old = cancel_conn; - /* be sure handle_sigint doesn't use pointer while freeing */ + /* be sure handle_interrupt doesn't use pointer while freeing */ cancel_conn = NULL; if (old != NULL) @@ -723,7 +723,7 @@ on_after_exec(PGcancel *thread_cancel_conn) //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); old = thread_cancel_conn; - /* be sure handle_sigint doesn't use pointer while freeing */ + /* be sure handle_interrupt doesn't use pointer while freeing */ thread_cancel_conn = NULL; if (old != NULL) @@ -733,7 +733,7 @@ on_after_exec(PGcancel *thread_cancel_conn) { old = cancel_conn; - /* be sure handle_sigint doesn't use pointer while freeing */ + /* be sure handle_interrupt doesn't use pointer while freeing */ cancel_conn = NULL; if (old != NULL) @@ -937,15 +937,18 @@ wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) #ifndef WIN32 static void -handle_sigint(SIGNAL_ARGS) +handle_interrupt(SIGNAL_ARGS) { on_interrupt(); } +/* Handle various inrerruptions in the same way */ static void init_cancel_handler(void) { - oldhandler = pqsignal(SIGINT, handle_sigint); + oldhandler = pqsignal(SIGINT, handle_interrupt); + pqsignal(SIGQUIT, handle_interrupt); + pqsignal(SIGTERM, handle_interrupt); } #else /* WIN32 */ From 7121f04e7130b8de86f5e3f10c23d434da59a82c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 23 Sep 2019 15:20:07 +0300 Subject: [PATCH 0975/2107] [Issue #102] New options for 'restore' command used for setting 'restore_command' GUC in recovery.conf: '--archive-host', '--archive-post', 'archive-user' and '--restore-command' --- Documentation.md | 51 ++++++++++++++++++++++++++++---- src/configure.c | 51 ++++++++++++++++++++------------ src/help.c | 34 ++++++++++++++++----- src/pg_probackup.c | 5 +++- src/pg_probackup.h | 12 ++++++++ src/restore.c | 37 +++++++++++++++++++++-- tests/archive.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 33 deletions(-) diff --git a/Documentation.md b/Documentation.md index 9e248424a..957774dee 100644 --- a/Documentation.md +++ b/Documentation.md @@ -68,6 +68,7 @@ Current version - 2.1.5 * [Compression Options](#compression-options) * [Archiving Options](#archiving-options) * [Remote Mode Options](#remote-mode-options) + * [Remote WAL Archive Options](#remote-wal-archive-options) * [Replica Options](#replica-options) 7. [Authors](#authors) @@ -546,6 +547,8 @@ Once the restore command is complete, start the database service. If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. +To restore cluster on remote host see the section [Using pg_probackup in the Remote Mode](#using-pg-probackup-in-the-remote-mode). + >NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` flag to skip validation and speed up the recovery. #### Partial Restore @@ -624,6 +627,24 @@ For example, to restore latest backup on remote system using remote mode through pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 +Restoring of ARCHIVE backup or performing PITR in remote mode require additional information: destination address, port and username for establishing ssh connection **from** a host with database **to** a host with backup catalog. This information will be used by `restore_command` to copy via ssh WAL segments from archive to PostgreSQL 'pg_wal' directory. + +To solve this problem you can use [Remote Wal Archive Options](#remote-wal-archive-options). + +For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302` and user `backup` on backup catalog host with address `192.168.0.3` via port `2303`, run: + + pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup + +Provided arguments will be used to construct 'restore_command' in recovery.conf: +``` +# recovery.conf generated by pg_probackup 2.1.5 +restore_command = 'pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +``` + +Alternatively you can use `--restore-command` option to provide an entire 'restore_command': + + pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' + >NOTE: The remote backup mode is currently unavailable for Windows systems. ### Running pg_probackup on Parallel Threads @@ -1054,7 +1075,7 @@ The sample output is as follows: Most fields are consistent with plain format, with some exceptions: - size is in bytes. -- 'closest-backup-id' attribute contain ID of valid backup closest to the timeline, located on some of the previous timelines. This backup is the closest starting point to reach the timeline from other timelines by PITR. If such backup do not exists, then string is empty. +- 'closest-backup-id' attribute contain ID of valid backup closest to the timeline, located on some of the previous timelines. This backup is the closest starting point to reach the timeline from other timelines by PITR. Closest backup always has a valid status, either OK or DONE. If such backup do not exists, then string is empty. - DEGRADED timelines contain 'lost-segments' array with information about intervals of missing segments. In OK timelines 'lost-segments' array is empty. - 'N backups' attribute is replaced with 'backups' array containing backups belonging to the timeline. If timeline has no backups, then 'backups' array is empty. @@ -1214,7 +1235,8 @@ Deletes all backups and WAL files associated with the specified instance. [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] [-d dbname] [-h host] [-p port] [-U username] [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [remote_options] [logging_options] + [--restore-command=cmdline] + [remote_options] [remote_archive_options] [logging_options] Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. @@ -1308,8 +1330,9 @@ For details on usage, see the section [Creating a Backup](#creating-a-backup). [-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + [--restore-command=cmdline] [recovery_options] [logging_options] [remote_options] - [partial_restore_options] + [partial_restore_options] [remote_archive_options] Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a [recovery target option](#recovery-target-options), pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. @@ -1333,7 +1356,10 @@ Disables block-level checksum verification to speed up validation. During automa --no-validate Skips backup validation. You can use this flag if you validate backups regularly and would like to save time when running restore operations. -Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. + --restore-command=cmdline +Set the [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) parameter to specified command. Example: `--restore-command='cp /mnt/server/archivedir/%f "%p"'` + +Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Remote WAL Archive Options](#remote-wal-archive-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). @@ -1656,9 +1682,24 @@ Specifies pg_probackup installation directory on the remote system. --ssh-options=ssh_options Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found on [ssh_config manual page](https://man.openbsd.org/ssh_config.5). +#### Remote WAL Archive Options + +This section describes the options used to provide the arguments for [Remote Mode Options](#remote-mode-options) in [archive-get](#archive-get) used in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) command when restoring ARCHIVE backup or performing PITR. + + --archive-host=destination +Provides the argument for `--remote-host` option in `archive-get` command. + + --archive-port=port + Default: 22 +Provides the argument for `--remote-port` option in `archive-get` command. + + --archive-user=username + Default: PostgreSQL user +Provides the argument for `--remote-user` option in `archive-get` command. If you omit this option, the the user running PostgreSQL cluster is used. + #### Partial Restore Options -This section describes the options related to partial restore of cluster from backup. These options can be used with [restore](#restore) command. +This section describes the options related to partial restore of a cluster from backup. These options can be used with [restore](#restore) command. --db-exclude=dbname Specifies database name to exclude from restore. All other databases in the cluster will be restored as usual, including `template0` and `template1`. This option can be specified multiple times for multiple databases. diff --git a/src/configure.c b/src/configure.c index cbe97e285..e3ab99ba3 100644 --- a/src/configure.c +++ b/src/configure.c @@ -122,97 +122,112 @@ ConfigOption instance_options[] = &instance_config.archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value }, + { + 's', 208, "archive-host", + &instance_config.archive.host, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + { + 's', 209, "archive-port", + &instance_config.archive.port, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + { + 's', 210, "archive-user", + &instance_config.archive.user, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, /* Logging options */ { - 'f', 208, "log-level-console", + 'f', 211, "log-level-console", assign_log_level_console, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, get_log_level_console }, { - 'f', 209, "log-level-file", + 'f', 212, "log-level-file", assign_log_level_file, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, get_log_level_file }, { - 's', 210, "log-filename", + 's', 213, "log-filename", &instance_config.logger.log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 211, "error-log-filename", + 's', 214, "error-log-filename", &instance_config.logger.error_log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 212, "log-directory", + 's', 215, "log-directory", &instance_config.logger.log_directory, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 213, "log-rotation-size", + 'U', 216, "log-rotation-size", &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 214, "log-rotation-age", + 'U', 217, "log-rotation-age", &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 215, "retention-redundancy", + 'u', 218, "retention-redundancy", &instance_config.retention_redundancy, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 216, "retention-window", + 'u', 219, "retention-window", &instance_config.retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 'f', 217, "compress-algorithm", + 'f', 220, "compress-algorithm", assign_compress_alg, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { - 'u', 218, "compress-level", + 'u', 221, "compress-level", &instance_config.compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 219, "remote-proto", + 's', 222, "remote-proto", &instance_config.remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 220, "remote-host", + 's', 223, "remote-host", &instance_config.remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 221, "remote-port", + 's', 224, "remote-port", &instance_config.remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 222, "remote-path", + 's', 225, "remote-path", &instance_config.remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 223, "remote-user", + 's', 226, "remote-user", &instance_config.remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "ssh-options", + 's', 227, "ssh-options", &instance_config.remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "ssh-config", + 's', 228, "ssh-config", &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, diff --git a/src/help.c b/src/help.c index b1136878f..6e7733475 100644 --- a/src/help.c +++ b/src/help.c @@ -97,6 +97,8 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--restore-command=cmdline] [--archive-host=destination]\n")); + printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -142,11 +144,13 @@ help_pg_probackup(void) printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--archive-host=hostname]\n")); + printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); @@ -331,7 +335,7 @@ help_backup(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-host=destination remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); printf(_(" (default: current binary path)\n")); @@ -363,10 +367,13 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--restore-command=cmdline]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--archive-host=hostname] [--archive-port=port]\n")); + printf(_(" [--archive-user=username]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -401,6 +408,7 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); @@ -430,13 +438,18 @@ help_restore(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-host=destination remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); printf(_(" (default: current binary path)\n")); printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); + + printf(_("\n Remote WAL archive options:\n")); + printf(_(" --archive-host=destination address or hostname for ssh connection to archive host\n")); + printf(_(" --archive-port=port port for ssh connection to archive host (default: 22)\n")); + printf(_(" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n\n")); } static void @@ -652,6 +665,7 @@ help_set_config(void) printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path]\n")); printf(_(" [-E external-directories-paths]\n")); + printf(_(" [--restore-command=cmdline]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -675,6 +689,7 @@ help_set_config(void) printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); + printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -722,7 +737,7 @@ help_set_config(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-host=destination remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); printf(_(" (default: current binary path)\n")); @@ -730,6 +745,11 @@ help_set_config(void) printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); + printf(_("\n Remote WAL archive options:\n")); + printf(_(" --archive-host=destination address or hostname for ssh connection to archive host\n")); + printf(_(" --archive-port=port port for ssh connection to archive host (default: 22)\n")); + printf(_(" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n")); + printf(_("\n Replica options:\n")); printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); @@ -769,7 +789,7 @@ help_add_instance(void) printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-host=destination remote host address or hostname\n")); printf(_(" --remote-port=port remote host port (default: 22)\n")); printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); printf(_(" (default: current binary path)\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0e1c07e99..72659048e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -87,6 +87,7 @@ static char *target_stop; static bool target_immediate; static char *target_name = NULL; static char *target_action = NULL; +static char *restore_command = NULL; static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; @@ -182,6 +183,7 @@ static ConfigOption cmd_options[] = { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, + { 's', 160, "restore-command", &restore_command, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -204,7 +206,7 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, - { 'b', 160, "archive", &show_archive, SOURCE_CMD_STRICT }, + { 'b', 161, "archive", &show_archive, SOURCE_CMD_STRICT }, /* options for backward compatibility */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, @@ -641,6 +643,7 @@ main(int argc, char *argv[]) restore_params->skip_external_dirs = skip_external_dirs; restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; + restore_params->restore_command = restore_command; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 38c32134c..93c1fa3e5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -220,6 +220,14 @@ typedef struct ConnectionArgs PGcancel *cancel_conn; } ConnectionArgs; +/* Store values for --remote-* option for 'restore_command' constructor */ +typedef struct ArchiveOptions +{ + const char *host; + const char *port; + const char *user; +} ArchiveOptions; + /* * An instance configuration. It can be stored in a configuration file or passed * from command line. @@ -256,6 +264,9 @@ typedef struct InstanceConfig CompressAlg compress_alg; int compress_level; + + /* Archive description */ + ArchiveOptions archive; } InstanceConfig; extern ConfigOption instance_options[]; @@ -357,6 +368,7 @@ typedef struct pgRestoreParams bool restore_as_replica; bool skip_external_dirs; bool skip_block_validation; + const char *restore_command; /* options for partial restore */ PartialRestoreType partial_restore_type; diff --git a/src/restore.c b/src/restore.c index fb9018880..cea4fb67a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -860,11 +860,44 @@ create_recovery_conf(time_t backup_id, if (need_restore_conf) { - fio_fprintf(fp, "restore_command = '%s archive-get -B %s --instance %s " - "--wal-file-path %%p --wal-file-name %%f'\n", + char restore_command_guc[16384]; + + /* If restore_command is provided, use it */ + if (params->restore_command) + sprintf(restore_command_guc, "%s", params->restore_command); + /* construct restore_command */ + else + { + /* default cmdline, ok for local restore */ + sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " + "--wal-file-path=%%p --wal-file-name=%%f", PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, backup_path, instance_name); + /* append --remote-* parameters provided via --archive-* settings */ + if (instance_config.archive.host) + { + strcat(restore_command_guc, " --remote-host="); + strcat(restore_command_guc, instance_config.archive.host); + } + + if (instance_config.archive.port) + { + strcat(restore_command_guc, " --remote-port="); + strcat(restore_command_guc, instance_config.archive.port); + } + + if (instance_config.archive.user) + { + strcat(restore_command_guc, " --remote-user="); + strcat(restore_command_guc, instance_config.archive.user); + } + } + + elog(LOG, "Setting restore_command to '%s'", restore_command_guc); + fio_fprintf(fp, "restore_command = '%s'\n", + restore_command_guc); + /* * We've already checked that only one of the four following mutually * exclusive options is specified, so the order of calls is insignificant. diff --git a/tests/archive.py b/tests/archive.py index 6a0d49a4e..179829982 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1541,6 +1541,79 @@ def test_archive_catalog_2(self): self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_options(self): + """ + check that '--archive-host', '--archive-user', '--archiver-port' + and '--restore-command' are working as expected. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + node.cleanup() + + wal_dir = os.path.join(backup_dir, 'wal', 'node') + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command="cp {0}/%f %p"'.format(wal_dir), + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user) + ]) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), + recovery_content) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-action=promote', + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user) + ]) + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + "restore_command = '{0} archive-get -B {1} --instance {2} " + "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " + "--remote-port=22 --remote-user={3}'".format( + self.probackup_path, backup_dir, 'node', self.user), + recovery_content) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + self.del_test_dir(module_name, fname) + # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. # so write validation code accordingly From cb6431369aff6379cf48380c2aee496f9fb2a5bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 24 Sep 2019 18:01:40 +0300 Subject: [PATCH 0976/2107] add double quotes to filenames in elog messages --- src/data.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index 680f38c12..494f0b67d 100644 --- a/src/data.c +++ b/src/data.c @@ -209,13 +209,13 @@ read_page_from_file(pgFile *file, BlockNumber blknum, /* The block could have been truncated. It is fine. */ if (read_len == 0) { - elog(VERBOSE, "File %s, block %u, file was truncated", + elog(VERBOSE, "File \"%s\", block %u, file was truncated", file->path, blknum); return 0; } else { - elog(WARNING, "File: %s, block %u, expected block size %u," + elog(WARNING, "File: \"%s\", block %u, expected block size %u," "but read %zu, try again", file->path, blknum, BLCKSZ, read_len); return -1; @@ -238,7 +238,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, /* Page is zeroed. No need to check header and checksum. */ if (i == BLCKSZ) { - elog(VERBOSE, "File: %s blknum %u, empty page", file->path, blknum); + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", file->path, blknum); return 1; } @@ -246,7 +246,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * If page is not completely empty and we couldn't parse it, * try again several times. If it didn't help, throw error */ - elog(LOG, "File: %s blknum %u have wrong page header, try again", + elog(LOG, "File: \"%s\" blknum %u have wrong page header, try again", file->path, blknum); return -1; } @@ -261,7 +261,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, */ if (pg_checksum_page(page, blkno) != ((PageHeader) page)->pd_checksum) { - elog(LOG, "File: %s blknum %u have wrong checksum, try again", + elog(LOG, "File: \"%s\" blknum %u have wrong checksum, try again", file->path, blknum); return -1; } @@ -342,7 +342,7 @@ prepare_page(ConnectionArgs *arguments, if (result == -1 && is_ptrack_support && strict) { - elog(WARNING, "File %s, block %u, try to fetch via SQL", + elog(WARNING, "File \"%s\", block %u, try to fetch via SQL", file->path, blknum); break; } @@ -389,7 +389,7 @@ prepare_page(ConnectionArgs *arguments, else if (page_size != BLCKSZ) { free(ptrack_page); - elog(ERROR, "File: %s, block %u, expected block size %d, but read %zu", + elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", file->path, absolute_blknum, BLCKSZ, page_size); } else @@ -422,7 +422,7 @@ prepare_page(ConnectionArgs *arguments, page_lsn && page_lsn < prev_backup_start_lsn) { - elog(VERBOSE, "Skipping blknum %u in file: %s", blknum, file->path); + elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, file->path); (*n_skipped)++; return SkipCurrentPage; } @@ -505,7 +505,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, fclose(in); fio_fclose(out); - elog(ERROR, "File: %s, cannot write backup at block %u: %s", + elog(ERROR, "File: \"%s\", cannot write backup at block %u: %s", file->path, blknum, strerror(errno_tmp)); } @@ -549,7 +549,7 @@ backup_data_file(backup_files_arg* arguments, * There are no changed blocks since last backup. We want make * incremental backup, so we should exit. */ - elog(VERBOSE, "Skipping the unchanged file: %s", file->path); + elog(VERBOSE, "Skipping the unchanged file: \"%s\"", file->path); return false; } @@ -1298,7 +1298,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, pg_crc32 crc; bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - elog(VERBOSE, "Validate relation blocks for file %s", file->path); + elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path); in = fopen(file->path, PG_BINARY_R); if (in == NULL) From 6fe844004e4cf972682f010ee45a834020372e19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 24 Sep 2019 18:28:10 +0300 Subject: [PATCH 0977/2107] Improvements for plain show, honest wal_bytes for STREAM backups and merged backups --- src/backup.c | 3 +- src/catalog.c | 10 ++++- src/merge.c | 26 ++----------- src/pg_probackup.h | 4 +- src/show.c | 92 +++++++++++++++++++++++++++++++++++++++------- 5 files changed, 94 insertions(+), 41 deletions(-) diff --git a/src/backup.c b/src/backup.c index e92b07dd7..3d7d61473 100644 --- a/src/backup.c +++ b/src/backup.c @@ -553,6 +553,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0, FIO_BACKUP_HOST); + /* TODO: Drop streamed WAL segments greater than stop_lsn */ for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); @@ -702,7 +703,7 @@ do_backup(time_t start_time, bool no_validate) } elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " - "wal-method: %s, remote: %s, compress-algorithm: %s, compress-level: %i", + "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); diff --git a/src/catalog.c b/src/catalog.c index a902621fc..a0c7123aa 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1121,6 +1121,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char buf[BUFFERSZ]; size_t write_len = 0; int64 backup_size_on_disk = 0; + int64 wal_size_on_disk = 0; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); @@ -1145,7 +1146,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* Count the amount of the data actually copied */ if (S_ISREG(file->mode) && file->write_size > 0) - backup_size_on_disk += file->write_size; + { + /* TODO: in 3.0 add attribute is_walfile */ + if (IsXLogFileName(file->name) && (file->external_dir_num == 0)) + wal_size_on_disk += file->write_size; + else + backup_size_on_disk += file->write_size; + } /* for files from PGDATA and external files use rel_path * streamed WAL files has rel_path relative not to "database/" @@ -1227,6 +1234,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; + backup->wal_bytes = wal_size_on_disk; } /* diff --git a/src/merge.c b/src/merge.c index 1cdfde090..ef4a77372 100644 --- a/src/merge.c +++ b/src/merge.c @@ -345,32 +345,12 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) to_backup->merge_time = merge_time; to_backup->end_time = time(NULL); - /* - * Target backup must inherit wal mode too. - */ + /* Target backup must inherit wal mode too. */ to_backup->stream = from_backup->stream; - /* Compute summary of size of regular files in the backup */ - to_backup->data_bytes = 0; - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - if (S_ISDIR(file->mode)) - { - to_backup->data_bytes += 4096; - continue; - } - /* Count the amount of the data actually copied */ - if (file->write_size > 0) - to_backup->data_bytes += file->write_size; - } - /* compute size of wal files of this backup stored in the archive */ + /* ARCHIVE backup must inherit wal_bytes. */ if (!to_backup->stream) - to_backup->wal_bytes = instance_config.xlog_seg_size * - (to_backup->stop_lsn / instance_config.xlog_seg_size - - to_backup->start_lsn / instance_config.xlog_seg_size + 1); - else - to_backup->wal_bytes = BYTES_INVALID; + to_backup->wal_bytes = from_backup->wal_bytes; write_backup_filelist(to_backup, files, from_database_path, NULL); write_backup(to_backup); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 93c1fa3e5..3776ac06f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -314,7 +314,7 @@ struct pgBackup * BYTES_INVALID means nothing was backed up. */ int64 data_bytes; - /* Size of WAL files in archive needed to restore this backup */ + /* Size of WAL files needed to restore this backup */ int64 wal_bytes; CompressAlg compress_alg; @@ -812,7 +812,7 @@ int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); extern void pretty_size(int64 size, char *buf, size_t len); - +extern void pretty_time_interval(int64 num_seconds, char *buf, size_t len); extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); extern void check_system_identifiers(PGconn *conn, char *pgdata); diff --git a/src/show.c b/src/show.c index 807fdb624..1cef0ad50 100644 --- a/src/show.c +++ b/src/show.c @@ -30,6 +30,7 @@ typedef struct ShowBackendRow char tli[20]; char duration[20]; char data_bytes[20]; + char wal_bytes[20]; char start_lsn[20]; char stop_lsn[20]; const char *status; @@ -147,9 +148,15 @@ pretty_size(int64 size, char *buf, size_t len) int64 limit2 = limit * 2 - 1; /* minus means the size is invalid */ - if (size < 0) +// if (size < 0) +// { +// strncpy(buf, "----", len); +// return; +// } + + if (size <= 0) { - strncpy(buf, "----", len); + strncpy(buf, "0", len); return; } @@ -180,6 +187,53 @@ pretty_size(int64 size, char *buf, size_t len) } } +void +pretty_time_interval(int64 num_seconds, char *buf, size_t len) +{ + int seconds = 0; + int minutes = 0; + int hours = 0; + int days = 0; + + if (num_seconds <= 0) + { + strncpy(buf, "0", len); + return; + } + + days = num_seconds / (24 * 3600); + num_seconds %= (24 * 3600); + + hours = num_seconds / 3600; + num_seconds %= 3600; + + minutes = num_seconds / 60; + num_seconds %= 60; + + seconds = num_seconds; + + if (days > 0) + { + snprintf(buf, len, "%dd:%dh", days, hours); + return; + } + + if (hours > 0) + { + snprintf(buf, len, "%dh:%dm", hours, minutes); + return; + } + + if (minutes > 0) + { + snprintf(buf, len, "%dm:%ds", minutes, seconds); + return; + } + + snprintf(buf, len, "%ds", seconds); + return; +} + /* * Initialize instance visualization. */ @@ -381,16 +435,16 @@ show_backup(const char *instance_name, time_t requested_backup_id) static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name) { -#define SHOW_FIELDS_COUNT 12 +#define SHOW_FIELDS_COUNT 13 int i; const char *names[SHOW_FIELDS_COUNT] = { "Instance", "Version", "ID", "Recovery Time", - "Mode", "WAL", "Current/Parent TLI", "Time", "Data", + "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL", "Start LSN", "Stop LSN", "Status" }; const char *field_formats[SHOW_FIELDS_COUNT] = { " %-*s ", " %-*s ", " %-*s ", " %-*s ", - " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", - " %*s ", " %*s ", " %-*s "}; + " %-*s ", " %-*s ", " %-*s ", " %*s ", " %-*s ", " %-*s ", + " %-*s ", " %-*s ", " %-*s "}; uint32 widths[SHOW_FIELDS_COUNT]; uint32 widths_sum = 0; ShowBackendRow *rows; @@ -443,7 +497,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na widths[cur] = Max(widths[cur], strlen(row->mode)); cur++; - /* WAL */ + /* WAL mode*/ row->wal_mode = backup->stream ? "STREAM": "ARCHIVE"; widths[cur] = Max(widths[cur], strlen(row->wal_mode)); cur++; @@ -453,7 +507,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na if (backup->parent_backup_link != NULL) parent_tli = backup->parent_backup_link->tli; - snprintf(row->tli, lengthof(row->tli), "%u / %u", + snprintf(row->tli, lengthof(row->tli), "%u/%u", backup->tli, backup->backup_mode == BACKUP_MODE_FULL ? 0 : parent_tli); widths[cur] = Max(widths[cur], strlen(row->tli)); @@ -461,14 +515,14 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na /* Time */ if (backup->status == BACKUP_STATUS_RUNNING) - snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, - difftime(current_time, backup->start_time)); + pretty_time_interval(difftime(current_time, backup->start_time), + row->duration, lengthof(row->duration)); else if (backup->merge_time != (time_t) 0) - snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, - difftime(backup->end_time, backup->merge_time)); + pretty_time_interval(difftime(backup->end_time, backup->merge_time), + row->duration, lengthof(row->duration)); else if (backup->end_time != (time_t) 0) - snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0, - difftime(backup->end_time, backup->start_time)); + pretty_time_interval(difftime(backup->end_time, backup->start_time), + row->duration, lengthof(row->duration)); else StrNCpy(row->duration, "----", sizeof(row->duration)); widths[cur] = Max(widths[cur], strlen(row->duration)); @@ -480,6 +534,12 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na widths[cur] = Max(widths[cur], strlen(row->data_bytes)); cur++; + /* WAL */ + pretty_size(backup->wal_bytes, row->wal_bytes, + lengthof(row->wal_bytes)); + widths[cur] = Max(widths[cur], strlen(row->wal_bytes)); + cur++; + /* Start LSN */ snprintf(row->start_lsn, lengthof(row->start_lsn), "%X/%X", (uint32) (backup->start_lsn >> 32), @@ -566,6 +626,10 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->data_bytes); cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->wal_bytes); + cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], row->start_lsn); cur++; From 4cf86c50c0b00ee8e7af4a0ffe4666df918a4bd5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Oct 2019 17:30:55 +0300 Subject: [PATCH 0978/2107] tests: minor fix --- tests/validate.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index 7855f1ef2..21175591c 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -81,14 +81,15 @@ def test_basic_validate_nullified_heap_page_backup(self): self.backup_node( backup_dir, 'node', node, options=['--log-level-file=verbose']) - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) + pgdata = self.pgdata_content(node.data_dir) if not self.remote: log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") with open(log_file_path) as f: - self.assertTrue( - '{0} blknum 1, empty page'.format(file_path) in f.read(), + log_content = f.read() + self.assertIn( + 'File: "{0}" blknum 1, empty page'.format(file), + log_content, 'Failed to detect nullified block') self.validate_pb(backup_dir, options=["-j", "4"]) @@ -96,9 +97,8 @@ def test_basic_validate_nullified_heap_page_backup(self): self.restore_node(backup_dir, 'node', node) - if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From c2d32da0a0f82178c40429873b46e84192f22052 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Oct 2019 17:32:27 +0300 Subject: [PATCH 0979/2107] [Issue #124] report correct backup status if corruption is detected during validation. Reported By Yuri Kurenkov --- src/validate.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/validate.c b/src/validate.c index e13abcd47..1352029c0 100644 --- a/src/validate.c +++ b/src/validate.c @@ -166,6 +166,8 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) parray_free(files); /* Update backup status */ + if (corrupted) + backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK, instance_name); From 56726b25fa66a3f5859559671f6e59c9ea5f0d53 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Oct 2019 13:07:47 +0300 Subject: [PATCH 0980/2107] [Issue #128] Get timeline ID via pg_control_checkpoint() function --- Documentation.md | 2 ++ src/backup.c | 8 +++++++- src/pg_probackup.h | 3 ++- src/util.c | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index 957774dee..2842a6920 100644 --- a/Documentation.md +++ b/Documentation.md @@ -245,6 +245,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; ``` @@ -263,6 +264,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; ``` diff --git a/src/backup.c b/src/backup.c index 3d7d61473..89f153901 100644 --- a/src/backup.c +++ b/src/backup.c @@ -183,7 +183,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) } /* Obtain current timeline */ - current.tli = get_current_timeline(false); + current.tli = get_current_timeline(backup_conn); + +#if PG_VERSION_NUM >= 90600 + current.tli = get_current_timeline(backup_conn); +#else + current.tli = get_current_timeline_from_control(false); +#endif /* * In incremental backup mode ensure that already-validated diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3776ac06f..957955a41 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -788,7 +788,8 @@ extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_ TimeLineID tli, uint32 wal_seg_size); /* in util.c */ -extern TimeLineID get_current_timeline(bool safe); +extern TimeLineID get_current_timeline(PGconn *conn); +extern TimeLineID get_current_timeline_from_control(bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); extern uint64 get_system_identifier(const char *pgdata_path); extern uint64 get_remote_system_identifier(PGconn *conn); diff --git a/src/util.c b/src/util.c index 93790283d..816e27f0d 100644 --- a/src/util.c +++ b/src/util.c @@ -146,7 +146,36 @@ writeControlFile(ControlFileData *ControlFile, char *path, fio_location location * used by a node. */ TimeLineID -get_current_timeline(bool safe) +get_current_timeline(PGconn *conn) +{ + + PGresult *res; + TimeLineID tli = 0; + char *val; + + res = pgut_execute_extended(conn, + "SELECT timeline_id FROM pg_control_checkpoint()", 0, NULL, true, true); + + if (PQresultStatus(res) == PGRES_TUPLES_OK) + val = PQgetvalue(res, 0, 0); + else + return get_current_timeline_from_control(false); + + if (!parse_uint32(val, &tli, 0)) + { + PQclear(res); + elog(WARNING, "Invalid value of timeline_id %s", val); + + /* TODO 3.0 remove it and just error out */ + return get_current_timeline_from_control(false); + } + + return tli; +} + +/* Get timeline from pg_control file */ +TimeLineID +get_current_timeline_from_control(bool safe) { ControlFileData ControlFile; char *buffer; @@ -164,6 +193,8 @@ get_current_timeline(bool safe) return ControlFile.checkPointCopy.ThisTimeLineID; } + + /* * Get last check point record ptr from pg_tonrol. */ From 544de123a4625ee72f5ab90a5d358260649deeac Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Oct 2019 23:56:47 +0300 Subject: [PATCH 0981/2107] [Issue #102] now it is possible to set restore_command via set-config --- src/backup.c | 5 +- src/configure.c | 105 +++++++++++++++++++++++++------------- src/pg_probackup.c | 3 -- src/pg_probackup.h | 3 ++ src/restore.c | 6 +-- src/utils/configuration.c | 13 +++++ tests/archive.py | 67 ++++++++++++++++++++++++ 7 files changed, 156 insertions(+), 46 deletions(-) diff --git a/src/backup.c b/src/backup.c index 89f153901..085bb4c99 100644 --- a/src/backup.c +++ b/src/backup.c @@ -702,11 +702,8 @@ do_backup(time_t start_time, bool no_validate) current.compress_level = instance_config.compress_level; /* Save list of external directories */ - if (instance_config.external_dir_str && - pg_strcasecmp(instance_config.external_dir_str, "none") != 0) - { + if (instance_config.external_dir_str) current.external_dir_str = instance_config.external_dir_str; - } elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", diff --git a/src/configure.c b/src/configure.c index e3ab99ba3..19d0c0af4 100644 --- a/src/configure.c +++ b/src/configure.c @@ -137,97 +137,102 @@ ConfigOption instance_options[] = &instance_config.archive.user, SOURCE_CMD, 0, OPTION_ARCHIVE_GROUP, 0, option_get_value }, + { + 's', 211, "restore-command", + &instance_config.restore_command, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, /* Logging options */ { - 'f', 211, "log-level-console", + 'f', 212, "log-level-console", assign_log_level_console, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, get_log_level_console }, { - 'f', 212, "log-level-file", + 'f', 213, "log-level-file", assign_log_level_file, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, get_log_level_file }, { - 's', 213, "log-filename", + 's', 214, "log-filename", &instance_config.logger.log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 214, "error-log-filename", + 's', 215, "error-log-filename", &instance_config.logger.error_log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 215, "log-directory", + 's', 216, "log-directory", &instance_config.logger.log_directory, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 216, "log-rotation-size", + 'U', 217, "log-rotation-size", &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 217, "log-rotation-age", + 'U', 218, "log-rotation-age", &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 218, "retention-redundancy", + 'u', 219, "retention-redundancy", &instance_config.retention_redundancy, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 219, "retention-window", + 'u', 220, "retention-window", &instance_config.retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 'f', 220, "compress-algorithm", + 'f', 221, "compress-algorithm", assign_compress_alg, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { - 'u', 221, "compress-level", + 'u', 222, "compress-level", &instance_config.compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 222, "remote-proto", + 's', 223, "remote-proto", &instance_config.remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 223, "remote-host", + 's', 224, "remote-host", &instance_config.remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "remote-port", + 's', 225, "remote-port", &instance_config.remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-path", + 's', 226, "remote-path", &instance_config.remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-user", + 's', 227, "remote-user", &instance_config.remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "ssh-options", + 's', 228, "ssh-options", &instance_config.remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "ssh-config", + 's', 229, "ssh-config", &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, @@ -450,97 +455,125 @@ readInstanceConfigFile(const char *instance_name) &instance->archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value }, + { + 's', 208, "archive-host", + &instance_config.archive.host, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + { + 's', 209, "archive-port", + &instance_config.archive.port, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + { + 's', 210, "archive-user", + &instance_config.archive.user, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + { + 's', 211, "restore-command", + &instance->restore_command, SOURCE_CMD, 0, + OPTION_ARCHIVE_GROUP, 0, option_get_value + }, + + /* Instance options */ + { + 's', 'D', "pgdata", + &instance->pgdata, SOURCE_CMD, 0, + OPTION_INSTANCE_GROUP, 0, option_get_value + }, + /* Logging options */ { - 's', 208, "log-level-console", + 's', 212, "log-level-console", &log_level_console, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 209, "log-level-file", + 's', 213, "log-level-file", &log_level_file, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 210, "log-filename", + 's', 214, "log-filename", &instance->logger.log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 211, "error-log-filename", + 's', 215, "error-log-filename", &instance->logger.error_log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 212, "log-directory", + 's', 216, "log-directory", &instance->logger.log_directory, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 213, "log-rotation-size", + 'U', 217, "log-rotation-size", &instance->logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 214, "log-rotation-age", + 'U', 218, "log-rotation-age", &instance->logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 215, "retention-redundancy", + 'u', 219, "retention-redundancy", &instance->retention_redundancy, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 216, "retention-window", + 'u', 220, "retention-window", &instance->retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 's', 217, "compress-algorithm", + 's', 221, "compress-algorithm", &compress_alg, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'u', 218, "compress-level", + 'u', 222, "compress-level", &instance->compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 219, "remote-proto", + 's', 223, "remote-proto", &instance->remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 220, "remote-host", + 's', 224, "remote-host", &instance->remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 221, "remote-port", + 's', 225, "remote-port", &instance->remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 222, "remote-path", + 's', 226, "remote-path", &instance->remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 223, "remote-user", + 's', 227, "remote-user", &instance->remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "ssh-options", + 's', 228, "ssh-options", &instance->remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "ssh-config", + 's', 229, "ssh-config", &instance->remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 72659048e..3cbd0d0c4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -87,7 +87,6 @@ static char *target_stop; static bool target_immediate; static char *target_name = NULL; static char *target_action = NULL; -static char *restore_command = NULL; static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; @@ -183,7 +182,6 @@ static ConfigOption cmd_options[] = { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, - { 's', 160, "restore-command", &restore_command, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -643,7 +641,6 @@ main(int argc, char *argv[]) restore_params->skip_external_dirs = skip_external_dirs; restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; - restore_params->restore_command = restore_command; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 957955a41..f9dbc687e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -252,6 +252,9 @@ typedef struct InstanceConfig /* Wait timeout for WAL segment archiving */ uint32 archive_timeout; + /* cmdline to be used as restore_command */ + char *restore_command; + /* Logger parameters */ LoggerConfig logger; diff --git a/src/restore.c b/src/restore.c index cea4fb67a..4167ae9f9 100644 --- a/src/restore.c +++ b/src/restore.c @@ -857,15 +857,15 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); + /* construct restore_command */ if (need_restore_conf) { char restore_command_guc[16384]; /* If restore_command is provided, use it */ - if (params->restore_command) - sprintf(restore_command_guc, "%s", params->restore_command); - /* construct restore_command */ + if (instance_config.restore_command) + sprintf(restore_command_guc, "%s", instance_config.restore_command); else { /* default cmdline, ok for local restore */ diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 958025a25..0edfda24c 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -250,6 +250,15 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src) case 's': if (orig_source != SOURCE_DEFAULT) free(*(char **) opt->var); + + /* 'none' and 'off' are always disable the string parameter */ + if ((pg_strcasecmp(optarg, "none") == 0) || + (pg_strcasecmp(optarg, "off") == 0)) + { + opt->var = NULL; + return; + } + *(char **) opt->var = pgut_strdup(optarg); if (strcmp(optarg,"") != 0) return; @@ -656,6 +665,10 @@ option_get_value(ConfigOption *opt) case 's': if (*((char **) opt->var) == NULL) return NULL; + /* 'none' and 'off' are always disable the string parameter */ + if ((pg_strcasecmp(*((char **) opt->var), "none") == 0) || + (pg_strcasecmp(*((char **) opt->var), "off") == 0)) + return NULL; return pstrdup(*((char **) opt->var)); case 't': { diff --git a/tests/archive.py b/tests/archive.py index 179829982..2d7567f7b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1614,6 +1614,73 @@ def test_archive_options(self): self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_options_1(self): + """ + check that '--archive-host', '--archive-user', '--archiver-port' + and '--restore-command' are working as expected with set-config + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + node.cleanup() + + wal_dir = os.path.join(backup_dir, 'wal', 'node') + self.set_config( + backup_dir, 'node', + options=[ + '--restore-command="cp {0}/%f %p"'.format(wal_dir), + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user)]) + self.restore_node(backup_dir, 'node', node) + + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), + recovery_content) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command=none'.format(wal_dir), + '--archive-host=localhost1', + '--archive-port=23', + '--archive-user={0}'.format(self.user) + ]) + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + "restore_command = '{0} archive-get -B {1} --instance {2} " + "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " + "--remote-port=23 --remote-user={3}'".format( + self.probackup_path, backup_dir, 'node', self.user), + recovery_content) + + self.del_test_dir(module_name, fname) + # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. # so write validation code accordingly From 8d6d1c6e404e084bdca5af3225896fbc9b27e0ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 4 Oct 2019 00:01:56 +0300 Subject: [PATCH 0982/2107] help: minor update --- src/help.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/help.c b/src/help.c index 6e7733475..e1261c9e5 100644 --- a/src/help.c +++ b/src/help.c @@ -409,6 +409,7 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); + printf(_(" 'none' disables; (default: 'none')\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); @@ -690,6 +691,7 @@ help_set_config(void) printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); + printf(_(" 'none' disables; (default: 'none')\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From b7d5d6c45946b63a54a5f57d8af09b4e0f676ae6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 4 Oct 2019 09:16:23 +0300 Subject: [PATCH 0983/2107] help: another minor update --- src/help.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/help.c b/src/help.c index e1261c9e5..c0069e001 100644 --- a/src/help.c +++ b/src/help.c @@ -408,8 +408,7 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); - printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); - printf(_(" 'none' disables; (default: 'none')\n")); + printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); @@ -690,8 +689,7 @@ help_set_config(void) printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); - printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf\n")); - printf(_(" 'none' disables; (default: 'none')\n")); + printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From d02fc64a14c551cb2951e6845488c8de576cde15 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 4 Oct 2019 10:00:33 +0300 Subject: [PATCH 0984/2107] Terminate ssh error by zero character --- src/utils/file.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 1b4989f2c..62d35f03b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -55,10 +55,13 @@ void fio_error(int rc, int size, char const* file, int line) } else { - char buf[PRINTF_BUF_SIZE]; - int err_size = read(fio_stderr, buf, sizeof(buf)); + char buf[PRINTF_BUF_SIZE+1]; + int err_size = read(fio_stderr, buf, PRINTF_BUF_SIZE); if (err_size > 0) + { + buf[err_size] = '\0'; elog(ERROR, "Agent error: %s", buf); + } else elog(ERROR, "Communication error: %s", rc >= 0 ? "end of data" : strerror(errno)); } From 6062362519522c78c273a94dcb8c0f52610e90dc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 4 Oct 2019 10:36:56 +0300 Subject: [PATCH 0985/2107] tests: fixes --- tests/delete.py | 1 - tests/helpers/ptrack_helpers.py | 20 ++++++++++++++++- tests/retention.py | 38 ++++++++++++++++----------------- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index ce6bbb98d..6df8269b1 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -411,7 +411,6 @@ def test_delete_interleaved_incremental_chains(self): node.slow_start() # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) backup_id_b = self.backup_node(backup_dir, 'node', node) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e8e27b8f0..0f9cb0970 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -949,7 +949,8 @@ def show_pb( def show_archive( self, backup_dir, instance=None, options=[], - as_text=False, as_json=True, old_binary=False + as_text=False, as_json=True, old_binary=False, + tli=0 ): cmd_list = [ @@ -973,6 +974,23 @@ def show_archive( data = self.run_pb(cmd_list + options, old_binary=old_binary) else: data = json.loads(self.run_pb(cmd_list + options, old_binary=old_binary)) + + if instance: + instance_timelines = None + for instance_name in data: + if instance_name['instance'] == instance: + instance_timelines = instance_name['timelines'] + break + + if tli > 0: + timeline_data = None + for timeline in instance_timelines: + if timeline['tli'] == tli: + return timeline + + if instance_timelines: + return instance_timelines + return data else: show_splitted = self.run_pb( diff --git a/tests/retention.py b/tests/retention.py index 11e4e0333..0447e0623 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -25,10 +25,8 @@ def test_retention_redundancy_1(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - with open(os.path.join( - backup_dir, 'backups', 'node', - "pg_probackup.conf"), "a") as conf: - conf.write("retention-redundancy = 1\n") + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) # Make backups to be purged self.backup_node(backup_dir, 'node', node) @@ -39,31 +37,31 @@ def test_retention_redundancy_1(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + output_before = self.show_archive(backup_dir, 'node', tli=1) + # Purge backups log = self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - # Check that WAL segments were deleted - min_wal = None - max_wal = None - for line in log.splitlines(): - if line.startswith("INFO: removed min WAL segment"): - min_wal = line[31:-1] - elif line.startswith("INFO: removed max WAL segment"): - max_wal = line[31:-1] + output_after = self.show_archive(backup_dir, 'node', tli=1) - if not min_wal: - self.assertTrue(False, "min_wal is empty") + self.assertEqual( + output_before['max-segno'], + output_after['max-segno']) - if not max_wal: - self.assertTrue(False, "max_wal is not set") + self.assertNotEqual( + output_before['min-segno'], + output_after['min-segno']) + + # Check that WAL segments were deleted + min_wal = output_after['min-segno'] + max_wal = output_after['max-segno'] for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - # wal_name_b = wal_name.encode('ascii') - self.assertEqual(wal_name[8:] > min_wal[8:], True) - self.assertEqual(wal_name[8:] > max_wal[8:], True) + self.assertTrue(wal_name[8:] >= min_wal) + self.assertTrue(wal_name[8:] <= max_wal) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1353,7 +1351,7 @@ def test_window_error_backups_2(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - gdb._execute('signal SIGTERM') + gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() page_id = self.show_pb(backup_dir, 'node')[1]['id'] From 65969267fdd4eb3292dd6869e9ae41d90fd61700 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 4 Oct 2019 17:20:55 +0300 Subject: [PATCH 0986/2107] [Issue #60] Calculate compression ratio("zratio") during backup --- Documentation.md | 47 +++++-- src/backup.c | 17 +++ src/catalog.c | 23 +++- src/data.c | 8 ++ src/dir.c | 1 + src/pg_probackup.h | 14 ++- src/show.c | 40 +++++- src/util.c | 1 + tests/archive.py | 4 +- tests/delete.py | 10 +- tests/show.py | 297 +++++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 435 insertions(+), 27 deletions(-) diff --git a/Documentation.md b/Documentation.md index 2842a6920..9e8b59c46 100644 --- a/Documentation.md +++ b/Documentation.md @@ -708,13 +708,14 @@ pg_probackup displays the list of all the available backups. For example: ``` BACKUP INSTANCE 'node' -============================================================================================================================================ - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -============================================================================================================================================ - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1 / 1 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1 / 1 21s 32MB 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1 / 1 31s 33MB 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1 / 0 11s 39MB 0/F000028 0/F000198 OK +====================================================================================================================================== + Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK + node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK + node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK + node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK + node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK ``` For each backup, the following information is provided: @@ -724,12 +725,14 @@ For each backup, the following information is provided: - ID — the backup identifier. - Recovery time — the earliest moment for which you can restore the state of the database cluster. - Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. -- WAL — the WAL delivery mode. Possible values: STREAM and ARCHIVE. -- Current/Parent TLI — timeline identifiers of current backup and its parent. +- WAL Mode — the WAL delivery mode. Possible values: STREAM and ARCHIVE. +- TLI — timeline identifiers of current backup and its parent. - Time — the time it took to perform the backup. -- Data — the size of the data files in this backup. This value does not include the size of WAL files. -- Start LSN — WAL log sequence number corresponding to the start of the backup process. -- Stop LSN — WAL log sequence number corresponding to the end of the backup process. +- Data — the size of the data files in this backup. This value does not include the size of WAL files. In case of STREAM backup the total size of backup can be calculated as 'Data' + 'WAL'. +- WAL — the uncompressed size of WAL files required to apply by PostgreSQL recovery process to reach consistency. +- Zratio — compression ratio calculated as 'uncompressed-bytes' / 'data-bytes'. +- Start LSN — WAL log sequence number corresponding to the start of the backup process. REDO point for PostgreSQL recovery process to start from. +- Stop LSN — WAL log sequence number corresponding to the end of the backup process. Consistency point for PostgreSQL recovery process. - Status — backup status. Possible values: - OK — the backup is complete and valid. @@ -774,11 +777,29 @@ recovery-xid = 597 recovery-time = '2017-05-16 12:57:31' data-bytes = 22288792 wal-bytes = 16777216 +uncompressed-bytes = 39961833 +pgdata-bytes = 39859393 status = OK parent-backup-id = 'PT8XFX' primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' ``` +Detailed output has additional attributes: +- compress-alg — compression algorithm used during backup. Possible values: 'zlib', 'pglz', 'none'. +- compress-level — compression level used during backup. +- from-replica — the fact that backup was taken from standby server. Possible values: '1', '0'. +- block-size — (block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. +- wal-block-size — (wal_block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. +- checksum-version — the fact that PostgreSQL cluster, from which backup is taken, has enabled [data block checksumms](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS). Possible values: '1', '0'. +- program-version — full version of pg_probackup binary used to create backup. +- start-time — the backup starting time. +- end-time — the backup ending time. +- uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used. +- pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'. +- recovery-xid — current transaction id at the moment of backup ending. +- parent-backup-id — backup ID of parent backup. Available only for incremental backups. +- primary_conninfo — libpq conninfo used for connection to PostgreSQL cluster during backup. The password is not included. + To get more detailed information about the backup in json format, run the show with the backup ID: pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id @@ -851,7 +872,7 @@ For each backup, the following information is provided: - Max Segno — number of the last existing WAL segment belonging to the timeline. - N segments — number of WAL segments belonging to the timeline. - Size — the size files take on disk. -- Zratio - compression ratio calculated as "N segments" * wal_seg_size / "Size". +- Zratio — compression ratio calculated as 'N segments' * wal_seg_size / 'Size'. - N backups — number of backups belonging to the timeline. To get the details about backups, use json format. - Status — archive status for this exact timeline. Possible values: - OK — all WAL segments between Min and Max are present. diff --git a/src/backup.c b/src/backup.c index 085bb4c99..54af8d1fa 100644 --- a/src/backup.c +++ b/src/backup.c @@ -547,6 +547,22 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* close ssh session in main thread */ fio_disconnect(); + /* Calculate pgdata_bytes */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* In case of FULL or DELTA backup we can trust read_size. + * In case of PAGE or PTRACK we are forced to trust datafile size, + * taken at the start of backup. + */ + if (current.backup_mode == BACKUP_MODE_FULL || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + current.pgdata_bytes += file->read_size; + else + current.pgdata_bytes += file->size; + } + /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { @@ -1934,6 +1950,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, file->crc = pgFileGetCRC(file->path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; + file->uncompressed_size = file->read_size; free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); diff --git a/src/catalog.c b/src/catalog.c index a0c7123aa..027be1232 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1052,6 +1052,12 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->wal_bytes != BYTES_INVALID) fio_fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); + if (backup->uncompressed_bytes >= 0) + fio_fprintf(out, "uncompressed-bytes = " INT64_FORMAT "\n", backup->uncompressed_bytes); + + if (backup->pgdata_bytes >= 0) + fio_fprintf(out, "pgdata-bytes = " INT64_FORMAT "\n", backup->pgdata_bytes); + fio_fprintf(out, "status = %s\n", status2str(backup->status)); /* 'parent_backup' is set if it is incremental backup */ @@ -1121,6 +1127,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char buf[BUFFERSZ]; size_t write_len = 0; int64 backup_size_on_disk = 0; + int64 uncompressed_size_on_disk = 0; int64 wal_size_on_disk = 0; pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); @@ -1142,16 +1149,25 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, i++; if (S_ISDIR(file->mode)) + { backup_size_on_disk += 4096; + uncompressed_size_on_disk += 4096; + } /* Count the amount of the data actually copied */ if (S_ISREG(file->mode) && file->write_size > 0) { - /* TODO: in 3.0 add attribute is_walfile */ + /* + * Size of WAL files in 'pg_wal' is counted separately + * TODO: in 3.0 add attribute is_walfile + */ if (IsXLogFileName(file->name) && (file->external_dir_num == 0)) wal_size_on_disk += file->write_size; else + { backup_size_on_disk += file->write_size; + uncompressed_size_on_disk += file->uncompressed_size; + } } /* for files from PGDATA and external files use rel_path @@ -1235,6 +1251,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; backup->wal_bytes = wal_size_on_disk; + backup->uncompressed_bytes = uncompressed_size_on_disk; } /* @@ -1269,6 +1286,8 @@ readBackupControlFile(const char *path) {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "uncompressed-bytes", &backup->uncompressed_bytes, SOURCE_FILE_STRICT}, + {'I', 0, "pgdata-bytes", &backup->pgdata_bytes, SOURCE_FILE_STRICT}, {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, @@ -1513,6 +1532,8 @@ pgBackupInit(pgBackup *backup) backup->data_bytes = BYTES_INVALID; backup->wal_bytes = BYTES_INVALID; + backup->uncompressed_bytes = 0; + backup->pgdata_bytes = 0; backup->compress_alg = COMPRESS_ALG_DEFAULT; backup->compress_level = COMPRESS_LEVEL_DEFAULT; diff --git a/src/data.c b/src/data.c index 494f0b67d..c4aa0efc0 100644 --- a/src/data.c +++ b/src/data.c @@ -510,6 +510,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, } file->write_size += write_buffer_size; + file->uncompressed_size += BLCKSZ; } /* @@ -556,6 +557,7 @@ backup_data_file(backup_files_arg* arguments, /* reset size summary */ file->read_size = 0; file->write_size = 0; + file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); /* open backup mode file for read */ @@ -625,6 +627,8 @@ backup_data_file(backup_files_arg* arguments, elog(ERROR, "Failed to read file \"%s\": %s", file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); n_blocks_read = rc; + + file->uncompressed_size = (n_blocks_read - n_blocks_skipped)*BLCKSZ; } else { @@ -959,6 +963,7 @@ copy_file(fio_location from_location, const char *to_root, /* reset size summary */ file->read_size = 0; file->write_size = 0; + file->uncompressed_size = 0; /* open backup mode file for read */ in = fio_fopen(file->path, PG_BINARY_R, from_location); @@ -1046,6 +1051,9 @@ copy_file(fio_location from_location, const char *to_root, } file->write_size = (int64) file->read_size; + + if (file->write_size > 0) + file->uncompressed_size = file->write_size; /* finish CRC calculation and store into pgFile */ FIN_FILE_CRC32(true, crc); file->crc = crc; diff --git a/src/dir.c b/src/dir.c index 571e26110..b9f4972cd 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1694,6 +1694,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ file->crc = pgFileGetCRC(database_map_path, true, false, &file->read_size, FIO_BACKUP_HOST); file->write_size = file->read_size; + file->uncompressed_size = file->read_size; parray_append(backup_files_list, file); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f9dbc687e..358dcd286 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -140,6 +140,9 @@ typedef struct pgFile int64 write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ + int64 uncompressed_size; /* size of the backed-up file before compression + * and adding block headers. + */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ @@ -317,8 +320,17 @@ struct pgBackup * BYTES_INVALID means nothing was backed up. */ int64 data_bytes; - /* Size of WAL files needed to restore this backup */ + /* Size of WAL files needed to replay on top of this + * backup to reach the consistency. + */ int64 wal_bytes; + /* Size of data files before applying compression and block header, + * WAL files are not included. + */ + int64 uncompressed_bytes; + + /* Size of data files in PGDATA at the moment of backup. */ + int64 pgdata_bytes; CompressAlg compress_alg; int compress_level; diff --git a/src/show.c b/src/show.c index 1cef0ad50..cb6db218d 100644 --- a/src/show.c +++ b/src/show.c @@ -31,6 +31,7 @@ typedef struct ShowBackendRow char duration[20]; char data_bytes[20]; char wal_bytes[20]; + char zratio[20]; char start_lsn[20]; char stop_lsn[20]; const char *status; @@ -385,6 +386,18 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); } + if (backup->uncompressed_bytes >= 0) + { + json_add_key(buf, "uncompressed-bytes", json_level); + appendPQExpBuffer(buf, INT64_FORMAT, backup->uncompressed_bytes); + } + + if (backup->uncompressed_bytes >= 0) + { + json_add_key(buf, "pgdata-bytes", json_level); + appendPQExpBuffer(buf, INT64_FORMAT, backup->pgdata_bytes); + } + if (backup->primary_conninfo) json_add_value(buf, "primary_conninfo", backup->primary_conninfo, json_level, true); @@ -435,16 +448,16 @@ show_backup(const char *instance_name, time_t requested_backup_id) static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name) { -#define SHOW_FIELDS_COUNT 13 +#define SHOW_FIELDS_COUNT 14 int i; const char *names[SHOW_FIELDS_COUNT] = { "Instance", "Version", "ID", "Recovery Time", "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL", - "Start LSN", "Stop LSN", "Status" }; + "Zratio", "Start LSN", "Stop LSN", "Status" }; const char *field_formats[SHOW_FIELDS_COUNT] = { " %-*s ", " %-*s ", " %-*s ", " %-*s ", - " %-*s ", " %-*s ", " %-*s ", " %*s ", " %-*s ", " %-*s ", - " %-*s ", " %-*s ", " %-*s "}; + " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", " %*s ", + " %*s ", " %-*s ", " %-*s ", " %-*s "}; uint32 widths[SHOW_FIELDS_COUNT]; uint32 widths_sum = 0; ShowBackendRow *rows; @@ -465,6 +478,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na pgBackup *backup = parray_get(backup_list, i); ShowBackendRow *row = &rows[i]; int cur = 0; + float zratio = 1; /* Instance */ row->instance = instance_name; @@ -503,7 +517,6 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na cur++; /* Current/Parent TLI */ - if (backup->parent_backup_link != NULL) parent_tli = backup->parent_backup_link->tli; @@ -540,6 +553,19 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na widths[cur] = Max(widths[cur], strlen(row->wal_bytes)); cur++; + /* Zratio (compression ratio) */ + if (backup->uncompressed_bytes != BYTES_INVALID && + (backup->uncompressed_bytes > 0 && backup->data_bytes > 0)) + { + zratio = (float)backup->uncompressed_bytes / (backup->data_bytes); + snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio); + } + else + snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio); + + widths[cur] = Max(widths[cur], strlen(row->zratio)); + cur++; + /* Start LSN */ snprintf(row->start_lsn, lengthof(row->start_lsn), "%X/%X", (uint32) (backup->start_lsn >> 32), @@ -630,6 +656,10 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->wal_bytes); cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->zratio); + cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], row->start_lsn); cur++; diff --git a/src/util.c b/src/util.c index 816e27f0d..e95051718 100644 --- a/src/util.c +++ b/src/util.c @@ -400,6 +400,7 @@ copy_pgcontrol_file(const char *from_root, fio_location from_location, file->crc = ControlFile.crc; file->read_size = size; file->write_size = size; + file->uncompressed_size = size; join_path_components(to_path, to_root, file->rel_path); writeControlFile(&ControlFile, to_path, to_location); diff --git a/tests/archive.py b/tests/archive.py index 2d7567f7b..626f4b671 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -49,12 +49,12 @@ def test_pgpro434_1(self): node.slow_start() # Recreate backup catalog + self.clean_pb(backup_dir) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) # Make backup - self.backup_node( - backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.cleanup() # Restore Database diff --git a/tests/delete.py b/tests/delete.py index 6df8269b1..61393a2d4 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -260,7 +260,7 @@ def test_delete_orphaned_wal_segments(self): # Check wals wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] original_wal_quantity = len(wals) # delete second full backup @@ -283,18 +283,18 @@ def test_delete_orphaned_wal_segments(self): result = self.delete_pb(backup_dir, 'node', options=['--wal']) # delete useless wals - self.assertTrue('INFO: removed min WAL segment' in result - and 'INFO: removed max WAL segment' in result) + self.assertTrue('WAL segments between ' in result + and 'on timeline 1 will be removed' in result) self.validate_pb(backup_dir) self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") # Check quantity, it should be lower than original - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") # Delete last backup self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] self.assertEqual (0, len(wals), "Number of wals should be equal to 0") # Clean after yourself diff --git a/tests/show.py b/tests/show.py index 6044e15b8..ad9cafb73 100644 --- a/tests/show.py +++ b/tests/show.py @@ -207,3 +207,300 @@ def test_corrupt_control_file(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness_1(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # change data + pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--remote-proto=none']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness_2(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=none', '--compress']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # change data + pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--remote-proto=none', '--compress']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--remote-proto=none', '--compress']) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 20d8820d7ae5c908e8ddeb400ea40d15d99e6f19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 6 Oct 2019 21:48:44 +0300 Subject: [PATCH 0987/2107] [Issue #129] Generate recovery.conf if --recovery-target='immediate' is provided --- src/restore.c | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/restore.c b/src/restore.c index 4167ae9f9..1aa2d1f6a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -193,7 +193,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { parray *timelines; - elog(LOG, "target timeline ID = %u", rt->target_tli); + // elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ timelines = read_timeline_history(arclog_path, rt->target_tli); @@ -835,11 +835,18 @@ create_recovery_conf(time_t backup_id, FILE *fp; bool need_restore_conf; bool target_latest; + bool target_immediate; + /* restore-target='latest' support */ target_latest = rt->target_stop != NULL && strcmp(rt->target_stop, "latest") == 0; - need_restore_conf = !backup->stream || - (rt->time_string || rt->xid_string || rt->lsn_string) || target_latest; + + target_immediate = rt->target_stop != NULL && + strcmp(rt->target_stop, "immediate") == 0; + + need_restore_conf = !backup->stream || rt->time_string || + rt->xid_string || rt->lsn_string || rt->target_name || + target_immediate || target_latest; /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || params->restore_as_replica)) @@ -914,7 +921,7 @@ create_recovery_conf(time_t backup_id, if (rt->lsn_string) fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - if (rt->target_stop && !target_latest) + if (rt->target_stop && target_immediate) fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); if (rt->inclusive_specified) @@ -1120,7 +1127,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_time(target_time, &dummy_time, false)) rt->target_time = dummy_time; else - elog(ERROR, "Invalid value for '--recovery-target-time' option %s", + elog(ERROR, "Invalid value for '--recovery-target-time' option '%s'", target_time); } @@ -1138,7 +1145,7 @@ parseRecoveryTargetOptions(const char *target_time, #endif rt->target_xid = dummy_xid; else - elog(ERROR, "Invalid value for '--recovery-target-xid' option %s", + elog(ERROR, "Invalid value for '--recovery-target-xid' option '%s'", target_xid); } @@ -1151,7 +1158,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_lsn(target_lsn, &dummy_lsn)) rt->target_lsn = dummy_lsn; else - elog(ERROR, "Invalid value of '--recovery-target-lsn' option %s", + elog(ERROR, "Invalid value of '--recovery-target-lsn' option '%s'", target_lsn); } @@ -1161,7 +1168,7 @@ parseRecoveryTargetOptions(const char *target_time, if (parse_bool(target_inclusive, &dummy_bool)) rt->target_inclusive = dummy_bool; else - elog(ERROR, "Invalid value for '--recovery-target-inclusive' option %s", + elog(ERROR, "Invalid value for '--recovery-target-inclusive' option '%s'", target_inclusive); } @@ -1170,7 +1177,7 @@ parseRecoveryTargetOptions(const char *target_time, { if ((strcmp(target_stop, "immediate") != 0) && (strcmp(target_stop, "latest") != 0)) - elog(ERROR, "Invalid value for '--recovery-target' option %s", + elog(ERROR, "Invalid value for '--recovery-target' option '%s'", target_stop); recovery_target_specified++; @@ -1188,7 +1195,7 @@ parseRecoveryTargetOptions(const char *target_time, if ((strcmp(target_action, "pause") != 0) && (strcmp(target_action, "promote") != 0) && (strcmp(target_action, "shutdown") != 0)) - elog(ERROR, "Invalid value for '--recovery-target-action' option %s", + elog(ERROR, "Invalid value for '--recovery-target-action' option '%s'", target_action); rt->target_action = target_action; @@ -1201,7 +1208,9 @@ parseRecoveryTargetOptions(const char *target_time, /* More than one mutually exclusive option was defined. */ if (recovery_target_specified > 1) - elog(ERROR, "At most one of --recovery-target, --recovery-target-name, --recovery-target-time, --recovery-target-xid, or --recovery-target-lsn can be specified"); + elog(ERROR, "At most one of '--recovery-target', '--recovery-target-name', " + "'--recovery-target-time', '--recovery-target-xid' or " + "'--recovery-target-lsn' options can be specified"); /* * If none of the options is defined, '--recovery-target-inclusive' option @@ -1209,7 +1218,17 @@ parseRecoveryTargetOptions(const char *target_time, */ if (!(rt->xid_string || rt->time_string || rt->lsn_string) && rt->target_inclusive) - elog(ERROR, "--recovery-target-inclusive option applies when either --recovery-target-time, --recovery-target-xid or --recovery-target-lsn is specified"); + elog(ERROR, "The '--recovery-target-inclusive' option can be applied only when " + "either of '--recovery-target-time', '--recovery-target-xid' or " + "'--recovery-target-lsn' options is specified"); + + /* If none of the options is defined, '--recovery-target-action' is meaningless */ + if (rt->target_action && recovery_target_specified == 0) + elog(ERROR, "The '--recovery-target-action' option can be applied only when " + "either of '--recovery-target', '--recovery-target-time', '--recovery-target-xid', " + "'--recovery-target-lsn' or '--recovery-target-name' options is specified"); + + /* TODO: sanity for recovery-target-timeline */ return rt; } From 0cd9e35028e2630352b32b2693f32db460322ca0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Oct 2019 11:47:56 +0300 Subject: [PATCH 0988/2107] [Issue #129] minor fix --- src/restore.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/restore.c b/src/restore.c index 1aa2d1f6a..8e3a78a05 100644 --- a/src/restore.c +++ b/src/restore.c @@ -933,6 +933,9 @@ create_recovery_conf(time_t backup_id, if (rt->target_action) fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); + else + /* default recovery_target_action is 'pause' */ + fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); } if (params->restore_as_replica) @@ -1200,11 +1203,6 @@ parseRecoveryTargetOptions(const char *target_time, rt->target_action = target_action; } - else - { - /* Default recovery target action is pause */ - rt->target_action = "pause"; - } /* More than one mutually exclusive option was defined. */ if (recovery_target_specified > 1) From c4b49c6abb105dbbd78201fd01cfb8ed1898b4db Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Oct 2019 14:28:03 +0300 Subject: [PATCH 0989/2107] [Issue #68] the backup pinning --- Documentation.md | 68 ++++++++- src/backup.c | 12 +- src/catalog.c | 82 +++++++++++ src/delete.c | 26 +++- src/help.c | 31 +++- src/pg_probackup.c | 49 ++++++- src/pg_probackup.h | 20 ++- src/show.c | 6 + src/utils/configuration.c | 4 +- tests/__init__.py | 3 +- tests/helpers/ptrack_helpers.py | 20 ++- tests/retention.py | 1 - tests/set_backup.py | 241 ++++++++++++++++++++++++++++++++ 13 files changed, 548 insertions(+), 15 deletions(-) create mode 100644 tests/set_backup.py diff --git a/Documentation.md b/Documentation.md index 9e8b59c46..442e155f5 100644 --- a/Documentation.md +++ b/Documentation.md @@ -38,6 +38,7 @@ Current version - 2.1.5 * [Managing the Backup Catalog](#managing-the-backup-catalog) * [Viewing WAL Archive Information](#viewing-wal-archive-information) * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) + * [Pinning a Backup] * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -49,6 +50,7 @@ Current version - 2.1.5 * [add-instance](#add-instance) * [del-instance](#del-instance) * [set-config](#set-config) + * [set-backup](#set-backup) * [show-config](#show-config) * [show](#show) * [backup](#backup) @@ -63,6 +65,7 @@ Current version - 2.1.5 * [Common Options](#common-options) * [Recovery Target Options](#recovery-target-options) * [Retention Options](#retention-options) + * [Pinning Options](#pinning-options) * [Logging Options](#logging-options) * [Connection Options](#connection-options) * [Compression Options](#compression-options) @@ -89,6 +92,8 @@ Current version - 2.1.5 `pg_probackup set-config -B backup_dir --instance instance_name [option...]` +`pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id [option...]` + `pg_probackup show-config -B backup_dir --instance instance_name [--format=format]` `pg_probackup show -B backup_dir [option...]` @@ -775,6 +780,7 @@ start-time = '2017-05-16 12:57:29' end-time = '2017-05-16 12:57:31' recovery-xid = 597 recovery-time = '2017-05-16 12:57:31' +expire-time = '2020-05-16 12:57:31' data-bytes = 22288792 wal-bytes = 16777216 uncompressed-bytes = 39961833 @@ -794,6 +800,7 @@ Detailed output has additional attributes: - program-version — full version of pg_probackup binary used to create backup. - start-time — the backup starting time. - end-time — the backup ending time. +- expire-time — if the backup was pinned, then until this point in time the backup cannot be removed by retention purge. - uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used. - pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'. - recovery-xid — current transaction id at the moment of backup ending. @@ -1171,6 +1178,42 @@ BACKUP INSTANCE 'node' >NOTE: The Time field for the merged backup displays the time required for the merge. +#### Pinning a Backup + +If you have the necessity to exclude certain backups from established retention policy then it is possible to pin a backup for an arbitrary amount of time. Example: + + pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d + +This command will set `expire-time` of specified backup to 30 days starting from backup `recovery-time` attribute. Basically 'expire-time = recovery-time + ttl'. + +You can set `expire-time` explicitly using `--expire-time` option. Example: + + pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' + +Alternatively you can use the `--ttl` and `--expire-time` options with the [backup](#backup) command to pin newly created backup: + + pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d + pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' + +You can determine the fact that backup is pinned and check due expire time by looking up 'expire-time' attribute in backup metadata via (show)[#show] command: + + pg_probackup show --instance instance_name -i backup_id + +Pinned backup has `expire-time` attribute: +``` +... +recovery-time = '2017-05-16 12:57:31' +expire-time = '2020-01-01 00:00:00+03' +data-bytes = 22288792 +... +``` + +You can unpin a backup by setting `--ttl` option to zero using `set-backup` command. Example: + + pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 + +Only pinned backups have `expire-time` attribute in backup metadata. + ### Merging Backups As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: @@ -1267,6 +1310,15 @@ For all available settings, see the [Options](#options) section. It is **not recommended** to edit pg_probackup.conf manually. +#### set-backup + + pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id + {--ttl=ttl | --expire-time=time} [--help] + +Sets the provided backup-specific settings into the backup.control configuration file, or modifies previously defined values. + +For all available settings, see the section [Pinning Options](#pinning-options). + #### show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] @@ -1296,7 +1348,7 @@ For details on usage, see the sections [Managing the Backup Catalog](#managing-t [-w --no-password] [-W --password] [--archive-timeout=timeout] [--external-dirs=external_directory_path] [connection_options] [compression_options] [remote_options] - [retention_options] [logging_options] + [retention_options] [pinning_options] [logging_options] Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. @@ -1342,7 +1394,7 @@ Disables block-level checksum verification to speed up backup. --no-validate Skips automatic validation after successfull backup. You can use this flag if you validate backups regularly and would like to save time when running backup operations. -Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. +Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Pinning Options](#pinning-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. For details on usage, see the section [Creating a Backup](#creating-a-backup). @@ -1565,6 +1617,18 @@ Merges the oldest incremental backup that satisfies the requirements of retentio --dry-run Displays the current status of all the available backups, without deleting or merging expired backups, if any. +##### Pinning Options + +You can use these options together with [backup](#backup) and [set-delete](#set-backup) commands. + +For details on backup pinning, see the section [Pinning a Backup](#pinning-a-backup). + + --ttl=ttl +Specifies the amount of time the backup should be pinned. Must be a positive integer. The zero value unpin already pinned backup. Supported units: ms, s, min, h, d (s by default). Example: `--ttl=30d`. + + --expire-time=time +Specifies the timestamp up to which the backup will stay pinned. Must be a ISO-8601 complaint timestamp. Example: `--expire-time='2020-01-01 00:00:00+03'` + #### Logging Options You can use these options with any command. diff --git a/src/backup.c b/src/backup.c index 54af8d1fa..32e810402 100644 --- a/src/backup.c +++ b/src/backup.c @@ -695,7 +695,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) * Entry point of pg_probackup BACKUP subcommand. */ int -do_backup(time_t start_time, bool no_validate) +do_backup(time_t start_time, bool no_validate, + pgSetBackupParams *set_backup_params) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -801,6 +802,15 @@ do_backup(time_t start_time, bool no_validate) current.status = BACKUP_STATUS_DONE; write_backup(¤t); + /* Pin backup if requested */ + if (set_backup_params && + (set_backup_params->ttl > 0 || + set_backup_params->expire_time > 0)) + { + if (!pin_backup(¤t, set_backup_params)) + elog(ERROR, "Failed to pin the backup %s", base36enc(current.backup_id)); + } + if (!no_validate) pgBackupValidate(¤t, NULL); diff --git a/src/catalog.c b/src/catalog.c index 027be1232..006b7e9ba 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -987,6 +987,81 @@ get_oldest_backup(timelineInfo *tlinfo) return oldest_backup; } +/* + * Overwrite backup metadata. + */ +void +do_set_backup(const char *instance_name, time_t backup_id, + pgSetBackupParams *set_backup_params) +{ + pgBackup *target_backup = NULL; + parray *backup_list = NULL; + + if (!set_backup_params) + elog(ERROR, "Nothing to set by 'set-backup' command"); + + backup_list = catalog_get_backup_list(instance_name, backup_id); + if (parray_num(backup_list) != 1) + elog(ERROR, "Failed to find backup %s", base36enc(backup_id)); + + target_backup = (pgBackup *) parray_get(backup_list, 0); + + if (!pin_backup(target_backup, set_backup_params)) + elog(ERROR, "Failed to pin the backup %s", base36enc(backup_id)); +} + +/* + * Set 'expire-time' attribute based on set_backup_params, or unpin backup + * if ttl is equal to zero. + */ +bool +pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) +{ + + if (target_backup->recovery_time <= 0) + elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'", + base36enc(target_backup->backup_id)); + + /* Pin comes from ttl */ + if (set_backup_params->ttl > 0) + target_backup->expire_time = target_backup->recovery_time + set_backup_params->ttl; + /* Unpin backup */ + else if (set_backup_params->ttl == 0) + { + /* If backup was not pinned in the first place, + * then there is nothing to unpin. + */ + if (target_backup->expire_time == 0) + { + elog(WARNING, "Backup %s is not pinned, nothing to unpin", + base36enc(target_backup->start_time)); + return false; + } + target_backup->expire_time = 0; + } + /* Pin comes from expire-time */ + else if (set_backup_params->expire_time > 0) + target_backup->expire_time = set_backup_params->expire_time; + else + return false; + + /* Update backup.control */ + write_backup(target_backup); + + if (set_backup_params->ttl > 0 || set_backup_params->expire_time > 0) + { + char expire_timestamp[100]; + + time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time); + elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), + expire_timestamp); + } + else + elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time)); + + return true; +} + /* * Write information about backup.in to stream "out". */ @@ -1041,6 +1116,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->recovery_time); fio_fprintf(out, "recovery-time = '%s'\n", timestamp); } + if (backup->expire_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->expire_time); + fio_fprintf(out, "expire-time = '%s'\n", timestamp); + } /* * Size of PGDATA directory. The size does not include size of related @@ -1284,6 +1364,7 @@ readBackupControlFile(const char *path) {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, + {'t', 0, "expire-time", &backup->expire_time, SOURCE_FILE_STRICT}, {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, {'I', 0, "uncompressed-bytes", &backup->uncompressed_bytes, SOURCE_FILE_STRICT}, @@ -1529,6 +1610,7 @@ pgBackupInit(pgBackup *backup) backup->end_time = (time_t) 0; backup->recovery_xid = 0; backup->recovery_time = (time_t) 0; + backup->expire_time = (time_t) 0; backup->data_bytes = BYTES_INVALID; backup->wal_bytes = BYTES_INVALID; diff --git a/src/delete.c b/src/delete.c index 325d321f5..e361e57ee 100644 --- a/src/delete.c +++ b/src/delete.c @@ -292,6 +292,20 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg * FULL CORRUPT out of retention */ + /* Save backup from purge if backup is pinned and + * expire date is not yet due. + */ + if ((backup->expire_time > 0) && + (backup->expire_time > current_time)) + { + char expire_timestamp[100]; + time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time); + + elog(LOG, "Backup %s is pinned until '%s', retain", + base36enc(backup->start_time), expire_timestamp); + continue; + } + /* Add backup to purge_list */ elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time)); parray_append(to_purge_list, backup); @@ -355,6 +369,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg for (i = 0; i < parray_num(backup_list); i++) { char *action = "Active"; + uint32 pinning_window = 0; pgBackup *backup = (pgBackup *) parray_get(backup_list, i); @@ -364,7 +379,11 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg if (backup->recovery_time == 0) actual_window = 0; else - actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24); + actual_window = (current_time - backup->recovery_time)/(3600 * 24); + + /* For pinned backups show expire date */ + if (backup->expire_time > 0 && backup->expire_time > backup->recovery_time) + pinning_window = (backup->expire_time - backup->recovery_time)/(3600 * 24); /* TODO: add ancestor(chain full backup) ID */ elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", @@ -373,7 +392,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg status2str(backup->status), cur_full_backup_num, instance_config.retention_redundancy, - actual_window, instance_config.retention_window, + actual_window, + pinning_window ? pinning_window : instance_config.retention_window, action); if (backup->backup_mode == BACKUP_MODE_FULL) @@ -576,7 +596,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) { /* We must not delete this backup, evict it from purge list */ - elog(LOG, "Retain backup %s from purge because his " + elog(LOG, "Retain backup %s because his " "descendant %s is guarded by retention", base36enc(delete_backup->start_time), keeped_backup_id); diff --git a/src/help.c b/src/help.c index c0069e001..2b25f4a3e 100644 --- a/src/help.c +++ b/src/help.c @@ -16,6 +16,7 @@ static void help_validate(void); static void help_show(void); static void help_delete(void); static void help_merge(void); +static void help_set_backup(void); static void help_set_config(void); static void help_show_config(void); static void help_add_instance(void); @@ -41,6 +42,8 @@ help_command(char *command) help_delete(); else if (strcmp(command, "merge") == 0) help_merge(); + else if (strcmp(command, "set-backup") == 0) + help_set_backup(); else if (strcmp(command, "set-config") == 0) help_set_config(); else if (strcmp(command, "show-config") == 0) @@ -101,6 +104,10 @@ help_pg_probackup(void) printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); + printf(_("\n %s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id [--ttl] [--expire-time]\n")); + printf(_(" [--help]\n")); + printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); printf(_(" [--help]\n")); @@ -130,8 +137,10 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--ttl] [--expire-time]\n")); printf(_(" [--help]\n")); + printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); @@ -262,7 +271,8 @@ help_backup(void) printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--ttl] [--expire-time]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -314,6 +324,12 @@ help_backup(void) printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); + printf(_("\n Pinning options:\n")); + printf(_(" --ttl=ttl pin backup for specified amount of time; 0 unpin\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); + printf(_(" --expire-time=time pin backup until specified time stamp\n")); + printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); + printf(_("\n Compression options:\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); @@ -659,6 +675,19 @@ help_merge(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); } +static void +help_set_backup(void) +{ + printf(_("\n%s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" -i backup-id\n")); + printf(_(" [--ttl] [--expire-time]\n\n")); + + printf(_(" --ttl=ttl pin backup for specified amount of time; 0 unpin\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); + printf(_(" --expire-time=time pin backup until specified time stamp\n")); + printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); +} + static void help_set_config(void) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3cbd0d0c4..dba7a93fd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -42,6 +42,7 @@ typedef enum ProbackupSubcmd MERGE_CMD, SHOW_CMD, SET_CONFIG_CMD, + SET_BACKUP_CMD, SHOW_CONFIG_CMD, CHECKDB_CMD } ProbackupSubcmd; @@ -128,6 +129,11 @@ static bool file_overwrite = false; ShowFormat show_format = SHOW_PLAIN; bool show_archive = false; +/* set-backup options */ +int64 ttl = -1; +static char *expire_time_string = NULL; +static pgSetBackupParams *set_backup_params = NULL; + /* current settings */ pgBackup current; static ProbackupSubcmd backup_subcmd = NO_CMD; @@ -205,6 +211,9 @@ static ConfigOption cmd_options[] = /* show options */ { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, { 'b', 161, "archive", &show_archive, SOURCE_CMD_STRICT }, + /* set-backup options */ + { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, + { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, /* options for backward compatibility */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, @@ -300,6 +309,8 @@ main(int argc, char *argv[]) backup_subcmd = SHOW_CMD; else if (strcmp(argv[1], "set-config") == 0) backup_subcmd = SET_CONFIG_CMD; + else if (strcmp(argv[1], "set-backup") == 0) + backup_subcmd = SET_BACKUP_CMD; else if (strcmp(argv[1], "show-config") == 0) backup_subcmd = SHOW_CONFIG_CMD; else if (strcmp(argv[1], "checkdb") == 0) @@ -361,7 +372,9 @@ main(int argc, char *argv[]) backup_subcmd == RESTORE_CMD || backup_subcmd == VALIDATE_CMD || backup_subcmd == DELETE_CMD || - backup_subcmd == MERGE_CMD) + backup_subcmd == MERGE_CMD || + backup_subcmd == SET_CONFIG_CMD || + backup_subcmd == SET_BACKUP_CMD) { int i, len = 0, @@ -588,6 +601,7 @@ main(int argc, char *argv[]) backup_subcmd != VALIDATE_CMD && backup_subcmd != DELETE_CMD && backup_subcmd != MERGE_CMD && + backup_subcmd != SET_BACKUP_CMD && backup_subcmd != SHOW_CMD) elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", command_name); @@ -658,6 +672,32 @@ main(int argc, char *argv[]) } } + /* + * Parse set-backup options into set_backup_params structure. + */ + if (backup_subcmd == SET_BACKUP_CMD || backup_subcmd == BACKUP_CMD) + { + time_t expire_time = 0; + + if (expire_time_string && ttl >= 0) + elog(ERROR, "You cannot specify '--expire-time' and '--ttl' options together"); + + /* Parse string to seconds */ + if (expire_time_string) + { + if (!parse_time(expire_time_string, &expire_time, false)) + elog(ERROR, "Invalid value for '--expire-time' option: '%s'", + expire_time_string); + } + + if (expire_time > 0 || ttl >= 0) + { + set_backup_params = pgut_new(pgSetBackupParams); + set_backup_params->ttl = ttl; + set_backup_params->expire_time = expire_time; + } + } + /* sanity */ if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", @@ -694,7 +734,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, no_validate); + return do_backup(start_time, no_validate, set_backup_params); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, @@ -738,6 +778,11 @@ main(int argc, char *argv[]) case SET_CONFIG_CMD: do_set_config(false); break; + case SET_BACKUP_CMD: + if (!backup_id_string) + elog(ERROR, "You must specify parameter (-i, --backup-id) for 'set-backup' command"); + do_set_backup(instance_name, current.backup_id, set_backup_params); + break; case CHECKDB_CMD: do_checkdb(need_amcheck, instance_config.conn_opt, instance_config.pgdata); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 358dcd286..a08f77df2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -310,6 +310,7 @@ struct pgBackup time_t recovery_time; /* Earliest moment for which you can restore * the state of the database cluster using * this backup */ + time_t expire_time; /* Backup expiration date */ TransactionId recovery_xid; /* Earliest xid for which you can restore * the state of the database cluster using * this backup */ @@ -390,6 +391,18 @@ typedef struct pgRestoreParams parray *partial_db_list; } pgRestoreParams; +/* Options needed for set-backup command */ +typedef struct pgSetBackupParams +{ + int64 ttl; /* amount of time backup must be pinned + * -1 - do nothing + * 0 - disable pinning + */ + time_t expire_time; /* Point in time before which backup + * must be pinned. + */ +} pgSetBackupParams; + typedef struct { const char *from_root; @@ -572,7 +585,8 @@ extern char** commands_args; extern const char *pgdata_exclude_dir[]; /* in backup.c */ -extern int do_backup(time_t start_time, bool no_validate); +extern int do_backup(time_t start_time, bool no_validate, + pgSetBackupParams *set_backup_params); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -668,6 +682,10 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); extern parray *catalog_get_timelines(InstanceConfig *instance); +extern void do_set_backup(const char *instance_name, time_t backup_id, + pgSetBackupParams *set_backup_params); +extern bool pin_backup(pgBackup *target_backup, + pgSetBackupParams *set_backup_params); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list); diff --git a/src/show.c b/src/show.c index cb6db218d..d57e881b3 100644 --- a/src/show.c +++ b/src/show.c @@ -374,6 +374,12 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add_value(buf, "recovery-time", timestamp, json_level, true); } + if (backup->expire_time > 0) + { + time2iso(timestamp, lengthof(timestamp), backup->expire_time); + json_add_value(buf, "expire-time", timestamp, json_level, true); + } + if (backup->data_bytes != BYTES_INVALID) { json_add_key(buf, "data-bytes", json_level); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 0edfda24c..bf5cd0140 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1095,6 +1095,8 @@ parse_uint64(const char *value, uint64 *result, int flags) * * If utc_default is true, then if timezone offset isn't specified tz will be * +00:00. + * + * TODO: '0' converted into '2000-01-01 00:00:00'. Example: set-backup --expire-time=0 */ bool parse_time(const char *value, time_t *result, bool utc_default) @@ -1202,7 +1204,7 @@ parse_time(const char *value, time_t *result, bool utc_default) &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); free(tmp); - if (i < 1 || 6 < i) + if (i < 3 || i > 6) return false; /* adjust year */ diff --git a/tests/__init__.py b/tests/__init__.py index e844d29b8..cd3088ee5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,7 @@ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external, config, checkdb + locking, remote, external, config, checkdb, set_backup def load_tests(loader, tests, pattern): @@ -48,6 +48,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(replica)) suite.addTests(loader.loadTestsFromModule(restore)) suite.addTests(loader.loadTestsFromModule(retention)) + suite.addTests(loader.loadTestsFromModule(set_backup)) suite.addTests(loader.loadTestsFromModule(show)) suite.addTests(loader.loadTestsFromModule(snapfs)) suite.addTests(loader.loadTestsFromModule(time_stamp)) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0f9cb0970..913781dbc 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -737,6 +737,22 @@ def set_config(self, backup_dir, instance, old_binary=False, options=[]): return self.run_pb(cmd + options, old_binary=old_binary) + def set_backup(self, backup_dir, instance, backup_id=False, + old_binary=False, options=[]): + + cmd = [ + 'set-backup', + '-B', backup_dir + ] + + if instance: + cmd = cmd + ['--instance={0}'.format(instance)] + + if backup_id: + cmd = cmd + ['-i', backup_id] + + return self.run_pb(cmd + options, old_binary=old_binary) + def del_instance(self, backup_dir, instance, old_binary=False): return self.run_pb([ @@ -788,7 +804,7 @@ def backup_node( def checkdb_node( self, backup_dir=False, instance=False, data_dir=False, - options=[], async=False, gdb=False, old_binary=False + options=[], asynchronous=False, gdb=False, old_binary=False ): cmd_list = ["checkdb"] @@ -802,7 +818,7 @@ def checkdb_node( if data_dir: cmd_list += ["-D", data_dir] - return self.run_pb(cmd_list + options, async, gdb, old_binary) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) def merge_backup( self, backup_dir, instance, backup_id, asynchronous=False, diff --git a/tests/retention.py b/tests/retention.py index 0447e0623..b3bdfc849 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -693,7 +693,6 @@ def test_window_merge_interleaved_incremental_chains_1(self): page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgdata_b3 = self.pgdata_content(node.data_dir) pgbench = node.pgbench(options=['-t', '10', '-c', '2']) diff --git a/tests/set_backup.py b/tests/set_backup.py new file mode 100644 index 000000000..52807f752 --- /dev/null +++ b/tests/set_backup.py @@ -0,0 +1,241 @@ +import unittest +import subprocess +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from sys import exit +from datetime import datetime, timedelta + +module_name = 'set_backup' + + +class SetBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_set_backup_sanity(self): + """general sanity for set-backup command""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + expire_time_1 = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=5)) + + try: + self.set_backup(backup_dir, False, options=['--ttl=30d']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing instance. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: required parameter not specified: --instance', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.set_backup( + backup_dir, 'node', + options=[ + "--ttl=30d", + "--expire-time='{0}'".format(expire_time_1)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because options cannot be mixed. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--expire-time' " + "and '--ttl' options together", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.set_backup(backup_dir, 'node', options=["--ttl=30d"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing backup_id. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You must specify parameter (-i, --backup-id) " + "for 'set-backup' command", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.set_backup( + backup_dir, 'node', backup_id, options=["--ttl=30d"]) + + actual_expire_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['expire-time'] + + self.assertNotEqual(expire_time_1, actual_expire_time) + + expire_time_2 = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + + self.set_backup( + backup_dir, 'node', backup_id, + options=["--expire-time={0}".format(expire_time_2)]) + + actual_expire_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['expire-time'] + + self.assertIn(expire_time_2, actual_expire_time) + + # unpin backup + self.set_backup( + backup_dir, 'node', backup_id, options=["--ttl=0"]) + + attr_list = self.show_pb( + backup_dir, 'node', backup_id=backup_id) + + self.assertNotIn('expire-time', attr_list) + + self.set_backup( + backup_dir, 'node', backup_id, options=["--expire-time={0}".format(recovery_time)]) + + # parse string to datetime object + #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_pinning(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + full_id = self.backup_node(backup_dir, 'node', node) + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.set_backup( + backup_dir, 'node', page_id, options=['--ttl=5d']) + + # Purge backups + log = self.delete_expired( + backup_dir, 'node', + options=['--delete-expired', '--log-level-console=LOG']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.assertIn('Time Window: 0d/5d', log) + self.assertIn( + 'LOG: Backup {0} is pinned until'.format(page_id), + log) + self.assertIn( + 'LOG: Retain backup {0} because his descendant ' + '{1} is guarded by retention'.format(full_id, page_id), + log) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_retention_window_pinning(self): + """purge all backups using window-based retention policy""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUP + backup_id_1 = self.backup_node(backup_dir, 'node', node) + page1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take second FULL BACKUP + backup_id_2 = self.backup_node(backup_dir, 'node', node) + page2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take third FULL BACKUP + backup_id_3 = self.backup_node(backup_dir, 'node', node) + page2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.set_backup( + backup_dir, 'node', page1, options=['--ttl=30d']) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=LOG', + '--retention-window=1', + '--delete-expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.assertIn( + 'LOG: Backup {0} is pinned until'.format(page1), out) + + self.assertIn( + 'LOG: Retain backup {0} because his descendant ' + '{1} is guarded by retention'.format(backup_id_1, page1), + out) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 59c341a2bb110a321d091ccc7eab51a338160014 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Oct 2019 20:24:40 +0300 Subject: [PATCH 0990/2107] bugfix: duplicate returned by last_dir_separator() string for file->name --- src/dir.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index b9f4972cd..0738b3d74 100644 --- a/src/dir.c +++ b/src/dir.c @@ -202,7 +202,8 @@ pgFileInit(const char *path, const char *rel_path) canonicalize_path(file->rel_path); /* Get file name from the path */ - file_name = last_dir_separator(file->path); + file_name = pgut_strdup(last_dir_separator(file->path)); + if (file_name == NULL) file->name = file->path; else @@ -1675,14 +1676,14 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", path, + elog(ERROR, "Cannot open database map \"%s\": %s", path, strerror(errno)); print_database_map(fp, database_map); if (fio_fflush(fp) || fio_fclose(fp)) { fio_unlink(database_map_path, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write file list \"%s\": %s", + elog(ERROR, "Cannot write database map \"%s\": %s", database_map_path, strerror(errno)); } From 9edb2e34985be1bb97c7465318e6eeb30076362c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Oct 2019 20:39:16 +0300 Subject: [PATCH 0991/2107] minor improvement: for STREAM backups include wal_bytes into message about resident data size --- src/backup.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 32e810402..063776938 100644 --- a/src/backup.c +++ b/src/backup.c @@ -815,8 +815,11 @@ do_backup(time_t start_time, bool no_validate, pgBackupValidate(¤t, NULL); /* Notify user about backup size */ - pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); - elog(INFO, "Backup %s real size: %s", base36enc(current.start_time), pretty_data_bytes); + if (current.stream) + pretty_size(current.data_bytes + current.wal_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); + else + pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); + elog(INFO, "Backup %s resident size: %s", base36enc(current.start_time), pretty_data_bytes); if (current.status == BACKUP_STATUS_OK || current.status == BACKUP_STATUS_DONE) @@ -1828,6 +1831,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, backup_in_progress = false; +// char *target_lsn = "2/F578A000"; +// XLogDataFromLSN(target_lsn, &lsn_hi, &lsn_lo); + /* Extract timeline and LSN from results of pg_stop_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 2), &lsn_hi, &lsn_lo); /* Calculate LSN */ From 0deff069127e7c06780a9169bc74b74087a097c9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 8 Oct 2019 14:33:52 +0300 Subject: [PATCH 0992/2107] tests: minor fixes --- tests/archive.py | 8 ++------ tests/compatibility.py | 26 ++++++++++---------------- tests/delta.py | 25 +++++++++++-------------- tests/merge.py | 16 ++++++---------- tests/page.py | 30 ++++++++++++------------------ tests/restore.py | 39 +++++++++++++++------------------------ tests/retention.py | 11 +++++++++-- 7 files changed, 65 insertions(+), 90 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 626f4b671..438a0efeb 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -58,9 +58,7 @@ def test_pgpro434_1(self): node.cleanup() # Restore Database - self.restore_node( - backup_dir, 'node', node, - options=["--recovery-target-action=promote"]) + self.restore_node(backup_dir, 'node', node) node.slow_start() self.assertEqual( @@ -1590,11 +1588,9 @@ def test_archive_options(self): self.restore_node( backup_dir, 'node', node, options=[ - '--recovery-target-action=promote', '--archive-host=localhost', '--archive-port=22', - '--archive-user={0}'.format(self.user) - ]) + '--archive-user={0}'.format(self.user)]) with open(recovery_conf, 'r') as f: recovery_content = f.read() diff --git a/tests/compatibility.py b/tests/compatibility.py index 7c7038828..2375d780f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -20,8 +20,8 @@ def test_backward_compatibility_page(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) + self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -51,8 +51,7 @@ def test_backward_compatibility_page(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -76,8 +75,7 @@ def test_backward_compatibility_page(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -101,8 +99,7 @@ def test_backward_compatibility_page(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -122,8 +119,8 @@ def test_backward_compatibility_delta(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) + self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -153,8 +150,7 @@ def test_backward_compatibility_delta(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -178,8 +174,7 @@ def test_backward_compatibility_delta(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -203,8 +198,7 @@ def test_backward_compatibility_delta(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) diff --git a/tests/delta.py b/tests/delta.py index e0f0c9f7e..ecc927053 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -465,9 +465,9 @@ def test_delta_multiple_segments(self): tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') - self.restore_node(backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"]) + self.restore_node( + backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM NODE_RESTORED pgdata_restored = self.pgdata_content(restored_node.data_dir) @@ -835,8 +835,8 @@ def test_alter_table_set_tablespace_delta(self): "postgres", "create table t_heap tablespace somedata as select i as id," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) + " from generate_series(0,100) i") + # FULL backup self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -844,8 +844,7 @@ def test_alter_table_set_tablespace_delta(self): self.create_tblspace_in_node(node, 'somedata_new') node.safe_psql( "postgres", - "alter table t_heap set tablespace somedata_new" - ) + "alter table t_heap set tablespace somedata_new") # DELTA BACKUP result = node.safe_psql( @@ -853,15 +852,14 @@ def test_alter_table_set_tablespace_delta(self): self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=["--stream"] - ) + options=["--stream"]) + if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -875,8 +873,7 @@ def test_alter_table_set_tablespace_delta(self): "-T", "{0}={1}".format( self.get_tblspace_path(node, 'somedata_new'), self.get_tblspace_path(node_restored, 'somedata_new') - ), - "--recovery-target-action=promote" + ) ] ) @@ -1275,7 +1272,7 @@ def test_delta_nullified_heap_page_backup(self): content = f.read() self.assertIn( - "VERBOSE: File: {0} blknum 1, empty page".format(file), + 'VERBOSE: File: "{0}" blknum 1, empty page'.format(file), content) self.assertNotIn( "Skipping blknum 1 in file: {0}".format(file), diff --git a/tests/merge.py b/tests/merge.py index ddcf2e850..8a6f2981e 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -642,9 +642,8 @@ def test_merge_page_truncate(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -694,8 +693,7 @@ def test_merge_page_truncate(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"]) + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) # Physical comparison if self.paranoia: @@ -735,9 +733,8 @@ def test_merge_delta_truncate(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -787,8 +784,7 @@ def test_merge_delta_truncate(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"]) + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) # Physical comparison if self.paranoia: diff --git a/tests/page.py b/tests/page.py index d064eb744..60eab0ac0 100644 --- a/tests/page.py +++ b/tests/page.py @@ -28,9 +28,8 @@ def test_basic_page_vacuum_truncate(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -78,8 +77,7 @@ def test_basic_page_vacuum_truncate(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"]) + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) # Physical comparison if self.paranoia: @@ -302,9 +300,7 @@ def test_page_multiple_segments(self): 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', 'autovacuum': 'off', - 'full_page_writes': 'off' - } - ) + 'full_page_writes': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -343,7 +339,6 @@ def test_page_multiple_segments(self): backup_dir, 'node', restored_node, options=[ "-j", "4", - "--recovery-target-action=promote", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM NODE_RESTORED @@ -958,15 +953,14 @@ def test_page_backup_with_alien_wal_segment(self): "Output: {0} \n CMD: {1}".format( self.output, self.cmd)) except ProbackupException as e: - self.assertTrue( - 'INFO: Wait for LSN' in e.message and - 'in archived WAL segment' in e.message and - 'Could not read WAL record at' in e.message and - 'WAL file is from different database system: WAL file database system identifier is' in e.message and - 'pg_control database system identifier is' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.assertIn('INFO: Wait for WAL segment', e.message) + self.assertIn('to be archived', e.message) + self.assertIn('Could not read WAL record at', e.message) + self.assertIn('WAL file is from different database system: ' + 'WAL file database system identifier is', e.message) + self.assertIn('pg_control database system identifier is', e.message) + self.assertIn('Possible WAL corruption. Error has occured ' + 'during reading WAL segment', e.message) self.assertEqual( 'ERROR', diff --git a/tests/restore.py b/tests/restore.py index bf3e028f7..e457f2be1 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -47,8 +47,7 @@ def test_restore_full_to_latest(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -98,8 +97,7 @@ def test_restore_full_page_to_latest(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -139,8 +137,7 @@ def test_restore_to_specific_timeline(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -162,8 +159,7 @@ def test_restore_to_specific_timeline(self): self.restore_node( backup_dir, 'node', node, options=[ - "-j", "4", "--timeline={0}".format(target_tli), - "--recovery-target-action=promote"] + "-j", "4", "--timeline={0}".format(target_tli)] ), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -883,8 +879,7 @@ def test_restore_with_tablespace_mapping_1(self): self.restore_node( backup_dir, 'node', node, options=[ - "-T", "%s=%s" % (tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"] + "-T", "%s=%s" % (tblspc_path, tblspc_path_new)] ), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -915,8 +910,7 @@ def test_restore_with_tablespace_mapping_1(self): self.restore_node( backup_dir, 'node', node, options=[ - "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page), - "--recovery-target-action=promote"]), + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -991,8 +985,7 @@ def test_restore_with_tablespace_mapping_2(self): self.restore_node( backup_dir, 'node', node, options=[ - "-T", "%s=%s" % (tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"]), + "-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() @@ -1804,25 +1797,23 @@ def test_restore_target_immediate_stream(self): recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - # restore page backup + # restore delta backup node.cleanup() self.restore_node( backup_dir, 'node', node, options=['--immediate']) - # For stream backup with immediate recovery target there is no need to - # create recovery.conf. Is it wise? - self.assertFalse( - os.path.isfile(recovery_conf)) + self.assertTrue( + os.path.isfile(recovery_conf), + "File {0} do not exists".format(recovery_conf)) - # restore page backup + # restore delta backup node.cleanup() self.restore_node( backup_dir, 'node', node, options=['--recovery-target=immediate']) - # For stream backup with immediate recovery target there is no need to - # create recovery.conf. Is it wise? - self.assertFalse( - os.path.isfile(recovery_conf)) + self.assertTrue( + os.path.isfile(recovery_conf), + "File {0} do not exists".format(recovery_conf)) # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/retention.py b/tests/retention.py index b3bdfc849..ba986f9d4 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -60,8 +60,15 @@ def test_retention_redundancy_1(self): for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): if not wal_name.endswith(".backup"): - self.assertTrue(wal_name[8:] >= min_wal) - self.assertTrue(wal_name[8:] <= max_wal) + + if self.archive_compress: + wal_name = wal_name[-19:] + wal_name = wal_name[:-3] + else: + wal_name = wal_name[-16:] + + self.assertTrue(wal_name >= min_wal) + self.assertTrue(wal_name <= max_wal) # Clean after yourself self.del_test_dir(module_name, fname) From 51f4e38d70d1576fca125ba64e1b2cd4667b93b9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 8 Oct 2019 14:34:56 +0300 Subject: [PATCH 0993/2107] new TODO --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index 063776938..ab077aba0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -750,6 +750,9 @@ do_backup(time_t start_time, bool no_validate, if (current.from_replica) elog(INFO, "Backup %s is going to be taken from standby", base36enc(start_time)); + /* TODO, print PostgreSQL full version */ + //elog(INFO, "PostgreSQL version: %s", nodeInfo.server_version_str); + /* * Ensure that backup directory was initialized for the same PostgreSQL * instance we opened connection to. And that target backup database PGDATA From 1d8a654726fa114fd7c7011b77ebe009dc8b9601 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 8 Oct 2019 17:47:41 +0300 Subject: [PATCH 0994/2107] [Issue #127] WAL retention --- src/backup.c | 2 - src/catalog.c | 343 ++++++++++++ src/configure.c | 47 +- src/delete.c | 67 ++- src/pg_probackup.h | 9 +- src/util.c | 2 - tests/delete.py | 5 +- tests/helpers/ptrack_helpers.py | 2 + tests/retention.py | 941 +++++++++++++++++++++++++++++++- 9 files changed, 1379 insertions(+), 39 deletions(-) diff --git a/src/backup.c b/src/backup.c index ab077aba0..e9fe3dfc2 100644 --- a/src/backup.c +++ b/src/backup.c @@ -183,8 +183,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) } /* Obtain current timeline */ - current.tli = get_current_timeline(backup_conn); - #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); #else diff --git a/src/catalog.c b/src/catalog.c index 006b7e9ba..b42cc60f0 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -36,6 +36,8 @@ timelineInfoNew(TimeLineID tli) tlinfo->switchpoint = InvalidXLogRecPtr; tlinfo->parent_link = NULL; tlinfo->xlog_filelist = parray_new(); + tlinfo->anchor_lsn = InvalidXLogRecPtr; + tlinfo->anchor_tli = 0; return tlinfo; } @@ -746,6 +748,7 @@ catalog_get_timelines(InstanceConfig *instance) wal_file->file = *file; wal_file->segno = segno; wal_file->type = BACKUP_HISTORY_FILE; + wal_file->keep = false; parray_append(tlinfo->xlog_filelist, wal_file); continue; } @@ -765,6 +768,7 @@ catalog_get_timelines(InstanceConfig *instance) wal_file->file = *file; wal_file->segno = segno; wal_file->type = PARTIAL_SEGMENT; + wal_file->keep = false; parray_append(tlinfo->xlog_filelist, wal_file); continue; } @@ -826,6 +830,7 @@ catalog_get_timelines(InstanceConfig *instance) wal_file->file = *file; wal_file->segno = segno; wal_file->type = SEGMENT; + wal_file->keep = false; parray_append(tlinfo->xlog_filelist, wal_file); } /* timeline history file */ @@ -895,6 +900,344 @@ catalog_get_timelines(InstanceConfig *instance) tlinfo->closest_backup = get_closest_backup(tlinfo); } + /* determine which WAL segments must be kept because of wal retention */ + if (instance->wal_depth <= 0) + return timelineinfos; + + /* + * WAL retention for now is fairly simple. + * User can set only one parameter - 'wal-depth'. + * It determines how many latest valid(!) backups on timeline + * must have an ability to perform PITR: + * Consider the example: + * + * ---B1-------B2-------B3-------B4--------> WAL timeline1 + * + * If 'wal-depth' is set to 2, then WAL purge should produce the following result: + * + * B1 B2 B3-------B4--------> WAL timeline1 + * + * Only valid backup can satisfy 'wal-depth' condition, so if B3 is not OK or DONE, + * then WAL purge should produce the following result: + * B1 B2-------B3-------B4--------> WAL timeline1 + * + * Complicated cases, such as branched timelines are taken into account. + * wal-depth is applied to each timeline independently: + * + * |---------> WAL timeline2 + * ---B1---|---B2-------B3-------B4--------> WAL timeline1 + * + * after WAL purge with wal-depth=2: + * + * |---------> WAL timeline2 + * B1---| B2 B3-------B4--------> WAL timeline1 + * + * In this example WAL retention prevents purge of WAL required by tli2 + * to stay reachable from backup B on tli1. + * + * To protect WAL from purge we try to set 'anchor_lsn' and 'anchor_tli' in every timeline. + * They are usually comes from 'start-lsn' and 'tli' attributes of backup + * calculated by 'wal-depth' parameter. + * With 'wal-depth=2' anchor_backup in tli1 is B3. + + * If timeline has not enough valid backups to satisfy 'wal-depth' condition, + * then 'anchor_lsn' and 'anchor_tli' taken from from 'start-lsn' and 'tli + * attribute of closest_backup. + * The interval of WAL starting from closest_backup to switchpoint is + * saved into 'keep_segments' attribute. + * If there is several intermediate timelines between timeline and its closest_backup + * then on every intermediate timeline WAL interval between switchpoint + * and starting segment is placed in 'keep_segments' attributes: + * + * |---------> WAL timeline3 + * |------| B5-----B6--> WAL timeline2 + * B1---| B2 B3-------B4------------> WAL timeline1 + * + * On timeline where closest_backup is located the WAL interval between + * closest_backup and switchpoint is placed into 'keep_segments'. + * If timeline has no 'closest_backup', then 'wal-depth' rules cannot be applied + * to this timeline and its WAL must be purged by following the basic rules of WAL purging. + * + * Third part is handling of ARCHIVE backups. + * If B1 and B2 have ARCHIVE wal-mode, then we must preserve WAL intervals + * between start_lsn and stop_lsn for each of them in 'keep_segments'. + */ + + /* determine anchor_lsn and keep_segments for every timeline */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + int count = 0; + timelineInfo *tlinfo = parray_get(timelineinfos, i); + + /* + * Iterate backward on backups belonging to this timeline to find + * anchor_backup. NOTE Here we rely on the fact that backups list + * is ordered by start_lsn DESC. + */ + if (tlinfo->backups) + { + for (int j = 0; j < parray_num(tlinfo->backups); j++) + { + pgBackup *backup = parray_get(tlinfo->backups, j); + + /* skip invalid backups */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + continue; + + /* sanity */ + if (XLogRecPtrIsInvalid(backup->start_lsn) || + backup->tli <= 0) + continue; + + count++; + + if (count == instance->wal_depth) + { + elog(LOG, "On timeline %i WAL is protected from purge at %X/%X", + tlinfo->tli, + (uint32) (backup->start_lsn >> 32), + (uint32) (backup->start_lsn)); + + tlinfo->anchor_lsn = backup->start_lsn; + tlinfo->anchor_tli = backup->tli; + break; + } + } + } + + /* + * Failed to find anchor backup for this timeline. + * We cannot just thrown it to the wolves, because by + * doing that we will violate our own guarantees. + * So check the existence of closest_backup for + * this timeline. If there is one, then + * set the 'anchor_lsn' and 'anchor_tli' to closest_backup + * 'start-lsn' and 'tli' respectively. + * |-------------B5----------> WAL timeline3 + * |-----|-------------------------> WAL timeline2 + * B1 B2---| B3 B4-------B6-----> WAL timeline1 + * + * wal-depth=2 + * + * If number of valid backups on timelines is less than 'wal-depth' + * then timeline must(!) stay reachable via parent timelines if any. + * If closest_backup is not available, then general WAL purge rules + * are applied. + */ + if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) + { + /* + * Failed to find anchor_lsn in our own timeline. + * Consider the case: + * -------------------------------------> tli5 + * ----------------------------B4-------> tli4 + * S3`--------------> tli3 + * S1`------------S3---B3-------B6-> tli2 + * B1---S1-------------B2--------B5-----> tli1 + * + * B* - backups + * S* - switchpoints + * wal-depth=2 + * + * Expected result: + * TLI5 will be purged entirely + * B4-------> tli4 + * S2`--------------> tli3 + * S1`------------S2 B3-------B6-> tli2 + * B1---S1 B2--------B5-----> tli1 + */ + pgBackup *closest_backup = NULL; + xlogInterval *interval = NULL; + TimeLineID tli = 0; + /* check if tli has closest_backup */ + if (!tlinfo->closest_backup) + /* timeline has no closest_backup, wal retention cannot be + * applied to this timeline. + * Timeline will be purged up to oldest_backup if any or + * purge entirely if there is none. + * In example above: tli5 and tli4. + */ + continue; + + /* sanity for closest_backup */ + if (XLogRecPtrIsInvalid(tlinfo->closest_backup->start_lsn) || + tlinfo->closest_backup->tli <= 0) + continue; + + /* + * Set anchor_lsn and anchor_tli to protect whole timeline from purge + * In the example above: tli3. + */ + tlinfo->anchor_lsn = tlinfo->closest_backup->start_lsn; + tlinfo->anchor_tli = tlinfo->closest_backup->tli; + + /* closest backup may be located not in parent timeline */ + closest_backup = tlinfo->closest_backup; + + tli = tlinfo->tli; + + /* + * Iterate over parent timeline chain and + * look for timeline where closest_backup belong + */ + while (tlinfo->parent_link) + { + /* In case of intermediate timeline save to keep_segments + * begin_segno and switchpoint segment. + * In case of final timelines save to keep_segments + * closest_backup start_lsn segment and switchpoint segment. + */ + XLogRecPtr switchpoint = tlinfo->switchpoint; + + tlinfo = tlinfo->parent_link; + + if (tlinfo->keep_segments == NULL) + tlinfo->keep_segments = parray_new(); + + /* in any case, switchpoint segment must be added to interval */ + interval = palloc(sizeof(xlogInterval)); + GetXLogSegNo(switchpoint, interval->end_segno, instance->xlog_seg_size); + + /* Save [S1`, S2] to keep_segments */ + if (tlinfo->tli != closest_backup->tli) + interval->begin_segno = tlinfo->begin_segno; + /* Save [B1, S1] to keep_segments */ + else + GetXLogSegNo(closest_backup->start_lsn, interval->begin_segno, instance->xlog_seg_size); + + /* + * TODO: check, maybe this interval is already here or + * covered by other larger interval. + */ + + elog(LOG, "Timeline %i to stay reachable from timeline %i " + "protect from purge WAL interval between " + "%08X%08X and %08X%08X on timeline %i", + tli, closest_backup->tli, + (uint32) interval->begin_segno / instance->xlog_seg_size, + (uint32) interval->begin_segno % instance->xlog_seg_size, + (uint32) interval->end_segno / instance->xlog_seg_size, + (uint32) interval->end_segno % instance->xlog_seg_size, + tlinfo->tli); + parray_append(tlinfo->keep_segments, interval); + continue; + } + continue; + } + + /* Iterate over backups left */ + for (int j = count; j < parray_num(tlinfo->backups); j++) + { + XLogSegNo segno = 0; + xlogInterval *interval = NULL; + pgBackup *backup = parray_get(tlinfo->backups, j); + + /* + * We must calculate keep_segments intervals for ARCHIVE backups + * with start_lsn less than anchor_lsn. + */ + + /* STREAM backups cannot contribute to keep_segments */ + if (backup->stream) + continue; + + /* sanity */ + if (XLogRecPtrIsInvalid(backup->start_lsn) || + backup->tli <= 0) + continue; + + /* no point in clogging keep_segments by backups protected by anchor_lsn */ + if (backup->start_lsn >= tlinfo->anchor_lsn) + continue; + + /* append interval to keep_segments */ + interval = palloc(sizeof(xlogInterval)); + GetXLogSegNo(backup->start_lsn, segno, instance->xlog_seg_size); + interval->begin_segno = segno; + GetXLogSegNo(backup->stop_lsn, segno, instance->xlog_seg_size); + + /* + * On replica it is possible to get STOP_LSN pointing to contrecord, + * so set end_segno to the next segment after STOP_LSN just to be safe. + */ + if (backup->from_replica) + interval->end_segno = segno + 1; + else + interval->end_segno = segno; + + elog(LOG, "Archive backup %s to stay consistent " + "protect from purge WAL interval " + "between %08X%08X and %08X%08X on timeline %i", + base36enc(backup->start_time), + (uint32) interval->begin_segno / instance->xlog_seg_size, + (uint32) interval->begin_segno % instance->xlog_seg_size, + (uint32) interval->end_segno / instance->xlog_seg_size, + (uint32) interval->end_segno % instance->xlog_seg_size, + backup->tli); + + if (tlinfo->keep_segments == NULL) + tlinfo->keep_segments = parray_new(); + + parray_append(tlinfo->keep_segments, interval); + } + } + + /* + * Protect WAL segments from deletion by setting 'keep' flag. + * We must keep all WAL segments after anchor_lsn (including), and also segments + * required by ARCHIVE backups for consistency - WAL between [start_lsn, stop_lsn]. + */ + for (int i = 0; i < parray_num(timelineinfos); i++) + { + XLogSegNo anchor_segno = 0; + timelineInfo *tlinfo = parray_get(timelineinfos, i); + + /* + * At this point invalid anchor_lsn can be only in one case: + * timeline is going to be purged by regular WAL purge rules. + */ + if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) + continue; + + /* + * anchor_lsn is located in another timeline, it means that the timeline + * will be protected from purge entirely. + */ + if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) + continue; + + GetXLogSegNo(tlinfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); + + for (int i = 0; i < parray_num(tlinfo->xlog_filelist); i++) + { + xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); + + if (wal_file->segno >= anchor_segno) + { + wal_file->keep = true; + continue; + } + + /* no keep segments */ + if (!tlinfo->keep_segments) + continue; + + /* Protect segments belonging to one of the keep invervals */ + for (int j = 0; j < parray_num(tlinfo->keep_segments); j++) + { + xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, j); + + if ((wal_file->segno >= keep_segments->begin_segno) && + wal_file->segno <= keep_segments->end_segno) + { + wal_file->keep = true; + break; + } + } + } + } + return timelineinfos; } diff --git a/src/configure.c b/src/configure.c index 19d0c0af4..1aae3df13 100644 --- a/src/configure.c +++ b/src/configure.c @@ -189,50 +189,55 @@ ConfigOption instance_options[] = &instance_config.retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, + { + 'u', 221, "wal-depth", + &instance_config.wal_depth, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, /* Compression options */ { - 'f', 221, "compress-algorithm", + 'f', 222, "compress-algorithm", assign_compress_alg, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { - 'u', 222, "compress-level", + 'u', 223, "compress-level", &instance_config.compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 223, "remote-proto", + 's', 224, "remote-proto", &instance_config.remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "remote-host", + 's', 225, "remote-host", &instance_config.remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-port", + 's', 226, "remote-port", &instance_config.remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-path", + 's', 227, "remote-path", &instance_config.remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-user", + 's', 228, "remote-user", &instance_config.remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "ssh-options", + 's', 229, "ssh-options", &instance_config.remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-config", + 's', 230, "ssh-config", &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, @@ -357,6 +362,7 @@ init_config(InstanceConfig *config, const char *instance_name) config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; config->retention_window = RETENTION_WINDOW_DEFAULT; + config->wal_depth = 0; config->compress_alg = COMPRESS_ALG_DEFAULT; config->compress_level = COMPRESS_LEVEL_DEFAULT; @@ -530,50 +536,55 @@ readInstanceConfigFile(const char *instance_name) &instance->retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, + { + 'u', 221, "wal-depth", + &instance->wal_depth, SOURCE_CMD, 0, + OPTION_RETENTION_GROUP, 0, option_get_value + }, /* Compression options */ { - 's', 221, "compress-algorithm", + 's', 222, "compress-algorithm", &compress_alg, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'u', 222, "compress-level", + 'u', 223, "compress-level", &instance->compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 223, "remote-proto", + 's', 224, "remote-proto", &instance->remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 224, "remote-host", + 's', 225, "remote-host", &instance->remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-port", + 's', 226, "remote-port", &instance->remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-path", + 's', 227, "remote-path", &instance->remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-user", + 's', 228, "remote-user", &instance->remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "ssh-options", + 's', 229, "ssh-options", &instance->remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-config", + 's', 230, "ssh-config", &instance->remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, diff --git a/src/delete.c b/src/delete.c index e361e57ee..d5a4615b9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -35,8 +35,6 @@ do_delete(time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; -// XLogRecPtr oldest_lsn = InvalidXLogRecPtr; -// TimeLineID oldest_tli = 0; /* Get complete list of backups */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -170,6 +168,9 @@ int do_retention(void) else elog(INFO, "There are no backups to delete by retention policy"); + if (!wal_deleted) + elog(INFO, "There is no WAL to purge by retention policy"); + /* Cleanup */ parray_walk(backup_list, pgBackupFree); parray_free(backup_list); @@ -655,8 +656,14 @@ do_retention_wal(bool dry_run) /* * If closest backup exists, then timeline is reachable from * at least one backup and no file should be removed. + * Unless wal-depth is enabled. */ - if (tlinfo->closest_backup) + if ((tlinfo->closest_backup) && instance_config.wal_depth <= 0) + continue; + + /* WAL retention keeps this timeline from purge */ + if (instance_config.wal_depth >= 0 && tlinfo->anchor_tli > 0 && + tlinfo->anchor_tli != tlinfo->tli) continue; /* @@ -665,13 +672,31 @@ do_retention_wal(bool dry_run) * can be safely purged. * Note, that oldest_backup is not necessarily valid here, * but still we keep wal for it. + * If wal-depth is enabled then use anchor_lsn instead + * of oldest_backup. */ if (tlinfo->oldest_backup) - delete_walfiles_in_tli(tlinfo->oldest_backup->start_lsn, - tlinfo, instance_config.xlog_seg_size, dry_run); + { + if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + { + delete_walfiles_in_tli(tlinfo->anchor_lsn, + tlinfo, instance_config.xlog_seg_size, dry_run); + } + else + { + delete_walfiles_in_tli(tlinfo->oldest_backup->start_lsn, + tlinfo, instance_config.xlog_seg_size, dry_run); + } + } else - delete_walfiles_in_tli(InvalidXLogRecPtr, - tlinfo, instance_config.xlog_seg_size, dry_run); + { + if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + delete_walfiles_in_tli(tlinfo->anchor_lsn, + tlinfo, instance_config.xlog_seg_size, dry_run); + else + delete_walfiles_in_tli(InvalidXLogRecPtr, + tlinfo, instance_config.xlog_seg_size, dry_run); + } } } @@ -772,6 +797,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, char wal_pretty_size[20]; bool purge_all = false; + /* Timeline is completely empty */ if (parray_num(tlinfo->xlog_filelist) == 0) { @@ -782,8 +808,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (XLogRecPtrIsInvalid(keep_lsn)) { /* Drop all files in timeline */ - elog(INFO, "All files on timeline %i %s be removed", tlinfo->tli, - dry_run?"can":"will"); + elog(INFO, "On timeline %i all files %s be removed", + tlinfo->tli, dry_run?"can":"will"); FirstToDeleteSegNo = tlinfo->begin_segno; OldestToKeepSegNo = tlinfo->end_segno; purge_all = true; @@ -796,15 +822,23 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, } if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo) - elog(INFO, "WAL segments between %08X%08X and %08X%08X on timeline %i %s be removed", - (uint32) FirstToDeleteSegNo / xlog_seg_size, (uint32) FirstToDeleteSegNo % xlog_seg_size, + elog(INFO, "On timeline %i WAL segments between %08X%08X and %08X%08X %s be removed", + tlinfo->tli, + (uint32) FirstToDeleteSegNo / xlog_seg_size, + (uint32) FirstToDeleteSegNo % xlog_seg_size, (uint32) (OldestToKeepSegNo - 1) / xlog_seg_size, (uint32) (OldestToKeepSegNo - 1) % xlog_seg_size, - tlinfo->tli, dry_run?"can":"will"); + dry_run?"can":"will"); /* sanity */ if (OldestToKeepSegNo > FirstToDeleteSegNo) + { wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; + + /* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */ + if (purge_all) + wal_size_logical += xlog_seg_size; + } else if (OldestToKeepSegNo < FirstToDeleteSegNo) { /* It is actually possible for OldestToKeepSegNo to be less than FirstToDeleteSegNo @@ -851,7 +885,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (wal_size_actual > 0) { pretty_size(wal_size_actual, wal_pretty_size, lengthof(wal_pretty_size)); - elog(INFO, "Resident data size to free on timeline %i : %s", + elog(INFO, "Resident WAL size to free on timeline %i : %s", tlinfo->tli, wal_pretty_size); } @@ -870,6 +904,13 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, */ if (purge_all || wal_file->segno < OldestToKeepSegNo) { + /* save segment from purging */ + if (instance_config.wal_depth >= 0 && wal_file->keep) + { + elog(VERBOSE, "Retain WAL segment \"%s\"", wal_file->file.path); + continue; + } + /* unlink segment */ rc = unlink(wal_file->file.path); if (rc < 0) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a08f77df2..d4a2e18c7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -267,6 +267,7 @@ typedef struct InstanceConfig /* Retention options. 0 disables the option. */ uint32 retention_redundancy; uint32 retention_window; + uint32 wal_depth; CompressAlg compress_alg; int compress_level; @@ -446,8 +447,11 @@ struct timelineInfo { parray *xlog_filelist; /* array of ordinary WAL segments, '.partial' * and '.backup' files belonging to this timeline */ parray *lost_segments; /* array of intervals of lost segments */ - pgBackup *closest_backup; /* link to backup, closest to timeline */ + parray *keep_segments; /* array of intervals of segments used by WAL retention */ + pgBackup *closest_backup; /* link to valid backup, closest to timeline */ pgBackup *oldest_backup; /* link to oldest backup on timeline */ + XLogRecPtr anchor_lsn; /* LSN belonging to the oldest segno to keep for 'wal-depth' */ + TimeLineID anchor_tli; /* timeline of anchor_lsn */ }; typedef struct xlogInterval @@ -468,6 +472,9 @@ typedef struct xlogFile pgFile file; XLogSegNo segno; xlogFileType type; + bool keep; /* Used to prevent removal of WAL segments + * required by ARCHIVE backups. + */ } xlogFile; diff --git a/src/util.c b/src/util.c index e95051718..46240d295 100644 --- a/src/util.c +++ b/src/util.c @@ -193,8 +193,6 @@ get_current_timeline_from_control(bool safe) return ControlFile.checkPointCopy.ThisTimeLineID; } - - /* * Get last check point record ptr from pg_tonrol. */ diff --git a/tests/delete.py b/tests/delete.py index 61393a2d4..e6e739cbd 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -283,8 +283,9 @@ def test_delete_orphaned_wal_segments(self): result = self.delete_pb(backup_dir, 'node', options=['--wal']) # delete useless wals - self.assertTrue('WAL segments between ' in result - and 'on timeline 1 will be removed' in result) + self.assertTrue('On timeline 1 WAL segments between ' in result + and 'will be removed' in result) + self.validate_pb(backup_dir) self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 913781dbc..8b7db6a77 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1004,6 +1004,8 @@ def show_archive( if timeline['tli'] == tli: return timeline + return {} + if instance_timelines: return instance_timelines diff --git a/tests/retention.py b/tests/retention.py index ba986f9d4..aa9766c9c 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from time import sleep +from distutils.dir_util import copy_tree module_name = 'retention' @@ -1415,6 +1416,8 @@ def test_retention_redundancy_overlapping_chains(self): backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.validate_pb(backup_dir, 'node') + # Clean after yourself self.del_test_dir(module_name, fname) @@ -1461,6 +1464,8 @@ def test_retention_redundancy_overlapping_chains(self): backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.validate_pb(backup_dir, 'node') + # Clean after yourself self.del_test_dir(module_name, fname) @@ -1501,7 +1506,23 @@ def test_wal_purge_victim(self): sleep(1) # Make FULL backup - self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) + full_id = self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "INFO: Backup {0} WAL segments are valid".format(full_id), + e.message) + self.assertIn( + "WARNING: Backup {0} has missing parent 0".format(page_id), + e.message) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1601,3 +1622,921 @@ def test_failed_merge_redundancy_retention(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_wal_depth(self): + """ + ARCHIVE replica: + + t6 |----------------------> + t5 | |------> + | | + t4 | |----|------> + | | + t3 | |--B1--|/|--B2-|/|-B3--> + | | + t2 |--A1-----|--A2---> + t1 ---------Y1--Y2-| + + ARCHIVE master: + t1 -Z1--Z2--> + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + + master.slow_start() + + # FULL + master.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'master', master) + + # PAGE + master.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'master', master, backup_type='page') + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + # Check data correctness on replica + replica.slow_start(replica=True) + + # FULL backup replica + Y1 = self.backup_node( + backup_dir, 'replica', replica, + options=['--stream', '--archive-timeout=60s']) + + master.pgbench_init(scale=5) + + # PAGE backup replica + Y2 = self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream', '--archive-timeout=60s']) + + # create timeline t2 + replica.promote() + + # do checkpoint to increment timeline ID in pg_control + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + + # FULL backup replica + A1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=5) + + replica.safe_psql( + 'postgres', + "CREATE TABLE t1 (a text)") + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # DELTA backup replica + A2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='delta') + + # create timeline t3 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + replica.slow_start() + + B1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=2) + + B2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + B3 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + # create timeline t4 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't2 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't3 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,10) i') + + # create timeline t5 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=4', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't4 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + # create timeline t6 + replica.cleanup() + + self.restore_node( + backup_dir, 'replica', replica, backup_id=A1, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + replica.slow_start() + + replica.pgbench_init(scale=2) + + show = self.show_archive(backup_dir, as_text=True) + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + if instance['instance'] == 'master': + master_timelines = instance['timelines'] + + # check that all timelines are ok + for timeline in replica_timelines: + self.assertTrue(timeline['status'], 'OK') + + # check that all timelines are ok + for timeline in master_timelines: + self.assertTrue(timeline['status'], 'OK') + + # create holes in t3 + wals_dir = os.path.join(backup_dir, 'wal', 'replica') + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) + and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') + ] + wals.sort() + + # check that t3 is ok + self.show_archive(backup_dir) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + # check that t3 is not OK + show = self.show_archive(backup_dir) + + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + # sanity + for timeline in replica_timelines: + if timeline['tli'] == 1: + timeline_1 = timeline + continue + + if timeline['tli'] == 2: + timeline_2 = timeline + continue + + if timeline['tli'] == 3: + timeline_3 = timeline + continue + + if timeline['tli'] == 4: + timeline_4 = timeline + continue + + if timeline['tli'] == 5: + timeline_5 = timeline + continue + + if timeline['tli'] == 6: + timeline_6 = timeline + continue + + self.assertEqual(timeline_6['status'], "OK") + self.assertEqual(timeline_5['status'], "OK") + self.assertEqual(timeline_4['status'], "OK") + self.assertEqual(timeline_3['status'], "DEGRADED") + self.assertEqual(timeline_2['status'], "OK") + self.assertEqual(timeline_1['status'], "OK") + + self.assertEqual(len(timeline_3['lost-segments']), 2) + self.assertEqual(timeline_3['lost-segments'][0]['begin-segno'], '0000000000000012') + self.assertEqual(timeline_3['lost-segments'][0]['end-segno'], '0000000000000013') + self.assertEqual(timeline_3['lost-segments'][1]['begin-segno'], '0000000000000017') + self.assertEqual(timeline_3['lost-segments'][1]['end-segno'], '0000000000000017') + + self.assertEqual(len(timeline_6['backups']), 0) + self.assertEqual(len(timeline_5['backups']), 0) + self.assertEqual(len(timeline_4['backups']), 0) + self.assertEqual(len(timeline_3['backups']), 3) + self.assertEqual(len(timeline_2['backups']), 2) + self.assertEqual(len(timeline_1['backups']), 2) + + # check closest backup correctness + self.assertEqual(timeline_6['closest-backup-id'], A1) + self.assertEqual(timeline_5['closest-backup-id'], B2) + self.assertEqual(timeline_4['closest-backup-id'], B2) + self.assertEqual(timeline_3['closest-backup-id'], A1) + self.assertEqual(timeline_2['closest-backup-id'], Y2) + + # check parent tli correctness + self.assertEqual(timeline_6['parent-tli'], 2) + self.assertEqual(timeline_5['parent-tli'], 4) + self.assertEqual(timeline_4['parent-tli'], 3) + self.assertEqual(timeline_3['parent-tli'], 2) + self.assertEqual(timeline_2['parent-tli'], 1) + self.assertEqual(timeline_1['parent-tli'], 0) + + output = self.delete_pb( + backup_dir, 'replica', + options=['--delete-wal', '--log-level-console=verbose']) + + self.validate_pb(backup_dir, 'node') + + self.del_test_dir(module_name, fname) + + def test_wal_depth_1(self): + """ + |-------------B5----------> WAL timeline3 + |-----|-------------------------> WAL timeline2 + B1 B2---| B3 B4-------B6-----> WAL timeline1 + + wal-depth=2 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # FULL + node.pgbench_init(scale=1) + B1 = self.backup_node(backup_dir, 'node', node) + + # PAGE + node.pgbench_init(scale=1) + B2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # generate_some more data + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").rstrip() + + node.pgbench_init(scale=1) + + B3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.pgbench_init(scale=1) + + B4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Timeline 2 + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node_restored.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_restored, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-action=promote']) + + self.assertIn( + 'Restore of backup {0} completed'.format(B2), + output) + + self.set_auto_conf(node_restored, options={'port': node_restored.port}) + + node_restored.slow_start() + + node_restored.pgbench_init(scale=1) + + target_xid = node_restored.safe_psql( + "postgres", + "select txid_current()").rstrip() + + node_restored.pgbench_init(scale=2) + + # Timeline 3 + node_restored.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_restored, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'Restore of backup {0} completed'.format(B2), + output) + + self.set_auto_conf(node_restored, options={'port': node_restored.port}) + + node_restored.slow_start() + + node_restored.pgbench_init(scale=1) + B5 = self.backup_node( + backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) + + node.pgbench_init(scale=1) + B6 = self.backup_node(backup_dir, 'node', node) + + lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] + + self.validate_pb( + backup_dir, 'node', backup_id=B2, + options=['--recovery-target-lsn={0}'.format(lsn)]) + + self.validate_pb(backup_dir, 'node') + + self.del_test_dir(module_name, fname) + + def test_wal_purge(self): + """ + -------------------------------------> tli5 + ---------------------------B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + B* - backups + S* - switchpoints + + Expected result: + TLI5 will be purged entirely + B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + wal-depth=2 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # STREAM FULL + stream_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.stop() + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + B1 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").rstrip() + node.pgbench_init(scale=5) + + # B2 FULL on TLI1 + B2 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + B3 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + + # TLI 2 + node_tli2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli2')) + node_tli2.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_tli2, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + + self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.slow_start() + node_tli2.pgbench_init(scale=4) + + target_xid = node_tli2.safe_psql( + "postgres", + "select txid_current()").rstrip() + node_tli2.pgbench_init(scale=1) + + B4 = self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=3) + + B5 = self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=1) + node_tli2.cleanup() + + # TLI3 + node_tli3 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli3')) + node_tli3.cleanup() + + # Note, that successful validation here is a happy coincidence + output = self.restore_node( + backup_dir, 'node', node_tli3, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.slow_start() + node_tli3.pgbench_init(scale=5) + node_tli3.cleanup() + + # TLI4 + node_tli4 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli4')) + node_tli4.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli4, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) + self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.slow_start() + + node_tli4.pgbench_init(scale=5) + + B6 = self.backup_node( + backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + node_tli4.pgbench_init(scale=5) + node_tli4.cleanup() + + # TLI5 + node_tli5 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli5')) + node_tli5.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli5, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) + self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.slow_start() + node_tli5.pgbench_init(scale=10) + + # delete '.history' file of TLI4 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + # delete '.history' file of TLI5 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--delete-wal', '--dry-run', + '--log-level-console=verbose']) + + self.assertIn( + 'INFO: On timeline 4 WAL segments between 0000000000000002 ' + 'and 0000000000000005 can be removed', + output) + + self.assertIn( + 'INFO: On timeline 5 all files can be removed', + output) + + show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + + self.assertTrue(show_tli1_before) + self.assertTrue(show_tli2_before) + self.assertTrue(show_tli3_before) + self.assertTrue(show_tli4_before) + self.assertTrue(show_tli5_before) + + output = self.delete_pb( + backup_dir, 'node', + options=['--delete-wal', '--log-level-console=verbose']) + + self.assertIn( + 'INFO: On timeline 4 WAL segments between 0000000000000002 ' + 'and 0000000000000005 will be removed', + output) + + self.assertIn( + 'INFO: On timeline 5 all files will be removed', + output) + + show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + + self.assertEqual(show_tli1_before, show_tli1_after) + self.assertEqual(show_tli2_before, show_tli2_after) + self.assertEqual(show_tli3_before, show_tli3_after) + self.assertNotEqual(show_tli4_before, show_tli4_after) + self.assertNotEqual(show_tli5_before, show_tli5_after) + + self.assertEqual( + show_tli4_before['min-segno'], + '0000000000000002') + + self.assertEqual( + show_tli4_after['min-segno'], + '0000000000000006') + + self.assertFalse(show_tli5_after) + + self.validate_pb(backup_dir, 'node') + + self.del_test_dir(module_name, fname) + + def test_wal_depth_2(self): + """ + -------------------------------------> tli5 + ---------------------------B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + B* - backups + S* - switchpoints + wal-depth=2 + + Expected result: + TLI5 will be purged entirely + B6--------> tli4 + S2`---------------> tli3 + S1`------------S2 B4-------B5--> tli2 + B1---S1 B2--------B3------> tli1 + + wal-depth=2 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # STREAM FULL + stream_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.stop() + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + B1 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").rstrip() + node.pgbench_init(scale=5) + + # B2 FULL on TLI1 + B2 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + B3 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + + # TLI 2 + node_tli2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli2')) + node_tli2.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_tli2, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + + self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.slow_start() + node_tli2.pgbench_init(scale=4) + + target_xid = node_tli2.safe_psql( + "postgres", + "select txid_current()").rstrip() + node_tli2.pgbench_init(scale=1) + + B4 = self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=3) + + B5 = self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=1) + node_tli2.cleanup() + + # TLI3 + node_tli3 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli3')) + node_tli3.cleanup() + + # Note, that successful validation here is a happy coincidence + output = self.restore_node( + backup_dir, 'node', node_tli3, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.slow_start() + node_tli3.pgbench_init(scale=5) + node_tli3.cleanup() + + # TLI4 + node_tli4 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli4')) + node_tli4.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli4, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) + self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.slow_start() + + node_tli4.pgbench_init(scale=5) + + B6 = self.backup_node( + backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + node_tli4.pgbench_init(scale=5) + node_tli4.cleanup() + + # TLI5 + node_tli5 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_tli5')) + node_tli5.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli5, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) + self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.slow_start() + node_tli5.pgbench_init(scale=10) + + # delete '.history' file of TLI4 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + # delete '.history' file of TLI5 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--delete-wal', '--dry-run', + '--wal-depth=2', '--log-level-console=verbose']) + + start_lsn_B2 = self.show_pb(backup_dir, 'node', B2)['start-lsn'] + self.assertIn( + 'On timeline 1 WAL is protected from purge at {0}'.format(start_lsn_B2), + output) + + self.assertIn( + 'VERBOSE: Archive backup {0} to stay consistent protect from ' + 'purge WAL interval between 0000000000000004 and 0000000000000004 ' + 'on timeline 1'.format(B1), output) + + start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] + self.assertIn( + 'On timeline 2 WAL is protected from purge at {0}'.format(start_lsn_B4), + output) + + self.assertIn( + 'VERBOSE: Timeline 3 to stay reachable from timeline 1 protect ' + 'from purge WAL interval between 0000000000000005 and ' + '0000000000000008 on timeline 2', output) + + self.assertIn( + 'VERBOSE: Timeline 3 to stay reachable from timeline 1 protect ' + 'from purge WAL interval between 0000000000000004 and ' + '0000000000000005 on timeline 1', output) + + show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + + self.assertTrue(show_tli1_before) + self.assertTrue(show_tli2_before) + self.assertTrue(show_tli3_before) + self.assertTrue(show_tli4_before) + self.assertTrue(show_tli5_before) + + output = self.delete_pb( + backup_dir, 'node', + options=['--delete-wal', '--wal-depth=2', '--log-level-console=verbose']) + + show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + + self.assertNotEqual(show_tli1_before, show_tli1_after) + self.assertNotEqual(show_tli2_before, show_tli2_after) + self.assertEqual(show_tli3_before, show_tli3_after) + self.assertNotEqual(show_tli4_before, show_tli4_after) + self.assertNotEqual(show_tli5_before, show_tli5_after) + + self.assertEqual( + show_tli4_before['min-segno'], + '0000000000000002') + + self.assertEqual( + show_tli4_after['min-segno'], + '0000000000000006') + + self.assertFalse(show_tli5_after) + + self.assertTrue(show_tli1_after['lost-segments']) + self.assertTrue(show_tli2_after['lost-segments']) + self.assertFalse(show_tli3_after['lost-segments']) + self.assertFalse(show_tli4_after['lost-segments']) + self.assertFalse(show_tli5_after) + + self.assertEqual(len(show_tli1_after['lost-segments']), 1) + self.assertEqual(len(show_tli2_after['lost-segments']), 1) + + self.assertEqual( + show_tli1_after['lost-segments'][0]['begin-segno'], + '0000000000000006') + + self.assertEqual( + show_tli1_after['lost-segments'][0]['end-segno'], + '0000000000000009') + + self.assertEqual( + show_tli2_after['lost-segments'][0]['begin-segno'], + '0000000000000009') + + self.assertEqual( + show_tli2_after['lost-segments'][0]['end-segno'], + '0000000000000009') + + self.validate_pb(backup_dir, 'node') + + self.del_test_dir(module_name, fname) From 1534590bbab139c8c192c39b0dbd08c1a13cf69d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Oct 2019 00:00:05 +0300 Subject: [PATCH 0995/2107] help: new option '--wal-depth' --- src/help.c | 14 +++++++++++++- tests/expected/option_help.out | 19 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/help.c b/src/help.c index 2b25f4a3e..195d27b8f 100644 --- a/src/help.c +++ b/src/help.c @@ -93,6 +93,7 @@ help_pg_probackup(void) printf(_(" [--log-rotation-age=log-rotation-age]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--archive-timeout=timeout]\n")); @@ -128,6 +129,7 @@ help_pg_probackup(void) printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); @@ -186,6 +188,7 @@ help_pg_probackup(void) printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--delete-wal] [-i backup-id | --delete-expired | --merge-expired]\n")); printf(_(" [--dry-run]\n")); printf(_(" [--help]\n")); @@ -263,6 +266,7 @@ help_backup(void) printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); @@ -322,6 +326,8 @@ help_backup(void) printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); + printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Pinning options:\n")); @@ -591,7 +597,8 @@ help_delete(void) printf(_(" [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n")); printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n\n")); + printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -609,6 +616,8 @@ help_delete(void) printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); + printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Logging options:\n")); @@ -704,6 +713,7 @@ help_set_config(void) printf(_(" [--log-rotation-age=log-rotation-age]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); + printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--archive-timeout=timeout]\n")); @@ -746,6 +756,8 @@ help_set_config(void) printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); + printf(_(" --wal-depth=wal-depth number of latest valid backups with ability to perform\n")); + printf(_(" the point in time recovery; disables; (default: 0)\n")); printf(_("\n Compression options:\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 9b1970ee8..46595b5a6 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -19,6 +19,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--log-rotation-age=log-rotation-age] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] + [--wal-depth=wal-depth] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] [--archive-timeout=timeout] @@ -26,6 +27,12 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--restore-command=cmdline] [--archive-host=destination] + [--archive-port=port] [--archive-user=username] + [--help] + + pg_probackup set-backup -B backup-path --instance=instance_name + -i backup-id [--ttl] [--expire-time] [--help] pg_probackup show-config -B backup-path --instance=instance_name @@ -48,6 +55,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] + [--wal-depth=wal-depth] [--compress] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] @@ -57,6 +65,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--ttl] [--expire-time] [--help] pg_probackup restore -B backup-path --instance=instance_name @@ -71,11 +80,13 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--no-validate] [--skip-block-validation] [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] - [--skip-external-dirs] + [--skip-external-dirs] [--restore-command=cmdline] [--db-include | --db-exclude] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--archive-host=hostname] + [--archive-port=port] [--archive-user=username] [--help] pg_probackup validate -B backup-path [--instance=instance_name] @@ -99,7 +110,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--help] pg_probackup delete -B backup-path --instance=instance_name - [--wal] [-i backup-id | --expired | --merge-expired] + [-j num-threads] [--progress] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [--delete-wal] [-i backup-id | --delete-expired | --merge-expired] [--dry-run] [--help] From 23f7e11544d002f5bde4d2417fb3850adb7fd1ba Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Oct 2019 13:23:14 +0300 Subject: [PATCH 0996/2107] tests: minor fixes --- tests/helpers/ptrack_helpers.py | 4 +- tests/retention.py | 109 +++++++++++++++++++++++++++++++- tests/show.py | 106 ++++++++++++++++++++----------- 3 files changed, 178 insertions(+), 41 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 8b7db6a77..bc02ba7ac 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -770,7 +770,7 @@ def backup_node( self, backup_dir, instance, node, data_dir=False, backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, - old_binary=False, return_id=True + old_binary=False, return_id=True, no_remote=False ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -792,7 +792,7 @@ def backup_node( cmd_list += ['-D', data_dir] # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: + if self.remote and not old_binary and not no_remote: options = options + [ '--remote-proto=ssh', '--remote-host=localhost'] diff --git a/tests/retention.py b/tests/retention.py index aa9766c9c..c1331d37e 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2455,7 +2455,7 @@ def test_wal_depth_2(self): output) self.assertIn( - 'VERBOSE: Archive backup {0} to stay consistent protect from ' + 'LOG: Archive backup {0} to stay consistent protect from ' 'purge WAL interval between 0000000000000004 and 0000000000000004 ' 'on timeline 1'.format(B1), output) @@ -2465,12 +2465,12 @@ def test_wal_depth_2(self): output) self.assertIn( - 'VERBOSE: Timeline 3 to stay reachable from timeline 1 protect ' + 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' 'from purge WAL interval between 0000000000000005 and ' '0000000000000008 on timeline 2', output) self.assertIn( - 'VERBOSE: Timeline 3 to stay reachable from timeline 1 protect ' + 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' 'from purge WAL interval between 0000000000000004 and ' '0000000000000005 on timeline 1', output) @@ -2540,3 +2540,106 @@ def test_wal_depth_2(self): self.validate_pb(backup_dir, 'node') self.del_test_dir(module_name, fname) + + def test_basic_wal_depth(self): + """ + B1---B1----B3-----B4----B5------> tli1 + + Expected result with wal-depth=1: + B1 B1 B3 B4 B5------> tli1 + + wal-depth=1 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + node.pgbench_init(scale=1) + B1 = self.backup_node(backup_dir, 'node', node) + + + # B2 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B3 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B4 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B5 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B5 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--wal-depth=1', '--delete-wal']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").rstrip() + + self.switch_wal_segment(node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + tli1 = self.show_archive(backup_dir, 'node', tli=1) + + # check that there are 4 lost_segments intervals + self.assertEqual(len(tli1['lost-segments']), 4) + + output = self.validate_pb( + backup_dir, 'node', B5, + options=['--recovery-target-xid={0}'.format(target_xid)]) + + print(output) + + self.assertIn( + 'INFO: Backup validation completed successfully on time', + output) + + self.assertIn( + 'xid {0} and LSN'.format(target_xid), + output) + + for backup_id in [B1, B2, B3, B4]: + try: + self.validate_pb( + backup_dir, 'node', backup_id, + options=['--recovery-target-xid={0}'.format(target_xid)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Not enough WAL records to xid {0}".format(target_xid), + e.message) + + self.validate_pb(backup_dir, 'node') + + self.del_test_dir(module_name, fname) diff --git a/tests/show.py b/tests/show.py index ad9cafb73..0da95dcbb 100644 --- a/tests/show.py +++ b/tests/show.py @@ -228,14 +228,17 @@ def test_corrupt_correctness(self): # FULL backup_local_id = self.backup_node( - backup_dir, 'node', node, options=['--remote-proto=none']) + backup_dir, 'node', node, no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node(backup_dir, 'node', node) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -252,15 +255,19 @@ def test_corrupt_correctness(self): # DELTA backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--remote-proto=none']) + backup_type='delta', no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -278,15 +285,19 @@ def test_corrupt_correctness(self): # PAGE backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['--remote-proto=none']) + backup_type='page', no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -324,14 +335,17 @@ def test_corrupt_correctness_1(self): # FULL backup_local_id = self.backup_node( - backup_dir, 'node', node, options=['--remote-proto=none']) + backup_dir, 'node', node, no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node(backup_dir, 'node', node) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -352,15 +366,19 @@ def test_corrupt_correctness_1(self): # DELTA backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--remote-proto=none']) + backup_type='delta', no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -378,15 +396,19 @@ def test_corrupt_correctness_1(self): # PAGE backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['--remote-proto=none']) + backup_type='page', no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -425,14 +447,18 @@ def test_corrupt_correctness_2(self): # FULL backup_local_id = self.backup_node( backup_dir, 'node', node, - options=['--remote-proto=none', '--compress']) + options=['--compress'], no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -453,15 +479,19 @@ def test_corrupt_correctness_2(self): # DELTA backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--remote-proto=none', '--compress']) + backup_type='delta', options=['--compress'], no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -479,15 +509,19 @@ def test_corrupt_correctness_2(self): # PAGE backup_local_id = self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['--remote-proto=none', '--compress']) + backup_type='page', options=['--compress'], no_remote=True) output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) From 7c7faf7b5ec18ac79a9d68005bf46fff90bf2e72 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Oct 2019 21:07:47 +0300 Subject: [PATCH 0997/2107] Documentation: update --- Documentation.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Documentation.md b/Documentation.md index 442e155f5..1e3d1cfb0 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.1.5 +Current version - 2.2.0 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) @@ -36,9 +36,10 @@ Current version - 2.1.5 * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) * [Configuring pg_probackup](#configuring-pg_probackup) * [Managing the Backup Catalog](#managing-the-backup-catalog) + * [Viewing Backup Information](#viewing-backup-information) * [Viewing WAL Archive Information](#viewing-wal-archive-information) * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) - * [Pinning a Backup] + * [Pinning a Backup](#pinning-a-backup) * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -698,12 +699,13 @@ If nothing is given, the default values are taken. By default pg_probackup tries With pg_probackup, you can manage backups from the command line: -- View available backups -- View available WAL Archive Information -- Validate backups -- Merge backups -- Delete backups -- Viewing Backup Information +- [View backup information](#viewing-backup-information) +- [View WAL Archive Information](#viewing-wal-archive-information) +- [Validate backups](#validating-a-backup) +- [Merge backups](#merging-backups) +- [Delete backups](#deleting-backups) + +#### Viewing Backup Information To view the list of existing backups for every instance, run the command: From 5c4ad52e6de954000ae930713bae74d0cee7c710 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Oct 2019 21:10:02 +0300 Subject: [PATCH 0998/2107] Version 2.2.0 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d4a2e18c7..dcc3e52f7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -205,8 +205,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.1.5" -#define AGENT_PROTOCOL_VERSION 20105 +#define PROGRAM_VERSION "2.2.0" +#define AGENT_PROTOCOL_VERSION 20200 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 9c96acc51..c1ee94fa4 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.1.5 \ No newline at end of file +pg_probackup 2.2.0 \ No newline at end of file From f725966c3b181ffce0471f50dfd512d91db978b1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 15:07:04 +0300 Subject: [PATCH 0999/2107] minor cleanup before 2.2.0 release --- Documentation.md | 146 ++++++++++++++++++++++++++++++++++++++++++----- README.md | 2 +- src/help.c | 32 +++++------ 3 files changed, 148 insertions(+), 32 deletions(-) diff --git a/Documentation.md b/Documentation.md index 1e3d1cfb0..6c086da50 100644 --- a/Documentation.md +++ b/Documentation.md @@ -17,9 +17,9 @@ Current version - 2.2.0 * [Setting up Continuous WAL Archiving](#setting-up-continuous-wal-archiving) * [Setting up Backup from Standby](#setting-up-backup-from-standby) * [Setting up Cluster Verification](#setting-up-cluster-verification) - * [Setting up PTRACK Backups](#setting-up-ptrack-backups) * [Setting up Partial Restore](#setting-up-partial-restore) * [Configuring the Remote Mode](#configuring-the-remote-mode) + * [Setting up PTRACK Backups](#setting-up-ptrack-backups) 5. [Usage](#usage) * [Creating a Backup](#creating-a-backup) @@ -73,10 +73,13 @@ Current version - 2.2.0 * [Archiving Options](#archiving-options) * [Remote Mode Options](#remote-mode-options) * [Remote WAL Archive Options](#remote-wal-archive-options) + * [Partial Restore Options](#partial-restore-options) * [Replica Options](#replica-options) -7. [Authors](#authors) -8. [Credits](#credits) +7. [HOWTO](#howto) + * [Minimal setup](#minimal-setup) +8. [Authors](#authors) +9. [Credits](#credits) ## Synopsis @@ -135,7 +138,7 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Backup from replica: avoid extra load on the master server by taking backups from a standby - External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files - Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats and view WAL Archive information. -- Partial Restore: restore the only specified databases or skip the specified databases. +- Partial Restore: restore only the specified databases or skip the specified databases. To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -280,7 +283,7 @@ Since pg_probackup needs to read cluster files directly, pg_probackup must be st Depending on whether you are plan to take [autonomous](#stream-mode) and/or [archive](#archive-mode) backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server, run pg_probackup in remote mode or create PTRACK backups, additional setup is required. -For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#setting-up-backup-from-standby), [Configuring the Remote Mode](#configuring-the-remote-mode) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). +For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#setting-up-backup-from-standby), [Configuring the Remote Mode](#configuring-the-remote-mode), [Setting up Partial Restore](#setting-up-partial-restore) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). ### Setting up STREAM Backups @@ -356,19 +359,13 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; ``` -### Setting up PTRACK Backups - -Backup mode PTACK can be used only on Postgrespro Standart and Postgrespro Enterprise installations or patched vanilla PostgreSQL. Links to ptrack patches can be found [here](https://github.com/postgrespro/pg_probackup#ptrack-support). +### Setting up Partial Restore -If you are going to use PTRACK backups, complete the following additional steps: - -- Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute `ptrack` functions to the *backup* role **in every database** of the cluster: +If you are plalling to use partial restore, complete the following additional step: - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; +- Grant the read-only acces to 'pg_catalog.pg_database' to the *backup* role only in database **used for connection** to PostgreSQL server: -- The *backup* role must have access to all the databases of the cluster. + GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ### Configuring the Remote Mode @@ -408,6 +405,20 @@ pg_probackup in remote mode via `ssh` works as follows: >NOTE: You can improse [additional restrictions](https://man.openbsd.org/OpenBSD-current/man8/sshd.8#AUTHORIZED_KEYS_FILE_FORMAT) on ssh settings to protect the system in the event of account compromise. +### Setting up PTRACK Backups + +Backup mode PTACK can be used only on Postgrespro Standart and Postgrespro Enterprise installations or patched vanilla PostgreSQL. Links to ptrack patches can be found [here](https://github.com/postgrespro/pg_probackup#ptrack-support). + +If you are going to use PTRACK backups, complete the following additional steps: + +- Set the parameter `ptrack_enable` to `on`. +- Grant the rights to execute `ptrack` functions to the *backup* role **in every database** of the cluster: + + GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; + GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; + +- The *backup* role must have access to all the databases of the cluster. + ## Usage ### Creating a Backup @@ -1821,6 +1832,111 @@ Deprecated. User name to connect as. Default: 300 sec Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. +## Howto + +All exaples below assume the remote mode of operations via `ssh`. If you are planning to run backup and restore operation locally then step `Setup passwordless SSH connection` can be skipped and all `--remote-*` options can be ommited. + +Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. + +backup_host - host with backup catalog. +backupman - user on `backup_host` running all pg_probackup operations. +/mnt/backups - directory on `backup_host` where backup catalog is stored. + +postgres_host - host with PostgreSQL cluster. +postgres - user on `postgres_host` which run PostgreSQL cluster. +/var/lib/postgresql/11/main - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. +backup_db - database used for connection to PostgreSQL cluster. + +### Minimal Setup + +This setup is relying on autonomous FULL and DELTA backups. + +1. Setup passwordless SSH connection from `backup_host` to `postgres_host`: +``` +[backupman@backup_host] ssh-copy-id postgres@postgres_host +``` + +2. Setup PostgreSQL cluster. +2.1. It is recommended from security purposes to use separate database for backup operations. +``` +postgres=# +CREATE DATABASE backupdb; +``` + +2.2. Connect to `backupdb` database, create role `probackup` and grant to it the following permissions: +``` +backupdb=# +BEGIN; +CREATE ROLE probackup WITH LOGIN REPLICATION; +GRANT USAGE ON SCHEMA pg_catalog TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; +COMMIT; +``` + +3. Init backup catalog: +``` +[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups +INFO: Backup catalog '/mnt/backups' successfully inited +``` + +4. Add instance 'pg-11' to backup catalog: +``` +[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main +INFO: Instance 'node' successfully inited +``` + +5. Take FULL backup: +``` +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ4LFC, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ4LFC +INFO: Backup PZ4LFC data files are valid +INFO: Backup PZ4LFC resident size: 62MB +INFO: Backup PZ4LFC completed +``` + +6. Lets take a look at the backup catalog: +``` +BACKUP INSTANCE 'pg-11' +================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================== + node 11 PZ4LFC 2019-10-10 00:09:17+03 FULL STREAM 1/0 7s 30MB 32MB 1.00 0/C000028 0/C000160 OK + +``` + +7. Lets hide some options into config, so cmdline can be less crowdy: +``` +[backupman@backup_host]$ pg_probackup-11 set-config -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres --pguser=probackup --pgdatabase=backupdb +``` + +8. Take incremental backup in DELTA mode: +``` +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream +``` + + + +#### + +### Setup with WAL archive + +The fact of having an WAL archive dramatically improve you scope of options toward backup and restore operations. + + ## Authors Postgres Professional, Moscow, Russia. diff --git a/README.md b/README.md index 156abe5ee..ed13b491b 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup). Slightly outdated documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). -Documentation for current devel can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md) +Documentation for current devel can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). ## License diff --git a/src/help.c b/src/help.c index 195d27b8f..21f27d9fa 100644 --- a/src/help.c +++ b/src/help.c @@ -321,7 +321,7 @@ help_backup(void) printf(_(" retention policy after successful backup completion\n")); printf(_(" --merge-expired merge backups expired according to current\n")); printf(_(" retention policy after successful backup completion\n")); - printf(_(" --delete-wal remove redundant wal files in WAL archive\n")); + printf(_(" --delete-wal remove redundant files in WAL archive\n")); printf(_(" --retention-redundancy=retention-redundancy\n")); printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); @@ -344,13 +344,13 @@ help_backup(void) printf(_(" level of compression [0-9] (default: 1)\n")); printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_("\n Connection options:\n")); - printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); printf(_(" -w, --no-password never prompt for password\n")); printf(_(" -W, --password force password prompt\n")); @@ -568,10 +568,10 @@ help_checkdb(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); printf(_("\n Connection options:\n")); - printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); printf(_(" -w, --no-password never prompt for password\n")); printf(_(" -W, --password force password prompt\n\n")); } @@ -611,7 +611,7 @@ help_delete(void) printf(_(" retention policy\n")); printf(_(" --merge-expired merge backups expired according to current\n")); printf(_(" retention policy\n")); - printf(_(" --delete-wal remove redundant wal files in WAL archive\n")); + printf(_(" --delete-wal remove redundant files in WAL archive\n")); printf(_(" --retention-redundancy=retention-redundancy\n")); printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); printf(_(" --retention-window=retention-window\n")); @@ -767,13 +767,13 @@ help_set_config(void) printf(_(" level of compression [0-9] (default: 1)\n")); printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); printf(_("\n Connection options:\n")); - printf(_(" -U, --username=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --dbname=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --port=PORT database server port (default: 5432)\n")); + printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); From e643b819319ea83c6113ebb0e5612d6b60ee3b17 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 15:11:55 +0300 Subject: [PATCH 1000/2107] minor fix --- src/backup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index e9fe3dfc2..21e9502cf 100644 --- a/src/backup.c +++ b/src/backup.c @@ -717,7 +717,8 @@ do_backup(time_t start_time, bool no_validate, current.compress_level = instance_config.compress_level; /* Save list of external directories */ - if (instance_config.external_dir_str) + if (instance_config.external_dir_str && + (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) current.external_dir_str = instance_config.external_dir_str; elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " From bd5226cf5a3bd62266c5ccb9f4aad8eaca059e2a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 15:14:50 +0300 Subject: [PATCH 1001/2107] minor fix --- src/utils/configuration.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index bf5cd0140..beca4fccb 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -252,12 +252,11 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src) free(*(char **) opt->var); /* 'none' and 'off' are always disable the string parameter */ - if ((pg_strcasecmp(optarg, "none") == 0) || - (pg_strcasecmp(optarg, "off") == 0)) - { - opt->var = NULL; - return; - } + //if (optarg && (pg_strcasecmp(optarg, "none") == 0)) + //{ + // *(char **) opt->var = "none"; + // return; + //} *(char **) opt->var = pgut_strdup(optarg); if (strcmp(optarg,"") != 0) @@ -666,9 +665,9 @@ option_get_value(ConfigOption *opt) if (*((char **) opt->var) == NULL) return NULL; /* 'none' and 'off' are always disable the string parameter */ - if ((pg_strcasecmp(*((char **) opt->var), "none") == 0) || - (pg_strcasecmp(*((char **) opt->var), "off") == 0)) - return NULL; + //if ((pg_strcasecmp(*((char **) opt->var), "none") == 0) || + // (pg_strcasecmp(*((char **) opt->var), "off") == 0)) + // return NULL; return pstrdup(*((char **) opt->var)); case 't': { From 4a5a9efe016dd183d9322cab9ed81eeb2bb1b3eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 15:47:35 +0300 Subject: [PATCH 1002/2107] honot the special value 'none' for restore_command --- src/restore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 8e3a78a05..4d8029eea 100644 --- a/src/restore.c +++ b/src/restore.c @@ -871,7 +871,8 @@ create_recovery_conf(time_t backup_id, char restore_command_guc[16384]; /* If restore_command is provided, use it */ - if (instance_config.restore_command) + if (instance_config.restore_command && + (pg_strcasecmp(instance_config.restore_command, "none") != 0)) sprintf(restore_command_guc, "%s", instance_config.restore_command); else { From 2f9dd3658b332fd92d598fc52d49ec4dfd1d20c1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 16:04:34 +0300 Subject: [PATCH 1003/2107] tests: minor fix --- tests/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index ee7811d40..4c1ee06aa 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2026,6 +2026,8 @@ def test_backup_with_less_privileges_role(self): bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] gdb_checkpointer = self.gdb_attach(bgwriter_pid) + self.switch_wal_segment(node) + # FULL backup from replica self.backup_node( backup_dir, 'replica', replica, From 81339c71e64048ca90d81834b3fbc3e5ed6a1d05 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 19:03:21 +0300 Subject: [PATCH 1004/2107] tests: fixes for backup.BackupTest.test_backup_with_less_privileges_role --- tests/backup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 4c1ee06aa..57bebb476 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1906,7 +1906,7 @@ def test_backup_with_less_privileges_role(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '1h'}) if self.ptrack: node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') @@ -2016,16 +2016,17 @@ def test_backup_with_less_privileges_role(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + copy_tree( os.path.join(backup_dir, 'wal', 'node'), os.path.join(backup_dir, 'wal', 'replica')) replica.slow_start(replica=True) - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) - + self.switch_wal_segment(node) self.switch_wal_segment(node) # FULL backup from replica @@ -2037,12 +2038,12 @@ def test_backup_with_less_privileges_role(self): self.backup_node( backup_dir, 'replica', replica, datname='backupdb', - options=['-U', 'backup', '--log-level-file=verbose', '--archive-timeout=100s']) + options=['-U', 'backup', '--log-level-file=verbose', '--archive-timeout=30s']) # PAGE backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=100s']) + datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) self.backup_node( backup_dir, 'replica', replica, backup_type='page', @@ -2051,7 +2052,7 @@ def test_backup_with_less_privileges_role(self): # DELTA backup from replica self.backup_node( backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=100s']) + datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) From 6ca984b63d7d2296b2082495536945adb6d4b083 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 19:03:55 +0300 Subject: [PATCH 1005/2107] tests: fix for "replica" module --- tests/replica.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index f357c5aca..b368b93ed 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -743,6 +743,10 @@ def test_archive_replica_null_offset(self): self.set_replica(master, replica, synchronous=True) self.set_archiving(backup_dir, 'replica', replica, replica=True) + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + copy_tree( os.path.join(backup_dir, 'wal', 'master'), os.path.join(backup_dir, 'wal', 'replica')) @@ -757,7 +761,7 @@ def test_archive_replica_null_offset(self): backup_dir, 'replica', replica, options=[ '--archive-timeout=30', - '--log-level-console=verbose', + '--log-level-console=LOG', '--no-validate'], return_id=False) From 0a1a2b676e5e506d9e0c61cf83401cacbd727ed8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 19:07:42 +0300 Subject: [PATCH 1006/2107] [Issue #131] '--dry-run' flag is honored by the deletion of specific backup --- src/delete.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/delete.c b/src/delete.c index d5a4615b9..55dfa126b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -35,6 +35,8 @@ do_delete(time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; + int size_to_delete = 0; + char size_to_delete_pretty[20]; /* Get complete list of backups */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -63,32 +65,50 @@ do_delete(time_t backup_id) pgBackup *backup = (pgBackup *) parray_get(backup_list, i); /* check if backup is descendant of delete target */ - if (is_parent(target_backup->start_time, backup, false)) + if (is_parent(target_backup->start_time, backup, true)) + { parray_append(delete_list, backup); + + elog(LOG, "Backup %s %s be deleted", + base36enc(backup->start_time), dry_run? "can":"will"); + + size_to_delete += backup->data_bytes; + if (backup->stream) + size_to_delete += backup->wal_bytes; + } } - parray_append(delete_list, target_backup); - /* Lock marked for delete backups */ - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); + /* Report the resident size to delete */ + if (size_to_delete >= 0) + { + pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); + elog(INFO, "Resident data size to free by delete of backup %s : %s", + base36enc(target_backup->start_time), size_to_delete_pretty); + } - /* Delete backups from the end of list */ - for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + if (!dry_run) { - pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + /* Lock marked for delete backups */ + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); - if (interrupted) - elog(ERROR, "interrupted during delete backup"); + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); - delete_backup_files(backup); - } + if (interrupted) + elog(ERROR, "interrupted during delete backup"); - parray_free(delete_list); + delete_backup_files(backup); + } + } /* Clean WAL segments */ if (delete_wal) - do_retention_wal(false); + do_retention_wal(dry_run); /* cleanup */ + parray_free(delete_list); parray_walk(backup_list, pgBackupFree); parray_free(backup_list); } From 295c5c844a422497b36b8e1acee99e3b4d7b8e2f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 19:08:15 +0300 Subject: [PATCH 1007/2107] [Issue #131] tests coverage --- tests/delete.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/delete.py b/tests/delete.py index e6e739cbd..d8a650dda 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -692,3 +692,106 @@ def test_delete_multiple_descendants(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_delete_multiple_descendants_dry_run(self): + """ + PAGEa3 + PAGEa2 / + \ / + PAGEa1 (delete target) + | + FULLa + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + node.pgbench_init(scale=1) + backup_id_a = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + + # Change PAGEa2 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # Delete PAGEa1 + output = self.delete_pb( + backup_dir, 'node', page_id_a1, + options=['--dry-run', '--log-level-console=LOG', '--delete-wal']) + + print(output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a3), + output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a2), + output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a1), + output) + + self.assertIn( + 'INFO: Resident data size to free by ' + 'delete of backup {0} :'.format(page_id_a1), + output) + + self.assertIn( + 'On timeline 1 WAL segments between 0000000000000001 ' + 'and 0000000000000002 can be removed', + output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + output = self.delete_pb( + backup_dir, 'node', page_id_a1, + options=['--log-level-console=LOG', '--delete-wal']) + + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a3), + output) + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a2), + output) + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a1), + output) + self.assertIn( + 'INFO: Resident data size to free by ' + 'delete of backup {0} :'.format(page_id_a1), + output) + + self.assertIn( + 'On timeline 1 WAL segments between 0000000000000001 ' + 'and 0000000000000002 will be removed', + output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 96ba94862a3b866c4ec61d58881b76177ac30eb1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 19:20:25 +0300 Subject: [PATCH 1008/2107] Version 2.2.1 --- Documentation.md | 4 ++-- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation.md b/Documentation.md index 6c086da50..7621ae8b9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.2.0 +Current version - 2.2.1 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) @@ -572,7 +572,7 @@ To restore cluster on remote host see the section [Using pg_probackup in the Rem #### Partial Restore -If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can restore or exclude from restore the arbitraty number of specific databases using [partial restore options](#partial-restore-options) with the [restore](#restore) commands. +If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can restore or exclude from restore the arbitrary number of specific databases using [partial restore options](#partial-restore-options) with the [restore](#restore) commands. To restore only one or more databases, run the restore command with the following options: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dcc3e52f7..111c95a80 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -205,8 +205,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.0" -#define AGENT_PROTOCOL_VERSION 20200 +#define PROGRAM_VERSION "2.2.1" +#define AGENT_PROTOCOL_VERSION 20201 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index c1ee94fa4..cb1388a2c 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.0 \ No newline at end of file +pg_probackup 2.2.1 \ No newline at end of file From b07a2505b551cc14f50e5c200fead77b3f52450f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 20:02:09 +0300 Subject: [PATCH 1009/2107] Documentation: update for 2.2.1 --- Documentation.md | 101 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 17 deletions(-) diff --git a/Documentation.md b/Documentation.md index 7621ae8b9..d41e6c325 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1834,7 +1834,7 @@ Deprecated. Wait time for WAL segment streaming via replication, in seconds. By ## Howto -All exaples below assume the remote mode of operations via `ssh`. If you are planning to run backup and restore operation locally then step `Setup passwordless SSH connection` can be skipped and all `--remote-*` options can be ommited. +All examples below assume the remote mode of operations via `ssh`. If you are planning to run backup and restore operation locally then step `Setup passwordless SSH connection` can be skipped and all `--remote-*` options can be ommited. Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. @@ -1842,7 +1842,7 @@ backup_host - host with backup catalog. backupman - user on `backup_host` running all pg_probackup operations. /mnt/backups - directory on `backup_host` where backup catalog is stored. -postgres_host - host with PostgreSQL cluster. +postgres-host - host with PostgreSQL cluster. postgres - user on `postgres_host` which run PostgreSQL cluster. /var/lib/postgresql/11/main - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. backup_db - database used for connection to PostgreSQL cluster. @@ -1882,7 +1882,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probac COMMIT; ``` -3. Init backup catalog: +3. Init the backup catalog: ``` [backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully inited @@ -1897,45 +1897,112 @@ INFO: Instance 'node' successfully inited 5. Take FULL backup: ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ4LFC, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Start transferring data files INFO: Data files are transferred INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ4LFC -INFO: Backup PZ4LFC data files are valid -INFO: Backup PZ4LFC resident size: 62MB -INFO: Backup PZ4LFC completed +INFO: Validating backup PZ7YK2 +INFO: Backup PZ7YK2 data files are valid +INFO: Backup PZ7YK2 resident size: 196MB +INFO: Backup PZ7YK2 completed ``` 6. Lets take a look at the backup catalog: ``` +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' + BACKUP INSTANCE 'pg-11' ================================================================================================================================== Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ================================================================================================================================== - node 11 PZ4LFC 2019-10-10 00:09:17+03 FULL STREAM 1/0 7s 30MB 32MB 1.00 0/C000028 0/C000160 OK + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK ``` -7. Lets hide some options into config, so cmdline can be less crowdy: +7. Take incremental backup in DELTA mode: ``` -[backupman@backup_host]$ pg_probackup-11 set-config -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres --pguser=probackup --pgdatabase=backupdb +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Parent backup: PZ7YK2 +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ7YMP +INFO: Backup PZ7YMP data files are valid +INFO: Backup PZ7YMP resident size: 32MB +INFO: Backup PZ7YMP completed ``` -8. Take incremental backup in DELTA mode: +8. Lets hide some parameters into config, so cmdline can looks less crodwy: ``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream +[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb ``` +9. Take another incremental backup in DELTA mode, omitting the `--remote-host`, `--remote-user`, `-U` and `-d` options: +``` +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Parent backup: PZ7YMP +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ7YR5 +INFO: Backup PZ7YR5 data files are valid +INFO: Backup PZ7YR5 resident size: 32MB +INFO: Backup PZ7YR5 completed +``` +10. Lets take a look at instance config: +``` +[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' + +# Backup instance information +pgdata = /var/lib/postgresql/11/main +system-identifier = 6746586934060931492 +xlog-seg-size = 16777216 +# Connection parameters +pgdatabase = backupdb +pghost = postgres_host +pguser = probackup +# Replica parameters +replica-timeout = 5min +# Archive parameters +archive-timeout = 5min +# Logging parameters +log-level-console = INFO +log-level-file = OFF +log-filename = pg_probackup.log +log-rotation-size = 0 +log-rotation-age = 0 +# Retention parameters +retention-redundancy = 0 +retention-window = 0 +wal-depth = 0 +# Compression parameters +compress-algorithm = none +compress-level = 1 +# Remote access parameters +remote-proto = ssh +remote-host = postgres_host +``` -#### - -### Setup with WAL archive +Note, that we are getting default values for other options, that were not overwritten by set-config command. -The fact of having an WAL archive dramatically improve you scope of options toward backup and restore operations. +10. Lets take a look at the backup catalog: +``` +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' + +==================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +==================================================================================================================================== + node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK + node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK +``` ## Authors Postgres Professional, Moscow, Russia. From 2b92f56bd3c468ead0ae5ff371ba19852063d9f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 20:26:06 +0300 Subject: [PATCH 1010/2107] Documentation: update --- Documentation.md | 53 ++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/Documentation.md b/Documentation.md index d41e6c325..73e8286a8 100644 --- a/Documentation.md +++ b/Documentation.md @@ -76,7 +76,7 @@ Current version - 2.2.1 * [Partial Restore Options](#partial-restore-options) * [Replica Options](#replica-options) -7. [HOWTO](#howto) +7. [Howto](#howto) * [Minimal setup](#minimal-setup) 8. [Authors](#authors) 9. [Credits](#credits) @@ -1691,24 +1691,24 @@ You can use these options together with [backup](#backup) and [checkdb](#checkdb All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. -d dbname - --dbname=dbname + --pgdatabase=dbname PGDATABASE Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. -h host - --host=host + --pghost=host PGHOST Default: local socket Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. -p port - --port=port + --pgport=port PGPORT Default: 5432 Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. -U username - --username=username + --pguser=username PGUSER User name to connect as. @@ -1839,31 +1839,37 @@ All examples below assume the remote mode of operations via `ssh`. If you are pl Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. backup_host - host with backup catalog. + backupman - user on `backup_host` running all pg_probackup operations. + /mnt/backups - directory on `backup_host` where backup catalog is stored. -postgres-host - host with PostgreSQL cluster. +postgres_host - host with PostgreSQL cluster. + postgres - user on `postgres_host` which run PostgreSQL cluster. + /var/lib/postgresql/11/main - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. -backup_db - database used for connection to PostgreSQL cluster. + +backupdb - database used for connection to PostgreSQL cluster. ### Minimal Setup This setup is relying on autonomous FULL and DELTA backups. -1. Setup passwordless SSH connection from `backup_host` to `postgres_host`: +#### Setup passwordless SSH connection from `backup_host` to `postgres_host` ``` [backupman@backup_host] ssh-copy-id postgres@postgres_host ``` -2. Setup PostgreSQL cluster. -2.1. It is recommended from security purposes to use separate database for backup operations. +#### Setup PostgreSQL cluster + +It is recommended from security purposes to use separate database for backup operations. ``` postgres=# CREATE DATABASE backupdb; ``` -2.2. Connect to `backupdb` database, create role `probackup` and grant to it the following permissions: +Connect to `backupdb` database, create role `probackup` and grant to it the following permissions: ``` backupdb=# BEGIN; @@ -1882,19 +1888,19 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probac COMMIT; ``` -3. Init the backup catalog: +#### Init the backup catalog ``` [backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully inited ``` -4. Add instance 'pg-11' to backup catalog: +#### Add instance 'pg-11' to backup catalog ``` [backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main INFO: Instance 'node' successfully inited ``` -5. Take FULL backup: +#### Take FULL backup ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -1908,7 +1914,7 @@ INFO: Backup PZ7YK2 resident size: 196MB INFO: Backup PZ7YK2 completed ``` -6. Lets take a look at the backup catalog: +#### Lets take a look at the backup catalog ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' @@ -1917,10 +1923,9 @@ BACKUP INSTANCE 'pg-11' Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ================================================================================================================================== node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - ``` -7. Take incremental backup in DELTA mode: +#### Take incremental backup in DELTA mode ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -1935,12 +1940,12 @@ INFO: Backup PZ7YMP resident size: 32MB INFO: Backup PZ7YMP completed ``` -8. Lets hide some parameters into config, so cmdline can looks less crodwy: +#### Lets hide some parameters into config, so cmdline can be less crodwy ``` [backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb ``` -9. Take another incremental backup in DELTA mode, omitting the `--remote-host`, `--remote-user`, `-U` and `-d` options: +#### Take another incremental backup in DELTA mode, omitting some of the previous parameters: ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -1955,7 +1960,7 @@ INFO: Backup PZ7YR5 resident size: 32MB INFO: Backup PZ7YR5 completed ``` -10. Lets take a look at instance config: +#### Lets take a look at instance config ``` [backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' @@ -1992,15 +1997,15 @@ remote-host = postgres_host Note, that we are getting default values for other options, that were not overwritten by set-config command. -10. Lets take a look at the backup catalog: +#### Lets take a look at the backup catalog ``` [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' ==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ==================================================================================================================================== - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK + node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK + node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK ``` From 2ea92ce8caeb9fbf44b44bccf2a3dc03ffd8f9d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 23:40:14 +0300 Subject: [PATCH 1011/2107] Documentation: update --- Documentation.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/Documentation.md b/Documentation.md index 73e8286a8..ca1263133 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1838,19 +1838,13 @@ All examples below assume the remote mode of operations via `ssh`. If you are pl Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. -backup_host - host with backup catalog. - -backupman - user on `backup_host` running all pg_probackup operations. - -/mnt/backups - directory on `backup_host` where backup catalog is stored. - -postgres_host - host with PostgreSQL cluster. - -postgres - user on `postgres_host` which run PostgreSQL cluster. - -/var/lib/postgresql/11/main - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. - -backupdb - database used for connection to PostgreSQL cluster. +- *backup_host* - host with backup catalog. +- *backupman* - user on `backup_host` running all pg_probackup operations. +- */mnt/backups* - directory on `backup_host` where backup catalog is stored. +- *postgres_host* - host with PostgreSQL cluster. +- *postgres* - user on `postgres_host` which run PostgreSQL cluster. +- */var/lib/postgresql/11/main* - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. +- *backupdb* - database used for connection to PostgreSQL cluster. ### Minimal Setup From baa88de2c56ebcaf771715b66f388af990850b16 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 23:49:34 +0300 Subject: [PATCH 1012/2107] make compiler happy --- src/catalog.c | 25 +++++++++++++------------ src/show.c | 4 +++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index b42cc60f0..49607681d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -688,6 +688,7 @@ pgBackupCreateDir(pgBackup *backup) parray * catalog_get_timelines(InstanceConfig *instance) { + int i,j; parray *xlog_files_list = parray_new(); parray *timelineinfos; parray *backups; @@ -703,7 +704,7 @@ catalog_get_timelines(InstanceConfig *instance) tlinfo = NULL; /* walk through files and collect info about timelines */ - for (int i = 0; i < parray_num(xlog_files_list); i++) + for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); TimeLineID tli; @@ -854,9 +855,9 @@ catalog_get_timelines(InstanceConfig *instance) tlinfo->parent_tli = tln->tli; /* find parent timeline to link it with this one */ - for (int i = 0; i < parray_num(timelineinfos); i++) + for (j = 0; j < parray_num(timelineinfos); j++) { - timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, i); + timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, j); if (cur->tli == tlinfo->parent_tli) { tlinfo->parent_link = cur; @@ -875,10 +876,10 @@ catalog_get_timelines(InstanceConfig *instance) /* save information about backups belonging to each timeline */ backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); - for (int i = 0; i < parray_num(timelineinfos); i++) + for (i = 0; i < parray_num(timelineinfos); i++) { timelineInfo *tlinfo = parray_get(timelineinfos, i); - for (int j = 0; j < parray_num(backups); j++) + for (j = 0; j < parray_num(backups); j++) { pgBackup *backup = parray_get(backups, j); if (tlinfo->tli == backup->tli) @@ -892,7 +893,7 @@ catalog_get_timelines(InstanceConfig *instance) } /* determine oldest backup and closest backup for every timeline */ - for (int i = 0; i < parray_num(timelineinfos); i++) + for (i = 0; i < parray_num(timelineinfos); i++) { timelineInfo *tlinfo = parray_get(timelineinfos, i); @@ -964,7 +965,7 @@ catalog_get_timelines(InstanceConfig *instance) */ /* determine anchor_lsn and keep_segments for every timeline */ - for (int i = 0; i < parray_num(timelineinfos); i++) + for (i = 0; i < parray_num(timelineinfos); i++) { int count = 0; timelineInfo *tlinfo = parray_get(timelineinfos, i); @@ -976,7 +977,7 @@ catalog_get_timelines(InstanceConfig *instance) */ if (tlinfo->backups) { - for (int j = 0; j < parray_num(tlinfo->backups); j++) + for (j = 0; j < parray_num(tlinfo->backups); j++) { pgBackup *backup = parray_get(tlinfo->backups, j); @@ -1127,7 +1128,7 @@ catalog_get_timelines(InstanceConfig *instance) } /* Iterate over backups left */ - for (int j = count; j < parray_num(tlinfo->backups); j++) + for (j = count; j < parray_num(tlinfo->backups); j++) { XLogSegNo segno = 0; xlogInterval *interval = NULL; @@ -1188,7 +1189,7 @@ catalog_get_timelines(InstanceConfig *instance) * We must keep all WAL segments after anchor_lsn (including), and also segments * required by ARCHIVE backups for consistency - WAL between [start_lsn, stop_lsn]. */ - for (int i = 0; i < parray_num(timelineinfos); i++) + for (i = 0; i < parray_num(timelineinfos); i++) { XLogSegNo anchor_segno = 0; timelineInfo *tlinfo = parray_get(timelineinfos, i); @@ -1209,7 +1210,7 @@ catalog_get_timelines(InstanceConfig *instance) GetXLogSegNo(tlinfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); - for (int i = 0; i < parray_num(tlinfo->xlog_filelist); i++) + for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) { xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); @@ -1224,7 +1225,7 @@ catalog_get_timelines(InstanceConfig *instance) continue; /* Protect segments belonging to one of the keep invervals */ - for (int j = 0; j < parray_num(tlinfo->keep_segments); j++) + for (j = 0; j < parray_num(tlinfo->keep_segments); j++) { xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, j); diff --git a/src/show.c b/src/show.c index d57e881b3..e718eec3c 100644 --- a/src/show.c +++ b/src/show.c @@ -77,6 +77,8 @@ static int32 json_level = 0; int do_show(const char *instance_name, time_t requested_backup_id, bool show_archive) { + int i; + if (instance_name == NULL && requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use (-i, --backup-id) option"); @@ -94,7 +96,7 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive parray *instances = catalog_get_instance_list(); show_instance_start(); - for (int i = 0; i < parray_num(instances); i++) + for (i = 0; i < parray_num(instances); i++) { InstanceConfig *instance = parray_get(instances, i); char backup_instance_path[MAXPGPATH]; From 6e2a1a73efc8cf1624f8349fae43404e343d92a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Oct 2019 23:57:51 +0300 Subject: [PATCH 1013/2107] make compiler happy --- src/show.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/show.c b/src/show.c index e718eec3c..4cd7cb498 100644 --- a/src/show.c +++ b/src/show.c @@ -937,7 +937,7 @@ static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, parray *tli_list) { - int i; + int i,j; PQExpBuffer buf = &show_buf; parray *actual_tli_list = parray_new(); @@ -1026,7 +1026,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, { json_add(buf, JT_BEGIN_ARRAY, &json_level); - for (int j = 0; j < parray_num(tlinfo->lost_segments); j++) + for (j = 0; j < parray_num(tlinfo->lost_segments); j++) { xlogInterval *lost_segments = (xlogInterval *) parray_get(tlinfo->lost_segments, j); @@ -1056,7 +1056,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, if (tlinfo->backups != NULL) { json_add(buf, JT_BEGIN_ARRAY, &json_level); - for (int j = 0; j < parray_num(tlinfo->backups); j++) + for (j = 0; j < parray_num(tlinfo->backups); j++) { pgBackup *backup = parray_get(tlinfo->backups, j); From ae4e64010c22f1dff80504edc8dd73437b26cdf4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Oct 2019 00:07:12 +0300 Subject: [PATCH 1014/2107] minor fix --- src/show.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/show.c b/src/show.c index 4cd7cb498..e96537b0c 100644 --- a/src/show.c +++ b/src/show.c @@ -294,7 +294,7 @@ show_instance(const char *instance_name, time_t requested_backup_id, bool show_n static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup) { - TimeLineID parent_tli; + TimeLineID parent_tli = 0; char timestamp[100] = "----"; char lsn[20]; @@ -344,10 +344,9 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add_key(buf, "parent-tli", json_level); /* Only incremental backup can have Parent TLI */ - if (backup->backup_mode == BACKUP_MODE_FULL) - parent_tli = 0; - else if (backup->parent_backup_link) + if (backup->parent_backup_link) parent_tli = backup->parent_backup_link->tli; + appendPQExpBuffer(buf, "%u", parent_tli); snprintf(lsn, lengthof(lsn), "%X/%X", From 948924dcab9f3f6e2fb91a3db0693eff6d708a06 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Oct 2019 12:31:09 +0300 Subject: [PATCH 1015/2107] Documentation: added section "WAL Retention Policy" --- Documentation.md | 81 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/Documentation.md b/Documentation.md index ca1263133..c04e716c4 100644 --- a/Documentation.md +++ b/Documentation.md @@ -40,6 +40,7 @@ Current version - 2.2.1 * [Viewing WAL Archive Information](#viewing-wal-archive-information) * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) * [Pinning a Backup](#pinning-a-backup) + * [WAL Retention Policy](#wal-retention-policy) * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -1227,6 +1228,75 @@ You can unpin a backup by setting `--ttl` option to zero using `set-backup` comm Only pinned backups have `expire-time` attribute in backup metadata. +### WAL Retention Policy + +By default, pg_probackup treatment of WAL is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any existing backup in the backup catalog. To save disk space, you can configure WAL retention policy. + +Suppose you have backed up the *node* instance in the *backup_dir* directory with configured [WAL archiving](#setting-up-continuous-wal-archiving): + + pg_probackup show -B backup_dir --instance node + +``` +BACKUP INSTANCE 'node' +==================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +==================================================================================================================================== + node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK + node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK + node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK + node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK + node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK +``` + +The state of WAL archive can be determined by using [show](#command) command with `--archive` flag: + + pg_probackup show -B backup_dir --instance node --archive + +``` +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK +``` + +WAL purge without WAL retention cannot achieve much, only one segment can be removed: + + pg_probackup delete -B backup_dir --instance node --delete-wal + +``` +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK +``` + +If you would like, for example, to keep only those WAL segments that can be applied to the last valid backup, use the `--wal-depth` option: + + pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 + +``` +ARCHIVE INSTANCE 'node' +================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================ + 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK +``` + +Alternatively you can use the `--wal-depth` option with the [backup](#backup) command: + + pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal + +``` +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK +``` + ### Merging Backups As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: @@ -1310,7 +1380,7 @@ Deletes all backups and WAL files associated with the specified instance. pg_probackup set-config -B backup_dir --instance instance_name [--help] [--pgdata=pgdata-path] - [--retention-redundancy=redundancy][--retention-window=window] + [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] [-d dbname] [-h host] [-p port] [-U username] [--archive-timeout=timeout] [--external-dirs=external_directory_path] @@ -1500,7 +1570,7 @@ For details, see the section [Merging Backups](#merging-backups). pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] - [--retention-redundancy=redundancy][--retention-window=window] + [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} [--dry-run] [logging_options] @@ -1618,6 +1688,10 @@ Specifies the number of full backup copies to keep in the data directory. Must b Default: 0 Number of days of recoverability. Must be a positive integer. The zero value disables this setting. + --wal-depth=wal_depth + Default: 0 +Number of latest valid backups on every timeline that must retain the ability to perform PITR. Must be a positive integer. The zero value disables this setting. + --delete-wal Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. @@ -1879,6 +1953,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO probackup; COMMIT; ``` @@ -1934,7 +2009,7 @@ INFO: Backup PZ7YMP resident size: 32MB INFO: Backup PZ7YMP completed ``` -#### Lets hide some parameters into config, so cmdline can be less crodwy +#### Lets hide some parameters into config, so cmdline can be less crowdy ``` [backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb ``` From 71a59fff6b3319bf1fd16a736db7a5b3e1f311b5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Oct 2019 12:50:00 +0300 Subject: [PATCH 1016/2107] Documentation: update --- Documentation.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index c04e716c4..5ffa5af6a 100644 --- a/Documentation.md +++ b/Documentation.md @@ -131,14 +131,15 @@ As compared to other backup solutions, pg_probackup offers the following benefit - Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow - Validation: automatic data consistency checks and on-demand backup validation without actual data recovery - Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` -- Retention: managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired` +- Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting 'time to live' for backups - Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads - Compression: storing backup data in a compressed state to save disk space - Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) - Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it - Backup from replica: avoid extra load on the master server by taking backups from a standby - External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats and view WAL Archive information. +- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats +- Archive Catalog: get list of all WAL timelines and corresponding meta information in `plain` or `json` formats - Partial Restore: restore only the specified databases or skip the specified databases. To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. From dd651c86c7f1b81156ff4e4290e40c6abb1879ec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Oct 2019 15:00:38 +0300 Subject: [PATCH 1017/2107] Readme: update --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ed13b491b..b6675c87d 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,19 @@ The utility is compatible with: * PostgreSQL 9.5, 9.6, 10, 11; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -* Choosing between full and page-level incremental backups to speed up backup and recovery -* Implementing a single backup strategy for multi-server PostgreSQL clusters -* Automatic data consistency checks and on-demand backup validation without actual data recovery -* Managing backups in accordance with retention policy -* Merging incremental into full backups without actual data recovery -* Running backup, restore, merge and validation processes on multiple parallel threads -* Storing backup data in a compressed state to save disk space -* Taking backups from a standby server to avoid extra load on the master server -* Extended logging settings -* Custom commands to simplify WAL log archiving -* External to PGDATA directories, such as directories with config files and scripts, can be included in backup -* Remote backup, restore, add-instance, archive-push and archive-get operations via ssh (beta) -* Checking running PostgreSQL instance for the sights of corruption in read-only mode via `checkdb` command. +* Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow +* Validation: automatic data consistency checks and on-demand backup validation without actual data recovery +* Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` +* Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting 'time to live' for backups +* Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads +* Compression: storing backup data in a compressed state to save disk space +* Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) +* Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it +* Backup from replica: avoid extra load on the master server by taking backups from a standby +* External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files +* Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats +* Archive Catalog: get list of all WAL timelines and corresponding meta information in `plain` or `json` formats +* Partial Restore: restore only the specified databases or exclude the specified databases from restore. To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. @@ -49,11 +49,11 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.1.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.5) +[2.2.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.1) ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.1.5). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.1). ### Linux Installation ```shell From d9b4da3e019662c269bb8d1497722c1d86a86197 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 12 Oct 2019 15:08:49 +0300 Subject: [PATCH 1018/2107] Readme: update --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b6675c87d..8c20dafc4 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. -* Remote mode is in beta stage. * Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. ## Current release From ffd5f45f12a6a99dd11d02ff6d2e4ce3a3d1ae42 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 13 Oct 2019 00:27:13 +0300 Subject: [PATCH 1019/2107] Documentation: update --- Documentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index 5ffa5af6a..bca5d2ba9 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1210,7 +1210,7 @@ Alternatively you can use the `--ttl` and `--expire-time` options with the [back pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' -You can determine the fact that backup is pinned and check due expire time by looking up 'expire-time' attribute in backup metadata via (show)[#show] command: +You can determine the fact that backup is pinned and check due expire time by looking up 'expire-time' attribute in backup metadata via [show](#show) command: pg_probackup show --instance instance_name -i backup_id @@ -1223,7 +1223,7 @@ data-bytes = 22288792 ... ``` -You can unpin a backup by setting `--ttl` option to zero using `set-backup` command. Example: +You can unpin a backup by setting `--ttl` option to zero using [set-backup](#set-backup) command. Example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 @@ -1231,7 +1231,7 @@ Only pinned backups have `expire-time` attribute in backup metadata. ### WAL Retention Policy -By default, pg_probackup treatment of WAL is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any existing backup in the backup catalog. To save disk space, you can configure WAL retention policy. +By default, pg_probackup treatment of WAL is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any existing backup in the backup catalog. To save disk space, you can configure WAL retention policy, to keep WAL of limited depth measured in backups. Suppose you have backed up the *node* instance in the *backup_dir* directory with configured [WAL archiving](#setting-up-continuous-wal-archiving): @@ -1262,7 +1262,7 @@ ARCHIVE INSTANCE 'node' 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK ``` -WAL purge without WAL retention cannot achieve much, only one segment can be removed: +General WAL purge without `wal-depth` cannot achieve much, only one segment can be removed: pg_probackup delete -B backup_dir --instance node --delete-wal From 4d61a6271a706a4003008097059021b89fc5ce33 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Oct 2019 21:54:31 +0300 Subject: [PATCH 1020/2107] [Issue #132] test coverage --- tests/backup.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 57bebb476..987fe30b9 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2059,3 +2059,40 @@ def test_backup_with_less_privileges_role(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_issue_132(self): + """ + https://github.com/postgrespro/pg_probackup/issues/132 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + for i in range(50000): + node.safe_psql( + 'postgres', + "CREATE TABLE t_{0} as select 1".format(i)) + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 17143835dc9b47115d7ba79058629cbfc9b61d3d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 00:19:52 +0300 Subject: [PATCH 1021/2107] speed up merge by skipping block validation during merge --- src/merge.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/merge.c b/src/merge.c index ef4a77372..acbdce032 100644 --- a/src/merge.c +++ b/src/merge.c @@ -190,6 +190,9 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) merge_time = time(NULL); elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + /* It's redundant to check block checksumms during merge */ + skip_block_validation = true; + /* * Validate to_backup only if it is BACKUP_STATUS_OK. If it has * BACKUP_STATUS_MERGING status then it isn't valid backup until merging @@ -672,13 +675,13 @@ merge_files(void *arg) */ file->compress_alg = to_backup->compress_alg; - if (file->write_size != BYTES_INVALID) - elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", - file->path, file->write_size); - else + if (file->write_size < 0) elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", file->path, BYTES_INVALID); + elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", + file->path, file->write_size); + /* Restore relative path */ file->path = prev_file_path; } From 60a9f3c1cde4a665245467e4431dc4445d7815be Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 00:45:09 +0300 Subject: [PATCH 1022/2107] tests: minor fix --- tests/backup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 987fe30b9..e58dcb656 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2078,10 +2078,11 @@ def test_issue_132(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - for i in range(50000): - node.safe_psql( - 'postgres', - "CREATE TABLE t_{0} as select 1".format(i)) + with node.connect("postgres") as conn: + for i in range(50000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() self.backup_node( backup_dir, 'node', node, options=['--stream']) From faed631523137977beabbfded3236b2a69ee14bd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 12:52:52 +0300 Subject: [PATCH 1023/2107] tests: added backup.BackupTest.test_issue_132_1 --- tests/backup.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e58dcb656..e6c263b68 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2097,3 +2097,157 @@ def test_issue_132(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_issue_132_1(self): + """ + https://github.com/postgrespro/pg_probackup/issues/132 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + for i in range(30000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() + + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream'], old_binary=True) + + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream'], old_binary=True) + + node.cleanup() + + # make sure that new binary can detect corruption + try: + self.validate_pb(backup_dir, 'node', backup_id=full_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir, 'node', backup_id=delta_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "ORPHAN"') + + # check that revalidation is working correctly + try: + self.restore_node( + backup_dir, 'node', node, backup_id=delta_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "ORPHAN"') + + # check that '--no-validate' do not allow to restore ORPHAN backup +# try: +# self.restore_node( +# backup_dir, 'node', node, backup_id=delta_id, +# options=['--no-validate']) +# # we should die here because exception is what we expect to happen +# self.assertEqual( +# 1, 0, +# "Expecting Error because FULL backup is CORRUPT" +# "\n Output: {0} \n CMD: {1}".format( +# repr(self.output), self.cmd)) +# except ProbackupException as e: +# self.assertIn( +# 'Insert data', +# e.message, +# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( +# repr(e.message), self.cmd)) + + node.cleanup() + + output = self.restore_node( + backup_dir, 'node', node, backup_id=full_id, options=['--force']) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), + output) + + self.assertIn( + 'INFO: Restore of backup {0} completed.'.format(full_id), + output) + + node.cleanup() + + output = self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, options=['--force']) + + self.assertIn( + 'WARNING: Backup {0} is orphan.'.format(delta_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(delta_id), + output) + + self.assertIn( + 'INFO: Restore of backup {0} completed.'.format(delta_id), + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 251fc8382a70b4f02bf9471e99f14c3dc06d245f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 12:53:53 +0300 Subject: [PATCH 1024/2107] [Issue #132] fix --- src/catalog.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 49607681d..cb4097cea 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1626,12 +1626,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len += sprintf(line+len, "}\n"); - if (write_len + len <= BUFFERSZ) - { - memcpy(buf+write_len, line, len); - write_len += len; - } - else + if (write_len + len >= BUFFERSZ) { /* write buffer to file */ if (fio_fwrite(out, buf, write_len) != write_len) @@ -1644,6 +1639,9 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* reset write_len */ write_len = 0; } + + memcpy(buf+write_len, line, len); + write_len += len; } /* write what is left in the buffer to file */ From f49c1970ef0c27d68ba689506950771e56c50eed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 13:00:24 +0300 Subject: [PATCH 1025/2107] [Issue #132] validate is now capable of detecting metadata corruption produced by #132 --- src/validate.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/validate.c b/src/validate.c index 1352029c0..663e1b39b 100644 --- a/src/validate.c +++ b/src/validate.c @@ -175,6 +175,28 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); else elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); + + /* Issue #132 kludge */ + if (!corrupted && + ((parse_program_version(backup->program_version) == 20104)|| + (parse_program_version(backup->program_version) == 20105)|| + (parse_program_version(backup->program_version) == 20201))) + { + char path[MAXPGPATH]; + + pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + + if (pgFileSize(path) >= (BLCKSZ*500)) + { + elog(WARNING, "Backup %s is a victim of metadata corruption. " + "Additional information can be found here: " + "https://github.com/postgrespro/pg_probackup/issues/132", + base36enc(backup->start_time)); + backup->status = BACKUP_STATUS_CORRUPT; + write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name); + } + + } } /* From 74e6a5fcd9cd48b63c53e2ee733eda00c2ff7e02 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 13:11:41 +0300 Subject: [PATCH 1026/2107] [Issue #132] new flag "--force" for "restore" command allows to ignore invalid backup status during restore --- src/help.c | 5 +++-- src/pg_probackup.c | 12 ++++++++++-- src/pg_probackup.h | 4 ++-- src/restore.c | 26 +++++++++++++++++++++----- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/help.c b/src/help.c index 21f27d9fa..5368a335b 100644 --- a/src/help.c +++ b/src/help.c @@ -151,7 +151,7 @@ help_pg_probackup(void) printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica]\n")); + printf(_(" [--restore-as-replica] [--force]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); @@ -384,7 +384,7 @@ help_restore(void) printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica]\n")); + printf(_(" [--restore-as-replica] [--force]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); @@ -422,6 +422,7 @@ help_restore(void) printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); + printf(_(" --force ignore invalid status of the restored backup\n")); printf(_(" --no-validate disable backup validation during restore\n")); printf(_(" --skip-block-validation set to validate only file-level checksum\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index dba7a93fd..0b2820b85 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -111,7 +111,7 @@ bool amcheck_parent = false; bool delete_wal = false; bool delete_expired = false; bool merge_expired = false; -bool force_delete = false; +bool force = false; bool dry_run = false; /* compression options */ @@ -196,7 +196,7 @@ static ConfigOption cmd_options[] = { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, /* TODO not implemented yet */ - { 'b', 147, "force", &force_delete, SOURCE_CMD_STRICT }, + { 'b', 147, "force", &force, SOURCE_CMD_STRICT }, /* compression options */ { 'b', 148, "compress", &compress_shortcut, SOURCE_CMD_STRICT }, /* connection options */ @@ -646,9 +646,17 @@ main(int argc, char *argv[]) (target_immediate) ? "immediate" : NULL, target_name, target_action); + if (force && backup_subcmd != RESTORE_CMD) + elog(ERROR, "You cannot specify \"--force\" flag with the \"%s\" command", + command_name); + + if (force) + no_validate = true; + /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); + restore_params->force = force; restore_params->no_validate = no_validate; restore_params->restore_as_replica = restore_as_replica; restore_params->skip_block_validation = skip_block_validation; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 111c95a80..482ceb8c8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -380,11 +380,12 @@ typedef struct pgRecoveryTarget /* Options needed for restore and validate commands */ typedef struct pgRestoreParams { + bool force; bool is_restore; bool no_validate; bool restore_as_replica; bool skip_external_dirs; - bool skip_block_validation; + bool skip_block_validation; //Start using it const char *restore_command; /* options for partial restore */ @@ -565,7 +566,6 @@ extern bool exclusive_backup; extern bool delete_wal; extern bool delete_expired; extern bool merge_expired; -extern bool force_delete; extern bool dry_run; /* compression options */ diff --git a/src/restore.c b/src/restore.c index 4d8029eea..da212f499 100644 --- a/src/restore.c +++ b/src/restore.c @@ -181,7 +181,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if ((current_backup->status == BACKUP_STATUS_ORPHAN || current_backup->status == BACKUP_STATUS_CORRUPT || current_backup->status == BACKUP_STATUS_RUNNING) - && !params->no_validate) + && (!params->no_validate || params->force)) elog(WARNING, "Backup %s has status: %s", base36enc(current_backup->start_time), status2str(current_backup->status)); else @@ -421,9 +421,19 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); } else if (dest_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + { + if (params->force) + elog(WARNING, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + } else if (dest_backup->status == BACKUP_STATUS_ORPHAN) - elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + { + if (params->force) + elog(WARNING, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + else + elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + } else elog(ERROR, "Backup %s has status: %s", base36enc(dest_backup->start_time), status2str(dest_backup->status)); @@ -552,8 +562,14 @@ restore_backup(pgBackup *backup, parray *dest_external_dirs, if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) - elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); + { + if (params->force) + elog(WARNING, "Backup %s is not valid, restore is forced", + base36enc(backup->start_time)); + else + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + } /* confirm block size compatibility */ if (backup->block_size != BLCKSZ) From b0a80c174ad0c8dbc483765d49fc39f2ebaa9622 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 13:11:58 +0300 Subject: [PATCH 1027/2107] tests: minor update --- tests/backup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e6c263b68..7f6556f9e 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2220,6 +2220,14 @@ def test_issue_132_1(self): output = self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=['--force']) + self.assertIn( + 'WARNING: Backup {0} has status: CORRUPT'.format(full_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is corrupt.'.format(full_id), + output) + self.assertIn( 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), output) From c1b4110d47e3d8c29f17a76bb0fa07d7c3916ab4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 15:39:03 +0300 Subject: [PATCH 1028/2107] Version 2.2.2 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 482ceb8c8..f1c07998c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -205,8 +205,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.1" -#define AGENT_PROTOCOL_VERSION 20201 +#define PROGRAM_VERSION "2.2.2" +#define AGENT_PROTOCOL_VERSION 20202 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index cb1388a2c..c2f509e09 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.1 \ No newline at end of file +pg_probackup 2.2.2 \ No newline at end of file From 3ae1fc2ca7de0733abfced534dda65b9369a9837 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 21:48:07 +0300 Subject: [PATCH 1029/2107] bigfix: WAL retention didn`t protected WAL segments entitled to protection --- src/catalog.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index cb4097cea..6538651aa 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -688,7 +688,7 @@ pgBackupCreateDir(pgBackup *backup) parray * catalog_get_timelines(InstanceConfig *instance) { - int i,j; + int i,j,k; parray *xlog_files_list = parray_new(); parray *timelineinfos; parray *backups; @@ -1210,9 +1210,9 @@ catalog_get_timelines(InstanceConfig *instance) GetXLogSegNo(tlinfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); - for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) + for (j = 0; j < parray_num(tlinfo->xlog_filelist); j++) { - xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); + xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, j); if (wal_file->segno >= anchor_segno) { @@ -1225,9 +1225,9 @@ catalog_get_timelines(InstanceConfig *instance) continue; /* Protect segments belonging to one of the keep invervals */ - for (j = 0; j < parray_num(tlinfo->keep_segments); j++) + for (k = 0; k < parray_num(tlinfo->keep_segments); k++) { - xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, j); + xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, k); if ((wal_file->segno >= keep_segments->begin_segno) && wal_file->segno <= keep_segments->end_segno) From a71829db3366a5412f5445d011a2a51713df6b01 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 21:48:37 +0300 Subject: [PATCH 1030/2107] tests: minor improvements --- tests/merge.py | 19 +++++++------------ tests/replica.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 8a6f2981e..81d609ad9 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1068,7 +1068,7 @@ def test_continue_failed_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge @@ -1091,8 +1091,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): "postgres", "create table t_heap as select i as id," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i" - ) + " from generate_series(0,1000) i") old_path = node.safe_psql( "postgres", @@ -1100,18 +1099,15 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # DELTA BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') node.safe_psql( "postgres", - "update t_heap set id = 100500" - ) + "update t_heap set id = 100500") node.safe_psql( "postgres", - "vacuum full t_heap" - ) + "vacuum full t_heap") new_path = node.safe_psql( "postgres", @@ -1119,8 +1115,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # DELTA BACKUP backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') backup_id = self.show_pb(backup_dir, "node")[1]["id"] @@ -1155,7 +1150,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # Try to continue failed MERGE try: - self.merge_backup(backup_dir, "node", backup_id) + print(self.merge_backup(backup_dir, "node", backup_id)) self.assertEqual( 1, 0, "Expecting Error because of incremental backup corruption.\n " diff --git a/tests/replica.py b/tests/replica.py index b368b93ed..c4d4cc950 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -890,6 +890,7 @@ def test_replica_toast(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ + 'autovacuum': 'off', 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) @@ -924,6 +925,8 @@ def test_replica_toast(self): self.switch_wal_segment(master) self.switch_wal_segment(master) + self.wait_until_replica_catch_with_master(master, replica) + master.safe_psql( 'postgres', 'CREATE TABLE t1 AS ' @@ -951,12 +954,23 @@ def test_replica_toast(self): 'LOG: Found prior LSN:', output) - print(output) + res1 = replica.safe_psql( + 'postgres', + 'select md5(fat_attr) from t1') replica.cleanup() - self.restore_node(backup_dir, 'replica', replica) + self.restore_node(backup_dir, 'replica', replica) pgdata_restored = self.pgdata_content(replica.data_dir) + + replica.slow_start() + + res2 = replica.safe_psql( + 'postgres', + 'select md5(fat_attr) from t1') + + self.assertEqual(res1, res2) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself From e60cb2041bc7ea6891dbba8ea2e76c51b8b0285d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 15 Oct 2019 21:51:15 +0300 Subject: [PATCH 1031/2107] Version 2.2.3 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f1c07998c..b7e29942b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -205,8 +205,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.2" -#define AGENT_PROTOCOL_VERSION 20202 +#define PROGRAM_VERSION "2.2.3" +#define AGENT_PROTOCOL_VERSION 20203 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index c2f509e09..9c025d295 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.2 \ No newline at end of file +pg_probackup 2.2.3 \ No newline at end of file From c1ebd8aa5f3b8f5770d7f61ebab8fe46b6243fce Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 16 Oct 2019 01:24:35 +0300 Subject: [PATCH 1032/2107] Readme: update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c20dafc4..4273cc5b9 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,11 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.2.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.1) +[2.2.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.3) ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.1). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.3). ### Linux Installation ```shell From 8ce5b7a2410b8be298cdaf5561e8d8a1bb62c365 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Oct 2019 17:13:28 +0300 Subject: [PATCH 1033/2107] Readme: update --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4273cc5b9..d5861147d 100644 --- a/README.md +++ b/README.md @@ -111,9 +111,8 @@ gen_probackup_project.pl C:\path_to_postgresql_source_tree ## Documentation -Currently the latest documentation can be found at [github](https://postgrespro.github.io/pg_probackup). -Slightly outdated documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup). -Documentation for current devel can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). +Documentation for the latest version can be found at [github](https://github.com/postgrespro/pg_probackup/blob/2.2.3/Documentation.md). +Documentation for the current devel version can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). ## License From 19ad13d777429a5e7913e39c0c8641c4dd4f5486 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Oct 2019 12:36:42 +0300 Subject: [PATCH 1034/2107] [Issue #134] PostgreSQL 12 support --- Makefile | 10 +++- src/data.c | 5 ++ src/dir.c | 1 + src/restore.c | 124 ++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 499255f8b..167f04c07 100644 --- a/Makefile +++ b/Makefile @@ -57,8 +57,6 @@ src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ -src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@ src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c @@ -72,6 +70,14 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +ifeq (12,$(MAJORVERSION)) +src/logging.h: $(top_srcdir)/src/include/common/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/include/common/logging.h $@ +else +src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h + rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ +endif + ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ diff --git a/src/data.c b/src/data.c index c4aa0efc0..bbdb570ba 100644 --- a/src/data.c +++ b/src/data.c @@ -111,7 +111,12 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, } #endif case PGLZ_COMPRESS: + +#if PG_VERSION_NUM >= 120000 + return pglz_decompress(src, src_size, dst, dst_size, true); +#else return pglz_decompress(src, src_size, dst, dst_size); +#endif } return -1; diff --git a/src/dir.c b/src/dir.c index 0738b3d74..7a7ff169f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -79,6 +79,7 @@ static char *pgdata_exclude_files[] = "recovery.conf", "postmaster.pid", "postmaster.opts", + "probackup_recovery.conf", NULL }; diff --git a/src/restore.c b/src/restore.c index da212f499..8a8ee7d68 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,6 +44,7 @@ static void create_recovery_conf(time_t backup_id, pgRestoreParams *params); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); +static void create_pg12_recovery_config(pgBackup *backup, bool add_include); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -866,24 +867,43 @@ create_recovery_conf(time_t backup_id, /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || params->restore_as_replica)) + { +#if PG_VERSION_NUM >= 120000 + /* + * Restoring STREAM backup without PITR and not as replica, + * recovery.signal and standby.signal are not needed + */ + create_pg12_recovery_config(backup, false); +#endif return; + } elog(LOG, "----------------------------------------"); +#if PG_VERSION_NUM >= 120000 + elog(LOG, "creating probackup_recovery.conf"); + create_pg12_recovery_config(backup, true); + snprintf(path, lengthof(path), "%s/probackup_recovery.conf", instance_config.pgdata); +#else elog(LOG, "creating recovery.conf"); - snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); +#endif + fp = fio_fopen(path, "w", FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open recovery.conf \"%s\": %s", path, + elog(ERROR, "cannot open file \"%s\": %s", path, strerror(errno)); +#if PG_VERSION_NUM >= 120000 + fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); +#else fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); +#endif /* construct restore_command */ if (need_restore_conf) { - char restore_command_guc[16384]; /* If restore_command is provided, use it */ @@ -957,7 +977,10 @@ create_recovery_conf(time_t backup_id, if (params->restore_as_replica) { + /* standby_mode was removed in PG12 */ +#if PG_VERSION_NUM < 120000 fio_fprintf(fp, "standby_mode = 'on'\n"); +#endif if (backup->primary_conninfo) fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); @@ -965,7 +988,100 @@ create_recovery_conf(time_t backup_id, if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write recovery.conf \"%s\": %s", path, + elog(ERROR, "cannot write file \"%s\": %s", path, + strerror(errno)); + +#if PG_VERSION_NUM >= 120000 + if (need_restore_conf) + { + elog(LOG, "creating recovery.signal file"); + snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); + + fp = fio_fopen(path, "w", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write file \"%s\": %s", path, + strerror(errno)); + } + + if (params->restore_as_replica) + { + elog(LOG, "creating standby.signal file"); + snprintf(path, lengthof(path), "%s/standby.signal", instance_config.pgdata); + + fp = fio_fopen(path, "w", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write file \"%s\": %s", path, + strerror(errno)); + } +#endif +} + +/* + * Create empty probackup_recovery.conf in PGDATA and + * add include directive to postgresql.auto.conf + + * When restoring PG12 we always(!) must do this, even + * when restoring STREAM backup without PITR options + * because restored instance may have been backed up + * and restored again and user didn`t cleaned up postgresql.auto.conf. + + * So for recovery to work regardless of all this factors + * we must always create empty probackup_recovery.conf file. + */ +static void +create_pg12_recovery_config(pgBackup *backup, bool add_include) +{ + char probackup_recovery_path[MAXPGPATH]; + char postgres_auto_path[MAXPGPATH]; + FILE *fp; + + if (add_include) + { + time_t current_time; + char current_time_str[100]; + + current_time = time(NULL); + time2iso(current_time_str, lengthof(current_time_str), current_time); + + snprintf(postgres_auto_path, lengthof(postgres_auto_path), + "%s/postgresql.auto.conf", instance_config.pgdata); + + fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + strerror(errno)); + + fio_fprintf(fp, "\n# created by pg_probackup restore of backup %s at '%s'\n", + base36enc(backup->start_time), current_time_str); + fio_fprintf(fp, "include '%s'\n", "probackup_recovery.conf"); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + strerror(errno)); + } + + /* Create empty probackup_recovery.conf */ + snprintf(probackup_recovery_path, lengthof(probackup_recovery_path), + "%s/probackup_recovery.conf", instance_config.pgdata); + fp = fio_fopen(probackup_recovery_path, "w", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", probackup_recovery_path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write to file \"%s\": %s", probackup_recovery_path, strerror(errno)); } From ff40fcc4be418845ca1c1739e6f297c7bb36b407 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Oct 2019 19:42:02 +0300 Subject: [PATCH 1035/2107] [Issue #134] do not backup "recovery.signal" and "standby.signal" files, and for PG12 and newer always set recovery_target_timeline to current --- src/dir.c | 2 ++ src/restore.c | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/dir.c b/src/dir.c index 7a7ff169f..bcf6293e9 100644 --- a/src/dir.c +++ b/src/dir.c @@ -80,6 +80,8 @@ static char *pgdata_exclude_files[] = "postmaster.pid", "postmaster.opts", "probackup_recovery.conf", + "recovery.signal", + "standby.signal", NULL }; diff --git a/src/restore.c b/src/restore.c index 8a8ee7d68..f25bdb986 100644 --- a/src/restore.c +++ b/src/restore.c @@ -967,6 +967,17 @@ create_recovery_conf(time_t backup_id, if (rt->target_tli) fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + else + { + /* + * In PG12 default recovery target timeline was changed to 'latest', which + * is extremely risky. Explicitly preserve old behavior of recovering to current + * timneline for PG12. + */ +#if PG_VERSION_NUM >= 120000 + fio_fprintf(fp, "recovery_target_timeline = '%u'\n", backup->tli); +#endif + } if (rt->target_action) fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); From af42fbf9b6889ab72fe4f83aaf075aadab0b5d4e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Oct 2019 19:43:13 +0300 Subject: [PATCH 1036/2107] remove spooky warning about first segment been greater than oldest to_keep segments --- src/delete.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/delete.c b/src/delete.c index 55dfa126b..b1053098c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -870,8 +870,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, * Assume the worst. */ if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0) - elog(WARNING, "On timeline %i first segment %08X%08X is greater than " - "oldest segment to keep %08X%08X. Possible WAL archive corruption!", + elog(LOG, "On timeline %i first segment %08X%08X is greater than " + "oldest segment to keep %08X%08X", tlinfo->tli, (uint32) FirstToDeleteSegNo / xlog_seg_size, (uint32) FirstToDeleteSegNo % xlog_seg_size, (uint32) OldestToKeepSegNo / xlog_seg_size, (uint32) OldestToKeepSegNo % xlog_seg_size); From effea35742f730b2724de569b799f8a4554c4bfa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 09:48:35 +0300 Subject: [PATCH 1037/2107] tests: support PG12 --- tests/archive.py | 13 ++-- tests/auth_test.py | 3 +- tests/backup.py | 4 +- tests/cfs_restore.py | 6 +- tests/delete.py | 3 +- tests/delta.py | 42 ++++-------- tests/helpers/ptrack_helpers.py | 112 ++++++++++++++++++++------------ tests/merge.py | 12 ++-- tests/page.py | 21 ++---- tests/pgpro2068.py | 12 ++-- tests/pgpro589.py | 2 +- tests/ptrack.py | 42 ++++++------ tests/replica.py | 53 +++++---------- tests/restore.py | 56 +++++++++------- tests/validate.py | 16 ++--- 15 files changed, 197 insertions(+), 200 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 438a0efeb..89813bdfc 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -245,8 +245,7 @@ def test_pgpro434_3(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - node.append_conf( - 'postgresql.auto.conf', "archive_command = 'exit 1'") + self.set_auto_conf(node, {'archive_command': "'exit 1'"}) node.reload() gdb.continue_execution_until_exit() @@ -306,8 +305,7 @@ def test_pgpro434_4(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - node.append_conf( - 'postgresql.auto.conf', "archive_command = 'exit 1'") + self.set_auto_conf(node, {'archive_command': "'exit 1'"}) node.reload() os.environ["PGAPPNAME"] = "foo" @@ -741,8 +739,8 @@ def test_replica_archive(self): base_dir=os.path.join(module_name, fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + + self.set_auto_conf(node, {'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS after = node.safe_psql("postgres", "SELECT * FROM t_heap") @@ -784,8 +782,7 @@ def test_replica_archive(self): self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + self.set_auto_conf(node, {'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS diff --git a/tests/auth_test.py b/tests/auth_test.py index d9b5c283e..eca62316b 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -124,7 +124,8 @@ def test_backup_via_unprivileged_user(self): node.safe_psql( "test1", "create table t1 as select generate_series(0,100)") - node.append_conf("postgresql.auto.conf", "ptrack_enable = 'on'") + if self.ptrack: + self.set_auto_conf(node, {'ptrack_enable': 'on'}) node.stop() node.slow_start() diff --git a/tests/backup.py b/tests/backup.py index 7f6556f9e..3d5c8a6c7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1539,7 +1539,7 @@ def test_backup_with_least_privileges_role(self): 'archive_timeout': '30s'}) if self.ptrack: - node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') + self.set_auto_conf(node, {'ptrack_enable': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1909,7 +1909,7 @@ def test_backup_with_less_privileges_role(self): 'checkpoint_timeout': '1h'}) if self.ptrack: - node.append_conf('postgresql.auto.conf', 'ptrack_enable = on') + self.set_auto_conf(node, {'ptrack_enable': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index f59c092af..07cf891aa 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -216,8 +216,7 @@ def test_restore_from_fullbackup_to_new_location(self): try: self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id) - node_new.append_conf("postgresql.auto.conf", - "port = {0}".format(node_new.port)) + self.set_auto_conf(node_new, {'port': node_new.port}) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( @@ -260,8 +259,7 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): try: self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) - node_new.append_conf("postgresql.auto.conf", - "port = {0}".format(node_new.port)) + self.set_auto_conf(node_new, {'port': node_new.port}) except ProbackupException as e: self.fail( "ERROR: Restore from full backup failed. \n {0} \n {1}".format( diff --git a/tests/delete.py b/tests/delete.py index d8a650dda..8b10d7302 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -332,8 +332,7 @@ def test_delete_wal_between_multiple_timelines(self): node2.cleanup() self.restore_node(backup_dir, 'node', node2) - node2.append_conf( - 'postgresql.auto.conf', "port = {0}".format(node2.port)) + self.set_auto_conf(node, {'port': node2.port}) node2.slow_start() # load some more data to node diff --git a/tests/delta.py b/tests/delta.py index ecc927053..12b2fadd6 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -84,8 +84,7 @@ def test_basic_delta_vacuum_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -173,8 +172,7 @@ def test_delta_vacuum_truncate_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -226,25 +224,21 @@ def test_delta_vacuum_truncate_2(self): os.unlink(os.path.join(node.data_dir, filepath + '.1')) self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, 'node', node_restored - ) + backup_dir, 'node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -473,8 +467,7 @@ def test_delta_multiple_segments(self): pgdata_restored = self.pgdata_content(restored_node.data_dir) # START RESTORED NODE - restored_node.append_conf( - "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.safe_psql( @@ -563,8 +556,7 @@ def test_delta_vacuum_full(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() @@ -644,8 +636,7 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # DROP DATABASE DB1 @@ -679,8 +670,7 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() try: @@ -801,8 +791,7 @@ def test_exists_in_previous_backup(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -883,8 +872,7 @@ def test_alter_table_set_tablespace_delta(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() result_new = node_restored.safe_psql( @@ -981,8 +969,7 @@ def test_alter_database_set_tablespace_delta(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -1065,8 +1052,7 @@ def test_delta_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bc02ba7ac..cc280afbe 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -130,8 +130,11 @@ def slow_start(self, replica=False): self.start() while True: try: - if self.safe_psql('template1', query) == 't\n': + output = self.safe_psql('template1', query).rstrip() + + if output == 't': break + except testgres.QueryException as e: if 'database system is starting up' in e[0]: continue @@ -323,32 +326,30 @@ def make_simple_node( initdb_params=initdb_params, allow_streaming=set_replication) # Sane default parameters - node.append_conf('postgresql.auto.conf', 'max_connections = 100') - node.append_conf('postgresql.auto.conf', 'shared_buffers = 10MB') - node.append_conf('postgresql.auto.conf', 'fsync = off') - - if 'wal_level' not in pg_options: - node.append_conf('postgresql.auto.conf', 'wal_level = logical') - node.append_conf('postgresql.auto.conf', 'hot_standby = off') - - node.append_conf( - 'postgresql.auto.conf', "log_line_prefix = '%t [%p]: [%l-1] '") - node.append_conf('postgresql.auto.conf', 'log_statement = none') - node.append_conf('postgresql.auto.conf', 'log_duration = on') - node.append_conf( - 'postgresql.auto.conf', 'log_min_duration_statement = 0') - node.append_conf('postgresql.auto.conf', 'log_connections = on') - node.append_conf('postgresql.auto.conf', 'log_disconnections = on') + options = {} + options['max_connections'] = 100 + options['shared_buffers'] = '10MB' + options['fsync'] = 'off' - # Apply given parameters - for key, value in six.iteritems(pg_options): - node.append_conf('postgresql.auto.conf', '%s = %s' % (key, value)) + options['wal_level'] = 'logical' + options['hot_standby'] = 'off' + + options['log_line_prefix'] = '"%t [%p]: [%l-1] "' + options['log_statement'] = 'none' + options['log_duration'] = 'on' + options['log_min_duration_statement'] = 0 + options['log_connections'] = 'on' + options['log_disconnections'] = 'on' # Allow replication in pg_hba.conf if set_replication: - node.append_conf( - 'postgresql.auto.conf', - 'max_wal_senders = 10') + options['max_wal_senders'] = 10 + + # set default values + self.set_auto_conf(node, options) + + # Apply given parameters + self.set_auto_conf(node, pg_options) # set major version with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: @@ -1072,8 +1073,15 @@ def show_config(self, backup_dir, instance, old_binary=False): def get_recovery_conf(self, node): out_dict = {} + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf_path = os.path.join( + node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf_path = os.path.join(node.data_dir, 'recovery.conf') + with open( - os.path.join(node.data_dir, 'recovery.conf'), 'r' + recovery_conf_path, 'r' ) as recovery_conf: for line in recovery_conf: try: @@ -1132,12 +1140,20 @@ def set_auto_conf(self, node, options): raw_content = f.read() current_options = {} + current_directives = [] for line in raw_content.splitlines(): # ignore comments if line.startswith('#'): continue + if line == '': + continue + + if line.startswith('include'): + current_directives.append(line) + continue + name, var = line.partition('=')[::2] name = name.strip() var = var.strip() @@ -1153,6 +1169,9 @@ def set_auto_conf(self, node, options): auto_conf += "{0} = '{1}'\n".format( option, current_options[option]) + for directive in current_directives: + auto_conf += directive + "\n" + with open(path, 'wt') as f: f.write(auto_conf) f.flush() @@ -1163,25 +1182,36 @@ def set_replica( replica_name='replica', synchronous=False ): + + self.set_auto_conf( + replica, + options={ + 'port': replica.port, + 'hot_standby': 'on'}) + + if self.get_version(replica) >= self.version_to_num('12.0'): + recovery_config = 'probackup_recovery.conf' + with open(os.path.join(replica.data_dir, "standby.signal"), 'w') as f: + f.flush() + f.close() + else: + recovery_config = 'recovery.conf' + replica.append_conf(recovery_config, 'standby_mode = on') + replica.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf('postgresql.auto.conf', 'hot_standby = on') - replica.append_conf('recovery.conf', 'standby_mode = on') - replica.append_conf( - 'recovery.conf', + recovery_config, "primary_conninfo = 'user={0} port={1} application_name={2}" " sslmode=prefer sslcompression=1'".format( - self.user, master.port, replica_name) - ) + self.user, master.port, replica_name)) + if synchronous: - master.append_conf( - 'postgresql.auto.conf', - "synchronous_standby_names='{0}'".format(replica_name) - ) - master.append_conf( - 'postgresql.auto.conf', - "synchronous_commit='remote_apply'" - ) + + self.set_auto_conf( + master, + options={ + 'synchronous_standby_names': replica_name, + 'synchronous_commit': 'remote_apply'}) + master.reload() def change_backup_status(self, backup_dir, instance, backup_id, status): @@ -1341,7 +1371,9 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): 'postmaster.pid', 'postmaster.opts', 'pg_internal.init', 'postgresql.auto.conf', 'backup_label', 'tablespace_map', 'recovery.conf', - 'ptrack_control', 'ptrack_init', 'pg_control' + 'ptrack_control', 'ptrack_init', 'pg_control', + 'probackup_recovery.conf', 'recovery.signal', + 'standby.signal' ] if exclude_dirs: diff --git a/tests/merge.py b/tests/merge.py index 81d609ad9..467104e32 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -700,8 +700,7 @@ def test_merge_page_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -791,8 +790,7 @@ def test_merge_delta_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -888,8 +886,7 @@ def test_merge_ptrack_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -987,8 +984,7 @@ def test_merge_delta_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself diff --git a/tests/page.py b/tests/page.py index 60eab0ac0..7a4ae3519 100644 --- a/tests/page.py +++ b/tests/page.py @@ -84,8 +84,7 @@ def test_basic_page_vacuum_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -345,8 +344,7 @@ def test_page_multiple_segments(self): pgdata_restored = self.pgdata_content(restored_node.data_dir) # START RESTORED NODE - restored_node.append_conf( - "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.safe_psql( @@ -432,8 +430,7 @@ def test_page_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -511,8 +508,7 @@ def test_page_delete_1(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Clean after yourself @@ -579,8 +575,7 @@ def test_parallel_pagemap(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() # Check restored node @@ -1066,8 +1061,7 @@ def test_page_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() node_restored.safe_psql('db1', 'select 1') @@ -1096,8 +1090,7 @@ def test_page_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() try: diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 0e6a8b98d..fa9a50875 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -51,10 +51,9 @@ def test_minrecpoint_on_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf( - 'postgresql.auto.conf', 'restart_after_crash = off') + self.set_auto_conf( + replica, + {'port': replica.port, 'restart_after_crash': 'off'}) # we need those later node.safe_psql( @@ -124,6 +123,11 @@ def test_minrecpoint_on_replica(self): # MinRecLSN = replica.get_control_data()['Minimum recovery ending location'] # Promote replica with 'immediate' target action + if self.get_version(replica) >= self.version_to_num('12.0'): + recovery_config = 'postgresql.auto.conf' + else: + recovery_config = 'recovery.conf' + replica.append_conf( 'recovery.conf', "recovery_target = 'immediate'") replica.append_conf( diff --git a/tests/pgpro589.py b/tests/pgpro589.py index 7b2b223e2..d6381a8b5 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -28,7 +28,7 @@ def test_pgpro589(self): self.set_archiving(backup_dir, 'node', node) # make erroneous archive_command - node.append_conf("postgresql.auto.conf", "archive_command = 'exit 0'") + self.set_auto_conf(node, {'archive_command': 'exit 0'}) node.slow_start() node.pgbench_init(scale=5) diff --git a/tests/ptrack.py b/tests/ptrack.py index 7025d994b..d33498890 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -172,8 +172,8 @@ def test_ptrack_uncommitted_xact(self): node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() @@ -258,8 +258,8 @@ def test_ptrack_vacuum_full(self): node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() @@ -345,8 +345,9 @@ def test_ptrack_vacuum_truncate(self): ) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + node_restored.slow_start() # Clean after yourself @@ -413,8 +414,9 @@ def test_ptrack_simple(self): node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + node_restored.slow_start() # Logical comparison @@ -988,8 +990,8 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() # DROP DATABASE DB1 @@ -1018,8 +1020,8 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() try: @@ -1225,8 +1227,8 @@ def test_alter_table_set_tablespace_ptrack(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() # result_new = node_restored.safe_psql( @@ -1453,8 +1455,8 @@ def test_ptrack_alter_tablespace(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - restored_node.append_conf( - "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + self.set_auto_conf( + restored_node, {'port': restored_node.port}) restored_node.slow_start() # COMPARE LOGICAL CONTENT @@ -1488,8 +1490,8 @@ def test_ptrack_alter_tablespace(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - restored_node.append_conf( - "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + self.set_auto_conf( + restored_node, {'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.safe_psql( @@ -1593,8 +1595,8 @@ def test_ptrack_multiple_segments(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - restored_node.append_conf( - "postgresql.auto.conf", "port = {0}".format(restored_node.port)) + self.set_auto_conf( + restored_node, {'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.safe_psql( diff --git a/tests/replica.py b/tests/replica.py index c4d4cc950..e25bbd371 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -86,8 +86,8 @@ def test_replica_stream_ptrack_backup(self): node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() # CHECK DATA CORRECTNESS @@ -121,8 +121,8 @@ def test_replica_stream_ptrack_backup(self): self.restore_node( backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() # CHECK DATA CORRECTNESS @@ -218,11 +218,7 @@ def test_replica_archive_page_backup(self): node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - - node.append_conf( - 'postgresql.auto.conf', 'archive_mode = off'.format(node.port)) + self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) node.slow_start() @@ -263,11 +259,7 @@ def test_replica_archive_page_backup(self): backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) - node.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node.port)) - - node.append_conf( - 'postgresql.auto.conf', 'archive_mode = off') + self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) node.slow_start() @@ -301,8 +293,6 @@ def test_basic_make_replica_via_restore(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) - # force more frequent wal switch - master.append_conf('postgresql.auto.conf', 'archive_timeout = 10') master.slow_start() replica = self.make_simple_node( @@ -327,10 +317,7 @@ def test_basic_make_replica_via_restore(self): # Settings for Replica self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) - replica.append_conf( - 'postgresql.auto.conf', 'hot_standby = on') + self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) @@ -390,15 +377,13 @@ def test_take_backup_from_delayed_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(replica.port)) + self.set_auto_conf(replica, {'port': replica.port}) replica.slow_start(replica=True) self.wait_until_replica_catch_with_master(master, replica) - replica.append_conf( - 'recovery.conf', "recovery_min_apply_delay = '300s'") + self.set_auto_conf(replica, {'recovery_min_apply_delay': '300s'}) replica.stop() replica.slow_start(replica=True) @@ -580,7 +565,7 @@ def test_replica_stop_lsn_null_offset(self): backup_dir, 'replica', replica, options=[ '--archive-timeout=30', - '--log-level-console=verbose', + '--log-level-console=LOG', '--no-validate', '--stream'], return_id=False) @@ -602,16 +587,12 @@ def test_replica_stop_lsn_null_offset(self): output) self.assertIn( - 'LOG: Record 0/2000160 has endpoint 0/3000000 which is ' + 'has endpoint 0/3000000 which is ' 'equal or greater than requested LSN 0/3000000', output) self.assertIn( - 'LOG: Found prior LSN: 0/2000160', - output) - - self.assertIn( - 'LOG: current.stop_lsn: 0/2000160', + 'LOG: Found prior LSN:', output) # Clean after yourself @@ -669,7 +650,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): backup_dir, 'replica', replica, options=[ '--archive-timeout=40', - '--log-level-file=verbose', + '--log-level-file=LOG', '--no-validate', '--stream'], gdb=True) @@ -782,12 +763,12 @@ def test_archive_replica_null_offset(self): output) self.assertIn( - 'LOG: Record 0/2000160 has endpoint 0/3000000 which is ' + 'has endpoint 0/3000000 which is ' 'equal or greater than requested LSN 0/3000000', output) self.assertIn( - 'LOG: Found prior LSN: 0/2000160', + 'LOG: Found prior LSN:', output) print(output) @@ -933,13 +914,11 @@ def test_replica_toast(self): 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' 'FROM generate_series(0,10) i') - # open connection to master output = self.backup_node( backup_dir, 'replica', replica, options=[ '--archive-timeout=30', - '--log-level-console=verbose', - '--log-level-file=verbose', + '--log-level-console=LOG', '--no-validate', '--stream'], return_id=False) diff --git a/tests/restore.py b/tests/restore.py index e457f2be1..b0c21817c 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -52,7 +52,10 @@ def test_restore_full_to_latest(self): repr(self.output), self.cmd)) # 2 - Test that recovery.conf was created - recovery_conf = os.path.join(node.data_dir, "recovery.conf") + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') self.assertEqual(os.path.isfile(recovery_conf), True) node.slow_start() @@ -181,13 +184,13 @@ def test_restore_to_time(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'TimeZone': 'Europe/Moscow'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node.append_conf("postgresql.auto.conf", "TimeZone = Europe/Moscow") node.slow_start() node.pgbench_init(scale=2) @@ -1189,8 +1192,7 @@ def test_archive_node_backup_archive_pitr_2(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() @@ -1320,10 +1322,9 @@ def test_zags_block_corrupt(self): self.restore_node( backup_dir, 'node', node_restored) - node_restored.append_conf("postgresql.auto.conf", "archive_mode = 'off'") - node_restored.append_conf("postgresql.auto.conf", "hot_standby = 'on'") - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, + {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) node_restored.slow_start() @@ -1403,10 +1404,9 @@ def test_zags_block_corrupt_1(self): self.restore_node( backup_dir, 'node', node_restored) - node_restored.append_conf("postgresql.auto.conf", "archive_mode = 'off'") - node_restored.append_conf("postgresql.auto.conf", "hot_standby = 'on'") - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, + {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) node_restored.slow_start() @@ -1795,7 +1795,10 @@ def test_restore_target_immediate_stream(self): pgdata = self.pgdata_content(node.data_dir) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') # restore delta backup node.cleanup() @@ -1847,7 +1850,10 @@ def test_restore_target_immediate_archive(self): pgdata = self.pgdata_content(node.data_dir) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') # restore page backup node.cleanup() @@ -1894,7 +1900,10 @@ def test_restore_target_latest_archive(self): self.backup_node( backup_dir, 'node', node) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') # restore node.cleanup() @@ -1944,7 +1953,11 @@ def test_restore_target_new_options(self): # Take FULL self.backup_node(backup_dir, 'node', node) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + node.pgbench_init(scale=2) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -2436,8 +2449,7 @@ def test_partial_restore_exclude(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - node_restored_2.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) node_restored_2.slow_start() @@ -2560,8 +2572,7 @@ def test_partial_restore_exclude_tablespace(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - node_restored_2.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) node_restored_2.slow_start() @@ -2680,8 +2691,7 @@ def test_partial_restore_include(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - node_restored_2.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored_2.port)) + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) node_restored_2.slow_start() node_restored_2.safe_psql( diff --git a/tests/validate.py b/tests/validate.py index 21175591c..28a02139f 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1754,14 +1754,14 @@ def test_pgpro561(self): backup_type='page', options=["--stream"]) self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) - node2.append_conf( - 'postgresql.auto.conf', 'port = {0}'.format(node2.port)) - node2.append_conf( - 'postgresql.auto.conf', 'archive_mode = off') + self.set_auto_conf( + node2, {'port': node2.port, 'archive_mode': 'off'}) + node2.slow_start() - node2.append_conf( - 'postgresql.auto.conf', 'archive_mode = on') + self.set_auto_conf( + node2, {'archive_mode': 'on'}) + node2.stop() node2.slow_start() @@ -3515,8 +3515,8 @@ def test_validate_target_lsn(self): self.restore_node(backup_dir, 'node', node_restored) - node_restored.append_conf( - "postgresql.auto.conf", "port = {0}".format(node_restored.port)) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) node_restored.slow_start() From a3b0bd7a4d8bcc2ae7badbe1a19b316ee6414f95 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 09:52:18 +0300 Subject: [PATCH 1038/2107] Documentation: update --- Documentation.md | 71 ++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/Documentation.md b/Documentation.md index bca5d2ba9..788ff106c 100644 --- a/Documentation.md +++ b/Documentation.md @@ -38,9 +38,10 @@ Current version - 2.2.1 * [Managing the Backup Catalog](#managing-the-backup-catalog) * [Viewing Backup Information](#viewing-backup-information) * [Viewing WAL Archive Information](#viewing-wal-archive-information) - * [Configuring Backup Retention Policy](#configuring-backup-retention-policy) - * [Pinning a Backup](#pinning-a-backup) - * [WAL Retention Policy](#wal-retention-policy) + * [Configuring Retention Policy](#configuring-retention-policy) + * [Backup Retention Policy](#backup-retention-policy) + * [Backup Pinning](#backup-pinning) + * [WAL Archive Retention Policy](#wal-archive-retention-policy) * [Merging Backups](#merging-backups) * [Deleting Backups](#deleting-backups) @@ -1124,7 +1125,11 @@ Most fields are consistent with plain format, with some exceptions: - DEGRADED timelines contain 'lost-segments' array with information about intervals of missing segments. In OK timelines 'lost-segments' array is empty. - 'N backups' attribute is replaced with 'backups' array containing backups belonging to the timeline. If timeline has no backups, then 'backups' array is empty. -### Configuring Backup Retention Policy +### Configuring Retention Policy + +With pg_probackup, you can set retention policies for backups and WAL archive. All policies can be combined together in any way. + +#### Backup Retention Policy By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. @@ -1162,16 +1167,16 @@ Suppose you have backed up the *node* instance in the *backup_dir* directory, wi ``` BACKUP INSTANCE 'node' -=========================================================================================================================================== - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -=========================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1 / 0 21s 32MB 0/13000028 0/13000198 OK - ---------------------------------------retention window------------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1 / 0 31s 33MB 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1 / 0 11s 200MB 0/D000028 0/D000198 OK +=================================================================================================================================== + Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status +=================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK + -------------------------------------------------------retention window-------------------------------------------------------- + node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK + node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK + node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK ``` Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` flag, only the P7XDFT full backup will be removed. @@ -1183,25 +1188,25 @@ With the `--merge-expired` option, the P7XDJA backup is merged with the underlyi ``` BACKUP INSTANCE 'node' -============================================================================================================================================ - Instance Version ID Recovery time Mode WAL Current/Parent TLI Time Data Start LSN Stop LSN Status -============================================================================================================================================ - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1 / 0 11s 200MB 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 DELTA STREAM 1 / 0 11s 19MB 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-04 05:28:36+03 FULL STREAM 1 / 0 5s 200MB 0/13000028 0/13000198 OK +================================================================================================================================== + Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK ``` >NOTE: The Time field for the merged backup displays the time required for the merge. -#### Pinning a Backup +#### Backup Pinning If you have the necessity to exclude certain backups from established retention policy then it is possible to pin a backup for an arbitrary amount of time. Example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d -This command will set `expire-time` of specified backup to 30 days starting from backup `recovery-time` attribute. Basically 'expire-time = recovery-time + ttl'. +This command will set `expire-time` of the specified backup to 30 days starting from backup `recovery-time` attribute. Basically `expire-time` = `recovery-time` + `ttl`. -You can set `expire-time` explicitly using `--expire-time` option. Example: +Also you can set `expire-time` explicitly using `--expire-time` option. Example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' @@ -1210,7 +1215,7 @@ Alternatively you can use the `--ttl` and `--expire-time` options with the [back pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' -You can determine the fact that backup is pinned and check due expire time by looking up 'expire-time' attribute in backup metadata via [show](#show) command: +You can determine the fact that backup is pinned and check due expire time by looking up `expire-time` attribute in backup metadata via [show](#show) command: pg_probackup show --instance instance_name -i backup_id @@ -1223,15 +1228,17 @@ data-bytes = 22288792 ... ``` -You can unpin a backup by setting `--ttl` option to zero using [set-backup](#set-backup) command. Example: +You can unpin the backup by setting the `--ttl` option to zero using [set-backup](#set-backup) command. Example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 -Only pinned backups have `expire-time` attribute in backup metadata. +Only pinned backups have the `expire-time` attribute in the backup metadata. + +>NOTE: Pinned incremental backup will also implicitly pin all its parent backups. -### WAL Retention Policy +#### WAL Archive Retention Policy -By default, pg_probackup treatment of WAL is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any existing backup in the backup catalog. To save disk space, you can configure WAL retention policy, to keep WAL of limited depth measured in backups. +By default, pg_probackup treatment of WAL Archive is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any of the existing backups in the backup catalog. To save disk space, you can configure WAL Archive retention policy, that allows to keep WAL of limited depth measured in backups per timeline. Suppose you have backed up the *node* instance in the *backup_dir* directory with configured [WAL archiving](#setting-up-continuous-wal-archiving): @@ -1578,7 +1585,7 @@ For details, see the section [Merging Backups](#merging-backups). Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. -For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-options) and [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-options) and [Configuring Retention Policy](#configuring-retention-policy). #### archive-push @@ -1678,8 +1685,8 @@ Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-s #### Retention Options You can use these options together with [backup](#backup) and [delete](#delete) commands. - -For details on configuring retention policy, see the sections [Configuring Backup Retention Policy](#configuring-backup-retention-policy). +f +For details on configuring retention policy, see the section [Configuring Retention Policy](#configuring-retention-policy). --retention-redundancy=redundancy Default: 0 @@ -1709,7 +1716,7 @@ Displays the current status of all the available backups, without deleting or me You can use these options together with [backup](#backup) and [set-delete](#set-backup) commands. -For details on backup pinning, see the section [Pinning a Backup](#pinning-a-backup). +For details on backup pinning, see the section [Backup Pinning](#backup-pinning). --ttl=ttl Specifies the amount of time the backup should be pinned. Must be a positive integer. The zero value unpin already pinned backup. Supported units: ms, s, min, h, d (s by default). Example: `--ttl=30d`. From 98f0c9ee48ccd00e342c26d3c715651be4bfdbe3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 12:22:07 +0300 Subject: [PATCH 1039/2107] [Issue #133] call pgpro_edition() function only if it exists --- src/backup.c | 66 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/src/backup.c b/src/backup.c index 21e9502cf..b38719d3d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -110,6 +110,9 @@ static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); static parray *get_database_map(PGconn *pg_startbackup_conn); +/* pgpro specific functions */ +static bool pgpro_support(PGconn *conn); + /* Ptrack functions */ static void pg_ptrack_clear(PGconn *backup_conn); static bool pg_ptrack_support(PGconn *backup_conn); @@ -658,6 +661,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) nodeInfo->block_size = BLCKSZ; nodeInfo->wal_block_size = XLOG_BLCKSZ; nodeInfo->is_superuser = pg_is_superuser(cur_conn); + nodeInfo->pgpro_support = pgpro_support(cur_conn); current.from_replica = pg_is_in_recovery(cur_conn); @@ -845,7 +849,7 @@ do_backup(time_t start_time, bool no_validate, static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) { - PGresult *res; + PGresult *res = NULL; /* confirm server version */ nodeInfo->server_version = PQserverVersion(conn); @@ -871,39 +875,45 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) "server version is %s, must be %s or higher for backup from replica", nodeInfo->server_version_str, "9.6"); - /* TODO: search pg_proc for pgpro_edition before calling */ - res = pgut_execute_extended(conn, "SELECT pgpro_edition()", - 0, NULL, true, true); + if (nodeInfo->pgpro_support) + res = pgut_execute(conn, "SELECT pgpro_edition()", 0, NULL); /* * Check major version of connected PostgreSQL and major version of * compiled PostgreSQL. */ #ifdef PGPRO_VERSION - if (PQresultStatus(res) == PGRES_FATAL_ERROR) + if (!res) /* It seems we connected to PostgreSQL (not Postgres Pro) */ elog(ERROR, "%s was built with Postgres Pro %s %s, " "but connection is made with PostgreSQL %s", PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); - else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && - strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with Postgres Pro %s %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, - nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); + else + { + if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && + strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with Postgres Pro %s %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, + nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); + } #else - if (PQresultStatus(res) != PGRES_FATAL_ERROR) + if (res) /* It seems we connected to Postgres Pro (not PostgreSQL) */ elog(ERROR, "%s was built with PostgreSQL %s, " "but connection is made with Postgres Pro %s %s", PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); - else if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) - elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", - PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str); + else + { + if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str); + } #endif - PQclear(res); + if (res) + PQclear(res); /* Do exclusive backup only for PostgreSQL 9.5 */ exclusive_backup = nodeInfo->server_version < 90600 || @@ -1106,6 +1116,30 @@ pg_ptrack_support(PGconn *backup_conn) return true; } +/* + * Check if the instance is PostgresPro fork. + */ +static bool +pgpro_support(PGconn *conn) +{ + PGresult *res; + + res = pgut_execute(conn, + "SELECT proname FROM pg_proc WHERE proname='pgpro_edition'", + 0, NULL); + + if (PQresultStatus(res) == PGRES_TUPLES_OK && + (PQntuples(res) == 1) && + (strcmp(PQgetvalue(res, 0, 0), "pgpro_edition") == 0)) + { + PQclear(res); + return true; + } + + PQclear(res); + return false; +} + /* * Fill 'datname to Oid' map * From 5c01dcf9c5a6e4c357e39130f809af0099b3f40b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 12:27:53 +0300 Subject: [PATCH 1040/2107] Documentation: update --- Documentation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation.md b/Documentation.md index 788ff106c..c6a0eac29 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.2.1 +Current version - 2.2.3 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) @@ -1685,7 +1685,7 @@ Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-s #### Retention Options You can use these options together with [backup](#backup) and [delete](#delete) commands. -f + For details on configuring retention policy, see the section [Configuring Retention Policy](#configuring-retention-policy). --retention-redundancy=redundancy From b9e76a1c4f3de6f4a0c3d1eecaf84c5835bd0651 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 12:28:41 +0300 Subject: [PATCH 1041/2107] [Issue #133] small fix --- src/catalog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/catalog.c b/src/catalog.c index 6538651aa..a0d78b5cd 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1930,6 +1930,7 @@ pgNodeInit(PGNodeInfo *node) node->checksum_version = 0; node->is_superuser = false; + node->pgpro_support = false; node->server_version = 0; node->server_version_str[0] = '\0'; From 0ad96362d089ef4bc539d2b0c5844e4822a03383 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 19 Oct 2019 12:30:58 +0300 Subject: [PATCH 1042/2107] rename create_pg12_recovery_config() to create_recovery_config() --- src/pg_probackup.h | 1 + src/restore.c | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b7e29942b..86223a753 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -285,6 +285,7 @@ typedef struct PGNodeInfo uint32 wal_block_size; uint32 checksum_version; bool is_superuser; + bool pgpro_support; int server_version; char server_version_str[100]; diff --git a/src/restore.c b/src/restore.c index f25bdb986..a05fed197 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,7 +44,8 @@ static void create_recovery_conf(time_t backup_id, pgRestoreParams *params); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); -static void create_pg12_recovery_config(pgBackup *backup, bool add_include); +static void create_recovery_config(pgBackup *backup, bool add_include); + /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -873,7 +874,7 @@ create_recovery_conf(time_t backup_id, * Restoring STREAM backup without PITR and not as replica, * recovery.signal and standby.signal are not needed */ - create_pg12_recovery_config(backup, false); + create_recovery_config(backup, false); #endif return; } @@ -881,7 +882,7 @@ create_recovery_conf(time_t backup_id, elog(LOG, "----------------------------------------"); #if PG_VERSION_NUM >= 120000 elog(LOG, "creating probackup_recovery.conf"); - create_pg12_recovery_config(backup, true); + create_recovery_config(backup, true); snprintf(path, lengthof(path), "%s/probackup_recovery.conf", instance_config.pgdata); #else elog(LOG, "creating recovery.conf"); @@ -1050,7 +1051,7 @@ create_recovery_conf(time_t backup_id, * we must always create empty probackup_recovery.conf file. */ static void -create_pg12_recovery_config(pgBackup *backup, bool add_include) +create_recovery_config(pgBackup *backup, bool add_include) { char probackup_recovery_path[MAXPGPATH]; char postgres_auto_path[MAXPGPATH]; From 3b2af9e83c0bc351412c3df0c144960765258c00 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 21 Oct 2019 13:42:51 +0300 Subject: [PATCH 1043/2107] [Issue #134] set recovery_target_timeline to "current" --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index a05fed197..9f83f8602 100644 --- a/src/restore.c +++ b/src/restore.c @@ -976,7 +976,7 @@ create_recovery_conf(time_t backup_id, * timneline for PG12. */ #if PG_VERSION_NUM >= 120000 - fio_fprintf(fp, "recovery_target_timeline = '%u'\n", backup->tli); + fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); #endif } From 5196fb25ec9028dcb70975ff696bd460713b9060 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 21 Oct 2019 13:46:48 +0300 Subject: [PATCH 1044/2107] tests: fixes --- tests/archive.py | 18 ++++++++++---- tests/backup.py | 42 +++++++++++++++++++++++++++++++++ tests/helpers/ptrack_helpers.py | 33 +++++++++++++++----------- 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 89813bdfc..b4d3a9d76 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -239,17 +239,19 @@ def test_pgpro434_3(self): backup_dir, 'node', node, options=[ "--archive-timeout=60", - "--log-level-file=info"], + "--log-level-file=LOG"], gdb=True) gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - self.set_auto_conf(node, {'archive_command': "'exit 1'"}) + self.set_auto_conf(node, {'archive_command': 'exit 1'}) node.reload() gdb.continue_execution_until_exit() + sleep(1) + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') with open(log_file, 'r') as f: log_content = f.read() @@ -1572,7 +1574,11 @@ def test_archive_options(self): '--archive-user={0}'.format(self.user) ]) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + with open(recovery_conf, 'r') as f: recovery_content = f.read() @@ -1643,7 +1649,11 @@ def test_archive_options_1(self): '--archive-user={0}'.format(self.user)]) self.restore_node(backup_dir, 'node', node) - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + with open(recovery_conf, 'r') as f: recovery_content = f.read() diff --git a/tests/backup.py b/tests/backup.py index 3d5c8a6c7..98a8e30b1 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2259,3 +2259,45 @@ def test_issue_132_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_streaming_timeout(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_sender_timeout': '5s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=LOG']) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + sleep(10) + gdb.continue_execution_until_error() + gdb._execute('detach') + sleep(2) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_content = f.read() + + self.assertIn( + 'could not receive data from WAL stream', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index cc280afbe..e3ce3c92e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1184,28 +1184,30 @@ def set_replica( ): self.set_auto_conf( - replica, - options={ - 'port': replica.port, - 'hot_standby': 'on'}) + replica, + options={ + 'port': replica.port, + 'hot_standby': 'on'}) if self.get_version(replica) >= self.version_to_num('12.0'): - recovery_config = 'probackup_recovery.conf' with open(os.path.join(replica.data_dir, "standby.signal"), 'w') as f: f.flush() f.close() - else: - recovery_config = 'recovery.conf' - replica.append_conf(recovery_config, 'standby_mode = on') - replica.append_conf( - recovery_config, - "primary_conninfo = 'user={0} port={1} application_name={2}" - " sslmode=prefer sslcompression=1'".format( - self.user, master.port, replica_name)) + self.set_auto_conf( + replica, + {'primary_conninfo': 'user={0} port={1} application_name={2} ' + ' sslmode=prefer sslcompression=1'.format( + self.user, master.port, replica_name)}) + else: + replica.append_conf('recovery.conf', 'standby_mode = on') + replica.append_conf( + 'recovery.conf', + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name)) if synchronous: - self.set_auto_conf( master, options={ @@ -1725,6 +1727,9 @@ def continue_execution_until_error(self): return if line.startswith('*stopped,reason="exited'): return + if line.startswith( + '*stopped,reason="signal-received",signal-name="SIGABRT"'): + return raise GdbException( 'Failed to continue execution until error.\n') From 6cbea998cca79c908fa8cb92d9910e8f5f52ade3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 22 Oct 2019 13:41:51 +0300 Subject: [PATCH 1045/2107] Readme: update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d5861147d..6a0eb6d71 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ As compared to other backup solutions, `pg_probackup` offers the following benef * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery * Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` -* Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting 'time to live' for backups +* Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting `time to live` for backups * Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads * Compression: storing backup data in a compressed state to save disk space * Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) From ef20e82a6f5185558936fba510b227e815e6f45a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 22 Oct 2019 21:16:49 +0300 Subject: [PATCH 1046/2107] [Issue #136] fix incorrect XlogSegNo calculation in catalog_get_timelines() --- src/catalog.c | 37 +++++++++++++++++++++---------------- src/delete.c | 33 +++++++++++++++++++-------------- src/pg_probackup.h | 14 ++++++++++++++ src/show.c | 34 ++++++++++++++++------------------ 4 files changed, 70 insertions(+), 48 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index a0d78b5cd..2c84d52b2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -695,6 +695,10 @@ catalog_get_timelines(InstanceConfig *instance) timelineInfo *tlinfo; char arclog_path[MAXPGPATH]; + /* for fancy reporting */ + char begin_segno_str[20]; + char end_segno_str[20]; + /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); @@ -716,19 +720,21 @@ catalog_get_timelines(InstanceConfig *instance) { int result = 0; uint32 log, seg; - XLogSegNo segno; - char suffix[MAXPGPATH]; + XLogSegNo segno = 0; + char suffix[MAXFNAMELEN]; result = sscanf(file->name, "%08X%08X%08X.%s", &tli, &log, &seg, (char *) &suffix); + /* sanity */ if (result < 3) { elog(WARNING, "unexpected WAL file name \"%s\"", file->name); continue; } - segno = log * instance->xlog_seg_size + seg; + /* get segno from log */ + GetXLogSegNoFromScrath(segno, log, seg, instance->xlog_seg_size); /* regular WAL file with suffix */ if (result == 4) @@ -1112,15 +1118,15 @@ catalog_get_timelines(InstanceConfig *instance) * covered by other larger interval. */ + GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size); + GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size); + elog(LOG, "Timeline %i to stay reachable from timeline %i " "protect from purge WAL interval between " - "%08X%08X and %08X%08X on timeline %i", - tli, closest_backup->tli, - (uint32) interval->begin_segno / instance->xlog_seg_size, - (uint32) interval->begin_segno % instance->xlog_seg_size, - (uint32) interval->end_segno / instance->xlog_seg_size, - (uint32) interval->end_segno % instance->xlog_seg_size, - tlinfo->tli); + "%s and %s on timeline %i", + tli, closest_backup->tli, begin_segno_str, + end_segno_str, tlinfo->tli); + parray_append(tlinfo->keep_segments, interval); continue; } @@ -1167,15 +1173,14 @@ catalog_get_timelines(InstanceConfig *instance) else interval->end_segno = segno; + GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size); + GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size); + elog(LOG, "Archive backup %s to stay consistent " "protect from purge WAL interval " - "between %08X%08X and %08X%08X on timeline %i", + "between %s and %s on timeline %i", base36enc(backup->start_time), - (uint32) interval->begin_segno / instance->xlog_seg_size, - (uint32) interval->begin_segno % instance->xlog_seg_size, - (uint32) interval->end_segno / instance->xlog_seg_size, - (uint32) interval->end_segno % instance->xlog_seg_size, - backup->tli); + begin_segno_str, end_segno_str, backup->tli); if (tlinfo->keep_segments == NULL) tlinfo->keep_segments = parray_new(); diff --git a/src/delete.c b/src/delete.c index b1053098c..bc4c93b2c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -810,6 +810,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { XLogSegNo FirstToDeleteSegNo; XLogSegNo OldestToKeepSegNo = 0; + char first_to_del_str[20]; + char oldest_to_keep_str[20]; int rc; int i; int wal_size_logical = 0; @@ -842,13 +844,15 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, } if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo) - elog(INFO, "On timeline %i WAL segments between %08X%08X and %08X%08X %s be removed", - tlinfo->tli, - (uint32) FirstToDeleteSegNo / xlog_seg_size, - (uint32) FirstToDeleteSegNo % xlog_seg_size, - (uint32) (OldestToKeepSegNo - 1) / xlog_seg_size, - (uint32) (OldestToKeepSegNo - 1) % xlog_seg_size, - dry_run?"can":"will"); + { + /* translate segno number into human readable format */ + GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size); + GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size); + + elog(INFO, "On timeline %i WAL segments between %s and %s %s be removed", + tlinfo->tli, first_to_del_str, + oldest_to_keep_str, dry_run?"can":"will"); + } /* sanity */ if (OldestToKeepSegNo > FirstToDeleteSegNo) @@ -866,15 +870,16 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, * 1. WAL archive corruption. * 2. There is no actual WAL archive to speak of and * 'keep_lsn' is coming from STREAM backup. - * - * Assume the worst. */ + if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0) - elog(LOG, "On timeline %i first segment %08X%08X is greater than " - "oldest segment to keep %08X%08X", - tlinfo->tli, - (uint32) FirstToDeleteSegNo / xlog_seg_size, (uint32) FirstToDeleteSegNo % xlog_seg_size, - (uint32) OldestToKeepSegNo / xlog_seg_size, (uint32) OldestToKeepSegNo % xlog_seg_size); + { + GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size); + GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size); + + elog(LOG, "On timeline %i first segment %s is greater than oldest segment to keep %s", + tlinfo->tli, first_to_del_str, oldest_to_keep_str); + } } else if (OldestToKeepSegNo == FirstToDeleteSegNo && !purge_all) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 86223a753..e48c39e62 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -526,6 +526,13 @@ typedef struct BackupPageHeader XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) #define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) +#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ + snprintf(fname, MAXFNAMELEN, "%08X%08X", \ + (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \ + (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes))) + +#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ + logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg #else #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo) @@ -535,6 +542,13 @@ typedef struct BackupPageHeader XLogFileName(fname, tli, logSegNo) #define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ XLByteInSeg(xlrp, logSegNo) +#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ + snprintf(fname, MAXFNAMELEN, "%08X%08X",\ + (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \ + (uint32) ((logSegNo) % XLogSegmentsPerXLogId)) + +#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ + logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg #endif #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) diff --git a/src/show.c b/src/show.c index e96537b0c..3ea8e3710 100644 --- a/src/show.c +++ b/src/show.c @@ -749,6 +749,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *tli_list, bool show_name) { + char segno_tmp[20]; parray *actual_tli_list = parray_new(); #define SHOW_ARCHIVE_FIELDS_COUNT 10 int i; @@ -807,16 +808,16 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* Min Segno */ - snprintf(row->min_segno, lengthof(row->min_segno), "%08X%08X", - (uint32) tlinfo->begin_segno / xlog_seg_size, - (uint32) tlinfo->begin_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size); + snprintf(row->min_segno, lengthof(row->min_segno), "%s",segno_tmp); + widths[cur] = Max(widths[cur], strlen(row->min_segno)); cur++; /* Max Segno */ - snprintf(row->max_segno, lengthof(row->max_segno), "%08X%08X", - (uint32) tlinfo->end_segno / xlog_seg_size, - (uint32) tlinfo->end_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size); + snprintf(row->max_segno, lengthof(row->max_segno), "%s", segno_tmp); + widths[cur] = Max(widths[cur], strlen(row->max_segno)); cur++; @@ -939,6 +940,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, int i,j; PQExpBuffer buf = &show_buf; parray *actual_tli_list = parray_new(); + char segno_tmp[20]; if (!first_instance) appendPQExpBufferChar(buf, ','); @@ -985,14 +987,12 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, (uint32) (tlinfo->switchpoint >> 32), (uint32) tlinfo->switchpoint); json_add_value(buf, "switchpoint", tmp_buf, json_level, true); - snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", - (uint32) tlinfo->begin_segno / xlog_seg_size, - (uint32) tlinfo->begin_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size); + snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "min-segno", tmp_buf, json_level, true); - snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", - (uint32) tlinfo->end_segno / xlog_seg_size, - (uint32) tlinfo->end_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size); + snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "max-segno", tmp_buf, json_level, true); json_add_key(buf, "n-segments", json_level); @@ -1034,14 +1034,12 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add(buf, JT_BEGIN_OBJECT, &json_level); - snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", - (uint32) lost_segments->begin_segno / xlog_seg_size, - (uint32) lost_segments->begin_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, lost_segments->begin_segno, xlog_seg_size); + snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "begin-segno", tmp_buf, json_level, true); - snprintf(tmp_buf, lengthof(tmp_buf), "%08X%08X", - (uint32) lost_segments->end_segno / xlog_seg_size, - (uint32) lost_segments->end_segno % xlog_seg_size); + GetXLogSegName(segno_tmp, lost_segments->end_segno, xlog_seg_size); + snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "end-segno", tmp_buf, json_level, true); json_add(buf, JT_END_OBJECT, &json_level); } From 2df5d27a66d48058e062eefd536d909f51de8624 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 23 Oct 2019 11:31:29 +0300 Subject: [PATCH 1047/2107] Add TODO in config_get_opt() --- src/utils/configuration.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index beca4fccb..1ef332ed5 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -434,6 +434,8 @@ get_username(void) /* * Process options passed from command line. + * TODO: currectly argument parsing treat missing argument for options + * as invalid option */ int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], From dd3775704958e0d75295772f26cce3e5704d4928 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 23 Oct 2019 11:32:23 +0300 Subject: [PATCH 1048/2107] tests: fixes --- tests/delete.py | 8 +++++--- tests/retention.py | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index 8b10d7302..7cefd22ab 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -332,7 +332,7 @@ def test_delete_wal_between_multiple_timelines(self): node2.cleanup() self.restore_node(backup_dir, 'node', node2) - self.set_auto_conf(node, {'port': node2.port}) + self.set_auto_conf(node2, {'port': node2.port}) node2.slow_start() # load some more data to node @@ -762,7 +762,7 @@ def test_delete_multiple_descendants_dry_run(self): self.assertIn( 'On timeline 1 WAL segments between 0000000000000001 ' - 'and 0000000000000002 can be removed', + 'and 0000000000000003 can be removed', output) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) @@ -787,10 +787,12 @@ def test_delete_multiple_descendants_dry_run(self): self.assertIn( 'On timeline 1 WAL segments between 0000000000000001 ' - 'and 0000000000000002 will be removed', + 'and 0000000000000003 will be removed', output) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + self.validate_pb(backup_dir, 'node') + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/retention.py b/tests/retention.py index c1331d37e..adbbd18a4 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1363,6 +1363,11 @@ def test_window_error_backups_2(self): page_id = self.show_pb(backup_dir, 'node')[1]['id'] + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'SELECT pg_catalog.pg_stop_backup()') + # Take DELTA backup delta_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', @@ -2230,7 +2235,7 @@ def test_wal_purge(self): self.assertIn( 'INFO: On timeline 4 WAL segments between 0000000000000002 ' - 'and 0000000000000005 can be removed', + 'and 0000000000000006 can be removed', output) self.assertIn( @@ -2255,7 +2260,7 @@ def test_wal_purge(self): self.assertIn( 'INFO: On timeline 4 WAL segments between 0000000000000002 ' - 'and 0000000000000005 will be removed', + 'and 0000000000000006 will be removed', output) self.assertIn( From 5a1ab20e03d8a5d0eb5ea9b952d9a3f8e39d7a57 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 23 Oct 2019 11:41:23 +0300 Subject: [PATCH 1049/2107] Documentation: update --- Documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index c6a0eac29..54e3003ce 100644 --- a/Documentation.md +++ b/Documentation.md @@ -217,7 +217,7 @@ For details on how to fine-tune pg_probackup configuration, see the section [Con The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. ->NOTE: For PostgreSQL >= 11 it is recommended to use [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature, so backup can be done by OS user with read-only permissions. +>NOTE: For PostgreSQL >= 11 it is recommended to use [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature, so backup can be done by any OS user in the same group as the cluster owner. In this case the user should have read permissions on the cluster directory. ### Configuring the Database Cluster From e29befec5ad655ce2b61b6d387e2f9197dfb112a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 24 Oct 2019 17:50:38 +0300 Subject: [PATCH 1050/2107] Documentation: add information about "--force" flag --- Documentation.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation.md b/Documentation.md index 54e3003ce..156eacc70 100644 --- a/Documentation.md +++ b/Documentation.md @@ -1495,7 +1495,7 @@ For details on usage, see the section [Creating a Backup](#creating-a-backup). [--help] [-D data_dir] [-i backup_id] [-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] - [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] + [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] [--restore-command=cmdline] [recovery_options] [logging_options] [remote_options] [partial_restore_options] [remote_archive_options] @@ -1525,6 +1525,9 @@ Skips backup validation. You can use this flag if you validate backups regularly --restore-command=cmdline Set the [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) parameter to specified command. Example: `--restore-command='cp /mnt/server/archivedir/%f "%p"'` + --force +Allows to ignore the invalid status of the backup. You can use this flag if you for some reason have the necessity to restore PostgreSQL cluster from corrupted or invalid backup. Use with caution. + Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Remote WAL Archive Options](#remote-wal-archive-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). From 2c4eb9d534b88d20fba750bb45da42a502f18b74 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 24 Oct 2019 17:53:20 +0300 Subject: [PATCH 1051/2107] [Issue #134] minor refactoring --- src/restore.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/restore.c b/src/restore.c index 9f83f8602..9591f7ede 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,7 +44,7 @@ static void create_recovery_conf(time_t backup_id, pgRestoreParams *params); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); -static void create_recovery_config(pgBackup *backup, bool add_include); +static void pg12_recovery_config(pgBackup *backup, bool add_include); /* @@ -869,20 +869,18 @@ create_recovery_conf(time_t backup_id, /* No need to generate recovery.conf at all. */ if (!(need_restore_conf || params->restore_as_replica)) { -#if PG_VERSION_NUM >= 120000 /* * Restoring STREAM backup without PITR and not as replica, - * recovery.signal and standby.signal are not needed + * recovery.signal and standby.signal for PG12 are not needed */ - create_recovery_config(backup, false); -#endif + pg12_recovery_config(backup, false); return; } elog(LOG, "----------------------------------------"); #if PG_VERSION_NUM >= 120000 elog(LOG, "creating probackup_recovery.conf"); - create_recovery_config(backup, true); + pg12_recovery_config(backup, true); snprintf(path, lengthof(path), "%s/probackup_recovery.conf", instance_config.pgdata); #else elog(LOG, "creating recovery.conf"); @@ -1040,19 +1038,20 @@ create_recovery_conf(time_t backup_id, /* * Create empty probackup_recovery.conf in PGDATA and - * add include directive to postgresql.auto.conf + * add "include" directive to postgresql.auto.conf * When restoring PG12 we always(!) must do this, even - * when restoring STREAM backup without PITR options - * because restored instance may have been backed up + * when restoring STREAM backup without PITR or replica options + * because restored instance may have been previously backed up * and restored again and user didn`t cleaned up postgresql.auto.conf. * So for recovery to work regardless of all this factors * we must always create empty probackup_recovery.conf file. */ static void -create_recovery_config(pgBackup *backup, bool add_include) +pg12_recovery_config(pgBackup *backup, bool add_include) { +#if PG_VERSION_NUM >= 120000 char probackup_recovery_path[MAXPGPATH]; char postgres_auto_path[MAXPGPATH]; FILE *fp; @@ -1095,6 +1094,8 @@ create_recovery_config(pgBackup *backup, bool add_include) fio_fclose(fp)) elog(ERROR, "cannot write to file \"%s\": %s", probackup_recovery_path, strerror(errno)); +#endif + return; } /* From 721b8434162a7a345daae66e85e6ea41c12ba0b4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 24 Oct 2019 17:55:52 +0300 Subject: [PATCH 1052/2107] tests: fixes for 9.5 --- tests/expected/option_help.out | 2 +- tests/remote.py | 3 +- tests/replica.py | 56 +++++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 46595b5a6..da7ebcd17 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -76,7 +76,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target=immediate|latest] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] - [--restore-as-replica] + [--restore-as-replica] [--force] [--no-validate] [--skip-block-validation] [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] diff --git a/tests/remote.py b/tests/remote.py index 064ee219f..f2d503a0e 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -16,6 +16,7 @@ def test_remote_sanity(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -26,7 +27,7 @@ def test_remote_sanity(self): try: self.backup_node( backup_dir, 'node', - node, options=['--remote-proto=ssh', '--stream']) + node, options=['--remote-proto=ssh', '--stream'], no_remote=True) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, diff --git a/tests/replica.py b/tests/replica.py index e25bbd371..a90386d56 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -11,7 +11,6 @@ module_name = 'replica' - class ReplicaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -33,6 +32,11 @@ def test_replica_stream_ptrack_backup(self): pg_options={ 'ptrack_enable': 'on'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + master.slow_start() self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -149,6 +153,11 @@ def test_replica_archive_page_backup(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -290,6 +299,11 @@ def test_basic_make_replica_via_restore(self): pg_options={ 'archive_timeout': '10s'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -343,6 +357,11 @@ def test_take_backup_from_delayed_replica(self): initdb_params=['--data-checksums'], pg_options={'archive_timeout': '10s'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -445,6 +464,11 @@ def test_replica_promote(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -530,6 +554,11 @@ def test_replica_stop_lsn_null_offset(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -612,6 +641,11 @@ def test_replica_stop_lsn_null_offset_next_record(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -706,6 +740,11 @@ def test_archive_replica_null_offset(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -790,6 +829,11 @@ def test_archive_replica_not_null_offset(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -875,6 +919,11 @@ def test_replica_toast(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -969,6 +1018,11 @@ def test_replica_promote_1(self): 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) # set replica True, so archive_mode 'always' is used. From dfdab27907d7301418fe201261e1289ca6376836 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 24 Oct 2019 18:01:31 +0300 Subject: [PATCH 1053/2107] tests: one more fix for 9.5 --- tests/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index 98a8e30b1..d5875dbb5 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2006,6 +2006,10 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'node', node, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) + if self.get_version(node) < 90600: + self.del_test_dir(module_name, fname) + return + # Restore as replica replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) @@ -2300,4 +2304,4 @@ def test_streaming_timeout(self): log_content) # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) From 386c2cb1b4226ebe4cc498b9cba473e173dbbd94 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 24 Oct 2019 18:28:18 +0300 Subject: [PATCH 1054/2107] Version 2.2.4 --- Documentation.md | 2 +- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation.md b/Documentation.md index 156eacc70..f32d3e784 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.2.3 +Current version - 2.2.4 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e48c39e62..f084e30a7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -205,8 +205,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.3" -#define AGENT_PROTOCOL_VERSION 20203 +#define PROGRAM_VERSION "2.2.4" +#define AGENT_PROTOCOL_VERSION 20204 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 9c025d295..e8cdf00c2 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.3 \ No newline at end of file +pg_probackup 2.2.4 \ No newline at end of file From 9a8285441e80a40785f13505f795654b7bd01204 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Oct 2019 19:59:46 +0300 Subject: [PATCH 1055/2107] [Issue #138] test coverage --- tests/helpers/ptrack_helpers.py | 26 +++++++++++ tests/restore.py | 82 +++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e3ce3c92e..07c1ad426 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1131,6 +1131,32 @@ def set_archiving( self.set_auto_conf(node, options) + def get_restore_command(self, backup_dir, instance, node): + + # parse postgresql.auto.conf + restore_command = '' + if os.name == 'posix': + restore_command += '{0} archive-get -B {1} --instance={2} '.format( + self.probackup_path, backup_dir, instance) + + elif os.name == 'nt': + restore_command += '"{0}" archive-get -B {1} --instance={2} '.format( + self.probackup_path.replace("\\","\\\\"), + backup_dir.replace("\\","\\\\"), instance) + + # don`t forget to kill old_binary after remote ssh release + if self.remote: + restore_command += '--remote-proto=ssh ' + restore_command += '--remote-host=localhost ' + + if os.name == 'posix': + restore_command += '--wal-file-path=%p --wal-file-name=%f' + + elif os.name == 'nt': + restore_command += '--wal-file-path="%p" --wal-file-name="%f"' + + return restore_command + def set_auto_conf(self, node, options): # parse postgresql.auto.conf diff --git a/tests/restore.py b/tests/restore.py index b0c21817c..b1634b516 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3246,3 +3246,85 @@ def test_missing_database_map(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_stream_restore_command_options(self): + """ + correct handling of restore command options + when restoring STREAM backup + + 1. Restore STREAM backup with --restore-command only + parameter, check that PostgreSQL recovery uses + restore_command to obtain WAL from archive. + + 2. Restore STREAM backup wuth --restore-command + as replica, check that PostgreSQL recovery uses + restore_command to obtain WAL from archive. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'max_wal_size': '32MB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # Take FULL + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + node.pgbench_init(scale=1) + + # restore backup + node.cleanup() + shutil.rmtree(os.path.join(node.logs_dir)) + +# self.restore_node(backup_dir, 'node', node) + restore_cmd = self.get_restore_command(backup_dir, 'node', node) + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command={0}'.format(restore_cmd), + '--recovery-target-action=pause', + '--recovery-target=latest']) + + self.assertTrue( + os.path.isfile(recovery_conf), + "File {0} do not exists".format(recovery_conf)) + + node.slow_start() + + exit(1) + +# self.set_config( +# backup_dir ,'node', +# options=['--restore-command="cp {0}/%f %p"'.format(wal_dir)]) +# +# # restore delta backup +# node.cleanup() +# self.restore_node( +# backup_dir, 'node', node, options=['--recovery-target=immediate']) +# +# self.assertTrue( +# os.path.isfile(recovery_conf), +# "File {0} do not exists".format(recovery_conf)) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From 4c52738cce51f12ef643057c614261d7c395fa3c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 27 Oct 2019 23:07:44 +0300 Subject: [PATCH 1056/2107] README: update --- README.md | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6a0eb6d71..dbfd86e59 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,18 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.2.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.3) +[2.2.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.4) + +## Documentation + +Documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.3). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.4). ### Linux Installation +#### pg_probackup for vanilla PostgreSQL ```shell #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list @@ -86,6 +91,26 @@ yum install pg_probackup-{11,10,9.6,9.5}-debuginfo yumdownloader --source pg_probackup-{11,10,9.6,9.5} ``` +#### pg_probackup for PostgresPro Standart and Enterprise +```shell +#DEB Ubuntu|Debian Packages +sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list +sudo wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update +sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} + +#RPM Centos Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm +yum install pg_probackup-{std,ent}-{11,10,9.6} + +#RPM RHEL Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm +yum install pg_probackup-{std,ent}-{11,10,9.6} + +#RPM Oracle Linux Packages +rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm +yum install pg_probackup-{std,ent}-{11,10,9.6} +``` + Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). ## Building from source @@ -109,11 +134,6 @@ SET PATH=%PATH%;C:\msys64\usr\bin gen_probackup_project.pl C:\path_to_postgresql_source_tree ``` -## Documentation - -Documentation for the latest version can be found at [github](https://github.com/postgrespro/pg_probackup/blob/2.2.3/Documentation.md). -Documentation for the current devel version can also be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). - ## License This module available under the [license](LICENSE) similar to [PostgreSQL](https://www.postgresql.org/about/license/). From c7110fdcb3123ef3ad466dc3f9d35e9ff7c4380e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 28 Oct 2019 00:49:44 +0300 Subject: [PATCH 1057/2107] Readme: update --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dbfd86e59..2803489d2 100644 --- a/README.md +++ b/README.md @@ -97,18 +97,22 @@ yumdownloader --source pg_probackup-{11,10,9.6,9.5} sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list sudo wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). From 3d4c2f23e5f8d5748c9eef45bb999c89e4f99f03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 28 Oct 2019 12:12:15 +0300 Subject: [PATCH 1058/2107] [Issue #134] use new logging infrastructure introduced in PG12 --- Makefile | 15 +++++++++------ src/pg_probackup.h | 4 ++++ src/utils/logger.c | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 167f04c07..627bf2861 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,11 @@ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o s OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ src/xlogreader.o -EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \ +EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c -INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h +INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -39,12 +39,18 @@ else srchome=$(top_srcdir) endif +ifneq (12,$(MAJORVERSION)) +EXTRA_CLEAN += src/logging.h +INCLUDES += src/logging.h +endif + ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) OBJS += src/walmethods.o EXTRA_CLEAN += src/walmethods.c src/walmethods.h INCLUDES += src/walmethods.h endif + PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} @@ -70,10 +76,7 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ -ifeq (12,$(MAJORVERSION)) -src/logging.h: $(top_srcdir)/src/include/common/logging.h - rm -f $@ && $(LN_S) $(srchome)/src/include/common/logging.h $@ -else +ifneq (12,$(MAJORVERSION)) src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ endif diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f084e30a7..8d6b1a998 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -17,6 +17,10 @@ #include "access/xlog_internal.h" #include "utils/pg_crc.h" +#if PG_VERSION_NUM >= 120000 +#include "common/logging.h" +#endif + #ifdef FRONTEND #undef FRONTEND #include diff --git a/src/utils/logger.c b/src/utils/logger.c index 0e0cba29d..c485af864 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -88,6 +88,31 @@ init_logger(const char *root_path, LoggerConfig *config) canonicalize_path(config->log_directory); logger_config = *config; + +#if PG_VERSION_NUM >= 120000 + /* Setup logging for functions from other modules called by pg_probackup */ + pg_logging_init(PROGRAM_NAME); + + switch (logger_config.log_level_console) + { + case VERBOSE: + pg_logging_set_level(PG_LOG_DEBUG); + break; + case INFO: + case NOTICE: + case LOG: + pg_logging_set_level(PG_LOG_INFO); + break; + case WARNING: + pg_logging_set_level(PG_LOG_WARNING); + break; + case ERROR: + pg_logging_set_level(PG_LOG_ERROR); + break; + default: + break; + }; +#endif } static void From 8134a6866f05a94f2be44a69ea8254634a644eec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 28 Oct 2019 14:14:27 +0300 Subject: [PATCH 1059/2107] [Issue #139] fix incorrect "Zratio" calculation in show --archive --- src/show.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/show.c b/src/show.c index 3ea8e3710..9782ec06d 100644 --- a/src/show.c +++ b/src/show.c @@ -835,7 +835,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, /* Zratio (compression ratio) */ if (tlinfo->size != 0) - zratio = (float) ((xlog_seg_size*tlinfo->n_xlog_files)/tlinfo->size); + zratio = ((float)xlog_seg_size*tlinfo->n_xlog_files) / tlinfo->size; snprintf(row->zratio, lengthof(row->n_segments), "%.2f", zratio); widths[cur] = Max(widths[cur], strlen(row->zratio)); @@ -1003,7 +1003,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add_key(buf, "zratio", json_level); if (tlinfo->size != 0) - zratio = (float) ((xlog_seg_size*tlinfo->n_xlog_files)/tlinfo->size); + zratio = ((float)xlog_seg_size*tlinfo->n_xlog_files) / tlinfo->size; appendPQExpBuffer(buf, "%.2f", zratio); if (tlinfo->closest_backup != NULL) From 56736b1d2ad3357d1d2a8c8b88ea5de0e7c6d0a1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 28 Oct 2019 16:13:58 +0300 Subject: [PATCH 1060/2107] make compiler happy --- src/pg_probackup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8d6b1a998..2c31c57b3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -531,7 +531,7 @@ typedef struct BackupPageHeader #define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) #define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ - snprintf(fname, MAXFNAMELEN, "%08X%08X", \ + snprintf(fname, 20, "%08X%08X", \ (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \ (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes))) @@ -547,7 +547,7 @@ typedef struct BackupPageHeader #define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ XLByteInSeg(xlrp, logSegNo) #define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ - snprintf(fname, MAXFNAMELEN, "%08X%08X",\ + snprintf(fname, 20, "%08X%08X",\ (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \ (uint32) ((logSegNo) % XLogSegmentsPerXLogId)) From e0871de361ee204bc81e72e9783291813f62493d Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 29 Oct 2019 18:10:31 +0300 Subject: [PATCH 1061/2107] PGPRO-3138: Added #undef open(a, b, c) for Windows build --- src/utils/file.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/file.c b/src/utils/file.c index 62d35f03b..0ee7aaac2 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -38,6 +38,11 @@ typedef struct /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) +#if defined(WIN32) +#undef open(a, b, c) +#undef fopen(a, b) +#endif + /* Use specified file descriptors as stdin/stdout for FIO functions */ void fio_redirect(int in, int out, int err) { From 7244ee3b86ffc10b8312e1f8755543c817ed4bde Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 30 Oct 2019 18:16:00 +0300 Subject: [PATCH 1062/2107] [Issue #138] always set "restore_command" in recovery.conf if "--restore-command" option is provided --- src/restore.c | 75 +++++++++++++++++++++++++++++++++++++----------- tests/restore.py | 46 ++++++++++++++++------------- 2 files changed, 85 insertions(+), 36 deletions(-) diff --git a/src/restore.c b/src/restore.c index 9591f7ede..8b2924dda 100644 --- a/src/restore.c +++ b/src/restore.c @@ -842,7 +842,10 @@ restore_files(void *arg) return NULL; } -/* Create recovery.conf with given recovery target parameters */ +/* + * Create recovery.conf (probackup_recovery.conf in case of PG12) + * with given recovery target parameters + */ static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, @@ -851,9 +854,17 @@ create_recovery_conf(time_t backup_id, { char path[MAXPGPATH]; FILE *fp; - bool need_restore_conf; + bool pitr_requested; bool target_latest; bool target_immediate; + bool restore_command_provided = false; + char restore_command_guc[16384]; + + if (instance_config.restore_command && + (pg_strcasecmp(instance_config.restore_command, "none") != 0)) + { + restore_command_provided = true; + } /* restore-target='latest' support */ target_latest = rt->target_stop != NULL && @@ -862,16 +873,39 @@ create_recovery_conf(time_t backup_id, target_immediate = rt->target_stop != NULL && strcmp(rt->target_stop, "immediate") == 0; - need_restore_conf = !backup->stream || rt->time_string || + /* + * Note that setting restore_command alone interpreted + * as PITR with target - "until all available WAL is replayed". + * We do this because of the following case: + * The user is restoring STREAM backup as replica but + * also relies on WAL archive to catch-up with master. + * If restore_command is provided, then it should be + * added to recovery config. + * In this scenario, "would be" replica will replay + * all WAL segments available in WAL archive, after that + * it will try to connect to master via repprotocol. + * + * The risk is obvious, what if masters current state is + * in "the past" relatively to latest state in the archive? + * We will get a replica that is "in the future" to the master. + * We accept this risk because its probability is low. + */ + pitr_requested = !backup->stream || rt->time_string || rt->xid_string || rt->lsn_string || rt->target_name || - target_immediate || target_latest; + target_immediate || target_latest || restore_command_provided; /* No need to generate recovery.conf at all. */ - if (!(need_restore_conf || params->restore_as_replica)) + if (!(pitr_requested || params->restore_as_replica)) { /* * Restoring STREAM backup without PITR and not as replica, * recovery.signal and standby.signal for PG12 are not needed + * + * We do not add "include" option in this case because + * here we are creating empty "probackup_recovery.conf" + * to handle possible already existing "include" + * directive pointing to "probackup_recovery.conf". + * If don`t do that, recovery will fail. */ pg12_recovery_config(backup, false); return; @@ -901,13 +935,10 @@ create_recovery_conf(time_t backup_id, #endif /* construct restore_command */ - if (need_restore_conf) + if (pitr_requested) { - char restore_command_guc[16384]; - - /* If restore_command is provided, use it */ - if (instance_config.restore_command && - (pg_strcasecmp(instance_config.restore_command, "none") != 0)) + /* If restore_command is provided, use it. Otherwise construct it from scratch. */ + if (restore_command_provided) sprintf(restore_command_guc, "%s", instance_config.restore_command); else { @@ -937,10 +968,6 @@ create_recovery_conf(time_t backup_id, } } - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", - restore_command_guc); - /* * We've already checked that only one of the four following mutually * exclusive options is specified, so the order of calls is insignificant. @@ -996,13 +1023,29 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); } + if (pitr_requested) + { + elog(LOG, "Setting restore_command to '%s'", restore_command_guc); + fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); + } + if (fio_fflush(fp) != 0 || fio_fclose(fp)) elog(ERROR, "cannot write file \"%s\": %s", path, strerror(errno)); #if PG_VERSION_NUM >= 120000 - if (need_restore_conf) + /* + * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. + * In older versions presense of recovery.conf alone was enough. + * To keep behaviour consistent with older versions, + * we are forced to create "recovery.signal" + * even when only restore_command is provided. + * Presense of "recovery.signal" by itself determine only + * one thing: do PostgreSQL must switch to a new timeline + * after successfull recovery or not? + */ + if (pitr_requested) { elog(LOG, "creating recovery.signal file"); snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); diff --git a/tests/restore.py b/tests/restore.py index b1634b516..02a3bd6d7 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3247,8 +3247,8 @@ def test_missing_database_map(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") - def test_stream_restore_command_options(self): + # @unittest.skip("skip") + def test_stream_restore_command_option(self): """ correct handling of restore command options when restoring STREAM backup @@ -3291,40 +3291,46 @@ def test_stream_restore_command_options(self): node.pgbench_init(scale=1) + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + node.safe_psql( + 'postgres', + 'create table t1()') + # restore backup node.cleanup() shutil.rmtree(os.path.join(node.logs_dir)) -# self.restore_node(backup_dir, 'node', node) restore_cmd = self.get_restore_command(backup_dir, 'node', node) self.restore_node( backup_dir, 'node', node, options=[ - '--restore-command={0}'.format(restore_cmd), - '--recovery-target-action=pause', - '--recovery-target=latest']) + '--restore-command={0}'.format(restore_cmd)]) self.assertTrue( os.path.isfile(recovery_conf), - "File {0} do not exists".format(recovery_conf)) + "File '{0}' do not exists".format(recovery_conf)) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_signal = os.path.join(node.data_dir, 'recovery.signal') + self.assertTrue( + os.path.isfile(recovery_signal), + "File '{0}' do not exists".format(recovery_signal)) node.slow_start() - exit(1) + node.safe_psql( + 'postgres', + 'select * from t1') + + timeline_id = node.safe_psql( + 'postgres', + 'select timeline_id from pg_control_checkpoint()').rstrip() -# self.set_config( -# backup_dir ,'node', -# options=['--restore-command="cp {0}/%f %p"'.format(wal_dir)]) -# -# # restore delta backup -# node.cleanup() -# self.restore_node( -# backup_dir, 'node', node, options=['--recovery-target=immediate']) -# -# self.assertTrue( -# os.path.isfile(recovery_conf), -# "File {0} do not exists".format(recovery_conf)) + self.assertEqual('2', timeline_id) # Clean after yourself self.del_test_dir(module_name, fname) \ No newline at end of file From b93317ae49cdd1d800172d4e4e064e393c30db55 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 30 Oct 2019 19:27:49 +0300 Subject: [PATCH 1063/2107] [SGML Documentation] initial commit --- doc/pgprobackup.xml | 5174 +++++++++++++++++++++++++++++++++++++++++++ doc/probackup.xml | 12 + doc/stylesheet.xsl | 48 + 3 files changed, 5234 insertions(+) create mode 100644 doc/pgprobackup.xml create mode 100644 doc/probackup.xml create mode 100644 doc/stylesheet.xsl diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml new file mode 100644 index 000000000..e2ebd739f --- /dev/null +++ b/doc/pgprobackup.xml @@ -0,0 +1,5174 @@ + + + + + pg_probackup + + + + pg_probackup + 1 + Application + + + + pg_probackup + manage backup and recovery of PostgreSQL database clusters + + + + + + pg_probackup + + + + pg_probackup + + command + + + pg_probackup + + backup_dir + + + pg_probackup + + backup_dir + data_dir + instance_name + + + pg_probackup + + backup_dir + instance_name + + + pg_probackup + + backup_dir + instance_name + option + + + pg_probackup + + backup_dir + instance_name + backup_id + option + + + pg_probackup + + backup_dir + instance_name + + + + pg_probackup + + backup_dir + option + + + pg_probackup + + backup_dir + instance_name + backup_mode + option + + + pg_probackup + + backup_dir + instance_name + option + + + pg_probackup + + backup_dir + instance_name + data_dir + + + + pg_probackup + + backup_dir + option + + + pg_probackup + + backup_dir + instance_name + backup_id + option + + + pg_probackup + + backup_dir + instance_name + + backup_id + + + + + option + + + pg_probackup + + backup_dir + instance_name + wal_file_path + wal_file_name + option + + + pg_probackup + + backup_dir + instance_name + wal_file_path + wal_file_name + option + + + + + + + Description + + + pg_probackup is a utility to manage backup and + recovery of PostgreSQL database clusters. + It is designed to perform periodic backups of the PostgreSQL + instance that enable you to restore the server in case of a failure. + pg_probackup supports PostgreSQL 9.5 or higher. + + + + + Overview + + + Installation and Setup + + + Command-Line Reference + + + Usage + + + + + + + + Overview + + + + As compared to other backup solutions, pg_probackup offers the + following benefits that can help you implement different backup + strategies and deal with large amounts of data: + + + + + Incremental backup: page-level incremental backup allows you + to save disk space, speed up backup and restore. With three + different incremental modes you can plan the backup strategy + in accordance with your data flow + + + + + Validation: automatic data consistency checks and on-demand + backup validation without actual data recovery + + + + + Verification: on-demand verification of PostgreSQL instance + via dedicated command checkdb + + + + + Retention: managing WAL archive and backups in accordance with + retention policies — Time and/or Redundancy based, with two + retention methods: delete expired and + merge expired. Additionally you can design + you own retention policy by setting 'time to live' for backups + + + + + Parallelization: running backup, restore, merge, delete, + verificaton and validation processes on multiple parallel + threads + + + + + Compression: storing backup data in a compressed state to save + disk space + + + + + Deduplication: saving disk space by not copying the not + changed non-data files (_vm, _fsm, etc) + + + + + Remote operations: backup PostgreSQL instance located on + remote machine or restore backup on it + + + + + Backup from replica: avoid extra load on the master server by + taking backups from a standby + + + + + External directories: add to backup content of directories + located outside of the PostgreSQL data directory (PGDATA), + such as scripts, configs, logs and pg_dump files + + + + + Backup Catalog: get list of backups and corresponding meta + information in plain or + json formats + + + + + Archive Catalog: get list of all WAL timelines and + corresponding meta information in plain or + json formats + + + + + Partial Restore: restore only the specified databases or skip + the specified databases. + + + + + To manage backup data, pg_probackup creates a + backup catalog. This is a directory that stores + all backup files with additional meta information, as well as WAL + archives required for point-in-time recovery. You can store + backups for different instances in separate subdirectories of a + single backup catalog. + + + Using pg_probackup, you can take full or incremental + backups: + + + + + FULL backups contain all the data files required to restore + the database cluster. + + + + + Incremental backups only store the data that has changed since + the previous backup. It allows to decrease the backup size and + speed up backup and restore operations. pg_probackup supports + the following modes of incremental backups: + + + + + DELTA backup. In this mode, pg_probackup reads all data + files in the data directory and copies only those pages + that has changed since the previous backup. Note that this + mode can impose read-only I/O pressure equal to a full + backup. + + + + + PAGE backup. In this mode, pg_probackup scans all WAL + files in the archive from the moment the previous full or + incremental backup was taken. Newly created backups + contain only the pages that were mentioned in WAL records. + This requires all the WAL files since the previous backup + to be present in the WAL archive. If the size of these + files is comparable to the total size of the database + cluster files, speedup is smaller, but the backup still + takes less space. You have to configure WAL archiving as + explained in the section + Setting + up continuous WAL archiving to make PAGE backups. + + + + + PTRACK backup. In this mode, PostgreSQL tracks page + changes on the fly. Continuous archiving is not necessary + for it to operate. Each time a relation page is updated, + this page is marked in a special PTRACK bitmap for this + relation. As one page requires just one bit in the PTRACK + fork, such bitmaps are quite small. Tracking implies some + minor overhead on the database server operation, but + speeds up incremental backups significantly. + + + + + + + pg_probackup can take only physical online backups, and online + backups require WAL for consistent recovery. So regardless of the + chosen backup mode (FULL, PAGE or DELTA), any backup taken with + pg_probackup must use one of the following + WAL delivery modes: + + + + + ARCHIVE. Such backups rely + on + continuous + archiving to ensure consistent recovery. This is the + default WAL delivery mode. + + + + + STREAM. Such backups + include all the files required to restore the cluster to a + consistent state at the time the backup was taken. Regardless + of + continuous + archiving been set up or not, the WAL segments required + for consistent recovery are streamed (hence STREAM) via + replication protocol during backup and included into the + backup files. Because of that backups of this WAL mode are + called autonomous or + standalone. + + + + + Limitations + + pg_probackup currently has the following limitations: + + + + Only PostgreSQL of versions 9.5 and newer are supported. + + + + + Currently remode mode of operations is not supported on + Windows systems. + + + + + On Unix systems backup of PostgreSQL verions =< 10 is + possible only by the same OS user PostgreSQL server is + running by. For example, if PostgreSQL server is running by + user postgres, then backup must be run + by user postgres. If backup is running + in + remote + mode using ssh, then this + limitation apply differently: value for + option should be + postgres. + + + + + During backup of PostgreSQL 9.5 functions + pg_create_restore_point(text) and + pg_switch_xlog() will be executed only if + backup role is superuser. Because of that backup of a + cluster with low amount of WAL traffic with non-superuser + role may take more time than backup of the same cluster with + superuser role. + + + + + The PostgreSQL server from which the backup was taken and + the restored server must be compatible by the + block_size + and + wal_block_size + parameters and have the same major release number. Also + depending on cluster configuration PostgreSQL itself may + apply additional restrictions such as CPU architecture + platform and libc/libicu versions. + + + + + Incremental chain can span only within one timeline. So if + you have backup incremental chain taken from replica and it + gets promoted, you would be forced to take another FULL + backup. + + + + + + + + + Installation and Setup + + Once you have pg_probackup installed, complete the following + setup: + + + + + Initialize the backup catalog. + + + + + Add a new backup instance to the backup catalog. + + + + + Configure the database cluster to enable pg_probackup backups. + + + + + Optionally, configure SSH for running pg_probackup operations + in remote mode. + + + + + Initializing the Backup Catalog + + pg_probackup stores all WAL and backup files in the + corresponding subdirectories of the backup catalog. + + + To initialize the backup catalog, run the following command: + + +pg_probackup init -B backup_dir + + + Where backup_dir is the path to backup + catalog. If the backup_dir already exists, + it must be empty. Otherwise, pg_probackup returns an error. + + + The user launching pg_probackup must have full access to + backup_dir directory. + + + pg_probackup creates the backup_dir backup + catalog, with the following subdirectories: + + + + + wal/ — directory for WAL files. + + + + + backups/ — directory for backup files. + + + + + Once the backup catalog is initialized, you can add a new backup + instance. + + + + Adding a New Backup Instance + + pg_probackup can store backups for multiple database clusters in + a single backup catalog. To set up the required subdirectories, + you must add a backup instance to the backup catalog for each + database cluster you are going to back up. + + + To add a new backup instance, run the following command: + + +pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] + + + Where: + + + + + data_dir is the data directory of the + cluster you are going to back up. To set up and use + pg_probackup, write access to this directory is required. + + + + + instance_name is the name of the + subdirectories that will store WAL and backup files for this + cluster. + + + + + The optional parameters + remote_options + should be used if data_dir is located + on remote machine. + + + + + pg_probackup creates the instance_name + subdirectories under the backups/ and wal/ directories of + the backup catalog. The + backups/instance_name directory contains + the pg_probackup.conf configuration file that controls + pg_probackup settings for this backup instance. If you run this + command with the + remote_options, used + parameters will be added to pg_probackup.conf. + + + For details on how to fine-tune pg_probackup configuration, see + the section Configuring + pg_probackup. + + + The user launching pg_probackup must have full access to + backup_dir directory and at least read-only + access to data_dir directory. If you + specify the path to the backup catalog in the + BACKUP_PATH environment variable, you can + omit the corresponding option when running pg_probackup + commands. + + + + For PostgreSQL >= 11 it is recommended to use + allow-group-access + feature, so backup can be done by any OS user in the same + group as the cluster owner. In this case the user should have + read permissions on the cluster directory. + + + + + Configuring the Database Cluster + + Although pg_probackup can be used by a superuser, it is + recommended to create a separate role with the minimum + permissions required for the chosen backup strategy. In these + configuration instructions, the backup role + is used as an example. + + + To perform backup, the following + permissions for role backup are required + only in database used for + connection to PostgreSQL server: + + + For PostgreSQL 9.5: + + +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +COMMIT; + + + For PostgreSQL 9.6: + + +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; +COMMIT; + + + For PostgreSQL >= 10: + + +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; +COMMIT; + + + In the + pg_hba.conf + file, allow connection to database cluster on behalf of the + backup role. + + + Since pg_probackup needs to read cluster files directly, + pg_probackup must be started by (in case of remote backup - + connected to) OS user that has read access to all files and + directories inside the data directory (PGDATA) you are going to + back up. + + + Depending on whether you are plan to take + autonomous and/or + archive backups, PostgreSQL + cluster configuration will differ, as specified in the sections + below. To back up the database cluster from a standby server, + run pg_probackup in remote mode or create PTRACK backups, + additional setup is required. + + + For details, see the sections + Setting up STREAM + Backups, + Setting up + continuous WAL archiving, + Setting up Backup + from Standby, + Configuring the + Remote Mode, + Setting up Partial + Restore and + Setting up PTRACK + Backups. + + + + Setting up STREAM Backups + + To set up the cluster for + STREAM backups, complete the + following steps: + + + + + Grant the REPLICATION privilege to the backup role: + + +ALTER ROLE backup WITH REPLICATION; + + + + + In the + pg_hba.conf + file, allow replication on behalf of the + backup role. + + + + + Make sure the parameter + max_wal_senders + is set high enough to leave at least one session available + for the backup process. + + + + + Set the parameter + wal_level + to be higher than minimal. + + + + + If you are planning to take PAGE backups in STREAM mode or + perform PITR with STREAM backups, you still have to configure + WAL archiving as explained in the section + Setting up + continuous WAL archiving. + + + Once these steps are complete, you can start taking FULL, PAGE, + DELTA and PTRACK backups with + STREAM WAL mode. + + + + Setting up continuous WAL archiving + + Making backups in PAGE backup mode, performing + PITR + and making backups with + ARCHIVE WAL delivery mode + require + continuous + WAL archiving to be enabled. To set up continuous + archiving in the cluster, complete the following steps: + + + + + Make sure the + wal_level + parameter is higher than minimal. + + + + + If you are configuring archiving on master, + archive_mode + must be set to on or + always. To perform archiving on standby, + set this parameter to always. + + + + + Set the + archive_command + parameter, as follows: + + +archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' + + + + + Where backup_dir and + instance_name refer to the already + initialized backup catalog instance for this database cluster + and optional parameters + remote_options should + be used to archive WAL to the remote host. For details about all + possible archive-push parameters, see the + section archive-push. + + + Once these steps are complete, you can start making backups with + ARCHIVE WAL-mode, backups in + PAGE backup mode and perform + PITR. + + + Current state of WAL Archive can be obtained via + show command. For details, see the + sections Viewing + WAL Archive information. + + + If you are planning to make PAGE backups and/or backups with + ARCHIVE WAL mode from a + standby of a server, that generates small amount of WAL traffic, + without long waiting for WAL segment to fill up, consider + setting + archive_timeout + PostgreSQL parameter on + master. It is advisable to set the value of this + setting slightly lower than pg_probackup parameter + (default 5 min), so there + should be enough time for rotated segment to be streamed to + replica and send to archive before backup is aborted because of + . + + + + Using pg_probackup command + archive-push for + continuous archiving is optional. You can use any other tool + you like as long as it delivers WAL segments into + backup_dir/wal/instance_name + directory. If compression is used, it should be + gzip, and .gz suffix in filename is + mandatory. + + + + + Instead of + archive_mode+archive_command + method you may opt to use the utility + pg_receivewal. + In this case pg_receivewal -D directory + option should point to + backup_dir/wal/instance_name + directory. WAL compression that could be done by pg_receivewal + is supported by pg_probackup. + Zero Data Loss archive strategy can be + achieved only by using pg_receivewal. + + + + + Setting up Backup from Standby + + For PostgreSQL 9.6 or higher, pg_probackup can take backups from + a standby server. This requires the following additional setup: + + + + + On the standby server, set the parameter + hot_standby + to on. + + + + + On the master server, set the parameter + full_page_writes + to on. + + + + + To perform autonomous backups on standby, complete all steps + in section Setting + up STREAM Backups + + + + + To perform archive backups on standby, complete all steps in + section + Setting + up continuous WAL archiving + + + + + Once these steps are complete, you can start taking FULL, PAGE, + DELTA or PTRACK backups with appropriate WAL delivery mode: + ARCHIVE or STREAM, from the standby server. + + + Backup from the standby server has the following limitations: + + + + + If the standby is promoted to the master during backup, the + backup fails. + + + + + All WAL records required for the backup must contain + sufficient full-page writes. This requires you to enable + full_page_writes on the master, and not + to use a tools like pg_compresslog as + archive_command + to remove full-page writes from WAL files. + + + + + + Setting up Cluster Verification + + Logical verification of database cluster requires the following + additional setup. Role backup is used as an + example: + + + + + Install extension + amcheck + or + amcheck_next + in every database of the + cluster: + + +CREATE EXTENSION amcheck; + + + + + To perform logical verification the following permissions + are required in every + database of the cluster: + + + + +GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; +GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; + + + + Setting up Partial Restore + + If you are planning to use partial restore, complete the + following additional step: + + + + + Grant the read-only access to pg_catalog.pg_database to the + backup role only in database + used for connection to + PostgreSQL server: + + +GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; + + + + + + Configuring the Remote Mode + + pg_probackup supports the remote mode that allows to perform + backup, restore and WAL archiving operations remotely. In this + mode, the backup catalog is stored on a local system, while + PostgreSQL instance to backup and/or to restore is located on a + remote system. Currently the only supported remote protocol is + SSH. + + + Setup SSH + + If you are going to use pg_probackup in remote mode via ssh, + complete the following steps: + + + + + Install pg_probackup on both systems: + backup_host and + db_host. + + + + + For communication between the hosts setup the passwordless + SSH connection between backup user on + backup_host and + postgres user on + db_host: + + +[backup@backup_host] ssh-copy-id postgres@db_host + + + + + If you are planning to rely on + continuous + WAL archiving, then setup passwordless SSH + connection between postgres user on + db_host and backup + user on backup_host: + + +[postgres@db_host] ssh-copy-id backup@backup_host + + + + + Where: + + + + + backup_host is the system with + backup catalog. + + + + + db_host is the system with PostgreSQL + cluster. + + + + + backup is the OS user on + backup_host used to run pg_probackup. + + + + + postgres is the OS user on + db_host used to run PostgreSQL + cluster. Note, that for PostgreSQL versions >= 11, a + more secure approach can be used thanks to + allow-group-access + feature. + + + + + pg_probackup in remote mode via ssh works + as follows: + + + + + only the following commands can be launched in remote + mode: add-instance, + backup, + restore, + archive-push, + archive-get. + + + + + when started in remote mode the main pg_probackup process + on local system connects via ssh to remote system and + launches there number of agent proccesses equal to + specified value of option /. + + + + + the main pg_probackup process uses remote agents to access + remote files and transfer data between local and remote + systems. + + + + + remote agents are smart and capable of handling some logic + on their own to minimize the network traffic and number of + round-trips between hosts. + + + + + usually the main proccess is started on + backup_host and connects to + db_host, but in case of + archive-push and + archive-get commands the main process + is started on db_host and connects to + backup_host. + + + + + after completition of data transfer the remote agents are + terminated and ssh connections are closed. + + + + + if an error condition is encountered by a remote agent, + then all agents are terminated and error details are + reported by the main pg_probackup process, which exits + with error. + + + + + compression is always done on + db_host. + + + + + decompression is always done on + backup_host. + + + + + + You can improse + additional + restrictions on ssh settings to protect the system + in the event of account compromise. + + + + + + Setting up PTRACK Backups + + Backup mode PTACK can be used only on Postgres Pro Standard and + Postgres Pro Enterprise installations or patched vanilla + PostgreSQL. Links to ptrack patches can be found + here. + + + If you are going to use PTRACK backups, complete the following + additional steps: + + + + + Set the parameter ptrack_enable to + on. + + + + + Grant the rights to execute ptrack + functions to the backup role + in every database of the + cluster: + + +GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; + + + + + The backup role must have access to all + the databases of the cluster. + + + + + + + + Usage + + Creating a Backup + + To create a backup, run the following command: + + +pg_probackup backup -B backup_dir --instance instance_name -b backup_mode + + + Where backup_mode can take one of the + following values: + + + + + FULL — creates a full backup that contains all the data + files of the cluster to be restored. + + + + + DELTA — reads all data files in the data directory and + creates an incremental backup for pages that have changed + since the previous backup. + + + + + PAGE — creates an incremental PAGE backup based on the WAL + files that have generated since the previous full or + incremental backup was taken. Only changed blocks are readed + from data files. + + + + + PTRACK — creates an incremental PTRACK backup tracking page + changes on the fly. + + + + + When restoring a cluster from an incremental backup, + pg_probackup relies on the parent full backup and all the + incremental backups between them, which is called + the backup chain. You must create at least + one full backup before taking incremental ones. + + + ARCHIVE mode + + ARCHIVE is the default WAL delivery mode. + + + For example, to make a FULL backup in ARCHIVE mode, run: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL + + + Unlike backup in STREAM mode, ARCHIVE backup rely on + continuous + archiving to provide WAL segments required to restore + the cluster to a consistent state at the time the backup was + taken. + + + During backup pg_probackup + ensures that WAL files containing WAL records between Start + LSN and Stop LSN are actually exists in + backup_dir/wal/instance_name + directory. Also pg_probackup ensures that WAL records between + Start LSN and Stop LSN can be parsed. This precations + eliminates the risk of silent WAL corruption. + + + + STREAM mode + + STREAM is the optional WAL delivery mode. + + + For example, to make a FULL backup in STREAM mode, add the + flag to the command from the + previous example: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot + + + The optional flag ensures that + the required segments remain available if the WAL is rotated + before the backup is complete. + + + Unlike backup in ARCHIVE mode, STREAM backup include all the + WAL segments required to restore the cluster to a consistent + state at the time the backup was taken. + + + During backup pg_probackup + streams WAL files containing WAL records between Start LSN and + Stop LSN to + backup_dir/backups/instance_name/backup_id/database/pg_wal directory. Also pg_probackup + ensures that WAL records between Start LSN and Stop LSN can be + parsed. This precations eliminates the risk of silent WAL + corruption. + + + Even if you are using + continuous + archiving, STREAM backups can still be useful in the + following cases: + + + + + STREAM backups can be restored on the server that has no + file access to WAL archive. + + + + + STREAM backups enable you to restore the cluster state at + the point in time for which WAL files in archive are no + longer available. + + + + + Backup in STREAM mode can be taken from standby of a + server, that generates small amount of WAL traffic, + without long waiting for WAL segment to fill up. + + + + + + Page validation + + If + data + checksums are enabled in the database cluster, + pg_probackup uses this information to check correctness of + data files during backup. While reading each page, + pg_probackup checks whether the calculated checksum coincides + with the checksum stored in the page header. This guarantees + that the PostgreSQL instance and backup itself are free of + corrupted pages. Note that pg_probackup reads database files + directly from filesystem, so under heavy write load during + backup it can show false positive checksum failures because of + partial writes. In case of page checksumm mismatch, page is + readed again and checksumm comparison repeated. + + + Page is considered corrupted if checksumm comparison failed + more than 100 times, in this case backup is aborted. + + + Redardless of data checksums been enabled or not, pg_probackup + always check page header "sanity". + + + + External directories + + To back up a directory located outside of the data directory, + use the optional parameter + that specifies the path to this directory. If you would like + to add more than one external directory, provide several paths + separated by colons, on Windows system paths must be separated + by semicolon instead. + + + For example, to include /etc/dir1/ and + /etc/dir2/ directories into the full + backup of your instance_name instance + that will be stored under the backup_dir + directory, run: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 + + + For example, to include C:\dir1\ and + C:\dir2\ directories into the full backup + of your instance_name instance that will + be stored under the backup_dir directory + on Windows system, run: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 + + + pg_probackup creates a separate subdirectory in the backup + directory for each external directory. Since external + directories included into different backups do not have to be + the same, when you are restoring the cluster from an + incremental backup, only those directories that belong to this + particular backup will be restored. Any external directories + stored in the previous backups will be ignored. + + + To include the same directories into each backup of your + instance, you can specify them in the pg_probackup.conf + configuration file using the + set-config command with the + option. + + + + + Verifying a Cluster + + To verify that PostgreSQL database cluster is free of + corruption, run the following command: + + +pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] + + + This physical verification works similar to + page validation that is + done during backup with several differences: + + + + + checkdb is read-only + + + + + if corrupted page is detected, checkdb is + not aborted, but carry on, until all pages in the cluster + are validated + + + + + checkdb do not strictly require + the backup catalog, so it can be used + to verify database clusters that are + not + added to the + backup catalog. + + + + + If backup_dir and + instance_name are omitted, then + connection options and + data_dir must be provided via environment + variables or command-line options. + + + Physical verification cannot detect logical inconsistencies, + missing and nullified blocks or entire files, repercussions from + PostgreSQL bugs and other wicked anomalies. Extensions + amcheck + and + amcheck_next + provide a partial solution to these problems. + + + If you would like, in addition to physical verification, to + verify all indexes in all databases using these extensions, you + can specify flag when running + checkdb command: + + +pg_probackup checkdb -D data_dir --amcheck + + + Physical verification can be skipped if + flag is used. For + logical only verification backup_dir and + data_dir are optional, only + connection options are + mandatory: + + +pg_probackup checkdb --amcheck --skip-block-validation [connection_options] + + + Logical verification can be done more thoroughly with flag + by checking that all heap + tuples that should be indexed are actually indexed, but at the + higher cost of CPU, memory and I/O comsumption. + + + + Validating a Backup + + pg_probackup calculates checksums for each file in a backup + during backup process. The process of checking checksumms of + backup data files is called + the backup validation. By default validation + is run immediately after backup is taken and right before + restore, to detect possible backup corruption. + + + If you would like to skip backup validation, you can specify the + flag when running + backup and + restore commands. + + + To ensure that all the required backup files are present and can + be used to restore the database cluster, you can run the + validate command with the exact + recovery target + options you are going to use for recovery. + + + For example, to check that you can restore the database cluster + from a backup copy up to the specified xid transaction ID, run + this command: + + +pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 + + + If validation completes successfully, pg_probackup displays the + corresponding message. If validation fails, you will receive an + error message with the exact time, transaction ID and LSN up to + which the recovery is possible. + + + If you specify backup_id via + -i/--backup-id option, then only backup copy + with specified backup ID will be validated. If + backup_id is specified with + recovery target + options then validate will check whether it is possible + to restore the specified backup to the specified + recovery target. + + + For example, to check that you can restore the database cluster + from a backup copy with backup_id up to the + specified timestamp, run this command: + + +pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' + + + If backup_id belong to incremental backup, + then all its parents starting from FULL backup will be + validated. + + + If you omit all the parameters, all backups are validated. + + + + Restoring a Cluster + + To restore the database cluster from a backup, run the restore + command with at least the following options: + + +pg_probackup restore -B backup_dir --instance instance_name -i backup_id + + + Where: + + + + + backup_dir is the backup catalog that + stores all backup files and meta information. + + + + + instance_name is the backup instance + for the cluster to be restored. + + + + + backup_id specifies the backup to + restore the cluster from. If you omit this option, + pg_probackup uses the latest valid backup available for the + specified instance. If you specify an incremental backup to + restore, pg_probackup automatically restores the underlying + full backup and then sequentially applies all the necessary + increments. + + + + + If the cluster to restore contains tablespaces, pg_probackup + restores them to their original location by default. To restore + tablespaces to a different location, use the + / option. Otherwise, + restoring the cluster on the same host will fail if tablespaces + are in use, because the backup would have to be written to the + same directories. + + + When using the / + option, you must provide absolute paths to the old and new + tablespace directories. If a path happens to contain an equals + sign (=), escape it with a backslash. This option can be + specified multiple times for multiple tablespaces. For example: + + +pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir + + + Once the restore command is complete, start the database + service. + + + If you are restoring an STREAM backup, the restore is complete + at once, with the cluster returned to a self-consistent state at + the point when the backup was taken. For ARCHIVE backups, + PostgreSQL replays all available archived WAL segments, so the + cluster is restored to the latest state possible. You can change + this behavior by using the + recovery target + options with the restore command. Note + that using the recovery + target options when restoring STREAM backup is possible + if the WAL archive is available at least starting from the time + the STREAM backup was taken. + + + To restore cluster on remote host see the section + Using + pg_probackup in the Remote Mode. + + + + By default, the restore + command validates the specified backup before restoring the + cluster. If you run regular backup validations and would like + to save time when restoring the cluster, you can specify the + flag to skip validation and + speed up the recovery. + + + + Partial Restore + + If you have enabled + partial + restore before taking backups, you can restore or + exclude from restore the arbitrary number of specific + databases using + partial restore + options with the restore + commands. + + + To restore only one or more databases, run the restore command + with the following options: + + +pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name + + + The option can be specified + multiple times. For example, to restore only databases + db1 and db2, run the + following command: + + +pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 + + + To exclude one or more specific databases from restore, run + the following options: + + +pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name + + + The option can be specified + multiple times. For example, to exclude the databases + db1 and db2 from + restore, run the following command: + + +pg_probackup restore -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 + + + Partial restore rely on lax behaviour of PostgreSQL recovery + process toward truncated files. Files of excluded databases + restored as null sized files, allowing recovery to work + properly. After successfull starting of PostgreSQL cluster, + you must drop excluded databases using + DROP DATABASE command. + + + + The databases template0 and + template1 are always restored. + + + + + + Performing Point-in-Time (PITR) Recovery + + If you have enabled + continuous + WAL archiving before taking backups, you can restore the + cluster to its state at an arbitrary point in time (recovery + target) using recovery + target options with the + restore and + validate commands. + + + If / option is omitted, + pg_probackup automatically chooses the backup that is the + closest to the specified recovery target and starts the restore + process, otherwise pg_probackup will try to restore + backup_id to the specified recovery target. + + + + + To restore the cluster state at the exact time, specify the + option, in the + timestamp format. For example: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' + + + + + To restore the cluster state up to a specific transaction + ID, use the option: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 + + + + + To restore the cluster state up to a specific LSN, use + option: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 + + + + + To restore the cluster state up to a specific named restore + point, use option: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' + + + + + To restore the backup to the latest state available in + archive, use option + with latest value: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' + + + + + To restore the cluster to the earliest point of consistency, + use option with + immediate value: + + +pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' + + + + + + Using <application>pg_probackup</application> in the Remote Mode + + pg_probackup supports the remote mode that allows to perform + backup and restore + operations remotely via SSH. In this mode, the backup catalog is + stored on a local system, while PostgreSQL instance to be backed + up is located on a remote system. You must have pg_probackup + installed on both systems. + + + Do note that pg_probackup rely on passwordless SSH connection + for communication between the hosts. + + + The typical workflow is as follows: + + + + + On your backup host, configure pg_probackup as explained in + the section + Installation and + Setup. For the + add-instance and + set-config commands, make + sure to specify remote + options that point to the database host with the + PostgreSQL instance. + + + + + If you would like to take remote backup in + PAGE mode, or rely + on ARCHIVE WAL delivery + mode, or use + PITR, + then configure continuous WAL archiving from database host + to the backup host as explained in the section + Setting + up continuous WAL archiving. For the + archive-push and + archive-get commands, you + must specify the remote + options that point to backup host with backup + catalog. + + + + + Run backup or + restore commands with + remote options + on backup host. + pg_probackup connects to the remote system via SSH and + creates a backup locally or restores the previously taken + backup on the remote system, respectively. + + + + + For example, to create archive full backup using remote mode + through SSH connection to user postgres on + host with address 192.168.0.2 via port + 2302, run: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 + + + For example, to restore latest backup on remote system using + remote mode through SSH connection to user + postgres on host with address + 192.168.0.2 via port 2302, + run: + + +pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 + + + Restoring of ARCHIVE backup or performing PITR in remote mode + require additional information: destination address, port and + username for establishing ssh connection + from a host with database + to a host with backup + catalog. This information will be used by + restore_command to copy via ssh WAL segments + from archive to PostgreSQL pg_wal directory. + + + To solve this problem you can use + Remote Wal Archive + Options. + + + For example, to restore latest backup on remote system using + remote mode through SSH connection to user + postgres on host with address + 192.168.0.2 via port 2302 + and user backup on backup catalog host with + address 192.168.0.3 via port + 2303, run: + + +pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup + + + Provided arguments will be used to construct restore_command + in recovery.conf: + + +# recovery.conf generated by pg_probackup 2.1.5 +restore_command = 'pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' + + + Alternatively you can use + option to provide an entire restore_command: + + +pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' + + + + The remote backup mode is currently unavailable for + Windows systems. + + + + + Running <application>pg_probackup</application> on Parallel Threads + + Backup, + restore, + merge, + delete, + checkdb and + validate processes can be + executed on several parallel threads. This can significantly + speed up pg_probackup operation given enough resources (CPU + cores, disk and network bandwidth). + + + Parallel execution is controlled by the + -j/--threads command line option. For + example, to create a backup using four parallel threads, run: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 + + + + Parallel restore applies only to copying data from the + backup catalog to the data directory of the cluster. When + PostgreSQL server is started, WAL records need to be replayed, + and this cannot be done in parallel. + + + + + Configuring <application>pg_probackup</application> + + Once the backup catalog is initialized and a new backup instance + is added, you can use the pg_probackup.conf configuration file + located in the + backup_dir/backups/instance_name + directory to fine-tune pg_probackup configuration. + + + For example, backup and + checkdb commands uses a regular + PostgreSQL connection. To avoid specifying these options each + time on the command line, you can set them in the + pg_probackup.conf configuration file using the + set-config command. + + + + It is not recommended + to edit pg_probackup.conf manually. + + + + Initially, pg_probackup.conf contains the following settings: + + + + + PGDATA — the path to the data directory of the cluster to + back up. + + + + + system-identifier — the unique identifier of the PostgreSQL + instance. + + + + + Additionally, you can define + remote, + retention, + logging and + compression settings + using the set-config command: + + +pg_probackup set-config -B backup_dir --instance instance_name +[--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] + + + To view the current settings, run the following command: + + +pg_probackup show-config -B backup_dir --instance instance_name + + + You can override the settings defined in pg_probackup.conf when + running pg_probackup commands + via corresponding environment variables and/or command line + options. + + + + Specifying Connection Settings + + If you define connection settings in the pg_probackup.conf + configuration file, you can omit connection options in all the + subsequent pg_probackup commands. However, if the corresponding + environment variables are set, they get higher priority. The + options provided on the command line overwrite both environment + variables and configuration file settings. + + + If nothing is given, the default values are taken. By default + pg_probackup tries to use local connection via Unix domain + socket (localhost on Windows) and tries to get the database name + and the user name from the PGUSER environment variable or the + current OS user name. + + + + Managing the Backup Catalog + + With pg_probackup, you can manage backups from the command line: + + + + + View backup + information + + + + + View WAL + Archive Information + + + + + Validate backups + + + + + Merge backups + + + + + Delete backups + + + + + Viewing Backup Information + + To view the list of existing backups for every instance, run + the command: + + +pg_probackup show -B backup_dir + + + pg_probackup displays the list of all the available backups. + For example: + + +BACKUP INSTANCE 'node' +====================================================================================================================================== + Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK + node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK + node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK + node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK + node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK + + + For each backup, the following information is provided: + + + + + Instance — the instance name. + + + + + Version — PostgreSQL major version. + + + + + ID — the backup identifier. + + + + + Recovery time — the earliest moment for which you can + restore the state of the database cluster. + + + + + Mode — the method used to take this backup. Possible + values: FULL, PAGE, DELTA, PTRACK. + + + + + WAL Mode — the WAL delivery mode. Possible values: STREAM + and ARCHIVE. + + + + + TLI — timeline identifiers of current backup and its + parent. + + + + + Time — the time it took to perform the backup. + + + + + Data — the size of the data files in this backup. This + value does not include the size of WAL files. In case of + STREAM backup the total size of backup can be calculated + as Data + WAL. + + + + + WAL — the uncompressed size of WAL files required to apply + by PostgreSQL recovery process to reach consistency. + + + + + Zratio — compression ratio calculated as + uncompressed-bytes / data-bytes. + + + + + Start LSN — WAL log sequence number corresponding to the + start of the backup process. REDO point for PostgreSQL + recovery process to start from. + + + + + Stop LSN — WAL log sequence number corresponding to the + end of the backup process. Consistency point for + PostgreSQL recovery process. + + + + + Status — backup status. Possible values: + + + + + OK — the backup is complete and valid. + + + + + DONE — the backup is complete, but was not validated. + + + + + RUNNING — the backup is in progress. + + + + + MERGING — the backup is being merged. + + + + + DELETING — the backup files are being deleted. + + + + + CORRUPT — some of the backup files are corrupted. + + + + + ERROR — the backup was aborted because of an + unexpected error. + + + + + ORPHAN — the backup is invalid because one of its + parent backups is corrupt or missing. + + + + + + + You can restore the cluster from the backup only if the backup + status is OK or DONE. + + + To get more detailed information about the backup, run the + show with the backup ID: + + +pg_probackup show -B backup_dir --instance instance_name -i backup_id + + + The sample output is as follows: + + +#Configuration +backup-mode = FULL +stream = false +compress-alg = zlib +compress-level = 1 +from-replica = false + +#Compatibility +block-size = 8192 +wal-block-size = 8192 +checksum-version = 1 +program-version = 2.1.3 +server-version = 10 + +#Result backup info +timelineid = 1 +start-lsn = 0/04000028 +stop-lsn = 0/040000f8 +start-time = '2017-05-16 12:57:29' +end-time = '2017-05-16 12:57:31' +recovery-xid = 597 +recovery-time = '2017-05-16 12:57:31' +expire-time = '2020-05-16 12:57:31' +data-bytes = 22288792 +wal-bytes = 16777216 +uncompressed-bytes = 39961833 +pgdata-bytes = 39859393 +status = OK +parent-backup-id = 'PT8XFX' +primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' + + + Detailed output has additional attributes: + + + + compress-alg — compression algorithm used during backup. Possible values: + zlib, pglz, none. + + + + + compress-level — compression level used during backup. + + + + + from-replica — the fact that backup was taken from standby server. Possible values: + 1, 0. + + + + + block-size — block_size + setting of PostgreSQL cluster at the moment of backup start. + + + + + checksum-version — the fact that PostgreSQL cluster, from + which backup is taken, has enabled + data + block checksumms. Possible values: 1, 0. + + + + + program-version — full version of pg_probackup binary used to create backup. + + + + + start-time — the backup starting time. + + + + + end-time — the backup ending time. + + + + + expire-time — if the + backup was pinned, then until this point in time the backup + cannot be removed by retention purge. + + + + + uncompressed-bytes — size of the data files before adding page headers and applying + compression. You can evaluate the effectiveness of compression + by comparing uncompressed-bytes to data-bytes if + compression if used. + + + + + pgdata-bytes — size of the PostgreSQL + cluster data files at the time of backup. You can evaluate the + effectiveness of incremental backup by comparing + pgdata-bytes to uncompressed-bytes. + + + + + recovery-xid — + current transaction id at the moment of backup ending. + + + + + parent-backup-id — backup ID of parent backup. Available only + for incremental backups. + + + + + primary_conninfo — libpq conninfo + used for connection to PostgreSQL cluster during backup. The + password is not included. + + + + + + To get more detailed information about the backup in json + format, run the show with the backup ID: + + +pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id + + + The sample output is as follows: + + +[ + { + "instance": "node", + "backups": [ + { + "id": "PT91HZ", + "parent-backup-id": "PT8XFX", + "backup-mode": "DELTA", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": false, + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.3", + "server-version": "10", + "current-tli": 16, + "parent-tli": 2, + "start-lsn": "0/8000028", + "stop-lsn": "0/8000160", + "start-time": "2019-06-17 18:25:11+03", + "end-time": "2019-06-17 18:25:16+03", + "recovery-xid": 0, + "recovery-time": "2019-06-17 18:25:15+03", + "data-bytes": 106733, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } +] + + + + Viewing WAL Archive Information + + To view the information about WAL archive for every instance, + run the command: + + +pg_probackup show -B backup_dir [--instance instance_name] --archive + + + pg_probackup displays the list of all the available WAL files + grouped by timelines. For example: + + +ARCHIVE INSTANCE 'node' +=================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=================================================================================================================== + 5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK + 4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK + 3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK + 2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED + 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK + + + For each backup, the following information is provided: + + + + + TLI — timeline identifier. + + + + + Parent TLI — identifier of timeline TLI branched off. + + + + + Switchpoint — LSN of the moment when the timeline branched + off from "Parent TLI". + + + + + Min Segno — number of the first existing WAL segment + belonging to the timeline. + + + + + Max Segno — number of the last existing WAL segment + belonging to the timeline. + + + + + N segments — number of WAL segments belonging to the + timeline. + + + + + Size — the size files take on disk. + + + + + Zratio — compression ratio calculated as N segments * + wal_seg_size / Size. + + + + + N backups — number of backups belonging to the timeline. + To get the details about backups, use the JSON format. + + + + + Status — archive status for this exact timeline. Possible + values: + + + + + OK — all WAL segments between Min and Max are present. + + + + + DEGRADED — some WAL segments between Min and Max are + lost. To get details about lost files, use the JSON + format. + + + + + + + To get more detailed information about the WAL archive in the JSON + format, run the command: + + +pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json + + + The sample output is as follows: + + +[ + { + "instance": "replica", + "timelines": [ + { + "tli": 5, + "parent-tli": 1, + "switchpoint": "0/B000000", + "min-segno": "000000000000000B", + "max-segno": "000000000000000C", + "n-segments": 2, + "size": 685320, + "zratio": 48.00, + "closest-backup-id": "PXS92O", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 4, + "parent-tli": 3, + "switchpoint": "0/18000000", + "min-segno": "0000000000000018", + "max-segno": "000000000000001A", + "n-segments": 3, + "size": 648625, + "zratio": 77.00, + "closest-backup-id": "PXS9CE", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 3, + "parent-tli": 2, + "switchpoint": "0/15000000", + "min-segno": "0000000000000015", + "max-segno": "0000000000000017", + "n-segments": 3, + "size": 648911, + "zratio": 77.00, + "closest-backup-id": "PXS9CE", + "status": "OK", + "lost-segments": [], + "backups": [] + }, + { + "tli": 2, + "parent-tli": 1, + "switchpoint": "0/B000108", + "min-segno": "000000000000000B", + "max-segno": "0000000000000015", + "n-segments": 5, + "size": 892173, + "zratio": 94.00, + "closest-backup-id": "PXS92O", + "status": "DEGRADED", + "lost-segments": [ + { + "begin-segno": "000000000000000D", + "end-segno": "000000000000000E" + }, + { + "begin-segno": "0000000000000010", + "end-segno": "0000000000000012" + } + ], + "backups": [ + { + "id": "PXS9CE", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 2, + "parent-tli": 0, + "start-lsn": "0/C000028", + "stop-lsn": "0/C000160", + "start-time": "2019-09-13 21:43:26+03", + "end-time": "2019-09-13 21:43:30+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:43:29+03", + "data-bytes": 104674852, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + }, + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "0000000000000001", + "max-segno": "000000000000000A", + "n-segments": 10, + "size": 8774805, + "zratio": 19.00, + "closest-backup-id": "", + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "PXS92O", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "true", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/4000028", + "stop-lsn": "0/6000028", + "start-time": "2019-09-13 21:37:36+03", + "end-time": "2019-09-13 21:38:45+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:30+03", + "data-bytes": 25987319, + "wal-bytes": 50331648, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } + ] + }, + { + "instance": "master", + "timelines": [ + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "0000000000000001", + "max-segno": "000000000000000B", + "n-segments": 11, + "size": 8860892, + "zratio": 20.00, + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "PXS92H", + "parent-backup-id": "PXS92C", + "backup-mode": "PAGE", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/4000028", + "stop-lsn": "0/50000B8", + "start-time": "2019-09-13 21:37:29+03", + "end-time": "2019-09-13 21:37:31+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:30+03", + "data-bytes": 1328461, + "wal-bytes": 33554432, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + }, + { + "id": "PXS92C", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "none", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.1.5", + "server-version": "10", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/2000028", + "stop-lsn": "0/2000160", + "start-time": "2019-09-13 21:37:24+03", + "end-time": "2019-09-13 21:37:29+03", + "recovery-xid": 0, + "recovery-time": "2019-09-13 21:37:28+03", + "data-bytes": 24871902, + "wal-bytes": 16777216, + "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", + "status": "OK" + } + ] + } + ] + } +] + + + Most fields are consistent with the plain format, with some + exceptions: + + + + + size is in bytes. + + + + + closest-backup-id attribute contain ID of valid backup + closest to the timeline, located on some of the previous + timelines. This backup is the closest starting point to + reach the timeline from other timelines by PITR. Closest + backup always has a valid status, either OK or DONE. If + such backup do not exists, then string is empty. + + + + + DEGRADED timelines contain lost-segments array with + information about intervals of missing segments. In OK + timelines lost-segments array is empty. + + + + + N backups attribute is replaced with backups array + containing backups belonging to the timeline. If timeline + has no backups, then backups array is empty. + + + + + + + Configuring Retention Policy + + With pg_probackup, you can set retention policies for backups + and WAL archive. All policies can be combined together in any + way. + + + Backup Retention Policy + + By default, all backup copies created with pg_probackup are + stored in the specified backup catalog. To save disk space, + you can configure retention policy and periodically clean up + redundant backup copies accordingly. + + + To configure retention policy, set one or more of the + following variables in the pg_probackup.conf file via + set-config: + + +--retention-redundancy=redundancy + + + Specifies the number of full backup + copies to keep in the backup catalog. + + +--retention-window=window + + + Defines the earliest point in time for which pg_probackup can + complete the recovery. This option is set in + the number of days from the + current moment. For example, if + retention-window=7, pg_probackup must + delete all backup copies that are older than seven days, with + all the corresponding WAL files. + + + If both and + options are set, + pg_probackup keeps backup copies that satisfy at least one + condition. For example, if you set + --retention-redundancy=2 and + --retention-window=7, pg_probackup purges + the backup catalog to keep only two full backup copies and all + backups that are newer than seven days: + + +pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 + + + To clean up the backup catalog in accordance with retention + policy, run: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-expired + + + pg_probackup deletes all backup copies that do not conform to + the defined retention policy. + + + If you would like to also remove the WAL files that are no + longer required for any of the backups, add the + flag: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal + + + + Alternatively, you can use the + , + , + flags and the + and + options together + with the backup command to + remove and merge the outdated backup copies once the new + backup is created. + + + + You can set or override the current retention policy by + specifying and + options directly when + running delete or backup + commands: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 + + + Since incremental backups require that their parent full + backup and all the preceding incremental backups are + available, if any of such backups expire, they still cannot be + removed while at least one incremental backup in this chain + satisfies the retention policy. To avoid keeping expired + backups that are still required to restore an active + incremental one, you can merge them with this backup using the + flag when running + backup or + delete commands. + + + Suppose you have backed up the node + instance in the backup_dir directory, + with the option is set + to 7, and you have the following backups + available on April 10, 2019: + + +BACKUP INSTANCE 'node' +=================================================================================================================================== + Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status +=================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK + -------------------------------------------------------retention window-------------------------------------------------------- + node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK + node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK + node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK + + + Even though P7XDHB and P7XDHU backups are outside the + retention window, they cannot be removed as it invalidates the + succeeding incremental backups P7XDJA and P7XDQV that are + still required, so, if you run the + delete command with the + flag, only the P7XDFT full + backup will be removed. + + + With the option, the P7XDJA + backup is merged with the underlying P7XDHU and P7XDHB backups + and becomes a full one, so there is no need to keep these + expired backups anymore: + + +pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired +pg_probackup show -B backup_dir + + +BACKUP INSTANCE 'node' +================================================================================================================================== + Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================== + node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK + node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK + node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK + + + + The Time field for the merged backup displays the time + required for the merge. + + + + + Backup Pinning + + If you have the necessity to exclude certain backups from + established retention policy then it is possible to pin a + backup for an arbitrary amount of time. Example: + + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d + + + This command will set expire-time of the + specified backup to 30 days starting from backup + recovery-time attribute. Basically + expire-time = + recovery-time + ttl. + + + Also you can set expire-time explicitly + using option. Example: + + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' + + + Alternatively you can use the and + options with the + backup command to pin newly + created backup: + + +pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d +pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' + + + You can determine the fact that backup is pinned and check due + expire time by looking up expire-time + attribute in backup metadata via + show command: + + +pg_probackup show -B backup_dir --instance instance_name -i backup_id + + + Pinned backup has expire-time attribute: + + +... +recovery-time = '2017-05-16 12:57:31' +expire-time = '2020-01-01 00:00:00+03' +data-bytes = 22288792 +... + + + You can unpin the backup by setting the + option to zero using + set-backup command. Example: + + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 + + + Only pinned backups have the expire-time + attribute in the backup metadata. + + + + Pinned incremental backup will also implicitly pin all + its parent backups. + + + + + WAL Archive Retention Policy + + By default, pg_probackup treatment of WAL Archive is very + conservative and only "redundant" WAL segments can + be purged, i.e. segments that cannot be applied to any of the + existing backups in the backup catalog. To save disk space, + you can configure WAL Archive retention policy, that allows to + keep WAL of limited depth measured in backups per timeline. + + + Suppose you have backed up the node + instance in the backup_dir directory with + configured + WAL + archiving: + + +pg_probackup show -B backup_dir --instance node + + +BACKUP INSTANCE 'node' +==================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +==================================================================================================================================== + node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK + node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK + node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK + node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK + node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK + + + The state of WAL archive can be determined by using + show command with + flag: + + +pg_probackup show -B backup_dir --instance node --archive + + +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK + + + General WAL purge without cannot + achieve much, only one segment can be removed: + + +pg_probackup delete -B backup_dir --instance node --delete-wal + + +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK + + + If you would like, for example, to keep only those WAL + segments that can be applied to the last valid backup, use the + option: + + +pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 + + +ARCHIVE INSTANCE 'node' +================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================ + 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK + + + Alternatively you can use the + option with the backup command: + + +pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal + + +ARCHIVE INSTANCE 'node' +=============================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================== + 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK + + + + + Merging Backups + + As you take more and more incremental backups, the total size of + the backup catalog can substantially grow. To save disk space, + you can merge incremental backups to their parent full backup by + running the merge command, specifying the backup ID of the most + recent incremental backup you would like to merge: + + +pg_probackup merge -B backup_dir --instance instance_name -i backup_id + + + This command merges the specified incremental backup to its + parent full backup, together with all incremental backups + between them. Once the merge is complete, the incremental + backups are removed as redundant. Thus, the merge operation is + virtually equivalent to retaking a full backup and removing all + the outdated backups, but it allows to save much time, + especially for large data volumes, I/O and network traffic in + case of + remote + backup. + + + Before the merge, pg_probackup validates all the affected + backups to ensure that they are valid. You can check the current + backup status by running the show + command with the backup ID: + + +pg_probackup show -B backup_dir --instance instance_name -i backup_id + + + If the merge is still in progress, the backup status is + displayed as MERGING. The merge is idempotent, so you can + restart the merge if it was interrupted. + + + + Deleting Backups + + To delete a backup that is no longer required, run the following + command: + + +pg_probackup delete -B backup_dir --instance instance_name -i backup_id + + + This command will delete the backup with the specified + backup_id, together with all the + incremental backups that descend from + backup_id if any. This way you can delete + some recent incremental backups, retaining the underlying full + backup and some of the incremental backups that follow it. + + + To delete obsolete WAL files that are not necessary to restore + any of the remaining backups, use the + flag: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-wal + + + To delete backups that are expired according to the current + retention policy, use the + flag: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-expired + + + Note that expired backups cannot be removed while at least one + incremental backup that satisfies the retention policy is based + on them. If you would like to minimize the number of backups + still required to keep incremental backups valid, specify the + flag when running this + command: + + +pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired + + + In this case, pg_probackup searches for the oldest incremental + backup that satisfies the retention policy and merges this + backup with the underlying full and incremental backups that + have already expired, thus making it a full backup. Once the + merge is complete, the remaining expired backups are deleted. + + + Before merging or deleting backups, you can run the + delete command with the + flag, which displays the status of + all the available backups according to the current retention + policy, without performing any irreversible actions. + + + + + +Command-Line Reference + + Commands + + This section describes pg_probackup commands. Some commands + require mandatory parameters and can take additional options. + Optional parameters encased in square brackets. For detailed + descriptions of options, see the section + Options. + + + version + +pg_probackup version + + + Prints pg_probackup version. + + + + help + +pg_probackup help [command] + + + Displays the synopsis of pg_probackup commands. If one of the + pg_probackup commands is specified, shows detailed information + about the options that can be used with this command. + + + + init + +pg_probackup init -B backup_dir [--help] + + + Initializes the backup catalog in + backup_dir that will store backup copies, + WAL archive and meta information for the backed up database + clusters. If the specified backup_dir + already exists, it must be empty. Otherwise, pg_probackup + displays a corresponding error message. + + + For details, see the secion + Initializing + the Backup Catalog. + + + + add-instance + +pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] + + + Initializes a new backup instance inside the backup catalog + backup_dir and generates the + pg_probackup.conf configuration file that controls + pg_probackup settings for the cluster with the specified + data_dir data directory. + + + For details, see the section + Adding a New + Backup Instance. + + + + del-instance + +pg_probackup del-instance -B backup_dir --instance instance_name [--help] + + + Deletes all backups and WAL files associated with the + specified instance. + + + + set-config + +pg_probackup set-config -B backup_dir --instance instance_name +[--help] [--pgdata=pgdata-path] +[--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] +[--compress-algorithm=compression_algorithm] [--compress-level=compression_level] +[-d dbname] [-h host] [-p port] [-U username] +[--archive-timeout=timeout] [--external-dirs=external_directory_path] +[--restore-command=cmdline] +[remote_options] [remote_archive_options] [logging_options] + + + Adds the specified connection, compression, retention, logging + and external directory settings into the pg_probackup.conf + configuration file, or modifies the previously defined values. + + + For all available settings, see the + Options section. + + + It is not recommended to + edit pg_probackup.conf manually. + + + + set-backup + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id +{--ttl=ttl | --expire-time=time} [--help] + + + Sets the provided backup-specific settings into the + backup.control configuration file, or modifies previously + defined values. + + + For all available settings, see the section + Pinning Options. + + + + show-config + +pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] + + + Displays the contents of the pg_probackup.conf configuration + file located in the + backup_dir/backups/instance_name + directory. You can specify the + --format=json option to return the result + in the JSON format. By default, configuration settings are + shown as plain text. + + + To edit pg_probackup.conf, use the + set-config command. + + + + show + +pg_probackup show -B backup_dir +[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] + + + Shows the contents of the backup catalog. If + instance_name and + backup_id are specified, shows detailed + information about this backup. You can specify the + --format=json option to return the result + in the JSON format. If option is + specified, shows the content of WAL archive of the backup + catalog. + + + By default, the contents of the backup catalog is shown as + plain text. + + + For details on usage, see the sections + Managing the + Backup Catalog and + Viewing WAL + Archive Information. + + + + backup + +pg_probackup backup -B backup_dir -b backup_mode --instance instance_name +[--help] [-j num_threads] [--progress] +[-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] +[--no-validate] [--skip-block-validation] +[-w --no-password] [-W --password] +[--archive-timeout=timeout] [--external-dirs=external_directory_path] +[connection_options] [compression_options] [remote_options] +[retention_options] [pinning_options] [logging_options] + + + Creates a backup copy of the PostgreSQL instance. The + backup_mode option specifies the backup + mode to use. + + + + + + + + + Specifies the backup mode to use. Possible values are: + + + + FULL — creates a full backup that contains all the data + files of the cluster to be restored. + + + + + DELTA — reads all data files in the data directory and + creates an incremental backup for pages that have changed + since the previous backup. + + + + + PAGE — creates an incremental PAGE backup based on the WAL + files that have changed since the previous full or + incremental backup was taken. + + + + + PTRACK — creates an incremental PTRACK backup tracking + page changes on the fly. + + + + + + + + + + + + + Spreads out the checkpoint over a period of time. By default, + pg_probackup tries to complete the checkpoint as soon as + possible. + + + + + + + + + Makes an STREAM backup that + includes all the necessary WAL files by streaming them from + the database server via replication protocol. + + + + + + + + + Creates a temporary physical replication slot for streaming + WAL from the backed up PostgreSQL instance. It ensures that + all the required WAL segments remain available if WAL is + rotated while the backup is in progress. This flag can only be + used together with the flag. + Default slot name is pg_probackup_slot, + which can be changed via option /. + + + + + + + + + + Specifies the replication slot for WAL streaming. This option + can only be used together with the + flag. + + + + + + + + + Includes the log directory into the backup. This directory + usually contains log messages. By default, log directory is + excluded. + + + + + + + + + + Includes the specified directory into the backup. This option + is useful to back up scripts, sql dumps and configuration + files located outside of the data directory. If you would like + to back up several external directories, separate their paths + by a colon on Unix and a semicolon on Windows. + + + + + + + + + Sets in seconds the timeout for WAL segment archiving and + streaming. By default pg_probackup waits 300 seconds. + + + + + + + + + + Disables block-level checksum verification to speed up backup. + + + + + + + + + Skips automatic validation after successfull backup. You can + use this flag if you validate backups regularly and would like + to save time when running backup operations. + + + + + + + + Additionally Connection + Options, Retention + Options, Pinning + Options, Remote + Mode Options, + Compression + Options, Logging + Options and Common + Options can be used. + + + For details on usage, see the section + Creating a Backup. + + + + restore + +pg_probackup restore -B backup_dir --instance instance_name +[--help] [-D data_dir] [-i backup_id] +[-j num_threads] [--progress] +[-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] +[-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] +[--restore-command=cmdline] +[recovery_options] [logging_options] [remote_options] +[partial_restore_options] [remote_archive_options] + + + Restores the PostgreSQL instance from a backup copy located in + the backup_dir backup catalog. If you + specify a recovery + target option, pg_probackup will find the closest + backup and restores it to the specified recovery target. + Otherwise, the most recent backup is used. + + + + + + + + + Writes a minimal recovery.conf in the output directory to + facilitate setting up a standby server. The password is not + included. If the replication connection requires a password, + you must specify the password manually. + + + + + + + + + + Relocates the tablespace from the OLDDIR to the NEWDIR + directory at the time of recovery. Both OLDDIR and NEWDIR must + be absolute paths. If the path contains the equals sign (=), + escape it with a backslash. This option can be specified + multiple times for multiple tablespaces. + + + + + + + + + Relocates an external directory included into the backup from + the OLDDIR to the NEWDIR directory at the time of recovery. + Both OLDDIR and NEWDIR must be absolute paths. If the path + contains the equals sign (=), escape it with a backslash. This + option can be specified multiple times for multiple + directories. + + + + + + + + + Skip external directories included into the backup with the + option. The contents of + these directories will not be restored. + + + + + + + + + Disables block-level checksum verification to speed up + validation. During automatic validation before restore only + file-level checksums will be verified. + + + + + + + + + Skips backup validation. You can use this flag if you validate + backups regularly and would like to save time when running + restore operations. + + + + + + + + + Set the + restore_command + parameter to specified command. Example: + --restore-command='cp /mnt/server/archivedir/%f "%p"' + + + + + + + + + Allows to ignore the invalid status of the backup. You can use + this flag if you for some reason have the necessity to restore + PostgreSQL cluster from corrupted or invalid backup. Use with + caution. + + + + + + + Additionally Recovery + Target Options, + Remote Mode + Options, + Remote WAL Archive + Options, Logging + Options, Partial + Restore and Common + Options can be used. + + + For details on usage, see the section + Restoring a + Cluster. + + + + checkdb + +pg_probackup checkdb +[-B backup_dir] [--instance instance_name] [-D data_dir] +[--help] [-j num_threads] [--progress] +[--skip-block-validation] [--amcheck] [--heapallindexed] +[connection_options] [logging_options] + + + Verifies the PostgreSQL database cluster correctness by + detecting physical and logical corruption. + + + + + + + + Performs logical verification of indexes for the specified + PostgreSQL instance if no corruption was found while checking + data files. You must have the amcheck + extention or the amcheck_next extension + installed in the database to check its indexes. For databases + without amcheck, index verification will be skipped. + + + + + + + + + Skip validation of data files. Can be used only with + flag, so only logical + verification of indexes is performed. + + + + + + + + + Checks that all heap tuples that should be indexed are + actually indexed. You can use this flag only together with the + flag. Can be used only with + amcheck extension of version 2.0 and + amcheck_next extension of any version. + + + + + + + Additionally Connection + Options and Logging + Options can be used. + + + For details on usage, see the section + Verifying a + Cluster. + + + + validate + +pg_probackup validate -B backup_dir +[--help] [--instance instance_name] [-i backup_id] +[-j num_threads] [--progress] +[--skip-block-validation] +[recovery_target_options] [logging_options] + + + Verifies that all the files required to restore the cluster + are present and not corrupted. If + instance_name is not specified, + pg_probackup validates all backups available in the backup + catalog. If you specify the instance_name + without any additional options, pg_probackup validates all the + backups available for this backup instance. If you specify the + instance_name with a + recovery target + options and/or a backup_id, + pg_probackup checks whether it is possible to restore the + cluster using these options. + + + For details, see the section + Validating a + Backup. + + + + merge + +pg_probackup merge -B backup_dir --instance instance_name -i backup_id +[--help] [-j num_threads] [--progress] +[logging_options] + + + Merges the specified incremental backup to its parent full + backup, together with all incremental backups between them, if + any. As a result, the full backup takes in all the merged + data, and the incremental backups are removed as redundant. + + + For details, see the section + Merging Backups. + + + + delete + +pg_probackup delete -B backup_dir --instance instance_name +[--help] [-j num_threads] [--progress] +[--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] +[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} +[--dry-run] +[logging_options] + + + Deletes backup with specified backip_id + or launches the retention purge of backups and archived WAL + that do not satisfy the current retention policies. + + + For details, see the sections + Deleting Backups, + Retention Options and + Configuring + Retention Policy. + + + + archive-push + +pg_probackup archive-push -B backup_dir --instance instance_name +--wal-file-path=wal_file_path --wal-file-name=wal_file_name +[--help] [--compress] [--compress-algorithm=compression_algorithm] +[--compress-level=compression_level] [--overwrite] +[remote_options] [logging_options] + + + Copies WAL files into the corresponding subdirectory of the + backup catalog and validates the backup instance by + instance_name and + system-identifier. If parameters of the + backup instance and the cluster do not match, this command + fails with the following error message: Refuse to push WAL + segment segment_name into archive. Instance parameters + mismatch. For each WAL file moved to the backup catalog, you + will see the following message in PostgreSQL logfile: + pg_probackup archive-push completed successfully. + + + If the files to be copied already exist in the backup catalog, + pg_probackup computes and compares their checksums. If the + checksums match, archive-push skips the corresponding file and + returns successful execution code. Otherwise, archive-push + fails with an error. If you would like to replace WAL files in + the case of checksum mismatch, run the archive-push command + with the flag. + + + Copying is done to temporary file with + .part suffix or, if + compression is + used, with .gz.part suffix. After copy is + done, atomic rename is performed. This algorihtm ensures that + failed archive-push will not stall continuous archiving and + that concurrent archiving from multiple sources into single + WAL archive has no risk of archive corruption. Copied to + archive WAL segments are synced to disk. + + + You can use archive-push in + archive_command + PostgreSQL parameter to set up + continous + WAl archiving. + + + For details, see sections + Archiving Options and + Compression + Options. + + + + archive-get + +pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name +[--help] [remote_options] [logging_options] + + + Copies WAL files from the corresponding subdirectory of the + backup catalog to the cluster's write-ahead log location. This + command is automatically set by pg_probackup as part of the + restore_command in recovery.conf when + restoring backups using a WAL archive. You do not need to set + it manually. + + + + + Options + + This section describes command-line options for pg_probackup + commands. If the option value can be derived from an environment + variable, this variable is specified below the command-line + option, in the uppercase. Some values can be taken from the + pg_probackup.conf configuration file located in the backup + catalog. + + + For details, see the section + Configuring + pg_probackup. + + + If an option is specified using more than one method, + command-line input has the highest priority, while the + pg_probackup.conf settings have the lowest priority. + + + Common Options + + The list of general options. + + + + + + +BACKUP_PATH + + + Specifies the absolute path to the backup catalog. Backup + catalog is a directory where all backup files and meta + information are stored. Since this option is required for most + of the pg_probackup commands, you are recommended to specify + it once in the BACKUP_PATH environment variable. In this case, + you do not need to use this option each time on the command + line. + + + + + + + +PGDATA + + + Specifies the absolute path to the data directory of the + database cluster. This option is mandatory only for the + add-instance command. + Other commands can take its value from the PGDATA environment + variable, or from the pg_probackup.conf configuration file. + + + + + + + + + + Specifies the unique identifier of the backup. + + + + + + + + + + Sets the number of parallel threads for backup, restore, + merge, validation and verification processes. + + + + + + + + + Shows the progress of operations. + + + + + + + + + Shows detailed information about the options that can be used + with this command. + + + + + + + + Recovery Target Options + + If + continuous + WAL archiving is configured, you can use one of these + options together with restore + or validate commands to + specify the moment up to which the database cluster must be + restored or validated. + + + + + + + + Defines when to stop the recovery: + + + + immediate value stops the recovery + after reaching the consistent state of the specified + backup, or the latest available backup if the + / option is omitted. + + + + + latest value continues the recovery + until all WAL segments available in the archive are + applied. + + + + + + Default value of depends + on WAL delivery method of restored backup, + immediate for STREAM backup and + latest for ARCHIVE. + + + + + + + + + Specifies a particular timeline to which recovery will + proceed. By default, the timeline of the specified backup is + used. + + + + + + + + + Specifies the LSN of the write-ahead log location up to which + recovery will proceed. Can be used only when restoring + database cluster of major version 10 or higher. + + + + + + + + + Specifies a named savepoint up to which to restore the cluster. + + + + + + + + + Specifies the timestamp up to which recovery will proceed. + + + + + + + + + Specifies the transaction ID up to which recovery will + proceed. + + + + + + + + + + Specifies whether to stop just after the specified recovery + target (true), or just before the recovery target (false). + This option can only be used together with + , + , + or + options. The default + depends on + recovery_target_inclusive + parameter. + + + + + + + + + Specifies + the + action the server should take when the recovery target + is reached. + + + Default: pause + + + + + + + + Retention Options + + You can use these options together with + backup and + delete commands. + + + For details on configuring retention policy, see the section + Configuring + Retention Policy. + + + + + + + + + + Specifies the number of full backup copies to keep in the data + directory. Must be a non-negative integer. The zero value disables + this setting. + + + Default: 0 + + + + + + + + + + Number of days of recoverability. Must be a non-negative integer. + The zero value disables this setting. + + + Default: 0 + + + + + + + + + Number of latest valid backups on every timeline that must + retain the ability to perform PITR. Must be a non-negative + integer. The zero value disables this setting. + + + Default: 0 + + + + + + + + + Deletes WAL files that are no longer required to restore the + cluster from any of the existing backups. + + + + + + + + + Deletes backups that do not conform to the retention policy + defined in the pg_probackup.conf configuration file. + + + + + + + + + + Merges the oldest incremental backup that satisfies the + requirements of retention policy with its parent backups that + have already expired. + + + + + + + + + Displays the current status of all the available backups, + without deleting or merging expired backups, if any. + + + + + + + + Pinning Options + + You can use these options together with + backup and + set-delete commands. + + + For details on backup pinning, see the section + Backup Pinning. + + + + + + + + Specifies the amount of time the backup should be pinned. + Must be a non-negative integer. The zero value unpin already + pinned backup. Supported units: ms, s, min, h, d (s by + default). Example: --ttl=30d. + + + + + + + + + Specifies the timestamp up to which the backup will stay + pinned. Must be a ISO-8601 complaint timestamp. Example: + --expire-time='2020-01-01 00:00:00+03' + + + + + + + + Logging Options + + You can use these options with any command. + + + + + + + + Controls which message levels are sent to the console log. + Valid values are verbose, + log, info, + warning, error and + off. Each level includes all the levels + that follow it. The later the level, the fewer messages are + sent. The off level disables console + logging. + + + Default: info + + + + All console log messages are going to stderr, so + output from show and + show-config commands do + not mingle with log messages. + + + + + + + + + + Controls which message levels are sent to a log file. Valid + values are verbose, log, + info, warning, + error, and off. Each + level includes all the levels that follow it. The later the + level, the fewer messages are sent. The off + level disables file logging. + + + Default: off + + + + + + + + + Defines the filenames of the created log files. The filenames + are treated as a strftime pattern, so you can use %-escapes to + specify time-varying filenames. + + + Default: pg_probackup.log + + + For example, if you specify the pg_probackup-%u.log pattern, + pg_probackup generates a separate log file for each day of the + week, with %u replaced by the corresponding decimal number: + pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, + and so on. + + + This option takes effect if file logging is enabled by the + option. + + + + + + + + + Defines the filenames of log files for error messages only. + The filenames are treated as a strftime pattern, so you can + use %-escapes to specify time-varying filenames. + + + Default: none + + + For example, if you specify the error-pg_probackup-%u.log + pattern, pg_probackup generates a separate log file for each + day of the week, with %u replaced by the corresponding decimal + number: error-pg_probackup-1.log for Monday, + error-pg_probackup-2.log for Tuesday, and so on. + + + This option is useful for troubleshooting and monitoring. + + + + + + + + + Defines the directory in which log files will be created. You + must specify the absolute path. This directory is created + lazily, when the first log message is written. + + + Default: $BACKUP_PATH/log/ + + + + + + + + + Maximum size of an individual log file. If this value is + reached, the log file is rotated once a pg_probackup command + is launched, except help and version commands. The zero value + disables size-based rotation. Supported units: kB, MB, GB, TB + (kB by default). + + + Default: 0 + + + + + + + + + Maximum lifetime of an individual log file. If this value is + reached, the log file is rotated once a pg_probackup command + is launched, except help and version commands. The time of the + last log file creation is stored in + $BACKUP_PATH/log/log_rotation. The zero value disables + time-based rotation. Supported units: ms, s, min, h, d (min by + default). + + + Default: 0 + + + + + + + + Connection Options + + You can use these options together with + backup and + checkdb commands. + + + All + libpq + environment variables are supported. + + + + + + +PGDATABASE + + + Specifies the name of the database to connect to. The + connection is used only for managing backup process, so you + can connect to any existing database. If this option is not + provided on the command line, PGDATABASE environment variable, + or the pg_probackup.conf configuration file, pg_probackup + tries to take this value from the PGUSER environment variable, + or from the current user name if PGUSER variable is not set. + + + + + + + +PGHOST + + + Specifies the host name of the system on which the server is + running. If the value begins with a slash, it is used as a + directory for the Unix domain socket. + + + Default: localhost + + + + + + + +PGPORT + + + Specifies the TCP port or the local Unix domain socket file + extension on which the server is listening for connections. + + + + Default: 5432 + + + + + + + +PGUSER + + + User name to connect as. + + + + + + + + + + Disables a password prompt. If the server requires password + authentication and a password is not available by other means + such as a + .pgpass + file or PGPASSWORD environment variable, the connection + attempt will fail. This flag can be useful in batch jobs and + scripts where no user is present to enter a password. + + + + + + + + + + + Forces a password prompt. + + + + + + + + Compression Options + + You can use these options together with + backup and + archive-push commands. + + + + + + + + Defines the algorithm to use for compressing data files. + Possible values are zlib, + pglz, and none. If set + to zlib or pglz, this option enables compression. By default, + compression is disabled. For the + archive-push command, the + pglz compression algorithm is not supported. + + + Default: none + + + + + + + + + Defines compression level (0 through 9, 0 being no compression + and 9 being best compression). This option can be used + together with option. + + + Default: 1 + + + + + + + + + Alias for --compress-algorithm=zlib and + --compress-level=1. + + + + + + + + Archiving Options + + These options can be used with + archive-push command in + archive_command + setting and archive-get + command in + restore_command + setting. + + + Additionally Remote Mode + Options and Logging + Options can be used. + + + + + + + + Provides the path to the WAL file in + archive_command and + restore_command. The %p + variable as value for this option is required for correct + processing. + + + + + + + + + Provides the name of the WAL file in + archive_command and + restore_command. The %f + variable as value is required for correct processing. + + + + + + + + + Overwrites archived WAL file. Use this flag together with the + archive-push command if + the specified subdirectory of the backup catalog already + contains this WAL file and it needs to be replaced with its + newer copy. Otherwise, archive-push reports that a WAL segment + already exists, and aborts the operation. If the file to + replace has not changed, archive-push skips this file + regardless of the flag. + + + + + + + + Remote Mode Options + + This section describes the options related to running + pg_probackup operations remotely via SSH. These options can be + used with add-instance, + set-config, + backup, + restore, + archive-push and + archive-get commands. + + + For details on configuring and usage of remote operation mode, + see the sections + Configuring the + Remote Mode and + Using + pg_probackup in the Remote Mode. + + + + + + + + Specifies the protocol to use for remote operations. Currently + only the SSH protocol is supported. Possible values are: + + + + + ssh enables the remote backup mode via + SSH. This is the default value. + + + + + none explicitly disables the remote + mode. + + + + + You can omit this option if the + option is specified. + + + + + + + + + + Specifies the remote host IP address or hostname to connect + to. + + + + + + + + + Specifies the remote host port to connect to. + + + Default: 22 + + + + + + + + + Specifies remote host user for SSH connection. If you omit + this option, the current user initiating the SSH connection is + used. + + + + + + + + + Specifies pg_probackup installation directory on the remote + system. + + + + + + + + + Specifies a string of SSH command-line options. For example, + the following options can used to set keep-alive for ssh + connections opened by pg_probackup: + --ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'. + Full list of possible options can be found on + ssh_config + manual page. + + + + + + + + Remote WAL Archive Options + + This section describes the options used to provide the + arguments for Remote Mode + Options in + archive-get used in + restore_command + command when restoring ARCHIVE backup or performing PITR. + + + + + + + + Provides the argument for + option in archive-get command. + + + + + + + + + + Provides the argument for + option in archive-get command. + + + Default: 22 + + + + + + + + + Provides the argument for + option in archive-get command. If you omit + this option, the the user running PostgreSQL cluster is used. + + + Default: PostgreSQL user + + + + + + + + Partial Restore Options + + This section describes the options related to partial restore + of a cluster from backup. These options can be used with + restore command. + + + + + + + + Specifies database name to exclude from restore. All other + databases in the cluster will be restored as usual, including + template0 and template1. + This option can be specified multiple times for multiple + databases. + + + + + + + + + Specifies database name to restore from backup. All other + databases in the cluster will not be restored, with exception + of template0 and + template1. This option can be specified + multiple times for multiple databases. + + + + + + + + Replica Options + + This section describes the options related to taking a backup + from standby. + + + + Starting from pg_probackup 2.0.24, backups can be + taken from standby without connecting to the master server, + so these options are no longer required. In lower versions, + pg_probackup had to connect to the master to determine + recovery time — the earliest moment for which you can + restore a consistent state of the database cluster. + + + + + + + + + Deprecated. Specifies the name of the database on the master + server to connect to. The connection is used only for managing + the backup process, so you can connect to any existing + database. Can be set in the pg_probackup.conf using the + set-config command. + + + Default: postgres, the default PostgreSQL database + + + + + + + + + Deprecated. Specifies the host name of the system on which the + master server is running. + + + + + + + + + Deprecated. Specifies the TCP port or the local Unix domain + socket file extension on which the master server is listening + for connections. + + + Default: 5432, the PostgreSQL default port + + + + + + + + + Deprecated. User name to connect as. + + + Default: postgres, + the PostgreSQL default user name + + + + + + + + + + Deprecated. Wait time for WAL segment streaming via + replication, in seconds. By default, pg_probackup waits 300 + seconds. You can also define this parameter in the + pg_probackup.conf configuration file using the + set-config command. + + + Default: 300 sec + + + + + + + + + + + Howto + + All examples below assume the remote mode of operations via + ssh. If you are planning to run backup and + restore operation locally then step + Setup passwordless SSH connection can be + skipped and all options can be + ommited. + + + Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup + 2.2.0. + + + + + backup_host — host with backup catalog. + + + + + backupman — user on + backup_host running all pg_probackup + operations. + + + + + /mnt/backups — directory on + backup_host where backup catalog is stored. + + + + + postgres_host — host with PostgreSQL + cluster. + + + + + postgres — user on + postgres_host which run PostgreSQL cluster. + + + + + /var/lib/postgresql/11/main — directory + on postgres_host where PGDATA of PostgreSQL + cluster is located. + + + + + backupdb — database used for connection + to PostgreSQL cluster. + + + + + Minimal Setup + + This setup is relying on autonomous FULL and DELTA backups. + + + + Setup passwordless SSH connection from + <literal>backup_host</literal> to + <literal>postgres_host</literal> + +[backupman@backup_host] ssh-copy-id postgres@postgres_host + + + + Setup <productname>PostgreSQL</productname> cluster + + It is recommended from security purposes to use separate + database for backup operations. + + +postgres=# +CREATE DATABASE backupdb; + + + Connect to backupdb database, create role + probackup and grant to it the following + permissions: + + +backupdb=# +BEGIN; +CREATE ROLE probackup WITH LOGIN REPLICATION; +GRANT USAGE ON SCHEMA pg_catalog TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO probackup; +COMMIT; + + + + Init the backup catalog + +[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups +INFO: Backup catalog '/mnt/backups' successfully inited + + + + Add instance <literal>pg-11</literal> to backup catalog + +[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main +INFO: Instance 'node' successfully inited + + + + Take FULL backup + +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ7YK2 +INFO: Backup PZ7YK2 data files are valid +INFO: Backup PZ7YK2 resident size: 196MB +INFO: Backup PZ7YK2 completed + + + + Lets take a look at the backup catalog + +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' + +BACKUP INSTANCE 'pg-11' +================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================== + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK + + + + Take incremental backup in DELTA mode + +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Parent backup: PZ7YK2 +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ7YMP +INFO: Backup PZ7YMP data files are valid +INFO: Backup PZ7YMP resident size: 32MB +INFO: Backup PZ7YMP completed + + + + Lets hide some parameters into config, so cmdline can be + less crowdy + +[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb + + + + Take another incremental backup in DELTA mode, omitting + some of the previous parameters: + +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream +INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Parent backup: PZ7YMP +INFO: Start transferring data files +INFO: Data files are transferred +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: Validating backup PZ7YR5 +INFO: Backup PZ7YR5 data files are valid +INFO: Backup PZ7YR5 resident size: 32MB +INFO: Backup PZ7YR5 completed + + + + Lets take a look at instance config + +[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' + +# Backup instance information +pgdata = /var/lib/postgresql/11/main +system-identifier = 6746586934060931492 +xlog-seg-size = 16777216 +# Connection parameters +pgdatabase = backupdb +pghost = postgres_host +pguser = probackup +# Replica parameters +replica-timeout = 5min +# Archive parameters +archive-timeout = 5min +# Logging parameters +log-level-console = INFO +log-level-file = OFF +log-filename = pg_probackup.log +log-rotation-size = 0 +log-rotation-age = 0 +# Retention parameters +retention-redundancy = 0 +retention-window = 0 +wal-depth = 0 +# Compression parameters +compress-algorithm = none +compress-level = 1 +# Remote access parameters +remote-proto = ssh +remote-host = postgres_host + + + Note, that we are getting default values for other options, + that were not overwritten by set-config command. + + + + Lets take a look at the backup catalog + +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' + +==================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +==================================================================================================================================== + node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK + node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK + node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK + + + + + + + + Versioning + + pg_probackup is following the + semantic versioning. + + + + + Authors + + + Postgres Professional, Moscow, Russia. + + + + Credits + + pg_probackup utility is based on pg_arman, + that was originally written by NTT and then developed and maintained by Michael Paquier. + + + + + + + \ No newline at end of file diff --git a/doc/probackup.xml b/doc/probackup.xml new file mode 100644 index 000000000..c839cb264 --- /dev/null +++ b/doc/probackup.xml @@ -0,0 +1,12 @@ + +]> + + + + pg_probackup Documentation +&pgprobackup; + + \ No newline at end of file diff --git a/doc/stylesheet.xsl b/doc/stylesheet.xsl new file mode 100644 index 000000000..332cb6f15 --- /dev/null +++ b/doc/stylesheet.xsl @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +book/reference toc, title + + + From 0608c7450dbd130ed6fd1da9fb3c5633aa5c911d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 30 Oct 2019 19:31:13 +0300 Subject: [PATCH 1064/2107] move to .xml --- doc/probackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/probackup.xml b/doc/probackup.xml index c839cb264..8ea3cdc46 100644 --- a/doc/probackup.xml +++ b/doc/probackup.xml @@ -1,7 +1,7 @@ + ]> From f8a6a26f989d56b82b7247a8fea6f45c6c91fbcb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Oct 2019 13:12:55 +0300 Subject: [PATCH 1065/2107] [Issue #140] report correctly the total size of data to delete during backup and WAL retention purge --- src/catalog.c | 1 + src/delete.c | 6 +++--- src/pg_probackup.h | 2 +- src/show.c | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 2c84d52b2..669efd9a3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -38,6 +38,7 @@ timelineInfoNew(TimeLineID tli) tlinfo->xlog_filelist = parray_new(); tlinfo->anchor_lsn = InvalidXLogRecPtr; tlinfo->anchor_tli = 0; + tlinfo->n_xlog_files = 0; return tlinfo; } diff --git a/src/delete.c b/src/delete.c index bc4c93b2c..be07554dc 100644 --- a/src/delete.c +++ b/src/delete.c @@ -35,7 +35,7 @@ do_delete(time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; - int size_to_delete = 0; + size_t size_to_delete = 0; char size_to_delete_pretty[20]; /* Get complete list of backups */ @@ -814,8 +814,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, char oldest_to_keep_str[20]; int rc; int i; - int wal_size_logical = 0; - int wal_size_actual = 0; + size_t wal_size_logical = 0; + size_t wal_size_actual = 0; char wal_pretty_size[20]; bool purge_all = false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2c31c57b3..fec8817ca 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -445,7 +445,7 @@ struct timelineInfo { * otherwise 0 */ XLogSegNo begin_segno; /* first present segment in this timeline */ XLogSegNo end_segno; /* last present segment in this timeline */ - int n_xlog_files; /* number of segments (only really existing) + size_t n_xlog_files; /* number of segments (only really existing) * does not include lost segments */ size_t size; /* space on disk taken by regular WAL files */ parray *backups; /* array of pgBackup sturctures with info diff --git a/src/show.c b/src/show.c index 9782ec06d..ed0cca16f 100644 --- a/src/show.c +++ b/src/show.c @@ -822,7 +822,7 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* N files */ - snprintf(row->n_segments, lengthof(row->n_segments), "%u", + snprintf(row->n_segments, lengthof(row->n_segments), "%lu", tlinfo->n_xlog_files); widths[cur] = Max(widths[cur], strlen(row->n_segments)); cur++; @@ -996,7 +996,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add_value(buf, "max-segno", tmp_buf, json_level, true); json_add_key(buf, "n-segments", json_level); - appendPQExpBuffer(buf, "%d", tlinfo->n_xlog_files); + appendPQExpBuffer(buf, "%lu", tlinfo->n_xlog_files); json_add_key(buf, "size", json_level); appendPQExpBuffer(buf, "%lu", tlinfo->size); From d2c60ca52ff21765c1ebfd98883476185b2c7e9a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 31 Oct 2019 20:12:33 +0300 Subject: [PATCH 1066/2107] Version 2.2.5 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fec8817ca..be8757a5b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -209,8 +209,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.4" -#define AGENT_PROTOCOL_VERSION 20204 +#define PROGRAM_VERSION "2.2.5" +#define AGENT_PROTOCOL_VERSION 20205 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e8cdf00c2..29b839daa 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.4 \ No newline at end of file +pg_probackup 2.2.5 \ No newline at end of file From 2d2da5010df2aa35d7095e45fcc8c414433b94a3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Nov 2019 15:23:31 +0300 Subject: [PATCH 1067/2107] tests: minor fixes --- tests/backup.py | 7 ++++--- tests/merge.py | 2 ++ tests/replica.py | 8 +++++++- tests/time_stamp.py | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index d5875dbb5..ad9e79482 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2019,6 +2019,7 @@ def test_backup_with_less_privileges_role(self): self.set_replica(node, replica) self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_auto_conf(replica, {'hot_standby': 'on'}) # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] @@ -2036,13 +2037,13 @@ def test_backup_with_less_privileges_role(self): # FULL backup from replica self.backup_node( backup_dir, 'replica', replica, - datname='backupdb', options=['--stream', '-U', 'backup']) + datname='backupdb', options=['--stream', '-U', 'backup', '--archive-timeout=30s']) - self.switch_wal_segment(node) +# self.switch_wal_segment(node) self.backup_node( backup_dir, 'replica', replica, datname='backupdb', - options=['-U', 'backup', '--log-level-file=verbose', '--archive-timeout=30s']) + options=['-U', 'backup', '--archive-timeout=300s']) # PAGE backup from replica self.backup_node( diff --git a/tests/merge.py b/tests/merge.py index 467104e32..000b483e0 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1051,6 +1051,8 @@ def test_continue_failed_merge(self): gdb.continue_execution_until_break(5) gdb._execute('signal SIGKILL') + gdb._execute('detach') + time.sleep(1) print(self.show_pb(backup_dir, as_text=True, as_json=False)) diff --git a/tests/replica.py b/tests/replica.py index a90386d56..0c47dba89 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -402,7 +402,13 @@ def test_take_backup_from_delayed_replica(self): self.wait_until_replica_catch_with_master(master, replica) - self.set_auto_conf(replica, {'recovery_min_apply_delay': '300s'}) + if self.get_version(master) >= self.version_to_num('12.0'): + self.set_auto_conf( + replica, {'recovery_min_apply_delay': '300s'}) + else: + replica.append_conf( + 'postgresql.auto.conf', + 'recovery_min_apply_delay = 300s') replica.stop() replica.slow_start(replica=True) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 72115e1d4..8abd55a2b 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -60,7 +60,7 @@ def test_server_date_style(self): base_dir="{0}/{1}/node".format(module_name, fname), set_replication=True, initdb_params=['--data-checksums'], - pg_options={"datestyle": "'GERMAN, DMY'"}) + pg_options={"datestyle": "GERMAN, DMY"}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 0d028b79eca5c90b46ccb310f425cdd67e00135b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 Nov 2019 18:42:20 +0300 Subject: [PATCH 1068/2107] wal_bytes for ARCHIVE backups was calculated incorrectly --- src/backup.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index b38719d3d..abd10a976 100644 --- a/src/backup.c +++ b/src/backup.c @@ -798,9 +798,19 @@ do_backup(time_t start_time, bool no_validate, /* compute size of wal files of this backup stored in the archive */ if (!current.stream) { - current.wal_bytes = instance_config.xlog_seg_size * - (current.stop_lsn / instance_config.xlog_seg_size - - current.start_lsn / instance_config.xlog_seg_size + 1); + XLogSegNo start_segno; + XLogSegNo stop_segno; + + GetXLogSegNo(current.start_lsn, start_segno, instance_config.xlog_seg_size); + GetXLogSegNo(current.stop_lsn, stop_segno, instance_config.xlog_seg_size); + current.wal_bytes = (stop_segno - start_segno) * instance_config.xlog_seg_size; + + /* + * If start_lsn and stop_lsn are located in the same segment, then + * set wal_bytes to the size of 1 segment. + */ + if (current.wal_bytes <= 0) + current.wal_bytes = instance_config.xlog_seg_size; } /* Backup is done. Update backup status */ From d4da7e88a784e7053a7008d5431ca061d73b2515 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Nov 2019 04:10:51 +0300 Subject: [PATCH 1069/2107] update current version to 2.2.5 in Readme and Documentation --- Documentation.md | 2 +- README.md | 46 +++++++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Documentation.md b/Documentation.md index f32d3e784..943ea0cfd 100644 --- a/Documentation.md +++ b/Documentation.md @@ -2,7 +2,7 @@ pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. -Current version - 2.2.4 +Current version - 2.2.5 1. [Synopsis](#synopsis) 2. [Versioning](#versioning) diff --git a/README.md b/README.md index 2803489d2..a1b8ea452 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.5, 9.6, 10, 11; +* PostgreSQL 9.5, 9.6, 10, 11, 12; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow @@ -37,8 +37,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* Postgres Pro Standard 9.5, 9.6, 10, 11 -* Postgres Pro Enterprise 9.5, 9.6, 10 +* Postgres Pro Standard 9.6, 10, 11 +* Postgres Pro Enterprise 9.6, 10, 11 ## Limitations @@ -48,7 +48,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.2.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.4) +[2.2.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.5) ## Documentation @@ -56,7 +56,7 @@ Documentation can be found at [github](https://github.com/postgrespro/pg_proback ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.4). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.5). ### Linux Installation #### pg_probackup for vanilla PostgreSQL @@ -64,31 +64,31 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list sudo wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{11,10,9.6,9.5} -sudo apt-get install pg-probackup-{11,10,9.6,9.5}-dbg +sudo apt-get install pg-probackup-{12,11,10,9.6,9.5} +sudo apt-get install pg-probackup-{12,11,10,9.6,9.5}-dbg #DEB-SRC Packages sudo echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list -sudo apt-get source pg-probackup-{11,10,9.6,9.5} +sudo apt-get source pg-probackup-{12,11,10,9.6,9.5} #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{11,10,9.6,9.5} -yum install pg_probackup-{11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{12,11,10,9.6,9.5} +yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{11,10,9.6,9.5} -yum install pg_probackup-{11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{12,11,10,9.6,9.5} +yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{11,10,9.6,9.5} -yum install pg_probackup-{11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{12,11,10,9.6,9.5} +yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #SRPM Packages -yumdownloader --source pg_probackup-{11,10,9.6,9.5} +yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} ``` #### pg_probackup for PostgresPro Standart and Enterprise @@ -96,23 +96,23 @@ yumdownloader --source pg_probackup-{11,10,9.6,9.5} #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list sudo wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). From 6ddfeaaf7eb271cb9448a72264dbdf75514d5e2e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 4 Nov 2019 18:24:52 +0300 Subject: [PATCH 1070/2107] Readme: update link to latest version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1b8ea452..a9f908074 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Documentation can be found at [github](https://github.com/postgrespro/pg_proback ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.5). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/latest). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 12cc64b567d56028fbe0c2ed543f331315b331ac Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Wed, 6 Nov 2019 15:38:41 +0300 Subject: [PATCH 1071/2107] DOC: fix grammar in new pg_probackup docs --- doc/pgprobackup.xml | 1093 +++++++++++++++++++++---------------------- 1 file changed, 531 insertions(+), 562 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index e2ebd739f..dde00b485 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -191,87 +191,87 @@ doc/src/sgml/pgprobackup.sgml Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three - different incremental modes you can plan the backup strategy - in accordance with your data flow + different incremental modes, you can plan the backup strategy + in accordance with your data flow. Validation: automatic data consistency checks and on-demand - backup validation without actual data recovery + backup validation without actual data recovery. Verification: on-demand verification of PostgreSQL instance - via dedicated command checkdb + with the checkdb command. Retention: managing WAL archive and backups in accordance with - retention policies — Time and/or Redundancy based, with two - retention methods: delete expired and - merge expired. Additionally you can design - you own retention policy by setting 'time to live' for backups + retention policy. You can configure retention policy based on recovery time + or the number of backups to keep, as well as specify time to live (TTL) + for a particular backup. Expired backups can be merged or deleted. - Parallelization: running backup, restore, merge, delete, - verificaton and validation processes on multiple parallel - threads + Parallelization: running backup, + restore, merge, + delete, validate, + and checkdb processes on multiple parallel threads. Compression: storing backup data in a compressed state to save - disk space + disk space. - Deduplication: saving disk space by not copying the not - changed non-data files (_vm, _fsm, etc) + Deduplication: saving disk space by not copying unchanged non-data + files, such as _vm or _fsm. - Remote operations: backup PostgreSQL instance located on - remote machine or restore backup on it + Remote operations: backing up PostgreSQL + instance located on a remote system or restoring a backup remotely. - Backup from replica: avoid extra load on the master server by - taking backups from a standby + Backup from standby: avoiding extra load on master by + taking backups from a standby server. - External directories: add to backup content of directories - located outside of the PostgreSQL data directory (PGDATA), - such as scripts, configs, logs and pg_dump files + External directories: backing up files and directories + located outside of the PostgreSQL data + directory (PGDATA), such as scripts, configuration + files, logs, or SQL dump files. - Backup Catalog: get list of backups and corresponding meta - information in plain or - json formats + Backup catalog: getting the list of backups and the corresponding meta + information in plain text or + JSON formats. - - Archive Catalog: get list of all WAL timelines and - corresponding meta information in plain or - json formats + + Archive catalog: getting the list of all WAL timelines and + the corresponding meta information in plain text or + JSON formats. - Partial Restore: restore only the specified databases or skip - the specified databases. + Partial restore: restoring only the specified databases. @@ -306,7 +306,7 @@ doc/src/sgml/pgprobackup.sgml DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages - that has changed since the previous backup. Note that this + that have changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. @@ -322,8 +322,7 @@ doc/src/sgml/pgprobackup.sgml files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as - explained in the section - Setting + explained in Setting up continuous WAL archiving to make PAGE backups. @@ -366,12 +365,11 @@ doc/src/sgml/pgprobackup.sgml consistent state at the time the backup was taken. Regardless of continuous - archiving been set up or not, the WAL segments required - for consistent recovery are streamed (hence STREAM) via + archiving having been set up or not, the WAL segments required + for consistent recovery are streamed via replication protocol during backup and included into the - backup files. Because of that backups of this WAL mode are - called autonomous or - standalone. + backup files. That's why such backups are called + autonomous, or standalone. @@ -382,39 +380,34 @@ doc/src/sgml/pgprobackup.sgml - Only PostgreSQL of versions 9.5 and newer are supported. + pg_probackup only supports PostgreSQL 9.5 and higher. - Currently remode mode of operations is not supported on - Windows systems. + The remote mode is not supported on Windows systems. - On Unix systems backup of PostgreSQL verions =< 10 is - possible only by the same OS user PostgreSQL server is - running by. For example, if PostgreSQL server is running by - user postgres, then backup must be run - by user postgres. If backup is running - in - remote - mode using ssh, then this - limitation apply differently: value for - option should be - postgres. + On Unix systems, for PostgreSQL 10 or higher, + a backup can be made only by the same OS user that has started the PostgreSQL + server. For example, if PostgreSQL server is started by + user postgres, the backup command must also be run + by user postgres. To satisfy this requirement when taking backups in the + remote mode using SSH, you must set + option to postgres. - During backup of PostgreSQL 9.5 functions + For PostgreSQL 9.5, functions pg_create_restore_point(text) and - pg_switch_xlog() will be executed only if - backup role is superuser. Because of that backup of a - cluster with low amount of WAL traffic with non-superuser - role may take more time than backup of the same cluster with - superuser role. + pg_switch_xlog() can be executed only if + the backup role is a superuser, so backup of a + cluster with low amount of WAL traffic by a non-superuser + role can take longer than the backup of the same cluster by + a superuser role. @@ -424,17 +417,17 @@ doc/src/sgml/pgprobackup.sgml block_size and wal_block_size - parameters and have the same major release number. Also - depending on cluster configuration PostgreSQL itself may - apply additional restrictions such as CPU architecture - platform and libc/libicu versions. + parameters and have the same major release number. + Depending on cluster configuration, PostgreSQL itself may + apply additional restrictions, such as CPU architecture + or libc/libicu versions. - Incremental chain can span only within one timeline. So if - you have backup incremental chain taken from replica and it - gets promoted, you would be forced to take another FULL + All backups in the incremental chain must belong to the same + timeline. For example, if you have taken incremental backups on a + standby server that gets promoted, you have to take another FULL backup. @@ -468,7 +461,7 @@ doc/src/sgml/pgprobackup.sgml Optionally, configure SSH for running pg_probackup operations - in remote mode. + in the remote mode. @@ -485,13 +478,13 @@ doc/src/sgml/pgprobackup.sgml pg_probackup init -B backup_dir - Where backup_dir is the path to backup + where backup_dir is the path to the backup catalog. If the backup_dir already exists, it must be empty. Otherwise, pg_probackup returns an error. The user launching pg_probackup must have full access to - backup_dir directory. + the backup_dir directory. pg_probackup creates the backup_dir backup @@ -529,7 +522,7 @@ pg_probackup init -B backup_dir pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] - Where: + where: @@ -548,10 +541,10 @@ pg_probackup add-instance -B backup_dir -D - The optional parameters remote_options - should be used if data_dir is located - on remote machine. + are optional parameters that need to be specified only if + data_dir is located + on a remote system. @@ -563,7 +556,7 @@ pg_probackup add-instance -B backup_dir -D pg_probackup.conf configuration file that controls pg_probackup settings for this backup instance. If you run this command with the - remote_options, used + remote_options, the specified parameters parameters will be added to pg_probackup.conf. @@ -582,11 +575,11 @@ pg_probackup add-instance -B backup_dir -D - For PostgreSQL >= 11 it is recommended to use + For PostgreSQL 11 or higher, it is recommended to use the allow-group-access - feature, so backup can be done by any OS user in the same - group as the cluster owner. In this case the user should have - read permissions on the cluster directory. + feature, so that backup can be done by any OS user in the same + group as the cluster owner. In this case, the user should have + read permissions for the cluster directory. @@ -600,13 +593,13 @@ pg_probackup add-instance -B backup_dir -D - To perform backup, the following + To perform a backup, the following permissions for role backup are required - only in database used for - connection to PostgreSQL server: + only in the database used for + connection to the PostgreSQL server: - For PostgreSQL 9.5: + For PostgreSQL 9.5: BEGIN; @@ -644,7 +637,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; - For PostgreSQL >= 10: + For PostgreSQL 10 or higher: BEGIN; @@ -666,23 +659,23 @@ COMMIT; In the pg_hba.conf - file, allow connection to database cluster on behalf of the + file, allow connection to the database cluster on behalf of the backup role. Since pg_probackup needs to read cluster files directly, - pg_probackup must be started by (in case of remote backup - - connected to) OS user that has read access to all files and + pg_probackup must be started by (or connected to, + if used in the remote mode) the OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. - Depending on whether you are plan to take - autonomous and/or + Depending on whether you plan to take + standalone or archive backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server, - run pg_probackup in remote mode or create PTRACK backups, + run pg_probackup in the remote mode, or create PTRACK backups, additional setup is required. @@ -742,20 +735,20 @@ ALTER ROLE backup WITH REPLICATION; - If you are planning to take PAGE backups in STREAM mode or + If you are planning to take PAGE backups in the STREAM mode or perform PITR with STREAM backups, you still have to configure - WAL archiving as explained in the section + WAL archiving, as explained in the section Setting up continuous WAL archiving. Once these steps are complete, you can start taking FULL, PAGE, - DELTA and PTRACK backups with + DELTA, and PTRACK backups in the STREAM WAL mode. - Setting up continuous WAL archiving + Setting up Continuous WAL Archiving Making backups in PAGE backup mode, performing PITR @@ -795,48 +788,45 @@ archive_command = 'pg_probackup archive-push -B backup_dir - Where backup_dir and + where backup_dir and instance_name refer to the already - initialized backup catalog instance for this database cluster - and optional parameters - remote_options should - be used to archive WAL to the remote host. For details about all + initialized backup catalog instance for this database cluster, + and remote_options + only need to be specified to archive WAL on a remote host. For details about all possible archive-push parameters, see the section archive-push. - Once these steps are complete, you can start making backups with - ARCHIVE WAL-mode, backups in - PAGE backup mode and perform + Once these steps are complete, you can start making backups in the + ARCHIVE WAL mode, backups in + the PAGE backup mode, as well as perform PITR. - Current state of WAL Archive can be obtained via + You can view the current state of the WAL archive using the show command. For details, see the - sections Viewing - WAL Archive information. + section Viewing + WAL Archive Information. If you are planning to make PAGE backups and/or backups with ARCHIVE WAL mode from a - standby of a server, that generates small amount of WAL traffic, + standby server that generates a small amount of WAL traffic, without long waiting for WAL segment to fill up, consider - setting + setting the archive_timeout PostgreSQL parameter on - master. It is advisable to set the value of this - setting slightly lower than pg_probackup parameter - (default 5 min), so there - should be enough time for rotated segment to be streamed to - replica and send to archive before backup is aborted because of - . + master. The value of this parameter should be slightly + lower than the setting (5 minutes by default), + so that there is enough time for the rotated + segment to be streamed to standby and sent to WAL archive before the + backup is aborted because of . - Using pg_probackup command - archive-push for - continuous archiving is optional. You can use any other tool - you like as long as it delivers WAL segments into + Instead of using the archive-push + command provided by pg_probackup, you can use + any other tool to set up continuous archiving as long as it delivers WAL segments into backup_dir/wal/instance_name directory. If compression is used, it should be gzip, and .gz suffix in filename is @@ -845,15 +835,15 @@ archive_command = 'pg_probackup archive-push -B backup_dir - Instead of - archive_mode+archive_command - method you may opt to use the utility - pg_receivewal. - In this case pg_receivewal -D directory + Instead of configuring continuous archiving by setting the + archive_mode and archive_command + parameters, you can opt for using the + pg_receivewal + utility. In this case, pg_receivewal -D directory option should point to backup_dir/wal/instance_name - directory. WAL compression that could be done by pg_receivewal - is supported by pg_probackup. + directory. pg_probackup supports WAL compression + that can be done by pg_receivewal. Zero Data Loss archive strategy can be achieved only by using pg_receivewal. @@ -868,21 +858,21 @@ archive_command = 'pg_probackup archive-push -B backup_dir - On the standby server, set the parameter + On the standby server, set the hot_standby - to on. + parameter to on. - On the master server, set the parameter + On the master server, set the full_page_writes - to on. + parameter to on. - To perform autonomous backups on standby, complete all steps + To perform standalone backups on standby, complete all steps in section Setting up STREAM Backups @@ -898,7 +888,7 @@ archive_command = 'pg_probackup archive-push -B backup_dir Once these steps are complete, you can start taking FULL, PAGE, - DELTA or PTRACK backups with appropriate WAL delivery mode: + DELTA, or PTRACK backups with appropriate WAL delivery mode: ARCHIVE or STREAM, from the standby server. @@ -916,7 +906,7 @@ archive_command = 'pg_probackup archive-push -B backup_dirfull_page_writes on the master, and not - to use a tools like pg_compresslog as + to use tools like pg_compresslog as archive_command to remove full-page writes from WAL files. @@ -926,17 +916,17 @@ archive_command = 'pg_probackup archive-push -B backup_dir Setting up Cluster Verification - Logical verification of database cluster requires the following + Logical verification of a database cluster requires the following additional setup. Role backup is used as an example: - Install extension + Install the amcheck or - amcheck_next + amcheck_next extension in every database of the cluster: @@ -946,9 +936,8 @@ CREATE EXTENSION amcheck; - To perform logical verification the following permissions - are required in every - database of the cluster: + Grant the following permissions to the backup + role in every database of the cluster: @@ -972,7 +961,7 @@ GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; Grant the read-only access to pg_catalog.pg_database to the - backup role only in database + backup role only in the database used for connection to PostgreSQL server: @@ -993,9 +982,9 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; SSH. - Setup SSH + Set up SSH - If you are going to use pg_probackup in remote mode via ssh, + If you are going to use pg_probackup in remote mode via SSH, complete the following steps: @@ -1008,7 +997,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - For communication between the hosts setup the passwordless + For communication between the hosts set up the passwordless SSH connection between backup user on backup_host and postgres user on @@ -1020,9 +1009,9 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - If you are planning to rely on + If you are going to rely on continuous - WAL archiving, then setup passwordless SSH + WAL archiving, set up passwordless SSH connection between postgres user on db_host and backup user on backup_host: @@ -1033,7 +1022,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - Where: + where: @@ -1057,8 +1046,8 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; postgres is the OS user on - db_host used to run PostgreSQL - cluster. Note, that for PostgreSQL versions >= 11, a + db_host used to start the PostgreSQL + cluster. Note that for PostgreSQL 11 or higher a more secure approach can be used thanks to allow-group-access feature. @@ -1066,13 +1055,13 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - pg_probackup in remote mode via ssh works + pg_probackup in the remote mode via SSH works as follows: - only the following commands can be launched in remote + Only the following commands can be launched in the remote mode: add-instance, backup, restore, @@ -1082,29 +1071,29 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - when started in remote mode the main pg_probackup process - on local system connects via ssh to remote system and - launches there number of agent proccesses equal to - specified value of option /. + When started in the remote mode, the main pg_probackup process + on the local system connects to the remote system via SSH and + launches one or more agent processes on the remote system, which are called + remote agents. The number of remote agents + is equal to the / setting. - the main pg_probackup process uses remote agents to access + The main pg_probackup process uses remote agents to access remote files and transfer data between local and remote systems. - remote agents are smart and capable of handling some logic - on their own to minimize the network traffic and number of + Remote agents try to minimize the network traffic and the number of round-trips between hosts. - usually the main proccess is started on + The main process is usually started on backup_host and connects to db_host, but in case of archive-push and @@ -1115,36 +1104,31 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - after completition of data transfer the remote agents are - terminated and ssh connections are closed. + Once data transfer is complete, remote agents are + terminated and SSH connections are closed. - if an error condition is encountered by a remote agent, + If an error condition is encountered by a remote agent, then all agents are terminated and error details are reported by the main pg_probackup process, which exits - with error. + with an error. - compression is always done on - db_host. - - - - - decompression is always done on + Compression is always done on + db_host, while decompression is always done on backup_host. - You can improse + You can impose additional - restrictions on ssh settings to protect the system + restrictions on SSH settings to protect the system in the event of account compromise. @@ -1153,9 +1137,9 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Setting up PTRACK Backups - Backup mode PTACK can be used only on Postgres Pro Standard and - Postgres Pro Enterprise installations or patched vanilla - PostgreSQL. Links to ptrack patches can be found + The PTACK backup mode can be used only for Postgres Pro Standard and + Postgres Pro Enterprise installations, or patched vanilla + PostgreSQL. Links to PTRACK patches can be found here. @@ -1165,13 +1149,13 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - Set the parameter ptrack_enable to + Set the ptrack_enable parameter to on. - Grant the rights to execute ptrack + Grant the rights to execute PTRACK functions to the backup role in every database of the cluster: @@ -1221,15 +1205,15 @@ pg_probackup backup -B backup_dir --instance - PAGE — creates an incremental PAGE backup based on the WAL - files that have generated since the previous full or - incremental backup was taken. Only changed blocks are readed + PAGE — creates an incremental backup based on the WAL + files that have been generated since the previous full or + incremental backup was taken. Only changed blocks are read from data files. - PTRACK — creates an incremental PTRACK backup tracking page + PTRACK — creates an incremental backup tracking page changes on the fly. @@ -1242,7 +1226,7 @@ pg_probackup backup -B backup_dir --instance - ARCHIVE mode + ARCHIVE Mode ARCHIVE is the default WAL delivery mode. @@ -1253,29 +1237,29 @@ pg_probackup backup -B backup_dir --instance backup_dir --instance instance_name -b FULL - Unlike backup in STREAM mode, ARCHIVE backup rely on + ARCHIVE backups rely on continuous - archiving to provide WAL segments required to restore + archiving to get WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - During backup pg_probackup + When a backup is taken, pg_probackup ensures that WAL files containing WAL records between Start - LSN and Stop LSN are actually exists in + LSN and Stop LSN actually exist in backup_dir/wal/instance_name - directory. Also pg_probackup ensures that WAL records between - Start LSN and Stop LSN can be parsed. This precations + directory. pg_probackup also ensures that WAL records between + Start LSN and Stop LSN can be parsed. This precaution eliminates the risk of silent WAL corruption. - STREAM mode + STREAM Mode STREAM is the optional WAL delivery mode. - For example, to make a FULL backup in STREAM mode, add the + For example, to make a FULL backup in the STREAM mode, add the flag to the command from the previous example: @@ -1288,7 +1272,7 @@ pg_probackup backup -B backup_dir --instance - Unlike backup in ARCHIVE mode, STREAM backup include all the + Unlike backups in ARCHIVE mode, STREAM backups include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. @@ -1296,10 +1280,10 @@ pg_probackup backup -B backup_dir --instance backup pg_probackup streams WAL files containing WAL records between Start LSN and Stop LSN to - backup_dir/backups/instance_name/backup_id/database/pg_wal directory. Also pg_probackup - ensures that WAL records between Start LSN and Stop LSN can be - parsed. This precations eliminates the risk of silent WAL - corruption. + backup_dir/backups/instance_name/backup_id/database/pg_wal directory. To eliminate the risk + of silent WAL corruption, pg_probackup also + checks that WAL records between Start LSN and + Stop LSN can be parsed. Even if you are using @@ -1331,7 +1315,7 @@ pg_probackup backup -B backup_dir --instance - Page validation + Page Validation If data @@ -1340,38 +1324,37 @@ pg_probackup backup -B backup_dir --instance pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees - that the PostgreSQL instance and backup itself are free of - corrupted pages. Note that pg_probackup reads database files - directly from filesystem, so under heavy write load during - backup it can show false positive checksum failures because of - partial writes. In case of page checksumm mismatch, page is - readed again and checksumm comparison repeated. + that the PostgreSQL instance and the backup itself have no + corrupt pages. Note that pg_probackup reads database files + directly from the filesystem, so under heavy write load during + backup it can show false-positive checksum mismatches because of + partial writes. If a page checksum mismatch occurs, the page is + re-read and checksum comparison is repeated. - Page is considered corrupted if checksumm comparison failed - more than 100 times, in this case backup is aborted. + A page is considered corrupt if checksum comparison has failed + more than 100 times. In this case, the backup is aborted. - Redardless of data checksums been enabled or not, pg_probackup - always check page header "sanity". + Even if data checksums are not enabled, pg_probackup + always performs sanity checks for page headers. - External directories + External Directories To back up a directory located outside of the data directory, use the optional parameter that specifies the path to this directory. If you would like - to add more than one external directory, provide several paths - separated by colons, on Windows system paths must be separated - by semicolon instead. + to add more than one external directory, you can provide several paths + separated by colons on Linux systems or semicolons on Windows systems. For example, to include /etc/dir1/ and /etc/dir2/ directories into the full backup of your instance_name instance that will be stored under the backup_dir - directory, run: + directory on Linux, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 @@ -1381,7 +1364,7 @@ pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-di C:\dir2\ directories into the full backup of your instance_name instance that will be stored under the backup_dir directory - on Windows system, run: + on Windows, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 @@ -1405,7 +1388,7 @@ pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-di - Verifying a Cluster + Performing Cluster Verification To verify that PostgreSQL database cluster is free of corruption, run the following command: @@ -1414,26 +1397,26 @@ pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-di pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] - This physical verification works similar to + This command performs physical verification works similar to page validation that is done during backup with several differences: - checkdb is read-only + checkdb is read-only. - if corrupted page is detected, checkdb is - not aborted, but carry on, until all pages in the cluster - are validated + Even if a corrupt page is detected, checkdb + continues cluster verification until all pages in the cluster + are validated. - checkdb do not strictly require + checkdb does not strictly require the backup catalog, so it can be used to verify database clusters that are not @@ -1444,7 +1427,7 @@ pg_probackup checkdb [-B backup_dir [--instance If backup_dir and - instance_name are omitted, then + instance_name are omitted, connection options and data_dir must be provided via environment variables or command-line options. @@ -1461,17 +1444,17 @@ pg_probackup checkdb [-B backup_dir [--instance If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you - can specify flag when running - checkdb command: + can specify the flag when running + the checkdb command: pg_probackup checkdb -D data_dir --amcheck - Physical verification can be skipped if - flag is used. For - logical only verification backup_dir and - data_dir are optional, only + You can skip physical verification by specifying the + flag. In this case, + you can omit backup_dir and + data_dir options, only connection options are mandatory: @@ -1479,20 +1462,20 @@ pg_probackup checkdb -D data_dir --amcheck pg_probackup checkdb --amcheck --skip-block-validation [connection_options] - Logical verification can be done more thoroughly with flag - by checking that all heap + Logical verification can be done more thoroughly with the + flag by checking that all heap tuples that should be indexed are actually indexed, but at the - higher cost of CPU, memory and I/O comsumption. + higher cost of CPU, memory, and I/O consumption. Validating a Backup pg_probackup calculates checksums for each file in a backup - during backup process. The process of checking checksumms of + during the backup process. The process of checking checksums of backup data files is called - the backup validation. By default validation - is run immediately after backup is taken and right before + the backup validation. By default, validation + is run immediately after the backup is taken and right before the restore, to detect possible backup corruption. @@ -1510,7 +1493,7 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ For example, to check that you can restore the database cluster - from a backup copy up to the specified xid transaction ID, run + from a backup copy up to transaction ID 4242, run this command: @@ -1519,30 +1502,30 @@ pg_probackup validate -B backup_dir --instance If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an - error message with the exact time, transaction ID and LSN up to + error message with the exact time, transaction ID, and LSN up to which the recovery is possible. If you specify backup_id via - -i/--backup-id option, then only backup copy + -i/--backup-id option, then only the backup copy with specified backup ID will be validated. If backup_id is specified with recovery target - options then validate will check whether it is possible + options, the validate command will check whether it is possible to restore the specified backup to the specified recovery target. For example, to check that you can restore the database cluster - from a backup copy with backup_id up to the + from a backup copy with the PT8XFX backup ID up to the specified timestamp, run this command: pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' - If backup_id belong to incremental backup, - then all its parents starting from FULL backup will be + If you specify the backup_id of an incremental backup, + all its parents starting from FULL backup will be validated. @@ -1559,7 +1542,7 @@ pg_probackup validate -B backup_dir --instance backup_dir --instance instance_name -i backup_id - Where: + where: @@ -1610,7 +1593,7 @@ pg_probackup restore -B backup_dir --instance - If you are restoring an STREAM backup, the restore is complete + If you are restoring a STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the @@ -1624,7 +1607,7 @@ pg_probackup restore -B backup_dir --instance - To restore cluster on remote host see the section + To restore the cluster on a remote host, see the section Using pg_probackup in the Remote Mode. @@ -1643,22 +1626,21 @@ pg_probackup restore -B backup_dir --instance If you have enabled partial - restore before taking backups, you can restore or - exclude from restore the arbitrary number of specific - databases using + restore before taking backups, you can restore + only some of the databases using partial restore options with the restore commands. - To restore only one or more databases, run the restore command + To restore the specified databases only, run the restore command with the following options: pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name - The option can be specified + The option can be specified multiple times. For example, to restore only databases db1 and db2, run the following command: @@ -1667,14 +1649,14 @@ pg_probackup restore -B backup_dir --instance backup_dir --instance instance_name --db-include=db1 --db-include=db2 - To exclude one or more specific databases from restore, run - the following options: + To exclude one or more databases from restore, use + the option: pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name - The option can be specified + The option can be specified multiple times. For example, to exclude the databases db1 and db2 from restore, run the following command: @@ -1683,17 +1665,16 @@ pg_probackup restore -B backup_dir --instance backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 - Partial restore rely on lax behaviour of PostgreSQL recovery - process toward truncated files. Files of excluded databases - restored as null sized files, allowing recovery to work - properly. After successfull starting of PostgreSQL cluster, - you must drop excluded databases using + Partial restore relies on lax behavior of PostgreSQL recovery + process toward truncated files. For recovery to work properly, files of excluded databases + are restored as files of zero size. After the PostgreSQL cluster is successfully started, + you must drop the excluded databases using DROP DATABASE command. - The databases template0 and - template1 are always restored. + The template0 and + template1 databases are always restored. @@ -1739,7 +1720,7 @@ pg_probackup restore -B backup_dir --instance - To restore the cluster state up to a specific LSN, use + To restore the cluster state up to the specific LSN, use option: @@ -1748,7 +1729,7 @@ pg_probackup restore -B backup_dir --instance - To restore the cluster state up to a specific named restore + To restore the cluster state up to the specific named restore point, use option: @@ -1758,7 +1739,7 @@ pg_probackup restore -B backup_dir --instance To restore the backup to the latest state available in - archive, use option + the WAL archive, use option with latest value: @@ -1768,7 +1749,7 @@ pg_probackup restore -B backup_dir --instance To restore the cluster to the earliest point of consistency, - use option with + use option with the immediate value: @@ -1787,10 +1768,12 @@ pg_probackup restore -B backup_dir --instance pg_probackup installed on both systems. - - Do note that pg_probackup rely on passwordless SSH connection + + + pg_probackup relies on passwordless SSH connection for communication between the hosts. - + + The typical workflow is as follows: @@ -1810,19 +1793,19 @@ pg_probackup restore -B backup_dir --instance - If you would like to take remote backup in + If you would like to take remote backups in PAGE mode, or rely on ARCHIVE WAL delivery mode, or use PITR, - then configure continuous WAL archiving from database host + configure continuous WAL archiving from the database host to the backup host as explained in the section Setting up continuous WAL archiving. For the archive-push and archive-get commands, you must specify the remote - options that point to backup host with backup + options that point to the backup host with the backup catalog. @@ -1831,7 +1814,7 @@ pg_probackup restore -B backup_dir --instance backup or restore commands with remote options - on backup host. + on the backup host. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. @@ -1839,36 +1822,35 @@ pg_probackup restore -B backup_dir --instance - For example, to create archive full backup using remote mode - through SSH connection to user postgres on - host with address 192.168.0.2 via port - 2302, run: + For example, to create an archive full backup of a + PostgreSQL cluster located on + a remote system with host address 192.168.0.2 + on behalf of the postgres user via SSH connection + through port 2302, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - For example, to restore latest backup on remote system using - remote mode through SSH connection to user - postgres on host with address - 192.168.0.2 via port 2302, - run: + To restore the latest available backup on a remote system with host address + 192.168.0.2 on behalf of the postgres + user via SSH connection through port 2302, run: pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - Restoring of ARCHIVE backup or performing PITR in remote mode + Restoring an ARCHIVE backup or performing PITR in the remote mode require additional information: destination address, port and - username for establishing ssh connection - from a host with database - to a host with backup - catalog. This information will be used by - restore_command to copy via ssh WAL segments - from archive to PostgreSQL pg_wal directory. + username for establishing an SSH connection + from the host with database + to the host with the backup + catalog. This information will be used by the + restore_command to copy WAL segments + from the archive to the PostgreSQL pg_wal directory. - To solve this problem you can use + To solve this problem, you can use Remote Wal Archive Options. @@ -1885,7 +1867,7 @@ pg_probackup restore -B backup_dir --instance backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup - Provided arguments will be used to construct restore_command + Provided arguments will be used to construct the restore_command in recovery.conf: @@ -1893,15 +1875,15 @@ pg_probackup restore -B backup_dir --instance - Alternatively you can use - option to provide an entire restore_command: + Alternatively, you can use the + option to provide the entire restore_command: pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' - The remote backup mode is currently unavailable for + The remote mode is currently unavailable for Windows systems. @@ -1917,11 +1899,11 @@ pg_probackup restore -B backup_dir --instance validate processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU - cores, disk and network bandwidth). + cores, disk, and network bandwidth). Parallel execution is controlled by the - -j/--threads command line option. For + -j/--threads command-line option. For example, to create a backup using four parallel threads, run: @@ -1947,7 +1929,7 @@ pg_probackup backup -B backup_dir --instance For example, backup and - checkdb commands uses a regular + checkdb commands use a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the @@ -1980,7 +1962,7 @@ pg_probackup backup -B backup_dir --instance remote, retention, - logging and + logging, and compression settings using the set-config command: @@ -1997,7 +1979,7 @@ pg_probackup show-config -B backup_dir --instance You can override the settings defined in pg_probackup.conf when running pg_probackup commands - via corresponding environment variables and/or command line + via the corresponding environment variables and/or command line options. @@ -2110,13 +2092,13 @@ BACKUP INSTANCE 'node' - WAL Mode — the WAL delivery mode. Possible values: STREAM + WAL Mode — WAL delivery mode. Possible values: STREAM and ARCHIVE. - TLI — timeline identifiers of current backup and its + TLI — timeline identifiers of the current backup and its parent. @@ -2128,15 +2110,15 @@ BACKUP INSTANCE 'node' Data — the size of the data files in this backup. This - value does not include the size of WAL files. In case of - STREAM backup the total size of backup can be calculated + value does not include the size of WAL files. For + STREAM backups, the total size of the backup can be calculated as Data + WAL. - WAL — the uncompressed size of WAL files required to apply - by PostgreSQL recovery process to reach consistency. + WAL — the uncompressed size of WAL files + that need to be applied during recovery for the backup to reach a consistent state. @@ -2191,7 +2173,7 @@ BACKUP INSTANCE 'node' - CORRUPT — some of the backup files are corrupted. + CORRUPT — some of the backup files are corrupt. @@ -2215,7 +2197,7 @@ BACKUP INSTANCE 'node' To get more detailed information about the backup, run the - show with the backup ID: + show command with the backup ID: pg_probackup show -B backup_dir --instance instance_name -i backup_id @@ -2271,49 +2253,47 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod - from-replica — the fact that backup was taken from standby server. Possible values: + from-replica — was this backup taken on standby? Possible values: 1, 0. - block-size — block_size - setting of PostgreSQL cluster at the moment of backup start. + block-size — the block_size + setting of PostgreSQL cluster at the backup start. - checksum-version — the fact that PostgreSQL cluster, from - which backup is taken, has enabled + checksum-version — are data - block checksumms. Possible values: 1, 0. + block checksums enabled in the backed up PostgreSQL cluster? Possible values: 1, 0. - program-version — full version of pg_probackup binary used to create backup. + program-version — full version of pg_probackup binary used to create the backup. - start-time — the backup starting time. + start-time — the backup start time. - end-time — the backup ending time. + end-time — the backup end time. - expire-time — if the - backup was pinned, then until this point in time the backup - cannot be removed by retention purge. + expire-time — the point in time + when a pinned backup can be removed by retention purge. - uncompressed-bytes — size of the data files before adding page headers and applying + uncompressed-bytes — the size of data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing uncompressed-bytes to data-bytes if compression if used. @@ -2321,36 +2301,35 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod - pgdata-bytes — size of the PostgreSQL + pgdata-bytes — the size of PostgreSQL cluster data files at the time of backup. You can evaluate the - effectiveness of incremental backup by comparing + effectiveness of an incremental backup by comparing pgdata-bytes to uncompressed-bytes. - recovery-xid — - current transaction id at the moment of backup ending. + recovery-xid — transaction ID at the backup end time. - parent-backup-id — backup ID of parent backup. Available only + parent-backup-id — backup ID of the parent backup. Available only for incremental backups. - primary_conninfo — libpq conninfo - used for connection to PostgreSQL cluster during backup. The + primary_conninfo — libpq connection parameters + used to connect to the PostgreSQL cluster to take this backup. The password is not included. - To get more detailed information about the backup in json - format, run the show with the backup ID: + You can also get the detailed information about the backup + in the JSON format: pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id @@ -2419,7 +2398,7 @@ ARCHIVE INSTANCE 'node' 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK - For each backup, the following information is provided: + For each timeline, the following information is provided: @@ -2429,24 +2408,24 @@ ARCHIVE INSTANCE 'node' - Parent TLI — identifier of timeline TLI branched off. + Parent TLI — identifier of the timeline from which this timeline branched off. Switchpoint — LSN of the moment when the timeline branched - off from "Parent TLI". + off from its parent timeline. - Min Segno — number of the first existing WAL segment + Min Segno — the first WAL segment belonging to the timeline. - Max Segno — number of the last existing WAL segment + Max Segno — the last WAL segment belonging to the timeline. @@ -2458,7 +2437,7 @@ ARCHIVE INSTANCE 'node' - Size — the size files take on disk. + Size — the size that files take on disk. @@ -2475,20 +2454,21 @@ ARCHIVE INSTANCE 'node' - Status — archive status for this exact timeline. Possible + Status — status of the WAL archive for this timeline. Possible values: - OK — all WAL segments between Min and Max are present. + OK — all WAL segments between Min Segno + and Max Segno are present. - DEGRADED — some WAL segments between Min and Max are - lost. To get details about lost files, use the JSON - format. + DEGRADED — some WAL segments between Min Segno + and Max Segno are missing. To find out which files are lost, + view this report in the JSON format. @@ -2721,31 +2701,29 @@ pg_probackup show -B backup_dir [--instance - size is in bytes. + The size is in bytes. - closest-backup-id attribute contain ID of valid backup - closest to the timeline, located on some of the previous - timelines. This backup is the closest starting point to - reach the timeline from other timelines by PITR. Closest - backup always has a valid status, either OK or DONE. If - such backup do not exists, then string is empty. + The closest-backup-id attribute + contains the ID of the most recent valid backup that belongs to + one of the previous timelines. You can use this backup to perform + point-in-time recovery to this timeline. If + such a backup does not exist, this string is empty. - DEGRADED timelines contain lost-segments array with - information about intervals of missing segments. In OK - timelines lost-segments array is empty. + The lost-segments array provides with + information about intervals of missing segments in DEGRADED timelines. In OK + timelines, the lost-segments array is empty. - N backups attribute is replaced with backups array - containing backups belonging to the timeline. If timeline - has no backups, then backups array is empty. + The backups array lists all backups + belonging to the timeline. If the timeline has no backups, this array is empty. @@ -2860,7 +2838,7 @@ pg_probackup delete -B backup_dir --instance Suppose you have backed up the node instance in the backup_dir directory, - with the option is set + with the option set to 7, and you have the following backups available on April 10, 2019: @@ -2915,31 +2893,29 @@ BACKUP INSTANCE 'node' Backup Pinning - If you have the necessity to exclude certain backups from - established retention policy then it is possible to pin a - backup for an arbitrary amount of time. Example: + If you need to keep certain backups longer than the + established retention policy allows, you can pin them + for arbitrary time. For example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d - This command will set expire-time of the - specified backup to 30 days starting from backup - recovery-time attribute. Basically - expire-time = - recovery-time + ttl. + This command sets the expiration time of the + specified backup to 30 days starting from the time + indicated in its recovery-time attribute. - Also you can set expire-time explicitly - using option. Example: + You can also explicitly set the expiration time for a backup + using the option. For example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' - Alternatively you can use the and + Alternatively, you can use the and options with the - backup command to pin newly + backup command to pin the newly created backup: @@ -2947,17 +2923,15 @@ pg_probackup backup -B backup_dir --instance backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' - You can determine the fact that backup is pinned and check due - expire time by looking up expire-time - attribute in backup metadata via - show command: - + To check if the backup is pinned, + run the show command: pg_probackup show -B backup_dir --instance instance_name -i backup_id - - Pinned backup has expire-time attribute: + + If the backup is pinned, the expire-time + attribute displays its expiration time: ... recovery-time = '2017-05-16 12:57:31' @@ -2965,40 +2939,41 @@ expire-time = '2020-01-01 00:00:00+03' data-bytes = 22288792 ... - - You can unpin the backup by setting the - option to zero using - set-backup command. Example: - -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 - Only pinned backups have the expire-time attribute in the backup metadata. - Pinned incremental backup will also implicitly pin all + A pinned incremental backup implicitly pins all its parent backups. + + You can unpin the backup by setting the + option to zero using the + set-backup command. For example: + + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 + + WAL Archive Retention Policy - By default, pg_probackup treatment of WAL Archive is very - conservative and only "redundant" WAL segments can - be purged, i.e. segments that cannot be applied to any of the - existing backups in the backup catalog. To save disk space, - you can configure WAL Archive retention policy, that allows to + By default, pg_probackup purges + only redundant WAL segments that cannot be applied to any of the + backups in the backup catalog. To save disk space, + you can configure WAL archive retention policy, which allows to keep WAL of limited depth measured in backups per timeline. Suppose you have backed up the node - instance in the backup_dir directory with + instance in the backup_dir directory and configured - WAL + continuous WAL archiving: @@ -3017,8 +2992,8 @@ BACKUP INSTANCE 'node' node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK - The state of WAL archive can be determined by using - show command with + You can check the state of the WAL archive by running the + show command with the flag: @@ -3032,8 +3007,8 @@ ARCHIVE INSTANCE 'node' 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK - General WAL purge without cannot - achieve much, only one segment can be removed: + WAL purge without cannot + achieve much, only one segment is removed: pg_probackup delete -B backup_dir --instance node --delete-wal @@ -3061,7 +3036,7 @@ ARCHIVE INSTANCE 'node' 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK - Alternatively you can use the + Alternatively, you can use the option with the backup command: @@ -3095,10 +3070,9 @@ pg_probackup merge -B backup_dir --instance remote - backup. + especially for large data volumes, as well as I/O and network traffic + if you are using pg_probackup in the + remote mode. Before the merge, pg_probackup validates all the affected @@ -3128,7 +3102,7 @@ pg_probackup delete -B backup_dir --instance backup_id, together with all the incremental backups that descend from - backup_id if any. This way you can delete + backup_id, if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. @@ -3181,11 +3155,9 @@ pg_probackup delete -B backup_dir --instance Commands - This section describes pg_probackup commands. Some commands - require mandatory parameters and can take additional options. - Optional parameters encased in square brackets. For detailed - descriptions of options, see the section - Options. + This section describes pg_probackup commands. + Optional parameters are enclosed in square brackets. For detailed + parameter descriptions, see the section Options. version @@ -3215,13 +3187,13 @@ pg_probackup init -B backup_dir [--help] Initializes the backup catalog in backup_dir that will store backup copies, - WAL archive and meta information for the backed up database + WAL archive, and meta information for the backed up database clusters. If the specified backup_dir already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. - For details, see the secion + For details, see the section Initializing the Backup Catalog. @@ -3267,7 +3239,7 @@ pg_probackup set-config -B backup_dir --instance remote_options] [remote_archive_options] [logging_options] - Adds the specified connection, compression, retention, logging + Adds the specified connection, compression, retention, logging, and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. @@ -3288,7 +3260,7 @@ pg_probackup set-backup -B backup_dir --instance Sets the provided backup-specific settings into the - backup.control configuration file, or modifies previously + backup.control configuration file, or modifies the previously defined values. @@ -3325,15 +3297,15 @@ pg_probackup show -B backup_dir Shows the contents of the backup catalog. If instance_name and backup_id are specified, shows detailed - information about this backup. You can specify the - --format=json option to return the result - in the JSON format. If option is - specified, shows the content of WAL archive of the backup + information about this backup. If the option is + specified, shows the contents of WAL archive of the backup catalog. By default, the contents of the backup catalog is shown as - plain text. + plain text. You can specify the + --format=json option to return the result + in the JSON format. For details on usage, see the sections @@ -3356,11 +3328,8 @@ pg_probackup backup -B backup_dir -b bac [retention_options] [pinning_options] [logging_options] - Creates a backup copy of the PostgreSQL instance. The - backup_mode option specifies the backup - mode to use. - - + Creates a backup copy of the PostgreSQL instance. + @@ -3368,6 +3337,7 @@ pg_probackup backup -B backup_dir -b bac Specifies the backup mode to use. Possible values are: + @@ -3416,7 +3386,7 @@ pg_probackup backup -B backup_dir -b bac - Makes an STREAM backup that + Makes a STREAM backup, which includes all the necessary WAL files by streaming them from the database server via replication protocol. @@ -3432,8 +3402,8 @@ pg_probackup backup -B backup_dir -b bac all the required WAL segments remain available if WAL is rotated while the backup is in progress. This flag can only be used together with the flag. - Default slot name is pg_probackup_slot, - which can be changed via option /. + The default slot name is pg_probackup_slot, + which can be changed using the / option. @@ -3467,7 +3437,7 @@ pg_probackup backup -B backup_dir -b bac Includes the specified directory into the backup. This option - is useful to back up scripts, sql dumps and configuration + is useful to back up scripts, SQL dump files, and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. @@ -3479,18 +3449,18 @@ pg_probackup backup -B backup_dir -b bac - Sets in seconds the timeout for WAL segment archiving and - streaming. By default pg_probackup waits 300 seconds. + Sets the timeout for WAL segment archiving and + streaming, in seconds. By default, pg_probackup waits 300 seconds. - - Disables block-level checksum verification to speed up backup. + Disables block-level checksum verification to speed up + the backup process. @@ -3499,7 +3469,7 @@ pg_probackup backup -B backup_dir -b bac - Skips automatic validation after successfull backup. You can + Skips automatic validation after the backup is taken. You can use this flag if you validate backups regularly and would like to save time when running backup operations. @@ -3509,15 +3479,15 @@ pg_probackup backup -B backup_dir -b bac - Additionally Connection - Options, Retention - Options, Pinning - Options, Remote - Mode Options, - Compression - Options, Logging - Options and Common - Options can be used. + Additionally, connection + options, retention + options, pinning + options, remote + mode options, + compression + options, logging + options, and common + options can be used. For details on usage, see the section @@ -3533,16 +3503,18 @@ pg_probackup restore -B backup_dir --instance OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] [--restore-command=cmdline] -[recovery_options] [logging_options] [remote_options] -[partial_restore_options] [remote_archive_options] +[recovery_target_options] [logging_options] [remote_options] +[partial_restore_options] [remote_wal_archive_options] Restores the PostgreSQL instance from a backup copy located in the backup_dir backup catalog. If you specify a recovery - target option, pg_probackup will find the closest + target option, pg_probackup finds the closest backup and restores it to the specified recovery target. - Otherwise, the most recent backup is used. + If neither the backup ID nor recovery target options are provided, + pg_probackup uses the most recent backup + to preform the recovery. @@ -3603,7 +3575,7 @@ pg_probackup restore -B backup_dir --instance Disables block-level checksum verification to speed up - validation. During automatic validation before restore only + validation. During automatic validation before the restore only file-level checksums will be verified. @@ -3624,9 +3596,9 @@ pg_probackup restore -B backup_dir --instance - Set the + Sets the restore_command - parameter to specified command. Example: + parameter to the specified command. For example: --restore-command='cp /mnt/server/archivedir/%f "%p"' @@ -3636,25 +3608,25 @@ pg_probackup restore -B backup_dir --instance - Allows to ignore the invalid status of the backup. You can use - this flag if you for some reason have the necessity to restore - PostgreSQL cluster from corrupted or invalid backup. Use with - caution. + Allows to ignore an invalid status of the backup. You can use + this flag if you need to restore the + PostgreSQL cluster from a corrupt or an invalid backup. + Use with caution. - Additionally Recovery - Target Options, - Remote Mode - Options, - Remote WAL Archive - Options, Logging - Options, Partial - Restore and Common - Options can be used. + Additionally, recovery + target options, + remote mode + options, + remote WAL archive + options, logging + options, partial + restore options, and common + options can be used. For details on usage, see the section @@ -3684,7 +3656,7 @@ pg_probackup checkdb Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the amcheck - extention or the amcheck_next extension + extension or the amcheck_next extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. @@ -3695,8 +3667,8 @@ pg_probackup checkdb - Skip validation of data files. Can be used only with - flag, so only logical + Skip validation of data files. You can use this flag only + together with the flag, so that only logical verification of indexes is performed. @@ -3708,18 +3680,21 @@ pg_probackup checkdb Checks that all heap tuples that should be indexed are actually indexed. You can use this flag only together with the - flag. Can be used only with - amcheck extension of version 2.0 and - amcheck_next extension of any version. + flag. + + + This check is only possible if you are using the + amcheck extension of version 2.0 or higher, or + the amcheck_next extension of any version. - Additionally Connection - Options and Logging - Options can be used. + Additionally, connection + options and logging + options can be used. For details on usage, see the section @@ -3738,7 +3713,7 @@ pg_probackup validate -B backup_dir Verifies that all the files required to restore the cluster - are present and not corrupted. If + are present and are not corrupt. If instance_name is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the instance_name @@ -3746,7 +3721,7 @@ pg_probackup validate -B backup_dir backups available for this backup instance. If you specify the instance_name with a recovery target - options and/or a backup_id, + option and/or a backup_id, pg_probackup checks whether it is possible to restore the cluster using these options. @@ -3785,7 +3760,7 @@ pg_probackup delete -B backup_dir --instance logging_options] - Deletes backup with specified backip_id + Deletes backup with specified backup_id or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. @@ -3815,31 +3790,29 @@ pg_probackup archive-push -B backup_dir --instance Refuse to push WAL segment segment_name into archive. Instance parameters mismatch. For each WAL file moved to the backup catalog, you - will see the following message in PostgreSQL logfile: + will see the following message in the PostgreSQL log file: pg_probackup archive-push completed successfully. If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and - returns successful execution code. Otherwise, archive-push + returns a successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the flag. - Copying is done to temporary file with - .part suffix or, if - compression is - used, with .gz.part suffix. After copy is - done, atomic rename is performed. This algorihtm ensures that + The files are copied to a temporary file with the + .part suffix. After the copy is + done, atomic rename is performed. This algorithm ensures that a failed archive-push will not stall continuous archiving and - that concurrent archiving from multiple sources into single - WAL archive has no risk of archive corruption. Copied to - archive WAL segments are synced to disk. + that concurrent archiving from multiple sources into a single + WAL archive have no risk of archive corruption. WAL segments copied to + the archive are synced to disk. - You can use archive-push in + You can use archive-push in the archive_command PostgreSQL parameter to set up continous @@ -3942,8 +3915,9 @@ pg_probackup archive-get -B backup_dir --instance - Sets the number of parallel threads for backup, restore, - merge, validation and verification processes. + Sets the number of parallel threads for backup, + restore, merge, + validate, and checkdb processes. @@ -3990,27 +3964,22 @@ pg_probackup archive-get -B backup_dir --instance - immediate value stops the recovery + The immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the / option is omitted. + This is the default behavior for STREAM backups. - latest value continues the recovery + The latest value continues the recovery until all WAL segments available in the archive are - applied. + applied. This is the default behavior for ARCHIVE backups. - - Default value of depends - on WAL delivery method of restored backup, - immediate for STREAM backup and - latest for ARCHIVE. - @@ -4018,9 +3987,8 @@ pg_probackup archive-get -B backup_dir --instance - Specifies a particular timeline to which recovery will - proceed. By default, the timeline of the specified backup is - used. + Specifies a particular timeline to be used for recovery. + By default, the timeline of the specified backup is used. @@ -4031,7 +3999,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring - database cluster of major version 10 or higher. + a database cluster of major version 10 or higher. @@ -4076,7 +4044,7 @@ pg_probackup archive-get -B backup_dir --instance --recovery-target-time, or options. The default - depends on + depends on the recovery_target_inclusive parameter. @@ -4207,7 +4175,7 @@ pg_probackup archive-get -B backup_dir --instance You can use these options together with backup and - set-delete commands. + set-backup commands. For details on backup pinning, see the section @@ -4220,9 +4188,12 @@ pg_probackup archive-get -B backup_dir --instance Specifies the amount of time the backup should be pinned. - Must be a non-negative integer. The zero value unpin already + Must be a non-negative integer. The zero value unpins the already pinned backup. Supported units: ms, s, min, h, d (s by - default). Example: --ttl=30d. + default). + + + Example: --ttl=30d @@ -4232,8 +4203,10 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which the backup will stay - pinned. Must be a ISO-8601 complaint timestamp. Example: - --expire-time='2020-01-01 00:00:00+03' + pinned. Must be an ISO-8601 complaint timestamp. + + + Example: --expire-time='2020-01-01 00:00:00+03' @@ -4266,8 +4239,8 @@ pg_probackup archive-get -B backup_dir --instance All console log messages are going to stderr, so - output from show and - show-config commands do + the output of show and + show-config commands does not mingle with log messages. @@ -4377,7 +4350,7 @@ pg_probackup archive-get -B backup_dir --instance Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command - is launched, except help and version commands. The time of the + is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by @@ -4446,7 +4419,6 @@ pg_probackup archive-get -B backup_dir --instance Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. - Default: 5432 @@ -4527,7 +4499,7 @@ pg_probackup archive-get -B backup_dir --instance Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used - together with option. + together with the option. Default: 1 @@ -4550,18 +4522,18 @@ pg_probackup archive-get -B backup_dir --instance Archiving Options - These options can be used with - archive-push command in + These options can be used with the + archive-push command in the archive_command - setting and archive-get - command in + setting and the archive-get + command in the restore_command setting. - Additionally Remote Mode - Options and Logging - Options can be used. + Additionally, remote mode + options and logging + options can be used. @@ -4571,9 +4543,8 @@ pg_probackup archive-get -B backup_dir --instance Provides the path to the WAL file in archive_command and - restore_command. The %p - variable as value for this option is required for correct - processing. + restore_command. Use the %p + variable as the value for this option for correct processing. @@ -4584,8 +4555,8 @@ pg_probackup archive-get -B backup_dir --instance Provides the name of the WAL file in archive_command and - restore_command. The %f - variable as value is required for correct processing. + restore_command. Use the %f + variable as the value for this option for correct processing. @@ -4617,11 +4588,11 @@ pg_probackup archive-get -B backup_dir --instance set-config, backup, restore, - archive-push and + archive-push, and archive-get commands. - For details on configuring and usage of remote operation mode, + For details on configuring and using the remote mode, see the sections Configuring the Remote Mode and @@ -4640,7 +4611,7 @@ pg_probackup archive-get -B backup_dir --instance - ssh enables the remote backup mode via + ssh enables the remote mode via SSH. This is the default value. @@ -4706,11 +4677,11 @@ pg_probackup archive-get -B backup_dir --instance - Specifies a string of SSH command-line options. For example, - the following options can used to set keep-alive for ssh + Provides a string of SSH command-line options. For example, + the following options can be used to set keep-alive for SSH connections opened by pg_probackup: --ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'. - Full list of possible options can be found on + For the full list of possible options, see ssh_config manual page. @@ -4723,11 +4694,11 @@ pg_probackup archive-get -B backup_dir --instance Remote WAL Archive Options This section describes the options used to provide the - arguments for Remote Mode - Options in - archive-get used in + arguments for remote mode + options in + archive-get used in the restore_command - command when restoring ARCHIVE backup or performing PITR. + command when restoring ARCHIVE backups or performing PITR. @@ -4735,19 +4706,18 @@ pg_probackup archive-get -B backup_dir --instance - Provides the argument for - option in archive-get command. + Provides the argument for the + option in the archive-get command. - - Provides the argument for - option in archive-get command. + Provides the argument for the + option in the archive-get command. Default: 22 @@ -4759,9 +4729,9 @@ pg_probackup archive-get -B backup_dir --instance - Provides the argument for - option in archive-get command. If you omit - this option, the the user running PostgreSQL cluster is used. + Provides the argument for the + option in the archive-get command. If you omit + this option, the user that has started the PostgreSQL cluster is used. Default: PostgreSQL user @@ -4774,8 +4744,8 @@ pg_probackup archive-get -B backup_dir --instance Partial Restore Options - This section describes the options related to partial restore - of a cluster from backup. These options can be used with + This section describes the options for partial cluster restore. + These options can be used with the restore command. @@ -4784,7 +4754,7 @@ pg_probackup archive-get -B backup_dir --instance - Specifies database name to exclude from restore. All other + Specifies the name of the database to exclude from restore. All other databases in the cluster will be restored as usual, including template0 and template1. This option can be specified multiple times for multiple @@ -4797,8 +4767,8 @@ pg_probackup archive-get -B backup_dir --instance - Specifies database name to restore from backup. All other - databases in the cluster will not be restored, with exception + Specifies the name of the database to restore from a backup. All other + databases in the cluster will not be restored, with the exception of template0 and template1. This option can be specified multiple times for multiple databases. @@ -4902,17 +4872,17 @@ pg_probackup archive-get -B backup_dir --instance - Howto + How-To All examples below assume the remote mode of operations via - ssh. If you are planning to run backup and - restore operation locally then step - Setup passwordless SSH connection can be - skipped and all options can be - ommited. + SSH. If you are planning to run backup and + restore operation locally, skip the + Setup passwordless SSH connection step + and omit all options. - Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup + Examples are based on Ubuntu 18.04, + PostgreSQL 11, and pg_probackup 2.2.0. @@ -4943,14 +4913,13 @@ pg_probackup archive-get -B backup_dir --instance postgres — user on - postgres_host which run PostgreSQL cluster. + postgres_host that has started the PostgreSQL cluster. - /var/lib/postgresql/11/main — directory - on postgres_host where PGDATA of PostgreSQL - cluster is located. + /var/lib/postgresql/11/main — PostgreSQL + data directory on postgres_host. @@ -4963,21 +4932,21 @@ pg_probackup archive-get -B backup_dir --instance Minimal Setup - This setup is relying on autonomous FULL and DELTA backups. + This scenario illustrates setting up standalone FULL and DELTA backups. - Setup passwordless SSH connection from + <title>Set up passwordless SSH connection from <literal>backup_host</literal> to - <literal>postgres_host</literal> + postgres_host: [backupman@backup_host] ssh-copy-id postgres@postgres_host - Setup <productname>PostgreSQL</productname> cluster + Configure your <productname>PostgreSQL</productname> cluster. - It is recommended from security purposes to use separate + For security purposes, it is recommended to use a separate database for backup operations. @@ -4985,9 +4954,9 @@ postgres=# CREATE DATABASE backupdb; - Connect to backupdb database, create role - probackup and grant to it the following - permissions: + Connect to the backupdb database, create the + probackup role, and grant the following + permissions to this role: backupdb=# @@ -5009,21 +4978,21 @@ COMMIT; - Init the backup catalog + Initialize the backup catalog: [backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully inited - Add instance <literal>pg-11</literal> to backup catalog + Add instance <literal>pg-11</literal> to the backup catalog: [backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main INFO: Instance 'node' successfully inited - Take FULL backup + Take a FULL backup: [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -5038,7 +5007,7 @@ INFO: Backup PZ7YK2 completed - Lets take a look at the backup catalog + Let's take a look at the backup catalog: [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' @@ -5050,7 +5019,7 @@ BACKUP INSTANCE 'pg-11' - Take incremental backup in DELTA mode + Take an incremental backup in the DELTA mode: [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -5066,14 +5035,14 @@ INFO: Backup PZ7YMP completed - Lets hide some parameters into config, so cmdline can be - less crowdy + Let's add some parameters to <application>pg_probackup</application> + configuration file, so that you can omit them from the command line: [backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb - Take another incremental backup in DELTA mode, omitting + <title>Take another incremental backup in the DELTA mode, omitting some of the previous parameters: [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream @@ -5090,7 +5059,7 @@ INFO: Backup PZ7YR5 completed - Lets take a look at instance config + Let's take a look at the instance configuration: [backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' @@ -5124,12 +5093,12 @@ remote-proto = ssh remote-host = postgres_host - Note, that we are getting default values for other options, - that were not overwritten by set-config command. + Note that we are getting the default values for other options + that were not overwritten by the set-config command. - Lets take a look at the backup catalog + Let's take a look at the backup catalog: [backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' @@ -5148,7 +5117,7 @@ remote-host = postgres_host Versioning - pg_probackup is following the + pg_probackup follows semantic versioning. @@ -5164,7 +5133,7 @@ remote-host = postgres_host Credits pg_probackup utility is based on pg_arman, - that was originally written by NTT and then developed and maintained by Michael Paquier. + which was originally written by NTT and then developed and maintained by Michael Paquier. From 83534d834da97bc4381026822e22c4da00ebe2e0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 Nov 2019 17:28:14 +0200 Subject: [PATCH 1072/2107] DOC: remove trailing space --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index dde00b485..186d54134 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -263,7 +263,7 @@ doc/src/sgml/pgprobackup.sgml - + Archive catalog: getting the list of all WAL timelines and the corresponding meta information in plain text or JSON formats. From a1bc007514f9a18beaa794a018ee72aec5a8ffdb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 Nov 2019 17:36:40 +0200 Subject: [PATCH 1073/2107] DOC: add README for docs and add '.html' files to '.gitignore' --- .gitignore | 3 +++ doc/Readme.md | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 doc/Readme.md diff --git a/.gitignore b/.gitignore index 4fc21d916..2f492c3c0 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ /src/xlogreader.c /src/walmethods.c /src/walmethods.h + +# Doc files +/doc/*html diff --git a/doc/Readme.md b/doc/Readme.md new file mode 100644 index 000000000..b9c74769e --- /dev/null +++ b/doc/Readme.md @@ -0,0 +1,5 @@ +# Generating documentation +``` +xmllint --noout --valid probackup.xml +xsltproc stylesheet.xsl probackup.xml >pg-probackup.html +``` \ No newline at end of file From 968bac6d64e469d257a589abaefdcd108e697e31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 Nov 2019 18:58:18 +0300 Subject: [PATCH 1074/2107] README: update --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a9f908074..a28fedbd4 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,18 @@ The utility is compatible with: * PostgreSQL 9.5, 9.6, 10, 11, 12; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: -* Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow +* Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery -* Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` -* Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting `time to live` for backups +* Verification: on-demand verification of PostgreSQL instance with the `checkdb` command. +* Retention: managing WAL archive and backups in accordance with retention policy. You can configure retention policy based on recovery time or the number of backups to keep, as well as specify `time to live` (TTL) for a particular backup. Expired backups can be merged or deleted. * Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads * Compression: storing backup data in a compressed state to save disk space -* Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) -* Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it -* Backup from replica: avoid extra load on the master server by taking backups from a standby -* External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -* Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats -* Archive Catalog: get list of all WAL timelines and corresponding meta information in `plain` or `json` formats +* Deduplication: saving disk space by not copying unchanged non-data files, such as `_vm` or `_fsm` +* Remote operations: backing up PostgreSQL instance located on a remote system or restoring a backup remotely +* Backup from standby: avoid extra load on master by taking backups from a standby server +* External directories: backing up files and directories located outside of the PostgreSQL `data directory` (PGDATA), such as scripts, configuration files, logs, or SQL dump files. +* Backup Catalog: get list of backups and corresponding meta information in plain text or JSON formats +* Archive catalog: getting the list of all WAL timelines and the corresponding meta information in plain text or JSON formats * Partial Restore: restore only the specified databases or exclude the specified databases from restore. To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog. From 060b84f5c956aad91ce20ffd085dd64cd6408381 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Thu, 7 Nov 2019 17:36:23 +0300 Subject: [PATCH 1075/2107] DOC: add xreflabel for all commands and change to to facilitate linking to command descriptions --- doc/pgprobackup.xml | 203 +++++++++++++++++++++----------------------- 1 file changed, 98 insertions(+), 105 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 186d54134..91c9bf224 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -561,8 +561,7 @@ pg_probackup add-instance -B backup_dir -D For details on how to fine-tune pg_probackup configuration, see - the section Configuring - pg_probackup. + . The user launching pg_probackup must have full access to @@ -593,7 +592,7 @@ pg_probackup add-instance -B backup_dir -D - To perform a backup, the following + To perform a , the following permissions for role backup are required only in the database used for connection to the PostgreSQL server: @@ -794,7 +793,7 @@ archive_command = 'pg_probackup archive-push -B backup_dirremote_options only need to be specified to archive WAL on a remote host. For details about all possible archive-push parameters, see the - section archive-push. + section . Once these steps are complete, you can start making backups in the @@ -804,9 +803,8 @@ archive_command = 'pg_probackup archive-push -B backup_dir You can view the current state of the WAL archive using the - show command. For details, see the - section Viewing - WAL Archive Information. + command. For details, see + . If you are planning to make PAGE backups and/or backups with @@ -824,7 +822,7 @@ archive_command = 'pg_probackup archive-push -B backup_dir - Instead of using the archive-push + Instead of using the command provided by pg_probackup, you can use any other tool to set up continuous archiving as long as it delivers WAL segments into backup_dir/wal/instance_name @@ -1062,11 +1060,11 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Only the following commands can be launched in the remote - mode: add-instance, - backup, - restore, - archive-push, - archive-get. + mode: , + , + , + , + . @@ -1277,7 +1275,7 @@ pg_probackup backup -B backup_dir --instance - During backup pg_probackup + During pg_probackup streams WAL files containing WAL records between Start LSN and Stop LSN to backup_dir/backups/instance_name/backup_id/database/pg_wal directory. To eliminate the risk @@ -1382,7 +1380,7 @@ pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-di To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the - set-config command with the + command with the option. @@ -1445,7 +1443,7 @@ pg_probackup checkdb [-B backup_dir [--instance --amcheck flag when running - the checkdb command: + the command: pg_probackup checkdb -D data_dir --amcheck @@ -1481,13 +1479,13 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ If you would like to skip backup validation, you can specify the flag when running - backup and - restore commands. + and + commands. To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the - validate command with the exact + command with the exact recovery target options you are going to use for recovery. @@ -1535,7 +1533,7 @@ pg_probackup validate -B backup_dir --instance Restoring a Cluster - To restore the database cluster from a backup, run the restore + To restore the database cluster from a backup, run the command with at least the following options: @@ -1613,7 +1611,7 @@ pg_probackup restore -B backup_dir --instance - By default, the restore + By default, the command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the @@ -1629,11 +1627,11 @@ pg_probackup restore -B backup_dir --instance before taking backups, you can restore only some of the databases using partial restore - options with the restore + options with the commands. - To restore the specified databases only, run the restore command + To restore the specified databases only, run the command with the following options: @@ -1688,8 +1686,8 @@ pg_probackup restore -B backup_dir --instance recovery target options with the - restore and - validate commands. + and + commands. If / option is omitted, @@ -1784,8 +1782,8 @@ pg_probackup restore -B backup_dir --instance Installation and Setup. For the - add-instance and - set-config commands, make + and + commands, make sure to specify remote options that point to the database host with the PostgreSQL instance. @@ -1802,8 +1800,8 @@ pg_probackup restore -B backup_dir --instance Setting up continuous WAL archiving. For the - archive-push and - archive-get commands, you + and + commands, you must specify the remote options that point to the backup host with the backup catalog. @@ -1811,8 +1809,8 @@ pg_probackup restore -B backup_dir --instance - Run backup or - restore commands with + Run or + commands with remote options on the backup host. pg_probackup connects to the remote system via SSH and @@ -1891,12 +1889,12 @@ pg_probackup restore -B backup_dir --instance Running <application>pg_probackup</application> on Parallel Threads - Backup, - restore, - merge, - delete, - checkdb and - validate processes can be + , + , + , + , + and + processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk, and network bandwidth). @@ -1928,12 +1926,12 @@ pg_probackup backup -B backup_dir --instance pg_probackup configuration. - For example, backup and - checkdb commands use a regular + For example, and + commands use a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the - set-config command. + command. @@ -2747,7 +2745,7 @@ pg_probackup show -B backup_dir [--instance To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via - set-config: + : --retention-redundancy=redundancy @@ -2808,7 +2806,7 @@ pg_probackup delete -B backup_dir --instance --delete-wal flags and the and options together - with the backup command to + with the command to remove and merge the outdated backup copies once the new backup is created. @@ -2832,8 +2830,8 @@ pg_probackup delete -B backup_dir --instance --merge-expired flag when running - backup or - delete commands. + or + commands. Suppose you have backed up the node @@ -2860,7 +2858,7 @@ BACKUP INSTANCE 'node' retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the - delete command with the + command with the flag, only the P7XDFT full backup will be removed. @@ -2915,7 +2913,7 @@ pg_probackup set-backup -B backup_dir --instance Alternatively, you can use the and options with the - backup command to pin the newly + command to pin the newly created backup: @@ -2924,7 +2922,7 @@ pg_probackup backup -B backup_dir --instance To check if the backup is pinned, - run the show command: + run the command: pg_probackup show -B backup_dir --instance instance_name -i backup_id @@ -2953,7 +2951,7 @@ data-bytes = 22288792 You can unpin the backup by setting the option to zero using the - set-backup command. For example: + command. For example: pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 @@ -2993,7 +2991,7 @@ BACKUP INSTANCE 'node' You can check the state of the WAL archive by running the - show command with the + command with the flag: @@ -3037,7 +3035,7 @@ ARCHIVE INSTANCE 'node' Alternatively, you can use the - option with the backup command: + option with the command: pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal @@ -3077,7 +3075,7 @@ pg_probackup merge -B backup_dir --instance Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current - backup status by running the show + backup status by running the command with the backup ID: @@ -3159,7 +3157,7 @@ pg_probackup delete -B backup_dir --instance Options. - + version pg_probackup version @@ -3168,7 +3166,7 @@ pg_probackup version Prints pg_probackup version. - + help pg_probackup help [command] @@ -3179,7 +3177,7 @@ pg_probackup help [command] about the options that can be used with this command. - + init pg_probackup init -B backup_dir [--help] @@ -3198,7 +3196,7 @@ pg_probackup init -B backup_dir [--help] the Backup Catalog. - + add-instance pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] @@ -3216,7 +3214,7 @@ pg_probackup add-instance -B backup_dir -D . - + del-instance pg_probackup del-instance -B backup_dir --instance instance_name [--help] @@ -3226,7 +3224,7 @@ pg_probackup del-instance -B backup_dir --instance - + set-config pg_probackup set-config -B backup_dir --instance instance_name @@ -3252,7 +3250,7 @@ pg_probackup set-config -B backup_dir --instance pg_probackup.conf manually. - + set-backup pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id @@ -3268,7 +3266,7 @@ pg_probackup set-backup -B backup_dir --instance Pinning Options. - + show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] @@ -3284,10 +3282,10 @@ pg_probackup show-config -B backup_dir --instance To edit pg_probackup.conf, use the - set-config command. + command. - + show pg_probackup show -B backup_dir @@ -3315,7 +3313,7 @@ pg_probackup show -B backup_dir Archive Information. - + backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name @@ -3494,7 +3492,7 @@ pg_probackup backup -B backup_dir -b bac Creating a Backup. - + restore pg_probackup restore -B backup_dir --instance instance_name @@ -3634,7 +3632,7 @@ pg_probackup restore -B backup_dir --instance . - + checkdb pg_probackup checkdb @@ -3702,7 +3700,7 @@ pg_probackup checkdb Cluster. - + validate pg_probackup validate -B backup_dir @@ -3731,7 +3729,7 @@ pg_probackup validate -B backup_dir Backup. - + merge pg_probackup merge -B backup_dir --instance instance_name -i backup_id @@ -3749,7 +3747,7 @@ pg_probackup merge -B backup_dir --instance Merging Backups. - + delete pg_probackup delete -B backup_dir --instance instance_name @@ -3772,7 +3770,7 @@ pg_probackup delete -B backup_dir --instance . - + archive-push pg_probackup archive-push -B backup_dir --instance instance_name @@ -3825,7 +3823,7 @@ pg_probackup archive-push -B backup_dir --instance . - + archive-get pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name @@ -3852,9 +3850,7 @@ pg_probackup archive-get -B backup_dir --instance - For details, see the section - Configuring - pg_probackup. + For details, see . If an option is specified using more than one method, @@ -3893,7 +3889,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the - add-instance command. + command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. @@ -3949,8 +3945,8 @@ pg_probackup archive-get -B backup_dir --instance continuous WAL archiving is configured, you can use one of these - options together with restore - or validate commands to + options together with + or commands to specify the moment up to which the database cluster must be restored or validated. @@ -4072,8 +4068,8 @@ pg_probackup archive-get -B backup_dir --instance Retention Options You can use these options together with - backup and - delete commands. + and + commands. For details on configuring retention policy, see the section @@ -4174,8 +4170,8 @@ pg_probackup archive-get -B backup_dir --instance Pinning Options You can use these options together with - backup and - set-backup commands. + and + commands. For details on backup pinning, see the section @@ -4239,8 +4235,8 @@ pg_probackup archive-get -B backup_dir --instance All console log messages are going to stderr, so - the output of show and - show-config commands does + the output of and + commands does not mingle with log messages. @@ -4368,8 +4364,8 @@ pg_probackup archive-get -B backup_dir --instance Connection Options You can use these options together with - backup and - checkdb commands. + and + commands. All @@ -4470,8 +4466,8 @@ pg_probackup archive-get -B backup_dir --instance Compression Options You can use these options together with - backup and - archive-push commands. + and + commands. @@ -4484,7 +4480,7 @@ pg_probackup archive-get -B backup_dir --instance pglz, and none. If set to zlib or pglz, this option enables compression. By default, compression is disabled. For the - archive-push command, the + command, the pglz compression algorithm is not supported. @@ -4523,9 +4519,9 @@ pg_probackup archive-get -B backup_dir --instance Archiving Options These options can be used with the - archive-push command in the + command in the archive_command - setting and the archive-get + setting and the command in the restore_command setting. @@ -4566,7 +4562,7 @@ pg_probackup archive-get -B backup_dir --instance Overwrites archived WAL file. Use this flag together with the - archive-push command if + command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment @@ -4584,20 +4580,17 @@ pg_probackup archive-get -B backup_dir --instance This section describes the options related to running pg_probackup operations remotely via SSH. These options can be - used with add-instance, - set-config, - backup, - restore, - archive-push, and - archive-get commands. + used with , + , + , + , + , and + commands. For details on configuring and using the remote mode, - see the sections - Configuring the - Remote Mode and - Using - pg_probackup in the Remote Mode. + see and + . @@ -4696,7 +4689,7 @@ pg_probackup archive-get -B backup_dir --instance remote mode options in - archive-get used in the + used in the restore_command command when restoring ARCHIVE backups or performing PITR. @@ -4746,7 +4739,7 @@ pg_probackup archive-get -B backup_dir --instance This section describes the options for partial cluster restore. These options can be used with the - restore command. + command. @@ -4804,7 +4797,7 @@ pg_probackup archive-get -B backup_dir --instance pg_probackup.conf using the - set-config command. + command. Default: postgres, the default PostgreSQL database @@ -4858,7 +4851,7 @@ pg_probackup archive-get -B backup_dir --instance pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the - set-config command. + command. Default: 300 sec @@ -5140,4 +5133,4 @@ remote-host = postgres_host - \ No newline at end of file + From 24d0224981783a766b84868788163283ff27658c Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Fri, 8 Nov 2019 12:41:40 +0300 Subject: [PATCH 1076/2107] DOC: remove _ from IDs --- doc/pgprobackup.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 91c9bf224..346f361ac 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1135,7 +1135,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Setting up PTRACK Backups - The PTACK backup mode can be used only for Postgres Pro Standard and + The PTRACK backup mode can be used only for Postgres Pro Standard and Postgres Pro Enterprise installations, or patched vanilla PostgreSQL. Links to PTRACK patches can be found here. @@ -1886,7 +1886,7 @@ pg_probackup restore -B backup_dir --instance - + Running <application>pg_probackup</application> on Parallel Threads , @@ -4928,7 +4928,7 @@ pg_probackup archive-get -B backup_dir --instance - + Set up passwordless SSH connection from <literal>backup_host</literal> to <literal>postgres_host</literal>: From b5564e8c3895aee0e1958acc7671e65e3214c3e8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 8 Nov 2019 19:18:38 +0300 Subject: [PATCH 1077/2107] Makefile update for future PG13 --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 627bf2861..81391ad64 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ else srchome=$(top_srcdir) endif +#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) ifneq (12,$(MAJORVERSION)) EXTRA_CLEAN += src/logging.h INCLUDES += src/logging.h @@ -76,6 +77,7 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ +#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) ifneq (12,$(MAJORVERSION)) src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ From 40003490a32bbb7dc51eba529fb87f1b07e500a7 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Wed, 13 Nov 2019 12:53:05 +0300 Subject: [PATCH 1078/2107] DOC: add pbk path to archive_command/restore_command - PGPRO-3240 --- doc/pgprobackup.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 346f361ac..1752d6fc7 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -782,12 +782,14 @@ ALTER ROLE backup WITH REPLICATION; parameter, as follows: -archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' +archive_command = 'install_dir/pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' - where backup_dir and + where install_dir is the + installation directory of the pg_probackup + version you are going to use, backup_dir and instance_name refer to the already initialized backup catalog instance for this database cluster, and remote_options @@ -1869,15 +1871,14 @@ pg_probackup restore -B backup_dir --instance recovery.conf: -# recovery.conf generated by pg_probackup 2.1.5 -restore_command = 'pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +restore_command = 'install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' Alternatively, you can use the option to provide the entire restore_command: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' From e1ed647c2146c7ef6ac7533405c57f025560ce0a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 Nov 2019 10:56:00 +0300 Subject: [PATCH 1079/2107] [Issue #149] Allow to set primary_slot_name during restore, option "-S,--slot" can now be used with "restore" command --- src/pg_probackup.c | 4 ++++ src/pg_probackup.h | 1 + src/restore.c | 15 ++++++++++----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0b2820b85..07d8917dd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -653,12 +653,16 @@ main(int argc, char *argv[]) if (force) no_validate = true; + if (replication_slot != NULL) + restore_as_replica = true; + /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); restore_params->force = force; restore_params->no_validate = no_validate; restore_params->restore_as_replica = restore_as_replica; + restore_params->primary_slot_name = replication_slot; restore_params->skip_block_validation = skip_block_validation; restore_params->skip_external_dirs = skip_external_dirs; restore_params->partial_db_list = NULL; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index be8757a5b..274442296 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -392,6 +392,7 @@ typedef struct pgRestoreParams bool skip_external_dirs; bool skip_block_validation; //Start using it const char *restore_command; + const char *primary_slot_name; /* options for partial restore */ PartialRestoreType partial_restore_type; diff --git a/src/restore.c b/src/restore.c index 8b2924dda..e0172c83d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -937,6 +937,7 @@ create_recovery_conf(time_t backup_id, /* construct restore_command */ if (pitr_requested) { + fio_fprintf(fp, "\n## recovery settings\n"); /* If restore_command is provided, use it. Otherwise construct it from scratch. */ if (restore_command_provided) sprintf(restore_command_guc, "%s", instance_config.restore_command); @@ -1012,8 +1013,15 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); } + if (pitr_requested) + { + elog(LOG, "Setting restore_command to '%s'", restore_command_guc); + fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); + } + if (params->restore_as_replica) { + fio_fprintf(fp, "\n## standby settings\n"); /* standby_mode was removed in PG12 */ #if PG_VERSION_NUM < 120000 fio_fprintf(fp, "standby_mode = 'on'\n"); @@ -1021,12 +1029,9 @@ create_recovery_conf(time_t backup_id, if (backup->primary_conninfo) fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); - } - if (pitr_requested) - { - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); + if (params->primary_slot_name != NULL) + fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); } if (fio_fflush(fp) != 0 || From 5f0c024422c7584aaefc9058457f9f61c7a0d95f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 17 Nov 2019 11:35:53 +0300 Subject: [PATCH 1080/2107] [Issue #150] Pinned backups should be ignored for the purpose of WAL retention fulfillment --- src/catalog.c | 14 +++++++++++--- src/delete.c | 4 ---- src/pg_probackup.c | 4 ++++ src/pg_probackup.h | 1 + src/restore.c | 2 -- src/show.c | 1 - 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 669efd9a3..6da5214ab 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -988,14 +988,22 @@ catalog_get_timelines(InstanceConfig *instance) { pgBackup *backup = parray_get(tlinfo->backups, j); + /* sanity */ + if (XLogRecPtrIsInvalid(backup->start_lsn) || + backup->tli <= 0) + continue; + /* skip invalid backups */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) continue; - /* sanity */ - if (XLogRecPtrIsInvalid(backup->start_lsn) || - backup->tli <= 0) + /* + * Pinned backups should be ignored for the + * purpose of retention fulfillment, so skip them. + */ + if (backup->expire_time > 0 && + backup->expire_time > current_time) continue; count++; diff --git a/src/delete.c b/src/delete.c index be07554dc..ad7443315 100644 --- a/src/delete.c +++ b/src/delete.c @@ -209,7 +209,6 @@ static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list) { int i; - time_t current_time; parray *redundancy_full_backup_list = NULL; @@ -221,9 +220,6 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* For fancy reporting */ uint32 actual_window = 0; - /* Get current time */ - current_time = time(NULL); - /* Calculate n_full_backups and days_threshold */ if (instance_config.retention_redundancy > 0) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0b2820b85..1b2a2292c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -92,6 +92,7 @@ static char *target_action = NULL; static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; +time_t current_time = 0; bool restore_as_replica = false; bool no_validate = false; @@ -267,6 +268,9 @@ main(int argc, char *argv[]) PROGRAM_NAME = get_progname(argv[0]); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); + /* Get current time */ + current_time = time(NULL); + //set_pglocale_pgservice(argv[0], "pgscripts"); #if PG_VERSION_NUM >= 110000 diff --git a/src/pg_probackup.h b/src/pg_probackup.h index be8757a5b..5d2a3c228 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -282,6 +282,7 @@ typedef struct InstanceConfig extern ConfigOption instance_options[]; extern InstanceConfig instance_config; +extern time_t current_time; typedef struct PGNodeInfo { diff --git a/src/restore.c b/src/restore.c index 8b2924dda..3d4e544c1 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1101,10 +1101,8 @@ pg12_recovery_config(pgBackup *backup, bool add_include) if (add_include) { - time_t current_time; char current_time_str[100]; - current_time = time(NULL); time2iso(current_time_str, lengthof(current_time_str), current_time); snprintf(postgres_auto_path, lengthof(postgres_auto_path), diff --git a/src/show.c b/src/show.c index ed0cca16f..e14196090 100644 --- a/src/show.c +++ b/src/show.c @@ -468,7 +468,6 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na uint32 widths[SHOW_FIELDS_COUNT]; uint32 widths_sum = 0; ShowBackendRow *rows; - time_t current_time = time(NULL); TimeLineID parent_tli = 0; for (i = 0; i < SHOW_FIELDS_COUNT; i++) From c513fd58ec5e452b0809dd4d8853b8606f9e62c9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 Nov 2019 12:32:49 +0300 Subject: [PATCH 1081/2107] Readme: ALT Linux packages installation instructions are added --- README.md | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a28fedbd4..525cdfe19 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,21 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #SRPM Packages yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} + +#RPM ALT Linux p7 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo + +#RPM ALT Linux p8 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo + +#RPM ALT Linux p9 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo ``` #### pg_probackup for PostgresPro Standart and Enterprise @@ -96,23 +111,38 @@ yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list sudo wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg #RPM Centos Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{11,10,9.6} +yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo + +#RPM ALT Linux p7 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo + +#RPM ALT Linux p8 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo + +#RPM ALT Linux p9 +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). From 088801a1f1fd1c2e5d35313781b4c5149aba396d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 Nov 2019 12:39:37 +0300 Subject: [PATCH 1082/2107] Readme: minor update --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 525cdfe19..600521e06 100644 --- a/README.md +++ b/README.md @@ -91,17 +91,20 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} #RPM ALT Linux p7 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux p8 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux p9 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo ``` @@ -130,12 +133,14 @@ yum install pg_probackup-{std,ent}-{11,10,9.6} yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM ALT Linux p7 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM ALT Linux p8 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo From 232dee2e884b765eb8f6644e61254d23977ac371 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 Nov 2019 18:19:14 +0300 Subject: [PATCH 1083/2107] DOC: css added --- doc/stylesheet.css | 402 +++++++++++++++++++++++++++++++++++++++++++++ doc/stylesheet.xsl | 2 + 2 files changed, 404 insertions(+) create mode 100644 doc/stylesheet.css diff --git a/doc/stylesheet.css b/doc/stylesheet.css new file mode 100644 index 000000000..0c3a5b3ad --- /dev/null +++ b/doc/stylesheet.css @@ -0,0 +1,402 @@ +/* ../media/css/docs.css */ +.navheader th { text-align: center; } /* anti-bootstrap */ + +.navheader tbody tr:nth-child(1) th { /* временно убрать ненужную строчку */ + display: none; +} + +/* PostgreSQL.org Documentation Style */ + +.docs div.NAVHEADER table { + margin-left: 0; +} + +.docs div.NAVHEADER th { + text-align: center; +} + +.docs { + font-size: 15px; + line-height: 1.6; +} + +/* Heading Definitions */ + +.docs h1, +.docs h2, +.docs h3 { + font-weight: bold; + margin-top: 2ex; +} + +.docs h1 a, +.docs h2 a, +.docs h3 a, +.docs h4 a + { + color: #EC5800; +} + +/* EKa --> */ +.docs h1 { + font-size: 1.4em; +} + +.docs h2 { + font-size: 1.25em; +} + +.docs h3 { + font-size: 1.2em; +} + +.docs h4 { + font-size: 1.15em; +} + +.docs h5 { + font-size: 1.1em; +} + +.docs h6 { + font-size: 1.0em; +} +/* <-- EKa */ + +.docs h1 a:hover { + color: #EC5800; + text-decoration: none; +} + +.docs h2 a:hover, +.docs h3 a:hover, +.docs h4 a:hover { + color: #666666; + text-decoration: none; +} + + + +/* Text Styles */ + +.docs div.SECT2 { + margin-top: 4ex; +} + +.docs div.SECT3 { + margin-top: 3ex; + margin-left: 3ex; +} + +.docs .txtCurrentLocation { + font-weight: bold; +} + +.docs p, +.docs ol, +.docs ul, +.docs li { + line-height: 1.5em; +} + +.docs code { + font-size: 1em; + padding: 0px; + color: #525f6c; + background-color: #FFF; + border-radius: 0px; +} + +.docs code, kbd, pre, samp { + font-family: monospace,monospace; +} + +.docs .txtCommentsWrap { + border: 2px solid #F5F5F5; + width: 100%; +} + +.docs .txtCommentsContent { + background: #F5F5F5; + padding: 3px; +} + +.docs .txtCommentsPoster { + float: left; +} + +.docs .txtCommentsDate { + float: right; +} + +.docs .txtCommentsComment { + padding: 3px; +} + +.docs #docContainer pre code, +.docs #docContainer pre tt, +.docs #docContainer pre pre, +.docs #docContainer tt tt, +.docs #docContainer tt code, +.docs #docContainer tt pre { + font-size: 1em; +} + +.docs pre.LITERALLAYOUT, +.docs .SCREEN, +.docs .SYNOPSIS, +.docs .PROGRAMLISTING, +.docs .REFSYNOPSISDIV p, +.docs table.CAUTION, +.docs table.WARNING, +.docs blockquote.NOTE, +.docs blockquote.TIP, +.docs div.note, +.docs div.tip, +.docs table.CALSTABLE { + -moz-box-shadow: 3px 3px 5px #DFDFDF; + -webkit-box-shadow: 3px 3px 5px #DFDFDF; + -khtml-box-shadow: 3px 3px 5px #DFDFDF; + -o-box-shadow: 3px 3px 5px #DFDFDF; + box-shadow: 3px 3px 5px #DFDFDF; +} + +.docs pre.LITERALLAYOUT, +.docs .SCREEN, +.docs .SYNOPSIS, +.docs .PROGRAMLISTING, +.docs .REFSYNOPSISDIV p, +.docs table.CAUTION, +.docs table.WARNING, +.docs blockquote.NOTE, +.docs blockquote.TIP +.docs div.note, +.docs div.tip { + color: black; + border-width: 1px; + border-style: solid; + padding: 2ex; + margin: 2ex 0 2ex 2ex; + overflow: auto; + -moz-border-radius: 8px; + -webkit-border-radius: 8px; + -khtml-border-radius: 8px; + border-radius: 8px; +} + +.docs div.note, +.docs div.tip { + -moz-border-radius: 8px !important; + -webkit-border-radius: 8px !important; + -khtml-border-radius: 8px !important; + border-radius: 8px !important; +} + + +.docs pre.LITERALLAYOUT, +.docs pre.SYNOPSIS, +.docs pre.PROGRAMLISTING, +.docs .REFSYNOPSISDIV p, +.docs .SCREEN { + border-color: #CFCFCF; + background-color: #F7F7F7; +} + +.docs blockquote.NOTE, +.docs blockquote.TIP, +.docs div.note, +.docs div.tip { + border-color: #DBDBCC; + background-color: #EEEEDD; + padding: 14px; + width: 572px; +/* font-size: 12px; */ +} + +.docs blockquote.NOTE, +.docs blockquote.TIP, +.docs table.CAUTION, +.docs table.WARNING { + margin: 4ex auto; +} + +.docs div.note, +.docs div.tip { + margin: 4ex auto !important; +} + + +.docs blockquote.NOTE p, +.docs blockquote.TIP p, +.docs div.note p, +.docs div.tip p { + margin: 0; +} + +.docs blockquote.NOTE pre, +.docs blockquote.NOTE code, +.docs div.note pre, +.docs div.note code, +.docs blockquote.TIP pre, +.docs blockquote.TIP code, +.docs div.tip pre, +.docs div.tio code { + margin-left: 0; + margin-right: 0; + -moz-box-shadow: none; + -webkit-box-shadow: none; + -khtml-box-shadow: none; + -o-box-shadow: none; + box-shadow: none; +} + +.docs .emphasis, +.docs .c2 { + font-weight: bold; +} + +.docs .REPLACEABLE { + font-style: italic; +} + +/* Table Styles */ + +.docs table { + margin-left: 2ex; +} + +.docs table.CALSTABLE td, +.docs table.CALSTABLE th, +.docs table.CAUTION td, +.docs table.CAUTION th, +.docs table.WARNING td, +.docs table.WARNING th { + border-style: solid; +} + +.docs table.CALSTABLE, +.docs table.CAUTION, +.docs table.WARNING { + border-spacing: 0; + border-collapse: collapse; +} + +.docs table.CALSTABLE +{ + margin: 2ex 0 2ex 2ex; + background-color: #E0ECEF; + border: 2px solid #A7C6DF; +} + +.docs table.CALSTABLE tr:hover td +{ + background-color: #EFEFEF; +} + +.docs table.CALSTABLE td { + background-color: #FFF; +} + +.docs table.CALSTABLE td, +.docs table.CALSTABLE th { + border: 1px solid #A7C6DF; + padding: 0.5ex 0.5ex; +} + +table.CAUTION, +.docs table.WARNING { + border-collapse: separate; + display: block; + padding: 0; + max-width: 600px; +} + +.docs table.CAUTION { + background-color: #F5F5DC; + border-color: #DEDFA7; +} + +.docs table.WARNING { + background-color: #FFD7D7; + border-color: #DF421E; +} + +.docs table.CAUTION td, +.docs table.CAUTION th, +.docs table.WARNING td, +.docs table.WARNING th { + border-width: 0; + padding-left: 2ex; + padding-right: 2ex; +} + +.docs table.CAUTION td, +.docs table.CAUTION th { + border-color: #F3E4D5 +} + +.docs table.WARNING td, +.docs table.WARNING th { + border-color: #FFD7D7; +} + +.docs td.c1, +.docs td.c2, +.docs td.c3, +.docs td.c4, +.docs td.c5, +.docs td.c6 { + font-size: 1.1em; + font-weight: bold; + border-bottom: 0px solid #FFEFEF; + padding: 1ex 2ex 0; +} + +.docs .table thead { + background: #E0ECEF; + border-bottom: 1px solid #000; +} +.docs .table > thead > tr > th { + border-bottom: 1px solid #000; +} + +.docs td, th { + padding: 0.1ex 0.5ex; +} + +.docs .book table tr:hover td { + background-color: #EFEFEF; +} + +/* Link Styles */ + +.docs #docNav a { + font-weight: bold; +} + +.docs code.FUNCTION tt { + font-size: 1em; +} + +.docs table.docs-compare { + align: center; + width: 90%; + border: 2px solid #999; + border-collapse: collapse; +} + +.docs table.docs-compare td { + padding: 12px; + border: 1px solid #DDD; +} + +.docs dd { + margin-left: 40px; +} + + +.docs .sidebar { + padding: 8px; + background: #FFF; + width: auto; +} diff --git a/doc/stylesheet.xsl b/doc/stylesheet.xsl index 332cb6f15..466127e9c 100644 --- a/doc/stylesheet.xsl +++ b/doc/stylesheet.xsl @@ -6,6 +6,8 @@ + + From 2d8c291c344819aa82c81c8c5f82814584efad15 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Nov 2019 13:54:49 +0300 Subject: [PATCH 1084/2107] DOC: CSS fix --- doc/stylesheet.css | 274 ++++++++++++++++++++++----------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/doc/stylesheet.css b/doc/stylesheet.css index 0c3a5b3ad..fe2c45df1 100644 --- a/doc/stylesheet.css +++ b/doc/stylesheet.css @@ -7,70 +7,70 @@ /* PostgreSQL.org Documentation Style */ -.docs div.NAVHEADER table { +.book div.NAVHEADER table { margin-left: 0; } -.docs div.NAVHEADER th { +.book div.NAVHEADER th { text-align: center; } -.docs { +.book { font-size: 15px; line-height: 1.6; } /* Heading Definitions */ -.docs h1, -.docs h2, -.docs h3 { +.book h1, +.book h2, +.book h3 { font-weight: bold; margin-top: 2ex; } -.docs h1 a, -.docs h2 a, -.docs h3 a, -.docs h4 a +.book h1 a, +.book h2 a, +.book h3 a, +.book h4 a { color: #EC5800; } /* EKa --> */ -.docs h1 { +.book h1 { font-size: 1.4em; } -.docs h2 { +.book h2 { font-size: 1.25em; } -.docs h3 { +.book h3 { font-size: 1.2em; } -.docs h4 { +.book h4 { font-size: 1.15em; } -.docs h5 { +.book h5 { font-size: 1.1em; } -.docs h6 { +.book h6 { font-size: 1.0em; } /* <-- EKa */ -.docs h1 a:hover { +.book h1 a:hover { color: #EC5800; text-decoration: none; } -.docs h2 a:hover, -.docs h3 a:hover, -.docs h4 a:hover { +.book h2 a:hover, +.book h3 a:hover, +.book h4 a:hover { color: #666666; text-decoration: none; } @@ -79,27 +79,27 @@ /* Text Styles */ -.docs div.SECT2 { +.book div.SECT2 { margin-top: 4ex; } -.docs div.SECT3 { +.book div.SECT3 { margin-top: 3ex; margin-left: 3ex; } -.docs .txtCurrentLocation { +.book .txtCurrentLocation { font-weight: bold; } -.docs p, -.docs ol, -.docs ul, -.docs li { +.book p, +.book ol, +.book ul, +.book li { line-height: 1.5em; } -.docs code { +.book code { font-size: 1em; padding: 0px; color: #525f6c; @@ -107,53 +107,53 @@ border-radius: 0px; } -.docs code, kbd, pre, samp { +.book code, kbd, pre, samp { font-family: monospace,monospace; } -.docs .txtCommentsWrap { +.book .txtCommentsWrap { border: 2px solid #F5F5F5; width: 100%; } -.docs .txtCommentsContent { +.book .txtCommentsContent { background: #F5F5F5; padding: 3px; } -.docs .txtCommentsPoster { +.book .txtCommentsPoster { float: left; } -.docs .txtCommentsDate { +.book .txtCommentsDate { float: right; } -.docs .txtCommentsComment { +.book .txtCommentsComment { padding: 3px; } -.docs #docContainer pre code, -.docs #docContainer pre tt, -.docs #docContainer pre pre, -.docs #docContainer tt tt, -.docs #docContainer tt code, -.docs #docContainer tt pre { +.book #docContainer pre code, +.book #docContainer pre tt, +.book #docContainer pre pre, +.book #docContainer tt tt, +.book #docContainer tt code, +.book #docContainer tt pre { font-size: 1em; } -.docs pre.LITERALLAYOUT, -.docs .SCREEN, -.docs .SYNOPSIS, -.docs .PROGRAMLISTING, -.docs .REFSYNOPSISDIV p, -.docs table.CAUTION, -.docs table.WARNING, -.docs blockquote.NOTE, -.docs blockquote.TIP, -.docs div.note, -.docs div.tip, -.docs table.CALSTABLE { +.book pre.LITERALLAYOUT, +.book .SCREEN, +.book .SYNOPSIS, +.book .PROGRAMLISTING, +.book .REFSYNOPSISDIV p, +.book table.CAUTION, +.book table.WARNING, +.book blockquote.NOTE, +.book blockquote.TIP, +.book div.note, +.book div.tip, +.book table.CALSTABLE { -moz-box-shadow: 3px 3px 5px #DFDFDF; -webkit-box-shadow: 3px 3px 5px #DFDFDF; -khtml-box-shadow: 3px 3px 5px #DFDFDF; @@ -161,17 +161,17 @@ box-shadow: 3px 3px 5px #DFDFDF; } -.docs pre.LITERALLAYOUT, -.docs .SCREEN, -.docs .SYNOPSIS, -.docs .PROGRAMLISTING, -.docs .REFSYNOPSISDIV p, -.docs table.CAUTION, -.docs table.WARNING, -.docs blockquote.NOTE, -.docs blockquote.TIP -.docs div.note, -.docs div.tip { +.book pre.LITERALLAYOUT, +.book .SCREEN, +.book .SYNOPSIS, +.book .PROGRAMLISTING, +.book .REFSYNOPSISDIV p, +.book table.CAUTION, +.book table.WARNING, +.book blockquote.NOTE, +.book blockquote.TIP +.book div.note, +.book div.tip { color: black; border-width: 1px; border-style: solid; @@ -184,8 +184,8 @@ border-radius: 8px; } -.docs div.note, -.docs div.tip { +.book div.note, +.book div.tip { -moz-border-radius: 8px !important; -webkit-border-radius: 8px !important; -khtml-border-radius: 8px !important; @@ -193,19 +193,19 @@ } -.docs pre.LITERALLAYOUT, -.docs pre.SYNOPSIS, -.docs pre.PROGRAMLISTING, -.docs .REFSYNOPSISDIV p, -.docs .SCREEN { +.book pre.LITERALLAYOUT, +.book pre.SYNOPSIS, +.book pre.PROGRAMLISTING, +.book .REFSYNOPSISDIV p, +.book .SCREEN { border-color: #CFCFCF; background-color: #F7F7F7; } -.docs blockquote.NOTE, -.docs blockquote.TIP, -.docs div.note, -.docs div.tip { +.book blockquote.NOTE, +.book blockquote.TIP, +.book div.note, +.book div.tip { border-color: #DBDBCC; background-color: #EEEEDD; padding: 14px; @@ -213,34 +213,34 @@ /* font-size: 12px; */ } -.docs blockquote.NOTE, -.docs blockquote.TIP, -.docs table.CAUTION, -.docs table.WARNING { +.book blockquote.NOTE, +.book blockquote.TIP, +.book table.CAUTION, +.book table.WARNING { margin: 4ex auto; } -.docs div.note, -.docs div.tip { +.book div.note, +.book div.tip { margin: 4ex auto !important; } -.docs blockquote.NOTE p, -.docs blockquote.TIP p, -.docs div.note p, -.docs div.tip p { +.book blockquote.NOTE p, +.book blockquote.TIP p, +.book div.note p, +.book div.tip p { margin: 0; } -.docs blockquote.NOTE pre, -.docs blockquote.NOTE code, -.docs div.note pre, -.docs div.note code, -.docs blockquote.TIP pre, -.docs blockquote.TIP code, -.docs div.tip pre, -.docs div.tio code { +.book blockquote.NOTE pre, +.book blockquote.NOTE code, +.book div.note pre, +.book div.note code, +.book blockquote.TIP pre, +.book blockquote.TIP code, +.book div.tip pre, +.book div.tio code { margin-left: 0; margin-right: 0; -moz-box-shadow: none; @@ -250,152 +250,152 @@ box-shadow: none; } -.docs .emphasis, -.docs .c2 { +.book .emphasis, +.book .c2 { font-weight: bold; } -.docs .REPLACEABLE { +.book .REPLACEABLE { font-style: italic; } /* Table Styles */ -.docs table { +.book table { margin-left: 2ex; } -.docs table.CALSTABLE td, -.docs table.CALSTABLE th, -.docs table.CAUTION td, -.docs table.CAUTION th, -.docs table.WARNING td, -.docs table.WARNING th { +.book table.CALSTABLE td, +.book table.CALSTABLE th, +.book table.CAUTION td, +.book table.CAUTION th, +.book table.WARNING td, +.book table.WARNING th { border-style: solid; } -.docs table.CALSTABLE, -.docs table.CAUTION, -.docs table.WARNING { +.book table.CALSTABLE, +.book table.CAUTION, +.book table.WARNING { border-spacing: 0; border-collapse: collapse; } -.docs table.CALSTABLE +.book table.CALSTABLE { margin: 2ex 0 2ex 2ex; background-color: #E0ECEF; border: 2px solid #A7C6DF; } -.docs table.CALSTABLE tr:hover td +.book table.CALSTABLE tr:hover td { background-color: #EFEFEF; } -.docs table.CALSTABLE td { +.book table.CALSTABLE td { background-color: #FFF; } -.docs table.CALSTABLE td, -.docs table.CALSTABLE th { +.book table.CALSTABLE td, +.book table.CALSTABLE th { border: 1px solid #A7C6DF; padding: 0.5ex 0.5ex; } table.CAUTION, -.docs table.WARNING { +.book table.WARNING { border-collapse: separate; display: block; padding: 0; max-width: 600px; } -.docs table.CAUTION { +.book table.CAUTION { background-color: #F5F5DC; border-color: #DEDFA7; } -.docs table.WARNING { +.book table.WARNING { background-color: #FFD7D7; border-color: #DF421E; } -.docs table.CAUTION td, -.docs table.CAUTION th, -.docs table.WARNING td, -.docs table.WARNING th { +.book table.CAUTION td, +.book table.CAUTION th, +.book table.WARNING td, +.book table.WARNING th { border-width: 0; padding-left: 2ex; padding-right: 2ex; } -.docs table.CAUTION td, -.docs table.CAUTION th { +.book table.CAUTION td, +.book table.CAUTION th { border-color: #F3E4D5 } -.docs table.WARNING td, -.docs table.WARNING th { +.book table.WARNING td, +.book table.WARNING th { border-color: #FFD7D7; } -.docs td.c1, -.docs td.c2, -.docs td.c3, -.docs td.c4, -.docs td.c5, -.docs td.c6 { +.book td.c1, +.book td.c2, +.book td.c3, +.book td.c4, +.book td.c5, +.book td.c6 { font-size: 1.1em; font-weight: bold; border-bottom: 0px solid #FFEFEF; padding: 1ex 2ex 0; } -.docs .table thead { +.book .table thead { background: #E0ECEF; border-bottom: 1px solid #000; } -.docs .table > thead > tr > th { +.book .table > thead > tr > th { border-bottom: 1px solid #000; } -.docs td, th { +.book td, th { padding: 0.1ex 0.5ex; } -.docs .book table tr:hover td { +.book .book table tr:hover td { background-color: #EFEFEF; } /* Link Styles */ -.docs #docNav a { +.book #docNav a { font-weight: bold; } -.docs code.FUNCTION tt { +.book code.FUNCTION tt { font-size: 1em; } -.docs table.docs-compare { +.book table.docs-compare { align: center; width: 90%; border: 2px solid #999; border-collapse: collapse; } -.docs table.docs-compare td { +.book table.docs-compare td { padding: 12px; border: 1px solid #DDD; } -.docs dd { +.book dd { margin-left: 40px; } -.docs .sidebar { +.book .sidebar { padding: 8px; background: #FFF; width: auto; From 8947a120a9531b3e8af6c26056f6c14b5286c7b8 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Fri, 22 Nov 2019 19:00:54 +0300 Subject: [PATCH 1085/2107] DOC: fix minor wording and tagging issues --- doc/pgprobackup.xml | 84 ++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1752d6fc7..83825c9b3 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -519,7 +519,7 @@ pg_probackup init -B backup_dir To add a new backup instance, run the following command: -pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] +pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] where: @@ -556,7 +556,7 @@ pg_probackup add-instance -B backup_dir -D pg_probackup.conf configuration file that controls pg_probackup settings for this backup instance. If you run this command with the - remote_options, the specified parameters + remote_options, the specified parameters will be added to pg_probackup.conf. @@ -688,7 +688,7 @@ COMMIT; Configuring the Remote Mode, Setting up Partial - Restore and + Restore, and Setting up PTRACK Backups. @@ -874,7 +874,7 @@ archive_command = 'install_dir/pg_probackup archive-p To perform standalone backups on standby, complete all steps in section Setting - up STREAM Backups + up STREAM Backups. @@ -882,7 +882,7 @@ archive_command = 'install_dir/pg_probackup archive-p To perform archive backups on standby, complete all steps in section Setting - up continuous WAL archiving + up continuous WAL archiving. @@ -991,17 +991,17 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Install pg_probackup on both systems: - backup_host and - db_host. + backup_host and + db_host. For communication between the hosts set up the passwordless SSH connection between backup user on - backup_host and + backup_host and postgres user on - db_host: + db_host: [backup@backup_host] ssh-copy-id postgres@db_host @@ -1013,8 +1013,8 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; continuous WAL archiving, set up passwordless SSH connection between postgres user on - db_host and backup - user on backup_host: + db_host and backup + user on backup_host: [postgres@db_host] ssh-copy-id backup@backup_host @@ -1027,26 +1027,26 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - backup_host is the system with + backup_host is the system with backup catalog. - db_host is the system with PostgreSQL + db_host is the system with PostgreSQL cluster. backup is the OS user on - backup_host used to run pg_probackup. + backup_host used to run pg_probackup. postgres is the OS user on - db_host used to start the PostgreSQL + db_host used to start the PostgreSQL cluster. Note that for PostgreSQL 11 or higher a more secure approach can be used thanks to allow-group-access @@ -1307,8 +1307,8 @@ pg_probackup backup -B backup_dir --instance - Backup in STREAM mode can be taken from standby of a - server, that generates small amount of WAL traffic, + Backup in STREAM mode can be taken from a standby of a + server that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. @@ -1350,24 +1350,24 @@ pg_probackup backup -B backup_dir --instance - For example, to include /etc/dir1/ and - /etc/dir2/ directories into the full + For example, to include /etc/dir1 and + /etc/dir2 directories into the full backup of your instance_name instance that will be stored under the backup_dir directory on Linux, run: -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 +pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 - For example, to include C:\dir1\ and - C:\dir2\ directories into the full backup + For example, to include C:\dir1 and + C:\dir2 directories into the full backup of your instance_name instance that will be stored under the backup_dir directory on Windows, run: -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 +pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 pg_probackup creates a separate subdirectory in the backup @@ -1513,7 +1513,7 @@ pg_probackup validate -B backup_dir --instance recovery target options, the validate command will check whether it is possible to restore the specified backup to the specified - recovery target. + recovery target. For example, to check that you can restore the database cluster @@ -1582,7 +1582,7 @@ pg_probackup restore -B backup_dir --instance --tablespace-mapping/ option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals - sign (=), escape it with a backslash. This option can be + sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: @@ -1650,7 +1650,7 @@ pg_probackup restore -B backup_dir --instance To exclude one or more databases from restore, use - the option: + the option: pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name @@ -1851,7 +1851,7 @@ pg_probackup restore -B backup_dir --instance To solve this problem, you can use - Remote Wal Archive + Remote WAL Archive Options. @@ -2313,7 +2313,7 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod - parent-backup-id — backup ID of the parent backup. Available only + parent-backup-id — ID of the parent backup. Available only for incremental backups. @@ -2969,7 +2969,7 @@ pg_probackup set-backup -B backup_dir --instance - Suppose you have backed up the node + Suppose you have backed up the node instance in the backup_dir directory and configured continuous WAL @@ -2996,7 +2996,7 @@ BACKUP INSTANCE 'node' flag: -pg_probackup show -B backup_dir --instance node --archive +pg_probackup show -B backup_dir --instance node --archive ARCHIVE INSTANCE 'node' @@ -3010,7 +3010,7 @@ ARCHIVE INSTANCE 'node' achieve much, only one segment is removed: -pg_probackup delete -B backup_dir --instance node --delete-wal +pg_probackup delete -B backup_dir --instance node --delete-wal ARCHIVE INSTANCE 'node' @@ -3025,7 +3025,7 @@ ARCHIVE INSTANCE 'node' option: -pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 +pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 ARCHIVE INSTANCE 'node' @@ -3039,7 +3039,7 @@ ARCHIVE INSTANCE 'node' option with the command: -pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal +pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal ARCHIVE INSTANCE 'node' @@ -3235,7 +3235,7 @@ pg_probackup set-config -B backup_dir --instance dbname] [-h host] [-p port] [-U username] [--archive-timeout=timeout] [--external-dirs=external_directory_path] [--restore-command=cmdline] -[remote_options] [remote_archive_options] [logging_options] +[remote_options] [remote_wal_archive_options] [logging_options] Adds the specified connection, compression, retention, logging, @@ -3277,7 +3277,7 @@ pg_probackup show-config -B backup_dir --instance backup_dir/backups/instance_name directory. You can specify the - --format=json option to return the result + --format=json option to get the result in the JSON format. By default, configuration settings are shown as plain text. @@ -3303,7 +3303,7 @@ pg_probackup show -B backup_dir By default, the contents of the backup catalog is shown as plain text. You can specify the - --format=json option to return the result + --format=json option to get the result in the JSON format. @@ -3318,7 +3318,7 @@ pg_probackup show -B backup_dir backup pg_probackup backup -B backup_dir -b backup_mode --instance instance_name -[--help] [-j num_threads] [--progress] +[--help] [-j num_threads] [--progress] [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] @@ -3498,7 +3498,7 @@ pg_probackup backup -B backup_dir -b bac pg_probackup restore -B backup_dir --instance instance_name [--help] [-D data_dir] [-i backup_id] -[-j num_threads] [--progress] +[-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] [--restore-command=cmdline] @@ -3513,7 +3513,7 @@ pg_probackup restore -B backup_dir --instance pg_probackup uses the most recent backup - to preform the recovery. + to perform the recovery. @@ -3537,7 +3537,7 @@ pg_probackup restore -B backup_dir --instance Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must - be absolute paths. If the path contains the equals sign (=), + be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. @@ -3551,7 +3551,7 @@ pg_probackup restore -B backup_dir --instance OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path - contains the equals sign (=), escape it with a backslash. This + contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. @@ -3815,7 +3815,7 @@ pg_probackup archive-push -B backup_dir --instance archive_command PostgreSQL parameter to set up continous - WAl archiving. + WAL archiving. For details, see sections From b3be13945673e7d6f9b03aa64ce422d6e29132ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 Nov 2019 12:09:34 +0300 Subject: [PATCH 1086/2107] DOC: css for code blocks --- doc/stylesheet.css | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/stylesheet.css b/doc/stylesheet.css index fe2c45df1..8178ecf7e 100644 --- a/doc/stylesheet.css +++ b/doc/stylesheet.css @@ -400,3 +400,10 @@ table.CAUTION, background: #FFF; width: auto; } + +.book pre { + background: #f5f5f5; + padding: 10px; + border: 1px solid #ccc; + border-radius: 4px; +} From 10c1f7ad3fdf502e497c58d445401c935e2c4750 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Mon, 25 Nov 2019 19:24:29 +0300 Subject: [PATCH 1087/2107] DOC: improve checkb docs and fix some other minor issues --- doc/pgprobackup.xml | 97 ++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 83825c9b3..3a4252b33 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -189,10 +189,12 @@ doc/src/sgml/pgprobackup.sgml - Incremental backup: page-level incremental backup allows you - to save disk space, speed up backup and restore. With three - different incremental modes, you can plan the backup strategy - in accordance with your data flow. + Incremental backup: with three different incremental modes, + you can plan the backup strategy in accordance with your data flow. + Incremental backups allow you to save disk space + and speed up backup as compared to taking full backups. + It is also faster to restore the cluster by applying incremental + backups than by replaying WAL files. @@ -296,9 +298,11 @@ doc/src/sgml/pgprobackup.sgml - Incremental backups only store the data that has changed since - the previous backup. It allows to decrease the backup size and - speed up backup and restore operations. pg_probackup supports + Incremental backups operate at the page level, only storing the data that has changed since + the previous backup. It allows you to save disk space + and speed up the backup process as compared to taking full backups. + It is also faster to restore the cluster by applying incremental + backups than by replaying WAL files. pg_probackup supports the following modes of incremental backups: @@ -306,7 +310,7 @@ doc/src/sgml/pgprobackup.sgml DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages - that have changed since the previous backup. Note that this + that have changed since the previous backup. This mode can impose read-only I/O pressure equal to a full backup. @@ -1047,7 +1051,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; postgres is the OS user on db_host used to start the PostgreSQL - cluster. Note that for PostgreSQL 11 or higher a + cluster. For PostgreSQL 11 or higher a more secure approach can be used thanks to allow-group-access feature. @@ -1390,52 +1394,46 @@ pg_probackup backup -B backup_dir --instance Performing Cluster Verification - To verify that PostgreSQL database cluster is free of - corruption, run the following command: + To verify that PostgreSQL database cluster is + not corrupt, run the following command: -pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] +pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] [connection_options] + - This command performs physical verification works similar to - page validation that is - done during backup with several differences: + This command performs physical verification of all data files + located in the specified data directory by running page header + sanity checks, as well as block-level checksum verification if checksums are enabled. + If a corrupt page is detected, checkdb + continues cluster verification until all pages in the cluster + are validated. - - - - checkdb is read-only. - - - - - Even if a corrupt page is detected, checkdb - continues cluster verification until all pages in the cluster - are validated. - - - - - checkdb does not strictly require - the backup catalog, so it can be used - to verify database clusters that are - not - added to the - backup catalog. - - - + - If backup_dir and - instance_name are omitted, + By default, similar page validation + is performed automatically while a backup is taken by + pg_probackup. The checkdb + command enables you to perform such page validation + on demand, without taking any backup copies, even if the cluster + is not backed up using pg_probackup at all. + + + + To perform cluster verification, pg_probackup + needs to connect to the cluster to be verified. In general, it is + enough to specify the backup instance of this cluster for + pg_probackup to determine the required + connection options. However, if -B and + --instance options are ommitted, you have to provide connection options and - data_dir must be provided via environment + data_dir via environment variables or command-line options. + Physical verification cannot detect logical inconsistencies, - missing and nullified blocks or entire files, repercussions from - PostgreSQL bugs and other wicked anomalies. Extensions + missing or nullified blocks and entire files, or similar anomalies. Extensions amcheck and amcheck_next @@ -1448,7 +1446,7 @@ pg_probackup checkdb [-B backup_dir [--instance command: -pg_probackup checkdb -D data_dir --amcheck +pg_probackup checkdb -D data_dir --amcheck [connection_options] You can skip physical verification by specifying the @@ -1929,8 +1927,9 @@ pg_probackup backup -B backup_dir --instance For example, and commands use a regular - PostgreSQL connection. To avoid specifying these options each - time on the command line, you can set them in the + PostgreSQL connection. To avoid specifying + connection options + each time on the command line, you can set them in the pg_probackup.conf configuration file using the command. @@ -2882,12 +2881,10 @@ BACKUP INSTANCE 'node' node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK - The Time field for the merged backup displays the time required for the merge. - Backup Pinning @@ -3122,7 +3119,7 @@ pg_probackup delete -B backup_dir --instance backup_dir --instance instance_name --delete-expired - Note that expired backups cannot be removed while at least one + Expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the From 6e5fe570b629650424d413d7b985ade9fa1b170e Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Tue, 26 Nov 2019 15:30:43 +0300 Subject: [PATCH 1088/2107] DOC: fix typo in continuous --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 3a4252b33..16d3a9add 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3811,7 +3811,7 @@ pg_probackup archive-push -B backup_dir --instance archive-push in the archive_command PostgreSQL parameter to set up - continous + continuous WAL archiving. From 3188aaf6b0cc1f2b99942d45864a1bb673e7b072 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Tue, 26 Nov 2019 16:44:32 +0300 Subject: [PATCH 1089/2107] DOC: fix wal_seg_size name --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 16d3a9add..138bb5e87 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2441,7 +2441,7 @@ ARCHIVE INSTANCE 'node' Zratio — compression ratio calculated as N segments * - wal_seg_size / Size. + wal_segment_size * wal_block_size / Size. From 715dbf8506090cdc2536f9e6d55d4908aa348651 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Nov 2019 23:11:32 +0300 Subject: [PATCH 1090/2107] DOC: minor fix --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 138bb5e87..da2c84cd6 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1660,7 +1660,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 +pg_probackup restore -B backup_dir --instance instance_name --db-exclude=db1 --db-exclude=db2 Partial restore relies on lax behavior of PostgreSQL recovery From 69f98672ad83288560c86ecef59e4a61c77cdfb5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Nov 2019 23:16:35 +0300 Subject: [PATCH 1091/2107] DOC: update fonts --- doc/stylesheet.css | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/stylesheet.css b/doc/stylesheet.css index 8178ecf7e..4d84058f5 100644 --- a/doc/stylesheet.css +++ b/doc/stylesheet.css @@ -1,3 +1,14 @@ +@import url('https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&subset=cyrillic'); + +body { + font-family: 'Roboto',Arial,sans-serif; +} + +body { + font-size: 18px; + font-weight: 300; +} + /* ../media/css/docs.css */ .navheader th { text-align: center; } /* anti-bootstrap */ From edd3df7f003e5cc624251cd403c61d7dd47a67ad Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Tue, 10 Dec 2019 16:42:02 +0300 Subject: [PATCH 1092/2107] DOC: modify restore-as-replica option description for v12 --- doc/pgprobackup.xml | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index da2c84cd6..0467dd717 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -233,8 +233,10 @@ doc/src/sgml/pgprobackup.sgml - Deduplication: saving disk space by not copying unchanged non-data - files, such as _vm or _fsm. + Deduplication: saving disk space by excluding non-data + files (such as _vm or _fsm) + from incremental backups if these files have not changed since + they were copied into one of the previous backups in this incremental chain. @@ -1364,20 +1366,18 @@ pg_probackup backup -B backup_dir --instance backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 - For example, to include C:\dir1 and + Similarly, to include C:\dir1 and C:\dir2 directories into the full backup - of your instance_name instance that will - be stored under the backup_dir directory on Windows, run: pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 - pg_probackup creates a separate subdirectory in the backup - directory for each external directory. Since external - directories included into different backups do not have to be - the same, when you are restoring the cluster from an + pg_probackup recursively copies the contents + of each external directory into a separate subdirectory in the backup + catalog. Since external directories included into different backups + do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. @@ -1865,8 +1865,7 @@ pg_probackup restore -B backup_dir --instance backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup - Provided arguments will be used to construct the restore_command - in recovery.conf: + Provided arguments will be used to construct the restore_command: restore_command = 'install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' @@ -3432,7 +3431,8 @@ pg_probackup backup -B backup_dir -b bac - Includes the specified directory into the backup. This option + Includes the specified directory into the backup by recursively + copying its contents into a separate subdirectory in the backup catalog. This option is useful to back up scripts, SQL dump files, and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths @@ -3519,10 +3519,15 @@ pg_probackup restore -B backup_dir --instance - Writes a minimal recovery.conf in the output directory to - facilitate setting up a standby server. The password is not - included. If the replication connection requires a password, - you must specify the password manually. + Creates a minimal recovery configuration file to facilitate setting up a + standby server. If the replication connection requires a password, + you must specify the password manually as it is not included. + For PostgreSQL 11 or lower, + recovery settings are written into the recovery.conf + file. Starting from PostgreSQL 12, + pg_probackup writes these settings into + the probackup_recovery.conf file in the data + directory. @@ -3831,7 +3836,7 @@ pg_probackup archive-get -B backup_dir --instance pg_probackup as part of the - restore_command in recovery.conf when + restore_command when restoring backups using a WAL archive. You do not need to set it manually. From 27f7d8bedfcca5e3e9660d72b76e608b6da70034 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 12 Dec 2019 18:19:15 +0300 Subject: [PATCH 1093/2107] Readme: redirect to new documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 600521e06..cce4a9343 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Documentation -Documentation can be found at [github](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md). +Documentation can be found at [github](https://postgrespro.github.io/pg_probackup). ## Installation and Setup ### Windows Installation From 29c457492d9fb385fd4ee3b0b03ae64d5deee7b6 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Wed, 18 Dec 2019 16:52:21 +0300 Subject: [PATCH 1094/2107] DOC: document new PTRACK requirements --- doc/pgprobackup.xml | 78 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 0467dd717..35aaaafab 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -338,8 +338,7 @@ doc/src/sgml/pgprobackup.sgml changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this - relation. As one page requires just one bit in the PTRACK - fork, such bitmaps are quite small. Tracking implies some + relation. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. @@ -1150,9 +1149,69 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; If you are going to use PTRACK backups, complete the following - additional steps: + additional steps. The role that will perform PTRACK backups + (the backup role in the examples below) must have + access to all the databases of the cluster. - + + For PostgreSQL 12 or higher: + + + + + Create PTRACK extension: + +CREATE EXTENSION ptrack; + + + + + + To enable tracking page updates, set ptrack_map_size + parameter to a positive integer and restart the server. + + + For optimal performance, it is recommended to set + ptrack_map_size to + N / 1024, where + N is the size of the + PostgreSQL cluster, in MB. If you set this + parameter to a lower value, PTRACK is more likely to map several blocks + together, which leads to false-positive results when tracking changed + blocks and increases the incremental backup size as unchanged blocks + can also be copied into the incremental backup. + Setting ptrack_map_size to a higher value + does not affect PTRACK operation. The maximum allowed value is 1024. + + + + + Grant the right to execute PTRACK + functions to the backup role + in the database used to connect to the cluster: + + +GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_pagemapset(pg_lsn) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_control_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; + + + + + + + If you change the ptrack_map_size parameter value, + the previously created PTRACK map file is cleared, + and tracking newly changed blocks starts from scratch. Thus, you have + to retake a full backup before taking incremental PTRACK backups after + changing ptrack_map_size. + + + + + For older PostgreSQL versions: + + Set the ptrack_enable parameter to @@ -1161,7 +1220,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - Grant the rights to execute PTRACK + Grant the right to execute PTRACK functions to the backup role in every database of the cluster: @@ -1171,13 +1230,8 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; - - - The backup role must have access to all - the databases of the cluster. - - - + + From d507c8ab0d68cdf4e2bb0f891bb83dae6565c83a Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Thu, 19 Dec 2019 13:14:13 +0300 Subject: [PATCH 1095/2107] DOC: fix PTRACK description --- doc/pgprobackup.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 35aaaafab..68037e2ec 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -337,8 +337,7 @@ doc/src/sgml/pgprobackup.sgml PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, - this page is marked in a special PTRACK bitmap for this - relation. Tracking implies some + this page is marked in a special PTRACK bitmap. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. From 64d35c5022e7a232ea4583fe5c10e118126f4883 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 23 Dec 2019 18:23:29 +0300 Subject: [PATCH 1096/2107] Added support for ptrack 2.0 --- Makefile | 2 +- README.md | 9 +- src/backup.c | 585 +-------- src/catalog.c | 4 + src/data.c | 47 +- src/dir.c | 19 +- src/merge.c | 3 +- src/parsexlog.c | 12 +- src/pg_probackup.h | 62 +- src/ptrack.c | 702 +++++++++++ tests/Readme.md | 2 +- tests/archive.py | 6 + tests/backup.py | 183 ++- tests/compatibility.py | 7 +- tests/delete.py | 9 +- tests/delta.py | 36 +- tests/external.py | 2 +- tests/false_positive.py | 24 +- tests/helpers/ptrack_helpers.py | 55 +- tests/merge.py | 24 +- tests/option.py | 7 - tests/pgpro2068.py | 4 +- tests/ptrack.py | 1954 ++++++++++++++++++------------- tests/replica.py | 27 +- tests/restore.py | 92 +- 25 files changed, 2370 insertions(+), 1507 deletions(-) create mode 100644 src/ptrack.c diff --git a/Makefile b/Makefile index 81391ad64..37cceb56e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ - src/parsexlog.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ + src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ src/validate.o # borrowed files diff --git a/README.md b/README.md index cce4a9343..e88e52118 100644 --- a/README.md +++ b/README.md @@ -155,11 +155,18 @@ Once you have `pg_probackup` installed, complete [the setup](https://github.com/ ## Building from source ### Linux -To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory: +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. Execute this in the module's directory: ```shell make USE_PGXS=1 PG_CONFIG= top_srcdir= ``` + +The alternative way, without using the PGXS infrastructure, is to place `pg_probackup` source directory into `contrib` directory and build it there. Example: + +```shell +cd && git clone https://github.com/postgrespro/pg_probackup contrib/pg_probackup && cd contrib/pg_probackup && make +``` + ### Windows Currently pg_probackup can be build using only MSVC 2013. diff --git a/src/backup.c b/src/backup.c index abd10a976..648a97dfd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -25,14 +25,6 @@ #include "utils/thread.h" #include "utils/file.h" - -/* - * Macro needed to parse ptrack. - * NOTE Keep those values synchronized with definitions in ptrack.h - */ -#define PTRACK_BITS_PER_HEAPBLOCK 1 -#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) - static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; @@ -74,8 +66,6 @@ typedef struct static pthread_t stream_thread; static StreamThreadArg stream_thread_arg = {"", NULL, 1}; -static int is_ptrack_enable = false; -bool is_ptrack_support = false; bool exclusive_backup = false; /* Is pg_start_backup() was executed */ @@ -102,7 +92,6 @@ static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir); -static void make_pagemap_from_ptrack(parray* files, PGconn* backup_conn); static void *StreamLog(void *arg); static void IdentifySystem(StreamThreadArg *stream_thread_arg); @@ -113,19 +102,6 @@ static parray *get_database_map(PGconn *pg_startbackup_conn); /* pgpro specific functions */ static bool pgpro_support(PGconn *conn); -/* Ptrack functions */ -static void pg_ptrack_clear(PGconn *backup_conn); -static bool pg_ptrack_support(PGconn *backup_conn); -static bool pg_ptrack_enable(PGconn *backup_conn); -static bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, - PGconn *backup_conn); -static char *pg_ptrack_get_and_clear(Oid tablespace_oid, - Oid db_oid, - Oid rel_oid, - size_t *result_size, - PGconn *backup_conn); -static XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn); - /* Check functions */ static bool pg_checksum_enable(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); @@ -177,6 +153,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) PGconn *master_conn = NULL; PGconn *pg_startbackup_conn = NULL; + /* for fancy reporting */ + time_t start_time, end_time; + elog(LOG, "Database backup start"); if(current.external_dir_str) { @@ -229,7 +208,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn); + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) { @@ -242,8 +221,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) } /* Clear ptrack files for FULL and PAGE backup */ - if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && is_ptrack_enable) - pg_ptrack_clear(backup_conn); + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable) + pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); /* notify start of backup to PostgreSQL server */ time2iso(label, lengthof(label), current.start_time); @@ -379,22 +358,41 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* * Build page mapping in incremental mode. */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) - { - /* - * Build the page map. Obtain information about changed pages - * reading WAL segments present in archives up to the point - * where this backup has started. - */ - extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, - prev_backup->start_lsn, current.start_lsn); - } - else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || + current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - /* - * Build the page map from ptrack information. - */ - make_pagemap_from_ptrack(backup_files_list, backup_conn); + elog(INFO, "Compiling pagemap of changed blocks"); + time(&start_time); + + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) + { + /* + * Build the page map. Obtain information about changed pages + * reading WAL segments present in archives up to the point + * where this backup has started. + */ + extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, + prev_backup->start_lsn, current.start_lsn); + } + else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + /* + * Build the page map from ptrack information. + */ + if (nodeInfo->ptrack_version_num == 20) + make_pagemap_from_ptrack_2(backup_files_list, backup_conn, + nodeInfo->ptrack_schema, + prev_backup_start_lsn); + else if (nodeInfo->ptrack_version_num == 15 || + nodeInfo->ptrack_version_num == 16 || + nodeInfo->ptrack_version_num == 17) + make_pagemap_from_ptrack_1(backup_files_list, backup_conn); + } + + time(&end_time); + elog(INFO, "Pagemap compiled, time elapsed %.0f sec", + difftime(end_time, start_time)); } /* @@ -454,6 +452,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) { backup_files_arg *arg = &(threads_args[i]); + arg->nodeInfo = nodeInfo; arg->from_root = instance_config.pgdata; arg->to_root = database_path; arg->external_prefix = external_prefix; @@ -769,19 +768,24 @@ do_backup(time_t start_time, bool no_validate, elog(ERROR, "Failed to retrieve wal_segment_size"); #endif - is_ptrack_support = pg_ptrack_support(backup_conn); - if (is_ptrack_support) + get_ptrack_version(backup_conn, &nodeInfo); +// elog(WARNING, "ptrack_version_num %d", ptrack_version_num); + + if (nodeInfo.ptrack_version_num > 0) { - is_ptrack_enable = pg_ptrack_enable(backup_conn); + if (nodeInfo.ptrack_version_num >= 20) + nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn); + else + nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn); } if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - if (!is_ptrack_support) + if (nodeInfo.ptrack_version_num == 0) elog(ERROR, "This PostgreSQL instance does not support ptrack"); else { - if(!is_ptrack_enable) + if (!nodeInfo.is_ptrack_enable) elog(ERROR, "Ptrack is disabled"); } } @@ -926,8 +930,7 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) PQclear(res); /* Do exclusive backup only for PostgreSQL 9.5 */ - exclusive_backup = nodeInfo->server_version < 90600 || - current.backup_mode == BACKUP_MODE_DIFF_PTRACK; + exclusive_backup = nodeInfo->server_version < 90600; } /* @@ -1084,48 +1087,6 @@ pg_switch_wal(PGconn *conn) PQclear(res); } -/* - * Check if the instance supports ptrack - * TODO Maybe we should rather check ptrack_version()? - */ -static bool -pg_ptrack_support(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, - "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", - 0, NULL); - if (PQntuples(res_db) == 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - - res_db = pgut_execute(backup_conn, - "SELECT pg_catalog.ptrack_version()", - 0, NULL); - if (PQntuples(res_db) == 0) - { - PQclear(res_db); - return false; - } - - /* Now we support only ptrack versions upper than 1.5 */ - if (strcmp(PQgetvalue(res_db, 0, 0), "1.5") != 0 && - strcmp(PQgetvalue(res_db, 0, 0), "1.6") != 0 && - strcmp(PQgetvalue(res_db, 0, 0), "1.7") != 0) - { - elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", PQgetvalue(res_db, 0, 0)); - PQclear(res_db); - return false; - } - - PQclear(res_db); - return true; -} - /* * Check if the instance is PostgresPro fork. */ @@ -1210,23 +1171,6 @@ get_database_map(PGconn *conn) return database_map; } -/* Check if ptrack is enabled in target instance */ -static bool -pg_ptrack_enable(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - return true; -} - /* Check if ptrack is enabled in target instance */ static bool pg_checksum_enable(PGconn *conn) @@ -1279,204 +1223,6 @@ pg_is_superuser(PGconn *conn) return false; } -/* Clear ptrack files in all databases of the instance we connected to */ -static void -pg_ptrack_clear(PGconn *backup_conn) -{ - PGresult *res_db, - *res; - const char *dbname; - int i; - Oid dbOid, tblspcOid; - char *params[2]; - - params[0] = palloc(64); - params[1] = palloc(64); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", - 0, NULL); - - for(i = 0; i < PQntuples(res_db); i++) - { - PGconn *tmp_conn; - - dbname = PQgetvalue(res_db, i, 0); - if (strcmp(dbname, "template0") == 0) - continue; - - dbOid = atoi(PQgetvalue(res_db, i, 1)); - tblspcOid = atoi(PQgetvalue(res_db, i, 2)); - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", - 0, NULL); - PQclear(res); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - PQclear(res); - - pgut_disconnect(tmp_conn); - } - - pfree(params[0]); - pfree(params[1]); - PQclear(res_db); -} - -static bool -pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) -{ - char *params[2]; - char *dbname; - PGresult *res_db; - PGresult *res; - bool result; - - params[0] = palloc(64); - params[1] = palloc(64); - - sprintf(params[0], "%i", dbOid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return false; - - dbname = PQgetvalue(res_db, 0, 0); - - /* Always backup all files from template0 database */ - if (strcmp(dbname, "template0") == 0) - { - PQclear(res_db); - return true; - } - PQclear(res_db); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); - - if (!parse_bool(PQgetvalue(res, 0, 0), &result)) - elog(ERROR, - "result of pg_ptrack_get_and_clear_db() is invalid: %s", - PQgetvalue(res, 0, 0)); - - PQclear(res); - pfree(params[0]); - pfree(params[1]); - - return result; -} - -/* Read and clear ptrack files of the target relation. - * Result is a bytea ptrack map of all segments of the target relation. - * case 1: we know a tablespace_oid, db_oid, and rel_filenode - * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) - * case 3: we know only rel_filenode (because file in pg_global) - */ -static char * -pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, - size_t *result_size, PGconn *backup_conn) -{ - PGconn *tmp_conn; - PGresult *res_db, - *res; - char *params[2]; - char *result; - char *val; - - params[0] = palloc(64); - params[1] = palloc(64); - - /* regular file (not in directory 'global') */ - if (db_oid != 0) - { - char *dbname; - - sprintf(params[0], "%i", db_oid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return NULL; - - dbname = PQgetvalue(res_db, 0, 0); - - if (strcmp(dbname, "template0") == 0) - { - PQclear(res_db); - return NULL; - } - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", - dbname, tablespace_oid, rel_filenode); - PQclear(res_db); - pgut_disconnect(tmp_conn); - } - /* file in directory 'global' */ - else - { - /* - * execute ptrack_get_and_clear for relation in pg_global - * Use backup_conn, cause we can do it from any database. - */ - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", - rel_filenode); - } - - val = PQgetvalue(res, 0, 0); - - /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. - * It should be fixed in future ptrack releases, but till then we - * can parse it. - */ - if (strcmp("x", val+1) == 0) - { - /* Ptrack file is missing */ - return NULL; - } - - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - PQclear(res); - pfree(params[0]); - pfree(params[1]); - - return result; -} - /* * Wait for target LSN or WAL segment, containing target LSN. * @@ -2212,7 +1958,7 @@ backup_files(void *arg) if (!pg_atomic_test_set_flag(&file->lock)) continue; - elog(VERBOSE, "Copying file: \"%s\" ", file->path); + elog(VERBOSE, "Copying file: \"%s\"", file->path); /* check for interrupt */ if (interrupted || thread_interrupted) @@ -2289,6 +2035,9 @@ backup_files(void *arg) current.backup_mode, instance_config.compress_alg, instance_config.compress_level, + arguments->nodeInfo->checksum_version, + arguments->nodeInfo->ptrack_version_num, + arguments->nodeInfo->ptrack_schema, true)) { /* disappeared file not to be confused with 'not changed' */ @@ -2549,132 +2298,6 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pg_free(path); } -/* - * Given a list of files in the instance to backup, build a pagemap for each - * data file that has ptrack. Result is saved in the pagemap field of pgFile. - * NOTE we rely on the fact that provided parray is sorted by file->path. - */ -static void -make_pagemap_from_ptrack(parray *files, PGconn *backup_conn) -{ - size_t i; - Oid dbOid_with_ptrack_init = 0; - Oid tblspcOid_with_ptrack_init = 0; - char *ptrack_nonparsed = NULL; - size_t ptrack_nonparsed_size = 0; - - elog(LOG, "Compiling pagemap"); - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - size_t start_addr; - - /* - * If there is a ptrack_init file in the database, - * we must backup all its files, ignoring ptrack files for relations. - */ - if (file->is_database) - { - char *filename = strrchr(file->path, '/'); - - Assert(filename != NULL); - filename++; - - /* - * The function pg_ptrack_get_and_clear_db returns true - * if there was a ptrack_init file. - * Also ignore ptrack files for global tablespace, - * to avoid any possible specific errors. - */ - if ((file->tblspcOid == GLOBALTABLESPACE_OID) || - pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid, backup_conn)) - { - dbOid_with_ptrack_init = file->dbOid; - tblspcOid_with_ptrack_init = file->tblspcOid; - } - } - - if (file->is_datafile) - { - if (file->tblspcOid == tblspcOid_with_ptrack_init && - file->dbOid == dbOid_with_ptrack_init) - { - /* ignore ptrack if ptrack_init exists */ - elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); - file->pagemap_isabsent = true; - continue; - } - - /* get ptrack bitmap once for all segments of the file */ - if (file->segno == 0) - { - /* release previous value */ - pg_free(ptrack_nonparsed); - ptrack_nonparsed_size = 0; - - ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, - file->relOid, &ptrack_nonparsed_size, backup_conn); - } - - if (ptrack_nonparsed != NULL) - { - /* - * pg_ptrack_get_and_clear() returns ptrack with VARHDR cut out. - * Compute the beginning of the ptrack map related to this segment - * - * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 - * RELSEG_SIZE. Number of Pages per segment: 131072 - * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed - * to keep track on one relsegment: 16384 - */ - start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; - - /* - * If file segment was created after we have read ptrack, - * we won't have a bitmap for this segment. - */ - if (start_addr > ptrack_nonparsed_size) - { - elog(VERBOSE, "Ptrack is missing for file: %s", file->path); - file->pagemap_isabsent = true; - } - else - { - - if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) - { - file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - else - { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - - file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); - memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); - } - } - else - { - /* - * If ptrack file is missing, try to copy the entire file. - * It can happen in two cases: - * - files were created by commands that bypass buffer manager - * and, correspondingly, ptrack mechanism. - * i.e. CREATE DATABASE - * - target relation was deleted. - */ - elog(VERBOSE, "Ptrack is missing for file: %s", file->path); - file->pagemap_isabsent = true; - } - } - } - elog(LOG, "Pagemap compiled"); -} - - /* * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is * set by pg_stop_backup(). @@ -2687,7 +2310,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during backup stop_streaming"); + elog(ERROR, "Interrupted during WAL streaming"); /* we assume that we get called once at the end of each segment */ if (segment_finished) @@ -2825,100 +2448,6 @@ StreamLog(void *arg) return NULL; } -/* - * Get lsn of the moment when ptrack was enabled the last time. - */ -static XLogRecPtr -get_last_ptrack_lsn(PGconn *backup_conn) - -{ - PGresult *res; - uint32 lsn_hi; - uint32 lsn_lo; - XLogRecPtr lsn; - - res = pgut_execute(backup_conn, "select pg_catalog.pg_ptrack_control_lsn()", - 0, NULL); - - /* Extract timeline and LSN from results of pg_start_backup() */ - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - - PQclear(res); - return lsn; -} - -char * -pg_ptrack_get_block(ConnectionArgs *arguments, - Oid dbOid, - Oid tblsOid, - Oid relOid, - BlockNumber blknum, - size_t *result_size) -{ - PGresult *res; - char *params[4]; - char *result; - - params[0] = palloc(64); - params[1] = palloc(64); - params[2] = palloc(64); - params[3] = palloc(64); - - /* - * Use tmp_conn, since we may work in parallel threads. - * We can connect to any database. - */ - sprintf(params[0], "%i", tblsOid); - sprintf(params[1], "%i", dbOid); - sprintf(params[2], "%i", relOid); - sprintf(params[3], "%u", blknum); - - if (arguments->conn == NULL) - { - arguments->conn = pgut_connect(instance_config.conn_opt.pghost, - instance_config.conn_opt.pgport, - instance_config.conn_opt.pgdatabase, - instance_config.conn_opt.pguser); - } - - if (arguments->cancel_conn == NULL) - arguments->cancel_conn = PQgetCancel(arguments->conn); - - //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - res = pgut_execute_parallel(arguments->conn, - arguments->cancel_conn, - "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", - 4, (const char **)params, true, false, false); - - if (PQnfields(res) != 1) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - if (PQgetisnull(res, 0, 0)) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - - PQclear(res); - - pfree(params[0]); - pfree(params[1]); - pfree(params[2]); - pfree(params[3]); - - return result; -} - static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) { diff --git a/src/catalog.c b/src/catalog.c index 669efd9a3..db88ab172 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1940,6 +1940,10 @@ pgNodeInit(PGNodeInfo *node) node->server_version = 0; node->server_version_str[0] = '\0'; + + node->ptrack_version_num = 0; + node->is_ptrack_enable = false; + node->ptrack_schema = NULL; } /* diff --git a/src/data.c b/src/data.c index bbdb570ba..de5c58079 100644 --- a/src/data.c +++ b/src/data.c @@ -300,9 +300,10 @@ prepare_page(ConnectionArgs *arguments, BlockNumber blknum, BlockNumber nblocks, FILE *in, BlockNumber *n_skipped, BackupMode backup_mode, - Page page, - bool strict, - uint32 checksum_version) + Page page, bool strict, + uint32 checksum_version, + int ptrack_version_num, + const char *ptrack_schema) { XLogRecPtr page_lsn = 0; int try_again = 100; @@ -319,7 +320,7 @@ prepare_page(ConnectionArgs *arguments, * Under high write load it's possible that we've read partly * flushed page, so try several times before throwing an error. */ - if (backup_mode != BACKUP_MODE_DIFF_PTRACK) + if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 20) { while(!page_is_valid && try_again) { @@ -345,9 +346,9 @@ prepare_page(ConnectionArgs *arguments, */ //elog(WARNING, "Checksum_Version: %i", checksum_version ? 1 : 0); - if (result == -1 && is_ptrack_support && strict) + if (result == -1 && strict && ptrack_version_num > 0) { - elog(WARNING, "File \"%s\", block %u, try to fetch via SQL", + elog(WARNING, "File \"%s\", block %u, try to fetch via shared buffer", file->path, blknum); break; } @@ -358,16 +359,16 @@ prepare_page(ConnectionArgs *arguments, */ if (!page_is_valid && - ((strict && !is_ptrack_support) || !strict)) + ((strict && ptrack_version_num == 0) || !strict)) { - /* show this message for checkdb or backup without ptrack support */ + /* show this message for checkdb, merge or backup without ptrack support */ elog(WARNING, "Corruption detected in file \"%s\", block %u", file->path, blknum); } /* Backup with invalid block and without ptrack support must throw error */ - if (!page_is_valid && strict && !is_ptrack_support) - elog(ERROR, "Data file corruption. Canceling backup"); + if (!page_is_valid && strict && ptrack_version_num == 0) + elog(ERROR, "Data file corruption, canceling backup"); /* Checkdb not going futher */ if (!strict) @@ -379,12 +380,15 @@ prepare_page(ConnectionArgs *arguments, } } - if (backup_mode == BACKUP_MODE_DIFF_PTRACK || (!page_is_valid && is_ptrack_support)) + if ((backup_mode == BACKUP_MODE_DIFF_PTRACK + && (ptrack_version_num >= 15 && ptrack_version_num < 20)) + || !page_is_valid) { size_t page_size = 0; Page ptrack_page = NULL; ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, - file->relOid, absolute_blknum, &page_size); + file->relOid, absolute_blknum, &page_size, + ptrack_version_num, ptrack_schema); if (ptrack_page == NULL) { @@ -530,7 +534,8 @@ bool backup_data_file(backup_files_arg* arguments, const char *to_path, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, bool missing_ok) + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, bool missing_ok) { FILE *in; FILE *out; @@ -621,12 +626,15 @@ backup_data_file(backup_files_arg* arguments, if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev) { + /* TODO: take into account PTRACK 2.0 */ if (backup_mode != BACKUP_MODE_DIFF_PTRACK && fio_is_remote_file(in)) { int rc = fio_send_pages(in, out, file, backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, &n_blocks_skipped, calg, clevel); - if (rc == PAGE_CHECKSUM_MISMATCH && is_ptrack_support) + + if (rc == PAGE_CHECKSUM_MISMATCH && ptrack_version_num >= 15) + /* only ptrack versions 1.5, 1.6, 1.7 and 2.x support this functionality */ goto RetryUsingPtrack; if (rc < 0) elog(ERROR, "Failed to read file \"%s\": %s", @@ -642,7 +650,9 @@ backup_data_file(backup_files_arg* arguments, { page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true, current.checksum_version); + backup_mode, curr_page, true, + checksum_version, ptrack_version_num, + ptrack_schema); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -666,7 +676,9 @@ backup_data_file(backup_files_arg* arguments, { page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true, current.checksum_version); + backup_mode, curr_page, true, + checksum_version, ptrack_version_num, + ptrack_schema); compress_and_backup_page(file, blknum, in, out, &(file->crc), page_state, curr_page, calg, clevel); n_blocks_read++; @@ -1268,7 +1280,8 @@ check_data_file(ConnectionArgs *arguments, { page_state = prepare_page(arguments, file, InvalidXLogRecPtr, blknum, nblocks, in, &n_blocks_skipped, - BACKUP_MODE_FULL, curr_page, false, checksum_version); + BACKUP_MODE_FULL, curr_page, false, checksum_version, + 0, NULL); if (page_state == PageIsTruncated) break; diff --git a/src/dir.c b/src/dir.c index bcf6293e9..754e6bde4 100644 --- a/src/dir.c +++ b/src/dir.c @@ -343,6 +343,16 @@ pgFileComparePath(const void *f1, const void *f2) return strcmp(f1p->path, f2p->path); } +/* Compare two pgFile with their path in ascending order of ASCII code. */ +int +pgFileMapComparePath(const void *f1, const void *f2) +{ + page_map_entry *f1p = *(page_map_entry **)f1; + page_map_entry *f2p = *(page_map_entry **)f2; + + return strcmp(f1p->path, f2p->path); +} + /* Compare two pgFile with their name in ascending order of ASCII code. */ int pgFileCompareName(const void *f1, const void *f2) @@ -677,6 +687,13 @@ dir_check_file(pgFile *file) { if (strcmp(file->name, "pg_internal.init") == 0) return CHECK_FALSE; + /* Do not backup ptrack2.x map files */ + else if (strcmp(file->name, "ptrack.map") == 0) + return CHECK_FALSE; + else if (strcmp(file->name, "ptrack.map.mmap") == 0) + return CHECK_FALSE; + else if (strcmp(file->name, "ptrack.map.tmp") == 0) + return CHECK_FALSE; /* Do not backup temp files */ else if (file->name[0] == 't' && isdigit(file->name[1])) return CHECK_FALSE; @@ -734,7 +751,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, bool follow_symlink, int external_dir_num, fio_location location) { - DIR *dir; + DIR *dir; struct dirent *dent; if (!S_ISDIR(parent->mode)) diff --git a/src/merge.c b/src/merge.c index acbdce032..4e871ddd5 100644 --- a/src/merge.c +++ b/src/merge.c @@ -617,7 +617,8 @@ merge_files(void *arg) to_backup->backup_mode, to_backup->compress_alg, to_backup->compress_level, - false); + from_backup->checksum_version, + 0, NULL, false); file->path = prev_path; diff --git a/src/parsexlog.c b/src/parsexlog.c index a383dc4b8..7c465db85 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -236,22 +236,12 @@ extractPageMap(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, XLogRecPtr startpoint, XLogRecPtr endpoint) { bool extract_isok = true; - time_t start_time, - end_time; - - elog(LOG, "Compiling pagemap"); - time(&start_time); extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, InvalidXLogRecPtr, tli, wal_seg_size, startpoint, endpoint, false, extractPageInfo, NULL); - - time(&end_time); - if (extract_isok) - elog(LOG, "Pagemap compiled, time elapsed %.0f sec", - difftime(end_time, start_time)); - else + if (!extract_isok) elog(ERROR, "Pagemap compiling failed"); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index be8757a5b..82e3b7e31 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -136,7 +136,7 @@ do { \ /* Information about single file (or dir) in backup */ typedef struct pgFile { - char *name; /* file or directory name */ + char *name; /* file or directory name */ mode_t mode; /* protection (file type and permission) */ size_t size; /* size of the file */ size_t read_size; /* size of the portion read (if only some pages are @@ -149,27 +149,34 @@ typedef struct pgFile */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ - char *linked; /* path of the linked file */ + char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ - char *path; /* absolute path of the file */ - char *rel_path; /* relative path of the file */ + char *path; /* absolute path of the file */ + char *rel_path; /* relative path of the file */ Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ Oid dbOid; /* dbOid extracted from path, if applicable */ Oid relOid; /* relOid extracted from path, if applicable */ - char *forkName; /* forkName extracted from path, if applicable */ + char *forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ int n_blocks; /* size of the file in blocks, readed during DELTA backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; - int external_dir_num; /* Number of external directory. 0 if not external */ - bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ - CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile pg_atomic_flag lock; /* lock for synchronization of parallel threads */ - datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ - bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, - * i.e. datafiles without _ptrack */ + int external_dir_num; /* Number of external directory. 0 if not external */ + bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ + CompressAlg compress_alg; /* compression algorithm applied to the file */ + volatile pg_atomic_flag lock;/* lock for synchronization of parallel threads */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, + * i.e. datafiles without _ptrack */ } pgFile; +typedef struct page_map_entry +{ + const char *path; /* file or directory name */ + char *pagemap; + size_t pagemapsize; +} page_map_entry; + /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ @@ -294,6 +301,10 @@ typedef struct PGNodeInfo int server_version; char server_version_str[100]; + int ptrack_version_num; + bool is_ptrack_enable; + const char *ptrack_schema; /* used only for ptrack 2.x */ + } PGNodeInfo; typedef struct pgBackup pgBackup; @@ -412,6 +423,8 @@ typedef struct pgSetBackupParams typedef struct { + PGNodeInfo *nodeInfo; + const char *from_root; const char *to_root; const char *external_prefix; @@ -578,7 +591,6 @@ extern bool smooth_checkpoint; /* remote probackup options */ extern char* remote_agent; -extern bool is_ptrack_support; extern bool exclusive_backup; /* delete options */ @@ -622,8 +634,8 @@ extern void process_block_change(ForkNumber forknum, RelFileNode rnode, extern char *pg_ptrack_get_block(ConnectionArgs *arguments, Oid dbOid, Oid tblsOid, Oid relOid, - BlockNumber blknum, - size_t *result_size); + BlockNumber blknum, size_t *result_size, + int ptrack_version_num, const char *ptrack_schema); /* in restore.c */ extern int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, @@ -797,6 +809,7 @@ extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read, fio_location location); extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); +extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileComparePathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); @@ -813,6 +826,9 @@ extern bool backup_data_file(backup_files_arg* arguments, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, + uint32 checksum_version, + int ptrack_version_num, + const char *ptrack_schema, bool missing_ok); extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, @@ -878,5 +894,21 @@ extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeIn extern void check_system_identifiers(PGconn *conn, char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); +/* in ptrack.c */ +extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); +extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, + const char *ptrack_schema, XLogRecPtr lsn); +extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); +extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); +extern bool pg_ptrack_enable(PGconn *backup_conn); +extern bool pg_ptrack_enable2(PGconn *backup_conn); +extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); +extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, + Oid db_oid, + Oid rel_oid, + size_t *result_size, + PGconn *backup_conn); +extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); +extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); #endif /* PG_PROBACKUP_H */ diff --git a/src/ptrack.c b/src/ptrack.c new file mode 100644 index 000000000..ee39d23b8 --- /dev/null +++ b/src/ptrack.c @@ -0,0 +1,702 @@ +/*------------------------------------------------------------------------- + * + * ptrack.c: support functions for ptrack backups + * + * Copyright (c) 2019 Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#if PG_VERSION_NUM < 110000 +#include "catalog/catalog.h" +#endif +#include "catalog/pg_tablespace.h" + +/* + * Macro needed to parse ptrack. + * NOTE Keep those values synchronized with definitions in ptrack.h + */ +#define PTRACK_BITS_PER_HEAPBLOCK 1 +#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * NOTE we rely on the fact that provided parray is sorted by file->path. + */ +void +make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) +{ + size_t i; + Oid dbOid_with_ptrack_init = 0; + Oid tblspcOid_with_ptrack_init = 0; + char *ptrack_nonparsed = NULL; + size_t ptrack_nonparsed_size = 0; + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + size_t start_addr; + + /* + * If there is a ptrack_init file in the database, + * we must backup all its files, ignoring ptrack files for relations. + */ + if (file->is_database) + { + char *filename = strrchr(file->path, '/'); + + Assert(filename != NULL); + filename++; + + /* + * The function pg_ptrack_get_and_clear_db returns true + * if there was a ptrack_init file. + * Also ignore ptrack files for global tablespace, + * to avoid any possible specific errors. + */ + if ((file->tblspcOid == GLOBALTABLESPACE_OID) || + pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid, backup_conn)) + { + dbOid_with_ptrack_init = file->dbOid; + tblspcOid_with_ptrack_init = file->tblspcOid; + } + } + + if (file->is_datafile) + { + if (file->tblspcOid == tblspcOid_with_ptrack_init && + file->dbOid == dbOid_with_ptrack_init) + { + /* ignore ptrack if ptrack_init exists */ + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + file->pagemap_isabsent = true; + continue; + } + + /* get ptrack bitmap once for all segments of the file */ + if (file->segno == 0) + { + /* release previous value */ + pg_free(ptrack_nonparsed); + ptrack_nonparsed_size = 0; + + ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, + file->relOid, &ptrack_nonparsed_size, backup_conn); + } + + if (ptrack_nonparsed != NULL) + { + /* + * pg_ptrack_get_and_clear() returns ptrack with VARHDR cut out. + * Compute the beginning of the ptrack map related to this segment + * + * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 + * RELSEG_SIZE. Number of Pages per segment: 131072 + * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed + * to keep track on one relsegment: 16384 + */ + start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; + + /* + * If file segment was created after we have read ptrack, + * we won't have a bitmap for this segment. + */ + if (start_addr > ptrack_nonparsed_size) + { + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + else + { + + if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) + { + file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + else + { + file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; + elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); + } + + file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); + memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); + } + } + else + { + /* + * If ptrack file is missing, try to copy the entire file. + * It can happen in two cases: + * - files were created by commands that bypass buffer manager + * and, correspondingly, ptrack mechanism. + * i.e. CREATE DATABASE + * - target relation was deleted. + */ + elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + file->pagemap_isabsent = true; + } + } + } +} + +/* Check if the instance supports compatible version of ptrack, + * fill-in version number if it does. + * Also for ptrack 2.x save schema namespace. + */ +void +get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) +{ + PGresult *res_db; + char *ptrack_version_str; + + res_db = pgut_execute(backup_conn, + "SELECT extnamespace::regnamespace, extversion " + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'", + 0, NULL); + + if (PQntuples(res_db) > 0) + { + /* ptrack 2.x is supported, save schema name and version */ + nodeInfo->ptrack_schema = pgut_strdup(PQgetvalue(res_db, 0, 0)); + + if (nodeInfo->ptrack_schema == NULL) + elog(ERROR, "Failed to obtain schema name of ptrack extension"); + + ptrack_version_str = PQgetvalue(res_db, 0, 1); + } + else + { + /* ptrack 1.x is supported, save version */ + PQclear(res_db); + res_db = pgut_execute(backup_conn, + "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + 0, NULL); + + if (PQntuples(res_db) == 0) + { + /* ptrack is not supported */ + PQclear(res_db); + return; + } + + res_db = pgut_execute(backup_conn, + "SELECT pg_catalog.ptrack_version()", + 0, NULL); + if (PQntuples(res_db) == 0) + { + /* TODO: Something went wrong, should we error out here? */ + PQclear(res_db); + return; + } + ptrack_version_str = PQgetvalue(res_db, 0, 0); + } + + if (strcmp(ptrack_version_str, "1.5") == 0) + nodeInfo->ptrack_version_num = 15; + else if (strcmp(ptrack_version_str, "1.6") == 0) + nodeInfo->ptrack_version_num = 16; + else if (strcmp(ptrack_version_str, "1.7") == 0) + nodeInfo->ptrack_version_num = 17; + else if (strcmp(ptrack_version_str, "2.0") == 0) + nodeInfo->ptrack_version_num = 20; + else + elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", + ptrack_version_str); + + PQclear(res_db); +} + +/* + * Check if ptrack is enabled in target instance + */ +bool +pg_ptrack_enable(PGconn *backup_conn) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + + +/* ---------------------------- + * Ptrack 1.* support functions + * ---------------------------- + */ + +/* Clear ptrack files in all databases of the instance we connected to */ +void +pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) +{ + PGresult *res_db, + *res; + const char *dbname; + int i; + Oid dbOid, tblspcOid; + char *params[2]; + + // FIXME Perform this check on caller's side + if (ptrack_version_num >= 20) + return; + + params[0] = palloc(64); + params[1] = palloc(64); + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + 0, NULL); + + for(i = 0; i < PQntuples(res_db); i++) + { + PGconn *tmp_conn; + + dbname = PQgetvalue(res_db, i, 0); + if (strcmp(dbname, "template0") == 0) + continue; + + dbOid = atoi(PQgetvalue(res_db, i, 1)); + tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + + tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, + dbname, + instance_config.conn_opt.pguser); + + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", + 0, NULL); + PQclear(res); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + PQclear(res); + + pgut_disconnect(tmp_conn); + } + + pfree(params[0]); + pfree(params[1]); + PQclear(res_db); +} + +bool +pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) +{ + char *params[2]; + char *dbname; + PGresult *res_db; + PGresult *res; + bool result; + + params[0] = palloc(64); + params[1] = palloc(64); + + sprintf(params[0], "%i", dbOid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return false; + + dbname = PQgetvalue(res_db, 0, 0); + + /* Always backup all files from template0 database */ + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return true; + } + PQclear(res_db); + + sprintf(params[0], "%i", dbOid); + sprintf(params[1], "%i", tblspcOid); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); + + if (!parse_bool(PQgetvalue(res, 0, 0), &result)) + elog(ERROR, + "result of pg_ptrack_get_and_clear_db() is invalid: %s", + PQgetvalue(res, 0, 0)); + + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* Read and clear ptrack files of the target relation. + * Result is a bytea ptrack map of all segments of the target relation. + * case 1: we know a tablespace_oid, db_oid, and rel_filenode + * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) + * case 3: we know only rel_filenode (because file in pg_global) + */ +char * +pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, + size_t *result_size, PGconn *backup_conn) +{ + PGconn *tmp_conn; + PGresult *res_db, + *res; + char *params[2]; + char *result; + char *val; + + params[0] = palloc(64); + params[1] = palloc(64); + + /* regular file (not in directory 'global') */ + if (db_oid != 0) + { + char *dbname; + + sprintf(params[0], "%i", db_oid); + res_db = pgut_execute(backup_conn, + "SELECT datname FROM pg_database WHERE oid=$1", + 1, (const char **) params); + /* + * If database is not found, it's not an error. + * It could have been deleted since previous backup. + */ + if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) + return NULL; + + dbname = PQgetvalue(res_db, 0, 0); + + if (strcmp(dbname, "template0") == 0) + { + PQclear(res_db); + return NULL; + } + + tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, + dbname, + instance_config.conn_opt.pguser); + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", + dbname, tablespace_oid, rel_filenode); + PQclear(res_db); + pgut_disconnect(tmp_conn); + } + /* file in directory 'global' */ + else + { + /* + * execute ptrack_get_and_clear for relation in pg_global + * Use backup_conn, cause we can do it from any database. + */ + sprintf(params[0], "%i", tablespace_oid); + sprintf(params[1], "%i", rel_filenode); + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", + 2, (const char **)params); + + if (PQnfields(res) != 1) + elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", + rel_filenode); + } + + val = PQgetvalue(res, 0, 0); + + /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. + * It should be fixed in future ptrack releases, but till then we + * can parse it. + */ + if (strcmp("x", val+1) == 0) + { + /* Ptrack file is missing */ + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + PQclear(res); + pfree(params[0]); + pfree(params[1]); + + return result; +} + +/* + * Get lsn of the moment when ptrack was enabled the last time. + */ +XLogRecPtr +get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) + +{ + PGresult *res; + uint32 lsn_hi; + uint32 lsn_lo; + XLogRecPtr lsn; + + if (nodeInfo->ptrack_version_num < 20) + res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_control_lsn()", + 0, NULL); + else + { + char query[128]; + + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + res = pgut_execute(backup_conn, query, 0, NULL); + } + + /* Extract timeline and LSN from results of pg_start_backup() */ + XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); + /* Calculate LSN */ + lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + + PQclear(res); + return lsn; +} + +char * +pg_ptrack_get_block(ConnectionArgs *arguments, + Oid dbOid, + Oid tblsOid, + Oid relOid, + BlockNumber blknum, + size_t *result_size, + int ptrack_version_num, + const char *ptrack_schema) +{ + PGresult *res; + char *params[4]; + char *result; + + params[0] = palloc(64); + params[1] = palloc(64); + params[2] = palloc(64); + params[3] = palloc(64); + + /* + * Use tmp_conn, since we may work in parallel threads. + * We can connect to any database. + */ + sprintf(params[0], "%i", tblsOid); + sprintf(params[1], "%i", dbOid); + sprintf(params[2], "%i", relOid); + sprintf(params[3], "%u", blknum); + + if (arguments->conn == NULL) + { + arguments->conn = pgut_connect(instance_config.conn_opt.pghost, + instance_config.conn_opt.pgport, + instance_config.conn_opt.pgdatabase, + instance_config.conn_opt.pguser); + } + + if (arguments->cancel_conn == NULL) + arguments->cancel_conn = PQgetCancel(arguments->conn); + + //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + + if (ptrack_version_num < 20) + res = pgut_execute_parallel(arguments->conn, + arguments->cancel_conn, + "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", + 4, (const char **)params, true, false, false); + else + { + char query[128]; + + /* sanity */ + if (!ptrack_schema) + elog(ERROR, "Schema name of ptrack extension is missing"); + + sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + + res = pgut_execute_parallel(arguments->conn, + arguments->cancel_conn, + query, 4, (const char **)params, + true, false, false); + } + + if (PQnfields(res) != 1) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + if (PQgetisnull(res, 0, 0)) + { + elog(VERBOSE, "cannot get file block for relation oid %u", + relOid); + return NULL; + } + + result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), + result_size); + + PQclear(res); + + pfree(params[0]); + pfree(params[1]); + pfree(params[2]); + pfree(params[3]); + + return result; +} + +/* ---------------------------- + * Ptrack 2.* support functions + * ---------------------------- + */ + +/* + * Check if ptrack is enabled in target instance + */ +bool +pg_ptrack_enable2(PGconn *backup_conn) +{ + PGresult *res_db; + + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + + if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) + { + PQclear(res_db); + return false; + } + PQclear(res_db); + return true; +} + +/* + * Fetch a list of changed files with their ptrack maps. + */ +parray * +pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn) +{ + PGresult *res; + char lsn_buf[17 + 1]; + char *params[1]; + parray *pagemapset = NULL; + int i; + char query[512]; + + snprintf(lsn_buf, sizeof lsn_buf, "%X/%X", (uint32) (lsn >> 32), (uint32) lsn); + params[0] = pstrdup(lsn_buf); + + if (!ptrack_schema) + elog(ERROR, "Schema name of ptrack extension is missing"); + + sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); + + res = pgut_execute(backup_conn, query, 1, (const char **) params); + pfree(params[0]); + + if (PQnfields(res) != 2) + elog(ERROR, "cannot get ptrack pagemapset"); + + /* sanity ? */ + + /* Construct database map */ + for (i = 0; i < PQntuples(res); i++) + { + page_map_entry *pm_entry = (page_map_entry *) pgut_malloc(sizeof(page_map_entry)); + + /* get path */ + pm_entry->path = pgut_strdup(PQgetvalue(res, i, 0)); + + /* get bytea */ + pm_entry->pagemap = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, i, 1), + &pm_entry->pagemapsize); + + if (pagemapset == NULL) + pagemapset = parray_new(); + + parray_append(pagemapset, pm_entry); + } + + PQclear(res); + + return pagemapset; +} + +/* + * Given a list of files in the instance to backup, build a pagemap for each + * data file that has ptrack. Result is saved in the pagemap field of pgFile. + * + * We fetch a list of changed files with their ptrack maps. After that files + * are merged with their bitmaps. File without bitmap is treated as unchanged. + */ +void +make_pagemap_from_ptrack_2(parray *files, + PGconn *backup_conn, + const char *ptrack_schema, + XLogRecPtr lsn) +{ + parray *filemaps; + int file_i = 0; + page_map_entry *dummy_map = NULL; + + /* Receive all available ptrack bitmaps at once */ + filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn); + + if (filemaps != NULL) + parray_qsort(filemaps, pgFileMapComparePath); + else + return; + + dummy_map = (page_map_entry *) pgut_malloc(sizeof(page_map_entry)); + + /* Iterate over files and look for corresponding pagemap if any */ + for (file_i = 0; file_i < parray_num(files); file_i++) + { + pgFile *file = (pgFile *) parray_get(files, file_i); + page_map_entry **res_map = NULL; + page_map_entry *map = NULL; + + /* + * For now nondata files are not entitled to have pagemap + * TODO It's possible to use ptrack for incremental backup of + * relation forks. Not implemented yet. + */ + if (!file->is_datafile || file->is_cfs) + continue; + + /* Consider only files from PGDATA (this check is probably redundant) */ + if (file->external_dir_num != 0) + continue; + + if (filemaps) + { + dummy_map->path = file->rel_path; + res_map = parray_bsearch(filemaps, dummy_map, pgFileMapComparePath); + map = (res_map) ? *res_map : NULL; + } + + /* Found map */ + if (map) + { + elog(VERBOSE, "Using ptrack pagemap for file \"%s\"", file->rel_path); + file->pagemap.bitmapsize = map->pagemapsize; + file->pagemap.bitmap = map->pagemap; + } + } + + free(dummy_map); +} diff --git a/tests/Readme.md b/tests/Readme.md index 3adf0c019..c1dd9a63d 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -34,7 +34,7 @@ Run ptrack tests: Usage: - pip install testgres + pip install testgres==1.8.2 export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` diff --git a/tests/archive.py b/tests/archive.py index b4d3a9d76..5650afe88 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -579,6 +579,10 @@ def test_archive_push_partial_file_exists(self): log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: log_content = f.read() + self.assertIn( + 'Cannot open destination temporary WAL file', + log_content) + self.assertIn( 'Reusing stale destination temporary WAL file', log_content) @@ -1320,6 +1324,8 @@ def test_archive_catalog(self): replica.pgbench_init(scale=2) + sleep(5) + show = self.show_archive(backup_dir, as_text=True) show = self.show_archive(backup_dir) diff --git a/tests/backup.py b/tests/backup.py index ad9e79482..0e448cf2b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -24,8 +24,7 @@ def test_backup_modes_archive(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -33,6 +32,11 @@ def test_backup_modes_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] @@ -121,7 +125,7 @@ def test_incremental_backup_without_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -129,6 +133,11 @@ def test_incremental_backup_without_full(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + try: self.backup_node(backup_dir, 'node', node, backup_type="page") # we should die here because exception is what we expect to happen @@ -244,7 +253,7 @@ def test_ptrack_threads(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -252,6 +261,11 @@ def test_ptrack_threads(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) @@ -276,14 +290,18 @@ def test_ptrack_threads_stream(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -307,6 +325,7 @@ def test_page_corruption_heal_via_ptrack_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -315,6 +334,11 @@ def test_page_corruption_heal_via_ptrack_1(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -324,9 +348,10 @@ def test_page_corruption_heal_via_ptrack_1(self): "create table t_heap as select 1 as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1000) i") + node.safe_psql( "postgres", - "CHECKPOINT;") + "CHECKPOINT") heap_path = node.safe_psql( "postgres", @@ -340,12 +365,12 @@ def test_page_corruption_heal_via_ptrack_1(self): self.backup_node( backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", "--log-level-file=verbose"]) + options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() - self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('block 1, try to fetch via shared buffer', log_content) self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) f.close @@ -366,6 +391,7 @@ def test_page_corruption_heal_via_ptrack_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -374,6 +400,11 @@ def test_page_corruption_heal_via_ptrack_2(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -414,12 +445,12 @@ def test_page_corruption_heal_via_ptrack_2(self): if self.remote: self.assertTrue( "WARNING: File" in e.message and - "try to fetch via SQL" in e.message and + "try to fetch via shared buffer" in e.message and "WARNING: page verification failed, " "calculated checksum" in e.message and "ERROR: query failed: " "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) else: @@ -427,12 +458,12 @@ def test_page_corruption_heal_via_ptrack_2(self): "LOG: File" in e.message and "blknum" in e.message and "have wrong checksum" in e.message and - "try to fetch via SQL" in e.message and + "try to fetch via shared buffer" in e.message and "WARNING: page verification failed, " "calculated checksum" in e.message and "ERROR: query failed: " "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block_2" in e.message, + "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -450,6 +481,7 @@ def test_backup_detect_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -458,6 +490,11 @@ def test_backup_detect_corruption(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + if self.ptrack and node.major_version > 11: + node.safe_psql( + "postgres", + "create extension ptrack") + node.safe_psql( "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, " @@ -922,7 +959,7 @@ def test_drop_rel_during_backup_delta(self): # DELTA backup gdb = self.backup_node( backup_dir, 'node', node, backup_type='delta', - gdb=True, options=['--log-level-file=verbose']) + gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') gdb.run_until_break() @@ -989,7 +1026,7 @@ def test_drop_rel_during_backup_page(self): # PAGE backup gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', - gdb=True, options=['--log-level-file=verbose']) + gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') gdb.run_until_break() @@ -1029,15 +1066,19 @@ def test_drop_rel_during_backup_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.safe_psql( "postgres", "create table t_heap as select i" @@ -1055,7 +1096,7 @@ def test_drop_rel_during_backup_ptrack(self): # PTRACK backup gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - gdb=True, options=['--log-level-file=verbose']) + gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') gdb.run_until_break() @@ -1304,7 +1345,7 @@ def test_sigint_handling(self): # FULL backup gdb = self.backup_node( backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=verbose']) + options=['--stream', '--log-level-file=LOG']) gdb.set_breakpoint('copy_file') gdb.run_until_break() @@ -1342,7 +1383,7 @@ def test_sigterm_handling(self): # FULL backup gdb = self.backup_node( backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=verbose']) + options=['--stream', '--log-level-file=LOG']) gdb.set_breakpoint('copy_file') gdb.run_until_break() @@ -1379,8 +1420,7 @@ def test_sigquit_handling(self): # FULL backup gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=verbose']) + backup_dir, 'node', node, gdb=True, options=['--stream']) gdb.set_breakpoint('copy_file') gdb.run_until_break() @@ -1534,12 +1574,9 @@ def test_backup_with_least_privileges_role(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s'}) - - if self.ptrack: - self.set_auto_conf(node, {'ptrack_enable': 'on'}) + pg_options={'archive_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1550,6 +1587,11 @@ def test_backup_with_least_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') + if self.ptrack and node.major_version >= 12: + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + # PG 9.5 if self.get_version(node) < 90600: node.safe_psql( @@ -1639,6 +1681,7 @@ def test_backup_with_least_privileges_role(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -1654,22 +1697,42 @@ def test_backup_with_least_privileges_role(self): ) if self.ptrack: - for fname in [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', + if node.major_version < 12: + for fname in [ + 'pg_catalog.oideq(oid, oid)', + 'pg_catalog.ptrack_version()', + 'pg_catalog.pg_ptrack_clear()', + 'pg_catalog.pg_ptrack_control_lsn()', + 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', + 'pg_catalog.pg_stop_backup()']: + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION {0} " + "TO backup".format(fname)) + else: + fnames = [ + 'pg_catalog.pg_ptrack_get_pagemapset(pg_lsn)', 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', - 'pg_catalog.pg_stop_backup()']: - # try: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) - # except: - # pass + 'pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint)' + ] + + for fname in fnames: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION {0} " + "TO backup".format(fname)) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") # FULL backup self.backup_node( @@ -1903,14 +1966,12 @@ def test_backup_with_less_privileges_role(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', 'checkpoint_timeout': '1h'}) - if self.ptrack: - self.set_auto_conf(node, {'ptrack_enable': 'on'}) - self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1920,6 +1981,11 @@ def test_backup_with_less_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') + if self.ptrack and node.major_version >= 12: + node.safe_psql( + 'backupdb', + 'CREATE EXTENSION ptrack') + # PG 9.5 if self.get_version(node) < 90600: node.safe_psql( @@ -2006,6 +2072,15 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'node', node, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) + # PTRACK + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) + if self.get_version(node) < 90600: self.del_test_dir(module_name, fname) return @@ -2062,6 +2137,15 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) + # PTRACK backup from replica + if self.ptrack: + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -2268,6 +2352,9 @@ def test_issue_132_1(self): # @unittest.skip("skip") def test_streaming_timeout(self): """ + Illustrate the problem of loosing exact error + message because our WAL streaming engine is "borrowed" + from pg_receivexlog """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2304,5 +2391,9 @@ def test_streaming_timeout(self): 'could not receive data from WAL stream', log_content) + self.assertIn( + 'ERROR: Problem in receivexlog', + log_content) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/compatibility.py b/tests/compatibility.py index 2375d780f..650321167 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -220,10 +220,10 @@ def test_backward_compatibility_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', - 'ptrack_enable': 'on'}) + 'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -254,8 +254,7 @@ def test_backward_compatibility_ptrack(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + backup_dir, 'node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) diff --git a/tests/delete.py b/tests/delete.py index 7cefd22ab..d6e9b2db7 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -194,8 +194,8 @@ def test_delete_increment_ptrack(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -203,6 +203,11 @@ def test_delete_increment_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + 'postgres', + 'CREATE EXTENSION ptrack') + # full backup mode self.backup_node(backup_dir, 'node', node) # ptrack backup mode diff --git a/tests/delta.py b/tests/delta.py index 12b2fadd6..2e91ef91d 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -28,7 +28,6 @@ def test_basic_delta_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', 'autovacuum': 'off'}) node_restored = self.make_simple_node( @@ -105,7 +104,6 @@ def test_delta_vacuum_truncate_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', 'autovacuum': 'off' } ) @@ -193,7 +191,6 @@ def test_delta_vacuum_truncate_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', 'autovacuum': 'off' } ) @@ -493,18 +490,14 @@ def test_delta_vacuum_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s' - } - ) + initdb_params=['--data-checksums']) + node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -577,7 +570,6 @@ def test_create_db(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'checkpoint_timeout': '5min', 'autovacuum': 'off' } ) @@ -1059,7 +1051,7 @@ def test_delta_delete(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_delta_corruption_heal_via_ptrack_1(self): + def test_delta_corruption_heal_via_ptrack(self): """make node, corrupt some page, check that backup failed""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') @@ -1076,6 +1068,11 @@ def test_delta_corruption_heal_via_ptrack_1(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -1107,7 +1104,7 @@ def test_delta_corruption_heal_via_ptrack_1(self): # open log file and check with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() - self.assertIn('block 1, try to fetch via SQL', log_content) + self.assertIn('block 1, try to fetch via shared buffer', log_content) self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) f.close @@ -1119,7 +1116,7 @@ def test_delta_corruption_heal_via_ptrack_1(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_2(self): + def test_page_corruption_heal_via_ptrack(self): """make node, corrupt some page, check that backup failed""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') @@ -1136,6 +1133,11 @@ def test_page_corruption_heal_via_ptrack_2(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -1176,7 +1178,7 @@ def test_page_corruption_heal_via_ptrack_2(self): if self.remote: self.assertTrue( "LOG: File" in e.message and - "try to fetch via SQL" in e.message and + "try to fetch via shared buffer" in e.message and "WARNING: page verification failed, " "calculated checksum" in e.message and "ERROR: query failed: " @@ -1189,7 +1191,7 @@ def test_page_corruption_heal_via_ptrack_2(self): "WARNING: File" in e.message and "blknum" in e.message and "have wrong checksum" in e.message and - "try to fetch via SQL" in e.message and + "try to fetch via shared buffer" in e.message and "WARNING: page verification failed, " "calculated checksum" in e.message and "ERROR: query failed: " diff --git a/tests/external.py b/tests/external.py index 1e383c709..e80935eb7 100644 --- a/tests/external.py +++ b/tests/external.py @@ -8,7 +8,7 @@ module_name = 'external' - +# TODO: add some ptrack tests class ExternalTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") diff --git a/tests/false_positive.py b/tests/false_positive.py index d3b27f877..571509db3 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -117,17 +117,17 @@ def test_ptrack_concurrent_get_and_clear_1(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) + ptrack_enable=True, + initdb_params=['--data-checksums']) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -202,17 +202,17 @@ def test_ptrack_concurrent_get_and_clear_2(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) + ptrack_enable=True, + initdb_params=['--data-checksums']) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 07c1ad426..b67b85407 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -311,6 +311,7 @@ def make_simple_node( self, base_dir=None, set_replication=False, + ptrack_enable=False, initdb_params=[], pg_options={}): @@ -325,6 +326,10 @@ def make_simple_node( node.init( initdb_params=initdb_params, allow_streaming=set_replication) + # set major version + with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: + node.major_version = int(f.read().rstrip()) + # Sane default parameters options = {} options['max_connections'] = 100 @@ -345,16 +350,18 @@ def make_simple_node( if set_replication: options['max_wal_senders'] = 10 + if ptrack_enable: + if node.major_version > 11: + options['ptrack_map_size'] = '128MB' + else: + options['ptrack_enable'] = 'on' + # set default values self.set_auto_conf(node, options) # Apply given parameters self.set_auto_conf(node, pg_options) - # set major version - with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version = f.read().rstrip() - return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): @@ -482,6 +489,31 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): os.close(file) return ptrack_bits_for_fork + def check_ptrack_map_sanity(self, node, idx_ptrack): + if node.major_version >= 12: + return + + success = True + for i in idx_ptrack: + # get new size of heap and indexes. size calculated in pages + idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate new md5sums for pages + idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], + [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + + # compare pages and check ptrack sanity + if not self.check_ptrack_sanity(idx_ptrack[i]): + success = False + + self.assertTrue( + success, 'Ptrack has failed to register changes in data files') + def check_ptrack_sanity(self, idx_dict): success = True if idx_dict['new_size'] > idx_dict['old_size']: @@ -1157,10 +1189,10 @@ def get_restore_command(self, backup_dir, instance, node): return restore_command - def set_auto_conf(self, node, options): + def set_auto_conf(self, node, options, config='postgresql.auto.conf'): # parse postgresql.auto.conf - path = os.path.join(node.data_dir, 'postgresql.auto.conf') + path = os.path.join(node.data_dir, config) with open(path, 'r') as f: raw_content = f.read() @@ -1220,11 +1252,18 @@ def set_replica( f.flush() f.close() + config = 'postgresql.auto.conf' + probackup_recovery_path = os.path.join(replica.data_dir, 'probackup_recovery.conf') + if os.path.exists(probackup_recovery_path): + if os.stat(probackup_recovery_path).st_size > 0: + config = 'probackup_recovery.conf' + self.set_auto_conf( replica, {'primary_conninfo': 'user={0} port={1} application_name={2} ' ' sslmode=prefer sslcompression=1'.format( - self.user, master.port, replica_name)}) + self.user, master.port, replica_name)}, + config) else: replica.append_conf('recovery.conf', 'standby_mode = on') replica.append_conf( @@ -1401,7 +1440,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): 'backup_label', 'tablespace_map', 'recovery.conf', 'ptrack_control', 'ptrack_init', 'pg_control', 'probackup_recovery.conf', 'recovery.signal', - 'standby.signal' + 'standby.signal', 'ptrack.map', 'ptrack.map.mmap' ] if exclude_dirs: diff --git a/tests/merge.py b/tests/merge.py index 000b483e0..541c5173c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -823,20 +823,19 @@ def test_merge_ptrack_truncate(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off', - 'ptrack_enable': 'on' - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + ptrack_enable=True, + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') node.safe_psql( @@ -871,6 +870,10 @@ def test_merge_ptrack_truncate(self): self.validate_pb(backup_dir) + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') @@ -878,8 +881,7 @@ def test_merge_ptrack_truncate(self): backup_dir, 'node', node_restored, options=[ "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace), - "--recovery-target-action=promote"]) + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) # Physical comparison if self.paranoia: diff --git a/tests/option.py b/tests/option.py index 71145e5f0..70d8fdd61 100644 --- a/tests/option.py +++ b/tests/option.py @@ -12,9 +12,6 @@ class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_help_1(self): """help options""" - self.maxDiff = None - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( self.run_pb(["--help"]), @@ -24,8 +21,6 @@ def test_help_1(self): # @unittest.skip("skip") def test_version_2(self): """help options""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertIn( version_out.read().decode("utf-8"), @@ -35,8 +30,6 @@ def test_version_2(self): # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') try: self.run_pb(["backup", "-b", "full"]) self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index fa9a50875..253be3441 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -129,9 +129,9 @@ def test_minrecpoint_on_replica(self): recovery_config = 'recovery.conf' replica.append_conf( - 'recovery.conf', "recovery_target = 'immediate'") + recovery_config, "recovery_target = 'immediate'") replica.append_conf( - 'recovery.conf', "recovery_target_action = 'pause'") + recovery_config, "recovery_target_action = 'pause'") replica.slow_start(replica=True) if self.get_version(node) < 100000: diff --git a/tests/ptrack.py b/tests/ptrack.py index d33498890..fa8311f36 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -15,6 +15,260 @@ class PtrackTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream']) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_unprivileged(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE DATABASE backupdb") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if self.ptrack: + fnames = [] + if node.major_version < 12: + fnames += [ + 'pg_catalog.oideq(oid, oid)', + 'pg_catalog.ptrack_version()', + 'pg_catalog.pg_ptrack_clear()', + 'pg_catalog.pg_ptrack_control_lsn()', + 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' + ] + else: + # TODO why backup works without these grants ? +# fnames += [ +# 'pg_ptrack_get_pagemapset(pg_lsn)', +# 'pg_ptrack_control_lsn()', +# 'pg_ptrack_get_block(oid, oid, oid, bigint)' +# ] + + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack") + + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") + + for fname in fnames: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) + + node.safe_psql( + "backupdb", + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', "-U", "backup"]) + + self.backup_node( + backup_dir, 'node', node, datname='backupdb', + backup_type='ptrack', options=['--stream', "-U", "backup"]) + + # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_enable(self): @@ -25,14 +279,18 @@ def test_ptrack_enable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s' - } - ) + 'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # PTRACK BACKUP try: self.backup_node( @@ -70,27 +328,38 @@ def test_ptrack_disable(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on' - } - ) + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={'checkpoint_timeout': '30s'}) + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) # DISABLE PTRACK - node.safe_psql('postgres', "alter system set ptrack_enable to off") + if node.major_version >= 12: + node.safe_psql('postgres', "alter system set ptrack_map_size to 0") + else: + node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.stop() node.slow_start() # ENABLE PTRACK - node.safe_psql('postgres', "alter system set ptrack_enable to on") + if node.major_version < 11: + node.safe_psql('postgres', "alter system set ptrack_enable to on") + else: + node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") node.stop() node.slow_start() @@ -129,39 +398,40 @@ def test_ptrack_uncommitted_xact(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + pg_options={'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'node', node) + con = node.connect("postgres") con.execute( "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) + " as id from generate_series(0,1) i") self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'] - ) + options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'] - ) + options=['--stream']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() self.restore_node( backup_dir, 'node', node_restored, options=["-j", "4"]) @@ -189,23 +459,21 @@ def test_ptrack_vacuum_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + ptrack_enable=True, + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'node', node) node.safe_psql( @@ -243,6 +511,10 @@ def test_ptrack_vacuum_full(self): gdb._execute('detach') process.join() + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') @@ -277,47 +549,43 @@ def test_ptrack_vacuum_truncate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.safe_psql( "postgres", "create sequence t_seq; " "create table t_heap tablespace somedata as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") + node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( backup_dir, 'node', node, backup_type='ptrack') @@ -328,6 +596,10 @@ def test_ptrack_vacuum_truncate(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') @@ -354,7 +626,7 @@ def test_ptrack_vacuum_truncate(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_ptrack_simple(self): + def test_ptrack_get_block(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" fname = self.id().split('.')[3] @@ -362,106 +634,34 @@ def test_ptrack_simple(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), - ) + ptrack_enable=True, + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() node.slow_start() - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'] - ) - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # Logical comparison - self.assertEqual( - result, - node_restored.safe_psql("postgres", "SELECT * FROM t_heap") - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_get_block(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s', - 'ptrack_enable': 'on' - } - ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.safe_psql( "postgres", "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) + " as id from generate_series(0,1) i") self.backup_node(backup_dir, 'node', node, options=['--stream']) gdb = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream'], - gdb=True - ) + gdb=True) - gdb.set_breakpoint('make_pagemap_from_ptrack') + if node.major_version > 11: + gdb.set_breakpoint('make_pagemap_from_ptrack_2') + else: + gdb.set_breakpoint('make_pagemap_from_ptrack_1') gdb.run_until_break() node.safe_psql( @@ -472,8 +672,8 @@ def test_ptrack_get_block(self): self.backup_node( backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) + backup_type='ptrack', options=['--stream']) + if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -507,26 +707,29 @@ def test_ptrack_stream(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP node.safe_psql("postgres", "create sequence t_seq") node.safe_psql( "postgres", "create table t_heap as select i as id, nextval('t_seq')" " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" - " as tsvector from generate_series(0,100) i" - ) + " as tsvector from generate_series(0,100) i") + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, options=['--stream']) @@ -536,14 +739,12 @@ def test_ptrack_stream(self): "postgres", "insert into t_heap select i as id, nextval('t_seq') as t_seq," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i" - ) + " from generate_series(100,200) i") + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") ptrack_backup_id = self.backup_node( - backup_dir, 'node', - node, backup_type='ptrack', - options=['--stream'] - ) + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -556,8 +757,7 @@ def test_ptrack_stream(self): "INFO: Restore of backup {0} completed.".format(full_backup_id), self.restore_node( backup_dir, 'node', node, - backup_id=full_backup_id, - options=["-j", "4", "--recovery-target-action=promote"] + backup_id=full_backup_id, options=["-j", "4"] ), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd) @@ -572,12 +772,10 @@ def test_ptrack_stream(self): "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), self.restore_node( backup_dir, 'node', node, - backup_id=ptrack_backup_id, - options=["-j", "4", "--recovery-target-action=promote"] + backup_id=ptrack_backup_id, options=["-j", "4"] ), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) + repr(self.output), self.cmd)) if self.paranoia: pgdata_restored = self.pgdata_content( @@ -600,19 +798,23 @@ def test_ptrack_archive(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP node.safe_psql( "postgres", @@ -620,8 +822,8 @@ def test_ptrack_archive(self): " select i as id," " md5(i::text) as text," " md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) + " from generate_series(0,100) i") + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") full_backup_id = self.backup_node(backup_dir, 'node', node) full_target_time = self.show_pb( @@ -633,8 +835,8 @@ def test_ptrack_archive(self): "insert into t_heap select i as id," " md5(i::text) as text," " md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i" - ) + " from generate_series(100,200) i") + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack') @@ -698,19 +900,20 @@ def test_ptrack_archive(self): def test_ptrack_pgpro417(self): """Make node, take full backup, take ptrack backup, delete ptrack backup. Try to take ptrack backup, - which should fail""" - self.maxDiff = None + which should fail. Actual only for PTRACK 1.x""" + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -778,20 +981,22 @@ def test_ptrack_pgpro417(self): def test_page_pgpro417(self): """ Make archive node, take full backup, take page backup, - delete page backup. Try to take ptrack backup, which should fail + delete page backup. Try to take ptrack backup, which should fail. + Actual only for PTRACK 1.x """ - self.maxDiff = None + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -848,20 +1053,22 @@ def test_page_pgpro417(self): def test_full_pgpro417(self): """ Make node, take two full backups, delete full second backup. - Try to take ptrack backup, which should fail + Try to take ptrack backup, which should fail. + Relevant only for PTRACK 1.x """ - self.maxDiff = None + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -925,26 +1132,27 @@ def test_create_db(self): Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense """ - self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'checkpoint_timeout': '5min', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP node.safe_psql( "postgres", @@ -967,16 +1175,14 @@ def test_create_db(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=["--stream"] - ) + options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1010,8 +1216,7 @@ def test_create_db(self): node_restored.cleanup() self.restore_node( backup_dir, 'node', node_restored, - backup_id=backup_id, options=["-j", "4"] - ) + backup_id=backup_id, options=["-j", "4"]) # COMPARE PHYSICAL CONTENT if self.paranoia: @@ -1031,14 +1236,12 @@ def test_create_db(self): 1, 0, "Expecting Error because we are connecting to deleted database" "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) + repr(self.output), self.cmd)) except QueryException as e: self.assertTrue( 'FATAL: database "db1" does not exist' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd) - ) + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1056,19 +1259,22 @@ def test_create_db_on_replica(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP node.safe_psql( "postgres", @@ -1118,6 +1324,7 @@ def test_create_db_on_replica(self): replica, backup_type='ptrack', options=[ '-j10', + '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(node.port) @@ -1129,8 +1336,7 @@ def test_create_db_on_replica(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1154,26 +1360,29 @@ def test_alter_table_set_tablespace_ptrack(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off' - } - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP self.create_tblspace_in_node(node, 'somedata') node.safe_psql( "postgres", "create table t_heap tablespace somedata as select i as id," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) + " from generate_series(0,100) i") # FULL backup self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1181,8 +1390,7 @@ def test_alter_table_set_tablespace_ptrack(self): self.create_tblspace_in_node(node, 'somedata_new') node.safe_psql( "postgres", - "alter table t_heap set tablespace somedata_new" - ) + "alter table t_heap set tablespace somedata_new") # sys.exit(1) # PTRACK BACKUP @@ -1200,8 +1408,7 @@ def test_alter_table_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1215,8 +1422,7 @@ def test_alter_table_set_tablespace_ptrack(self): "-T", "{0}={1}".format( self.get_tblspace_path(node, 'somedata_new'), self.get_tblspace_path(node_restored, 'somedata_new') - ), - "--recovery-target-action=promote" + ) ] ) @@ -1244,23 +1450,26 @@ def test_alter_database_set_tablespace_ptrack(self): """Make node, create tablespace with database," " take full backup, alter tablespace location," " take ptrack backup, restore database.""" - self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1313,23 +1522,26 @@ def test_drop_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'ptrack_enable': 'on', - 'autovacuum': 'off'} - ) + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # CREATE TABLE @@ -1366,10 +1578,19 @@ def test_drop_tablespace(self): backup_dir, 'node', node, backup_type='ptrack', options=["--stream"]) + if self.paranoia: + pgdata = self.pgdata_content( + node.data_dir, ignore_ptrack=True) + tblspace = self.get_tblspace_path(node, 'somedata') node.cleanup() shutil.rmtree(tblspace, ignore_errors=True) self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=True) + node.slow_start() tblspc_exist = node.safe_psql( @@ -1386,6 +1607,9 @@ def test_drop_tablespace(self): result_new = node.safe_psql("postgres", "select * from t_heap") self.assertEqual(result, result_new) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -1395,21 +1619,26 @@ def test_ptrack_alter_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - self.maxDiff = None fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', 'ptrack_enable': 'on', + 'checkpoint_timeout': '30s', 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') tblspc_path = self.get_tblspace_path(node, 'somedata') @@ -1425,7 +1654,8 @@ def test_ptrack_alter_tablespace(self): # Move table to separate tablespace node.safe_psql( - "postgres", "alter table t_heap set tablespace somedata") + "postgres", + "alter table t_heap set tablespace somedata") # GET LOGICAL CONTENT FROM NODE result = node.safe_psql("postgres", "select * from t_heap") @@ -1445,8 +1675,7 @@ def test_ptrack_alter_tablespace(self): tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') self.restore_node(backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"]) + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: @@ -1479,9 +1708,10 @@ def test_ptrack_alter_tablespace(self): pgdata = self.pgdata_content(node.data_dir) # Restore second ptrack backup and check table consistency - self.restore_node(backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"]) + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT if self.paranoia: @@ -1512,19 +1742,22 @@ def test_ptrack_multiple_segments(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'fsync': 'off', 'autovacuum': 'off', - 'full_page_writes': 'off' - } - ) + 'full_page_writes': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # CREATE TABLE @@ -1533,44 +1766,43 @@ def test_ptrack_multiple_segments(self): self.backup_node(backup_dir, 'node', node) # PTRACK STUFF - idx_ptrack = {'type': 'heap'} - idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') - idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') - idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack['path'], idx_ptrack['old_size']) - - pgbench = node.pgbench(options=['-T', '150', '-c', '2', '--no-vacuum']) + if node.major_version < 12: + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench( + options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() node.safe_psql("postgres", "checkpoint") - idx_ptrack['new_size'] = self.get_fork_size( - node, - 'pgbench_accounts' - ) - idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack['path'], - idx_ptrack['new_size'] - ) - idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, - idx_ptrack['path'] - ) + if node.major_version < 12: + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts') - if not self.check_ptrack_sanity(idx_ptrack): - self.assertTrue( - False, 'Ptrack has failed to register changes in data files' - ) + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size']) + + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path']) + + if not self.check_ptrack_sanity(idx_ptrack): + self.assertTrue( + False, 'Ptrack has failed to register changes in data files') # GET LOGICAL CONTENT FROM NODE # it`s stupid, because hint`s are ignored by ptrack - #result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.safe_psql("postgres", "select * from pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack') - node.safe_psql("postgres", "checkpoint") - # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -1581,18 +1813,18 @@ def test_ptrack_multiple_segments(self): tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( restored_node, - 'somedata_restored' - ) + 'somedata_restored') - self.restore_node(backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new), - "--recovery-target-action=promote"]) + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format( + tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM NODE_RESTORED if self.paranoia: pgdata_restored = self.pgdata_content( restored_node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE self.set_auto_conf( @@ -1601,11 +1833,13 @@ def test_ptrack_multiple_segments(self): result_new = restored_node.safe_psql( "postgres", - "select * from pgbench_accounts" - ) + "select * from pgbench_accounts") # COMPARE RESTORED FILES - #self.assertEqual(result, result_new, 'data is lost') + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1614,15 +1848,19 @@ def test_ptrack_multiple_segments(self): # @unittest.expectedFailure def test_atexit_fail(self): """ - Take backups of every available types and check that PTRACK is clean + Take backups of every available types and check that PTRACK is clean. + Relevant only for PTRACK 1.x """ + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'max_connections': '15'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1637,8 +1875,7 @@ def test_atexit_fail(self): try: self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=[ - "--stream", "-j 30"]) + options=["--stream", "-j 30"]) # we should die here because exception is what we expect to happen self.assertEqual( @@ -1667,14 +1904,19 @@ def test_atexit_fail(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): - """Take backups of every available types and check that PTRACK is clean""" + """ + Take backups of every available types and check that PTRACK is clean + Relevant only for PTRACK 1.x + """ + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1729,8 +1971,8 @@ def test_ptrack_clean(self): # Take PTRACK backup to clean every ptrack backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['-j10']) + backup_dir, 'node', node, backup_type='ptrack', options=['-j10']) + node.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -1777,15 +2019,19 @@ def test_ptrack_clean(self): def test_ptrack_clean_replica(self): """ Take backups of every available types from - master and check that PTRACK on replica is clean + master and check that PTRACK on replica is clean. + Relevant only for PTRACK 1.x """ + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'archive_timeout': '30s'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1918,15 +2164,19 @@ def test_ptrack_cluster_on_btree(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -1949,14 +2199,15 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) @@ -1966,27 +2217,8 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1997,15 +2229,19 @@ def test_ptrack_cluster_on_gist(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # Create table and indexes node.safe_psql( "postgres", @@ -2042,27 +2278,7 @@ def test_ptrack_cluster_on_gist(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # Compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + self.check_ptrack_map_sanity(node, idx_ptrack) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2073,15 +2289,19 @@ def test_ptrack_cluster_on_btree_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2140,27 +2360,7 @@ def test_ptrack_cluster_on_btree_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) - - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False - - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + self.check_ptrack_map_sanity(replica, idx_ptrack) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2171,15 +2371,18 @@ def test_ptrack_cluster_on_gist_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2233,36 +2436,35 @@ def test_ptrack_cluster_on_gist_replica(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'cluster t_heap using t_gist') - master.safe_psql('postgres', 'checkpoint') + master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1') + master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist') + + if master.major_version < 12: + master.safe_psql('postgres', 'CHECKPOINT') # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if master.major_version < 12: + replica.safe_psql('postgres', 'CHECKPOINT') + self.check_ptrack_map_sanity(replica, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) - # Compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + if self.paranoia: + pgdata = self.pgdata_content(replica.data_dir) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node) + + if self.paranoia: + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2275,16 +2477,22 @@ def test_ptrack_empty(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'autovacuum': 'off'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table @@ -2333,8 +2541,7 @@ def test_ptrack_empty(self): backup_id=backup_id, options=[ "-j", "4", - "-T{0}={1}".format(tblspace1, tblspace2)] - ) + "-T{0}={1}".format(tblspace1, tblspace2)]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -2355,14 +2562,20 @@ def test_ptrack_empty_replica(self): base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], + ptrack_enable=True, pg_options={ - 'ptrack_enable': 'on'}) + 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2405,13 +2618,8 @@ def test_ptrack_empty_replica(self): idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - self.wait_until_replica_catch_with_master(master, replica) - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - # Take PTRACK backup backup_id = self.backup_node( backup_dir, @@ -2419,19 +2627,21 @@ def test_ptrack_empty_replica(self): replica, backup_type='ptrack', options=[ - '-j10', '--stream', + '-j1', '--stream', '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port)]) - + if self.paranoia: pgdata = self.pgdata_content(replica.data_dir) + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + self.restore_node( backup_dir, 'replica', node_restored, - backup_id=backup_id, - options=["-j", "4"] - ) + backup_id=backup_id, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -2447,15 +2657,19 @@ def test_ptrack_truncate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -2467,48 +2681,67 @@ def test_ptrack_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + if node.major_version < 12: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) node.safe_psql('postgres', 'truncate t_heap') node.safe_psql('postgres', 'checkpoint') - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make full backup to clean every ptrack + if node.major_version < 12: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make backup to clean every ptrack self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + pgdata = self.pgdata_content(node.data_dir) + + if node.major_version < 12: + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_ptrack_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'max_wal_size': '32MB', 'archive_timeout': '10s', 'checkpoint_timeout': '30s'}) @@ -2518,6 +2751,11 @@ def test_ptrack_truncate_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2532,19 +2770,19 @@ def test_ptrack_truncate_replica(self): replica.slow_start(replica=True) # Create table and indexes - self.create_tblspace_in_node(master, 'somedata') master.safe_psql( "postgres", "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " + "create table t_heap " "as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") + for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': master.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( + "postgres", + "create index {0} on {1} using {2}({3}) ".format( i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) @@ -2552,16 +2790,17 @@ def test_ptrack_truncate_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make full backup to clean every ptrack + if replica.major_version < 12: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, options=[ @@ -2571,39 +2810,37 @@ def test_ptrack_truncate_replica(self): '--master-db=postgres', '--master-port={0}'.format(master.port)]) - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + if replica.major_version < 12: + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) master.safe_psql('postgres', 'truncate t_heap') - master.safe_psql('postgres', 'checkpoint') # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '-j10', + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + pgdata = self.pgdata_content(replica.data_dir) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2615,15 +2852,19 @@ def test_ptrack_vacuum(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -2651,17 +2892,18 @@ def test_ptrack_vacuum(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) # Delete some rows, vacuum it and make checkpoint node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') @@ -2669,27 +2911,24 @@ def test_ptrack_vacuum(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2700,9 +2939,9 @@ def test_ptrack_vacuum_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'checkpoint_timeout': '30'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2710,6 +2949,11 @@ def test_ptrack_vacuum_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2754,17 +2998,18 @@ def test_ptrack_vacuum_replica(self): '--master-port={0}'.format(master.port), '--stream']) - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + if replica.major_version < 12: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) # Delete some rows, vacuum it and make checkpoint master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') @@ -2776,27 +3021,23 @@ def test_ptrack_vacuum_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if replica.major_version < 12: + self.check_ptrack_map_sanity(master, idx_ptrack) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2808,15 +3049,19 @@ def test_ptrack_vacuum_bits_frozen(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -2845,37 +3090,34 @@ def test_ptrack_vacuum_bits_frozen(self): node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2886,15 +3128,19 @@ def test_ptrack_vacuum_bits_frozen_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -2930,7 +3176,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - # Take PTRACK backup to clean every ptrack + # Take backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, options=[ @@ -2940,14 +3186,15 @@ def test_ptrack_vacuum_bits_frozen_replica(self): '--master-port={0}'.format(master.port), '--stream']) - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if replica.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) master.safe_psql('postgres', 'vacuum freeze t_heap') master.safe_psql('postgres', 'checkpoint') @@ -2957,27 +3204,20 @@ def test_ptrack_vacuum_bits_frozen_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if replica.major_version < 12: + self.check_ptrack_map_sanity(master, idx_ptrack) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=['-j10', '--stream']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata = self.pgdata_content(replica.data_dir) + replica.cleanup() + + self.restore_node(backup_dir, 'replica', replica) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2989,15 +3229,19 @@ def test_ptrack_vacuum_bits_visibility(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -3023,40 +3267,37 @@ def test_ptrack_vacuum_bits_visibility(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -3068,15 +3309,18 @@ def test_ptrack_vacuum_full(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(node, 'somedata') # Create table and indexes @@ -3101,41 +3345,38 @@ def test_ptrack_vacuum_full(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'vacuum full t_heap') node.safe_psql('postgres', 'checkpoint') - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) - # compare pages and check ptrack sanity, the most important part - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -3147,17 +3388,20 @@ def test_ptrack_vacuum_full_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'autovacuum': 'off', - 'archive_timeout': '30s'} - ) + pg_options={'autovacuum': 'off'}) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) @@ -3177,14 +3421,16 @@ def test_ptrack_vacuum_full_replica(self): "create table t_heap as select i as id, " "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " "tsvector from generate_series(0,256000) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) + + if master.major_version < 12: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) master.safe_psql('postgres', 'vacuum t_heap') master.safe_psql('postgres', 'checkpoint') @@ -3201,18 +3447,17 @@ def test_ptrack_vacuum_full_replica(self): '--master-host=localhost', '--master-db=postgres', '--master-port={0}'.format(master.port), - '--stream' - ] - ) - # TODO: check that all ptrack are nullified - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + '--stream']) + + if replica.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') master.safe_psql('postgres', 'vacuum full t_heap') @@ -3222,28 +3467,20 @@ def test_ptrack_vacuum_full_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if replica.major_version < 12: + self.check_ptrack_map_sanity(master, idx_ptrack) - # compare pages and check ptrack sanity, the most important part - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + pgdata = self.pgdata_content(replica.data_dir) + replica.cleanup() + + self.restore_node(backup_dir, 'replica', replica) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -3255,74 +3492,73 @@ def test_ptrack_vacuum_truncate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - self.create_tblspace_in_node(node, 'somedata') + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table and indexes res = node.safe_psql( "postgres", "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " + "create table t_heap " "as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') + if node.major_version < 12: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'VACUUM t_heap') self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if node.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - node.safe_psql('postgres', 'delete from t_heap where id > 128;') - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') + node.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128') + node.safe_psql('postgres', 'VACUUM t_heap') + node.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if node.major_version < 12: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + pgdata = self.pgdata_content(node.data_dir) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files' - ) + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -3334,16 +3570,19 @@ def test_ptrack_vacuum_truncate_replica(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on', - 'checkpoint_timeout': '30'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( @@ -3376,52 +3615,57 @@ def test_ptrack_vacuum_truncate_replica(self): master.safe_psql('postgres', 'vacuum t_heap') master.safe_psql('postgres', 'checkpoint') - # Take PTRACK backup to clean every ptrack + # Take FULL backup to clean every ptrack self.backup_node( backup_dir, 'replica', replica, options=[ '-j10', + '--stream', '--master-host=localhost', '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream' + '--master-port={0}'.format(master.port) ] ) - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + if master.major_version < 12: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - master.safe_psql('postgres', 'delete from t_heap where id > 128;') - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') + master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128;') + master.safe_psql('postgres', 'VACUUM t_heap') + master.safe_psql('postgres', 'CHECKPOINT') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - success = True - for i in idx_ptrack: - # get new size of heap and indexes. size calculated in pages - idx_ptrack[i]['new_size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate new md5sums for pages - idx_ptrack[i]['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['new_size']) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], - [idx_ptrack[i]['old_size'], idx_ptrack[i]['new_size']]) + if master.major_version < 12: + self.check_ptrack_map_sanity(master, idx_ptrack) - # compare pages and check ptrack sanity - if not self.check_ptrack_sanity(idx_ptrack[i]): - success = False + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--log-level-file=INFO', + '--archive-timeout=30']) - self.assertTrue( - success, 'Ptrack has failed to register changes in data files') + pgdata = self.pgdata_content(replica.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'replica', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -3429,13 +3673,17 @@ def test_ptrack_vacuum_truncate_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery(self): + if self.pg_config_version > self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL =< 11 for this test') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on'}) + 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3486,3 +3734,141 @@ def test_ptrack_recovery(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery_1(self): + if self.pg_config_version < self.version_to_num('12.0'): + return unittest.skip('You need PostgreSQL >= 12 for this test') + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "CREATE INDEX {0} ON {1} USING {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + node.safe_psql( + 'postgres', + "create extension pg_buffercache") + + print(node.safe_psql( + 'postgres', + "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_zero_changes(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create table t_heap " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# TODO: ptrack.map corruption diff --git a/tests/replica.py b/tests/replica.py index 0c47dba89..8b076d402 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -23,24 +23,28 @@ def test_replica_stream_ptrack_backup(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') + if self.pg_config_version > self.version_to_num('9.6.0'): + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( - 'Skipped because backup from replica is not supported in PG 9.5') - - master.slow_start() self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # CREATE TABLE master.psql( "postgres", @@ -427,6 +431,8 @@ def test_take_backup_from_delayed_replica(self): data_dir=replica.data_dir, backup_type='page', options=['--archive-timeout=60s']) + sleep(1) + self.backup_node( backup_dir, 'replica', replica, backup_type='delta', options=['--archive-timeout=60s']) @@ -499,8 +505,7 @@ def test_replica_promote(self): self.add_instance(backup_dir, 'replica', replica) self.set_archiving(backup_dir, 'replica', replica, replica=True) self.set_replica( - master, replica, - replica_name='replica', synchronous=True) + master, replica, replica_name='replica', synchronous=True) replica.slow_start(replica=True) diff --git a/tests/restore.py b/tests/restore.py index 02a3bd6d7..115da13b9 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -500,7 +500,7 @@ def test_restore_full_ptrack_archive(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -508,6 +508,11 @@ def test_restore_full_ptrack_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.pgbench_init(scale=2) self.backup_node(backup_dir, 'node', node) @@ -529,8 +534,7 @@ def test_restore_full_ptrack_archive(self): "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( backup_dir, 'node', node, - options=[ - "-j", "4", "--recovery-target-action=promote"]), + options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -551,7 +555,7 @@ def test_restore_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -559,6 +563,11 @@ def test_restore_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.pgbench_init(scale=2) self.backup_node(backup_dir, 'node', node) @@ -587,8 +596,7 @@ def test_restore_ptrack(self): "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( backup_dir, 'node', node, - options=[ - "-j", "4", "--recovery-target-action=promote"]), + options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -609,8 +617,8 @@ def test_restore_full_ptrack_stream(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -618,6 +626,11 @@ def test_restore_full_ptrack_stream(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.pgbench_init(scale=2) self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -639,8 +652,7 @@ def test_restore_full_ptrack_stream(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -664,9 +676,8 @@ def test_restore_full_ptrack_under_load(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -674,6 +685,11 @@ def test_restore_full_ptrack_under_load(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.pgbench_init(scale=2) self.backup_node(backup_dir, 'node', node) @@ -703,8 +719,7 @@ def test_restore_full_ptrack_under_load(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) @@ -731,9 +746,8 @@ def test_restore_full_under_load_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'ptrack_enable': 'on'}) + ptrack_enable=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -741,6 +755,11 @@ def test_restore_full_under_load_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # wal_segment_size = self.guc_wal_segment_size(node) node.pgbench_init(scale=2) @@ -773,8 +792,7 @@ def test_restore_full_under_load_ptrack(self): self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--recovery-target-action=promote"]), + backup_dir, 'node', node, options=["-j", "4"]), '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() @@ -2493,7 +2511,7 @@ def test_partial_restore_exclude_tablespace(self): node.slow_start() cat_version = node.get_control_data()["Catalog version number"] - version_specific_dir = 'PG_' + node.major_version + '_' + cat_version + version_specific_dir = 'PG_' + str(node.major_version) + '_' + cat_version # PG_10_201707211 # pg_tblspc/33172/PG_9.5_201510051/16386/ @@ -3162,6 +3180,7 @@ def test_missing_database_map(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -3177,20 +3196,41 @@ def test_missing_database_map(self): ) if self.ptrack: - for fname in [ + fnames = [] + if node.major_version < 12: + fnames += [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', 'pg_catalog.pg_ptrack_clear()', 'pg_catalog.pg_ptrack_control_lsn()', 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', - 'pg_catalog.pg_stop_backup()']: + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' + ] + else: + # TODO why backup works without these grants ? +# fnames += [ +# 'pg_ptrack_get_pagemapset(pg_lsn)', +# 'pg_ptrack_control_lsn()', +# 'pg_ptrack_get_block(oid, oid, oid, bigint)' +# ] + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + for fname in fnames: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) + "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") # FULL backup without database_map backup_id = self.backup_node( From 75fefb0858f0195c244a8847b636ad4f0a9735c0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 24 Dec 2019 15:03:59 +0300 Subject: [PATCH 1097/2107] tests: minor fixes --- tests/backup.py | 2 +- tests/helpers/ptrack_helpers.py | 2 +- tests/replica.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 0e448cf2b..44b69b60e 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2187,7 +2187,7 @@ def test_issue_132(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_issue_132_1(self): """ https://github.com/postgrespro/pg_probackup/issues/132 diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b67b85407..c9a8361c0 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -328,7 +328,7 @@ def make_simple_node( # set major version with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version = int(f.read().rstrip()) + node.major_version = str(f.read().rstrip()) # Sane default parameters options = {} diff --git a/tests/replica.py b/tests/replica.py index 8b076d402..af126865b 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -411,7 +411,7 @@ def test_take_backup_from_delayed_replica(self): replica, {'recovery_min_apply_delay': '300s'}) else: replica.append_conf( - 'postgresql.auto.conf', + 'recovery.conf', 'recovery_min_apply_delay = 300s') replica.stop() From 1c791f3433e72f728342660c64891f32b75ec4ad Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 25 Dec 2019 17:41:19 +0300 Subject: [PATCH 1098/2107] tests: added ptrack.PtrackTest.test_corrupt_ptrack_map and ptrack.PtrackTest.test_ptrack_pg_resetxlog --- tests/ptrack.py | 255 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 215 insertions(+), 40 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index fa8311f36..b9c86e312 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -100,7 +100,7 @@ def test_ptrack_unprivileged(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) node.slow_start() node.safe_psql( @@ -283,7 +283,6 @@ def test_ptrack_enable(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -335,7 +334,6 @@ def test_ptrack_disable(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -404,7 +402,6 @@ def test_ptrack_uncommitted_xact(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -412,7 +409,7 @@ def test_ptrack_uncommitted_xact(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, options=['--stream']) con = node.connect("postgres") con.execute( @@ -464,7 +461,6 @@ def test_ptrack_vacuum_full(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -474,7 +470,7 @@ def test_ptrack_vacuum_full(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", @@ -499,10 +495,10 @@ def test_ptrack_vacuum_full(self): gdb.continue_execution_until_break(20) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -555,7 +551,6 @@ def test_ptrack_vacuum_truncate(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -577,7 +572,7 @@ def test_ptrack_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", @@ -588,10 +583,10 @@ def test_ptrack_vacuum_truncate(self): "vacuum t_heap") self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -639,7 +634,6 @@ def test_ptrack_get_block(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -1009,7 +1003,6 @@ def test_page_pgpro417(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") node.safe_psql("postgres", "SELECT * FROM t_heap") - self.backup_node(backup_dir, 'node', node) # PAGE BACKUP node.safe_psql( @@ -1145,7 +1138,6 @@ def test_create_db(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -1174,8 +1166,7 @@ def test_create_db(self): # PTRACK BACKUP backup_id = self.backup_node( backup_dir, 'node', node, - backup_type='ptrack', - options=["--stream"]) + backup_type='ptrack', options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1267,7 +1258,6 @@ def test_create_db_on_replica(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -1286,14 +1276,13 @@ def test_create_db_on_replica(self): replica.cleanup() self.backup_node( - backup_dir, 'node', node, options=['-j10']) + backup_dir, 'node', node, options=['-j10', '--stream']) self.restore_node(backup_dir, 'node', replica) # Add replica self.add_instance(backup_dir, 'replica', replica) self.set_replica(node, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) self.backup_node( @@ -1750,7 +1739,6 @@ def test_ptrack_multiple_segments(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -1763,7 +1751,7 @@ def test_ptrack_multiple_segments(self): # CREATE TABLE node.pgbench_init(scale=100, options=['--tablespace=somedata']) # FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, options=['--stream']) # PTRACK STUFF if node.major_version < 12: @@ -1801,7 +1789,7 @@ def test_ptrack_multiple_segments(self): result = node.safe_psql("postgres", "select * from pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -1921,7 +1909,6 @@ def test_ptrack_clean(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -1971,7 +1958,7 @@ def test_ptrack_clean(self): # Take PTRACK backup to clean every ptrack backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['-j10']) + backup_dir, 'node', node, backup_type='ptrack', options=['-j10', '--stream']) node.safe_psql('postgres', 'checkpoint') @@ -1997,7 +1984,7 @@ def test_ptrack_clean(self): # Take PAGE backup to clean every ptrack self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['-j10']) + backup_type='page', options=['-j10', '--stream']) node.safe_psql('postgres', 'checkpoint') for i in idx_ptrack: @@ -2049,7 +2036,6 @@ def test_ptrack_clean_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -2312,7 +2298,6 @@ def test_ptrack_cluster_on_btree_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -2393,7 +2378,6 @@ def test_ptrack_cluster_on_gist_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -2485,7 +2469,6 @@ def test_ptrack_empty(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() if node.major_version >= 12: @@ -2531,7 +2514,7 @@ def test_ptrack_empty(self): # Take PTRACK backup backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', - options=['-j10']) + options=['-j10', '--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -2586,7 +2569,6 @@ def test_ptrack_empty_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table @@ -2766,7 +2748,6 @@ def test_ptrack_truncate_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -2964,7 +2945,6 @@ def test_ptrack_vacuum_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -3151,7 +3131,6 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -3411,7 +3390,6 @@ def test_ptrack_vacuum_full_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -3593,7 +3571,6 @@ def test_ptrack_vacuum_truncate_replica(self): self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) replica.slow_start(replica=True) # Create table and indexes @@ -3754,7 +3731,6 @@ def test_ptrack_recovery_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() node.safe_psql( @@ -3871,4 +3847,203 @@ def test_ptrack_zero_changes(self): # Clean after yourself self.del_test_dir(module_name, fname) -# TODO: ptrack.map corruption + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_pg_resetxlog(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "CREATE INDEX {0} ON {1} USING {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + +# node.safe_psql( +# 'postgres', +# "create extension pg_buffercache") +# +# print(node.safe_psql( +# 'postgres', +# "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='ptrack', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'Insert error message', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(module_name, fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_ptrack_map(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # Let`s do index corruption. ptrack.map, ptrack.map.mmap + with open(os.path.join(node.data_dir, 'global', 'ptrack.map'), "rb+", 0) as f: + f.seek(42) + f.write(b"blablahblahs") + f.flush() + f.close + + with open(os.path.join(node.data_dir, 'global', 'ptrack.map.mmap'), "rb+", 0) as f: + f.seek(42) + f.write(b"blablahblahs") + f.flush() + f.close + +# os.remove(os.path.join(node.logs_dir, node.pg_log_name)) + + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From eccd847d3028a5be44140e08cd4e2b80fe9fe21c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 25 Dec 2019 17:43:12 +0300 Subject: [PATCH 1099/2107] tests: added delta.DeltaTest.test_delta_pg_resetxlog and page.PageTest.test_page_pg_resetxlog --- tests/delta.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++ tests/page.py | 106 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 1 deletion(-) diff --git a/tests/delta.py b/tests/delta.py index 2e91ef91d..4447b9a81 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1335,3 +1335,106 @@ def test_delta_backup_from_past(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_delta_pg_resetxlog(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='delta', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'Insert error message', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(module_name, fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py index 7a4ae3519..54e601271 100644 --- a/tests/page.py +++ b/tests/page.py @@ -10,7 +10,7 @@ module_name = 'page' -class PageBackupTest(ProbackupTest, unittest.TestCase): +class PageTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_basic_page_vacuum_truncate(self): @@ -1111,3 +1111,107 @@ def test_page_create_db(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_page_pg_resetxlog(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + self.switch_wal_segment(node) + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='page', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'Insert error message', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(module_name, fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 2e6aa530ad7a6cd86ce391e7c0050540a7b159be Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 26 Dec 2019 00:08:57 +0300 Subject: [PATCH 1100/2107] [Issue #150] added set_backup.SetBackupTest.test_wal_retention_and_pinning and set_backup.SetBackupTest.test_wal_retention_and_pinning_1 --- src/catalog.c | 5 ++ tests/retention.py | 2 +- tests/set_backup.py | 141 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 3377c53f8..db92e7931 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1004,7 +1004,12 @@ catalog_get_timelines(InstanceConfig *instance) */ if (backup->expire_time > 0 && backup->expire_time > current_time) + { + elog(LOG, "Pinned backup %s is ignored for the " + "purpose of WAL retention", + base36enc(backup->start_time)); continue; + } count++; diff --git a/tests/retention.py b/tests/retention.py index adbbd18a4..8f429db22 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1629,7 +1629,7 @@ def test_failed_merge_redundancy_retention(self): self.del_test_dir(module_name, fname) # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_wal_depth(self): """ ARCHIVE replica: diff --git a/tests/set_backup.py b/tests/set_backup.py index 52807f752..5e98e9181 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -239,3 +239,144 @@ def test_retention_window_pinning(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_wal_retention_and_pinning(self): + """ + B1---B2---P---B3---> + wal-depth=2 + P - pinned backup + + expected result after WAL purge: + B1 B2---P---B3---> + + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUP + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + # Take PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + node.pgbench_init(scale=1) + + # Take DELTA BACKUP and pin it + expire_time = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + backup_id_pinned = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + '--stream', + '--expire-time={0}'.format(expire_time)]) + + node.pgbench_init(scale=1) + + # Take second PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + node.pgbench_init(scale=1) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=LOG', + '--delete-wal', '--wal-depth=2']) + + # print(out) + self.assertIn( + 'Pinned backup {0} is ignored for the ' + 'purpose of WAL retention'.format(backup_id_pinned), + out) + + for instance in self.show_archive(backup_dir): + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual(timeline['min-segno'], '0000000000000004') + self.assertEqual(timeline['status'], 'OK') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_wal_retention_and_pinning_1(self): + """ + P---B1---> + wal-depth=2 + P - pinned backup + + expected result after WAL purge: + P---B1---> + + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + expire_time = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + + # take FULL BACKUP + backup_id_pinned = self.backup_node( + backup_dir, 'node', node, + options=['--expire-time={0}'.format(expire_time)]) + + node.pgbench_init(scale=2) + + # Take second PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + node.pgbench_init(scale=2) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=verbose', + '--delete-wal', '--wal-depth=2']) + + print(out) + self.assertIn( + 'Pinned backup {0} is ignored for the ' + 'purpose of WAL retention'.format(backup_id_pinned), + out) + + for instance in self.show_archive(backup_dir): + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual(timeline['min-segno'], '0000000000000002') + self.assertEqual(timeline['status'], 'OK') + + self.validate_pb(backup_dir) + + # Clean after yourself + self.del_test_dir(module_name, fname) \ No newline at end of file From 4f10eb3c59c4ad3f553e8bb9d43eba8937914237 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 26 Dec 2019 01:03:06 +0300 Subject: [PATCH 1101/2107] [Issue #150] Documentation update --- doc/pgprobackup.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 68037e2ec..52f8c43ad 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3097,6 +3097,12 @@ ARCHIVE INSTANCE 'node' =============================================================================================================== 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK + + + Pinned backups are + ignored for the purpose of WAL Archive Retention Policy fulfilment. + + From 2b23095c101f142c179c62005ec47b7bac082740 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 27 Dec 2019 00:38:55 +0300 Subject: [PATCH 1102/2107] [Issue #155] Use full filename as segno --- src/catalog.c | 12 ++++++------ src/delete.c | 12 ++++++------ src/show.c | 22 +++++++++++----------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index db88ab172..0421d346a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -697,8 +697,8 @@ catalog_get_timelines(InstanceConfig *instance) char arclog_path[MAXPGPATH]; /* for fancy reporting */ - char begin_segno_str[20]; - char end_segno_str[20]; + char begin_segno_str[XLOG_FNAME_LEN]; + char end_segno_str[XLOG_FNAME_LEN]; /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); @@ -1119,8 +1119,8 @@ catalog_get_timelines(InstanceConfig *instance) * covered by other larger interval. */ - GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size); - GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size); + GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); + GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); elog(LOG, "Timeline %i to stay reachable from timeline %i " "protect from purge WAL interval between " @@ -1174,8 +1174,8 @@ catalog_get_timelines(InstanceConfig *instance) else interval->end_segno = segno; - GetXLogSegName(begin_segno_str, interval->begin_segno, instance->xlog_seg_size); - GetXLogSegName(end_segno_str, interval->end_segno, instance->xlog_seg_size); + GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); + GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); elog(LOG, "Archive backup %s to stay consistent " "protect from purge WAL interval " diff --git a/src/delete.c b/src/delete.c index be07554dc..5fc75bca2 100644 --- a/src/delete.c +++ b/src/delete.c @@ -810,8 +810,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { XLogSegNo FirstToDeleteSegNo; XLogSegNo OldestToKeepSegNo = 0; - char first_to_del_str[20]; - char oldest_to_keep_str[20]; + char first_to_del_str[XLOG_FNAME_LEN]; + char oldest_to_keep_str[XLOG_FNAME_LEN]; int rc; int i; size_t wal_size_logical = 0; @@ -846,8 +846,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo) { /* translate segno number into human readable format */ - GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size); - GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size); + GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size); + GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size); elog(INFO, "On timeline %i WAL segments between %s and %s %s be removed", tlinfo->tli, first_to_del_str, @@ -874,8 +874,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0) { - GetXLogSegName(first_to_del_str, FirstToDeleteSegNo, xlog_seg_size); - GetXLogSegName(oldest_to_keep_str, OldestToKeepSegNo, xlog_seg_size); + GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size); + GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size); elog(LOG, "On timeline %i first segment %s is greater than oldest segment to keep %s", tlinfo->tli, first_to_del_str, oldest_to_keep_str); diff --git a/src/show.c b/src/show.c index ed0cca16f..ed4828bd2 100644 --- a/src/show.c +++ b/src/show.c @@ -43,8 +43,8 @@ typedef struct ShowArchiveRow char tli[20]; char parent_tli[20]; char switchpoint[20]; - char min_segno[20]; - char max_segno[20]; + char min_segno[XLOG_FNAME_LEN+1]; + char max_segno[XLOG_FNAME_LEN+1]; char n_segments[20]; char size[20]; char zratio[20]; @@ -749,7 +749,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *tli_list, bool show_name) { - char segno_tmp[20]; + char segno_tmp[XLOG_FNAME_LEN]; parray *actual_tli_list = parray_new(); #define SHOW_ARCHIVE_FIELDS_COUNT 10 int i; @@ -808,14 +808,14 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, cur++; /* Min Segno */ - GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size); snprintf(row->min_segno, lengthof(row->min_segno), "%s",segno_tmp); widths[cur] = Max(widths[cur], strlen(row->min_segno)); cur++; /* Max Segno */ - GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size); snprintf(row->max_segno, lengthof(row->max_segno), "%s", segno_tmp); widths[cur] = Max(widths[cur], strlen(row->max_segno)); @@ -940,7 +940,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, int i,j; PQExpBuffer buf = &show_buf; parray *actual_tli_list = parray_new(); - char segno_tmp[20]; + char segno_tmp[XLOG_FNAME_LEN+1]; if (!first_instance) appendPQExpBufferChar(buf, ','); @@ -969,7 +969,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) { timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); - char tmp_buf[20]; + char tmp_buf[XLOG_FNAME_LEN+1]; float zratio = 0; if (i != (parray_num(actual_tli_list) - 1)) @@ -987,11 +987,11 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, (uint32) (tlinfo->switchpoint >> 32), (uint32) tlinfo->switchpoint); json_add_value(buf, "switchpoint", tmp_buf, json_level, true); - GetXLogSegName(segno_tmp, tlinfo->begin_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size); snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "min-segno", tmp_buf, json_level, true); - GetXLogSegName(segno_tmp, tlinfo->end_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size); snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "max-segno", tmp_buf, json_level, true); @@ -1034,11 +1034,11 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, json_add(buf, JT_BEGIN_OBJECT, &json_level); - GetXLogSegName(segno_tmp, lost_segments->begin_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->begin_segno, xlog_seg_size); snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "begin-segno", tmp_buf, json_level, true); - GetXLogSegName(segno_tmp, lost_segments->end_segno, xlog_seg_size); + GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->end_segno, xlog_seg_size); snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); json_add_value(buf, "end-segno", tmp_buf, json_level, true); json_add(buf, JT_END_OBJECT, &json_level); From 6dc2eaf9db642db486a0cb600757cd6d52d3bce0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 27 Dec 2019 00:39:35 +0300 Subject: [PATCH 1103/2107] [Issue #155] tests fixes --- tests/archive.py | 85 ++++++++++++++++++++++++++++++++++++++++++---- tests/delete.py | 8 ++--- tests/retention.py | 40 +++++++++++----------- 3 files changed, 102 insertions(+), 31 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 5650afe88..16f9aa8be 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1413,10 +1413,18 @@ def test_archive_catalog(self): self.assertEqual(timeline_1['status'], "OK") self.assertEqual(len(timeline_3['lost-segments']), 2) - self.assertEqual(timeline_3['lost-segments'][0]['begin-segno'], '0000000000000012') - self.assertEqual(timeline_3['lost-segments'][0]['end-segno'], '0000000000000013') - self.assertEqual(timeline_3['lost-segments'][1]['begin-segno'], '0000000000000017') - self.assertEqual(timeline_3['lost-segments'][1]['end-segno'], '0000000000000017') + self.assertEqual( + timeline_3['lost-segments'][0]['begin-segno'], + '000000030000000000000012') + self.assertEqual( + timeline_3['lost-segments'][0]['end-segno'], + '000000030000000000000013') + self.assertEqual( + timeline_3['lost-segments'][1]['begin-segno'], + '000000030000000000000017') + self.assertEqual( + timeline_3['lost-segments'][1]['end-segno'], + '000000030000000000000017') self.assertEqual(len(timeline_6['backups']), 0) self.assertEqual(len(timeline_5['backups']), 0) @@ -1487,7 +1495,9 @@ def test_archive_catalog_1(self): # sanity for timeline in timelines: - self.assertEqual(timeline['min-segno'], '0000000000000001') + self.assertEqual( + timeline['min-segno'], + '000000010000000000000001') self.assertEqual(timeline['status'], 'OK') self.del_test_dir(module_name, fname) @@ -1539,7 +1549,9 @@ def test_archive_catalog_2(self): # sanity for timeline in timelines: - self.assertEqual(timeline['min-segno'], '0000000000000002') + self.assertEqual( + timeline['min-segno'], + '000000010000000000000002') self.assertEqual(timeline['status'], 'OK') self.del_test_dir(module_name, fname) @@ -1690,8 +1702,67 @@ def test_archive_options_1(self): self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_hexadecimal_timeline(self): + """ + Check that timelines are correct. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + # create timelines + for i in range(1, 13): + # print(i) + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + node.pgbench_init(scale=2) + + show = self.show_archive(backup_dir) + + timelines = show[0]['timelines'] + + print(timelines[0]) + + tli13 = timelines[0] + + self.assertEqual( + 13, + tli13['tli']) + + self.assertEqual( + 12, + tli13['parent-tli']) + + self.assertEqual( + backup_id, + tli13['closest-backup-id']) + + self.assertEqual( + '0000000D000000000000001B', + tli13['max-segno']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. -# so write validation code accordingly +# so write WAL validation code accordingly # change wal-seg-size # diff --git a/tests/delete.py b/tests/delete.py index d6e9b2db7..389584e8a 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -766,8 +766,8 @@ def test_delete_multiple_descendants_dry_run(self): output) self.assertIn( - 'On timeline 1 WAL segments between 0000000000000001 ' - 'and 0000000000000003 can be removed', + 'On timeline 1 WAL segments between 000000010000000000000001 ' + 'and 000000010000000000000003 can be removed', output) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) @@ -791,8 +791,8 @@ def test_delete_multiple_descendants_dry_run(self): output) self.assertIn( - 'On timeline 1 WAL segments between 0000000000000001 ' - 'and 0000000000000003 will be removed', + 'On timeline 1 WAL segments between 000000010000000000000001 ' + 'and 000000010000000000000003 will be removed', output) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) diff --git a/tests/retention.py b/tests/retention.py index adbbd18a4..d6ac66531 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -63,10 +63,10 @@ def test_retention_redundancy_1(self): if not wal_name.endswith(".backup"): if self.archive_compress: - wal_name = wal_name[-19:] + wal_name = wal_name[-27:] wal_name = wal_name[:-3] else: - wal_name = wal_name[-16:] + wal_name = wal_name[-24:] self.assertTrue(wal_name >= min_wal) self.assertTrue(wal_name <= max_wal) @@ -2234,8 +2234,8 @@ def test_wal_purge(self): '--log-level-console=verbose']) self.assertIn( - 'INFO: On timeline 4 WAL segments between 0000000000000002 ' - 'and 0000000000000006 can be removed', + 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' + 'and 000000040000000000000006 can be removed', output) self.assertIn( @@ -2259,8 +2259,8 @@ def test_wal_purge(self): options=['--delete-wal', '--log-level-console=verbose']) self.assertIn( - 'INFO: On timeline 4 WAL segments between 0000000000000002 ' - 'and 0000000000000006 will be removed', + 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' + 'and 000000040000000000000006 will be removed', output) self.assertIn( @@ -2281,11 +2281,11 @@ def test_wal_purge(self): self.assertEqual( show_tli4_before['min-segno'], - '0000000000000002') + '000000040000000000000002') self.assertEqual( show_tli4_after['min-segno'], - '0000000000000006') + '000000040000000000000006') self.assertFalse(show_tli5_after) @@ -2461,8 +2461,8 @@ def test_wal_depth_2(self): self.assertIn( 'LOG: Archive backup {0} to stay consistent protect from ' - 'purge WAL interval between 0000000000000004 and 0000000000000004 ' - 'on timeline 1'.format(B1), output) + 'purge WAL interval between 000000010000000000000004 ' + 'and 000000010000000000000004 on timeline 1'.format(B1), output) start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] self.assertIn( @@ -2471,13 +2471,13 @@ def test_wal_depth_2(self): self.assertIn( 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' - 'from purge WAL interval between 0000000000000005 and ' - '0000000000000008 on timeline 2', output) + 'from purge WAL interval between 000000020000000000000005 and ' + '000000020000000000000008 on timeline 2', output) self.assertIn( 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' - 'from purge WAL interval between 0000000000000004 and ' - '0000000000000005 on timeline 1', output) + 'from purge WAL interval between 000000010000000000000004 and ' + '000000010000000000000005 on timeline 1', output) show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) @@ -2509,11 +2509,11 @@ def test_wal_depth_2(self): self.assertEqual( show_tli4_before['min-segno'], - '0000000000000002') + '000000040000000000000002') self.assertEqual( show_tli4_after['min-segno'], - '0000000000000006') + '000000040000000000000006') self.assertFalse(show_tli5_after) @@ -2528,19 +2528,19 @@ def test_wal_depth_2(self): self.assertEqual( show_tli1_after['lost-segments'][0]['begin-segno'], - '0000000000000006') + '000000010000000000000006') self.assertEqual( show_tli1_after['lost-segments'][0]['end-segno'], - '0000000000000009') + '000000010000000000000009') self.assertEqual( show_tli2_after['lost-segments'][0]['begin-segno'], - '0000000000000009') + '000000020000000000000009') self.assertEqual( show_tli2_after['lost-segments'][0]['end-segno'], - '0000000000000009') + '000000020000000000000009') self.validate_pb(backup_dir, 'node') From b08e9b468e88dde203f5fce566384eb2eb53aead Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 27 Dec 2019 00:53:28 +0300 Subject: [PATCH 1104/2107] [Issue #155] Documentation update --- doc/pgprobackup.xml | 80 ++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 68037e2ec..d2adef5b8 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2438,14 +2438,14 @@ pg_probackup show -B backup_dir [--instance ARCHIVE INSTANCE 'node' -=================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=================================================================================================================== - 5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK - 4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK - 3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK - 2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED - 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK +=================================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=================================================================================================================================== + 5 1 0/B000000 00000005000000000000000B 00000005000000000000000C 2 685kB 48.00 0 OK + 4 3 0/18000000 000000040000000000000018 00000004000000000000001A 3 648kB 77.00 0 OK + 3 2 0/15000000 000000030000000000000015 000000030000000000000017 3 648kB 77.00 0 OK + 2 1 0/B000108 00000002000000000000000B 000000020000000000000015 5 892kB 94.00 1 DEGRADED + 1 0 0/0 000000010000000000000001 00000001000000000000000A 10 8774kB 19.00 1 OK For each timeline, the following information is provided: @@ -2543,8 +2543,8 @@ pg_probackup show -B backup_dir [--instance backup_dir [--instance backup_dir [--instance backup_dir [--instance backup_dir [--instance backup_dir [--instance backup_dir [--instance backup_dir --instance node --arc ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK +=============================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================================== + 1 0 0/0 000000010000000000000001 000000010000000000000047 71 36MB 31.00 6 OK WAL purge without cannot @@ -3063,10 +3063,10 @@ pg_probackup delete -B backup_dir --instance node --d ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK +=============================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================================== + 1 0 0/0 000000010000000000000002 000000010000000000000047 70 34MB 32.00 6 OK If you would like, for example, to keep only those WAL @@ -3078,10 +3078,10 @@ pg_probackup delete -B backup_dir --instance node --d ARCHIVE INSTANCE 'node' -================================================================================================================ - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -================================================================================================================ - 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000046 000000010000000000000047 2 143kB 228.00 6 OK Alternatively, you can use the @@ -3092,10 +3092,10 @@ pg_probackup backup -B backup_dir --instance node -b ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK +=============================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================================== + 1 0 0/0 000000010000000000000048 000000010000000000000049 1 72kB 228.00 7 OK From ee3e8d11e34c1d7502fb9ef94df5bf3999684ff4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 27 Dec 2019 01:17:22 +0300 Subject: [PATCH 1105/2107] [Issue #165] Report errno correctly when failed to access instance directory --- src/pg_probackup.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1b2a2292c..32260d930 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -497,8 +497,14 @@ main(int argc, char *argv[]) if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) { if (fio_access(backup_instance_path, F_OK, FIO_BACKUP_HOST) != 0) + { + elog(WARNING, "Failed to access directory \"%s\": %s", + backup_instance_path, strerror(errno)); + + // TODO: redundant message, should we get rid of it? elog(ERROR, "Instance '%s' does not exist in this backup catalog", instance_name); + } } } From 1f4dc1bf4c1993ae6fabd7452d7bae4396b914c1 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Fri, 27 Dec 2019 17:27:04 +0300 Subject: [PATCH 1106/2107] DOC: fix for PGPRO-3397 and PGPRO-3398 --- doc/pgprobackup.xml | 60 +++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index aa7813ed2..6e2eed7df 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1208,7 +1208,11 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint) - For older PostgreSQL versions: + For older PostgreSQL versions, + PTRACK required taking backups in the exclusive mode + to provide exclusive access to bitmaps with changed blocks. + To set up PTRACK backups for PostgreSQL 11 + or lower, do the following: @@ -1227,6 +1231,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint) GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; @@ -4941,6 +4946,19 @@ pg_probackup archive-get -B backup_dir --instance + + + backup — PostgreSQL + role used for connection to PostgreSQL + cluster. + + + + + backupdb — database used for connection + to PostgreSQL cluster. + + backup_host — host with backup catalog. @@ -4977,12 +4995,6 @@ pg_probackup archive-get -B backup_dir --instance postgres_host. - - - backupdb — database used for connection - to PostgreSQL cluster. - - Minimal Setup @@ -5016,19 +5028,19 @@ CREATE DATABASE backupdb; backupdb=# BEGIN; -CREATE ROLE probackup WITH LOGIN REPLICATION; -GRANT USAGE ON SCHEMA pg_catalog TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO probackup; +CREATE ROLE backup WITH LOGIN REPLICATION; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; @@ -5049,7 +5061,7 @@ INFO: Instance 'node' successfully inited Take a FULL backup: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Start transferring data files INFO: Data files are transferred @@ -5076,7 +5088,7 @@ BACKUP INSTANCE 'pg-11' Take an incremental backup in the DELTA mode: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Parent backup: PZ7YK2 INFO: Start transferring data files @@ -5093,7 +5105,7 @@ INFO: Backup PZ7YMP completed Let's add some parameters to <application>pg_probackup</application> configuration file, so that you can omit them from the command line: -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb +[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb @@ -5125,7 +5137,7 @@ xlog-seg-size = 16777216 # Connection parameters pgdatabase = backupdb pghost = postgres_host -pguser = probackup +pguser = backup # Replica parameters replica-timeout = 5min # Archive parameters From c6c605345870044aab4a77f2fca7d72d0bb6edae Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 27 Dec 2019 19:56:44 +0300 Subject: [PATCH 1107/2107] [Issue #159] pgdata_bytes for FULL and DELTA remote backups was calculated incorrectly --- src/backup.c | 43 +++++++++++++++++++++---------------------- src/data.c | 5 ++++- src/utils/file.c | 1 - 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/backup.c b/src/backup.c index 648a97dfd..40900c475 100644 --- a/src/backup.c +++ b/src/backup.c @@ -155,6 +155,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* for fancy reporting */ time_t start_time, end_time; + char pretty_bytes[20]; elog(LOG, "Database backup start"); if(current.external_dir_str) @@ -330,6 +331,20 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) elog(ERROR, "PGDATA is almost empty. Either it was concurrently deleted or " "pg_probackup do not possess sufficient permissions to list PGDATA content"); + /* Calculate pgdata_bytes */ + for (i = 0; i < parray_num(backup_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + if (file->external_dir_num != 0) + continue; + + current.pgdata_bytes += file->size; + } + + pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); + elog(INFO, "PGDATA size: %s", pretty_bytes); + /* * Sort pathname ascending. It is necessary to create intermediate * directories sequentially. @@ -547,22 +562,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* close ssh session in main thread */ fio_disconnect(); - /* Calculate pgdata_bytes */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - /* In case of FULL or DELTA backup we can trust read_size. - * In case of PAGE or PTRACK we are forced to trust datafile size, - * taken at the start of backup. - */ - if (current.backup_mode == BACKUP_MODE_FULL || - current.backup_mode == BACKUP_MODE_DIFF_DELTA) - current.pgdata_bytes += file->read_size; - else - current.pgdata_bytes += file->size; - } - /* Add archived xlog files into the list of files of this backup */ if (stream_wal) { @@ -699,9 +698,9 @@ int do_backup(time_t start_time, bool no_validate, pgSetBackupParams *set_backup_params) { - PGconn *backup_conn = NULL; - PGNodeInfo nodeInfo; - char pretty_data_bytes[20]; + PGconn *backup_conn = NULL; + PGNodeInfo nodeInfo; + char pretty_bytes[20]; /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); @@ -836,10 +835,10 @@ do_backup(time_t start_time, bool no_validate, /* Notify user about backup size */ if (current.stream) - pretty_size(current.data_bytes + current.wal_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); + pretty_size(current.data_bytes + current.wal_bytes, pretty_bytes, lengthof(pretty_bytes)); else - pretty_size(current.data_bytes, pretty_data_bytes, lengthof(pretty_data_bytes)); - elog(INFO, "Backup %s resident size: %s", base36enc(current.start_time), pretty_data_bytes); + pretty_size(current.data_bytes, pretty_bytes, lengthof(pretty_bytes)); + elog(INFO, "Backup %s resident size: %s", base36enc(current.start_time), pretty_bytes); if (current.status == BACKUP_STATUS_OK || current.status == BACKUP_STATUS_DONE) diff --git a/src/data.c b/src/data.c index de5c58079..a68197cd1 100644 --- a/src/data.c +++ b/src/data.c @@ -453,7 +453,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, char write_buffer[BLCKSZ+sizeof(header)]; char compressed_page[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ - if(page_state == SkipCurrentPage) + if (page_state == SkipCurrentPage) return; header.block = blknum; @@ -641,6 +641,7 @@ backup_data_file(backup_files_arg* arguments, file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); n_blocks_read = rc; + file->read_size = n_blocks_read * BLCKSZ; file->uncompressed_size = (n_blocks_read - n_blocks_skipped)*BLCKSZ; } else @@ -658,6 +659,8 @@ backup_data_file(backup_files_arg* arguments, n_blocks_read++; if (page_state == PageIsTruncated) break; + + file->read_size += BLCKSZ; } } if (backup_mode == BACKUP_MODE_DIFF_DELTA) diff --git a/src/utils/file.c b/src/utils/file.c index 0ee7aaac2..a2bed130a 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1236,7 +1236,6 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, blknum += 1; break; } - file->read_size += BLCKSZ; } *nBlocksSkipped = blknum - n_blocks_read; return blknum; From 489c447862e1b454f098e405eb7f592b04b425c0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 28 Dec 2019 14:18:02 +0300 Subject: [PATCH 1108/2107] DOC: for ptrack 2.0 functions schema name depends on extension schema --- doc/pgprobackup.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 6e2eed7df..da69cf191 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1190,9 +1190,9 @@ CREATE EXTENSION ptrack; in the database used to connect to the cluster: -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_pagemapset(pg_lsn) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_control_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; +GRANT EXECUTE ON FUNCTION pg_ptrack_get_pagemapset(pg_lsn) TO backup; +GRANT EXECUTE ON FUNCTION pg_ptrack_control_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; From d785bea5fb2169eb3a6bdc7de49184528beea390 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 28 Dec 2019 14:41:20 +0300 Subject: [PATCH 1109/2107] fix windows build --- gen_probackup_project.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index 4189b1b07..cf82543d0 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -169,7 +169,8 @@ sub build_pgprobackup 'show.c', 'util.c', 'validate.c', - 'checkdb.c' + 'checkdb.c', + 'ptrack.c' ); $probackup->AddFiles( "$currpath/src/utils", From 68922f2151ea2fe927a458c31eeb249c0fd034ba Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 28 Dec 2019 18:53:33 +0300 Subject: [PATCH 1110/2107] tests: various fixes --- tests/merge.py | 16 ++++++++++++++++ tests/restore.py | 1 + tests/set_backup.py | 8 ++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 541c5173c..1f5538193 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1865,6 +1865,10 @@ def test_merge_backup_from_future(self): backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) + result = node.safe_psql( + 'postgres', + 'SELECT * from pgbench_accounts') + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -1883,6 +1887,18 @@ def test_merge_backup_from_future(self): backup_dir, 'node', node_restored, backup_id=backup_id) pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf( + node_restored, + {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + 'postgres', + 'SELECT * from pgbench_accounts') + + self.assertTrue(result, result_new) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself diff --git a/tests/restore.py b/tests/restore.py index 115da13b9..918800bc2 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2748,6 +2748,7 @@ def test_partial_restore_include(self): def test_partial_restore_backward_compatibility_1(self): """ + old binary should be version < 2.2.0 """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/set_backup.py b/tests/set_backup.py index 5e98e9181..d93d0d71b 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -311,7 +311,9 @@ def test_wal_retention_and_pinning(self): # sanity for timeline in timelines: - self.assertEqual(timeline['min-segno'], '0000000000000004') + self.assertEqual( + timeline['min-segno'], + '000000010000000000000004') self.assertEqual(timeline['status'], 'OK') # Clean after yourself @@ -373,7 +375,9 @@ def test_wal_retention_and_pinning_1(self): # sanity for timeline in timelines: - self.assertEqual(timeline['min-segno'], '0000000000000002') + self.assertEqual( + timeline['min-segno'], + '000000010000000000000002') self.assertEqual(timeline['status'], 'OK') self.validate_pb(backup_dir) From b0efb76003bedcde95fc96e96c82f34993bc410d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 29 Dec 2019 02:11:10 +0300 Subject: [PATCH 1111/2107] use portable unlink in delete_walfiles_in_tli() --- src/delete.c | 2 +- src/utils/file.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index a1ee2948f..dcbc5645c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -933,7 +933,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, } /* unlink segment */ - rc = unlink(wal_file->file.path); + rc = fio_unlink(wal_file->file.path, FIO_BACKUP_HOST); if (rc < 0) { /* Missing file is not considered as error condition */ diff --git a/src/utils/file.c b/src/utils/file.c index a2bed130a..f7c3eab78 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -816,6 +816,7 @@ int fio_unlink(char const* path, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + // TODO: error is swallowed ? return 0; } else From e1dfb614cf25ee681c128b0b72468ab463480f69 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 29 Dec 2019 14:23:28 +0300 Subject: [PATCH 1112/2107] fix stack buffer overflow --- src/catalog.c | 4 ++-- src/delete.c | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index dffb4302c..4ade1bca6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -697,8 +697,8 @@ catalog_get_timelines(InstanceConfig *instance) char arclog_path[MAXPGPATH]; /* for fancy reporting */ - char begin_segno_str[XLOG_FNAME_LEN]; - char end_segno_str[XLOG_FNAME_LEN]; + char begin_segno_str[MAXFNAMELEN]; + char end_segno_str[MAXFNAMELEN]; /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); diff --git a/src/delete.c b/src/delete.c index dcbc5645c..96ab26fd2 100644 --- a/src/delete.c +++ b/src/delete.c @@ -806,9 +806,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { XLogSegNo FirstToDeleteSegNo; XLogSegNo OldestToKeepSegNo = 0; - char first_to_del_str[XLOG_FNAME_LEN]; - char oldest_to_keep_str[XLOG_FNAME_LEN]; - int rc; + char first_to_del_str[MAXFNAMELEN]; + char oldest_to_keep_str[MAXFNAMELEN]; int i; size_t wal_size_logical = 0; size_t wal_size_actual = 0; @@ -933,8 +932,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, } /* unlink segment */ - rc = fio_unlink(wal_file->file.path, FIO_BACKUP_HOST); - if (rc < 0) + if (fio_unlink(wal_file->file.path, FIO_BACKUP_HOST) < 0) { /* Missing file is not considered as error condition */ if (errno != ENOENT) From 07557566f46e0ec919cd29e4cef0709c6a93b6e8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 29 Dec 2019 14:49:41 +0300 Subject: [PATCH 1113/2107] leftover fix for stack buffer overflow --- src/show.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/show.c b/src/show.c index d0c9e8cbd..a0628790a 100644 --- a/src/show.c +++ b/src/show.c @@ -748,7 +748,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *tli_list, bool show_name) { - char segno_tmp[XLOG_FNAME_LEN]; + char segno_tmp[XLOG_FNAME_LEN+1]; parray *actual_tli_list = parray_new(); #define SHOW_ARCHIVE_FIELDS_COUNT 10 int i; From 2a89a60e2f94ac9c4c4c08f9323e3232bf881d35 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Sun, 29 Dec 2019 17:04:48 +0300 Subject: [PATCH 1114/2107] Fixed call_atexit_callbacks function --- src/utils/pgut.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index cdc90e30f..a050a5128 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -834,9 +834,11 @@ static void call_atexit_callbacks(bool fatal) { pgut_atexit_item *item; - - for (item = pgut_atexit_stack; item; item = item->next) + pgut_atexit_item *next; + for (item = pgut_atexit_stack; item; item = next){ + next = item->next; item->callback(fatal, item->userdata); + } } static void From b7f69ae2067abdde48b405fefaf20d244a4b8b31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 29 Dec 2019 21:41:47 +0300 Subject: [PATCH 1115/2107] tests: update comments --- tests/restore.py | 3 ++- tests/retention.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index 918800bc2..e1d866fcd 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2748,7 +2748,7 @@ def test_partial_restore_include(self): def test_partial_restore_backward_compatibility_1(self): """ - old binary should be version < 2.2.0 + old binary should be of version < 2.2.0 """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -2851,6 +2851,7 @@ def test_partial_restore_backward_compatibility_1(self): def test_partial_restore_backward_compatibility_merge(self): """ + old binary should be of version < 2.2.0 """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/retention.py b/tests/retention.py index 4e5296574..d6ac66531 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1629,7 +1629,7 @@ def test_failed_merge_redundancy_retention(self): self.del_test_dir(module_name, fname) # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_wal_depth(self): """ ARCHIVE replica: From f431452ea94281d33426c539f3c051e4f57d87eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 29 Dec 2019 21:43:23 +0300 Subject: [PATCH 1116/2107] Version 2.2.6 --- README.md | 2 +- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88e52118..f48a90906 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.2.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.5) +[2.2.6](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.6) ## Documentation diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3deb69af7..7b71f7733 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -216,8 +216,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.5" -#define AGENT_PROTOCOL_VERSION 20205 +#define PROGRAM_VERSION "2.2.6" +#define AGENT_PROTOCOL_VERSION 20206 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 29b839daa..c256d255c 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.5 \ No newline at end of file +pg_probackup 2.2.6 \ No newline at end of file From c2f52dc86db70d8f4700ef40972feebbdd9664b3 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 30 Dec 2019 12:29:18 +0300 Subject: [PATCH 1117/2107] Fix for [Issue #155]. Allocate destination buffer size correctly --- src/show.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/show.c b/src/show.c index a0628790a..f2af77255 100644 --- a/src/show.c +++ b/src/show.c @@ -43,8 +43,8 @@ typedef struct ShowArchiveRow char tli[20]; char parent_tli[20]; char switchpoint[20]; - char min_segno[XLOG_FNAME_LEN+1]; - char max_segno[XLOG_FNAME_LEN+1]; + char min_segno[MAXFNAMELEN]; + char max_segno[MAXFNAMELEN]; char n_segments[20]; char size[20]; char zratio[20]; @@ -748,7 +748,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *tli_list, bool show_name) { - char segno_tmp[XLOG_FNAME_LEN+1]; + char segno_tmp[MAXFNAMELEN]; parray *actual_tli_list = parray_new(); #define SHOW_ARCHIVE_FIELDS_COUNT 10 int i; @@ -939,7 +939,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, int i,j; PQExpBuffer buf = &show_buf; parray *actual_tli_list = parray_new(); - char segno_tmp[XLOG_FNAME_LEN+1]; + char segno_tmp[MAXFNAMELEN]; if (!first_instance) appendPQExpBufferChar(buf, ','); @@ -968,7 +968,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) { timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); - char tmp_buf[XLOG_FNAME_LEN+1]; + char tmp_buf[MAXFNAMELEN]; float zratio = 0; if (i != (parray_num(actual_tli_list) - 1)) From 95459a8363685e6a2a746950439a64f78557d492 Mon Sep 17 00:00:00 2001 From: Anastasia Date: Mon, 30 Dec 2019 12:59:14 +0300 Subject: [PATCH 1118/2107] fix buffer size in write_backup_filelist --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 4ade1bca6..6f2720076 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1566,7 +1566,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char path_temp[MAXPGPATH]; int errno_temp; size_t i = 0; - #define BUFFERSZ BLCKSZ*500 + #define BUFFERSZ 1024*1024 char buf[BUFFERSZ]; size_t write_len = 0; int64 backup_size_on_disk = 0; From 31293691127ed21b9fd27d9a15f54e866684fe7b Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Mon, 30 Dec 2019 13:03:12 +0300 Subject: [PATCH 1119/2107] DOC: fixes for PGPRO-3413 --- doc/pgprobackup.xml | 79 ++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index da69cf191..07169e559 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1625,6 +1625,44 @@ pg_probackup restore -B backup_dir --instance + + + Once the restore command is complete, start + the database service. + + + + If you restore ARCHIVE backups, + perform PITR, + or specify the --restore-as-replica option with the + restore command to set up a standby server, + pg_probackup creates a recovery configuration + file once all data files are copied into the target directory. This file + includes the minimal settings required for recovery, except for the password in the + primary_conninfo + parameter; you have to add the password manually, if required. + For PostgreSQL 11 or lower, + recovery settings are written into the recovery.conf + file. Starting from PostgreSQL 12, + pg_probackup writes these settings into + the probackup_recovery.conf file in the data + directory, and then includes them into the + postgresql.auto.conf when the cluster is + is started. + + + + If you are restoring a STREAM backup, the restore is complete + at once, with the cluster returned to a self-consistent state at + the point when the backup was taken. For ARCHIVE backups, + PostgreSQL replays all available archived WAL + segments, so the cluster is restored to the latest state possible + within the current timeline. You can change this behavior by using the + recovery target + options with the restore command, + as explained in . + + If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore @@ -1644,28 +1682,10 @@ pg_probackup restore -B backup_dir --instance pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir + - Once the restore command is complete, start the database - service. - - - If you are restoring a STREAM backup, the restore is complete - at once, with the cluster returned to a self-consistent state at - the point when the backup was taken. For ARCHIVE backups, - PostgreSQL replays all available archived WAL segments, so the - cluster is restored to the latest state possible. You can change - this behavior by using the - recovery target - options with the restore command. Note - that using the recovery - target options when restoring STREAM backup is possible - if the WAL archive is available at least starting from the time - the STREAM backup was taken. - - - To restore the cluster on a remote host, see the section - Using - pg_probackup in the Remote Mode. + To restore the cluster on a remote host, follow the instructions in + . @@ -1744,15 +1764,18 @@ pg_probackup restore -B backup_dir --instance recovery target options with the - and - commands. + command. + + You can use both STREAM and ARCHIVE backups for point in time + recovery as long as the WAL archive is available at least starting + from the time the backup was taken. If / option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore - backup_id to the specified recovery target. + the specified backup to the specified recovery target. @@ -3585,13 +3608,17 @@ pg_probackup restore -B backup_dir --instance Creates a minimal recovery configuration file to facilitate setting up a standby server. If the replication connection requires a password, - you must specify the password manually as it is not included. + you must specify the password manually in the + primary_conninfo + parameter as it is not included. For PostgreSQL 11 or lower, recovery settings are written into the recovery.conf file. Starting from PostgreSQL 12, pg_probackup writes these settings into the probackup_recovery.conf file in the data - directory. + directory, and then includes them into the + postgresql.auto.conf when the cluster is + is started. From 6ad38c62ce29c64d30b485e96ed4e9574d7ef0f7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 2 Jan 2020 20:51:50 +0300 Subject: [PATCH 1120/2107] use IsXLogFileName() when parsing xlog filenames --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 6f2720076..5d731f2d6 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -717,7 +717,7 @@ catalog_get_timelines(InstanceConfig *instance) xlogFile *wal_file = NULL; /* regular WAL file */ - if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) + if (IsXLogFileName(file->name)) { int result = 0; uint32 log, seg; From 9cce8639b4a0fd915f71a3aed071dfb8378184a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 4 Jan 2020 14:48:10 +0300 Subject: [PATCH 1121/2107] [Issue #168] use heap variable as buffer in write_backup_filelist() instead of stack variable --- src/catalog.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 5d731f2d6..44c30283c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1567,7 +1567,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int errno_temp; size_t i = 0; #define BUFFERSZ 1024*1024 - char buf[BUFFERSZ]; + char *buf; size_t write_len = 0; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; @@ -1581,6 +1581,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, strerror(errno)); + buf = pgut_malloc(BUFFERSZ); + /* print each file in the list */ while(i < parray_num(files)) { @@ -1693,6 +1695,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, backup->data_bytes = backup_size_on_disk; backup->wal_bytes = wal_size_on_disk; backup->uncompressed_bytes = uncompressed_size_on_disk; + + free(buf); } /* From 9af67ed097513609a2a23e4fd4bb7a216967eb49 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 4 Jan 2020 19:50:35 +0300 Subject: [PATCH 1122/2107] Revert "use IsXLogFileName() when parsing xlog filenames" This reverts commit 6ad38c62ce29c64d30b485e96ed4e9574d7ef0f7. --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 44c30283c..ae302d487 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -717,7 +717,7 @@ catalog_get_timelines(InstanceConfig *instance) xlogFile *wal_file = NULL; /* regular WAL file */ - if (IsXLogFileName(file->name)) + if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) { int result = 0; uint32 log, seg; From 20d09c47867ae847b31a1c57846e8f3a36e3da00 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 4 Jan 2020 19:52:11 +0300 Subject: [PATCH 1123/2107] Add some comments --- src/catalog.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index ae302d487..1fe8e4b2e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -716,7 +716,10 @@ catalog_get_timelines(InstanceConfig *instance) parray *timelines; xlogFile *wal_file = NULL; - /* regular WAL file */ + /* + * Regular WAL file. + * IsXLogFileName() cannot be used here + */ if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) { int result = 0; From 9830b629e7b7d52582bb4884cb0d63adc2b24f3c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 Jan 2020 21:15:53 +0300 Subject: [PATCH 1124/2107] tests: fixes for 9.5 --- tests/backup.py | 3 +++ tests/restore.py | 1 + tests/set_backup.py | 1 + 3 files changed, 5 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 44b69b60e..bc8f89ba0 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1613,6 +1613,7 @@ def test_backup_with_least_privileges_role(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " @@ -1644,6 +1645,7 @@ def test_backup_with_least_privileges_role(self): "CREATE ROLE backup WITH LOGIN REPLICATION; " "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " @@ -1680,6 +1682,7 @@ def test_backup_with_least_privileges_role(self): "CREATE ROLE backup WITH LOGIN REPLICATION; " "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack diff --git a/tests/restore.py b/tests/restore.py index e1d866fcd..7ff4465c1 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3114,6 +3114,7 @@ def test_missing_database_map(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " diff --git a/tests/set_backup.py b/tests/set_backup.py index d93d0d71b..b8d97ad5a 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -254,6 +254,7 @@ def test_wal_retention_and_pinning(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 00ea17704820f413763b160cb0ba28303c43203a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 Jan 2020 21:18:28 +0300 Subject: [PATCH 1125/2107] Version 2.2.7 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7b71f7733..af782f1dc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -216,8 +216,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.6" -#define AGENT_PROTOCOL_VERSION 20206 +#define PROGRAM_VERSION "2.2.7" +#define AGENT_PROTOCOL_VERSION 20207 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index c256d255c..dd345e045 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.6 \ No newline at end of file +pg_probackup 2.2.7 \ No newline at end of file From 5a069739c4f70c0b52b60170372ffebe90cf340e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 10 Jan 2020 17:01:28 +0300 Subject: [PATCH 1126/2107] Readme: update current release version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f48a90906..b64ea8db5 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.2.6](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.6) +[2.2.7](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.7) ## Documentation From c6e7490caf8e6c82b02781421e4f90617fd0c788 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 13 Jan 2020 13:22:50 +0300 Subject: [PATCH 1127/2107] Readme: update installation instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b64ea8db5..155ae74e3 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ sudo apt-get install pg-probackup-{12,11,10,9.6,9.5}-dbg #DEB-SRC Packages sudo echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ - /etc/apt/sources.list.d/pg_probackup.list + /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update sudo apt-get source pg-probackup-{12,11,10,9.6,9.5} #RPM Centos Packages From 7f983c599af1397812040d8219a590558e3524a7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Jan 2020 02:58:40 +0300 Subject: [PATCH 1128/2107] [Issue #169] Restore optimization --- src/data.c | 189 +++++++++++++++++- src/pg_probackup.h | 6 + src/restore.c | 471 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 653 insertions(+), 13 deletions(-) diff --git a/src/data.c b/src/data.c index a68197cd1..1d0fada29 100644 --- a/src/data.c +++ b/src/data.c @@ -757,7 +757,7 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } /* - * Open backup file for write. We use "r+" at first to overwrite only + * Open backup file for write. We use "r+" at first to overwrite only * modified pages for differential restore. If the file does not exist, * re-open it with "w" to create an empty file. */ @@ -961,6 +961,193 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, fclose(in); } +/* + * Restore files in the from_root directory to the to_root directory with + * same relative path. + * + * If write_header is true then we add header to each restored block, currently + * it is used for MERGE command. + * + * to_fullpath and from_fullpath are provided strictly for ERROR reporting + */ +void +restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, + const char *from_fullpath, const char *to_fullpath, int nblocks) +{ + BackupPageHeader header; + BlockNumber blknum = 0; + size_t write_len = 0; + + while (true) + { + off_t write_pos; + size_t read_len; + DataPage compressed_page; /* used as read buffer */ + DataPage page; + int32 uncompressed_size = 0; + + /* read BackupPageHeader */ + read_len = fread(&header, 1, sizeof(header), in); + + if (read_len != sizeof(header)) + { + int errno_tmp = errno; + if (read_len == 0 && feof(in)) + break; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, "Odd size page found at block %u of \"%s\"", + blknum, from_fullpath); + else + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", + blknum, from_fullpath, strerror(errno_tmp)); + } + + /* Consider empty block */ + if (header.block == 0 && header.compressed_size == 0) + { + elog(VERBOSE, "Skip empty block of \"%s\"", from_fullpath); + continue; + } + + /* sanity? */ + if (header.block < blknum) + elog(ERROR, "Backup is broken at block %u of \"%s\"", + blknum, from_fullpath); + + blknum = header.block; + + /* no point in writing redundant data */ + if (nblocks > 0 && blknum >= nblocks) + return; + + if (header.compressed_size > BLCKSZ) + elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); + + /* read a page from file */ + read_len = fread(compressed_page.data, 1, + MAXALIGN(header.compressed_size), in); + + if (read_len != MAXALIGN(header.compressed_size)) + elog(ERROR, "Cannot read block %u of \"%s\", read %zu of %d", + blknum, from_fullpath, read_len, header.compressed_size); + + /* + * if page size is smaller than BLCKSZ, decompress the page. + * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. + * we have to check, whether it is compressed or not using + * page_may_be_compressed() function. + */ + if (header.compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) + { + const char *errormsg = NULL; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + header.compressed_size, + file->compress_alg, &errormsg); + + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, from_fullpath, errormsg); + + if (uncompressed_size != BLCKSZ) + elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + from_fullpath, uncompressed_size); + } + + write_pos = blknum * BLCKSZ; + + /* + * Seek and write the restored page. + * TODO: invent fio_pwrite(). + */ + if (fio_fseek(out, write_pos) < 0) + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + + /* if we uncompressed the page - write page.data, + * if page wasn't compressed - + * write what we've read - compressed_page.data + */ + if (uncompressed_size == BLCKSZ) + { + if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + else + { + if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + + write_len += BLCKSZ; + } + + elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); +} + +/* + * Copy file to backup. + * We do not apply compression to these files, because + * it is either small control file or already compressed cfs file. + */ +void +restore_non_data_file(FILE *in, FILE *out, pgFile *file, + const char *from_fullpath, const char *to_fullpath) +{ + size_t read_len = 0; + int errno_tmp; + char buf[BLCKSZ]; + + /* copy content */ + for (;;) + { + read_len = 0; + + if ((read_len = fio_fread(in, buf, sizeof(buf))) != sizeof(buf)) + break; + + if (fio_fwrite(out, buf, read_len) != read_len) + { + errno_tmp = errno; + /* oops */ + fio_fclose(in); + fio_fclose(out); + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, + strerror(errno_tmp)); + } + } + + errno_tmp = errno; + if (read_len < 0) + { + fio_fclose(in); + fio_fclose(out); + elog(ERROR, "Cannot read backup mode file \"%s\": %s", + from_fullpath, strerror(errno_tmp)); + } + + /* copy odd part. */ + if (read_len > 0) + { + if (fio_fwrite(out, buf, read_len) != read_len) + { + errno_tmp = errno; + /* oops */ + fio_fclose(in); + fio_fclose(out); + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, + strerror(errno_tmp)); + } + } + + elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); +} + /* * Copy file to backup. * We do not apply compression to these files, because diff --git a/src/pg_probackup.h b/src/pg_probackup.h index af782f1dc..d5d5f2b4e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -372,6 +372,8 @@ struct pgBackup * in the format suitable for recovery.conf */ char *external_dir_str; /* List of external directories, * separated by ':' */ + parray *files; /* list of files belonging to this backup + * must be populated by calling backup_populate() */ }; /* Recovery target for restore and validate subcommands */ @@ -835,6 +837,10 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); +extern void restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, + const char *from_fullpath, const char *to_fullpath, int nblocks); +extern void restore_non_data_file(FILE *in, FILE *out, pgFile *file, + const char *from_fullpath, const char *to_fullpath); extern bool copy_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file, bool missing_ok); extern bool create_empty_file(fio_location from_location, const char *to_root, diff --git a/src/restore.c b/src/restore.c index 3d4e544c1..13b6184dc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -35,6 +35,24 @@ typedef struct int ret; } restore_files_arg; +typedef struct +{ + parray *dest_files; + pgBackup *dest_backup; + char *external_prefix; + parray *dest_external_dirs; + parray *parent_chain; + parray *dbOid_exclude_list; + bool skip_external_dirs; + const char *to_root; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} restore_files_arg_new; + static void restore_backup(pgBackup *backup, parray *dest_external_dirs, parray *dest_files, parray *dbOid_exclude_list, pgRestoreParams *params); @@ -46,6 +64,12 @@ static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); static void pg12_recovery_config(pgBackup *backup, bool add_include); +static void restore_chain(pgBackup *dest_backup, parray *dest_files, + parray *parent_chain, parray *dbOid_exclude_list, + pgRestoreParams *params, const char *pgdata_path); + +static void *restore_files_new(void *arg); + /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -499,29 +523,74 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, DIR_PERMISSION, FIO_DB_HOST); } + if (rt->lsn_string && + parse_server_version(dest_backup->server_version) < 100000) + elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", + base36enc(dest_backup->start_time), + dest_backup->server_version); + /* * Restore backups files starting from the parent backup. */ +// for (i = parray_num(parent_chain) - 1; i >= 0; i--) +// { +// pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); +// +// /* +// * Backup was locked during validation if no-validate wasn't +// * specified. +// */ +// if (params->no_validate && !lock_backup(backup)) +// elog(ERROR, "Cannot lock backup directory"); +// +// restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); +// } + + // lock entire chain + + // sanity: + // 1. check status of every backup in chain for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - if (rt->lsn_string && - parse_server_version(backup->server_version) < 100000) - elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", - base36enc(dest_backup->start_time), - dest_backup->server_version); + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + { + if (params->force) + elog(WARNING, "Backup %s is not valid, restore is forced", + base36enc(backup->start_time)); + else + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + } - /* - * Backup was locked during validation if no-validate wasn't - * specified. - */ - if (params->no_validate && !lock_backup(backup)) - elog(ERROR, "Cannot lock backup directory"); + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); - restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); + /* populate backup filelist */ + if (backup->start_time != dest_backup->start_time) + { + pgBackupGetPath(backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); + backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + } + else + backup->files = dest_files; + + parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } + restore_chain(dest_backup, dest_files, parent_chain, dbOid_exclude_list, + params, instance_config.pgdata); + if (dest_external_dirs != NULL) free_dir_list(dest_external_dirs); @@ -542,6 +611,384 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, return 0; } +/* + * Restore backup chain. + */ +void +restore_chain(pgBackup *dest_backup, parray *dest_files, + parray *parent_chain, parray *dbOid_exclude_list, + pgRestoreParams *params, const char *pgdata_path) +{ + char timestamp[100]; + char external_prefix[MAXPGPATH]; + parray *external_dirs = NULL; + int i; + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + restore_files_arg_new *threads_args; + bool restore_isok = true; + + time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); + elog(LOG, "Restoring database from backup %s", timestamp); + + if (dest_backup->external_dir_str) + external_dirs = make_external_directory_list(dest_backup->external_dir_str, + true); + + /* Restore directories first */ + parray_qsort(dest_files, pgFileCompareRelPathWithExternal); + + /* + * Setup file locks + */ + for (i = 0; i < parray_num(dest_files); i++) + { + pgFile *file = (pgFile *) parray_get(dest_files, i); + + /* setup threads */ + pg_atomic_clear_flag(&file->lock); + } + + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) * + num_threads); + + /* Restore files into target directory */ + thread_interrupted = false; + for (i = 0; i < num_threads; i++) + { + restore_files_arg_new *arg = &(threads_args[i]); + + arg->dest_files = dest_files; + arg->dest_backup = dest_backup; + arg->external_prefix = external_prefix; + arg->dest_external_dirs = external_dirs; + arg->parent_chain = parent_chain; + arg->dbOid_exclude_list = dbOid_exclude_list; + arg->skip_external_dirs = params->skip_external_dirs; + arg->to_root = pgdata_path; + /* By default there are some error */ + threads_args[i].ret = 1; + + /* Useless message TODO: rewrite */ + elog(LOG, "Start thread %i", i + 1); + + pthread_create(&threads[i], NULL, restore_files_new, arg); + } + + /* Wait theads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + restore_isok = false; + } + if (!restore_isok) + elog(ERROR, "Data files restoring failed"); + + pfree(threads); + pfree(threads_args); + + if (external_dirs != NULL) + free_dir_list(external_dirs); + +// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time)); +} + +/* + * Restore files into $PGDATA. + */ +static void * +restore_files_new(void *arg) +{ + int i, j; + char to_fullpath[MAXPGPATH]; + FILE *out = NULL; + + restore_files_arg_new *arguments = (restore_files_arg_new *) arg; + +// for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) +// { +// pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); +// +// for (j = 0; j < parray_num(backup->files); j++) +// { +// pgFile *file = (pgFile *) parray_get(backup->files, j); +// +// elog(INFO, "Backup %s;File: %s, Size: %li", +// base36enc(backup->start_time), file->name, file->write_size); +// } +// } +// +// elog(ERROR, "HELLO"); + + for (i = 0; i < parray_num(arguments->dest_files); i++) + { + pgFile *file = (pgFile *) parray_get(arguments->dest_files, i); + + /* Directories were created before */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during restore"); + + if (progress) + elog(INFO, "Progress: (%d/%lu). Process file %s ", + i + 1, (unsigned long) parray_num(arguments->dest_files), + file->rel_path); + + /* Only files from pgdata can be skipped by partial restore */ + if (arguments->dbOid_exclude_list && file->external_dir_num == 0) + { + /* Check if the file belongs to the database we exclude */ + if (parray_bsearch(arguments->dbOid_exclude_list, + &file->dbOid, pgCompareOid)) + { + /* + * We cannot simply skip the file, because it may lead to + * failure during WAL redo; hence, create empty file. + */ + create_empty_file(FIO_BACKUP_HOST, + arguments->to_root, FIO_DB_HOST, file); + + elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", + file->rel_path); + continue; + } + } + + /* Do not restore tablespace_map file */ + if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) + { + elog(VERBOSE, "Skip tablespace_map"); + continue; + } + + /* Do not restore database_map file */ + if ((file->external_dir_num == 0) && + strcmp(DATABASE_MAP, file->rel_path) == 0) + { + elog(VERBOSE, "Skip database_map"); + continue; + } + + /* Do no restore external directory file if a user doesn't want */ + if (arguments->skip_external_dirs && file->external_dir_num > 0) + continue; + + //set max_blknum based on file->n_blocks + /* set fullpath of destination file */ + if (file->external_dir_num == 0) + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + //else + // TODO + + /* open destination file */ + out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); + if (out == NULL) + { + int errno_tmp = errno; + elog(ERROR, "Cannot open restore target file \"%s\": %s", + to_fullpath, strerror(errno_tmp)); + } + + if (!file->is_datafile || file->is_cfs) + elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); + else + elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); + + // if dest file is 0 sized, then just close it and go for the next + if (file->write_size == 0) + goto done; + + // TODO + // optimize copying of non-data files: + // lookup latest backup with file that has not BYTES_INVALID size + // and copy only it. + + if (!file->is_datafile || file->is_cfs) + { + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile *tmp_file = NULL; + + if (file->write_size > 0) + { + tmp_file = file; + pgBackupGetPath(arguments->dest_backup, from_root, lengthof(from_root), DATABASE_DIR); + join_path_components(from_fullpath, from_root, file->rel_path); + } + else + { + for (j = 0; j < parray_num(arguments->parent_chain); j++) + { + pgFile **res_file = NULL; + pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + if (!tmp_file) + continue; + + if (tmp_file->write_size == 0) + goto done; + + if (tmp_file->write_size > 0) + { + pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); + join_path_components(from_fullpath, from_root, file->rel_path); + break; + } + } + } + + if (!tmp_file) + elog(ERROR, "Something went wrong"); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + } + + if (strcmp(file->name, "pg_control") == 0) + copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, + instance_config.pgdata, FIO_DB_HOST, + file); + else + restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); + + goto done; + } + + for (j = parray_num(arguments->parent_chain) - 1; j >= 0; j--) + { + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile **res_file = NULL; + pgFile *tmp_file = NULL; + + pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + /* destination file is not exists in this intermediate backup */ + if (tmp_file == NULL) + continue; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during restore"); + + /* + * For incremental backups skip files which haven't changed + * since previous backup and thus were not backed up. + */ + if (tmp_file->write_size == BYTES_INVALID) + { + elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", from_fullpath); + continue; + } + + /* + * At this point we are sure, that something is going to be copied + * Open source file. + */ + + /* TODO: special handling for files from external directories */ + pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); + join_path_components(from_fullpath, from_root, file->rel_path); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + } + + /* + * restore the file. + * We treat datafiles separately, cause they were backed up block by + * block and have BackupPageHeader meta information, so we cannot just + * copy the file from backup. + */ +// if (file->is_datafile && file->is_cfs) + restore_data_file_new(in, out, tmp_file, + parse_program_version(backup->program_version), + from_fullpath, to_fullpath, file->n_blocks); +// else if (strcmp(file->name, "pg_control") == 0) +// copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, +// instance_config.pgdata, FIO_DB_HOST, +// file); +// else +// restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); + } + + done: + // chmod + // fsync + // close + + /* truncate file up to n_blocks. NOTE: no need, we just should not write + * blocks that are exceeding n_blocks. + * But for this to work, n_blocks should be trusted. + */ + + /* update file permission */ + if (fio_chmod(to_fullpath, file->mode, FIO_DB_HOST) == -1) + { + int errno_tmp = errno; + fio_fclose(out); + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno_tmp)); + } + + /* flush file */ + if (fio_fflush(out) != 0) + elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, + strerror(errno)); + + /* fsync file */ + + /* close file */ + if (fio_fclose(out) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, + strerror(errno)); + + /* print size of restored file */ +// if (file->write_size != BYTES_INVALID) +// elog(VERBOSE, "Restored file %s : " INT64_FORMAT " bytes", +// file->path, file->write_size); + } + + /* Data files restoring is successful */ + arguments->ret = 0; + + return NULL; +} + /* * Restore one backup. */ From 577e8763bc447d5b5bc419a182fad2d244c9b03e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Jan 2020 03:41:55 +0300 Subject: [PATCH 1129/2107] [Issue #169] Refactoring, added support for external directories and bugfixes --- src/catalog.c | 5 + src/data.c | 217 +++++++++++++++++-- src/dir.c | 3 +- src/pg_probackup.h | 12 +- src/restore.c | 517 ++++++++++++++++++--------------------------- src/utils/file.c | 9 +- 6 files changed, 425 insertions(+), 338 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 1fe8e4b2e..d6bc2dd41 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -444,6 +444,9 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) base36enc(backup->start_time), backup_conf_path); } + backup->root_dir = pgut_strdup(data_path); + + /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID && requested_backup_id != backup->start_time) @@ -2005,6 +2008,7 @@ pgBackupInit(pgBackup *backup) backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; backup->external_dir_str = NULL; + backup->root_dir = NULL; } /* free pgBackup object */ @@ -2015,6 +2019,7 @@ pgBackupFree(void *backup) pfree(b->primary_conninfo); pfree(b->external_dir_str); + pfree(b->root_dir); pfree(backup); } diff --git a/src/data.c b/src/data.c index 1d0fada29..572ee750c 100644 --- a/src/data.c +++ b/src/data.c @@ -962,23 +962,88 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, } /* - * Restore files in the from_root directory to the to_root directory with - * same relative path. - * - * If write_header is true then we add header to each restored block, currently - * it is used for MERGE command. - * - * to_fullpath and from_fullpath are provided strictly for ERROR reporting + * Iterate over parent backup chain and lookup given destination file in + * filelist of every chain member starting with FULL backup. + * Apply changed blocks to destination file from every backup in parent chain. */ void -restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) +{ + int i; + + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile **res_file = NULL; + pgFile *tmp_file = NULL; + + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during restore"); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + /* Destination file is not exists yet at this moment */ + if (tmp_file == NULL) + continue; + + /* + * Skip file if it haven't changed since previous backup + * and thus was not backed up. + */ + if (tmp_file->write_size == BYTES_INVALID) + { +// elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", tmp_file->rel_path); + continue; + } + + /* + * At this point we are sure, that something is going to be copied + * Open source file. + */ + join_path_components(from_root, backup->root_dir, DATABASE_DIR); + join_path_components(from_fullpath, from_root, tmp_file->rel_path); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + Assert(0); + } + + /* + * restore the file. + * Datafiles are backed up block by block and every block + * have BackupPageHeader with meta information, so we cannot just + * copy the file from backup. + */ + restore_data_file_internal(in, out, tmp_file, + parse_program_version(backup->program_version), + from_fullpath, to_fullpath, dest_file->n_blocks); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); + } +} + +void +restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks) { BackupPageHeader header; BlockNumber blknum = 0; size_t write_len = 0; - while (true) + for (;;) { off_t write_pos; size_t read_len; @@ -1016,9 +1081,44 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, blknum = header.block; + /* + * Backupward compatibility kludge: in the good old days + * n_blocks attribute was available only in DELTA backups. + * File truncate in PAGE and PTRACK happened on the fly when + * special value PageIsTruncated is encountered. + * It is inefficient. + * + * Nowadays every backup type has n_blocks, so instead + * writing and then truncating redundant data, writing + * is not happening in the first place. + * TODO: remove in 3.0.0 + */ + if (header.compressed_size == PageIsTruncated) + { + /* + * Block header contains information that this block was truncated. + * We need to truncate file to this length. + */ + + elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, header.block); + + /* To correctly truncate file, we must first flush STDIO buffers */ + if (fio_fflush(out) != 0) + elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); + + /* Set position to the start of file */ + if (fio_fseek(out, 0) < 0) + elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); + + if (fio_ftruncate(out, header.block * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); + + break; + } + /* no point in writing redundant data */ if (nblocks > 0 && blknum >= nblocks) - return; + break; if (header.compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); @@ -1061,7 +1161,6 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, /* * Seek and write the restored page. - * TODO: invent fio_pwrite(). */ if (fio_fseek(out, write_pos) < 0) elog(ERROR, "Cannot seek block %u of \"%s\": %s", @@ -1096,7 +1195,7 @@ restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, * it is either small control file or already compressed cfs file. */ void -restore_non_data_file(FILE *in, FILE *out, pgFile *file, +restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath) { size_t read_len = 0; @@ -1148,6 +1247,100 @@ restore_non_data_file(FILE *in, FILE *out, pgFile *file, elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } +void +restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, + pgFile *dest_file, FILE *out, const char *to_fullpath) +{ + int i; + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; + + pgFile *tmp_file = NULL; + pgBackup *tmp_backup = NULL; + + /* Check if full copy of destination file is available in destination backup */ + if (dest_file->write_size > 0) + { + tmp_file = dest_file; + tmp_backup = dest_backup; + } + else + { + /* + * Iterate over parent chain starting from direct parent of destination + * backup to oldest backup in chain, and look for the first + * full copy of destination file. + * Full copy is latest possible destination file with size equal or + * greater than zero. + */ + for (i = 1; i < parray_num(parent_chain); i++) + { + pgFile **res_file = NULL; + + tmp_backup = (pgBackup *) parray_get(parent_chain, i); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); + tmp_file = (res_file) ? *res_file : NULL; + + /* + * It should not be possible not to find destination file in intermediate + * backup, without encountering full copy first. + */ + if (!tmp_file) + { + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + dest_file->rel_path, base36enc(tmp_backup->start_time)); + continue; + } + + /* Full copy is found and it is null sized, nothing to do here */ + if (tmp_file->write_size == 0) + return; + + /* Full copy is found */ + if (tmp_file->write_size > 0) + break; + } + } + + /* sanity */ + if (!tmp_backup) + elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", + to_fullpath); + + if (!tmp_file) + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); + + if (tmp_file->external_dir_num == 0) +// pgBackupGetPath(tmp_backup, from_root, lengthof(from_root), DATABASE_DIR); + join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); + else + { + // get external prefix for tmp_backup + char external_prefix[MAXPGPATH]; + + join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR); + makeExternalDirPathByNum(from_root, external_prefix, tmp_file->external_dir_num); + } + + join_path_components(from_fullpath, from_root, dest_file->rel_path); + + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, + strerror(errno)); + } + + restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); + + if (fio_fclose(in) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, + strerror(errno)); +} + /* * Copy file to backup. * We do not apply compression to these files, because diff --git a/src/dir.c b/src/dir.c index 754e6bde4..496518ad0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1643,8 +1643,7 @@ free_dir_list(parray *list) /* Append to string "path_prefix" int "dir_num" */ void -makeExternalDirPathByNum(char *ret_path, const char *path_prefix, - const int dir_num) +makeExternalDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num) { sprintf(ret_path, "%s%d", path_prefix, dir_num); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d5d5f2b4e..d21d8540d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -374,6 +374,8 @@ struct pgBackup * separated by ':' */ parray *files; /* list of files belonging to this backup * must be populated by calling backup_populate() */ + char *root_dir; /* Full path for root backup directory: + backup_path/instance_name/backup_id */ }; /* Recovery target for restore and validate subcommands */ @@ -837,10 +839,14 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern void restore_data_file_new(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +extern void restore_data_file_new(parray *parent_chain, pgFile *dest_file, + FILE *out, const char *to_fullpath); +extern void restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks); -extern void restore_non_data_file(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath); +extern void restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, + pgFile *dest_file, FILE *out, const char *to_fullpath); +extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, + const char *from_fullpath, const char *to_fullpath); extern bool copy_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file, bool missing_ok); extern bool create_empty_file(fio_location from_location, const char *to_root, diff --git a/src/restore.c b/src/restore.c index 13b6184dc..16f3186cf 100644 --- a/src/restore.c +++ b/src/restore.c @@ -39,7 +39,6 @@ typedef struct { parray *dest_files; pgBackup *dest_backup; - char *external_prefix; parray *dest_external_dirs; parray *parent_chain; parray *dbOid_exclude_list; @@ -64,9 +63,9 @@ static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); static void pg12_recovery_config(pgBackup *backup, bool add_include); -static void restore_chain(pgBackup *dest_backup, parray *dest_files, - parray *parent_chain, parray *dbOid_exclude_list, - pgRestoreParams *params, const char *pgdata_path); +static void restore_chain(pgBackup *dest_backup, parray *parent_chain, + parray *dbOid_exclude_list, pgRestoreParams *params, + const char *pgdata_path); static void *restore_files_new(void *arg); @@ -469,21 +468,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - parray *dest_external_dirs = NULL; - parray *dest_files; - char control_file[MAXPGPATH], - dest_backup_path[MAXPGPATH]; - int i; - - /* - * Preparations for actual restoring. - */ - pgBackupGetPath(dest_backup, control_file, lengthof(control_file), - DATABASE_FILE_LIST); - dest_files = dir_read_file_list(NULL, NULL, control_file, - FIO_BACKUP_HOST); - parray_qsort(dest_files, pgFileCompareRelPathWithExternal); - /* * Get a list of dbOids to skip if user requested the partial restore. * It is important that we do this after(!) validation so @@ -497,31 +481,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->partial_db_list) dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list, - params->partial_restore_type); - - /* - * Restore dest_backup internal directories. - */ - pgBackupGetPath(dest_backup, dest_backup_path, - lengthof(dest_backup_path), NULL); - create_data_directories(dest_files, instance_config.pgdata, dest_backup_path, true, - FIO_DB_HOST); - - /* - * Restore dest_backup external directories. - */ - if (dest_backup->external_dir_str && !params->skip_external_dirs) - { - dest_external_dirs = make_external_directory_list( - dest_backup->external_dir_str, - true); - if (parray_num(dest_external_dirs) > 0) - elog(LOG, "Restore external directories"); - - for (i = 0; i < parray_num(dest_external_dirs); i++) - fio_mkdir(parray_get(dest_external_dirs, i), - DIR_PERMISSION, FIO_DB_HOST); - } + params->partial_restore_type); if (rt->lsn_string && parse_server_version(dest_backup->server_version) < 100000) @@ -529,80 +489,15 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), dest_backup->server_version); - /* - * Restore backups files starting from the parent backup. - */ -// for (i = parray_num(parent_chain) - 1; i >= 0; i--) -// { -// pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); -// -// /* -// * Backup was locked during validation if no-validate wasn't -// * specified. -// */ -// if (params->no_validate && !lock_backup(backup)) -// elog(ERROR, "Cannot lock backup directory"); -// -// restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); -// } - - // lock entire chain - - // sanity: - // 1. check status of every backup in chain - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - { - if (params->force) - elog(WARNING, "Backup %s is not valid, restore is forced", - base36enc(backup->start_time)); - else - elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); - } - - /* confirm block size compatibility */ - if (backup->block_size != BLCKSZ) - elog(ERROR, - "BLCKSZ(%d) is not compatible(%d expected)", - backup->block_size, BLCKSZ); - - if (backup->wal_block_size != XLOG_BLCKSZ) - elog(ERROR, - "XLOG_BLCKSZ(%d) is not compatible(%d expected)", - backup->wal_block_size, XLOG_BLCKSZ); - - /* populate backup filelist */ - if (backup->start_time != dest_backup->start_time) - { - pgBackupGetPath(backup, control_file, lengthof(control_file), DATABASE_FILE_LIST); - backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - } - else - backup->files = dest_files; - - parray_qsort(backup->files, pgFileCompareRelPathWithExternal); - } - - restore_chain(dest_backup, dest_files, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata); - - if (dest_external_dirs != NULL) - free_dir_list(dest_external_dirs); - - parray_walk(dest_files, pgFileFree); - parray_free(dest_files); + restore_chain(dest_backup, parent_chain, dbOid_exclude_list, + params, instance_config.pgdata); /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup, params); } /* cleanup */ - parray_walk(backups, pgBackupFree); + parray_walk(backups, pgBackupFree); /* free backup->files */ parray_free(backups); parray_free(parent_chain); @@ -615,36 +510,129 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * Restore backup chain. */ void -restore_chain(pgBackup *dest_backup, parray *dest_files, - parray *parent_chain, parray *dbOid_exclude_list, - pgRestoreParams *params, const char *pgdata_path) +restore_chain(pgBackup *dest_backup, parray *parent_chain, + parray *dbOid_exclude_list, pgRestoreParams *params, + const char *pgdata_path) { - char timestamp[100]; - char external_prefix[MAXPGPATH]; - parray *external_dirs = NULL; int i; + char control_file[MAXPGPATH]; + char timestamp[100]; + parray *dest_files = NULL; + parray *external_dirs = NULL; /* arrays with meta info for multi threaded backup */ pthread_t *threads; restore_files_arg_new *threads_args; bool restore_isok = true; + time_t start_time, end_time; + + /* Preparations for actual restoring */ time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); - elog(LOG, "Restoring database from backup %s", timestamp); + elog(LOG, "Restoring database from backup at %s", timestamp); - if (dest_backup->external_dir_str) - external_dirs = make_external_directory_list(dest_backup->external_dir_str, - true); + join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST); + dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + + // TODO lock entire chain +// for (i = parray_num(parent_chain) - 1; i >= 0; i--) +// { +// pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); +// +// /* +// * Backup was locked during validation if no-validate wasn't +// * specified. +// */ +// if (params->no_validate && !lock_backup(backup)) +// elog(ERROR, "Cannot lock backup directory"); +// +// restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); +// } + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + { + if (params->force) + elog(WARNING, "Backup %s is not valid, restore is forced", + base36enc(backup->start_time)); + else + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + } + + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); - /* Restore directories first */ - parray_qsort(dest_files, pgFileCompareRelPathWithExternal); + /* populate backup filelist */ + if (backup->start_time != dest_backup->start_time) + { + join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); + backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + } + else + backup->files = dest_files; + + /* this sorting is important */ + parray_qsort(backup->files, pgFileCompareRelPathWithExternal); + } /* - * Setup file locks + * Restore dest_backup internal directories. + */ + create_data_directories(dest_files, instance_config.pgdata, + dest_backup->root_dir, true, FIO_DB_HOST); + + /* + * Restore dest_backup external directories. + */ + if (dest_backup->external_dir_str && !params->skip_external_dirs) + { + external_dirs = make_external_directory_list(dest_backup->external_dir_str, true); + + if (!external_dirs) + elog(ERROR, "Failed to get a list of external directories"); + + if (parray_num(external_dirs) > 0) + elog(LOG, "Restore external directories"); + + for (i = 0; i < parray_num(external_dirs); i++) + fio_mkdir(parray_get(external_dirs, i), + DIR_PERMISSION, FIO_DB_HOST); + } + + /* + * Setup directory structure for external directories and file locks */ for (i = 0; i < parray_num(dest_files); i++) { pgFile *file = (pgFile *) parray_get(dest_files, i); + if (!params->skip_external_dirs && + file->external_dir_num && S_ISDIR(file->mode)) + { + char *external_path; + char dirpath[MAXPGPATH]; + + if (parray_num(external_dirs) < file->external_dir_num - 1) + elog(ERROR, "Inconsistent external directory backup metadata"); + + external_path = parray_get(external_dirs, file->external_dir_num - 1); + join_path_components(dirpath, external_path, file->rel_path); + + elog(VERBOSE, "Create external directory \"%s\"", dirpath); + fio_mkdir(dirpath, file->mode, FIO_DB_HOST); + } + /* setup threads */ pg_atomic_clear_flag(&file->lock); } @@ -655,13 +643,14 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, /* Restore files into target directory */ thread_interrupted = false; + elog(INFO, "Start restoring backup files"); + time(&start_time); for (i = 0; i < num_threads; i++) { restore_files_arg_new *arg = &(threads_args[i]); arg->dest_files = dest_files; arg->dest_backup = dest_backup; - arg->external_prefix = external_prefix; arg->dest_external_dirs = external_dirs; arg->parent_chain = parent_chain; arg->dbOid_exclude_list = dbOid_exclude_list; @@ -683,15 +672,59 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, if (threads_args[i].ret == 1) restore_isok = false; } - if (!restore_isok) - elog(ERROR, "Data files restoring failed"); + time(&end_time); + if (restore_isok) + elog(INFO, "Backup files are restored, time elapsed: %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec", + difftime(end_time, start_time)); + + + elog(INFO, "Sync restored backup files to disk"); + time(&start_time); + + for (i = 0; i < parray_num(dest_files); i++) + { + int out; + char to_fullpath[MAXPGPATH]; + pgFile *dest_file = (pgFile *) parray_get(dest_files, i); + + if (S_ISDIR(dest_file->mode) || + dest_file->external_dir_num > 0 || + (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) || + (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)) + continue; + + join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + + /* open destination file */ + out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST); + if (out < 0) + elog(ERROR, "Cannot open file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* sync file */ + if (fio_flush(out) != 0 || fio_close(out) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", + to_fullpath, strerror(errno)); + } + + time(&end_time); + elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec", + difftime(end_time, start_time)); + + /* cleanup */ pfree(threads); pfree(threads_args); if (external_dirs != NULL) free_dir_list(external_dirs); + parray_walk(dest_files, pgFileFree); + parray_free(dest_files); + // elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time)); } @@ -701,36 +734,21 @@ restore_chain(pgBackup *dest_backup, parray *dest_files, static void * restore_files_new(void *arg) { - int i, j; + int i; char to_fullpath[MAXPGPATH]; FILE *out = NULL; restore_files_arg_new *arguments = (restore_files_arg_new *) arg; -// for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) -// { -// pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); -// -// for (j = 0; j < parray_num(backup->files); j++) -// { -// pgFile *file = (pgFile *) parray_get(backup->files, j); -// -// elog(INFO, "Backup %s;File: %s, Size: %li", -// base36enc(backup->start_time), file->name, file->write_size); -// } -// } -// -// elog(ERROR, "HELLO"); - for (i = 0; i < parray_num(arguments->dest_files); i++) { - pgFile *file = (pgFile *) parray_get(arguments->dest_files, i); + pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ - if (S_ISDIR(file->mode)) + if (S_ISDIR(dest_file->mode)) continue; - if (!pg_atomic_test_set_flag(&file->lock)) + if (!pg_atomic_test_set_flag(&dest_file->lock)) continue; /* check for interrupt */ @@ -740,53 +758,57 @@ restore_files_new(void *arg) if (progress) elog(INFO, "Progress: (%d/%lu). Process file %s ", i + 1, (unsigned long) parray_num(arguments->dest_files), - file->rel_path); + dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && file->external_dir_num == 0) + if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) { /* Check if the file belongs to the database we exclude */ if (parray_bsearch(arguments->dbOid_exclude_list, - &file->dbOid, pgCompareOid)) + &dest_file->dbOid, pgCompareOid)) { /* * We cannot simply skip the file, because it may lead to - * failure during WAL redo; hence, create empty file. + * failure during WAL redo; hence, create empty file. */ create_empty_file(FIO_BACKUP_HOST, - arguments->to_root, FIO_DB_HOST, file); + arguments->to_root, FIO_DB_HOST, dest_file); - elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", - file->rel_path); + elog(INFO, "Skip file due to partial restore: \"%s\"", + dest_file->rel_path); continue; } } /* Do not restore tablespace_map file */ - if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) + if ((dest_file->external_dir_num == 0) && + strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) { elog(VERBOSE, "Skip tablespace_map"); continue; } /* Do not restore database_map file */ - if ((file->external_dir_num == 0) && - strcmp(DATABASE_MAP, file->rel_path) == 0) + if ((dest_file->external_dir_num == 0) && + strcmp(DATABASE_MAP, dest_file->rel_path) == 0) { elog(VERBOSE, "Skip database_map"); continue; } /* Do no restore external directory file if a user doesn't want */ - if (arguments->skip_external_dirs && file->external_dir_num > 0) + if (arguments->skip_external_dirs && dest_file->external_dir_num > 0) continue; - //set max_blknum based on file->n_blocks /* set fullpath of destination file */ - if (file->external_dir_num == 0) - join_path_components(to_fullpath, arguments->to_root, file->rel_path); - //else - // TODO + if (dest_file->external_dir_num == 0) + join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path); + else + { + char *external_path = parray_get(arguments->dest_external_dirs, + dest_file->external_dir_num - 1); + join_path_components(to_fullpath, external_path, dest_file->rel_path); + } /* open destination file */ out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); @@ -797,158 +819,32 @@ restore_files_new(void *arg) to_fullpath, strerror(errno_tmp)); } - if (!file->is_datafile || file->is_cfs) + if (!dest_file->is_datafile || dest_file->is_cfs) elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); else elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); - // if dest file is 0 sized, then just close it and go for the next - if (file->write_size == 0) + // If destination file is 0 sized, then just close it and go for the next + if (dest_file->write_size == 0) goto done; - // TODO - // optimize copying of non-data files: - // lookup latest backup with file that has not BYTES_INVALID size - // and copy only it. - - if (!file->is_datafile || file->is_cfs) - { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile *tmp_file = NULL; - - if (file->write_size > 0) - { - tmp_file = file; - pgBackupGetPath(arguments->dest_backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - } - else - { - for (j = 0; j < parray_num(arguments->parent_chain); j++) - { - pgFile **res_file = NULL; - pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - if (!tmp_file) - continue; - - if (tmp_file->write_size == 0) - goto done; - - if (tmp_file->write_size > 0) - { - pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - break; - } - } - } - - if (!tmp_file) - elog(ERROR, "Something went wrong"); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - } - - if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, - file); - else - restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); - - if (fio_fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - - goto done; - } - - for (j = parray_num(arguments->parent_chain) - 1; j >= 0; j--) - { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile **res_file = NULL; - pgFile *tmp_file = NULL; - - pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, j); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - /* destination file is not exists in this intermediate backup */ - if (tmp_file == NULL) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during restore"); - - /* - * For incremental backups skip files which haven't changed - * since previous backup and thus were not backed up. - */ - if (tmp_file->write_size == BYTES_INVALID) - { - elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", from_fullpath); - continue; - } - - /* - * At this point we are sure, that something is going to be copied - * Open source file. - */ - - /* TODO: special handling for files from external directories */ - pgBackupGetPath(backup, from_root, lengthof(from_root), DATABASE_DIR); - join_path_components(from_fullpath, from_root, file->rel_path); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - } + /* Restore destination file */ + if (dest_file->is_datafile && !dest_file->is_cfs) + /* Destination file is data file */ + restore_data_file_new(arguments->parent_chain, dest_file, out, to_fullpath); + else + /* Destination file is non-data file */ + restore_non_data_file(arguments->parent_chain, arguments->dest_backup, + dest_file, out, to_fullpath); - /* - * restore the file. - * We treat datafiles separately, cause they were backed up block by - * block and have BackupPageHeader meta information, so we cannot just - * copy the file from backup. - */ -// if (file->is_datafile && file->is_cfs) - restore_data_file_new(in, out, tmp_file, - parse_program_version(backup->program_version), - from_fullpath, to_fullpath, file->n_blocks); -// else if (strcmp(file->name, "pg_control") == 0) -// copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, -// instance_config.pgdata, FIO_DB_HOST, -// file); -// else -// restore_non_data_file(in, out, tmp_file, from_fullpath, to_fullpath); - - if (fio_fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - } + /* + * Destination file is data file. + * Iterate over incremental chain and lookup given destination file. + * Apply changed blocks to destination file from every backup in parent chain. + */ done: // chmod - // fsync // close /* truncate file up to n_blocks. NOTE: no need, we just should not write @@ -957,7 +853,7 @@ restore_files_new(void *arg) */ /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_DB_HOST) == -1) + if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) { int errno_tmp = errno; fio_fclose(out); @@ -965,13 +861,6 @@ restore_files_new(void *arg) strerror(errno_tmp)); } - /* flush file */ - if (fio_fflush(out) != 0) - elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, - strerror(errno)); - - /* fsync file */ - /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, @@ -1903,7 +1792,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", base36enc(backup->start_time)); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); /* check database_map CRC */ diff --git a/src/utils/file.c b/src/utils/file.c index f7c3eab78..bae08ef2b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -447,17 +447,12 @@ int fio_fprintf(FILE* f, char const* format, ...) return rc; } -/* Flush stream data (does nothing for remote file) */ +/* Flush stream data (does nothing for remote file) */ int fio_fflush(FILE* f) { int rc = 0; if (!fio_is_remote_file(f)) - { rc = fflush(f); - if (rc == 0) { - rc = fsync(fileno(f)); - } - } return rc; } @@ -560,7 +555,7 @@ int fio_pread(FILE* f, void* buf, off_t offs) int fio_fseek(FILE* f, off_t offs) { return fio_is_remote_file(f) - ? fio_seek(fio_fileno(f), offs) + ? fio_seek(fio_fileno(f), offs) : fseek(f, offs, SEEK_SET); } From bf5b8f6b48488379ed6feb4900f96125a26d01a7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Jan 2020 21:39:13 +0300 Subject: [PATCH 1130/2107] [Issue #169] fio_sync() added --- src/pg_probackup.c | 15 ++++---- src/pg_probackup.h | 3 +- src/restore.c | 89 ++++++++++++++++++++++++++-------------------- src/utils/file.c | 59 ++++++++++++++++++++++++++++++ src/utils/file.h | 2 ++ 5 files changed, 122 insertions(+), 46 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 32260d930..811758736 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -68,6 +68,7 @@ static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; bool progress = false; +bool no_sync = false; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; #endif @@ -155,13 +156,14 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg); static ConfigOption cmd_options[] = { /* directory options */ - { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, + { 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, /* common options */ { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, - { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, - { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, + { 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT }, + { 'b', 122, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, + { 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -756,8 +758,8 @@ main(int argc, char *argv[]) } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, - recovery_target_options, - restore_params); + recovery_target_options, + restore_params, no_sync); case VALIDATE_CMD: if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn) { @@ -771,7 +773,8 @@ main(int argc, char *argv[]) /* PITR validation and, optionally, partial validation */ return do_restore_or_validate(current.backup_id, recovery_target_options, - restore_params); + restore_params, + no_sync); case SHOW_CMD: return do_show(instance_name, current.backup_id, show_archive); case DELETE_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d21d8540d..d16336171 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -644,7 +644,8 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments, /* in restore.c */ extern int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - pgRestoreParams *params); + pgRestoreParams *params, + bool no_sync); extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); diff --git a/src/restore.c b/src/restore.c index 16f3186cf..0c0403b51 100644 --- a/src/restore.c +++ b/src/restore.c @@ -65,7 +65,7 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path); + const char *pgdata_path, bool no_sync); static void *restore_files_new(void *arg); @@ -118,7 +118,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) */ int do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, - pgRestoreParams *params) + pgRestoreParams *params, bool no_sync) { int i = 0; int j = 0; @@ -490,14 +490,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup->server_version); restore_chain(dest_backup, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata); + params, instance_config.pgdata, no_sync); /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup, params); } /* cleanup */ - parray_walk(backups, pgBackupFree); /* free backup->files */ + parray_walk(backups, pgBackupFree); /* TODO: free backup->files */ parray_free(backups); parray_free(parent_chain); @@ -512,7 +512,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path) + const char *pgdata_path, bool no_sync) { int i; char control_file[MAXPGPATH]; @@ -524,6 +524,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, restore_files_arg_new *threads_args; bool restore_isok = true; + char pretty_time[20]; time_t start_time, end_time; /* Preparations for actual restoring */ @@ -637,6 +638,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_atomic_clear_flag(&file->lock); } + fio_disconnect(); + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) * num_threads); @@ -674,46 +677,56 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); if (restore_isok) - elog(INFO, "Backup files are restored, time elapsed: %.0f sec", - difftime(end_time, start_time)); + elog(INFO, "Backup files are restored, time elapsed: %s", pretty_time); else - elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec", - difftime(end_time, start_time)); + elog(ERROR, "Backup files restoring failed, time elapsed: %s", pretty_time); + if (no_sync) + elog(WARNING, "Restored files are not synced to disk"); + else + { + elog(INFO, "Syncing restored files to disk"); + time(&start_time); - elog(INFO, "Sync restored backup files to disk"); - time(&start_time); + for (i = 0; i < parray_num(dest_files); i++) + { + char to_fullpath[MAXPGPATH]; + pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - for (i = 0; i < parray_num(dest_files); i++) - { - int out; - char to_fullpath[MAXPGPATH]; - pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - - if (S_ISDIR(dest_file->mode) || - dest_file->external_dir_num > 0 || - (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) || - (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)) - continue; + if (S_ISDIR(dest_file->mode)) + continue; - join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + if (params->skip_external_dirs && dest_file->external_dir_num > 0) + continue; - /* open destination file */ - out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST); - if (out < 0) - elog(ERROR, "Cannot open file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* sync file */ - if (fio_flush(out) != 0 || fio_close(out) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", - to_fullpath, strerror(errno)); - } + /* construct fullpath */ + if (dest_file->external_dir_num == 0) + { + if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) + continue; + if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0) + continue; + join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); + } + else + { + char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); + join_path_components(to_fullpath, external_path, dest_file->rel_path); + } - time(&end_time); - elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec", - difftime(end_time, start_time)); + /* TODO: write test for case: file to be synced is missing */ + if (fio_sync(to_fullpath, FIO_DB_HOST) != 0) + elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); + } + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Restored backup files are synced, time elapsed: %s", pretty_time); + } /* cleanup */ pfree(threads); @@ -724,8 +737,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_walk(dest_files, pgFileFree); parray_free(dest_files); - -// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time)); } /* diff --git a/src/utils/file.c b/src/utils/file.c index bae08ef2b..b6d1e16d6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -797,6 +797,48 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location } } +/* Sync file to disk */ +int fio_sync(char const* path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + hdr.cop = FIO_SYNC; + hdr.handle = -1; + hdr.size = path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + + return 0; + } + else + { + int fd; + + fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); + if (fd < 0) + return -1; + + if (fsync(fd) < 0) + { + close(fd); + return -1; + } + close(fd); + + return 0; + } +} + /* Remove file */ int fio_unlink(char const* path, fio_location location) { @@ -1348,6 +1390,7 @@ void fio_communicate(int in, int out) fio_header hdr; struct stat st; int rc; + int tmp_fd; #ifdef WIN32 SYS_CHECK(setmode(in, _O_BINARY)); @@ -1470,6 +1513,22 @@ void fio_communicate(int in, int out) Assert(hdr.size == sizeof(fio_send_request)); fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf); break; + case FIO_SYNC: + /* open file and fsync it */ + tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); + if (tmp_fd < 0) + hdr.arg = errno; + else + { + if (fsync(tmp_fd) == 0) + hdr.arg = 0; + else + hdr.arg = errno; + } + close(tmp_fd); + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 70dce4255..9b3321a50 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -15,6 +15,7 @@ typedef enum FIO_OPEN, FIO_CLOSE, FIO_WRITE, + FIO_SYNC, FIO_RENAME, FIO_SYMLINK, FIO_UNLINK, @@ -92,6 +93,7 @@ extern int fio_fstat(int fd, struct stat* st); extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); +extern int fio_sync(char const* path, fio_location location); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, fio_location location); From c0541a60687adbfb4ab404790c83afb02733fff0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Jan 2020 22:04:07 +0300 Subject: [PATCH 1131/2107] [Issue #169] Minor improvements --- src/restore.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/restore.c b/src/restore.c index 0c0403b51..bb88933a0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -497,7 +497,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* cleanup */ - parray_walk(backups, pgBackupFree); /* TODO: free backup->files */ + parray_walk(backups, pgBackupFree); parray_free(backups); parray_free(parent_chain); @@ -529,29 +529,19 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, /* Preparations for actual restoring */ time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); - elog(LOG, "Restoring database from backup at %s", timestamp); + elog(INFO, "Restoring the database from backup at %s", timestamp); join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST); dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - // TODO lock entire chain -// for (i = parray_num(parent_chain) - 1; i >= 0; i--) -// { -// pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); -// -// /* -// * Backup was locked during validation if no-validate wasn't -// * specified. -// */ -// if (params->no_validate && !lock_backup(backup)) -// elog(ERROR, "Cannot lock backup directory"); -// -// restore_backup(backup, dest_external_dirs, dest_files, dbOid_exclude_list, params); -// } + /* Lock backup chain and make sanity checks */ for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + if (!lock_backup(backup)) + elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); + if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) { @@ -638,6 +628,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_atomic_clear_flag(&file->lock); } + /* + * Close ssh connection belonging to the main thread + * to avoid the possibility of been killed for idleness + */ fio_disconnect(); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -735,8 +729,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (external_dirs != NULL) free_dir_list(external_dirs); - parray_walk(dest_files, pgFileFree); - parray_free(dest_files); + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + parray_walk(backup->files, pgFileFree); + parray_free(backup->files); + } } /* From ee36dd9d293e3c3d720b4a02262ef1a7eb361f1f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 27 Jan 2020 11:41:49 +0300 Subject: [PATCH 1132/2107] [Issue #169] Calculate appoximate efficiency ratio when restoring backup --- src/data.c | 31 +++++++++++++++--------------- src/pg_probackup.h | 6 +++--- src/restore.c | 48 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/data.c b/src/data.c index 572ee750c..69a7c65cc 100644 --- a/src/data.c +++ b/src/data.c @@ -966,10 +966,11 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * filelist of every chain member starting with FULL backup. * Apply changed blocks to destination file from every backup in parent chain. */ -void +size_t restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) { int i; + size_t total_write_len = 0; for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -1013,29 +1014,27 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) - { elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - Assert(0); - } /* - * restore the file. + * Restore the file. * Datafiles are backed up block by block and every block * have BackupPageHeader with meta information, so we cannot just * copy the file from backup. */ - restore_data_file_internal(in, out, tmp_file, + total_write_len += restore_data_file_internal(in, out, tmp_file, parse_program_version(backup->program_version), from_fullpath, to_fullpath, dest_file->n_blocks); - if (fio_fclose(in) != 0) + if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, strerror(errno)); } + return total_write_len; } -void +size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks) { @@ -1067,7 +1066,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers blknum, from_fullpath, strerror(errno_tmp)); } - /* Consider empty block */ + /* Consider empty blockm. wtf empty block ? */ if (header.block == 0 && header.compressed_size == 0) { elog(VERBOSE, "Skip empty block of \"%s\"", from_fullpath); @@ -1187,6 +1186,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); + return write_len; } /* @@ -1247,7 +1247,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } -void +size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath) { @@ -1297,7 +1297,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* Full copy is found and it is null sized, nothing to do here */ if (tmp_file->write_size == 0) - return; + return 0; /* Full copy is found */ if (tmp_file->write_size > 0) @@ -1314,11 +1314,9 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); if (tmp_file->external_dir_num == 0) -// pgBackupGetPath(tmp_backup, from_root, lengthof(from_root), DATABASE_DIR); join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); else { - // get external prefix for tmp_backup char external_prefix[MAXPGPATH]; join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR); @@ -1329,16 +1327,17 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) - { elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - } + /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); - if (fio_fclose(in) != 0) + if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, strerror(errno)); + + return tmp_file->write_size; } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d16336171..27f92e4d3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -840,11 +840,11 @@ extern void restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, bool write_header, uint32 backup_version); -extern void restore_data_file_new(parray *parent_chain, pgFile *dest_file, +extern size_t restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath); -extern void restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, +extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks); -extern void restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, +extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath); extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath); diff --git a/src/restore.c b/src/restore.c index bb88933a0..cd166e5d2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -44,6 +44,7 @@ typedef struct parray *dbOid_exclude_list; bool skip_external_dirs; const char *to_root; + size_t restored_bytes; /* * Return value from the thread. @@ -524,6 +525,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, restore_files_arg_new *threads_args; bool restore_isok = true; + /* fancy reporting */ + char pretty_dest_bytes[20]; + char pretty_total_bytes[20]; + size_t dest_bytes = 0; + size_t total_bytes = 0; char pretty_time[20]; time_t start_time, end_time; @@ -608,6 +614,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgFile *file = (pgFile *) parray_get(dest_files, i); + if (S_ISDIR(file->mode)) + total_bytes += 4096; + if (!params->skip_external_dirs && file->external_dir_num && S_ISDIR(file->mode)) { @@ -639,9 +648,15 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, num_threads); /* Restore files into target directory */ - thread_interrupted = false; - elog(INFO, "Start restoring backup files"); + if (dest_backup->stream) + dest_bytes = dest_backup->pgdata_bytes + dest_backup->wal_bytes; + else + dest_bytes = dest_backup->pgdata_bytes; + + pretty_size(dest_bytes, pretty_dest_bytes, lengthof(pretty_dest_bytes)); + elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); time(&start_time); + thread_interrupted = false; for (i = 0; i < num_threads; i++) { restore_files_arg_new *arg = &(threads_args[i]); @@ -653,6 +668,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, arg->dbOid_exclude_list = dbOid_exclude_list; arg->skip_external_dirs = params->skip_external_dirs; arg->to_root = pgdata_path; + threads_args[i].restored_bytes = 0; /* By default there are some error */ threads_args[i].ret = 1; @@ -668,15 +684,27 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pthread_join(threads[i], NULL); if (threads_args[i].ret == 1) restore_isok = false; + + total_bytes += threads_args[i].restored_bytes; } time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); + pretty_size(total_bytes, pretty_total_bytes, lengthof(pretty_total_bytes)); + if (restore_isok) - elog(INFO, "Backup files are restored, time elapsed: %s", pretty_time); + { + elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s", + pretty_total_bytes, pretty_time); + + elog(INFO, "Approximate restore efficiency ratio: %.f%% (%s/%s)", + ((float) dest_bytes / total_bytes) * 100, + pretty_dest_bytes, pretty_total_bytes); + } else - elog(ERROR, "Backup files restoring failed, time elapsed: %s", pretty_time); + elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s", + pretty_total_bytes, pretty_time); if (no_sync) elog(WARNING, "Restored files are not synced to disk"); @@ -841,11 +869,12 @@ restore_files_new(void *arg) /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) /* Destination file is data file */ - restore_data_file_new(arguments->parent_chain, dest_file, out, to_fullpath); + arguments->restored_bytes += restore_data_file_new(arguments->parent_chain, + dest_file, out, to_fullpath); else /* Destination file is non-data file */ - restore_non_data_file(arguments->parent_chain, arguments->dest_backup, - dest_file, out, to_fullpath); + arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, + arguments->dest_backup, dest_file, out, to_fullpath); /* * Destination file is data file. @@ -875,11 +904,6 @@ restore_files_new(void *arg) if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, strerror(errno)); - - /* print size of restored file */ -// if (file->write_size != BYTES_INVALID) -// elog(VERBOSE, "Restored file %s : " INT64_FORMAT " bytes", -// file->path, file->write_size); } /* Data files restoring is successful */ From 9c3d2998542ed1f5db5e9909a6662e13b2aab96f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 Jan 2020 20:42:47 +0300 Subject: [PATCH 1133/2107] [Issue #169] check for interrupt more frequently during restore --- src/data.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index 69a7c65cc..37aa8f253 100644 --- a/src/data.c +++ b/src/data.c @@ -983,10 +983,6 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during restore"); - /* lookup file in intermediate backup */ res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); tmp_file = (res_file) ? *res_file : NULL; @@ -1050,6 +1046,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers DataPage page; int32 uncompressed_size = 0; + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during data file restore"); + /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); @@ -1207,6 +1207,10 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, { read_len = 0; + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during non-data file restore"); + if ((read_len = fio_fread(in, buf, sizeof(buf))) != sizeof(buf)) break; From 506afe5c12efb81e15df38d1122d992889812790 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 Jan 2020 20:43:37 +0300 Subject: [PATCH 1134/2107] added some comments in backup_data_file() --- src/data.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 37aa8f253..3328bc1b8 100644 --- a/src/data.c +++ b/src/data.c @@ -542,7 +542,7 @@ backup_data_file(backup_files_arg* arguments, BlockNumber blknum = 0; BlockNumber nblocks = 0; BlockNumber n_blocks_skipped = 0; - BlockNumber n_blocks_read = 0; + BlockNumber n_blocks_read = 0; /* number of blocks actually readed */ int page_state; char curr_page[BLCKSZ]; @@ -570,7 +570,7 @@ backup_data_file(backup_files_arg* arguments, file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); - /* open backup mode file for read */ + /* open source mode file for read */ in = fio_fopen(file->path, PG_BINARY_R, FIO_DB_HOST); if (in == NULL) { @@ -639,6 +639,8 @@ backup_data_file(backup_files_arg* arguments, if (rc < 0) elog(ERROR, "Failed to read file \"%s\": %s", file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); + + /* TODO: check that fio_send_pages ain`t bullshitting about number of readed blocks */ n_blocks_read = rc; file->read_size = n_blocks_read * BLCKSZ; @@ -714,6 +716,9 @@ backup_data_file(backup_files_arg* arguments, /* * If we have pagemap then file in the backup can't be a zero size. * Otherwise, we will clear the last file. + * + * TODO: maybe we should just check write_size to be >= 0. + * Or better yeat, open file only there is something to write. */ if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) { From f165001c8dc2654c442779069b2e0a066947377a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 28 Jan 2020 20:44:24 +0300 Subject: [PATCH 1135/2107] honest file->n_blocks for PAGE and PTRACK backups --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index 40900c475..ff5157198 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2028,6 +2028,9 @@ backup_files(void *arg) join_path_components(to_path, arguments->to_root, file->path + strlen(arguments->from_root) + 1); + if (current.backup_mode != BACKUP_MODE_FULL) + file->n_blocks = file->size/BLCKSZ; + /* backup block by block if datafile AND not compressed by cfs*/ if (!backup_data_file(arguments, to_path, file, arguments->prev_start_lsn, From 7d78d501a53cb65ed975691351f0aac00eda76f9 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 6 Feb 2020 14:39:52 +0300 Subject: [PATCH 1136/2107] Decompress on agent's size --- src/data.c | 56 +++++++++++++++++----------------------- src/pg_probackup.h | 10 +++++--- src/utils/file.c | 64 +++++++++++++++++++++++++++++++++++++++++++++- src/utils/file.h | 4 ++- 4 files changed, 95 insertions(+), 39 deletions(-) diff --git a/src/data.c b/src/data.c index 3328bc1b8..effdb9a69 100644 --- a/src/data.c +++ b/src/data.c @@ -89,7 +89,7 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, * Decompresses source into dest using algorithm. Returns the number of bytes * decompressed in the destination buffer, or -1 if decompression fails. */ -static int32 +int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, const char **errormsg) { @@ -1047,9 +1047,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers { off_t write_pos; size_t read_len; - DataPage compressed_page; /* used as read buffer */ DataPage page; - int32 uncompressed_size = 0; + int32 compressed_size = 0; + bool is_compressed = false; /* check for interrupt */ if (interrupted || thread_interrupted) @@ -1090,14 +1090,16 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * n_blocks attribute was available only in DELTA backups. * File truncate in PAGE and PTRACK happened on the fly when * special value PageIsTruncated is encountered. - * It is inefficient. + * It was inefficient. * * Nowadays every backup type has n_blocks, so instead * writing and then truncating redundant data, writing * is not happening in the first place. * TODO: remove in 3.0.0 */ - if (header.compressed_size == PageIsTruncated) + compressed_size = header.compressed_size; + + if (compressed_size == PageIsTruncated) { /* * Block header contains information that this block was truncated. @@ -1124,16 +1126,15 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (nblocks > 0 && blknum >= nblocks) break; - if (header.compressed_size > BLCKSZ) + if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); /* read a page from file */ - read_len = fread(compressed_page.data, 1, - MAXALIGN(header.compressed_size), in); + read_len = fread(page.data, 1, MAXALIGN(compressed_size), in); - if (read_len != MAXALIGN(header.compressed_size)) + if (read_len != MAXALIGN(compressed_size)) elog(ERROR, "Cannot read block %u of \"%s\", read %zu of %d", - blknum, from_fullpath, read_len, header.compressed_size); + blknum, from_fullpath, read_len, compressed_size); /* * if page size is smaller than BLCKSZ, decompress the page. @@ -1142,23 +1143,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * page_may_be_compressed() function. */ if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, + || page_may_be_compressed(page.data, file->compress_alg, backup_version)) { - const char *errormsg = NULL; - - uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, - file->compress_alg, &errormsg); - - if (uncompressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, from_fullpath, errormsg); - - if (uncompressed_size != BLCKSZ) - elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - from_fullpath, uncompressed_size); + is_compressed = true; } write_pos = blknum * BLCKSZ; @@ -1170,19 +1158,21 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); - /* if we uncompressed the page - write page.data, - * if page wasn't compressed - - * write what we've read - compressed_page.data + /* If page is compressed and restore is in remote mode, send compressed + * page to the remote side. */ - if (uncompressed_size == BLCKSZ) + if (is_compressed) { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); + ssize_t rc; + rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg); + + if (!fio_is_remote_file(out) && rc != BLCKSZ) + elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", + blknum, to_fullpath, strerror(errno), compressed_size); } else { - if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) + if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "Cannot write block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 27f92e4d3..2f3a73dde 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -216,8 +216,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.7" -#define AGENT_PROTOCOL_VERSION 20207 +#define PROGRAM_VERSION "2.2.8" +#define AGENT_PROTOCOL_VERSION 20208 typedef struct ConnectionOptions @@ -898,8 +898,10 @@ extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); extern bool parse_page(Page page, XLogRecPtr *lsn); -int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, int level, const char **errormsg); +extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, int level, const char **errormsg); +extern int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, + CompressAlg alg, const char **errormsg); extern void pretty_size(int64 size, char *buf, size_t len); extern void pretty_time_interval(int64 num_seconds, char *buf, size_t len); diff --git a/src/utils/file.c b/src/utils/file.c index b6d1e16d6..44bed31cb 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -55,7 +55,7 @@ void fio_error(int rc, int size, char const* file, int line) { if (remote_agent) { - fprintf(stderr, "%s:%d: proceeds %d bytes instead of %d: %s\n", file, line, rc, size, rc >= 0 ? "end of data" : strerror(errno)); + fprintf(stderr, "%s:%d: processed %d bytes instead of %d: %s\n", file, line, rc, size, rc >= 0 ? "end of data" : strerror(errno)); exit(EXIT_FAILURE); } else @@ -611,6 +611,65 @@ ssize_t fio_write(int fd, void const* buf, size_t size) } } +static int32 +fio_decompress(void* dst, void const* src, size_t size, int compress_alg) +{ + const char *errormsg = NULL; + int32 uncompressed_size = do_decompress(dst, BLCKSZ, + src, + size, + compress_alg, &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + { + elog(WARNING, "An error occured during decompressing block: %s", errormsg); + return -1; + } + + if (uncompressed_size != BLCKSZ) + { + elog(ERROR, "Page uncompressed to %d bytes != BLCKSZ", + uncompressed_size); + return -1; + } + return uncompressed_size; +} + +/* Write data to the file */ +ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + hdr.cop = FIO_WRITE_COMPRESSED; + hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; + hdr.size = size; + hdr.arg = compress_alg; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + + return size; + } + else + { + char uncompressed_buf[BLCKSZ]; + int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); + + return (uncompressed_size < 0) + ? uncompressed_size + : fwrite(uncompressed_buf, 1, uncompressed_size, f); + } +} + +static ssize_t +fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) +{ + char uncompressed_buf[BLCKSZ]; + int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); + return fio_write_all(fd, uncompressed_buf, uncompressed_size); +} + /* Read data from stdio file */ ssize_t fio_fread(FILE* f, void* buf, size_t size) { @@ -1447,6 +1506,9 @@ void fio_communicate(int in, int out) case FIO_WRITE: /* Write to the current position in file */ IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); break; + case FIO_WRITE_COMPRESSED: /* Write to the current position in file */ + IO_CHECK(fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg), BLCKSZ); + break; case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { buf_size = hdr.arg; diff --git a/src/utils/file.h b/src/utils/file.h index 9b3321a50..563eb79f6 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -34,7 +34,8 @@ typedef enum FIO_READDIR, FIO_CLOSEDIR, FIO_SEND_PAGES, - FIO_PAGE + FIO_PAGE, + FIO_WRITE_COMPRESSED, } fio_operations; typedef enum @@ -70,6 +71,7 @@ extern void fio_communicate(int in, int out); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); +extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); From d6567a2d67f7d0352f36d36c6f473a0243902992 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Feb 2020 14:49:58 +0300 Subject: [PATCH 1137/2107] [Issue #169] Set stdio buffers to 64KB when running "restore" --- src/data.c | 8 +++++++- src/restore.c | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index effdb9a69..8ab42714a 100644 --- a/src/data.c +++ b/src/data.c @@ -976,6 +976,7 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const { int i; size_t total_write_len = 0; + char buffer[65536]; for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -1018,6 +1019,8 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); + setbuf(in, buffer); + /* * Restore the file. * Datafiles are backed up block by block and every block @@ -1195,7 +1198,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, { size_t read_len = 0; int errno_tmp; - char buf[BLCKSZ]; + char buf[65536]; /* 64kB buffer */ /* copy content */ for (;;) @@ -1257,6 +1260,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *tmp_file = NULL; pgBackup *tmp_backup = NULL; + char buffer[65536]; /* Check if full copy of destination file is available in destination backup */ if (dest_file->write_size > 0) @@ -1329,6 +1333,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); + setbuf(in, buffer); + /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); diff --git a/src/restore.c b/src/restore.c index cd166e5d2..e5a05e985 100644 --- a/src/restore.c +++ b/src/restore.c @@ -775,6 +775,7 @@ restore_files_new(void *arg) int i; char to_fullpath[MAXPGPATH]; FILE *out = NULL; + char buffer[65536]; restore_files_arg_new *arguments = (restore_files_arg_new *) arg; @@ -857,6 +858,9 @@ restore_files_new(void *arg) to_fullpath, strerror(errno_tmp)); } + if (!fio_is_remote_file(out)) + setbuf(out, buffer); + if (!dest_file->is_datafile || dest_file->is_cfs) elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); else From 385c16b0d2168bfc00ecdd736492ac53a5233e1f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 6 Feb 2020 21:35:16 +0300 Subject: [PATCH 1138/2107] tests: fix elog messages --- tests/restore.py | 4 ++-- tests/validate.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index 7ff4465c1..93b28c800 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2371,11 +2371,11 @@ def test_lost_non_data_file(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'is not found', e.message, + 'No such file or directory', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Data files restoring failed', e.message, + 'ERROR: Backup files restoring failed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/validate.py b/tests/validate.py index 28a02139f..9a6a48f16 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -2699,9 +2699,8 @@ def test_file_size_corruption_no_validate(self): options=["--no-validate"]) except ProbackupException as e: self.assertTrue( - "ERROR: Data files restoring failed" in e.message, + "ERROR: Backup files restoring failed" in e.message, repr(e.message)) - # print "\nExpected error: \n" + e.message # Clean after yourself self.del_test_dir(module_name, fname) From a1e005efa273156b07b1d4cda48126ef72f81519 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 9 Feb 2020 21:13:42 +0300 Subject: [PATCH 1139/2107] [Issue #169] Minor improvements to restore --- src/restore.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/restore.c b/src/restore.c index e5a05e985..fae458125 100644 --- a/src/restore.c +++ b/src/restore.c @@ -721,9 +721,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (S_ISDIR(dest_file->mode)) continue; - if (params->skip_external_dirs && dest_file->external_dir_num > 0) - continue; - /* construct fullpath */ if (dest_file->external_dir_num == 0) { @@ -813,7 +810,7 @@ restore_files_new(void *arg) create_empty_file(FIO_BACKUP_HOST, arguments->to_root, FIO_DB_HOST, dest_file); - elog(INFO, "Skip file due to partial restore: \"%s\"", + elog(VERBOSE, "Skip file due to partial restore: \"%s\"", dest_file->rel_path); continue; } @@ -887,15 +884,15 @@ restore_files_new(void *arg) */ done: - // chmod - // close /* truncate file up to n_blocks. NOTE: no need, we just should not write * blocks that are exceeding n_blocks. * But for this to work, n_blocks should be trusted. */ - /* update file permission */ + /* update file permission + * TODO: chmod must be done right after fopen() + */ if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) { int errno_tmp = errno; @@ -910,6 +907,9 @@ restore_files_new(void *arg) strerror(errno)); } + /* ssh connection to longer needed */ + fio_disconnect(); + /* Data files restoring is successful */ arguments->ret = 0; From 2ad2af98f1bc28d01f6ff2340d4fc4599936e531 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 9 Feb 2020 21:34:13 +0300 Subject: [PATCH 1140/2107] [Issue #169] added function slurpFileFullPath() --- src/fetch.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/fetch.c b/src/fetch.c index 4dfd3a700..d65cdf7f2 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -13,6 +13,67 @@ #include #include +/* + * Read a file into memory. + * The file contents are returned in a malloc'd buffer, and *filesize + * is set to the length of the file. + * + * The returned buffer is always zero-terminated; the size of the returned + * buffer is actually *filesize + 1. That's handy when reading a text file. + * This function can be used to read binary files as well, you can just + * ignore the zero-terminator in that case. + * + */ +char * +slurpFileFullPath(const char *from_fullpath, size_t *filesize, bool safe, fio_location location) +{ + int fd; + char *buffer; + int len; + struct stat statbuf; + + if ((fd = fio_open(from_fullpath, O_RDONLY | PG_BINARY, location)) == -1) + { + if (safe) + return NULL; + else + elog(ERROR, "Could not open file \"%s\" for reading: %s", + from_fullpath, strerror(errno)); + } + + if (fio_fstat(fd, &statbuf) < 0) + { + if (safe) + return NULL; + else + elog(ERROR, "Could not stat file \"%s\": %s", + from_fullpath, strerror(errno)); + } + + len = statbuf.st_size; + + buffer = pg_malloc(len + 1); + + if (fio_read(fd, buffer, len) != len) + { + if (safe) + return NULL; + else + elog(ERROR, "Could not read file \"%s\": %s\n", + from_fullpath, strerror(errno)); + } + + fio_close(fd); + + /* Zero-terminate the buffer. */ + buffer[len] = '\0'; + + if (filesize) + *filesize = len; + + return buffer; +} + /* * Read a file into memory. The file to be read is /. * The file contents are returned in a malloc'd buffer, and *filesize From d37ade9096cdf7bdd05cc73b5d9df603ae9a85c6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 9 Feb 2020 21:35:21 +0300 Subject: [PATCH 1141/2107] validation should ignore empty and not changed files --- src/validate.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/validate.c b/src/validate.c index 663e1b39b..a5fae50b5 100644 --- a/src/validate.c +++ b/src/validate.c @@ -271,6 +271,10 @@ pgBackupValidateFiles(void *arg) continue; } + /* no point in trying to open empty or non-changed files */ + if (file->write_size <= 0) + continue; + if (stat(file->path, &st) == -1) { if (errno == ENOENT) From 3962b02253c15eaedd34789dd74b1c8c72fbd0ec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 17 Feb 2020 20:14:42 +0300 Subject: [PATCH 1142/2107] Allow validation of incremental backups with MERGING status --- src/validate.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/validate.c b/src/validate.c index a5fae50b5..67dbdf8b8 100644 --- a/src/validate.c +++ b/src/validate.c @@ -78,6 +78,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE && backup->status != BACKUP_STATUS_ORPHAN && + backup->status != BACKUP_STATUS_MERGING && backup->status != BACKUP_STATUS_CORRUPT) { elog(WARNING, "Backup %s has status %s. Skip validation.", @@ -86,14 +87,18 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) return; } - if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) - elog(INFO, "Validating backup %s", base36enc(backup->start_time)); - /* backups in MERGING status must have an option of revalidation without losing MERGING status - else if (backup->status == BACKUP_STATUS_MERGING) + /* additional sanity */ + if (backup->backup_mode == BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_MERGING) { - some message here + elog(WARNING, "Full backup %s has status %s, skip validation", + base36enc(backup->start_time), status2str(backup->status)); + return; } - */ + + if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE || + backup->status == BACKUP_STATUS_MERGING) + elog(INFO, "Validating backup %s", base36enc(backup->start_time)); else elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); From e1b0f1450f46f3c47781b0e6e5990753cb7b517b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Feb 2020 19:48:35 +0300 Subject: [PATCH 1143/2107] New fio command FIO_GET_CRC32 added and local behavior changed in fio_pread() --- src/utils/file.c | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/utils/file.h | 2 ++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 44bed31cb..73fd780f2 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -548,7 +548,15 @@ int fio_pread(FILE* f, void* buf, off_t offs) return hdr.arg; } else - return pread(fileno(f), buf, BLCKSZ, offs); + { + int rc; + rc = fseek(f, offs, SEEK_SET); + + if (rc < 0) + return rc; + + return fread(buf, 1, BLCKSZ, f); + } } /* Set position in stdio file */ @@ -898,6 +906,28 @@ int fio_sync(char const* path, fio_location location) } } +/* Get crc32 of file */ +pg_crc32 fio_get_crc32(const char *file_path, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + size_t path_len = strlen(file_path) + 1; + pg_crc32 crc = 0; + hdr.cop = FIO_GET_CRC32; + hdr.handle = -1; + hdr.size = path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); + IO_CHECK(fio_read_all(fio_stdin, &crc, sizeof(crc)), sizeof(crc)); + + return crc; + } + else + return pgFileGetCRCnew(file_path, true, true); +} + /* Remove file */ int fio_unlink(char const* path, fio_location location) { @@ -1403,7 +1433,11 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) return; } } - /* horizonLsn is not 0 for delta backup. As far as unsigned number are always greater or equal than zero, there is no sense to add more checks */ + /* + * horizonLsn is not 0 for delta backup. + * As far as unsigned number are always greater or equal than zero, + * there is no sense to add more checks. + */ if (page_lsn >= req->horizonLsn || page_lsn == InvalidXLogRecPtr) { char write_buffer[BLCKSZ*2]; @@ -1450,6 +1484,7 @@ void fio_communicate(int in, int out) struct stat st; int rc; int tmp_fd; + pg_crc32 crc; #ifdef WIN32 SYS_CHECK(setmode(in, _O_BINARY)); @@ -1591,6 +1626,11 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; + case FIO_GET_CRC32: + /* calculate crc32 for a file */ + crc = pgFileGetCRCnew(buf, true, true); + IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 563eb79f6..c919a2e69 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -36,6 +36,7 @@ typedef enum FIO_SEND_PAGES, FIO_PAGE, FIO_WRITE_COMPRESSED, + FIO_GET_CRC32 } fio_operations; typedef enum @@ -96,6 +97,7 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); +extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, fio_location location); From 9c662d5622dd3c8a5e844c67aa398e4805765e5e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 Feb 2020 19:59:57 +0300 Subject: [PATCH 1144/2107] [Issue #169] Major refactoring of backup, restore and merge internal algorihtms --- src/backup.c | 269 ++++----- src/catalog.c | 22 +- src/checkdb.c | 54 +- src/data.c | 833 ++++++++++------------------ src/delete.c | 68 +-- src/dir.c | 103 +++- src/merge.c | 1314 ++++++++++++++++++++++++++++++-------------- src/pg_probackup.c | 2 +- src/pg_probackup.h | 67 ++- src/restore.c | 346 +----------- src/util.c | 15 +- 11 files changed, 1549 insertions(+), 1544 deletions(-) diff --git a/src/backup.c b/src/backup.c index ff5157198..150503842 100644 --- a/src/backup.c +++ b/src/backup.c @@ -80,7 +80,7 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo); +static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn); @@ -129,7 +129,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) * Move files from 'pgdata' to a subdirectory in 'backup_path'. */ static void -do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) +do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) { int i; char database_path[MAXPGPATH]; @@ -155,6 +155,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* for fancy reporting */ time_t start_time, end_time; + char pretty_time[20]; char pretty_bytes[20]; elog(LOG, "Database backup start"); @@ -452,7 +453,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) parray_qsort(backup_files_list, pgFileCompareSize); /* Sort the array for binary search */ if (prev_backup_filelist) - parray_qsort(prev_backup_filelist, pgFileComparePathWithExternal); + parray_qsort(prev_backup_filelist, pgFileCompareRelPathWithExternal); /* write initial backup_content.control file and update backup.control */ write_backup_filelist(¤t, backup_files_list, @@ -485,6 +486,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* Run threads */ thread_interrupted = false; elog(INFO, "Start transferring data files"); + time(&start_time); for (i = 0; i < num_threads; i++) { backup_files_arg *arg = &(threads_args[i]); @@ -500,10 +502,16 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) if (threads_args[i].ret == 1) backup_isok = false; } + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); if (backup_isok) - elog(INFO, "Data files are transferred"); + elog(INFO, "Data files are transferred, time elapsed: %s", + pretty_time); else - elog(ERROR, "Data files transferring failed"); + elog(ERROR, "Data files transferring failed, time elapsed: %s", + pretty_time); /* Remove disappeared during backup files from backup_list */ for (i = 0; i < parray_num(backup_files_list); i++) @@ -589,7 +597,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) { char *ptr = file->path; - file->path = pstrdup(GetRelativePath(ptr, database_path)); + file->path = pgut_strdup(GetRelativePath(ptr, database_path)); + file->rel_path = pgut_strdup(file->path); free(ptr); } } @@ -613,6 +622,48 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* update backup control file to update size info */ write_backup(¤t); + /* Sync all copied files unless '--no-sync' flag is used */ + if (no_sync) + elog(WARNING, "Backup files are not synced to disk"); + else + { + elog(INFO, "Syncing backup files to disk"); + time(&start_time); + + for (i = 0; i < parray_num(backup_files_list); i++) + { + char to_fullpath[MAXPGPATH]; + pgFile *file = (pgFile *) parray_get(backup_files_list, i); + + /* TODO: sync directory ? */ + if (S_ISDIR(file->mode)) + continue; + + if (file->write_size <= 0) + continue; + + /* construct fullpath */ + if (file->external_dir_num == 0) + join_path_components(to_fullpath, database_path, file->rel_path); + else + { + char external_dst[MAXPGPATH]; + + makeExternalDirPathByNum(external_dst, external_prefix, + file->external_dir_num); + join_path_components(to_fullpath, external_dst, file->rel_path); + } + + if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); + } + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time); + } + /* clean external directories list */ if (external_dirs) free_dir_list(external_dirs); @@ -696,7 +747,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) */ int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params) + pgSetBackupParams *set_backup_params, bool no_sync) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -795,7 +846,7 @@ do_backup(time_t start_time, bool no_validate, elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); /* backup data */ - do_backup_instance(backup_conn, &nodeInfo); + do_backup_instance(backup_conn, &nodeInfo, no_sync); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -1928,18 +1979,24 @@ static void * backup_files(void *arg) { int i; - backup_files_arg *arguments = (backup_files_arg *) arg; - int n_backup_files_list = parray_num(arguments->files_list); + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; static time_t prev_time; + backup_files_arg *arguments = (backup_files_arg *) arg; + int n_backup_files_list = parray_num(arguments->files_list); + prev_time = current.start_time; /* backup a file */ for (i = 0; i < n_backup_files_list; i++) { - int ret; - struct stat buf; - pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + pgFile *prev_file = NULL; + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; if (arguments->thread_num == 1) { @@ -1957,7 +2014,6 @@ backup_files(void *arg) if (!pg_atomic_test_set_flag(&file->lock)) continue; - elog(VERBOSE, "Copying file: \"%s\"", file->path); /* check for interrupt */ if (interrupted || thread_interrupted) @@ -1965,139 +2021,84 @@ backup_files(void *arg) if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_backup_files_list, file->path); + i + 1, n_backup_files_list, file->rel_path); - /* stat file to check its current state */ - ret = fio_stat(file->path, &buf, true, FIO_DB_HOST); - if (ret == -1) + /* Handle zero sized files */ + if (file->size == 0) { - if (errno == ENOENT) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - file->write_size = FILE_NOT_FOUND; - elog(LOG, "File \"%s\" is not found", file->path); - continue; - } - else - { - elog(ERROR, - "can't stat file to backup \"%s\": %s", - file->path, strerror(errno)); - } - } - - /* We have already copied all directories */ - if (S_ISDIR(buf.st_mode)) + file->write_size = 0; continue; + } - if (S_ISREG(buf.st_mode)) + /* construct destination filepath */ + if (file->external_dir_num == 0) + { + join_path_components(from_fullpath, arguments->from_root, file->rel_path); + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + } + else { - pgFile **prev_file = NULL; - char *external_path = NULL; + char external_dst[MAXPGPATH]; + char *external_path = parray_get(arguments->external_dirs, + file->external_dir_num - 1); - if (file->external_dir_num) - external_path = parray_get(arguments->external_dirs, - file->external_dir_num - 1); + makeExternalDirPathByNum(external_dst, + arguments->external_prefix, + file->external_dir_num); - /* Check that file exist in previous backup */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - char *relative; - pgFile key; - - relative = GetRelativePath(file->path, file->external_dir_num ? - external_path : arguments->from_root); - key.path = relative; - key.external_dir_num = file->external_dir_num; - - prev_file = (pgFile **) parray_bsearch(arguments->prev_filelist, - &key, pgFileComparePathWithExternal); - if (prev_file) - /* File exists in previous backup */ - file->exists_in_prev = true; - } + join_path_components(to_fullpath, external_dst, file->rel_path); + join_path_components(from_fullpath, external_path, file->rel_path); + } - /* copy the file into backup */ - if (file->is_datafile && !file->is_cfs) - { - char to_path[MAXPGPATH]; - - join_path_components(to_path, arguments->to_root, - file->path + strlen(arguments->from_root) + 1); - - if (current.backup_mode != BACKUP_MODE_FULL) - file->n_blocks = file->size/BLCKSZ; - - /* backup block by block if datafile AND not compressed by cfs*/ - if (!backup_data_file(arguments, to_path, file, - arguments->prev_start_lsn, - current.backup_mode, - instance_config.compress_alg, - instance_config.compress_level, - arguments->nodeInfo->checksum_version, - arguments->nodeInfo->ptrack_version_num, - arguments->nodeInfo->ptrack_schema, - true)) - { - /* disappeared file not to be confused with 'not changed' */ - if (file->write_size != FILE_NOT_FOUND) - file->write_size = BYTES_INVALID; - elog(VERBOSE, "File \"%s\" was not copied to backup", file->path); - continue; - } - } - else if (!file->external_dir_num && - strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(arguments->from_root, FIO_DB_HOST, - arguments->to_root, FIO_BACKUP_HOST, - file); - else - { - const char *dst; - bool skip = false; - char external_dst[MAXPGPATH]; + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); - /* If non-data file has not changed since last backup... */ - if (prev_file && file->exists_in_prev && - buf.st_mtime < current.parent_backup) - { - file->crc = pgFileGetCRC(file->path, true, false, - &file->read_size, FIO_DB_HOST); - file->write_size = file->read_size; - /* ...and checksum is the same... */ - if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc)) - skip = true; /* ...skip copying file. */ - } - /* Set file paths */ - if (file->external_dir_num) - { - makeExternalDirPathByNum(external_dst, - arguments->external_prefix, - file->external_dir_num); - dst = external_dst; - } - else - dst = arguments->to_root; - if (skip || - !copy_file(FIO_DB_HOST, dst, FIO_BACKUP_HOST, file, true)) - { - /* disappeared file not to be confused with 'not changed' */ - if (file->write_size != FILE_NOT_FOUND) - file->write_size = BYTES_INVALID; - elog(VERBOSE, "File \"%s\" was not copied to backup", - file->path); - continue; - } + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + pgFile **prev_file_tmp = NULL; + prev_file_tmp = (pgFile **) parray_bsearch(arguments->prev_filelist, + file, pgFileCompareRelPathWithExternal); + if (prev_file_tmp) + { + /* File exists in previous backup */ + file->exists_in_prev = true; + prev_file = *prev_file_tmp; } + } - elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", - file->path, file->write_size); + /* backup file */ + if (file->is_datafile && !file->is_cfs) + { + backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, + arguments->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arguments->nodeInfo->checksum_version, + arguments->nodeInfo->ptrack_version_num, + arguments->nodeInfo->ptrack_schema, + true); } else - elog(WARNING, "unexpected file type %d", buf.st_mode); + { + backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, true); + } + + if (file->write_size == FILE_NOT_FOUND) + continue; + + if (file->write_size == BYTES_INVALID) + { + elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + from_fullpath, file->write_size); } /* ssh connection to longer needed */ diff --git a/src/catalog.c b/src/catalog.c index d6bc2dd41..5221ccab7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -127,7 +127,7 @@ lock_backup(pgBackup *backup) pid_t my_pid, my_p_pid; - pgBackupGetPath(backup, lock_file, lengthof(lock_file), BACKUP_CATALOG_PID); + join_path_components(lock_file, backup->root_dir, BACKUP_CATALOG_PID); /* * If the PID in the lockfile is our own PID or our parent's or @@ -674,6 +674,7 @@ pgBackupCreateDir(pgBackup *backup) elog(ERROR, "backup destination is not empty \"%s\"", path); fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); + backup->root_dir = pgut_strdup(path); /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) @@ -1491,6 +1492,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fio_fprintf(out, "expire-time = '%s'\n", timestamp); } + if (backup->merge_dest_backup != 0) + fio_fprintf(out, "merge-dest-id = '%s'\n", base36enc(backup->merge_dest_backup)); + /* * Size of PGDATA directory. The size does not include size of related * WAL segments in archive 'wal' directory. @@ -1699,9 +1703,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; - backup->wal_bytes = wal_size_on_disk; backup->uncompressed_bytes = uncompressed_size_on_disk; + if (backup->stream) + backup->wal_bytes = wal_size_on_disk; + free(buf); } @@ -1719,6 +1725,7 @@ readBackupControlFile(const char *path) char *stop_lsn = NULL; char *status = NULL; char *parent_backup = NULL; + char *merge_dest_backup = NULL; char *program_version = NULL; char *server_version = NULL; char *compress_alg = NULL; @@ -1748,6 +1755,7 @@ readBackupControlFile(const char *path) {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, {'s', 0, "status", &status, SOURCE_FILE_STRICT}, {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, + {'s', 0, "merge-dest-id", &merge_dest_backup, SOURCE_FILE_STRICT}, {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, @@ -1820,6 +1828,8 @@ readBackupControlFile(const char *path) backup->status = BACKUP_STATUS_RUNNING; else if (strcmp(status, "MERGING") == 0) backup->status = BACKUP_STATUS_MERGING; + else if (strcmp(status, "MERGED") == 0) + backup->status = BACKUP_STATUS_MERGED; else if (strcmp(status, "DELETING") == 0) backup->status = BACKUP_STATUS_DELETING; else if (strcmp(status, "DELETED") == 0) @@ -1841,6 +1851,12 @@ readBackupControlFile(const char *path) free(parent_backup); } + if (merge_dest_backup) + { + backup->merge_dest_backup = base36dec(merge_dest_backup); + free(merge_dest_backup); + } + if (program_version) { StrNCpy(backup->program_version, program_version, @@ -2003,12 +2019,14 @@ pgBackupInit(pgBackup *backup) backup->stream = false; backup->from_replica = false; backup->parent_backup = INVALID_BACKUP_ID; + backup->merge_dest_backup = INVALID_BACKUP_ID; backup->parent_backup_link = NULL; backup->primary_conninfo = NULL; backup->program_version[0] = '\0'; backup->server_version[0] = '\0'; backup->external_dir_str = NULL; backup->root_dir = NULL; + backup->files = NULL; } /* free pgBackup object */ diff --git a/src/checkdb.c b/src/checkdb.c index e878d688f..8ae4255cf 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -39,6 +39,8 @@ typedef struct ConnectionArgs conn_arg; /* number of thread for debugging */ int thread_num; + /* pgdata path */ + const char *from_root; /* * Return value from the thread: * 0 everything is ok @@ -130,6 +132,7 @@ check_files(void *arg) int i; check_files_arg *arguments = (check_files_arg *) arg; int n_files_list = 0; + char from_fullpath[MAXPGPATH]; if (arguments->files_list) n_files_list = parray_num(arguments->files_list); @@ -137,49 +140,28 @@ check_files(void *arg) /* check a file */ for (i = 0; i < n_files_list; i++) { - int ret; - struct stat buf; pgFile *file = (pgFile *) parray_get(arguments->files_list, i); + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "interrupted during checkdb"); + + /* No need to check directories */ + if (S_ISDIR(file->mode)) + continue; + if (!pg_atomic_test_set_flag(&file->lock)) continue; - elog(VERBOSE, "Checking file: \"%s\" ", file->path); + join_path_components(from_fullpath, arguments->from_root, file->rel_path); - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during checkdb"); + elog(VERBOSE, "Checking file: \"%s\" ", from_fullpath); if (progress) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_files_list, file->path); - - /* stat file to check its current state */ - ret = stat(file->path, &buf); - if (ret == -1) - { - if (errno == ENOENT) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - elog(LOG, "File \"%s\" is not found", file->path); - continue; - } - else - { - elog(ERROR, - "can't stat file to check \"%s\": %s", - file->path, strerror(errno)); - } - } - - /* No need to check directories */ - if (S_ISDIR(buf.st_mode)) - continue; + i + 1, n_files_list, from_fullpath); - if (S_ISREG(buf.st_mode)) + if (S_ISREG(file->mode)) { /* check only uncompressed by cfs datafiles */ if (file->is_datafile && !file->is_cfs) @@ -189,13 +171,14 @@ check_files(void *arg) * uses global variables to set connections. * Need refactoring. */ - if (!check_data_file(&(arguments->conn_arg), file, + if (!check_data_file(&(arguments->conn_arg), + file, from_fullpath, arguments->checksum_version)) arguments->ret = 2; /* corruption found */ } } else - elog(WARNING, "unexpected file type %d", buf.st_mode); + elog(WARNING, "unexpected file type %d", file->mode); } /* Ret values: @@ -258,6 +241,7 @@ do_block_validation(char *pgdata, uint32 checksum_version) arg->files_list = files_list; arg->checksum_version = checksum_version; + arg->from_root = pgdata; arg->conn_arg.conn = NULL; arg->conn_arg.cancel_conn = NULL; diff --git a/src/data.c b/src/data.c index 8ab42714a..780d97449 100644 --- a/src/data.c +++ b/src/data.c @@ -194,9 +194,14 @@ parse_page(Page page, XLogRecPtr *lsn) /* Read one page from file directly accessing disk * return value: - * 0 - if the page is not found + * 2 - if the page is found but zeroed * 1 - if the page is found and valid - * -1 - if the page is found but invalid + * 0 - if the page is not found, probably truncated + * -1 - if the page is found but read size is not multiple of BLKSIZE + * -2 - if the page is found but page header is "insane" + * -3 - if the page is found but page checksumm is wrong + * -4 - something went wrong, check errno + * */ static int read_page_from_file(pgFile *file, BlockNumber blknum, @@ -211,20 +216,14 @@ read_page_from_file(pgFile *file, BlockNumber blknum, if (read_len != BLCKSZ) { + /* The block could have been truncated. It is fine. */ if (read_len == 0) - { - elog(VERBOSE, "File \"%s\", block %u, file was truncated", - file->path, blknum); return 0; - } - else - { - elog(WARNING, "File: \"%s\", block %u, expected block size %u," - "but read %zu, try again", - file->path, blknum, BLCKSZ, read_len); + else if (read_len > 0) return -1; - } + else + return -4; } /* @@ -242,18 +241,9 @@ read_page_from_file(pgFile *file, BlockNumber blknum, /* Page is zeroed. No need to check header and checksum. */ if (i == BLCKSZ) - { - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", file->path, blknum); - return 1; - } + return 2; - /* - * If page is not completely empty and we couldn't parse it, - * try again several times. If it didn't help, throw error - */ - elog(LOG, "File: \"%s\" blknum %u have wrong page header, try again", - file->path, blknum); - return -1; + return -2; } /* Verify checksum */ @@ -265,22 +255,14 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * several times. If it didn't help, throw error */ if (pg_checksum_page(page, blkno) != ((PageHeader) page)->pd_checksum) - { - elog(LOG, "File: \"%s\" blknum %u have wrong checksum, try again", - file->path, blknum); - return -1; - } + return -3; else - { /* page header and checksum are correct */ return 1; - } } else - { /* page header is correct and checksum check is disabled */ return 1; - } } /* @@ -295,15 +277,15 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * PageIsCorrupted(-4) if the page check mismatch */ static int32 -prepare_page(ConnectionArgs *arguments, +prepare_page(ConnectionArgs *conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, BlockNumber nblocks, - FILE *in, BlockNumber *n_skipped, - BackupMode backup_mode, + FILE *in, BackupMode backup_mode, Page page, bool strict, uint32 checksum_version, int ptrack_version_num, - const char *ptrack_schema) + const char *ptrack_schema, + const char *from_fullpath) { XLogRecPtr page_lsn = 0; int try_again = 100; @@ -322,36 +304,67 @@ prepare_page(ConnectionArgs *arguments, */ if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 20) { - while(!page_is_valid && try_again) + while (!page_is_valid && try_again) { int result = read_page_from_file(file, blknum, in, page, &page_lsn, checksum_version); - try_again--; - if (result == 0) + switch (result) { - /* This block was truncated.*/ - page_is_truncated = true; - /* Page is not actually valid, but it is absent - * and we're not going to reread it or validate */ - page_is_valid = true; - } + case 2: + page_is_valid = true; + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + break; + + case 1: + page_is_valid = true; + break; + + case 0: + /* This block was truncated.*/ + page_is_truncated = true; + /* Page is not actually valid, but it is absent + * and we're not going to reread it or validate */ + page_is_valid = true; + + elog(VERBOSE, "File \"%s\", block %u, file was truncated", + from_fullpath, blknum); + break; + + case -1: + elog(WARNING, "File: \"%s\", block %u, partial read, try again", + from_fullpath, blknum); + break; + + case -2: + elog(LOG, "File: \"%s\" blknum %u have wrong page header, try again", + from_fullpath, blknum); + break; + + case -3: + elog(LOG, "File: \"%s\" blknum %u have wrong checksum, try again", + from_fullpath, blknum); + break; - if (result == 1) - page_is_valid = true; + case -4: + elog(LOG, "File: \"%s\" access error: %s", + from_fullpath, strerror(errno)); + break; + } /* * If ptrack support is available use it to get invalid block * instead of rereading it 99 times */ - //elog(WARNING, "Checksum_Version: %i", checksum_version ? 1 : 0); - if (result == -1 && strict && ptrack_version_num > 0) + if (result < 0 && strict && ptrack_version_num > 0) { elog(WARNING, "File \"%s\", block %u, try to fetch via shared buffer", - file->path, blknum); + from_fullpath, blknum); break; } + + try_again--; } /* * If page is not valid after 100 attempts to read it @@ -363,7 +376,7 @@ prepare_page(ConnectionArgs *arguments, { /* show this message for checkdb, merge or backup without ptrack support */ elog(WARNING, "Corruption detected in file \"%s\", block %u", - file->path, blknum); + from_fullpath, blknum); } /* Backup with invalid block and without ptrack support must throw error */ @@ -374,7 +387,7 @@ prepare_page(ConnectionArgs *arguments, if (!strict) { if (page_is_valid) - return 0; + return PageIsOk; else return PageIsCorrupted; } @@ -386,7 +399,7 @@ prepare_page(ConnectionArgs *arguments, { size_t page_size = 0; Page ptrack_page = NULL; - ptrack_page = (Page) pg_ptrack_get_block(arguments, file->dbOid, file->tblspcOid, + ptrack_page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size, ptrack_version_num, ptrack_schema); @@ -399,7 +412,7 @@ prepare_page(ConnectionArgs *arguments, { free(ptrack_page); elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", - file->path, absolute_blknum, BLCKSZ, page_size); + from_fullpath, absolute_blknum, BLCKSZ, page_size); } else { @@ -424,99 +437,77 @@ prepare_page(ConnectionArgs *arguments, } - /* Nullified pages must be copied by DELTA backup, just to be safe */ + if (page_is_truncated) + return PageIsTruncated; + + /* + * Skip page if page lsn is less than START_LSN of parent backup. + * Nullified pages must be copied by DELTA backup, just to be safe. + */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - !page_is_truncated && page_lsn && page_lsn < prev_backup_start_lsn) { - elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, file->path); - (*n_skipped)++; + elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, from_fullpath); return SkipCurrentPage; } - if (page_is_truncated) - return PageIsTruncated; - - return 0; + return PageIsOk; } static void compress_and_backup_page(pgFile *file, BlockNumber blknum, FILE *in, FILE *out, pg_crc32 *crc, int page_state, Page page, - CompressAlg calg, int clevel) + CompressAlg calg, int clevel, + const char *from_fullpath, const char *to_fullpath) { BackupPageHeader header; size_t write_buffer_size = sizeof(header); char write_buffer[BLCKSZ+sizeof(header)]; char compressed_page[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ - - if (page_state == SkipCurrentPage) - return; + const char *errormsg = NULL; header.block = blknum; header.compressed_size = page_state; - if(page_state == PageIsTruncated) + + /* The page was not truncated, so we need to compress it */ + header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), + page, BLCKSZ, calg, clevel, + &errormsg); + /* Something went wrong and errormsg was assigned, throw a warning */ + if (header.compressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", + blknum, from_fullpath, errormsg); + + file->compress_alg = calg; + + /* The page was successfully compressed. */ + if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) { - /* - * The page was truncated. Write only header - * to know that we must truncate restored file - */ memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), + compressed_page, header.compressed_size); + write_buffer_size += MAXALIGN(header.compressed_size); } + /* Non-positive value means that compression failed. Write it as is. */ else { - const char *errormsg = NULL; - - /* The page was not truncated, so we need to compress it */ - header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), - page, BLCKSZ, calg, clevel, - &errormsg); - /* Something went wrong and errormsg was assigned, throw a warning */ - if (header.compressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", - blknum, file->path, errormsg); - - file->compress_alg = calg; - file->read_size += BLCKSZ; - - /* The page was successfully compressed. */ - if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) - { - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), - compressed_page, header.compressed_size); - write_buffer_size += MAXALIGN(header.compressed_size); - } - /* Non-positive value means that compression failed. Write it as is. */ - else - { - header.compressed_size = BLCKSZ; - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), page, BLCKSZ); - write_buffer_size += header.compressed_size; - } + header.compressed_size = BLCKSZ; + memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer + sizeof(header), page, BLCKSZ); + write_buffer_size += header.compressed_size; } - /* elog(VERBOSE, "backup blkno %u, compressed_size %d write_buffer_size %ld", - blknum, header.compressed_size, write_buffer_size); */ - /* Update CRC */ COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); /* write data page */ if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) - { - int errno_tmp = errno; - - fclose(in); - fio_fclose(out); - elog(ERROR, "File: \"%s\", cannot write backup at block %u: %s", - file->path, blknum, strerror(errno_tmp)); - } + elog(ERROR, "File: \"%s\", cannot write at block %u: %s", + to_fullpath, blknum, strerror(errno)); file->write_size += write_buffer_size; file->uncompressed_size += BLCKSZ; @@ -530,9 +521,9 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, * incremental backup), validate checksum, optionally compress and write to * backup with special header. */ -bool -backup_data_file(backup_files_arg* arguments, - const char *to_path, pgFile *file, +void +backup_data_file(ConnectionArgs* conn_arg, pgFile *file, + const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok) @@ -540,12 +531,31 @@ backup_data_file(backup_files_arg* arguments, FILE *in; FILE *out; BlockNumber blknum = 0; - BlockNumber nblocks = 0; + BlockNumber nblocks = 0; /* number of blocks in file */ BlockNumber n_blocks_skipped = 0; - BlockNumber n_blocks_read = 0; /* number of blocks actually readed */ + BlockNumber n_blocks_read = 0; /* number of blocks actually readed + * TODO: we should report them */ int page_state; char curr_page[BLCKSZ]; + /* stdio buffers */ + char in_buffer[STDIO_BUFSIZE]; + char out_buffer[STDIO_BUFSIZE]; + + /* sanity */ + if (file->size % BLCKSZ != 0) + elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + nblocks = file->size/BLCKSZ; + + /* set n_blocks for a file */ + file->n_blocks = nblocks; + /* * Skip unchanged file only if it exists in previous backup. * This way we can correctly handle null-sized files which are @@ -557,11 +567,11 @@ backup_data_file(backup_files_arg* arguments, file->exists_in_prev && !file->pagemap_isabsent) { /* - * There are no changed blocks since last backup. We want make + * There are no changed blocks since last backup. We want to make * incremental backup, so we should exit. */ - elog(VERBOSE, "Skipping the unchanged file: \"%s\"", file->path); - return false; + file->write_size = BYTES_INVALID; + return; } /* reset size summary */ @@ -570,8 +580,8 @@ backup_data_file(backup_files_arg* arguments, file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); - /* open source mode file for read */ - in = fio_fopen(file->path, PG_BINARY_R, FIO_DB_HOST); + /* open source file for read */ + in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); if (in == NULL) { FIN_FILE_CRC32(true, file->crc); @@ -584,53 +594,58 @@ backup_data_file(backup_files_arg* arguments, { if (missing_ok) { - elog(LOG, "File \"%s\" is not found", file->path); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; - return false; + return; } else - elog(ERROR, "File \"%s\" is not found", file->path); + elog(ERROR, "File \"%s\" is not found", from_fullpath); } - elog(ERROR, "cannot open file \"%s\": %s", - file->path, strerror(errno)); + /* In all other cases throw an error */ + elog(ERROR, "Cannot open file \"%s\": %s", + from_fullpath, strerror(errno)); } - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; + if (!fio_is_remote_file(in)) + setbuf(in, in_buffer); /* open backup file for write */ - out = fio_fopen(to_path, PG_BINARY_W, FIO_BACKUP_HOST); + out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) - { - int errno_tmp = errno; - fio_fclose(in); - elog(ERROR, "cannot open backup file \"%s\": %s", - to_path, strerror(errno_tmp)); - } + elog(ERROR, "Cannot open backup file \"%s\": %s", + to_fullpath, strerror(errno)); + + setbuf(out, out_buffer); + + /* update file permission */ + if (chmod(to_fullpath, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); /* * Read each page, verify checksum and write it to backup. * If page map is empty or file is not present in previous backup * backup all pages of the relation. * - * We will enter here if backup_mode is FULL or DELTA. + * Usually enter here if backup_mode is FULL or DELTA. + * Also in some cases even PAGE backup is going here, + * becase not all data files are logged into WAL, + * for example CREATE DATABASE. + * Such files should be fully copied. + + * In PTRACK 1.x there was a problem + * of data files with missing _ptrack map. + * Such files should be fully copied. */ if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev) { - /* TODO: take into account PTRACK 2.0 */ - if (backup_mode != BACKUP_MODE_DIFF_PTRACK && fio_is_remote_file(in)) + if (fio_is_remote_file(in)) { int rc = fio_send_pages(in, out, file, - backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, &n_blocks_skipped, calg, clevel); if (rc == PAGE_CHECKSUM_MISMATCH && ptrack_version_num >= 15) @@ -638,9 +653,10 @@ backup_data_file(backup_files_arg* arguments, goto RetryUsingPtrack; if (rc < 0) elog(ERROR, "Failed to read file \"%s\": %s", - file->path, rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); + from_fullpath, + rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); - /* TODO: check that fio_send_pages ain`t bullshitting about number of readed blocks */ + /* TODO: check that fio_send_pages ain`t lying about number of readed blocks */ n_blocks_read = rc; file->read_size = n_blocks_read * BLCKSZ; @@ -651,22 +667,31 @@ backup_data_file(backup_files_arg* arguments, RetryUsingPtrack: for (blknum = 0; blknum < nblocks; blknum++) { - page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, - blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true, - checksum_version, ptrack_version_num, - ptrack_schema); - compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel); - n_blocks_read++; + page_state = prepare_page(conn_arg, file, prev_backup_start_lsn, + blknum, nblocks, in, backup_mode, + curr_page, true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath); + if (page_state == PageIsTruncated) break; + else if (page_state == SkipCurrentPage) + n_blocks_skipped++; + + else if (page_state == PageIsOk) + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel, + from_fullpath, to_fullpath); + else + elog(ERROR, "Illegal page state: %i, file: %s, blknum %i", + page_state, file->rel_path, blknum); + + n_blocks_read++; file->read_size += BLCKSZ; } } - if (backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = n_blocks_read; + file->n_blocks = n_blocks_read; } /* * If page map is not empty we scan only changed blocks. @@ -679,291 +704,98 @@ backup_data_file(backup_files_arg* arguments, iter = datapagemap_iterate(&file->pagemap); while (datapagemap_next(iter, &blknum)) { - page_state = prepare_page(&(arguments->conn_arg), file, prev_backup_start_lsn, - blknum, nblocks, in, &n_blocks_skipped, - backup_mode, curr_page, true, - checksum_version, ptrack_version_num, - ptrack_schema); - compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel); - n_blocks_read++; + page_state = prepare_page(conn_arg, file, prev_backup_start_lsn, + blknum, nblocks, in, backup_mode, + curr_page, true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath); + if (page_state == PageIsTruncated) break; + + else if (page_state == SkipCurrentPage) + n_blocks_skipped++; + + else if (page_state == PageIsOk) + compress_and_backup_page(file, blknum, in, out, &(file->crc), + page_state, curr_page, calg, clevel, + from_fullpath, to_fullpath); + else + elog(ERROR, "Illegal page state: %i, file: %s, blknum %i", + page_state, file->rel_path, blknum); + + n_blocks_read++; + file->read_size += BLCKSZ; } pg_free(file->pagemap.bitmap); pg_free(iter); } - /* update file permission */ - if (fio_chmod(to_path, FILE_PERMISSION, FIO_BACKUP_HOST) == -1) - { - int errno_tmp = errno; - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot change mode of \"%s\": %s", file->path, - strerror(errno_tmp)); - } + if (fclose(out)) + elog(ERROR, "Cannot close the backup file \"%s\": %s", + to_fullpath, strerror(errno)); - if (fio_fflush(out) != 0 || - fio_fclose(out)) - elog(ERROR, "cannot write backup file \"%s\": %s", - to_path, strerror(errno)); fio_fclose(in); FIN_FILE_CRC32(true, file->crc); + /* Determine that file didn`t changed in case of incremental backup */ + if (backup_mode != BACKUP_MODE_FULL && + file->exists_in_prev && + file->write_size == 0 && + file->n_blocks > 0) + { + file->write_size = BYTES_INVALID; + } + /* - * If we have pagemap then file in the backup can't be a zero size. - * Otherwise, we will clear the last file. - * - * TODO: maybe we should just check write_size to be >= 0. - * Or better yeat, open file only there is something to write. + * No point in storing empty files. */ - if (n_blocks_read != 0 && n_blocks_read == n_blocks_skipped) + if (file->write_size <= 0) { - if (fio_unlink(to_path, FIO_BACKUP_HOST) == -1) - elog(ERROR, "cannot remove file \"%s\": %s", to_path, + if (unlink(to_fullpath) == -1) + elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath, strerror(errno)); - return false; } - - return true; } /* - * Restore files in the from_root directory to the to_root directory with - * same relative path. - * - * If write_header is true then we add header to each restored block, currently - * it is used for MERGE command. + * Backup non data file + * We do not apply compression to this file. + * If file exists in previous backup, then compare checksums + * and make a decision about copying or skiping the file. */ void -restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, - bool write_header, uint32 backup_version) +backup_non_data_file(pgFile *file, pgFile *prev_file, + const char *from_fullpath, const char *to_fullpath, + BackupMode backup_mode, time_t parent_backup_time, + bool missing_ok) { - FILE *in = NULL; - FILE *out = NULL; - BackupPageHeader header; - BlockNumber blknum = 0, - truncate_from = 0; - bool need_truncate = false; - - /* BYTES_INVALID allowed only in case of restoring file from DELTA backup */ - if (file->write_size != BYTES_INVALID) - { - /* open backup mode file for read */ - in = fopen(file->path, PG_BINARY_R); - if (in == NULL) - { - elog(ERROR, "Cannot open backup file \"%s\": %s", file->path, - strerror(errno)); - } - } + /* special treatment for global/pg_control */ + if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) + return copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, + to_fullpath, FIO_BACKUP_HOST, file); /* - * Open backup file for write. We use "r+" at first to overwrite only - * modified pages for differential restore. If the file does not exist, - * re-open it with "w" to create an empty file. - */ - out = fio_fopen(to_path, PG_BINARY_R "+", FIO_DB_HOST); - if (out == NULL) - { - int errno_tmp = errno; - fclose(in); - elog(ERROR, "Cannot open restore target file \"%s\": %s", - to_path, strerror(errno_tmp)); - } - - while (true) + * If non-data file exists in previous backup + * and its mtime is less than parent backup start time ... */ + if (prev_file && file->exists_in_prev && + file->mtime <= parent_backup_time) { - off_t write_pos; - size_t read_len; - DataPage compressed_page; /* used as read buffer */ - DataPage page; - int32 uncompressed_size = 0; - - /* File didn`t changed. Nothing to copy */ - if (file->write_size == BYTES_INVALID) - break; - - /* - * We need to truncate result file if data file in an incremental backup - * less than data file in a full backup. We know it thanks to n_blocks. - * - * It may be equal to -1, then we don't want to truncate the result - * file. - */ - if (file->n_blocks != BLOCKNUM_INVALID && - (blknum + 1) > file->n_blocks) - { - truncate_from = blknum; - need_truncate = true; - break; - } - - /* read BackupPageHeader */ - read_len = fread(&header, 1, sizeof(header), in); - if (read_len != sizeof(header)) - { - int errno_tmp = errno; - if (read_len == 0 && feof(in)) - break; /* EOF found */ - else if (read_len != 0 && feof(in)) - elog(ERROR, - "Odd size page found at block %u of \"%s\"", - blknum, file->path); - else - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - blknum, file->path, strerror(errno_tmp)); - } - - if (header.block == 0 && header.compressed_size == 0) - { - elog(VERBOSE, "Skip empty block of \"%s\"", file->path); - continue; - } - - if (header.block < blknum) - elog(ERROR, "Backup is broken at block %u of \"%s\"", - blknum, file->path); - - blknum = header.block; - - if (header.compressed_size == PageIsTruncated) - { - /* - * Backup contains information that this block was truncated. - * We need to truncate file to this length. - */ - truncate_from = blknum; - need_truncate = true; - break; - } - - Assert(header.compressed_size <= BLCKSZ); - - /* read a page from file */ - read_len = fread(compressed_page.data, 1, - MAXALIGN(header.compressed_size), in); - if (read_len != MAXALIGN(header.compressed_size)) - elog(ERROR, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, file->path, read_len, header.compressed_size); - - /* - * if page size is smaller than BLCKSZ, decompress the page. - * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. - * we have to check, whether it is compressed or not using - * page_may_be_compressed() function. - */ - if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, - backup_version)) - { - const char *errormsg = NULL; - uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, - file->compress_alg, &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, file->path, errormsg); - - if (uncompressed_size != BLCKSZ) - elog(ERROR, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - file->path, uncompressed_size); - } - - write_pos = (write_header) ? blknum * (BLCKSZ + sizeof(header)) : - blknum * BLCKSZ; + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST); - /* - * Seek and write the restored page. - */ - if (fio_fseek(out, write_pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_path, strerror(errno)); - - if (write_header) - { - /* We uncompressed the page, so its size is BLCKSZ */ - header.compressed_size = BLCKSZ; - if (fio_fwrite(out, &header, sizeof(header)) != sizeof(header)) - elog(ERROR, "Cannot write header of block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); - } - - /* if we uncompressed the page - write page.data, - * if page wasn't compressed - - * write what we've read - compressed_page.data - */ - if (uncompressed_size == BLCKSZ) - { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); - } - else - { - if (fio_fwrite(out, compressed_page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); - } - } - - /* - * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do - * But during DELTA backup we read every file in PGDATA and thus DELTA backup - * knows exact size of every file at the time of backup. - * So when restoring file from DELTA backup we, knowing it`s size at - * a time of a backup, can truncate file to this size. - */ - if (allow_truncate && file->n_blocks != BLOCKNUM_INVALID && !need_truncate) - { - struct stat st; - if (fio_ffstat(out, &st) == 0 && st.st_size > file->n_blocks * BLCKSZ) + /* ...and checksum is the same... */ + if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) { - truncate_from = file->n_blocks; - need_truncate = true; + file->write_size = BYTES_INVALID; + return; /* ...skip copying file. */ } } - if (need_truncate) - { - off_t write_pos; - - write_pos = (write_header) ? truncate_from * (BLCKSZ + sizeof(header)) : - truncate_from * BLCKSZ; - - /* - * Truncate file to this length. - */ - if (fio_ftruncate(out, write_pos) != 0) - elog(ERROR, "Cannot truncate \"%s\": %s", - file->path, strerror(errno)); - elog(VERBOSE, "Delta truncate file %s to block %u", - file->path, truncate_from); - } - - /* update file permission */ - if (fio_chmod(to_path, file->mode, FIO_DB_HOST) == -1) - { - int errno_tmp = errno; - - if (in) - fclose(in); - fio_fclose(out); - elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, - strerror(errno_tmp)); - } - - if (fio_fflush(out) != 0 || - fio_fclose(out)) - elog(ERROR, "Cannot write \"%s\": %s", to_path, strerror(errno)); - - if (in) - fclose(in); + backup_non_data_file_internal(from_fullpath, FIO_DB_HOST, + to_fullpath, file, true); } /* @@ -972,11 +804,11 @@ restore_data_file(const char *to_path, pgFile *file, bool allow_truncate, * Apply changed blocks to destination file from every backup in parent chain. */ size_t -restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) +restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) { int i; size_t total_write_len = 0; - char buffer[65536]; + char buffer[STDIO_BUFSIZE]; for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -1002,10 +834,10 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const * and thus was not backed up. */ if (tmp_file->write_size == BYTES_INVALID) - { -// elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", tmp_file->rel_path); continue; - } + + if (tmp_file->write_size == 0) + continue; /* * At this point we are sure, that something is going to be copied @@ -1016,7 +848,7 @@ restore_data_file_new(parray *parent_chain, pgFile *dest_file, FILE *out, const in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) - elog(INFO, "Cannot open backup file \"%s\": %s", from_fullpath, + elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); setbuf(in, buffer); @@ -1196,9 +1028,8 @@ void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath) { - size_t read_len = 0; - int errno_tmp; - char buf[65536]; /* 64kB buffer */ + ssize_t read_len = 0; + char buf[STDIO_BUFSIZE]; /* 64kB buffer */ /* copy content */ for (;;) @@ -1209,41 +1040,18 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during non-data file restore"); - if ((read_len = fio_fread(in, buf, sizeof(buf))) != sizeof(buf)) - break; + read_len = fread(buf, 1, sizeof(buf), in); - if (fio_fwrite(out, buf, read_len) != read_len) - { - errno_tmp = errno; - /* oops */ - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno_tmp)); - } - } + if (read_len == 0 && feof(in)) + break; - errno_tmp = errno; - if (read_len < 0) - { - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "Cannot read backup mode file \"%s\": %s", - from_fullpath, strerror(errno_tmp)); - } + if (read_len < 0) + elog(ERROR, "Cannot read backup mode file \"%s\": %s", + from_fullpath, strerror(errno)); - /* copy odd part. */ - if (read_len > 0) - { if (fio_fwrite(out, buf, read_len) != read_len) - { - errno_tmp = errno; - /* oops */ - fio_fclose(in); - fio_fclose(out); elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno_tmp)); - } + strerror(errno)); } elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); @@ -1260,7 +1068,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *tmp_file = NULL; pgBackup *tmp_backup = NULL; - char buffer[65536]; + char buffer[STDIO_BUFSIZE]; /* Check if full copy of destination file is available in destination backup */ if (dest_file->write_size > 0) @@ -1350,16 +1158,16 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, * We do not apply compression to these files, because * it is either small control file or already compressed cfs file. */ -bool -copy_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file, bool missing_ok) +void +backup_non_data_file_internal(const char *from_fullpath, + fio_location from_location, + const char *to_fullpath, pgFile *file, + bool missing_ok) { - char to_path[MAXPGPATH]; FILE *in; FILE *out; - size_t read_len = 0; - int errno_tmp; - char buf[BLCKSZ]; + ssize_t read_len = 0; + char buf[STDIO_BUFSIZE]; /* 64kB buffer */ pg_crc32 crc; INIT_FILE_CRC32(true, crc); @@ -1369,8 +1177,8 @@ copy_file(fio_location from_location, const char *to_root, file->write_size = 0; file->uncompressed_size = 0; - /* open backup mode file for read */ - in = fio_fopen(file->path, PG_BINARY_R, from_location); + /* open source file for read */ + in = fio_fopen(from_fullpath, PG_BINARY_R, from_location); if (in == NULL) { FIN_FILE_CRC32(true, crc); @@ -1381,73 +1189,45 @@ copy_file(fio_location from_location, const char *to_root, { if (missing_ok) { - elog(LOG, "File \"%s\" is not found", file->path); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; - return false; + return; } else - elog(ERROR, "File \"%s\" is not found", file->path); + elog(ERROR, "File \"%s\" is not found", from_fullpath); } - elog(ERROR, "cannot open source file \"%s\": %s", file->path, + elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); } /* open backup file for write */ - join_path_components(to_path, to_root, file->rel_path); - out = fio_fopen(to_path, PG_BINARY_W, to_location); + out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) - { - int errno_tmp = errno; - fio_fclose(in); - elog(ERROR, "cannot open destination file \"%s\": %s", - to_path, strerror(errno_tmp)); - } + elog(ERROR, "Cannot open destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath, file->mode) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); /* copy content and calc CRC */ for (;;) { - read_len = 0; + read_len = fio_fread(in, buf, sizeof(buf)); - if ((read_len = fio_fread(in, buf, sizeof(buf))) != sizeof(buf)) + if (read_len == 0) break; - if (fio_fwrite(out, buf, read_len) != read_len) - { - errno_tmp = errno; - /* oops */ - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot write to \"%s\": %s", to_path, - strerror(errno_tmp)); - } - /* update CRC */ - COMP_FILE_CRC32(true, crc, buf, read_len); + if (read_len < 0) + elog(ERROR, "Cannot read from source file \"%s\": %s", + from_fullpath, strerror(errno)); - file->read_size += read_len; - } - - errno_tmp = errno; - if (read_len < 0) - { - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot read backup mode file \"%s\": %s", - file->path, strerror(errno_tmp)); - } + if (fwrite(buf, 1, read_len, out) != read_len) + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, + strerror(errno)); - /* copy odd part. */ - if (read_len > 0) - { - if (fio_fwrite(out, buf, read_len) != read_len) - { - errno_tmp = errno; - /* oops */ - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot write to \"%s\": %s", to_path, - strerror(errno_tmp)); - } /* update CRC */ COMP_FILE_CRC32(true, crc, buf, read_len); @@ -1462,22 +1242,9 @@ copy_file(fio_location from_location, const char *to_root, FIN_FILE_CRC32(true, crc); file->crc = crc; - /* update file permission */ - if (fio_chmod(to_path, file->mode, to_location) == -1) - { - errno_tmp = errno; - fio_fclose(in); - fio_fclose(out); - elog(ERROR, "cannot change mode of \"%s\": %s", to_path, - strerror(errno_tmp)); - } - - if (fio_fflush(out) != 0 || - fio_fclose(out)) - elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno)); + if (fclose(out)) + elog(ERROR, "Cannot write \"%s\": %s", to_fullpath, strerror(errno)); fio_fclose(in); - - return true; } /* @@ -1624,18 +1391,17 @@ validate_one_page(Page page, pgFile *file, * also returns true if the file was not found */ bool -check_data_file(ConnectionArgs *arguments, - pgFile *file, uint32 checksum_version) +check_data_file(ConnectionArgs *arguments, pgFile *file, + const char *from_fullpath, uint32 checksum_version) { FILE *in; BlockNumber blknum = 0; BlockNumber nblocks = 0; - BlockNumber n_blocks_skipped = 0; int page_state; char curr_page[BLCKSZ]; bool is_valid = true; - in = fopen(file->path, PG_BINARY_R); + in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) { /* @@ -1648,8 +1414,8 @@ check_data_file(ConnectionArgs *arguments, return true; } - elog(WARNING, "cannot open file \"%s\": %s", - file->path, strerror(errno)); + elog(WARNING, "Cannot open file \"%s\": %s", + from_fullpath, strerror(errno)); return false; } @@ -1665,10 +1431,11 @@ check_data_file(ConnectionArgs *arguments, for (blknum = 0; blknum < nblocks; blknum++) { - page_state = prepare_page(arguments, file, InvalidXLogRecPtr, - blknum, nblocks, in, &n_blocks_skipped, - BACKUP_MODE_FULL, curr_page, false, checksum_version, - 0, NULL); + + page_state = prepare_page(NULL, file, InvalidXLogRecPtr, + blknum, nblocks, in, BACKUP_MODE_FULL, + curr_page, false, checksum_version, + 0, NULL, from_fullpath); if (page_state == PageIsTruncated) break; diff --git a/src/delete.c b/src/delete.c index 96ab26fd2..4ca958e5d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -482,14 +482,14 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l */ keep_backup_id = base36enc_dup(keep_backup->start_time); - elog(INFO, "Merge incremental chain between FULL backup %s and backup %s", + elog(INFO, "Merge incremental chain between full backup %s and backup %s", base36enc(full_backup->start_time), keep_backup_id); pg_free(keep_backup_id); merge_list = parray_new(); /* Form up a merge list */ - while(keep_backup->parent_backup_link) + while (keep_backup->parent_backup_link) { parray_append(merge_list, keep_backup); keep_backup = keep_backup->parent_backup_link; @@ -515,6 +515,19 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l /* Lock merge chain */ catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); + /* Consider this extreme case */ + // PAGEa1 PAGEb1 both valid + // \ / + // FULL + + /* Check that FULL backup do not has multiple descendants + * full_backup always point to current full_backup after merge + */ +// if (is_prolific(backup_list, full_backup)) +// { +// elog(WARNING, "Backup %s has multiple valid descendants. " +// "Automatic merge is not possible.", base36enc(full_backup->start_time)); +// } /* Merge list example: * 0 PAGE3 @@ -522,38 +535,26 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l * 2 PAGE1 * 3 FULL * - * Consequentially merge incremental backups from PAGE1 to PAGE3 - * into FULL. + * Merge incremental chain from PAGE3 into FULL. */ + keep_backup = parray_get(merge_list, 0); + merge_chain(merge_list, full_backup, keep_backup); + backup_merged = true; + for (j = parray_num(merge_list) - 2; j >= 0; j--) { - pgBackup *from_backup = (pgBackup *) parray_get(merge_list, j); - - - /* Consider this extreme case */ - // PAGEa1 PAGEb1 both valid - // \ / - // FULL - - /* Check that FULL backup do not has multiple descendants - * full_backup always point to current full_backup after merge - */ - if (is_prolific(backup_list, full_backup)) - { - elog(WARNING, "Backup %s has multiple valid descendants. " - "Automatic merge is not possible.", base36enc(full_backup->start_time)); - break; - } - - merge_backups(full_backup, from_backup); - backup_merged = true; + pgBackup *tmp_backup = (pgBackup *) parray_get(merge_list, j); /* Try to remove merged incremental backup from both keep and purge lists */ - parray_rm(to_purge_list, from_backup, pgBackupCompareId); + parray_rm(to_purge_list, tmp_backup, pgBackupCompareId); parray_set(to_keep_list, i, NULL); } + pgBackupValidate(full_backup, NULL); + if (full_backup->status == BACKUP_STATUS_CORRUPT) + elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time)); + /* Cleanup */ parray_free(merge_list); } @@ -724,10 +725,10 @@ void delete_backup_files(pgBackup *backup) { size_t i; - char path[MAXPGPATH]; char timestamp[100]; - parray *files; + parray *files; size_t num_files; + char full_path[MAXPGPATH]; /* * If the backup was deleted already, there is nothing to do. @@ -752,8 +753,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); - pgBackupGetPath(backup, path, lengthof(path), NULL); - dir_list_file(files, path, false, true, true, 0, FIO_BACKUP_HOST); + dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); @@ -762,14 +762,16 @@ delete_backup_files(pgBackup *backup) { pgFile *file = (pgFile *) parray_get(files, i); - if (progress) - elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"", - i + 1, num_files, file->path); + join_path_components(full_path, backup->root_dir, file->rel_path); if (interrupted) elog(ERROR, "interrupted during delete backup"); - pgFileDelete(file); + if (progress) + elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", + i + 1, num_files, full_path); + + pgFileDelete(file, full_path); } parray_walk(files, pgFileFree); diff --git a/src/dir.c b/src/dir.c index 496518ad0..25dfcfaed 100644 --- a/src/dir.c +++ b/src/dir.c @@ -181,6 +181,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, file = pgFileInit(path, rel_path); file->size = st.st_size; file->mode = st.st_mode; + file->mtime = st.st_mtime; file->external_dir_num = external_dir_num; return file; @@ -226,33 +227,95 @@ pgFileInit(const char *path, const char *rel_path) * If the pgFile points directory, the directory must be empty. */ void -pgFileDelete(pgFile *file) +pgFileDelete(pgFile *file, const char *full_path) { if (S_ISDIR(file->mode)) { - if (rmdir(file->path) == -1) + if (rmdir(full_path) == -1) { if (errno == ENOENT) return; else if (errno == ENOTDIR) /* could be symbolic link */ goto delete_file; - elog(ERROR, "cannot remove directory \"%s\": %s", - file->path, strerror(errno)); + elog(ERROR, "Cannot remove directory \"%s\": %s", + full_path, strerror(errno)); } return; } delete_file: - if (remove(file->path) == -1) + if (remove(full_path) == -1) { if (errno == ENOENT) return; - elog(ERROR, "cannot remove file \"%s\": %s", file->path, + elog(ERROR, "Cannot remove file \"%s\": %s", full_path, strerror(errno)); } } +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + * TODO: add decompression option. + */ +pg_crc32 +pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok) +{ + FILE *fp; + pg_crc32 crc = 0; + char buf[STDIO_BUFSIZE]; + size_t len = 0; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(&buf, 1, sizeof(buf), fp); + + if (len == 0) + { + /* we either run into eof or error */ + if (feof(fp)) + break; + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + } + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + } + + FIN_FILE_CRC32(use_crc32c, crc); + fclose(fp); + + return crc; +} + /* * Read the file to compute its CRC. * As a handy side effect, we return filesize via bytes_read parameter. @@ -263,7 +326,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, { FILE *fp; pg_crc32 crc = 0; - char buf[1024]; + char buf[STDIO_BUFSIZE]; size_t len = 0; size_t total = 0; int errno_tmp; @@ -291,7 +354,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, elog(ERROR, "interrupted during CRC calculation"); len = fio_fread(fp, buf, sizeof(buf)); - if(len == 0) + if (len == 0) break; /* update CRC */ COMP_FILE_CRC32(use_crc32c, crc, buf, len); @@ -1150,7 +1213,7 @@ read_tablespace_map(parray *files, const char *backup_dir) void check_tablespace_mapping(pgBackup *backup) { - char this_backup_path[MAXPGPATH]; +// char this_backup_path[MAXPGPATH]; parray *links; size_t i; TablespaceListCell *cell; @@ -1158,8 +1221,8 @@ check_tablespace_mapping(pgBackup *backup) links = parray_new(); - pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); - read_tablespace_map(links, this_backup_path); +// pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); + read_tablespace_map(links, backup->root_dir); /* Sort links by the path of a linked file*/ parray_qsort(links, pgFileCompareLinked); @@ -1687,15 +1750,16 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ { FILE *fp; pgFile *file; - char path[MAXPGPATH]; + char database_dir[MAXPGPATH]; char database_map_path[MAXPGPATH]; - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); - join_path_components(database_map_path, path, DATABASE_MAP); +// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(database_dir, backup->root_dir, DATABASE_DIR); + join_path_components(database_map_path, database_dir, DATABASE_MAP); fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "Cannot open database map \"%s\": %s", path, + elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path, strerror(errno)); print_database_map(fp, database_map); @@ -1710,9 +1774,9 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); pfree(file->path); - file->path = strdup(DATABASE_MAP); - file->crc = pgFileGetCRC(database_map_path, true, false, - &file->read_size, FIO_BACKUP_HOST); + file->path = pgut_strdup(DATABASE_MAP); + file->crc = pgFileGetCRCnew(database_map_path, true, false); + file->write_size = file->read_size; file->uncompressed_size = file->read_size; parray_append(backup_files_list, file); @@ -1730,7 +1794,8 @@ read_database_map(pgBackup *backup) char path[MAXPGPATH]; char database_map_path[MAXPGPATH]; - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); +// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); diff --git a/src/merge.c b/src/merge.c index 4e871ddd5..2551104ce 100644 --- a/src/merge.c +++ b/src/merge.c @@ -16,16 +16,18 @@ typedef struct { - parray *to_files; - parray *files; - parray *from_external; + parray *merge_filelist; + parray *parent_chain; - pgBackup *to_backup; - pgBackup *from_backup; - const char *to_root; - const char *from_root; - const char *to_external_prefix; - const char *from_external_prefix; + pgBackup *dest_backup; + pgBackup *full_backup; + + const char *full_database_dir; + const char *full_external_prefix; + +// size_t in_place_merge_bytes; + bool compression_match; + bool program_version_match; /* * Return value from the thread. @@ -34,6 +36,7 @@ typedef struct int ret; } merge_files_arg; + static void *merge_files(void *arg); static void reorder_external_dirs(pgBackup *to_backup, parray *to_external, @@ -41,6 +44,17 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, static int get_external_index(const char *key, const parray *list); +static void +merge_data_file(parray *parent_chain, pgBackup *full_backup, + pgBackup *dest_backup, pgFile *dest_file, + pgFile *tmp_file, const char *to_root); + +static void +merge_non_data_file(parray *parent_chain, pgBackup *full_backup, + pgBackup *dest_backup, pgFile *dest_file, + pgFile *tmp_file, const char *full_database_dir, + const char *full_external_prefix); + /* * Implementation of MERGE command. * @@ -54,6 +68,7 @@ do_merge(time_t backup_id) parray *backups; parray *merge_list = parray_new(); pgBackup *dest_backup = NULL; + pgBackup *dest_backup_tmp = NULL; pgBackup *full_backup = NULL; int i; @@ -81,70 +96,290 @@ do_merge(time_t backup_id) backup->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ backup->status != BACKUP_STATUS_MERGING && + backup->status != BACKUP_STATUS_MERGED && backup->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", base36enc(backup->start_time), status2str(backup->status)); - if (backup->backup_mode == BACKUP_MODE_FULL) - elog(ERROR, "Backup %s is full backup", - base36enc(backup->start_time)); - dest_backup = backup; break; } } - /* sanity */ if (dest_backup == NULL) elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); - /* get full backup */ - full_backup = find_parent_full_backup(dest_backup); + /* It is possible to use FULL backup as target backup for merge. + * There are two possible cases: + * 1. The user want to merge FULL backup with closest incremental backup. + * In this case we must find suitable destination backup and merge them. + * + * 2. Previous merge has failed after destination backup was deleted, + * but before FULL backup was renamed: + * Example A: + * PAGE2_1 OK + * FULL2 OK + * PAGE1_1 MISSING/DELETING <- + * FULL1 MERGED/MERGING + */ + if (dest_backup->backup_mode == BACKUP_MODE_FULL) + { + full_backup = dest_backup; + dest_backup = NULL; + elog(INFO, "Merge target backup %s is full backup", + base36enc(full_backup->start_time)); + + /* sanity */ + if (full_backup->status == BACKUP_STATUS_DELETING) + elog(ERROR, "Backup %s has status: %s", + base36enc(full_backup->start_time), + status2str(full_backup->status)); + + /* Case #1 */ + if (full_backup->status == BACKUP_STATUS_OK || + full_backup->status == BACKUP_STATUS_DONE) + { + /* Check the case of FULL backup having more than one direct children */ + if (is_prolific(backups, full_backup)) + elog(ERROR, "Merge target is full backup and has multiple direct children, " + "you must specify child backup id you want to merge with"); + + elog(LOG, "Looking for closest incremental backup to merge with"); + + /* Look for closest child backup */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + /* skip unsuitable candidates */ + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) + continue; + + if (backup->parent_backup == full_backup->start_time) + { + dest_backup = backup; + break; + } + } + + /* sanity */ + if (dest_backup == NULL) + elog(ERROR, "Failed to find merge candidate, " + "backup %s has no valid children", + base36enc(full_backup->start_time)); + + } + /* Case #2 */ + else if (full_backup->status == BACKUP_STATUS_MERGING) + { + /* + * MERGING - merge was ongoing at the moment of crash. + * We must find destination backup and rerun merge. + * If destination backup is missing, then merge must be aborted, + * there is no recovery from this situation. + */ + + if (full_backup->merge_dest_backup == INVALID_BACKUP_ID) + elog(ERROR, "Failed to determine merge destination backup"); + + /* look up destination backup */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time == full_backup->merge_dest_backup) + { + dest_backup = backup; + break; + } + } + if (!dest_backup) + { + char *tmp_backup_id = base36enc_dup(full_backup->start_time); + elog(ERROR, "Full backup %s has unfinished merge with missing backup %s", + tmp_backup_id, + base36enc(full_backup->merge_dest_backup)); + pg_free(tmp_backup_id); + } + } + else if (full_backup->status == BACKUP_STATUS_MERGED) + { + /* + * MERGED - merge crashed after files were transfered, but + * before rename could take place. + * If destination backup is missing, this is ok. + * If destination backup is present, then it should be deleted. + * After that FULL backup must acquire destination backup ID. + */ + + /* destination backup may or may not exists */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->start_time == full_backup->merge_dest_backup) + { + dest_backup = backup; + break; + } + } + if (!dest_backup) + { + char *tmp_backup_id = base36enc_dup(full_backup->start_time); + elog(WARNING, "Full backup %s has unfinished merge with missing backup %s", + tmp_backup_id, + base36enc(full_backup->merge_dest_backup)); + pg_free(tmp_backup_id); + } + } + else + elog(ERROR, "Backup %s has status: %s", + base36enc(full_backup->start_time), + status2str(full_backup->status)); + } + else + { + /* + * Legal Case #1: + * PAGE2 OK <- target + * PAGE1 OK + * FULL OK + * Legal Case #2: + * PAGE2 MERGING <- target + * PAGE1 MERGING + * FULL MERGING + * Legal Case #3: + * PAGE2 MERGING <- target + * PAGE1 DELETING + * FULL MERGED + * Legal Case #4: + * PAGE2 MERGING <- target + * PAGE1 missing + * FULL MERGED + * Legal Case #5: + * PAGE2 DELETING <- target + * FULL MERGED + * Legal Case #6: + * PAGE2 MERGING <- target + * PAGE1 missing + * FULL MERGED + * Illegal Case #7: + * PAGE2 MERGING <- target + * PAGE1 missing + * FULL MERGING + */ + + if (dest_backup->status == BACKUP_STATUS_MERGING || + dest_backup->status == BACKUP_STATUS_DELETING) + elog(WARNING, "Rerun unfinished merge for backup %s", + base36enc(dest_backup->start_time)); + + /* First we should try to find parent FULL backup */ + full_backup = find_parent_full_backup(dest_backup); + + /* Chain is broken, one or more member of parent chain is missing */ + if (full_backup == NULL) + { + /* It is the legal state of affairs in Case #4, but + * only for MERGING incremental target backup and only + * if FULL backup has MERGED status. + */ + if (dest_backup->status != BACKUP_STATUS_MERGING) + elog(ERROR, "Failed to find parent full backup for %s", + base36enc(dest_backup->start_time)); + + /* Find FULL backup that has unfinished merge with dest backup */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->merge_dest_backup == dest_backup->start_time) + { + full_backup = backup; + break; + } + } + + if (!full_backup) + elog(ERROR, "Failed to find full backup that has unfinished merge" + "with backup %s, cannot rerun merge", + base36enc(dest_backup->start_time)); + + if (full_backup->status == BACKUP_STATUS_MERGED) + elog(WARNING, "Incremental chain is broken, try to recover unfinished merge"); + else + elog(ERROR, "Incremental chain is broken, merge is impossible to finish"); + } + else + { + if ((full_backup->status == BACKUP_STATUS_MERGED || + full_backup->status == BACKUP_STATUS_MERGED) && + dest_backup->start_time != full_backup->merge_dest_backup) + { + char *tmp_backup_id = base36enc_dup(full_backup->start_time); + elog(ERROR, "Full backup %s has unfinished merge with backup %s", + tmp_backup_id, base36enc(full_backup->merge_dest_backup)); + pg_free(tmp_backup_id); + } + + } + } /* sanity */ if (full_backup == NULL) elog(ERROR, "Parent full backup for the given backup %s was not found", base36enc(backup_id)); + /* At this point NULL as dest_backup is allowed only in case of full backup + * having status MERGED */ + if (dest_backup == NULL && full_backup->status != BACKUP_STATUS_MERGED) + elog(ERROR, "Cannot run merge for full backup %s", + base36enc(full_backup->start_time)); + /* sanity */ if (full_backup->status != BACKUP_STATUS_OK && full_backup->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ + full_backup->status != BACKUP_STATUS_MERGED && full_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Backup %s has status: %s", base36enc(full_backup->start_time), status2str(full_backup->status)); - /* form merge list */ - while(dest_backup->parent_backup_link) + /* Form merge list */ + dest_backup_tmp = dest_backup; + + /* While loop below may looks strange, it is done so on purpose + * to handle both whole and broken incremental chains. + */ + while (dest_backup_tmp) { /* sanity */ - if (dest_backup->status != BACKUP_STATUS_OK && - dest_backup->status != BACKUP_STATUS_DONE && + if (dest_backup_tmp->status != BACKUP_STATUS_OK && + dest_backup_tmp->status != BACKUP_STATUS_DONE && /* It is possible that previous merging was interrupted */ - dest_backup->status != BACKUP_STATUS_MERGING && - dest_backup->status != BACKUP_STATUS_DELETING) + dest_backup_tmp->status != BACKUP_STATUS_MERGING && + dest_backup_tmp->status != BACKUP_STATUS_MERGED && + dest_backup_tmp->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup->start_time), status2str(dest_backup->status)); + base36enc(dest_backup_tmp->start_time), + status2str(dest_backup_tmp->status)); + + if (dest_backup_tmp->backup_mode == BACKUP_MODE_FULL) + break; - parray_append(merge_list, dest_backup); - dest_backup = dest_backup->parent_backup_link; + parray_append(merge_list, dest_backup_tmp); + dest_backup_tmp = dest_backup_tmp->parent_backup_link; } - /* Add FULL backup for easy locking */ + /* Add FULL backup */ parray_append(merge_list, full_backup); /* Lock merge chain */ catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); - /* - * Found target and full backups, merge them and intermediate backups - */ - for (i = parray_num(merge_list) - 2; i >= 0; i--) - { - pgBackup *from_backup = (pgBackup *) parray_get(merge_list, i); - - merge_backups(full_backup, from_backup); - } + /* do actual merge */ + merge_chain(merge_list, full_backup, dest_backup); pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) @@ -159,130 +394,193 @@ do_merge(time_t backup_id) } /* - * Merge two backups data files using threads. - * - to_backup - FULL, from_backup - incremental. - * - move instance files from from_backup to to_backup - * - remove unnecessary directories and files from to_backup - * - update metadata of from_backup, it becames FULL backup + * Merge backup chain. + * dest_backup - incremental backup. + * parent_chain - array of backups starting with dest_backup and + * ending with full_backup. + * + * Copy backup files from incremental backups from parent_chain into + * full backup directory. + * Remove unnecessary directories and files from full backup directory. + * Update metadata of full backup to represent destination backup. */ void -merge_backups(pgBackup *to_backup, pgBackup *from_backup) +merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) { - char *to_backup_id = base36enc_dup(to_backup->start_time), - *from_backup_id = base36enc_dup(from_backup->start_time); - char to_backup_path[MAXPGPATH], - to_database_path[MAXPGPATH], - to_external_prefix[MAXPGPATH], - from_backup_path[MAXPGPATH], - from_database_path[MAXPGPATH], - from_external_prefix[MAXPGPATH], - control_file[MAXPGPATH]; - parray *files, - *to_files; - parray *to_external = NULL, - *from_external = NULL; - pthread_t *threads = NULL; - merge_files_arg *threads_args = NULL; int i; - time_t merge_time; - bool merge_isok = true; + char *dest_backup_id; + char full_external_prefix[MAXPGPATH]; + char full_database_dir[MAXPGPATH]; + parray *full_externals = NULL, + *dest_externals = NULL; - merge_time = time(NULL); - elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id); + parray *result_filelist = NULL; +// size_t total_in_place_merge_bytes = 0; + pthread_t *threads = NULL; + merge_files_arg *threads_args = NULL; + time_t merge_time; + bool merge_isok = true; + /* for fancy reporting */ + time_t end_time; + char pretty_time[20]; + /* in-place flags */ + bool compression_match = false; + bool program_version_match = false; /* It's redundant to check block checksumms during merge */ skip_block_validation = true; - /* - * Validate to_backup only if it is BACKUP_STATUS_OK. If it has - * BACKUP_STATUS_MERGING status then it isn't valid backup until merging - * finished. - */ - if (to_backup->status == BACKUP_STATUS_OK || - to_backup->status == BACKUP_STATUS_DONE) + /* Handle corner cases missing destination backup */ + if (dest_backup == NULL && + full_backup->status == BACKUP_STATUS_MERGED) + goto merge_rename; + + if (!dest_backup) + elog(ERROR, "Destination backup is missing, cannot continue merge"); + + elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); + + /* sanity */ + if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && + full_backup->merge_dest_backup != dest_backup->start_time) { - pgBackupValidate(to_backup, NULL); - if (to_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Interrupt merging"); + char *merge_dest_backup_current = base36enc_dup(dest_backup->start_time); + char *merge_dest_backup = base36enc_dup(full_backup->merge_dest_backup); + + elog(ERROR, "Cannot run merge for %s, because full backup %s has " + "unfinished merge with backup %s", + merge_dest_backup_current, + base36enc(full_backup->start_time), + merge_dest_backup); + + pg_free(merge_dest_backup_current); + pg_free(merge_dest_backup); } /* - * It is OK to validate from_backup if it has BACKUP_STATUS_OK or - * BACKUP_STATUS_MERGING status. + * Previous merging was interrupted during deleting source backup. It is + * safe just to delete it again. */ - Assert(from_backup->status == BACKUP_STATUS_OK || - from_backup->status == BACKUP_STATUS_DONE || - from_backup->status == BACKUP_STATUS_MERGING || - from_backup->status == BACKUP_STATUS_DELETING); - pgBackupValidate(from_backup, NULL); - if (from_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Interrupt merging"); + if (full_backup->status == BACKUP_STATUS_MERGED) + goto merge_delete; - /* - * Make backup paths. + /* Forward compatibility is not supported */ + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + if (parse_program_version(backup->program_version) > + parse_program_version(PROGRAM_VERSION)) + { + elog(ERROR, "Backup %s has been produced by pg_probackup version %s, " + "but current program version is %s. Forward compatibility " + "is not supported.", + base36enc(backup->start_time), + backup->program_version, + PROGRAM_VERSION); + } + } + + /* TODO: Should we keep relying on caller to provide valid parent_chain? */ + + /* If destination backup compression algorihtm differs from + * full backup compression algorihtm, then in-place merge is + * not possible. */ - pgBackupGetPath(to_backup, to_backup_path, lengthof(to_backup_path), NULL); - pgBackupGetPath(to_backup, to_database_path, lengthof(to_database_path), - DATABASE_DIR); - pgBackupGetPath(to_backup, to_external_prefix, lengthof(to_database_path), - EXTERNAL_DIR); - pgBackupGetPath(from_backup, from_backup_path, lengthof(from_backup_path), NULL); - pgBackupGetPath(from_backup, from_database_path, lengthof(from_database_path), - DATABASE_DIR); - pgBackupGetPath(from_backup, from_external_prefix, lengthof(from_database_path), - EXTERNAL_DIR); + if (full_backup->compress_alg == dest_backup->compress_alg) + compression_match = true; + else + elog(WARNING, "In-place merge is disabled because of compression " + "algorihtms mismatch"); /* - * Get list of files which will be modified or removed. + * If current program version differs from destination backup version, + * then in-place merge is not possible. */ - pgBackupGetPath(to_backup, control_file, lengthof(control_file), - DATABASE_FILE_LIST); - to_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - /* To delete from leaf, sort in reversed order */ - parray_qsort(to_files, pgFileCompareRelPathWithExternalDesc); + if (parse_program_version(dest_backup->program_version) == + parse_program_version(PROGRAM_VERSION)) + program_version_match = true; + else + elog(WARNING, "In-place merge is disabled because of program " + "versions mismatch"); + + /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ + join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); + /* Construct path to external dir: /backup_dir/instance_name/FULL/external */ + join_path_components(full_external_prefix, full_backup->root_dir, EXTERNAL_DIR); + /* - * Get list of files which need to be moved. + * Validate or revalidate all members of parent chain + * with sole exception of FULL backup. If it has MERGING status + * then it isn't valid backup until merging is finished. */ - pgBackupGetPath(from_backup, control_file, lengthof(control_file), - DATABASE_FILE_LIST); - files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - /* sort by size for load balancing */ - parray_qsort(files, pgFileCompareSize); + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + /* FULL backup is not to be validated if its status is MERGING */ + if (backup->backup_mode == BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_MERGING) + { + continue; + } + + pgBackupValidate(backup, NULL); + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status %s, merge is aborted", + base36enc(backup->start_time), status2str(backup->status)); + } /* - * Previous merging was interrupted during deleting source backup. It is - * safe just to delete it again. + * Get backup files. */ - if (from_backup->status == BACKUP_STATUS_DELETING) - goto delete_source_backup; + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + char control_file[MAXPGPATH]; - write_backup_status(to_backup, BACKUP_STATUS_MERGING, instance_name); - write_backup_status(from_backup, BACKUP_STATUS_MERGING, instance_name); + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - create_data_directories(files, to_database_path, from_backup_path, false, FIO_BACKUP_HOST); + join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); + backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + parray_qsort(backup->files, pgFileCompareRelPathWithExternal); + + /* Set MERGING status for every member of the chain */ + if (backup->backup_mode == BACKUP_MODE_FULL) + { + /* In case of FULL backup also remember backup_id of + * of destination backup we are merging with, so + * we can safely allow rerun merge in case of failure. + */ + backup->merge_dest_backup = dest_backup->start_time; + backup->status = BACKUP_STATUS_MERGING; + write_backup(backup); + } + else + write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name); + } - /* Create external directories lists */ - if (to_backup->external_dir_str) - to_external = make_external_directory_list(to_backup->external_dir_str, - false); - if (from_backup->external_dir_str) - from_external = make_external_directory_list(from_backup->external_dir_str, - false); + /* Create directories */ + create_data_directories(dest_backup->files, full_database_dir, + dest_backup->root_dir, false, FIO_BACKUP_HOST); + /* External directories stuff */ + if (dest_backup->external_dir_str) + dest_externals = make_external_directory_list(dest_backup->external_dir_str, false); + if (full_backup->external_dir_str) + full_externals = make_external_directory_list(full_backup->external_dir_str, false); /* - * Rename external directories in to_backup (if exists) - * according to numeration of external dirs in from_backup. + * Rename external directories in FULL backup (if exists) + * according to numeration of external dirs in destionation backup. */ - if (to_external) - reorder_external_dirs(to_backup, to_external, from_external); + if (full_externals && dest_externals) + reorder_external_dirs(full_backup, full_externals, dest_externals); /* Setup threads */ - for (i = 0; i < parray_num(files); i++) + for (i = 0; i < parray_num(dest_backup->files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(dest_backup->files, i); /* if the entry was an external directory, create it in the backup */ if (file->external_dir_num && S_ISDIR(file->mode)) @@ -290,28 +588,33 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) char dirpath[MAXPGPATH]; char new_container[MAXPGPATH]; - makeExternalDirPathByNum(new_container, to_external_prefix, + makeExternalDirPathByNum(new_container, full_external_prefix, file->external_dir_num); - join_path_components(dirpath, new_container, file->path); + join_path_components(dirpath, new_container, file->rel_path); dir_create_dir(dirpath, DIR_PERMISSION); } + pg_atomic_init_flag(&file->lock); } + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); + thread_interrupted = false; + merge_time = time(NULL); + elog(INFO, "Start merging backup files"); for (i = 0; i < num_threads; i++) { merge_files_arg *arg = &(threads_args[i]); - - arg->to_files = to_files; - arg->files = files; - arg->to_backup = to_backup; - arg->from_backup = from_backup; - arg->to_root = to_database_path; - arg->from_root = from_database_path; - arg->from_external = from_external; - arg->to_external_prefix = to_external_prefix; - arg->from_external_prefix = from_external_prefix; + arg->merge_filelist = parray_new(); + arg->parent_chain = parent_chain; + arg->dest_backup = dest_backup; + arg->full_backup = full_backup; + arg->full_database_dir = full_database_dir; + arg->full_external_prefix = full_external_prefix; + + arg->compression_match = compression_match; + arg->program_version_match = program_version_match; /* By default there are some error */ arg->ret = 1; @@ -321,374 +624,374 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup) } /* Wait threads */ + result_filelist = parray_new(); for (i = 0; i < num_threads; i++) { pthread_join(threads[i], NULL); if (threads_args[i].ret == 1) merge_isok = false; + + /* Compile final filelist */ + parray_concat(result_filelist, threads_args[i].merge_filelist); + + /* cleanup */ + parray_free(threads_args[i].merge_filelist); + //total_in_place_merge_bytes += threads_args[i].in_place_merge_bytes; } - if (!merge_isok) - elog(ERROR, "Data files merging failed"); + + time(&end_time); + pretty_time_interval(difftime(end_time, merge_time), + pretty_time, lengthof(pretty_time)); + + if (merge_isok) + elog(INFO, "Backup files are successfully merged, time elapsed: %s", + pretty_time); + else + elog(ERROR, "Backup files merging failed, time elapsed: %s", + pretty_time); /* - * Update to_backup metadata. + * Update FULL backup metadata. * We cannot set backup status to OK just yet, * because it still has old start_time. */ - StrNCpy(to_backup->program_version, PROGRAM_VERSION, - sizeof(to_backup->program_version)); - to_backup->parent_backup = INVALID_BACKUP_ID; - to_backup->start_lsn = from_backup->start_lsn; - to_backup->stop_lsn = from_backup->stop_lsn; - to_backup->recovery_time = from_backup->recovery_time; - to_backup->recovery_xid = from_backup->recovery_xid; - pfree(to_backup->external_dir_str); - to_backup->external_dir_str = from_backup->external_dir_str; - from_backup->external_dir_str = NULL; /* For safe pgBackupFree() */ - to_backup->merge_time = merge_time; - to_backup->end_time = time(NULL); - - /* Target backup must inherit wal mode too. */ - to_backup->stream = from_backup->stream; - - /* ARCHIVE backup must inherit wal_bytes. */ - if (!to_backup->stream) - to_backup->wal_bytes = from_backup->wal_bytes; - - write_backup_filelist(to_backup, files, from_database_path, NULL); - write_backup(to_backup); - -delete_source_backup: - /* - * Files were copied into to_backup. It is time to remove source backup - * entirely. + StrNCpy(full_backup->program_version, PROGRAM_VERSION, + sizeof(full_backup->program_version)); + full_backup->parent_backup = INVALID_BACKUP_ID; + full_backup->start_lsn = dest_backup->start_lsn; + full_backup->stop_lsn = dest_backup->stop_lsn; + full_backup->recovery_time = dest_backup->recovery_time; + full_backup->recovery_xid = dest_backup->recovery_xid; + + pfree(full_backup->external_dir_str); + full_backup->external_dir_str = pgut_strdup(dest_backup->external_dir_str); + pfree(full_backup->primary_conninfo); + full_backup->primary_conninfo = pgut_strdup(dest_backup->primary_conninfo); + + full_backup->merge_time = merge_time; + full_backup->end_time = time(NULL); + + full_backup->compress_alg = dest_backup->compress_alg; + full_backup->compress_level = dest_backup->compress_level; + + /* FULL backup must inherit wal mode. */ + full_backup->stream = dest_backup->stream; + + /* ARCHIVE backup must inherit wal_bytes too. + * STREAM backup will have its wal_bytes calculated by + * write_backup_filelist(). */ - delete_backup_files(from_backup); + if (!dest_backup->stream) + full_backup->wal_bytes = dest_backup->wal_bytes; - /* - * Delete files which are not in from_backup file list. + parray_qsort(result_filelist, pgFileCompareRelPathWithExternal); + + write_backup_filelist(full_backup, result_filelist, full_database_dir, NULL); + write_backup(full_backup); + + /* Delete FULL backup files, that do not exists in destination backup + * Both arrays must be sorted in in reversed order to delete from leaf */ - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - for (i = 0; i < parray_num(to_files); i++) + parray_qsort(dest_backup->files, pgFileCompareRelPathWithExternalDesc); + parray_qsort(full_backup->files, pgFileCompareRelPathWithExternalDesc); + for (i = 0; i < parray_num(full_backup->files); i++) { - pgFile *file = (pgFile *) parray_get(to_files, i); + pgFile *full_file = (pgFile *) parray_get(full_backup->files, i); - if (file->external_dir_num && to_external) + if (full_file->external_dir_num && full_externals) { - char *dir_name = parray_get(to_external, file->external_dir_num - 1); - if (backup_contains_external(dir_name, from_external)) + char *dir_name = parray_get(full_externals, full_file->external_dir_num - 1); + if (backup_contains_external(dir_name, full_externals)) /* Dir already removed*/ continue; } - if (parray_bsearch(files, file, pgFileCompareRelPathWithExternalDesc) == NULL) + if (parray_bsearch(dest_backup->files, full_file, pgFileCompareRelPathWithExternalDesc) == NULL) { - char to_file_path[MAXPGPATH]; - char *prev_path; + char full_file_path[MAXPGPATH]; /* We need full path, file object has relative path */ - join_path_components(to_file_path, to_database_path, file->path); - prev_path = file->path; - file->path = to_file_path; + join_path_components(full_file_path, full_database_dir, full_file->rel_path); - pgFileDelete(file); - elog(VERBOSE, "Deleted \"%s\"", file->path); - - file->path = prev_path; + pgFileDelete(full_file, full_file_path); + elog(VERBOSE, "Deleted \"%s\"", full_file_path); } } + /* Critical section starts. + * Change status of FULL backup. + * Files are merged into FULL backup. It is time to remove incremental chain. + */ + full_backup->status = BACKUP_STATUS_MERGED; + write_backup(full_backup); + +merge_delete: + for (i = parray_num(parent_chain) - 2; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + delete_backup_files(backup); + } + + /* + * If we crash now, automatic rerun of failed merge will be impossible. + * The user must have to manually change start_time of FULL backup + * to start_time of destination backup: + * PAGE2 DELETED + * PAGE1 DELETED + * FULL MERGED + */ + +merge_rename: /* - * Rename FULL backup directory. + * Rename FULL backup directory to destination backup directory. */ - elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id); - if (rename(to_backup_path, from_backup_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - to_backup_path, from_backup_path, strerror(errno)); + if (dest_backup) + { + elog(LOG, "Rename %s to %s", full_backup->root_dir, dest_backup->root_dir); + if (rename(full_backup->root_dir, dest_backup->root_dir) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + full_backup->root_dir, dest_backup->root_dir, strerror(errno)); + } + else + { + /* Ugly */ + char backups_dir[MAXPGPATH]; + char instance_dir[MAXPGPATH]; + char destination_path[MAXPGPATH]; + + join_path_components(backups_dir, backup_path, BACKUPS_DIR); + join_path_components(instance_dir, backups_dir, instance_name); + join_path_components(destination_path, instance_dir, + base36enc(full_backup->merge_dest_backup)); + + elog(LOG, "Rename %s to %s", full_backup->root_dir, destination_path); + if (rename(full_backup->root_dir, destination_path) == -1) + elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", + full_backup->root_dir, destination_path, strerror(errno)); + } /* - * Merging finished, now we can safely update ID of the destination backup. - * TODO: for this critical section we must save incremental backup start_tome - * to FULL backup meta, so even if crash happens after incremental backup removal - * but before full backup obtaining new start_time we could safely continue - * this failed backup. + * Merging finished, now we can safely update ID of the FULL backup */ - to_backup->status = BACKUP_STATUS_OK; - to_backup->start_time = from_backup->start_time; - write_backup(to_backup); + dest_backup_id = base36enc_dup(full_backup->merge_dest_backup); + elog(INFO, "Rename merged full backup %s to %s", + base36enc(full_backup->start_time), dest_backup_id); + + full_backup->status = BACKUP_STATUS_OK; + full_backup->start_time = full_backup->merge_dest_backup; + full_backup->merge_dest_backup = INVALID_BACKUP_ID; + write_backup(full_backup); + /* Critical section end */ /* Cleanup */ + pg_free(dest_backup_id); if (threads) { pfree(threads_args); pfree(threads); } - parray_walk(to_files, pgFileFree); - parray_free(to_files); + if (result_filelist && parray_num(result_filelist) > 0) + { + parray_walk(result_filelist, pgFileFree); + parray_free(result_filelist); + } - parray_walk(files, pgFileFree); - parray_free(files); + if (dest_externals != NULL) + free_dir_list(dest_externals); - pfree(to_backup_id); - pfree(from_backup_id); + if (full_externals != NULL) + free_dir_list(full_externals); + + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + + if (backup->files) + { + parray_walk(backup->files, pgFileFree); + parray_free(backup->files); + } + } } /* - * Thread worker of merge_backups(). + * Thread worker of merge_chain(). */ static void * merge_files(void *arg) { - merge_files_arg *argument = (merge_files_arg *) arg; - pgBackup *to_backup = argument->to_backup; - pgBackup *from_backup = argument->from_backup; - int i, - num_files = parray_num(argument->files); + int i; + merge_files_arg *arguments = (merge_files_arg *) arg; - for (i = 0; i < num_files; i++) + for (i = 0; i < parray_num(arguments->dest_backup->files); i++) { - pgFile *file = (pgFile *) parray_get(argument->files, i); - pgFile *to_file; - pgFile **res_file; - char to_file_path[MAXPGPATH]; /* Path of target file */ - char from_file_path[MAXPGPATH]; - char *prev_file_path; + pgFile *dest_file = (pgFile *) parray_get(arguments->dest_backup->files, i); + pgFile *tmp_file; + bool in_place = false; /* keep file as it is */ /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during merging backups"); + elog(ERROR, "Interrupted during merge"); - /* Directories were created before */ - if (S_ISDIR(file->mode)) + if (!pg_atomic_test_set_flag(&dest_file->lock)) continue; - if (!pg_atomic_test_set_flag(&file->lock)) - continue; + tmp_file = pgFileInit(dest_file->rel_path, dest_file->rel_path); + tmp_file->mode = dest_file->mode; + tmp_file->is_datafile = dest_file->is_datafile; + tmp_file->is_cfs = dest_file->is_cfs; + tmp_file->external_dir_num = dest_file->external_dir_num; + tmp_file->dbOid = dest_file->dbOid; - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, num_files, file->path); + /* Directories were created before */ + if (S_ISDIR(dest_file->mode)) + goto done; - res_file = parray_bsearch(argument->to_files, file, - pgFileCompareRelPathWithExternalDesc); - to_file = (res_file) ? *res_file : NULL; + if (progress) + elog(INFO, "Progress: (%d/%lu). Process file \"%s\"", + i + 1, (unsigned long) parray_num(arguments->dest_backup->files), dest_file->rel_path); - join_path_components(to_file_path, argument->to_root, file->path); + if (dest_file->is_datafile && !dest_file->is_cfs) + tmp_file->segno = dest_file->segno; - /* - * Skip files which haven't changed since previous backup. But in case - * of DELTA backup we must truncate the target file to n_blocks. - * Unless it is a non data file, in this case truncation is not needed. - */ - if (file->write_size == BYTES_INVALID) + // If destination file is 0 sized, then go for the next + if (dest_file->write_size == 0) { - /* sanity */ - if (!to_file) - elog(ERROR, "The file \"%s\" is missing in FULL backup %s", - file->rel_path, base36enc(to_backup->start_time)); - - /* for not changed files of all types in PAGE and PTRACK */ - if (from_backup->backup_mode != BACKUP_MODE_DIFF_DELTA || - /* and not changed non-data files in DELTA */ - (!file->is_datafile || file->is_cfs)) - { - elog(VERBOSE, "Skip merging file \"%s\", the file didn't change", - file->rel_path); - - /* - * If the file wasn't changed, retreive its - * write_size and compression algorihtm from previous FULL backup. - */ - file->compress_alg = to_file->compress_alg; - file->write_size = to_file->write_size; - - /* - * Recalculate crc for backup prior to 2.0.25. - */ - if (parse_program_version(from_backup->program_version) < 20025) - file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST); - /* Otherwise just get it from the previous file */ - else - file->crc = to_file->crc; + if (!dest_file->is_datafile || dest_file->is_cfs) + tmp_file->crc = dest_file->crc; - continue; - } - } - - /* TODO optimization: file from incremental backup has size 0, then - * just truncate the file from FULL backup - */ - - /* We need to make full path, file object has relative path */ - if (file->external_dir_num) - { - char temp[MAXPGPATH]; - makeExternalDirPathByNum(temp, argument->from_external_prefix, - file->external_dir_num); + tmp_file->write_size = 0; - join_path_components(from_file_path, temp, file->path); + goto done; } - else - join_path_components(from_file_path, argument->from_root, - file->path); - prev_file_path = file->path; - file->path = from_file_path; /* - * Move the file. We need to decompress it and compress again if - * necessary. + * If file didn`t changed over the course of all incremental chain, + * then do in-place merge, unless destination backup has + * different compression algorihtm. + * In-place merge is also impossible, if program version of destination + * backup differs from PROGRAM_VERSION */ - elog(VERBOSE, "Merging file \"%s\", is_datafile %d, is_cfs %d", - file->path, file->is_database, file->is_cfs); - - if (file->is_datafile && !file->is_cfs) + if (arguments->program_version_match && arguments->compression_match) { /* - * We need more complicate algorithm if target file should be - * compressed. + * Case 1: + * in this case in place merge is possible: + * 0 PAGE; file, size BYTES_INVALID + * 1 PAGE; file, size BYTES_INVALID + * 2 FULL; file, size 100500 + * + * Case 2: + * in this case in place merge is possible: + * 0 PAGE; file, size BYTES_INVALID (should not be possible) + * 1 PAGE; file, size 0 + * 2 FULL; file, size 100500 + * + * Case 3: + * in this case in place merge is impossible: + * 0 PAGE; file, size BYTES_INVALID + * 1 PAGE; file, size 100501 + * 2 FULL; file, size 100500 + * + * Case 4: + * in this case in place merge is impossible: + * 0 PAGE; file, size BYTES_INVALID + * 1 PAGE; file, size 100501 + * 2 FULL; file, missing */ - if (to_backup->compress_alg == PGLZ_COMPRESS || - to_backup->compress_alg == ZLIB_COMPRESS) + + in_place = true; + + for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) { - char merge_to_file_path[MAXPGPATH]; - char tmp_file_path[MAXPGPATH]; - char *prev_path; - - snprintf(merge_to_file_path, MAXPGPATH, "%s_merge", to_file_path); - snprintf(tmp_file_path, MAXPGPATH, "%s_tmp", to_file_path); - - /* Start the magic */ - - /* - * Merge files: - * - if to_file in FULL backup exists, restore and decompress it to to_file_merge - * - decompress source file if necessary and merge it with the - * target decompressed file in to_file_merge. - * - compress result file to to_file_tmp - * - rename to_file_tmp to to_file - */ + pgFile **res_file = NULL; + pgFile *file = NULL; + + pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); - /* - * We need to decompress target file in FULL backup if it exists. + /* lookup file in intermediate backup */ + res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); + file = (res_file) ? *res_file : NULL; + + /* Destination file is not exists yet, + * in-place merge is impossible */ - if (to_file) + if (file == NULL) { - elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"", - merge_to_file_path); - - // TODO: truncate merge_to_file_path just in case? - - /* - * file->path is relative, to_file_path - is absolute. - * Substitute them. - */ - prev_path = to_file->path; - to_file->path = to_file_path; - /* Decompress target file into temporary one */ - restore_data_file(merge_to_file_path, to_file, false, false, - parse_program_version(to_backup->program_version)); - to_file->path = prev_path; + in_place = false; + break; + } + + /* Skip file from FULL backup */ + if (backup->backup_mode == BACKUP_MODE_FULL) + continue; + + if (file->write_size != BYTES_INVALID) + { + in_place = false; + break; } - else - elog(VERBOSE, "Restore source file into the temporary path \"%s\"", - merge_to_file_path); - - /* TODO: Optimize merge of new files */ - - /* Merge source file with target file */ - restore_data_file(merge_to_file_path, file, - from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false, - parse_program_version(from_backup->program_version)); - - elog(VERBOSE, "Compress file and save it into the directory \"%s\"", - argument->to_root); - - /* Again we need to change path */ - prev_path = file->path; - file->path = merge_to_file_path; - /* backup_data_file() requires file size to calculate nblocks */ - file->size = pgFileSize(file->path); - /* Now we can compress the file */ - backup_data_file(NULL, /* We shouldn't need 'arguments' here */ - tmp_file_path, file, - to_backup->start_lsn, - to_backup->backup_mode, - to_backup->compress_alg, - to_backup->compress_level, - from_backup->checksum_version, - 0, NULL, false); - - file->path = prev_path; - - /* rename temp file */ - if (rename(tmp_file_path, to_file_path) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - file->path, tmp_file_path, strerror(errno)); - - /* We can remove temporary file */ - if (unlink(merge_to_file_path)) - elog(ERROR, "Could not remove temporary file \"%s\": %s", - merge_to_file_path, strerror(errno)); - } - /* - * Otherwise merging algorithm is simpler. - */ - else - { - /* We can merge in-place here */ - restore_data_file(to_file_path, file, - from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - true, - parse_program_version(from_backup->program_version)); - - /* - * We need to calculate write_size, restore_data_file() doesn't - * do that. - */ - file->write_size = pgFileSize(to_file_path); - file->crc = pgFileGetCRC(to_file_path, true, true, NULL, FIO_LOCAL_HOST); } } - else if (file->external_dir_num) - { - char to_root[MAXPGPATH]; - int new_dir_num; - char *file_external_path = parray_get(argument->from_external, - file->external_dir_num - 1); - - Assert(argument->from_external); - new_dir_num = get_external_index(file_external_path, - argument->from_external); - makeExternalDirPathByNum(to_root, argument->to_external_prefix, - new_dir_num); - copy_file(FIO_LOCAL_HOST, to_root, FIO_LOCAL_HOST, file, false); - } - else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(argument->from_root, FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file); - else - copy_file(FIO_LOCAL_HOST, argument->to_root, FIO_LOCAL_HOST, file, false); /* - * We need to save compression algorithm type of the target backup to be - * able to restore in the future. + * In-place merge means that file in FULL backup stays as it is, + * no additional actions are required. */ - file->compress_alg = to_backup->compress_alg; + if (in_place) + { + pgFile **res_file = NULL; + pgFile *file = NULL; + res_file = parray_bsearch(arguments->full_backup->files, dest_file, + pgFileCompareRelPathWithExternal); + file = (res_file) ? *res_file : NULL; + + /* If file didn`t changed in any way, then in-place merge is possible */ + if (file && + file->n_blocks == dest_file->n_blocks) + { + + elog(VERBOSE, "The file didn`t changed since FULL backup, skip merge: \"%s\"", + file->rel_path); - if (file->write_size < 0) - elog(ERROR, "Merge of file \"%s\" failed. Invalid size: %i", - file->path, BYTES_INVALID); + tmp_file->crc = file->crc; + tmp_file->write_size = file->write_size; + + if (dest_file->is_datafile && !dest_file->is_cfs) + { + tmp_file->n_blocks = file->n_blocks; + tmp_file->compress_alg = file->compress_alg; + tmp_file->uncompressed_size = file->n_blocks * BLCKSZ; + } + else + tmp_file->uncompressed_size = tmp_file->write_size; - elog(VERBOSE, "Merged file \"%s\": " INT64_FORMAT " bytes", - file->path, file->write_size); + //TODO: report in_place merge bytes. + goto done; + } + } - /* Restore relative path */ - file->path = prev_file_path; + if (dest_file->is_datafile && !dest_file->is_cfs) + merge_data_file(arguments->parent_chain, + arguments->full_backup, + arguments->dest_backup, + dest_file, tmp_file, + arguments->full_database_dir); + else + merge_non_data_file(arguments->parent_chain, + arguments->full_backup, + arguments->dest_backup, + dest_file, tmp_file, + arguments->full_database_dir, + arguments->full_external_prefix); + +done: + parray_append(arguments->merge_filelist, tmp_file); } /* Data files merging is successful */ - argument->ret = 0; + arguments->ret = 0; return NULL; } @@ -699,6 +1002,7 @@ remove_dir_with_files(const char *path) { parray *files = parray_new(); int i; + char full_path[MAXPGPATH]; dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST); parray_qsort(files, pgFileComparePathDesc); @@ -706,9 +1010,15 @@ remove_dir_with_files(const char *path) { pgFile *file = (pgFile *) parray_get(files, i); - pgFileDelete(file); - elog(VERBOSE, "Deleted \"%s\"", file->path); + join_path_components(full_path, path, file->rel_path); + + pgFileDelete(file, full_path); + elog(VERBOSE, "Deleted \"%s\"", full_path); } + + /* cleanup */ + parray_walk(files, pgFileFree); + parray_free(files); } /* Get index of external directory */ @@ -735,8 +1045,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, char externaldir_template[MAXPGPATH]; int i; - pgBackupGetPath(to_backup, externaldir_template, - lengthof(externaldir_template), EXTERNAL_DIR); + join_path_components(externaldir_template, to_backup->root_dir, EXTERNAL_DIR); for (i = 0; i < parray_num(to_external); i++) { int from_num = get_external_index(parray_get(to_external, i), @@ -760,3 +1069,166 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, } } } + +/* Merge is usually happens as usual backup/restore via temp files, unless + * file didn`t changed since FULL backup AND full a dest backup have the + * same compression algorihtm. In this case file can be left as it is. + */ +void +merge_data_file(parray *parent_chain, pgBackup *full_backup, + pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, + const char *full_database_dir) +{ + FILE *out = NULL; + char to_fullpath[MAXPGPATH]; + char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ + char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ + char buffer[STDIO_BUFSIZE]; + + /* The next possible optimization is copying "as is" the file + * from intermediate incremental backup, that didn`t changed in + * subsequent incremental backups. TODO. + */ + + /* set fullpath of destination file and temp files */ + join_path_components(to_fullpath, full_database_dir, tmp_file->rel_path); + snprintf(to_fullpath_tmp1, MAXPGPATH, "%s_tmp1", to_fullpath); + snprintf(to_fullpath_tmp2, MAXPGPATH, "%s_tmp2", to_fullpath); + + /* open temp file */ + out = fopen(to_fullpath_tmp1, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open merge target file \"%s\": %s", + to_fullpath_tmp1, strerror(errno)); + setbuf(out, buffer); + + /* restore file into temp file */ + tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); + fclose(out); + + backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + InvalidXLogRecPtr, BACKUP_MODE_FULL, + dest_backup->compress_alg, dest_backup->compress_level, + dest_backup->checksum_version, 0, NULL, false); + + /* + * In old (=<2.2.7) versions of pg_probackup n_blocks attribute of files + * in PAGE and PTRACK wasn`t filled. + */ +// Assert(tmp_file->n_blocks == dest_file->n_blocks); + + if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot fsync merge temp file \"%s\": %s", + to_fullpath_tmp2, strerror(errno)); + + /* Do atomic rename from second temp file to destination file */ + if (rename(to_fullpath_tmp2, to_fullpath) == -1) + elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", + to_fullpath_tmp2, to_fullpath, strerror(errno)); + + /* drop temp file */ + unlink(to_fullpath_tmp1); +} + +/* + * For every destionation file lookup the newest file in chain and + * copy it. + * Additional pain is external directories. + */ +void +merge_non_data_file(parray *parent_chain, pgBackup *full_backup, + pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, + const char *full_database_dir, const char *to_external_prefix) +{ + int i; + char to_fullpath[MAXPGPATH]; + char to_fullpath_tmp[MAXPGPATH]; /* used for backup */ + char from_fullpath[MAXPGPATH]; + pgBackup *from_backup = NULL; + pgFile *from_file = NULL; + + /* We need to make full path to destination file */ + if (dest_file->external_dir_num) + { + char temp[MAXPGPATH]; + makeExternalDirPathByNum(temp, to_external_prefix, + dest_file->external_dir_num); + join_path_components(to_fullpath, temp, dest_file->rel_path); + } + else + join_path_components(to_fullpath, full_database_dir, dest_file->rel_path); + + snprintf(to_fullpath_tmp, MAXPGPATH, "%s_tmp", to_fullpath); + + /* + * Iterate over parent chain starting from direct parent of destination + * backup to oldest backup in chain, and look for the first + * full copy of destination file. + * Full copy is latest possible destination file with size equal(!) + * or greater than zero. + */ + for (i = 0; i < parray_num(parent_chain); i++) + { + pgFile **res_file = NULL; + from_backup = (pgBackup *) parray_get(parent_chain, i); + + /* lookup file in intermediate backup */ + res_file = parray_bsearch(from_backup->files, dest_file, pgFileCompareRelPathWithExternal); + from_file = (res_file) ? *res_file : NULL; + + /* + * It should not be possible not to find source file in intermediate + * backup, without encountering full copy first. + */ + if (!from_file) + { + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + dest_file->rel_path, base36enc(from_backup->start_time)); + continue; + } + + if (from_file->write_size > 0) + break; + } + + /* sanity */ + if (!from_backup) + elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", + dest_file->rel_path); + + if (!from_file) + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", dest_file->rel_path); + + /* set path to source file */ + if (from_file->external_dir_num) + { + char temp[MAXPGPATH]; + char external_prefix[MAXPGPATH]; + + join_path_components(external_prefix, from_backup->root_dir, EXTERNAL_DIR); + makeExternalDirPathByNum(temp, external_prefix, dest_file->external_dir_num); + + join_path_components(from_fullpath, temp, from_file->rel_path); + } + else + { + char backup_database_dir[MAXPGPATH]; + join_path_components(backup_database_dir, from_backup->root_dir, DATABASE_DIR); + join_path_components(from_fullpath, backup_database_dir, from_file->rel_path); + } + + /* Copy file to FULL backup directory into temp file */ + backup_non_data_file(tmp_file, NULL, from_fullpath, + to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); + + /* TODO: --no-sync support */ + if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot fsync merge temp file \"%s\": %s", + to_fullpath_tmp, strerror(errno)); + + /* Do atomic rename from second temp file to destination file */ + if (rename(to_fullpath_tmp, to_fullpath) == -1) + elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", + to_fullpath_tmp, to_fullpath, strerror(errno)); + +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 811758736..a39a4bbf0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -754,7 +754,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, no_validate, set_backup_params); + return do_backup(start_time, no_validate, set_backup_params, no_sync); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2f3a73dde..6fbd77736 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -85,6 +85,9 @@ extern const char *PROGRAM_EMAIL; #define STDOUT_FILENO 1 #endif +/* stdio buffer size */ +#define STDIO_BUFSIZE 65536 + /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) @@ -139,6 +142,8 @@ typedef struct pgFile char *name; /* file or directory name */ mode_t mode; /* protection (file type and permission) */ size_t size; /* size of the file */ + time_t mtime; /* file st_mtime attribute, can be used only + during backup */ size_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ int64 write_size; /* size of the backed-up file. BYTES_INVALID means @@ -188,6 +193,8 @@ typedef enum BackupStatus BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ BACKUP_STATUS_RUNNING, /* running backup */ BACKUP_STATUS_MERGING, /* merging backups */ + BACKUP_STATUS_MERGED, /* backup has been successfully merged and now awaits + * the assignment of new start_time */ BACKUP_STATUS_DELETING, /* data files are being deleted */ BACKUP_STATUS_DELETED, /* data files have been deleted */ BACKUP_STATUS_DONE, /* completed but not validated yet */ @@ -322,6 +329,10 @@ struct pgBackup XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ time_t start_time; /* since this moment backup has status * BACKUP_STATUS_RUNNING */ + time_t merge_dest_backup; /* start_time of incremental backup, + * this backup is merging with. + * Only available for FULL backups + * with MERGING or MERGED statuses */ time_t merge_time; /* the moment when merge was started or 0 */ time_t end_time; /* the moment when backup was finished, or the moment * when we realized that backup is broken */ @@ -372,10 +383,10 @@ struct pgBackup * in the format suitable for recovery.conf */ char *external_dir_str; /* List of external directories, * separated by ':' */ - parray *files; /* list of files belonging to this backup - * must be populated by calling backup_populate() */ char *root_dir; /* Full path for root backup directory: backup_path/instance_name/backup_id */ + parray *files; /* list of files belonging to this backup + * must be populated explicitly */ }; /* Recovery target for restore and validate subcommands */ @@ -513,6 +524,7 @@ typedef struct BackupPageHeader } BackupPageHeader; /* Special value for compressed_size field */ +#define PageIsOk 0 #define PageIsTruncated -2 #define SkipCurrentPage -3 #define PageIsCorrupted -4 /* used by checkdb */ @@ -629,7 +641,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params); + pgSetBackupParams *set_backup_params, bool no_sync); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -664,6 +676,8 @@ extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetT /* in merge.c */ extern void do_merge(time_t backup_id); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); +extern void merge_chain(parray *parent_chain, + pgBackup *full_backup, pgBackup *dest_backup); extern parray *read_database_map(pgBackup *backup); @@ -698,6 +712,9 @@ extern char *slurpFile(const char *datadir, size_t *filesize, bool safe, fio_location location); +extern char *slurpFileFullPath(const char *from_fullpath, + size_t *filesize, bool safe, + fio_location location); extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ @@ -809,10 +826,13 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location); extern pgFile *pgFileInit(const char *path, const char *rel_path); -extern void pgFileDelete(pgFile *file); +extern void pgFileDelete(pgFile *file, const char *full_path); + extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, size_t *bytes_read, fio_location location); +extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c); +//extern pg_crc32 pgFileGetCRC_compressed(const char *file_path, bool use_crc32c, bool missing_ok); extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileMapComparePath(const void *f1, const void *f2); @@ -826,21 +846,24 @@ extern int pgFileCompareSize(const void *f1, const void *f2); extern int pgCompareOid(const void *f1, const void *f2); /* in data.c */ -extern bool check_data_file(ConnectionArgs* arguments, pgFile* file, uint32 checksum_version); -extern bool backup_data_file(backup_files_arg* arguments, - const char *to_path, pgFile *file, - XLogRecPtr prev_backup_start_lsn, - BackupMode backup_mode, - CompressAlg calg, int clevel, - uint32 checksum_version, - int ptrack_version_num, - const char *ptrack_schema, - bool missing_ok); -extern void restore_data_file(const char *to_path, - pgFile *file, bool allow_truncate, - bool write_header, - uint32 backup_version); -extern size_t restore_data_file_new(parray *parent_chain, pgFile *dest_file, +extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, + const char *from_fullpath, uint32 checksum_version); + +extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, + const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, bool missing_ok); +extern void backup_non_data_file(pgFile *file, pgFile *prev_file, + const char *from_fullpath, const char *to_fullpath, + BackupMode backup_mode, time_t parent_backup_time, + bool missing_ok); +extern void backup_non_data_file_internal(const char *from_fullpath, + fio_location from_location, + const char *to_fullpath, pgFile *file, + bool missing_ok); + +extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks); @@ -848,8 +871,6 @@ extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath); extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath); -extern bool copy_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file, bool missing_ok); extern bool create_empty_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); @@ -887,8 +908,8 @@ extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); extern uint32 get_xlog_seg_size(char *pgdata_path); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); -extern void copy_pgcontrol_file(const char *from_root, fio_location location, const char *to_root, fio_location to_location, - pgFile *file); +extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, + const char *to_fullpath, fio_location to_location, pgFile *file); extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); diff --git a/src/restore.c b/src/restore.c index fae458125..c25a4711a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -17,24 +17,6 @@ #include "utils/thread.h" -typedef struct -{ - parray *files; - pgBackup *backup; - parray *external_dirs; - char *external_prefix; - parray *dest_external_dirs; - parray *dest_files; - parray *dbOid_exclude_list; - bool skip_external_dirs; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} restore_files_arg; - typedef struct { parray *dest_files; @@ -51,11 +33,8 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; -} restore_files_arg_new; +} restore_files_arg; -static void restore_backup(pgBackup *backup, parray *dest_external_dirs, - parray *dest_files, parray *dbOid_exclude_list, - pgRestoreParams *params); static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, @@ -68,9 +47,6 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync); -static void *restore_files_new(void *arg); - - /* * Iterate over backup list to find all ancestors of the broken parent_backup * and update their status to BACKUP_STATUS_ORPHAN @@ -522,7 +498,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *external_dirs = NULL; /* arrays with meta info for multi threaded backup */ pthread_t *threads; - restore_files_arg_new *threads_args; + restore_files_arg *threads_args; bool restore_isok = true; /* fancy reporting */ @@ -644,7 +620,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, fio_disconnect(); threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) * + threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * num_threads); /* Restore files into target directory */ @@ -659,7 +635,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, thread_interrupted = false; for (i = 0; i < num_threads; i++) { - restore_files_arg_new *arg = &(threads_args[i]); + restore_files_arg *arg = &(threads_args[i]); arg->dest_files = dest_files; arg->dest_backup = dest_backup; @@ -675,7 +651,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, /* Useless message TODO: rewrite */ elog(LOG, "Start thread %i", i + 1); - pthread_create(&threads[i], NULL, restore_files_new, arg); + pthread_create(&threads[i], NULL, restore_files, arg); } /* Wait theads */ @@ -767,14 +743,14 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * Restore files into $PGDATA. */ static void * -restore_files_new(void *arg) +restore_files(void *arg) { int i; char to_fullpath[MAXPGPATH]; FILE *out = NULL; - char buffer[65536]; + char buffer[STDIO_BUFSIZE]; - restore_files_arg_new *arguments = (restore_files_arg_new *) arg; + restore_files_arg *arguments = (restore_files_arg *) arg; for (i = 0; i < parray_num(arguments->dest_files); i++) { @@ -870,8 +846,8 @@ restore_files_new(void *arg) /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) /* Destination file is data file */ - arguments->restored_bytes += restore_data_file_new(arguments->parent_chain, - dest_file, out, to_fullpath); + arguments->restored_bytes += restore_data_file(arguments->parent_chain, + dest_file, out, to_fullpath); else /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, @@ -885,7 +861,7 @@ restore_files_new(void *arg) done: - /* truncate file up to n_blocks. NOTE: no need, we just should not write + /* Truncate file up to n_blocks. NOTE: no need, we just should not write * blocks that are exceeding n_blocks. * But for this to work, n_blocks should be trusted. */ @@ -916,306 +892,6 @@ restore_files_new(void *arg) return NULL; } -/* - * Restore one backup. - */ -void -restore_backup(pgBackup *backup, parray *dest_external_dirs, - parray *dest_files, parray *dbOid_exclude_list, - pgRestoreParams *params) -{ - char timestamp[100]; - char database_path[MAXPGPATH]; - char external_prefix[MAXPGPATH]; - char list_path[MAXPGPATH]; - parray *files; - parray *external_dirs = NULL; - int i; - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - restore_files_arg *threads_args; - bool restore_isok = true; - - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - { - if (params->force) - elog(WARNING, "Backup %s is not valid, restore is forced", - base36enc(backup->start_time)); - else - elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); - } - - /* confirm block size compatibility */ - if (backup->block_size != BLCKSZ) - elog(ERROR, - "BLCKSZ(%d) is not compatible(%d expected)", - backup->block_size, BLCKSZ); - if (backup->wal_block_size != XLOG_BLCKSZ) - elog(ERROR, - "XLOG_BLCKSZ(%d) is not compatible(%d expected)", - backup->wal_block_size, XLOG_BLCKSZ); - - time2iso(timestamp, lengthof(timestamp), backup->start_time); - elog(LOG, "Restoring database from backup %s", timestamp); - - if (backup->external_dir_str) - external_dirs = make_external_directory_list(backup->external_dir_str, - true); - - /* - * Get list of files which need to be restored. - */ - pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR); - pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), - EXTERNAL_DIR); - pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST); - files = dir_read_file_list(database_path, external_prefix, list_path, - FIO_BACKUP_HOST); - - /* Restore directories in do_backup_instance way */ - parray_qsort(files, pgFileComparePath); - - /* - * Make external directories before restore - * and setup threads at the same time - */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - /* - * If the entry was an external directory, create it in the backup. - */ - if (!params->skip_external_dirs && - file->external_dir_num && S_ISDIR(file->mode) && - /* Do not create unnecessary external directories */ - parray_bsearch(dest_files, file, pgFileCompareRelPathWithExternal)) - { - char *external_path; - - if (!external_dirs || - parray_num(external_dirs) < file->external_dir_num - 1) - elog(ERROR, "Inconsistent external directory backup metadata"); - - external_path = parray_get(external_dirs, - file->external_dir_num - 1); - if (backup_contains_external(external_path, dest_external_dirs)) - { - char container_dir[MAXPGPATH]; - char dirpath[MAXPGPATH]; - char *dir_name; - - makeExternalDirPathByNum(container_dir, external_prefix, - file->external_dir_num); - dir_name = GetRelativePath(file->path, container_dir); - elog(VERBOSE, "Create directory \"%s\"", dir_name); - join_path_components(dirpath, external_path, dir_name); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_DB_HOST); - } - } - - /* setup threads */ - pg_atomic_clear_flag(&file->lock); - } - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * - num_threads); - - /* Restore files into target directory */ - thread_interrupted = false; - for (i = 0; i < num_threads; i++) - { - restore_files_arg *arg = &(threads_args[i]); - - arg->files = files; - arg->backup = backup; - arg->external_dirs = external_dirs; - arg->external_prefix = external_prefix; - arg->dest_external_dirs = dest_external_dirs; - arg->dest_files = dest_files; - arg->dbOid_exclude_list = dbOid_exclude_list; - arg->skip_external_dirs = params->skip_external_dirs; - /* By default there are some error */ - threads_args[i].ret = 1; - - /* Useless message TODO: rewrite */ - elog(LOG, "Start thread for num:%zu", parray_num(files)); - - pthread_create(&threads[i], NULL, restore_files, arg); - } - - /* Wait theads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - restore_isok = false; - } - if (!restore_isok) - elog(ERROR, "Data files restoring failed"); - - pfree(threads); - pfree(threads_args); - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); - - if (external_dirs != NULL) - free_dir_list(external_dirs); - - elog(LOG, "Restore %s backup completed", base36enc(backup->start_time)); -} - -/* - * Restore files into $PGDATA. - */ -static void * -restore_files(void *arg) -{ - int i; - restore_files_arg *arguments = (restore_files_arg *)arg; - - for (i = 0; i < parray_num(arguments->files); i++) - { - char from_root[MAXPGPATH]; - pgFile *file = (pgFile *) parray_get(arguments->files, i); - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - pgBackupGetPath(arguments->backup, from_root, - lengthof(from_root), DATABASE_DIR); - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during restore database"); - - /* Directories were created before */ - if (S_ISDIR(file->mode)) - continue; - - if (progress) - elog(INFO, "Progress: (%d/%lu). Process file %s ", - i + 1, (unsigned long) parray_num(arguments->files), - file->rel_path); - - /* Only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && file->external_dir_num == 0) - { - /* Check if the file belongs to the database we exclude */ - if (parray_bsearch(arguments->dbOid_exclude_list, - &file->dbOid, pgCompareOid)) - { - /* - * We cannot simply skip the file, because it may lead to - * failure during WAL redo; hence, create empty file. - */ - create_empty_file(FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, file); - - elog(VERBOSE, "Exclude file due to partial restore: \"%s\"", - file->rel_path); - continue; - } - } - - /* - * For PAGE and PTRACK backups skip datafiles which haven't changed - * since previous backup and thus were not backed up. - * We cannot do the same when restoring DELTA backup because we need information - * about every datafile to correctly truncate them. - */ - if (file->write_size == BYTES_INVALID) - { - /* data file, only PAGE and PTRACK can skip */ - if (((file->is_datafile && !file->is_cfs) && - (arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE || - arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK)) || - /* non-data file can be skipped regardless of backup type */ - !(file->is_datafile && !file->is_cfs)) - { - elog(VERBOSE, "The file didn`t change. Skip restore: \"%s\"", file->path); - continue; - } - } - - /* Do not restore tablespace_map file */ - if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, file->rel_path)) - { - elog(VERBOSE, "Skip tablespace_map"); - continue; - } - - /* Do not restore database_map file */ - if ((file->external_dir_num == 0) && - strcmp(DATABASE_MAP, file->rel_path) == 0) - { - elog(VERBOSE, "Skip database_map"); - continue; - } - - /* Do no restore external directory file if a user doesn't want */ - if (arguments->skip_external_dirs && file->external_dir_num > 0) - continue; - - /* Skip unnecessary file */ - if (parray_bsearch(arguments->dest_files, file, - pgFileCompareRelPathWithExternal) == NULL) - continue; - - /* - * restore the file. - * We treat datafiles separately, cause they were backed up block by - * block and have BackupPageHeader meta information, so we cannot just - * copy the file from backup. - */ - elog(VERBOSE, "Restoring file \"%s\", is_datafile %i, is_cfs %i", - file->path, file->is_datafile?1:0, file->is_cfs?1:0); - - if (file->is_datafile && !file->is_cfs) - { - char to_path[MAXPGPATH]; - - join_path_components(to_path, instance_config.pgdata, - file->rel_path); - restore_data_file(to_path, file, - arguments->backup->backup_mode == BACKUP_MODE_DIFF_DELTA, - false, - parse_program_version(arguments->backup->program_version)); - } - else if (file->external_dir_num) - { - char *external_path = parray_get(arguments->external_dirs, - file->external_dir_num - 1); - if (backup_contains_external(external_path, - arguments->dest_external_dirs)) - copy_file(FIO_BACKUP_HOST, - external_path, FIO_DB_HOST, file, false); - } - else if (strcmp(file->name, "pg_control") == 0) - copy_pgcontrol_file(from_root, FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, - file); - else - copy_file(FIO_BACKUP_HOST, - instance_config.pgdata, FIO_DB_HOST, - file, false); - - /* print size of restored file */ - if (file->write_size != BYTES_INVALID) - elog(VERBOSE, "Restored file %s : " INT64_FORMAT " bytes", - file->path, file->write_size); - } - - /* Data files restoring is successful */ - arguments->ret = 0; - - return NULL; -} - /* * Create recovery.conf (probackup_recovery.conf in case of PG12) * with given recovery target parameters diff --git a/src/util.c b/src/util.c index 46240d295..871ad1051 100644 --- a/src/util.c +++ b/src/util.c @@ -109,7 +109,7 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) * Write ControlFile to pg_control */ static void -writeControlFile(ControlFileData *ControlFile, char *path, fio_location location) +writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location) { int fd; char *buffer = NULL; @@ -135,7 +135,7 @@ writeControlFile(ControlFileData *ControlFile, char *path, fio_location location elog(ERROR, "Failed to overwrite file: %s", path); if (fio_flush(fd) != 0) - elog(ERROR, "Failed to fsync file: %s", path); + elog(ERROR, "Failed to sync file: %s", path); fio_close(fd); pg_free(buffer); @@ -383,15 +383,14 @@ set_min_recovery_point(pgFile *file, const char *backup_path, * Copy pg_control file to backup. We do not apply compression to this file. */ void -copy_pgcontrol_file(const char *from_root, fio_location from_location, - const char *to_root, fio_location to_location, pgFile *file) +copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, + const char *to_fullpath, fio_location to_location, pgFile *file) { ControlFileData ControlFile; char *buffer; size_t size; - char to_path[MAXPGPATH]; - buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false, from_location); + buffer = slurpFileFullPath(from_fullpath, &size, false, from_location); digestControlFile(&ControlFile, buffer, size); @@ -400,8 +399,7 @@ copy_pgcontrol_file(const char *from_root, fio_location from_location, file->write_size = size; file->uncompressed_size = size; - join_path_components(to_path, to_root, file->rel_path); - writeControlFile(&ControlFile, to_path, to_location); + writeControlFile(&ControlFile, to_fullpath, to_location); pg_free(buffer); } @@ -471,6 +469,7 @@ status2str(BackupStatus status) "ERROR", "RUNNING", "MERGING", + "MERGED", "DELETING", "DELETED", "DONE", From 2d3f19e622464201c94f26f81a344744df6b49d8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 19 Feb 2020 20:22:12 +0300 Subject: [PATCH 1145/2107] [Issue #169] minor improvements --- src/data.c | 29 +++++++++++++++-------------- src/restore.c | 41 ++++++++++++++--------------------------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/src/data.c b/src/data.c index 780d97449..2bebc8126 100644 --- a/src/data.c +++ b/src/data.c @@ -237,7 +237,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum, { int i; /* Check if the page is zeroed. */ - for(i = 0; i < BLCKSZ && page[i] == 0; i++); + for (i = 0; i < BLCKSZ && page[i] == 0; i++); /* Page is zeroed. No need to check header and checksum. */ if (i == BLCKSZ) @@ -271,9 +271,10 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * should be a pointer to allocated BLCKSZ of bytes. * * Prints appropriate warnings/errors/etc into log. - * Returns 0 if page was successfully retrieved - * SkipCurrentPage(-3) if we need to skip this page + * Returns: + * PageIsOk(0) if page was successfully retrieved * PageIsTruncated(-2) if the page was truncated + * SkipCurrentPage(-3) if we need to skip this page * PageIsCorrupted(-4) if the page check mismatch */ static int32 @@ -312,9 +313,8 @@ prepare_page(ConnectionArgs *conn_arg, switch (result) { case 2: - page_is_valid = true; elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); - break; + return PageIsOk; case 1: page_is_valid = true; @@ -393,6 +393,12 @@ prepare_page(ConnectionArgs *conn_arg, } } + /* Get page via ptrack interface from PostgreSQL shared buffer. + * We do this in following cases: + * 1. PTRACK backup of 1.x versions + * 2. During backup, regardless of backup mode, of PostgreSQL instance + * with ptrack support we encountered invalid page. + */ if ((backup_mode == BACKUP_MODE_DIFF_PTRACK && (ptrack_version_num >= 15 && ptrack_version_num < 20)) || !page_is_valid) @@ -434,7 +440,6 @@ prepare_page(ConnectionArgs *conn_arg, !parse_page(page, &page_lsn)) elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " "Possible risk of a memory corruption"); - } if (page_is_truncated) @@ -1260,22 +1265,18 @@ create_empty_file(fio_location from_location, const char *to_root, /* open file for write */ join_path_components(to_path, to_root, file->rel_path); out = fio_fopen(to_path, PG_BINARY_W, to_location); + if (out == NULL) - { - elog(ERROR, "cannot open destination file \"%s\": %s", + elog(ERROR, "Cannot open destination file \"%s\": %s", to_path, strerror(errno)); - } /* update file permission */ if (fio_chmod(to_path, file->mode, to_location) == -1) - { - fio_fclose(out); - elog(ERROR, "cannot change mode of \"%s\": %s", to_path, + elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, strerror(errno)); - } if (fio_fclose(out)) - elog(ERROR, "cannot close \"%s\": %s", to_path, strerror(errno)); + elog(ERROR, "Cannot close \"%s\": %s", to_path, strerror(errno)); return true; } diff --git a/src/restore.c b/src/restore.c index c25a4711a..7c6dfecec 100644 --- a/src/restore.c +++ b/src/restore.c @@ -697,6 +697,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (S_ISDIR(dest_file->mode)) continue; + /* skip external files if ordered to do so */ + if (dest_file->external_dir_num > 0 && + params->skip_external_dirs) + continue; + /* construct fullpath */ if (dest_file->external_dir_num == 0) { @@ -708,7 +713,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } else { - char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); + char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); join_path_components(to_fullpath, external_path, dest_file->rel_path); } @@ -831,8 +836,10 @@ restore_files(void *arg) to_fullpath, strerror(errno_tmp)); } - if (!fio_is_remote_file(out)) - setbuf(out, buffer); + /* update file permission */ + if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); if (!dest_file->is_datafile || dest_file->is_cfs) elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); @@ -843,6 +850,9 @@ restore_files(void *arg) if (dest_file->write_size == 0) goto done; + if (!fio_is_remote_file(out)) + setbuf(out, buffer); + /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) /* Destination file is data file */ @@ -853,30 +863,7 @@ restore_files(void *arg) arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, arguments->dest_backup, dest_file, out, to_fullpath); - /* - * Destination file is data file. - * Iterate over incremental chain and lookup given destination file. - * Apply changed blocks to destination file from every backup in parent chain. - */ - - done: - - /* Truncate file up to n_blocks. NOTE: no need, we just should not write - * blocks that are exceeding n_blocks. - * But for this to work, n_blocks should be trusted. - */ - - /* update file permission - * TODO: chmod must be done right after fopen() - */ - if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) - { - int errno_tmp = errno; - fio_fclose(out); - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno_tmp)); - } - +done: /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, From d648100d55cbe1e7e426abda59d5c4ed2663f580 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Thu, 20 Feb 2020 15:17:49 +0300 Subject: [PATCH 1146/2107] DOC: fix style and grammar in docs --- doc/pgprobackup.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 07169e559..89d540b99 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1645,10 +1645,8 @@ pg_probackup restore -B backup_dir --instance recovery.conf file. Starting from PostgreSQL 12, pg_probackup writes these settings into - the probackup_recovery.conf file in the data - directory, and then includes them into the - postgresql.auto.conf when the cluster is - is started. + the probackup_recovery.conf file and then includes + it into postgresql.auto.conf. From e9af50464bca8be625bc425bcf48737f0e0b13e1 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 20 Feb 2020 15:23:04 +0300 Subject: [PATCH 1147/2107] Added --primary-conninfo option for restoring as replica --- src/pg_probackup.c | 4 ++++ src/pg_probackup.h | 1 + src/restore.c | 8 ++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a39a4bbf0..4956bac16 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -90,6 +90,8 @@ static bool target_immediate; static char *target_name = NULL; static char *target_action = NULL; +static char *primary_conninfo = NULL; + static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; @@ -191,6 +193,7 @@ static ConfigOption cmd_options[] = { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, + { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -675,6 +678,7 @@ main(int argc, char *argv[]) restore_params->skip_external_dirs = skip_external_dirs; restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; + restore_params->primary_conninfo = primary_conninfo; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6fbd77736..c2cf55df9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -423,6 +423,7 @@ typedef struct pgRestoreParams /* options for partial restore */ PartialRestoreType partial_restore_type; parray *partial_db_list; + const char *primary_conninfo; } pgRestoreParams; /* Options needed for set-backup command */ diff --git a/src/restore.c b/src/restore.c index c25a4711a..edb15bb67 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1069,8 +1069,12 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "standby_mode = 'on'\n"); #endif - if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + if(params->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); + else{ + if (backup->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + } } if (pitr_requested) From 0b16fc0576f00911fa3ffd452693f17ad5d89f4d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Feb 2020 17:50:15 +0300 Subject: [PATCH 1148/2107] [Issue #169] fix merge of 0 sized files --- src/data.c | 7 +++++-- src/merge.c | 22 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/data.c b/src/data.c index 2bebc8126..44f6d3376 100644 --- a/src/data.c +++ b/src/data.c @@ -646,6 +646,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev) { + /* remote FULL and DELTA */ if (fio_is_remote_file(in)) { int rc = fio_send_pages(in, out, file, @@ -669,6 +670,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, } else { + /* local FULL and DELTA */ RetryUsingPtrack: for (blknum = 0; blknum < nblocks; blknum++) { @@ -689,7 +691,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, page_state, curr_page, calg, clevel, from_fullpath, to_fullpath); else - elog(ERROR, "Illegal page state: %i, file: %s, blknum %i", + elog(ERROR, "Invalid page state: %i, file: %s, blknum %i", page_state, file->rel_path, blknum); n_blocks_read++; @@ -718,6 +720,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, if (page_state == PageIsTruncated) break; + /* TODO: PAGE and PTRACK should never get SkipCurrentPage */ else if (page_state == SkipCurrentPage) n_blocks_skipped++; @@ -726,7 +729,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, page_state, curr_page, calg, clevel, from_fullpath, to_fullpath); else - elog(ERROR, "Illegal page state: %i, file: %s, blknum %i", + elog(ERROR, "Invalid page state: %i, file: %s, blknum %i", page_state, file->rel_path, blknum); n_blocks_read++; diff --git a/src/merge.c b/src/merge.c index 2551104ce..0d0719ada 100644 --- a/src/merge.c +++ b/src/merge.c @@ -849,7 +849,7 @@ merge_files(void *arg) goto done; if (progress) - elog(INFO, "Progress: (%d/%lu). Process file \"%s\"", + elog(INFO, "Progress: (%d/%lu). Merging file \"%s\"", i + 1, (unsigned long) parray_num(arguments->dest_backup->files), dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) @@ -1111,14 +1111,28 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, false); + /* drop restored temp file */ + if (unlink(to_fullpath_tmp1) == -1) + elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath_tmp1, + strerror(errno)); + /* * In old (=<2.2.7) versions of pg_probackup n_blocks attribute of files * in PAGE and PTRACK wasn`t filled. */ -// Assert(tmp_file->n_blocks == dest_file->n_blocks); + //Assert(tmp_file->n_blocks == dest_file->n_blocks); + + /* Backward compatibility kludge: + * When merging old backups, it is possible that + * to_fullpath_tmp2 size will be 0, and so it will be + * truncated in backup_data_file(). + * TODO: remove in 3.0.0 + */ + if (tmp_file->write_size == 0) + return; if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot fsync merge temp file \"%s\": %s", + elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); /* Do atomic rename from second temp file to destination file */ @@ -1223,7 +1237,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, /* TODO: --no-sync support */ if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot fsync merge temp file \"%s\": %s", + elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp, strerror(errno)); /* Do atomic rename from second temp file to destination file */ From fa326c96e1b07aa67e02c6e85517090bc888a9f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Feb 2020 18:32:14 +0300 Subject: [PATCH 1149/2107] [Issue #169] minor spelling fixes --- src/merge.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/merge.c b/src/merge.c index 0d0719ada..fcd23e041 100644 --- a/src/merge.c +++ b/src/merge.c @@ -483,15 +483,15 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* TODO: Should we keep relying on caller to provide valid parent_chain? */ - /* If destination backup compression algorihtm differs from - * full backup compression algorihtm, then in-place merge is + /* If destination backup compression algorithm differs from + * full backup compression algorithm, then in-place merge is * not possible. */ if (full_backup->compress_alg == dest_backup->compress_alg) compression_match = true; else elog(WARNING, "In-place merge is disabled because of compression " - "algorihtms mismatch"); + "algorithms mismatch"); /* * If current program version differs from destination backup version, @@ -823,8 +823,9 @@ merge_files(void *arg) { int i; merge_files_arg *arguments = (merge_files_arg *) arg; + size_t n_files = parray_num(arguments->dest_backup->files); - for (i = 0; i < parray_num(arguments->dest_backup->files); i++) + for (i = 0; i < n_files; i++) { pgFile *dest_file = (pgFile *) parray_get(arguments->dest_backup->files, i); pgFile *tmp_file; @@ -850,7 +851,7 @@ merge_files(void *arg) if (progress) elog(INFO, "Progress: (%d/%lu). Merging file \"%s\"", - i + 1, (unsigned long) parray_num(arguments->dest_backup->files), dest_file->rel_path); + i + 1, n_files, dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) tmp_file->segno = dest_file->segno; @@ -862,14 +863,13 @@ merge_files(void *arg) tmp_file->crc = dest_file->crc; tmp_file->write_size = 0; - goto done; } /* * If file didn`t changed over the course of all incremental chain, * then do in-place merge, unless destination backup has - * different compression algorihtm. + * different compression algorithm. * In-place merge is also impossible, if program version of destination * backup differs from PROGRAM_VERSION */ @@ -1072,7 +1072,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, /* Merge is usually happens as usual backup/restore via temp files, unless * file didn`t changed since FULL backup AND full a dest backup have the - * same compression algorihtm. In this case file can be left as it is. + * same compression algorithm. In this case file can be left as it is. */ void merge_data_file(parray *parent_chain, pgBackup *full_backup, From 859b40dfe15b50d5322c4cf8eed322e2a5e900aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 20 Feb 2020 19:34:29 +0300 Subject: [PATCH 1150/2107] minor spelling fix --- src/backup.c | 4 ++-- src/pg_probackup.c | 3 ++- src/validate.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 150503842..f30c55ba4 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1450,8 +1450,8 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, if (!stream_wal && is_start_lsn && try_count == 30) elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " - "If continius archiving is not set up, use '--stream' option to make autonomous backup. " - "Otherwise check that continius archiving works correctly."); + "If continuous archiving is not set up, use '--stream' option to make autonomous backup. " + "Otherwise check that continuous archiving works correctly."); if (timeout > 0 && try_count > timeout) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a39a4bbf0..a9132e53c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -860,7 +860,8 @@ compress_init(void) { if (instance_config.compress_level != COMPRESS_LEVEL_DEFAULT && instance_config.compress_alg == NOT_DEFINED_COMPRESS) - elog(ERROR, "Cannot specify compress-level option without compress-alg option"); + elog(ERROR, "Cannot specify compress-level option alone without " + "compress-algorithm option"); } if (instance_config.compress_level < 0 || instance_config.compress_level > 9) diff --git a/src/validate.c b/src/validate.c index 67dbdf8b8..4f16edc68 100644 --- a/src/validate.c +++ b/src/validate.c @@ -255,7 +255,7 @@ pgBackupValidateFiles(void *arg) continue; if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + elog(INFO, "Progress: (%d/%d). Validate file \"%s\"", i + 1, num_files, file->path); /* From 5a17d07329a49ca15fd176cb5ca1f258c75b4630 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 21 Feb 2020 17:28:27 +0300 Subject: [PATCH 1151/2107] merge: improve wording in elog() messages --- src/merge.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/merge.c b/src/merge.c index fcd23e041..91474fbbc 100644 --- a/src/merge.c +++ b/src/merge.c @@ -403,6 +403,9 @@ do_merge(time_t backup_id) * full backup directory. * Remove unnecessary directories and files from full backup directory. * Update metadata of full backup to represent destination backup. + * + * TODO: stop relying on caller to provide valid parent_chain, make sure + * that chain is ok. */ void merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) @@ -481,8 +484,6 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) } } - /* TODO: Should we keep relying on caller to provide valid parent_chain? */ - /* If destination backup compression algorithm differs from * full backup compression algorithm, then in-place merge is * not possible. @@ -502,13 +503,19 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) program_version_match = true; else elog(WARNING, "In-place merge is disabled because of program " - "versions mismatch"); + "versions mismatch: backup %s was produced by version %s, " + "but current program version is %s", + base36enc(dest_backup->start_time), + dest_backup->program_version, PROGRAM_VERSION); /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); /* Construct path to external dir: /backup_dir/instance_name/FULL/external */ join_path_components(full_external_prefix, full_backup->root_dir, EXTERNAL_DIR); + elog(INFO, "Validate parent chain for backup %s", + base36enc(dest_backup->start_time)); + /* * Validate or revalidate all members of parent chain * with sole exception of FULL backup. If it has MERGING status From ba8c4caae0394239708373288ecb4e3df739a076 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 21 Feb 2020 22:18:49 +0300 Subject: [PATCH 1152/2107] use pgBackup->root_dir when procuring filelist of previous backup --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index f30c55ba4..8677f2f8f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -192,8 +192,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) "Create new FULL backup before an incremental one.", current.tli); - pgBackupGetPath(prev_backup, prev_backup_filelist_path, - lengthof(prev_backup_filelist_path), DATABASE_FILE_LIST); + join_path_components(prev_backup_filelist_path, prev_backup->root_dir, + DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST); From b15c7f1587e0f36c84a99856e32b7708fcb3dbd0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 21 Feb 2020 22:19:52 +0300 Subject: [PATCH 1153/2107] [Issue #169] test coverage --- tests/backup.py | 22 ++- tests/compatibility.py | 170 +++++++++++++++- tests/helpers/ptrack_helpers.py | 6 + tests/locking.py | 10 +- tests/merge.py | 210 +++++++++++++++++--- tests/replica.py | 7 +- tests/retention.py | 341 ++++++++++++++++++++++++++------ 7 files changed, 653 insertions(+), 113 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index bc8f89ba0..e1d451120 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1023,6 +1023,11 @@ def test_drop_rel_during_backup_page(self): # FULL backup self.backup_node(backup_dir, 'node', node, options=['--stream']) + node.safe_psql( + "postgres", + "insert into t_heap select i" + " as id from generate_series(101,102) i") + # PAGE backup gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', @@ -1039,11 +1044,10 @@ def test_drop_rel_during_backup_page(self): pgdata = self.pgdata_content(node.data_dir) - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( - 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + self.assertNotIn(relative_path, filelist) node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) @@ -1347,7 +1351,7 @@ def test_sigint_handling(self): backup_dir, 'node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -1385,7 +1389,7 @@ def test_sigterm_handling(self): backup_dir, 'node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -1422,7 +1426,7 @@ def test_sigquit_handling(self): gdb = self.backup_node( backup_dir, 'node', node, gdb=True, options=['--stream']) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -1513,7 +1517,7 @@ def test_basic_missing_file_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: cannot open file', + 'ERROR: Cannot open file', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/compatibility.py b/tests/compatibility.py index 650321167..c5b5fcc52 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -85,8 +85,8 @@ def test_backward_compatibility_page(self): pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) + options=["-c", "4", "-T", "20"]) + pgbench.wait() pgbench.stdout.close() @@ -105,6 +105,44 @@ def test_backward_compatibility_page(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + node.safe_psql( + 'postgres', + 'create table tmp as select * from pgbench_accounts where aid < 1000') + + node.safe_psql( + 'postgres', + 'delete from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM') + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'insert into pgbench_accounts select * from pgbench_accounts') + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -118,8 +156,7 @@ def test_backward_compatibility_delta(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -189,8 +226,7 @@ def test_backward_compatibility_delta(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.backup_node(backup_dir, 'node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -204,6 +240,44 @@ def test_backward_compatibility_delta(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + node.safe_psql( + 'postgres', + 'create table tmp as select * from pgbench_accounts where aid < 1000') + + node.safe_psql( + 'postgres', + 'delete from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM') + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'insert into pgbench_accounts select * from pgbench_accounts') + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -530,3 +604,87 @@ def test_backward_compatibility_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_1(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, + old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + # PAGE2 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + # PAGE3 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + # merge chain created by old binary with new binary + output = self.merge_backup( + backup_dir, "node", backup_id) + + # check that in-place is disabled + self.assertIn( + "WARNING: In-place merge is disabled " + "because of program versions mismatch", output) + + # restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index c9a8361c0..595c8f152 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -833,6 +833,9 @@ def backup_node( if backup_type: cmd_list += ['-b', backup_type] + if not old_binary: + cmd_list += ['--no-sync'] + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id) def checkdb_node( @@ -889,6 +892,9 @@ def restore_node( if backup_id: cmd_list += ['-i', backup_id] + if not old_binary: + cmd_list += ['--no-sync'] + return self.run_pb(cmd_list + options, old_binary=old_binary) def show_pb( diff --git a/tests/locking.py b/tests/locking.py index 54389bde6..5044575ba 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -33,7 +33,7 @@ def test_locking_running_validate_1(self): gdb = self.backup_node( backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -87,7 +87,7 @@ def test_locking_running_validate_2(self): gdb = self.backup_node( backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -156,7 +156,7 @@ def test_locking_running_validate_2_specific_id(self): gdb = self.backup_node( backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -253,7 +253,7 @@ def test_locking_running_3(self): gdb = self.backup_node( backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) @@ -399,7 +399,7 @@ def test_locking_restore_locked_without_validation(self): restore_id) in e.message and 'is using backup {0} and still is running'.format( backup_id) in e.message and - 'ERROR: Cannot lock backup directory' in e.message, + 'ERROR: Cannot lock backup' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/merge.py b/tests/merge.py index 1f5538193..5cd30c5c2 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -31,7 +31,7 @@ def test_basic_merge_full_page(self): node.slow_start() # Do full backup - self.backup_node(backup_dir, "node", node) + self.backup_node(backup_dir, "node", node, options=['--compress']) show_backup = self.show_pb(backup_dir, "node")[0] self.assertEqual(show_backup["status"], "OK") @@ -45,7 +45,7 @@ def test_basic_merge_full_page(self): conn.commit() # Do first page backup - self.backup_node(backup_dir, "node", node, backup_type="page") + self.backup_node(backup_dir, "node", node, backup_type="page", options=['--compress']) show_backup = self.show_pb(backup_dir, "node")[1] # sanity check @@ -60,7 +60,9 @@ def test_basic_merge_full_page(self): conn.commit() # Do second page backup - self.backup_node(backup_dir, "node", node, backup_type="page") + self.backup_node( + backup_dir, "node", node, + backup_type="page", options=['--compress']) show_backup = self.show_pb(backup_dir, "node")[2] page_id = show_backup["id"] @@ -1047,7 +1049,7 @@ def test_continue_failed_merge(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(5) @@ -1068,7 +1070,7 @@ def test_continue_failed_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge @@ -1121,7 +1123,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): # Failed MERGE gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(2) @@ -1158,7 +1160,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "ERROR: Merging of backup {0} failed".format( + "ERROR: Backup {0} has status CORRUPT, merge is aborted".format( backup_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1217,8 +1219,12 @@ def test_continue_failed_merge_2(self): gdb.run_until_break() + gdb._execute('thread apply all bt') + gdb.continue_execution_until_break(20) + gdb._execute('thread apply all bt') + gdb._execute('signal SIGKILL') print(self.show_pb(backup_dir, as_text=True, as_json=False)) @@ -1234,8 +1240,8 @@ def test_continue_failed_merge_2(self): def test_continue_failed_merge_3(self): """ - Check that failed MERGE can`t be continued after target backup deleting - Create FULL and 2 PAGE backups + Check that failed MERGE cannot be continued if intermediate + backup is missing. """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1297,14 +1303,14 @@ def test_continue_failed_merge_3(self): gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') print(self.show_pb(backup_dir, as_text=True, as_json=False)) - print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) + # print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) # DELETE PAGE1 shutil.rmtree( @@ -1320,8 +1326,8 @@ def test_continue_failed_merge_3(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "ERROR: Parent full backup for the given backup {0} was not found".format( - backup_id_merge) in e.message, + "ERROR: Incremental chain is broken, " + "merge is impossible to finish" in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1545,7 +1551,7 @@ def test_crash_after_opening_backup_control_2(self): backup_dir, 'backups', 'node', full_id, 'database', fsm_path) - print(file_to_remove) + # print(file_to_remove) os.remove(file_to_remove) @@ -1701,9 +1707,6 @@ def test_failed_merge_after_delete(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('parray_bsearch') - gdb.continue_execution_until_break() - gdb.set_breakpoint('pgFileDelete') gdb.continue_execution_until_break(20) @@ -1711,7 +1714,7 @@ def test_failed_merge_after_delete(self): # backup half-merged self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) self.assertEqual( full_id, self.show_pb(backup_dir, 'node')[0]['id']) @@ -1731,9 +1734,8 @@ def test_failed_merge_after_delete(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "ERROR: Parent full backup for the given " - "backup {0} was not found".format( - page_id_2) in e.message, + "ERROR: Full backup {0} has unfinished merge with backup {1}".format( + full_id, page_id) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1764,7 +1766,7 @@ def test_failed_merge_after_delete_1(self): page_1 = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change FULL B backup status to ERROR + # Change PAGE1 backup status to ERROR self.change_backup_status(backup_dir, 'node', page_1, 'ERROR') pgdata = self.pgdata_content(node.data_dir) @@ -1773,11 +1775,11 @@ def test_failed_merge_after_delete_1(self): pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) pgbench.wait() - # take PAGE backup + # take PAGE2 backup page_id = self.backup_node( backup_dir, 'node', node, backup_type='page') - # Change FULL B backup status to ERROR + # Change PAGE1 backup status to OK self.change_backup_status(backup_dir, 'node', page_1, 'OK') gdb = self.merge_backup( @@ -1787,8 +1789,8 @@ def test_failed_merge_after_delete_1(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('parray_bsearch') - gdb.continue_execution_until_break() +# gdb.set_breakpoint('parray_bsearch') +# gdb.continue_execution_until_break() gdb.set_breakpoint('pgFileDelete') gdb.continue_execution_until_break(30) @@ -1800,6 +1802,7 @@ def test_failed_merge_after_delete_1(self): # restore node.cleanup() try: + #self.restore_node(backup_dir, 'node', node, backup_id=page_1) self.restore_node(backup_dir, 'node', node) self.assertEqual( 1, 0, @@ -1815,6 +1818,158 @@ def test_failed_merge_after_delete_1(self): self.del_test_dir(module_name, fname) + def test_failed_merge_after_delete_2(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + page_1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # add data + pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # take PAGE2 backup + page_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_2, gdb=True, + options=['--log-level-console=VERBOSE']) + + gdb.set_breakpoint('pgFileDelete') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + gdb._execute('signal SIGKILL') + + self.delete_pb(backup_dir, 'node', backup_id=page_2) + + # rerun merge + try: + #self.restore_node(backup_dir, 'node', node, backup_id=page_1) + self.merge_backup(backup_dir, 'node', page_1) + self.assertEqual( + 1, 0, + "Expecting Error because of backup is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Full backup {0} has unfinished merge " + "with backup {1}".format(full_id, page_2), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.del_test_dir(module_name, fname) + + def test_failed_merge_after_delete_3(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + dboid = node.safe_psql( + "postgres", + "select oid from pg_database where datname = 'testdb'").rstrip() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # drop database + node.safe_psql( + 'postgres', + 'DROP DATABASE testdb') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb') + + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + + # backup half-merged + self.assertEqual( + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + db_path = os.path.join( + backup_dir, 'backups', 'node', full_id) + + # FULL backup is missing now + shutil.rmtree(db_path) + + try: + self.merge_backup( + backup_dir, 'node', page_id_2, + options=['--log-level-console=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of missing parent.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Failed to find parent full backup for {0}".format( + page_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_merge_backup_from_future(self): """ @@ -2085,8 +2240,7 @@ def test_merge_multiple_descendants(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "ERROR: Parent full backup for the given " - "backup {0} was not found".format( + "ERROR: Failed to find parent full backup for {0}".format( page_id_a3) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/replica.py b/tests/replica.py index af126865b..7b37b6861 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -928,7 +928,8 @@ def test_replica_toast(self): pg_options={ 'autovacuum': 'off', 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) + 'wal_level': 'replica', + 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) @@ -966,14 +967,14 @@ def test_replica_toast(self): self.switch_wal_segment(master) self.switch_wal_segment(master) - self.wait_until_replica_catch_with_master(master, replica) - master.safe_psql( 'postgres', 'CREATE TABLE t1 AS ' 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' 'FROM generate_series(0,10) i') + self.wait_until_replica_catch_with_master(master, replica) + output = self.backup_node( backup_dir, 'replica', replica, options=[ diff --git a/tests/retention.py b/tests/retention.py index d6ac66531..ef17d3e04 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -585,46 +585,22 @@ def test_window_merge_interleaved_incremental_chains(self): options=['--retention-window=1', '--expired', '--merge-expired']) self.assertIn( - "Merge incremental chain between FULL backup {0} and backup {1}".format( + "Merge incremental chain between full backup {0} and backup {1}".format( backup_id_a, page_id_a2), output) self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_a1, backup_id_a), output) + "Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a2), output) self.assertIn( - "Rename {0} to {1}".format( - backup_id_a, page_id_a1), output) - - self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_a2, page_id_a1), output) - - self.assertIn( - "Rename {0} to {1}".format( - page_id_a1, page_id_a2), output) - - self.assertIn( - "Merge incremental chain between FULL backup {0} and backup {1}".format( + "Merge incremental chain between full backup {0} and backup {1}".format( backup_id_b, page_id_b2), output) self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_b1, backup_id_b), output) - - self.assertIn( - "Rename {0} to {1}".format( - backup_id_b, page_id_b1), output) - - self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_b2, page_id_b1), output) - - self.assertIn( - "Rename {0} to {1}".format( - page_id_b1, page_id_b2), output) + "Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b2), output) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) @@ -979,64 +955,295 @@ def test_basic_window_merge_multiple_descendants(self): output = self.delete_expired( backup_dir, 'node', options=[ - '--retention-window=1', '--expired', + '--retention-window=1', '--delete-expired', '--merge-expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) # Merging chain A self.assertIn( - "Merge incremental chain between FULL backup {0} and backup {1}".format( + "Merge incremental chain between full backup {0} and backup {1}".format( backup_id_a, page_id_a3), output) self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_a1, backup_id_a), output) + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a3), output) - self.assertIn( - "INFO: Rename {0} to {1}".format( - backup_id_a, page_id_a1), output) +# self.assertIn( +# "WARNING: Backup {0} has multiple valid descendants. " +# "Automatic merge is not possible.".format( +# page_id_a1), output) self.assertIn( - "WARNING: Backup {0} has multiple valid descendants. " - "Automatic merge is not possible.".format( - page_id_a1), output) + "LOG: Consider backup {0} for purge".format( + page_id_a2), output) # Merge chain B self.assertIn( - "Merge incremental chain between FULL backup {0} and backup {1}".format( + "Merge incremental chain between full backup {0} and backup {1}".format( backup_id_b, page_id_b3), output) self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_b1, backup_id_b), output) + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b3), output) self.assertIn( - "INFO: Rename {0} to {1}".format( - backup_id_b, page_id_b1), output) + "Delete: {0}".format(page_id_a2), output) - self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_b2, page_id_b1), output) + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'FULL') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_window_merge_multiple_descendants_1(self): + """ + PAGEb3 + | PAGEa3 + -----------------------------retention window + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEb2 and PAGEb1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # and FULL stuff + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa3 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2, PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Merging chain A self.assertIn( - "INFO: Rename {0} to {1}".format( - page_id_b1, page_id_b2), output) + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_a, page_id_a3), + output) self.assertIn( - "Merging backup {0} with backup {1}".format( - page_id_b3, page_id_b2), output) + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a3), output) +# self.assertIn( +# "WARNING: Backup {0} has multiple valid descendants. " +# "Automatic merge is not possible.".format( +# page_id_a1), output) + + # Merge chain B self.assertIn( - "INFO: Rename {0} to {1}".format( - page_id_b2, page_id_b3), output) + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_b, page_id_b3), output) - # this backup deleted because it is not guarded by retention self.assertIn( - "INFO: Delete: {0}".format( - page_id_a1), output) + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b3), output) self.assertEqual( self.show_pb(backup_dir, 'node')[2]['id'], @@ -1048,7 +1255,7 @@ def test_basic_window_merge_multiple_descendants(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['id'], - page_id_a1) + page_id_a2) self.assertEqual( self.show_pb(backup_dir, 'node')[2]['backup-mode'], @@ -1056,11 +1263,17 @@ def test_basic_window_merge_multiple_descendants(self): self.assertEqual( self.show_pb(backup_dir, 'node')[1]['backup-mode'], - 'PAGE') + 'FULL') self.assertEqual( self.show_pb(backup_dir, 'node')[0]['backup-mode'], - 'FULL') + 'PAGE') + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--delete-expired', '--log-level-console=log']) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1596,7 +1809,7 @@ def test_failed_merge_redundancy_retention(self): # create pair of MERGING backup as a result of failed merge gdb = self.merge_backup( backup_dir, 'node', delta_id, gdb=True) - gdb.set_breakpoint('copy_file') + gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -2491,10 +2704,14 @@ def test_wal_depth_2(self): self.assertTrue(show_tli4_before) self.assertTrue(show_tli5_before) + sleep(5) + output = self.delete_pb( backup_dir, 'node', options=['--delete-wal', '--wal-depth=2', '--log-level-console=verbose']) +# print(output) + show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) From 3d81b532422d8ffdb2419de1f13e6298967363e7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 22 Feb 2020 17:33:34 +0300 Subject: [PATCH 1154/2107] [Issue #175] Incorrect signature for function bt_index_check() in documentation --- doc/pgprobackup.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 89d540b99..fe5109e60 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -951,8 +951,8 @@ GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup; From c2f39687368821aa21b74de29e18e775aafaade6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 22 Feb 2020 18:17:41 +0300 Subject: [PATCH 1155/2107] [Issue #175] test coverage --- tests/checkdb.py | 163 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/tests/checkdb.py b/tests/checkdb.py index 64a733ac7..033a6c253 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -495,3 +495,166 @@ def test_checkdb_sigint_handling(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_checkdb_with_least_privileges(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + try: + node.safe_psql( + "backupdb", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "backupdb", + "create extension amcheck_next") + + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC;") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) + # >= 10 + else: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) + +# if ProbackupTest.enterprise: +# node.safe_psql( +# "backupdb", +# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") +# +# node.safe_psql( +# "backupdb", +# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + + # checkdb + try: + self.checkdb_node( + backup_dir, 'node', + options=[ + '--amcheck', '-U', 'backup', + '-d', 'backupdb', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because permissions are missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "INFO: Amcheck succeeded for database 'backupdb'", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Extension 'amcheck' or 'amcheck_next' are " + "not installed in database postgres", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "ERROR: Some databases were not amchecked", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) From 6f1ba680abfd56514936020a96451fb0624fe011 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Feb 2020 22:04:01 +0300 Subject: [PATCH 1156/2107] [Issue #171] minor fixes --- src/restore.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/restore.c b/src/restore.c index 391741d6b..583279a4a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1056,12 +1056,10 @@ create_recovery_conf(time_t backup_id, fio_fprintf(fp, "standby_mode = 'on'\n"); #endif - if(params->primary_conninfo) + if (params->primary_conninfo) fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); - else{ - if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); - } + else if (backup->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); } if (pitr_requested) From d46e96b808864df2d48786e23d73b57730126186 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Feb 2020 22:04:14 +0300 Subject: [PATCH 1157/2107] [Issue #171] test coverage --- tests/restore.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/restore.py b/tests/restore.py index 93b28c800..54b6d44e4 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3376,4 +3376,54 @@ def test_stream_restore_command_option(self): self.assertEqual('2', timeline_id) # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_primary_conninfo(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' + + self.restore_node( + backup_dir, 'node', replica, + options=['-R', '--primary-conninfo={0}'.format(str_conninfo)]) + + if self.get_version(node) >= self.version_to_num('12.0'): + standby_signal = os.path.join(replica.data_dir, 'standby.signal') + self.assertTrue( + os.path.isfile(standby_signal), + "File '{0}' do not exists".format(standby_signal)) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(replica.data_dir, 'probackup_recovery.conf') + else: + recovery_conf = os.path.join(replica.data_dir, 'recovery.conf') + + with open(os.path.join(replica.data_dir, recovery_conf), 'r') as f: + recovery_conf_content = f.read() + + self.assertIn(str_conninfo, recovery_conf_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) From d42e72016c9fcd0db00dafa7951a33bb2dc340f1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 23 Feb 2020 22:51:58 +0300 Subject: [PATCH 1158/2107] [Issue #171] documentation --- doc/pgprobackup.xml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index fe5109e60..26ce99c9c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1634,13 +1634,14 @@ pg_probackup restore -B backup_dir --instance If you restore ARCHIVE backups, perform PITR, - or specify the --restore-as-replica option with the + or specify the --restore-as-replica flag with the restore command to set up a standby server, pg_probackup creates a recovery configuration file once all data files are copied into the target directory. This file includes the minimal settings required for recovery, except for the password in the primary_conninfo - parameter; you have to add the password manually, if required. + parameter; you have to add the password manually or use + the --primary-conninfo option, if required. For PostgreSQL 11 or lower, recovery settings are written into the recovery.conf file. Starting from PostgreSQL 12, @@ -3584,6 +3585,7 @@ pg_probackup restore -B backup_dir --instance OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] [--restore-command=cmdline] +[--primary-conninfo=primary_conninfo] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -3621,6 +3623,21 @@ pg_probackup restore -B backup_dir --instance + + + + + Sets the + primary_conninfo + parameter to the specified value. + This option will be ignored if used without the flag. + + + Example: --primary-conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' + + + + From 1494a8e872f607326ed9f8189c629416fb099021 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 25 Feb 2020 18:29:51 +0300 Subject: [PATCH 1159/2107] Added --error-state option for delete command --- src/delete.c | 33 +++++++++++++++++++++++++++++++++ src/pg_probackup.c | 16 +++++++++++----- src/pg_probackup.h | 1 + 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/delete.c b/src/delete.c index 4ca958e5d..ed9d56a3b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1023,3 +1023,36 @@ do_delete_instance(void) elog(INFO, "Instance '%s' successfully deleted", instance_name); return 0; } + +/* Delete all error backup files of given instance. */ +int +do_delete_error(void) +{ + parray *backup_list; + parray *xlog_files_list; + int i; + int rc; + char instance_config_path[MAXPGPATH]; + + + /* Delete all error backups. */ + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + if (backup->status == BACKUP_STATUS_ERROR){ + /* elog(INFO, "Delete error backup '%s' ", base36enc(backup->backup_id)); */ + catalog_lock_backup_list(backup_list, i, i); + delete_backup_files(backup); + } + } + + /* Cleanup */ + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + + + elog(INFO, "Error backups from instance '%s' successfully deleted", instance_name); + return 0; +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 88db6f900..ed06ac49e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -117,7 +117,7 @@ bool delete_expired = false; bool merge_expired = false; bool force = false; bool dry_run = false; - +bool delete_error = false; /* compression options */ bool compress_shortcut = false; @@ -201,6 +201,7 @@ static ConfigOption cmd_options[] = /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, + { 'b', 172, "error-state", &delete_error, SOURCE_CMD_STRICT }, /* TODO not implemented yet */ { 'b', 147, "force", &force, SOURCE_CMD_STRICT }, /* compression options */ @@ -786,13 +787,18 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); if (merge_expired && backup_id_string) elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together"); - if (!delete_expired && !merge_expired && !delete_wal && !backup_id_string) + if (delete_error && backup_id_string) + elog(ERROR, "You cannot specify --error-state and (-i, --backup-id) options together"); + if (!delete_expired && !merge_expired && !delete_wal && !delete_error && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: " - "--delete-expired |--delete-wal |--merge-expired |(-i, --backup-id)"); + "--delete-expired |--delete-wal |--merge-expired |--error-state |(-i, --backup-id)"); if (!backup_id_string) - return do_retention(); + if(delete_error) + do_delete_error(); + else + return do_retention(); else - do_delete(current.backup_id); + do_delete(current.backup_id); break; case MERGE_CMD: do_merge(current.backup_id); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c2cf55df9..58b538d43 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -706,6 +706,7 @@ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); extern int do_retention(void); extern int do_delete_instance(void); +extern int do_delete_error(void); /* in fetch.c */ extern char *slurpFile(const char *datadir, From 529a1878d0efd9c45473004ad1fad34fc2d29f14 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 27 Feb 2020 15:19:47 +0300 Subject: [PATCH 1160/2107] Fixes test for --error-state option --- tests/delete.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/delete.py b/tests/delete.py index 389584e8a..b4f8db6e6 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -801,3 +801,50 @@ def test_delete_multiple_descendants_dry_run(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_delete_error_backups(self): + """delete increment and all after him""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Take FULL BACKUP + backup_id_a = self.backup_node(backup_dir, 'node', node) + # Take PAGE BACKUP + backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Change status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete error backups + self.delete_pb(backup_dir, 'node', options=['--error-state']) + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, fname) + From 2cc3d03c4c456fdff2cf90a3af88dbb227302589 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 27 Feb 2020 16:19:09 +0300 Subject: [PATCH 1161/2107] [Issue #120] documentation: --error-state option for delete command --- doc/pgprobackup.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 26ce99c9c..5e9de6eca 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3228,6 +3228,13 @@ pg_probackup delete -B backup_dir --instance + + To delete backups with the ERROR status, use the flag: + + +pg_probackup delete -B backup_dir --instance instance_name --error-state + + @@ -3862,7 +3869,7 @@ pg_probackup merge -B backup_dir --instance backup_dir --instance instance_name [--help] [-j num_threads] [--progress] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] -[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} +[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --error-state} [--dry-run] [logging_options] From d57bf8ef05e6237b88732c1b39c8f9b8fb37e84b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 2 Mar 2020 20:04:02 +0300 Subject: [PATCH 1162/2107] [Issue #169] Addressing some issues, pointed out by Anastasia Lubennikova review: setbuf, comments, tests --- src/data.c | 12 ++++++++---- src/help.c | 6 ++++++ src/merge.c | 2 +- src/pg_probackup.c | 25 ++++++++++++++----------- src/restore.c | 13 ++++++++----- src/utils/file.c | 1 + src/validate.c | 6 ++++-- 7 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/data.c b/src/data.c index 44f6d3376..fc9a92af6 100644 --- a/src/data.c +++ b/src/data.c @@ -613,7 +613,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, } if (!fio_is_remote_file(in)) - setbuf(in, in_buffer); + setbuffer(in, in_buffer, STDIO_BUFSIZE); /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); @@ -621,7 +621,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, elog(ERROR, "Cannot open backup file \"%s\": %s", to_fullpath, strerror(errno)); - setbuf(out, out_buffer); + setbuffer(out, out_buffer, STDIO_BUFSIZE); /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) @@ -844,6 +844,10 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char if (tmp_file->write_size == BYTES_INVALID) continue; + /* If file was truncated in intermediate backup, + * it is ok not to truncate it now, because old blocks will be + * overwritten by new blocks from next backup. + */ if (tmp_file->write_size == 0) continue; @@ -859,7 +863,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuf(in, buffer); + setbuffer(in, buffer, STDIO_BUFSIZE); /* * Restore the file. @@ -1149,7 +1153,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuf(in, buffer); + setbuffer(in, buffer, STDIO_BUFSIZE); /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); diff --git a/src/help.c b/src/help.c index 5368a335b..14ede6dd6 100644 --- a/src/help.c +++ b/src/help.c @@ -119,6 +119,7 @@ help_pg_probackup(void) printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -156,6 +157,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); @@ -256,6 +258,7 @@ help_backup(void) printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-E external-directories-paths]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -294,6 +297,7 @@ help_backup(void) printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); + printf(_(" --no-sync do not sync backed up files to disk\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -390,6 +394,7 @@ help_restore(void) printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); printf(_(" [--restore-command=cmdline]\n")); + printf(_(" [--no-sync]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); @@ -432,6 +437,7 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); + printf(_(" --no-sync do not sync restored files to disk\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); diff --git a/src/merge.c b/src/merge.c index 91474fbbc..2d286062c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1107,7 +1107,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, if (out == NULL) elog(ERROR, "Cannot open merge target file \"%s\": %s", to_fullpath_tmp1, strerror(errno)); - setbuf(out, buffer); + setbuffer(out, buffer, STDIO_BUFSIZE); /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index a9132e53c..97ef973e4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -152,28 +152,29 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg); /* * Short name should be non-printable ASCII character. + * Use values between 128 and 255. */ static ConfigOption cmd_options[] = { /* directory options */ - { 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT }, + { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, /* common options */ { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, - { 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT }, - { 'b', 122, "progress", &progress, SOURCE_CMD_STRICT }, + { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, + { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, - { 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT }, + { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, /* backup options */ - { 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, + { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, - { 'b', 234, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, - { 'b', 134, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, - { 'b', 135, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 'b', 235, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, - { 'b', 237, "dry-run", &dry_run, SOURCE_CMD_STRICT }, + { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, + { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, + { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, + { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, + { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -218,7 +219,9 @@ static ConfigOption cmd_options[] = { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, - /* options for backward compatibility */ + /* options for backward compatibility + * TODO: remove in 3.0.0 + */ { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, diff --git a/src/restore.c b/src/restore.c index 7c6dfecec..ce9c650fb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -441,7 +441,6 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required - * TODO: before restore - lock entire parent chain */ if (params->is_restore) { @@ -555,7 +554,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, else backup->files = dest_files; - /* this sorting is important */ + /* + * this sorting is important, because we rely on it to find + * destination file in intermediate backups file lists + * using bsearch. + */ parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } @@ -622,8 +625,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * num_threads); - - /* Restore files into target directory */ if (dest_backup->stream) dest_bytes = dest_backup->pgdata_bytes + dest_backup->wal_bytes; else @@ -633,6 +634,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); time(&start_time); thread_interrupted = false; + + /* Restore files into target directory */ for (i = 0; i < num_threads; i++) { restore_files_arg *arg = &(threads_args[i]); @@ -851,7 +854,7 @@ restore_files(void *arg) goto done; if (!fio_is_remote_file(out)) - setbuf(out, buffer); + setbuffer(out, buffer, STDIO_BUFSIZE); /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) diff --git a/src/utils/file.c b/src/utils/file.c index 73fd780f2..031730ad8 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -549,6 +549,7 @@ int fio_pread(FILE* f, void* buf, off_t offs) } else { + /* For local file, opened by fopen, we should use stdio operations */ int rc; rc = fseek(f, offs, SEEK_SET); diff --git a/src/validate.c b/src/validate.c index 4f16edc68..167f86b9d 100644 --- a/src/validate.c +++ b/src/validate.c @@ -264,6 +264,7 @@ pgBackupValidateFiles(void *arg) */ if (file->write_size == BYTES_INVALID) { + /* TODO: lookup corresponding merge bug */ if (arguments->backup_mode == BACKUP_MODE_FULL) { /* It is illegal for file in FULL backup to have BYTES_INVALID */ @@ -276,10 +277,11 @@ pgBackupValidateFiles(void *arg) continue; } - /* no point in trying to open empty or non-changed files */ - if (file->write_size <= 0) + /* no point in trying to open empty file */ + if (file->write_size == 0) continue; + /* TODO: it is redundant to check file existence using stat */ if (stat(file->path, &st) == -1) { if (errno == ENOENT) From ac017dd8da08b24dbfc909633032909ed49db136 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 2 Mar 2020 20:05:30 +0300 Subject: [PATCH 1163/2107] [Issue #169] added tests for issues, pointed out by review --- tests/compatibility.py | 77 ++++++++++++++++++++++++++++++++++++++++++ tests/page.py | 75 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index c5b5fcc52..0b75d1a0e 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -688,3 +688,80 @@ def test_backward_compatibility_merge_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py index 54e601271..e815068d2 100644 --- a/tests/page.py +++ b/tests/page.py @@ -54,6 +54,7 @@ def test_basic_page_vacuum_truncate(self): self.backup_node(backup_dir, 'node', node) + # TODO: make it dynamic node.safe_psql( "postgres", "delete from t_heap where ctid >= '(11,0)'") @@ -101,6 +102,80 @@ def test_basic_page_vacuum_truncate(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_page_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup and check data correctness + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_page_stream(self): """ From 251e46c2ab014a2f3a5462b3af1675374065f5dd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 2 Mar 2020 21:07:09 +0300 Subject: [PATCH 1164/2107] [Issue #169] improve comments in merge --- src/merge.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/merge.c b/src/merge.c index 2d286062c..102807a46 100644 --- a/src/merge.c +++ b/src/merge.c @@ -427,13 +427,13 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* for fancy reporting */ time_t end_time; char pretty_time[20]; - /* in-place flags */ + /* in-place merge flags */ bool compression_match = false; bool program_version_match = false; /* It's redundant to check block checksumms during merge */ skip_block_validation = true; - /* Handle corner cases missing destination backup */ + /* Handle corner cases of missing destination backup */ if (dest_backup == NULL && full_backup->status == BACKUP_STATUS_MERGED) goto merge_rename; @@ -740,12 +740,11 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) } /* - * If we crash now, automatic rerun of failed merge will be impossible. - * The user must have to manually change start_time of FULL backup - * to start_time of destination backup: * PAGE2 DELETED * PAGE1 DELETED * FULL MERGED + * If we crash now, automatic rerun of failed merge is still possible: + * The user should start merge with full backup ID as an argument to option '-i'. */ merge_rename: @@ -891,7 +890,7 @@ merge_files(void *arg) * * Case 2: * in this case in place merge is possible: - * 0 PAGE; file, size BYTES_INVALID (should not be possible) + * 0 PAGE; file, size 0 * 1 PAGE; file, size 0 * 2 FULL; file, size 100500 * @@ -901,11 +900,11 @@ merge_files(void *arg) * 1 PAGE; file, size 100501 * 2 FULL; file, size 100500 * - * Case 4: + * Case 4 (good candidate for future optimization): * in this case in place merge is impossible: * 0 PAGE; file, size BYTES_INVALID * 1 PAGE; file, size 100501 - * 2 FULL; file, missing + * 2 FULL; file, not exists yet */ in_place = true; @@ -1138,6 +1137,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, if (tmp_file->write_size == 0) return; + /* sync second temp file to disk */ if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); @@ -1242,7 +1242,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, backup_non_data_file(tmp_file, NULL, from_fullpath, to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); - /* TODO: --no-sync support */ + /* sync temp file to disk */ if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp, strerror(errno)); From 8110715dbfafbfbbc37341d770358b74eb8cb6ef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 3 Mar 2020 15:59:19 +0300 Subject: [PATCH 1165/2107] [Issue #169] review: remove slurpFileFullPath --- src/fetch.c | 77 +++------------------------------------------- src/pg_probackup.h | 3 -- src/util.c | 2 +- 3 files changed, 5 insertions(+), 77 deletions(-) diff --git a/src/fetch.c b/src/fetch.c index d65cdf7f2..6a9c2db51 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -13,67 +13,6 @@ #include #include -/* - * Read a file into memory. - * The file contents are returned in a malloc'd buffer, and *filesize - * is set to the length of the file. - * - * The returned buffer is always zero-terminated; the size of the returned - * buffer is actually *filesize + 1. That's handy when reading a text file. - * This function can be used to read binary files as well, you can just - * ignore the zero-terminator in that case. - * - */ -char * -slurpFileFullPath(const char *from_fullpath, size_t *filesize, bool safe, fio_location location) -{ - int fd; - char *buffer; - int len; - struct stat statbuf; - - if ((fd = fio_open(from_fullpath, O_RDONLY | PG_BINARY, location)) == -1) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not open file \"%s\" for reading: %s", - from_fullpath, strerror(errno)); - } - - if (fio_fstat(fd, &statbuf) < 0) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not stat file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - len = statbuf.st_size; - - buffer = pg_malloc(len + 1); - - if (fio_read(fd, buffer, len) != len) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not read file \"%s\": %s\n", - from_fullpath, strerror(errno)); - } - - fio_close(fd); - - /* Zero-terminate the buffer. */ - buffer[len] = '\0'; - - if (filesize) - *filesize = len; - - return buffer; -} - /* * Read a file into memory. The file to be read is /. * The file contents are returned in a malloc'd buffer, and *filesize @@ -93,23 +32,15 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi struct stat statbuf; char fullpath[MAXPGPATH]; int len; - snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); - if (fio_access(fullpath, R_OK, location) != 0) - { - if (safe) - return NULL; - else - elog(ERROR, "could not open file \"%s\" for reading: %s", - fullpath, strerror(errno)); - } + join_path_components(fullpath, datadir, path); if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) { if (safe) return NULL; else - elog(ERROR, "could not open file \"%s\" for reading: %s", + elog(ERROR, "Could not open file \"%s\" for reading: %s", fullpath, strerror(errno)); } @@ -118,7 +49,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi if (safe) return NULL; else - elog(ERROR, "could not open file \"%s\" for reading: %s", + elog(ERROR, "Could not stat file \"%s\": %s", fullpath, strerror(errno)); } @@ -130,7 +61,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi if (safe) return NULL; else - elog(ERROR, "could not read file \"%s\": %s\n", + elog(ERROR, "Could not read file \"%s\": %s\n", fullpath, strerror(errno)); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6fbd77736..7a7db97e7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -712,9 +712,6 @@ extern char *slurpFile(const char *datadir, size_t *filesize, bool safe, fio_location location); -extern char *slurpFileFullPath(const char *from_fullpath, - size_t *filesize, bool safe, - fio_location location); extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ diff --git a/src/util.c b/src/util.c index 871ad1051..35799e2ca 100644 --- a/src/util.c +++ b/src/util.c @@ -390,7 +390,7 @@ copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, char *buffer; size_t size; - buffer = slurpFileFullPath(from_fullpath, &size, false, from_location); + buffer = slurpFile(from_fullpath, "", &size, false, from_location); digestControlFile(&ControlFile, buffer, size); From e9c31759c571f9daadd7ae97f854c0dd0ee7e7c1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 3 Mar 2020 16:57:09 +0300 Subject: [PATCH 1166/2107] [Issue #169] pgFileGetCRC refactoring, now it operates only locally --- src/archive.c | 4 +-- src/backup.c | 23 +++++++++-------- src/dir.c | 64 ++-------------------------------------------- src/pg_probackup.h | 7 +++-- src/utils/file.c | 4 +-- src/validate.c | 2 +- 6 files changed, 23 insertions(+), 81 deletions(-) diff --git a/src/archive.c b/src/archive.c index a42163227..658c3dfba 100644 --- a/src/archive.c +++ b/src/archive.c @@ -594,11 +594,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) else #endif { - crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST); + crc2 = fio_get_crc32(path2, FIO_BACKUP_HOST); } /* Get checksum of original file */ - crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST); + crc1 = fio_get_crc32(path1, FIO_DB_HOST); return EQ_CRC32C(crc1, crc2); } diff --git a/src/backup.c b/src/backup.c index 8677f2f8f..183f1b2a3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -575,6 +575,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) { parray *xlog_files_list; char pg_xlog_path[MAXPGPATH]; + char wal_full_path[MAXPGPATH]; /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); @@ -586,11 +587,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) for (i = 0; i < parray_num(xlog_files_list); i++) { pgFile *file = (pgFile *) parray_get(xlog_files_list, i); + + join_path_components(wal_full_path, pg_xlog_path, file->rel_path); + if (S_ISREG(file->mode)) { - file->crc = pgFileGetCRC(file->path, true, false, - &file->read_size, FIO_BACKUP_HOST); - file->write_size = file->read_size; + file->crc = pgFileGetCRC(wal_full_path, true, false); + file->write_size = file->size; } /* Remove file path root prefix*/ if (strstr(file->path, database_path) == file->path) @@ -1805,10 +1808,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, { file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0, FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(file->path, true, false, - &file->read_size, FIO_BACKUP_HOST); - file->write_size = file->read_size; - file->uncompressed_size = file->read_size; + + file->crc = pgFileGetCRC(backup_label, true, false); + + file->write_size = file->size; + file->uncompressed_size = file->size; free(file->path); file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); @@ -1854,9 +1858,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, FIO_BACKUP_HOST); if (S_ISREG(file->mode)) { - file->crc = pgFileGetCRC(file->path, true, false, - &file->read_size, FIO_BACKUP_HOST); - file->write_size = file->read_size; + file->crc = pgFileGetCRC(tablespace_map, true, false); + file->write_size = file->size; } free(file->path); file->path = strdup(PG_TABLESPACE_MAP_FILE); diff --git a/src/dir.c b/src/dir.c index 25dfcfaed..bd0c09303 100644 --- a/src/dir.c +++ b/src/dir.c @@ -259,10 +259,9 @@ pgFileDelete(pgFile *file, const char *full_path) * We cannot make decision about file decompression because * user may ask to backup already compressed files and we should be * obvious about it. - * TODO: add decompression option. */ pg_crc32 -pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok) +pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) { FILE *fp; pg_crc32 crc = 0; @@ -316,65 +315,6 @@ pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok) return crc; } -/* - * Read the file to compute its CRC. - * As a handy side effect, we return filesize via bytes_read parameter. - */ -pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted, - size_t *bytes_read, fio_location location) -{ - FILE *fp; - pg_crc32 crc = 0; - char buf[STDIO_BUFSIZE]; - size_t len = 0; - size_t total = 0; - int errno_tmp; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = fio_fopen(file_path, PG_BINARY_R, location); - if (fp == NULL) - { - if (!raise_on_deleted && errno == ENOENT) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - else - elog(ERROR, "cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fio_fread(fp, buf, sizeof(buf)); - if (len == 0) - break; - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - total += len; - } - - if (bytes_read) - *bytes_read = total; - - errno_tmp = errno; - if (len < 0) - elog(WARNING, "cannot read \"%s\": %s", file_path, - strerror(errno_tmp)); - - FIN_FILE_CRC32(use_crc32c, crc); - fio_fclose(fp); - - return crc; -} - void pgFileFree(void *file) { @@ -1775,7 +1715,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ FIO_BACKUP_HOST); pfree(file->path); file->path = pgut_strdup(DATABASE_MAP); - file->crc = pgFileGetCRCnew(database_map_path, true, false); + file->crc = pgFileGetCRC(database_map_path, true, false); file->write_size = file->read_size; file->uncompressed_size = file->read_size; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7a7db97e7..e620950e7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -826,10 +826,9 @@ extern pgFile *pgFileInit(const char *path, const char *rel_path); extern void pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, - bool raise_on_deleted, size_t *bytes_read, fio_location location); -extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c); -//extern pg_crc32 pgFileGetCRC_compressed(const char *file_path, bool use_crc32c, bool missing_ok); + +extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c); + extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileMapComparePath(const void *f1, const void *f2); diff --git a/src/utils/file.c b/src/utils/file.c index 031730ad8..dd7f0845f 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -926,7 +926,7 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location) return crc; } else - return pgFileGetCRCnew(file_path, true, true); + return pgFileGetCRC(file_path, true, true); } /* Remove file */ @@ -1629,7 +1629,7 @@ void fio_communicate(int in, int out) break; case FIO_GET_CRC32: /* calculate crc32 for a file */ - crc = pgFileGetCRCnew(buf, true, true); + crc = pgFileGetCRC(buf, true, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; default: diff --git a/src/validate.c b/src/validate.c index 167f86b9d..1129f0481 100644 --- a/src/validate.c +++ b/src/validate.c @@ -328,7 +328,7 @@ pgBackupValidateFiles(void *arg) crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021 || arguments->backup_version >= 20025, - true, NULL, FIO_LOCAL_HOST); + false); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", From c3d716383a4b75696916d6c890e2635488720978 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 3 Mar 2020 23:42:10 +0300 Subject: [PATCH 1167/2107] [Issue #178] enabled logging into file breaks recovery via archive-get --- src/pg_probackup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 32260d930..53f9b11ac 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -533,6 +533,14 @@ main(int argc, char *argv[]) setMyLocation(); } + /* disable logging into file for archive-push and archive-get */ + if (backup_subcmd == ARCHIVE_GET_CMD || + backup_subcmd == ARCHIVE_PUSH_CMD) + { + instance_config.logger.log_level_file = LOG_OFF; + } + + /* Just read environment variables */ if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) config_get_opt_env(instance_options); From 4f1a76bcf431039b39346508f1bf3862a50d5ad8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 5 Mar 2020 11:25:52 +0300 Subject: [PATCH 1168/2107] [Issue #184] Don`t issue fsync when copying file by archive-get --- src/archive.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archive.c b/src/archive.c index a42163227..9f002b415 100644 --- a/src/archive.c +++ b/src/archive.c @@ -479,7 +479,7 @@ get_wal_file(const char *from_path, const char *to_path) } } - if (fio_flush(out) != 0 || fio_close(out) != 0) + if (fio_close(out) != 0) { errno_temp = errno; fio_unlink(to_path_temp, FIO_DB_HOST); From 1102e578d9601d35fa903eec4058b46c98695a7a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Mar 2020 02:01:43 +0300 Subject: [PATCH 1169/2107] [Issue #169] Documentation update --- doc/pgprobackup.xml | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index fe5109e60..269c2f676 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2242,6 +2242,13 @@ BACKUP INSTANCE 'node' MERGING — the backup is being merged. + + + MERGED — the backup data files were + successfully merged, but its metadata is in the process + of been updated. Only full backup can have this status. + + DELETING — the backup files are being deleted. @@ -3146,7 +3153,9 @@ pg_probackup merge -B backup_dir --instance This command merges the specified incremental backup to its parent full backup, together with all incremental backups - between them. Once the merge is complete, the incremental + between them. If the specified backup ID belong to the full backup, + then it will be merged with the closest incremental backup. + Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, @@ -3165,7 +3174,8 @@ pg_probackup show -B backup_dir --instance If the merge is still in progress, the backup status is - displayed as MERGING. The merge is idempotent, so you can + displayed as MERGING or, at the final stage, + MERGED. The merge is idempotent, so you can restart the merge if it was interrupted. @@ -3404,6 +3414,7 @@ pg_probackup backup -B backup_dir -b bac [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] [--archive-timeout=timeout] [--external-dirs=external_directory_path] +[--no-sync] [connection_options] [compression_options] [remote_options] [retention_options] [pinning_options] [logging_options] @@ -3556,6 +3567,18 @@ pg_probackup backup -B backup_dir -b bac + + + + + + Do not sync backed up files to disk. You can use this flag to speed + up backup process. Using this flag can result in data + corruption in case of operating system or hardware crash. + Corruption can be detected by backup validation. + + + @@ -3583,7 +3606,7 @@ pg_probackup restore -B backup_dir --instance num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] -[--restore-command=cmdline] +[--restore-command=cmdline] [--no-sync] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -3705,6 +3728,17 @@ pg_probackup restore -B backup_dir --instance + + + + + + Do not sync restored files to disk. You can use this flag to speed + up restore process. Using this flag can result in data + corruption in case of operating system or hardware crash. + + + @@ -3831,7 +3865,9 @@ pg_probackup merge -B backup_dir --instance Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if - any. As a result, the full backup takes in all the merged + any. If the specified backup ID belong to the full backup, + then it will be merged with the closest incremental backup. + As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. From 15228f35533a476338eb2c40d4b7fbf2ac0b1f30 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Mar 2020 11:47:25 +0300 Subject: [PATCH 1170/2107] [Issue #169] merge: handle the case of crash right after deletion of the target incremental backup. It should be possible to rerun merge by using the backup ID of deleted backup as an argument --- src/merge.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/merge.c b/src/merge.c index 102807a46..295d4230c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -106,6 +106,27 @@ do_merge(time_t backup_id) } } + /* + * Handle the case of crash right after deletion of the target + * incremental backup. We still can recover from this. + * Iterate over backups and look for the FULL backup with + * MERGED status, that has merge-target-id eqial to backup_id. + */ + if (dest_backup == NULL) + { + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backups, i); + + if (backup->status == BACKUP_STATUS_MERGED && + backup->merge_dest_backup == backup_id) + { + dest_backup = backup; + break; + } + } + } + if (dest_backup == NULL) elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); @@ -776,6 +797,11 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->root_dir, destination_path, strerror(errno)); } + /* If we crash here, it will produce full backup in MERGED + * status, located in directory with wrong backup id. + * It should not be a problem. + */ + /* * Merging finished, now we can safely update ID of the FULL backup */ From da283d3eadd534fbe3afbad6e06afb868d9284ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 6 Mar 2020 11:48:11 +0300 Subject: [PATCH 1171/2107] tests: added tests.merge.MergeTest.test_idempotent_merge --- tests/merge.py | 84 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 5cd30c5c2..8f6ccbb57 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2308,17 +2308,81 @@ def test_smart_merge(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_idempotent_merge(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb2') + + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id_2, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + gdb.remove_all_breakpoints() + + gdb.set_breakpoint('rename') + gdb.continue_execution_until_break() + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + show_backups = self.show_pb(backup_dir, "node") + self.assertEqual(len(show_backups), 1) + + self.assertEqual( + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + self.merge_backup(backup_dir, 'node', page_id_2) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) -# 1. always use parent link when merging (intermediates may be from different chain) -# 2. page backup we are merging with may disappear after failed merge, -# it should not be possible to continue merge after that -# PAGE_A MERGING (disappear) -# FULL MERGING + self.assertEqual( + page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) -# FULL MERGING -# PAGE_B OK (new backup) -# FULL MERGING + self.del_test_dir(module_name, fname) -# 3. Need new test with corrupted FULL backup -# 4. different compression levels +# 1. Need new test with corrupted FULL backup +# 2. different compression levels From a7943c1b4637a508988a7cfa5d5cf6b072da2d40 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 12 Mar 2020 14:54:03 +0300 Subject: [PATCH 1172/2107] Added --note option for backup command --- src/backup.c | 5 ++++- src/catalog.c | 7 +++++++ src/pg_probackup.c | 5 +++-- src/pg_probackup.h | 4 +++- src/show.c | 5 +++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 183f1b2a3..a2fa44795 100644 --- a/src/backup.c +++ b/src/backup.c @@ -750,7 +750,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) */ int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync) + pgSetBackupParams *set_backup_params, bool no_sync, char *note) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -766,6 +766,9 @@ do_backup(time_t start_time, bool no_validate, /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; + + current.note = note; + StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); diff --git a/src/catalog.c b/src/catalog.c index 5221ccab7..dd6bade6a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1524,6 +1524,10 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* print external directories list */ if (backup->external_dir_str) fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); + + if (backup->note) + fio_fprintf(out, "note = %s\n", backup->note); + } /* @@ -1761,6 +1765,7 @@ readBackupControlFile(const char *path) {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, + {'s', 0, "note", &backup->note, SOURCE_FILE_STRICT}, {0} }; @@ -2027,6 +2032,8 @@ pgBackupInit(pgBackup *backup) backup->external_dir_str = NULL; backup->root_dir = NULL; backup->files = NULL; + backup->note = NULL; + } /* free pgBackup object */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2af64b62c..dbeda715e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,7 +78,7 @@ bool temp_slot = false; bool backup_logs = false; bool smooth_checkpoint; char *remote_agent; - +static char *backup_note = NULL; /* restore options */ static char *target_time = NULL; static char *target_xid = NULL; @@ -177,6 +177,7 @@ static ConfigOption cmd_options[] = { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, + { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -769,7 +770,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, no_validate, set_backup_params, no_sync); + return do_backup(start_time, no_validate, set_backup_params, no_sync, backup_note); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 33b2373ef..1e33eb2da 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -387,6 +387,8 @@ struct pgBackup backup_path/instance_name/backup_id */ parray *files; /* list of files belonging to this backup * must be populated explicitly */ + char *note; + }; /* Recovery target for restore and validate subcommands */ @@ -642,7 +644,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync); + pgSetBackupParams *set_backup_params, bool no_sync, char *note); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); diff --git a/src/show.c b/src/show.c index f2af77255..bfc2195d3 100644 --- a/src/show.c +++ b/src/show.c @@ -416,6 +416,11 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add_value(buf, "status", status2str(backup->status), json_level, true); + if (backup->note){ + json_add_value(buf, "note", backup->note, + json_level, true); + + } json_add(buf, JT_END_OBJECT, &json_level); } From 884fed1a647e35e4357101d4cee23c81b2e7210d Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 12 Mar 2020 15:06:27 +0300 Subject: [PATCH 1173/2107] Some fix for --note option --- src/catalog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/catalog.c b/src/catalog.c index dd6bade6a..87e72a698 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2045,6 +2045,7 @@ pgBackupFree(void *backup) pfree(b->primary_conninfo); pfree(b->external_dir_str); pfree(b->root_dir); + pfree(b->note); pfree(backup); } From 9bd2bea0b720240d4b5eac9b30faf13c0b7d1893 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 13 Mar 2020 19:57:15 +0300 Subject: [PATCH 1174/2107] Wait remote agent to complete its work --- src/utils/file.c | 12 +++++++++++- src/utils/file.h | 4 +++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index dd7f0845f..0031f2c26 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -368,6 +368,12 @@ fio_disconnect(void) { if (fio_stdin) { + fio_header hdr; + hdr.cop = FIO_DISCONNECT; + hdr.size = 0; + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_DISCONNECTED); SYS_CHECK(close(fio_stdin)); SYS_CHECK(close(fio_stdout)); fio_stdin = 0; @@ -1492,7 +1498,7 @@ void fio_communicate(int in, int out) SYS_CHECK(setmode(out, _O_BINARY)); #endif - /* Main loop until command of processing master command */ + /* Main loop until end of processing all master commands */ while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { if (hdr.size != 0) { if (hdr.size > buf_size) { @@ -1632,6 +1638,10 @@ void fio_communicate(int in, int out) crc = pgFileGetCRC(buf, true, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; + case FIO_DISCONNECT: + hdr.cop = FIO_DISCONNECTED; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index c919a2e69..848a018e6 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -36,7 +36,9 @@ typedef enum FIO_SEND_PAGES, FIO_PAGE, FIO_WRITE_COMPRESSED, - FIO_GET_CRC32 + FIO_GET_CRC32, + FIO_DISCONNECT, + FIO_DISCONNECTED, } fio_operations; typedef enum From 6d0eb79323d3e2822e2449e41105dfde33cfdfa6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Mar 2020 01:50:49 +0300 Subject: [PATCH 1175/2107] restore: close ssh connection before exit --- src/restore.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/restore.c b/src/restore.c index f8685bc05..616f6589c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -472,6 +472,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, create_recovery_conf(target_backup_id, rt, dest_backup, params); } + /* ssh connection to longer needed */ + fio_disconnect(); + /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); From 86fd79512a2779e515fbb1574ac2f207a1011e56 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Mar 2020 14:35:57 +0300 Subject: [PATCH 1176/2107] [Issue #172] Optimize remote backup in PAGE and PTRACK modes --- src/data.c | 607 +++++++++++++++++++--------------------- src/dir.c | 3 +- src/pg_probackup.h | 39 ++- src/utils/file.c | 311 ++++++++++++++------ src/utils/file.h | 13 +- tests/backup.py | 446 ++++++++++++++++++++++++----- tests/compatibility.py | 80 ++++++ tests/false_positive.py | 254 +++++++++++++++++ tests/page.py | 20 +- tests/pgpro560.py | 2 +- tests/restore.py | 79 +----- tests/validate.py | 154 ---------- 12 files changed, 1286 insertions(+), 722 deletions(-) diff --git a/src/data.c b/src/data.c index fc9a92af6..f9c83e5d9 100644 --- a/src/data.c +++ b/src/data.c @@ -192,77 +192,67 @@ parse_page(Page page, XLogRecPtr *lsn) return false; } -/* Read one page from file directly accessing disk - * return value: - * 2 - if the page is found but zeroed - * 1 - if the page is found and valid - * 0 - if the page is not found, probably truncated - * -1 - if the page is found but read size is not multiple of BLKSIZE - * -2 - if the page is found but page header is "insane" - * -3 - if the page is found but page checksumm is wrong - * -4 - something went wrong, check errno - * +/* We know that header is invalid, store specific + * details in errormsg. */ -static int -read_page_from_file(pgFile *file, BlockNumber blknum, - FILE *in, Page page, XLogRecPtr *page_lsn, - uint32 checksum_version) +void +get_header_errormsg(Page page, char **errormsg) { - off_t offset = blknum * BLCKSZ; - ssize_t read_len = 0; - - /* read the block */ - read_len = fio_pread(in, page, offset); - - if (read_len != BLCKSZ) - { - - /* The block could have been truncated. It is fine. */ - if (read_len == 0) - return 0; - else if (read_len > 0) - return -1; - else - return -4; - } - - /* - * If we found page with invalid header, at first check if it is zeroed, - * which is a valid state for page. If it is not, read it and check header - * again, because it's possible that we've read a partly flushed page. - * If after several attempts page header is still invalid, throw an error. - * The same idea is applied to checksum verification. - */ - if (!parse_page(page, page_lsn)) - { - int i; - /* Check if the page is zeroed. */ - for (i = 0; i < BLCKSZ && page[i] == 0; i++); + PageHeader phdr = (PageHeader) page; + *errormsg = pgut_malloc(MAXPGPATH); + + if (PageGetPageSize(phdr) != BLCKSZ) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "page size %lu is not equal to block size %u", + PageGetPageSize(phdr), BLCKSZ); + + else if (phdr->pd_lower < SizeOfPageHeaderData) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_lower %i is less than page header size %lu", + phdr->pd_lower, SizeOfPageHeaderData); + + else if (phdr->pd_lower > phdr->pd_upper) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_lower %u is greater than pd_upper %u", + phdr->pd_lower, phdr->pd_upper); + + else if (phdr->pd_upper > phdr->pd_special) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_upper %u is greater than pd_special %u", + phdr->pd_upper, phdr->pd_special); + + else if (phdr->pd_special > BLCKSZ) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_special %u is greater than block size %u", + phdr->pd_special, BLCKSZ); + + else if (phdr->pd_special != MAXALIGN(phdr->pd_special)) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_special %i is misaligned, expected %lu", + phdr->pd_special, MAXALIGN(phdr->pd_special)); + + else if (phdr->pd_flags & ~PD_VALID_FLAG_BITS) + snprintf(*errormsg, MAXPGPATH, "page header invalid, " + "pd_flags mask contain illegal bits"); - /* Page is zeroed. No need to check header and checksum. */ - if (i == BLCKSZ) - return 2; + else + snprintf(*errormsg, MAXPGPATH, "page header invalid"); +} - return -2; - } +/* We know that checksumms are mismatched, store specific + * details in errormsg. + */ +void +get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) +{ + PageHeader phdr = (PageHeader) page; + *errormsg = pgut_malloc(MAXPGPATH); - /* Verify checksum */ - if (checksum_version) - { - BlockNumber blkno = file->segno * RELSEG_SIZE + blknum; - /* - * If checksum is wrong, sleep a bit and then try again - * several times. If it didn't help, throw error - */ - if (pg_checksum_page(page, blkno) != ((PageHeader) page)->pd_checksum) - return -3; - else - /* page header and checksum are correct */ - return 1; - } - else - /* page header is correct and checksum check is disabled */ - return 1; + snprintf(*errormsg, MAXPGPATH, + "page verification failed, " + "calculated checksum %u but expected %u", + phdr->pd_checksum, + pg_checksum_page(page, absolute_blkno)); } /* @@ -273,15 +263,20 @@ read_page_from_file(pgFile *file, BlockNumber blknum, * Prints appropriate warnings/errors/etc into log. * Returns: * PageIsOk(0) if page was successfully retrieved - * PageIsTruncated(-2) if the page was truncated - * SkipCurrentPage(-3) if we need to skip this page - * PageIsCorrupted(-4) if the page check mismatch + * PageIsTruncated(-1) if the page was truncated + * SkipCurrentPage(-2) if we need to skip this page, + * only used for DELTA backup + * PageIsCorrupted(-3) if the page checksum mismatch + * or header corruption, + * only used for checkdb + * TODO: probably we should always + * return it to the caller */ static int32 prepare_page(ConnectionArgs *conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, - BlockNumber blknum, BlockNumber nblocks, - FILE *in, BackupMode backup_mode, + BlockNumber blknum, FILE *in, + BackupMode backup_mode, Page page, bool strict, uint32 checksum_version, int ptrack_version_num, @@ -289,9 +284,8 @@ prepare_page(ConnectionArgs *conn_arg, const char *from_fullpath) { XLogRecPtr page_lsn = 0; - int try_again = 100; + int try_again = PAGE_READ_ATTEMPTS; bool page_is_valid = false; - bool page_is_truncated = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; /* check for interrupt */ @@ -305,83 +299,104 @@ prepare_page(ConnectionArgs *conn_arg, */ if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 20) { - while (!page_is_valid && try_again) + int rc = 0; + while (!page_is_valid && try_again--) { - int result = read_page_from_file(file, blknum, in, page, - &page_lsn, checksum_version); + /* read the block */ + int read_len = fio_pread(in, page, blknum * BLCKSZ); + page_lsn = 0; - switch (result) + /* The block could have been truncated. It is fine. */ + if (read_len == 0) { - case 2: - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); - return PageIsOk; - - case 1: - page_is_valid = true; - break; - - case 0: - /* This block was truncated.*/ - page_is_truncated = true; - /* Page is not actually valid, but it is absent - * and we're not going to reread it or validate */ - page_is_valid = true; - - elog(VERBOSE, "File \"%s\", block %u, file was truncated", - from_fullpath, blknum); - break; - - case -1: - elog(WARNING, "File: \"%s\", block %u, partial read, try again", - from_fullpath, blknum); - break; - - case -2: - elog(LOG, "File: \"%s\" blknum %u have wrong page header, try again", - from_fullpath, blknum); - break; - - case -3: - elog(LOG, "File: \"%s\" blknum %u have wrong checksum, try again", - from_fullpath, blknum); - break; - - case -4: - elog(LOG, "File: \"%s\" access error: %s", - from_fullpath, strerror(errno)); - break; + elog(VERBOSE, "Cannot read block %u of \"%s\": " + "block truncated", blknum, from_fullpath); + return PageIsTruncated; + } + else if (read_len < 0) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + blknum, from_fullpath, strerror(errno)); + else if (read_len != BLCKSZ) + elog(WARNING, "Cannot read block %u of \"%s\": " + "read %i of %d, try again", + blknum, from_fullpath, read_len, BLCKSZ); + else + { + /* We have BLCKSZ of raw data, validate it */ + rc = validate_one_page(page, absolute_blknum, + InvalidXLogRecPtr, &page_lsn, + checksum_version); + switch (rc) + { + case PAGE_IS_ZEROED: + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + return PageIsOk; + + case PAGE_IS_VALID: + /* in DELTA mode we must compare lsn */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA) + page_is_valid = true; + else + return PageIsOk; + break; + + case PAGE_HEADER_IS_INVALID: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", + from_fullpath, blknum); + break; + + case PAGE_CHECKSUM_MISMATCH: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", + from_fullpath, blknum); + break; + default: + Assert(false); + } } /* - * If ptrack support is available use it to get invalid block + * If ptrack support is available, use it to get invalid block * instead of rereading it 99 times */ - - if (result < 0 && strict && ptrack_version_num > 0) + if (!page_is_valid && strict && ptrack_version_num > 0) { elog(WARNING, "File \"%s\", block %u, try to fetch via shared buffer", from_fullpath, blknum); break; } - - try_again--; } + /* * If page is not valid after 100 attempts to read it * throw an error. */ - - if (!page_is_valid && - ((strict && ptrack_version_num == 0) || !strict)) + if (!page_is_valid) { - /* show this message for checkdb, merge or backup without ptrack support */ - elog(WARNING, "Corruption detected in file \"%s\", block %u", - from_fullpath, blknum); - } + int elevel = ERROR; + char *errormsg = NULL; + + /* Get the details of corruption */ + if (rc == PAGE_HEADER_IS_INVALID) + get_header_errormsg(page, &errormsg); + else if (rc == PAGE_CHECKSUM_MISMATCH) + get_checksum_errormsg(page, &errormsg, + file->segno * RELSEG_SIZE + blknum); + + /* Error out in case of merge or backup without ptrack support; + * issue warning in case of checkdb or backup with ptrack support + */ + if (!strict || (strict && ptrack_version_num > 0)) + elevel = WARNING; - /* Backup with invalid block and without ptrack support must throw error */ - if (!page_is_valid && strict && ptrack_version_num == 0) - elog(ERROR, "Data file corruption, canceling backup"); + if (errormsg) + elog(elevel, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, blknum, errormsg); + else + elog(elevel, "Corruption detected in file \"%s\", block %u", + from_fullpath, blknum); + + pg_free(errormsg); + } /* Checkdb not going futher */ if (!strict) @@ -412,7 +427,7 @@ prepare_page(ConnectionArgs *conn_arg, if (ptrack_page == NULL) { /* This block was truncated.*/ - page_is_truncated = true; + return PageIsTruncated; } else if (page_size != BLCKSZ) { @@ -433,18 +448,15 @@ prepare_page(ConnectionArgs *conn_arg, if (checksum_version) ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } + /* get lsn from page, provided by pg_ptrack_get_block() */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - !page_is_truncated && !parse_page(page, &page_lsn)) elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " "Possible risk of a memory corruption"); } - if (page_is_truncated) - return PageIsTruncated; - /* * Skip page if page lsn is less than START_LSN of parent backup. * Nullified pages must be copied by DELTA backup, just to be safe. @@ -475,10 +487,8 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, const char *errormsg = NULL; header.block = blknum; - header.compressed_size = page_state; - - /* The page was not truncated, so we need to compress it */ + /* Compress the page */ header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), page, BLCKSZ, calg, clevel, &errormsg); @@ -487,7 +497,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", blknum, from_fullpath, errormsg); - file->compress_alg = calg; + file->compress_alg = calg; /* TODO: wtf? why here? */ /* The page was successfully compressed. */ if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) @@ -533,15 +543,15 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok) { - FILE *in; - FILE *out; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; /* number of blocks in file */ - BlockNumber n_blocks_skipped = 0; - BlockNumber n_blocks_read = 0; /* number of blocks actually readed - * TODO: we should report them */ - int page_state; - char curr_page[BLCKSZ]; + FILE *in; + FILE *out; + BlockNumber blknum = 0; + BlockNumber nblocks = 0; /* number of blocks in source file */ + BlockNumber n_blocks_skipped = 0; + int page_state; + char curr_page[BLCKSZ]; + bool use_pagemap; + datapagemap_iterator_t *iter = NULL; /* stdio buffers */ char in_buffer[STDIO_BUFSIZE]; @@ -633,113 +643,114 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, * If page map is empty or file is not present in previous backup * backup all pages of the relation. * - * Usually enter here if backup_mode is FULL or DELTA. - * Also in some cases even PAGE backup is going here, - * becase not all data files are logged into WAL, - * for example CREATE DATABASE. - * Such files should be fully copied. - * In PTRACK 1.x there was a problem * of data files with missing _ptrack map. * Such files should be fully copied. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev) + + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev || + !file->pagemap.bitmap) + use_pagemap = false; + else + use_pagemap = true; + + + /* Remote mode */ + if (fio_is_remote_file(in)) { - /* remote FULL and DELTA */ - if (fio_is_remote_file(in)) + char *errmsg = NULL; + BlockNumber err_blknum = 0; + + /* TODO: retrying via ptrack should be implemented on the agent */ + int rc = fio_send_pages(in, out, file, + /* send prev backup START_LSN */ + backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + calg, clevel, checksum_version, + /* send pagemap if any */ + use_pagemap ? &file->pagemap : NULL, + /* variables for error reporting */ + &err_blknum, &errmsg); + + /* check for errors */ + if (rc == REMOTE_ERROR) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + err_blknum, from_fullpath, strerror(errno)); + + else if (rc == PAGE_CORRUPTION) { - int rc = fio_send_pages(in, out, file, - backup_mode == BACKUP_MODE_DIFF_DELTA && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - &n_blocks_skipped, calg, clevel); - - if (rc == PAGE_CHECKSUM_MISMATCH && ptrack_version_num >= 15) - /* only ptrack versions 1.5, 1.6, 1.7 and 2.x support this functionality */ - goto RetryUsingPtrack; - if (rc < 0) - elog(ERROR, "Failed to read file \"%s\": %s", - from_fullpath, - rc == PAGE_CHECKSUM_MISMATCH ? "data file checksum mismatch" : strerror(-rc)); - - /* TODO: check that fio_send_pages ain`t lying about number of readed blocks */ - n_blocks_read = rc; - - file->read_size = n_blocks_read * BLCKSZ; - file->uncompressed_size = (n_blocks_read - n_blocks_skipped)*BLCKSZ; + if (errmsg) + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, err_blknum, errmsg); + else + elog(ERROR, "Corruption detected in file \"%s\", block %u", + from_fullpath, err_blknum); } - else - { - /* local FULL and DELTA */ - RetryUsingPtrack: - for (blknum = 0; blknum < nblocks; blknum++) - { - page_state = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, nblocks, in, backup_mode, - curr_page, true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath); - - if (page_state == PageIsTruncated) - break; - else if (page_state == SkipCurrentPage) - n_blocks_skipped++; + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + err_blknum, to_fullpath, strerror(errno)); - else if (page_state == PageIsOk) - compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel, - from_fullpath, to_fullpath); - else - elog(ERROR, "Invalid page state: %i, file: %s, blknum %i", - page_state, file->rel_path, blknum); + file->read_size = rc * BLCKSZ; + pg_free(errmsg); - n_blocks_read++; - file->read_size += BLCKSZ; - } - } - file->n_blocks = n_blocks_read; } - /* - * If page map is not empty we scan only changed blocks. - * - * We will enter here if backup_mode is PAGE or PTRACK. - */ + /* Local mode */ else { - datapagemap_iterator_t *iter; - iter = datapagemap_iterate(&file->pagemap); - while (datapagemap_next(iter, &blknum)) + if (use_pagemap) + { + iter = datapagemap_iterate(&file->pagemap); + datapagemap_next(iter, &blknum); /* set first block */ + } + + while (blknum < nblocks) { page_state = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, nblocks, in, backup_mode, - curr_page, true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath); + blknum, in, backup_mode, curr_page, + true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath); if (page_state == PageIsTruncated) break; - /* TODO: PAGE and PTRACK should never get SkipCurrentPage */ + /* TODO: remove */ else if (page_state == SkipCurrentPage) n_blocks_skipped++; else if (page_state == PageIsOk) compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel, - from_fullpath, to_fullpath); + page_state, curr_page, calg, clevel, + from_fullpath, to_fullpath); + /* TODO: handle PageIsCorrupted, currently it is done in prepare_page */ else - elog(ERROR, "Invalid page state: %i, file: %s, blknum %i", - page_state, file->rel_path, blknum); + Assert(false); + - n_blocks_read++; file->read_size += BLCKSZ; - } - pg_free(file->pagemap.bitmap); - pg_free(iter); + /* next block */ + if (use_pagemap) + { + /* exit if pagemap is exhausted */ + if (!datapagemap_next(iter, &blknum)) + break; + } + else + blknum++; + } } + pg_free(file->pagemap.bitmap); + pg_free(iter); + + /* refresh n_blocks for FULL and DELTA */ + if (backup_mode == BACKUP_MODE_FULL || + backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = file->read_size / BLCKSZ; + if (fclose(out)) elog(ERROR, "Cannot close the backup file \"%s\": %s", to_fullpath, strerror(errno)); @@ -1290,106 +1301,55 @@ create_empty_file(fio_location from_location, const char *to_root, /* * Validate given page. - * - * Returns value: - * 0 - if the page is not found - * 1 - if the page is found and valid - * -1 - if the page is found but invalid + * This function is expected to be executed multiple times, + * so avoid using elog within it. + * lsn from page is assigned to page_lsn pointer. + * TODO: switch to enum for return codes. */ -#define PAGE_IS_NOT_FOUND 0 -#define PAGE_IS_FOUND_AND_VALID 1 -#define PAGE_IS_FOUND_AND_NOT_VALID -1 -static int -validate_one_page(Page page, pgFile *file, - BlockNumber blknum, XLogRecPtr stop_lsn, - uint32 checksum_version) +int +validate_one_page(Page page, BlockNumber absolute_blkno, + XLogRecPtr stop_lsn, XLogRecPtr *page_lsn, + uint32 checksum_version) { - PageHeader phdr; - XLogRecPtr lsn; - /* new level of paranoia */ if (page == NULL) - { - elog(LOG, "File \"%s\", block %u, page is NULL", file->path, blknum); return PAGE_IS_NOT_FOUND; - } - phdr = (PageHeader) page; - - if (PageIsNew(page)) + /* check that page header is ok */ + if (!parse_page(page, page_lsn)) { - int i; - + int i; /* Check if the page is zeroed. */ - for(i = 0; i < BLCKSZ && page[i] == 0; i++); + for (i = 0; i < BLCKSZ && page[i] == 0; i++); + /* Page is zeroed. No need to verify checksums */ if (i == BLCKSZ) - { - elog(LOG, "File: %s blknum %u, page is New, empty zeroed page", - file->path, blknum); - return PAGE_IS_FOUND_AND_VALID; - } - else - { - elog(WARNING, "File: %s blknum %u, page is New, but not zeroed", - file->path, blknum); - } + return PAGE_IS_ZEROED; - /* Page is zeroed. No sense in checking header and checksum. */ - return PAGE_IS_FOUND_AND_VALID; + /* Page does not looking good */ + return PAGE_HEADER_IS_INVALID; } /* Verify checksum */ if (checksum_version) { /* Checksums are enabled, so check them. */ - if (!(pg_checksum_page(page, file->segno * RELSEG_SIZE + blknum) - == ((PageHeader) page)->pd_checksum)) - { - elog(WARNING, "File: %s blknum %u have wrong checksum", - file->path, blknum); - return PAGE_IS_FOUND_AND_NOT_VALID; - } - } - - /* Check page for the sights of insanity. - * TODO: We should give more information about what exactly is looking "wrong" - */ - if (!(PageGetPageSize(phdr) == BLCKSZ && - // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && - (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && - phdr->pd_lower >= SizeOfPageHeaderData && - phdr->pd_lower <= phdr->pd_upper && - phdr->pd_upper <= phdr->pd_special && - phdr->pd_special <= BLCKSZ && - phdr->pd_special == MAXALIGN(phdr->pd_special))) - { - /* Page does not looking good */ - elog(WARNING, "Page header is looking insane: %s, block %i", - file->path, blknum); - return PAGE_IS_FOUND_AND_NOT_VALID; + if (pg_checksum_page(page, absolute_blkno) != ((PageHeader) page)->pd_checksum) + return PAGE_CHECKSUM_MISMATCH; } /* At this point page header is sane, if checksums are enabled - the`re ok. * Check that page is not from future. + * Note, this check should be used only by validate command. */ if (stop_lsn > 0) { /* Get lsn from page header. Ensure that page is from our time. */ - lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (lsn > stop_lsn) - { - elog(WARNING, "File: %s, block %u, checksum is %s. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->path, blknum, checksum_version ? "correct" : "not enabled", - (uint32) (lsn >> 32), (uint32) lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - return PAGE_IS_FOUND_AND_NOT_VALID; - } + if (*page_lsn > stop_lsn) + return PAGE_LSN_FROM_FUTURE; } - return PAGE_IS_FOUND_AND_VALID; + return PAGE_IS_VALID; } /* @@ -1441,7 +1401,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, { page_state = prepare_page(NULL, file, InvalidXLogRecPtr, - blknum, nblocks, in, BACKUP_MODE_FULL, + blknum, in, BACKUP_MODE_FULL, curr_page, false, checksum_version, 0, NULL, from_fullpath); @@ -1456,19 +1416,6 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, is_valid = false; continue; } - - /* At this point page is found and its checksum is ok, if any - * but could be 'insane' - * TODO: between prepare_page and validate_one_page we - * compute and compare checksum twice, it`s ineffective - */ - if (validate_one_page(curr_page, file, blknum, - InvalidXLogRecPtr, - 0) == PAGE_IS_FOUND_AND_NOT_VALID) - { - /* Page is corrupted */ - is_valid = false; - } } fclose(in); @@ -1507,10 +1454,12 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, /* read and validate pages one by one */ while (true) { + int rc = 0; DataPage compressed_page; /* used as read buffer */ DataPage page; BackupPageHeader header; BlockNumber blknum = 0; + XLogRecPtr page_lsn = 0; if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file validation"); @@ -1597,15 +1546,39 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, return false; } - if (validate_one_page(page.data, file, blknum, - stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) - is_valid = false; + rc = validate_one_page(page.data, + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_lsn, checksum_version); } else + rc = validate_one_page(compressed_page.data, + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_lsn, checksum_version); + + switch (rc) { - if (validate_one_page(compressed_page.data, file, blknum, - stop_lsn, checksum_version) == PAGE_IS_FOUND_AND_NOT_VALID) + case PAGE_IS_NOT_FOUND: + elog(LOG, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); + break; + case PAGE_IS_ZEROED: + elog(LOG, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); + break; + case PAGE_HEADER_IS_INVALID: + elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); is_valid = false; + break; + case PAGE_CHECKSUM_MISMATCH: + elog(WARNING, "File: %s blknum %u have wrong checksum", file->rel_path, blknum); + is_valid = false; + break; + case PAGE_LSN_FROM_FUTURE: + elog(WARNING, "File: %s, block %u, checksum is %s. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", + file->rel_path, blknum, + checksum_version ? "correct" : "not enabled", + (uint32) (page_lsn >> 32), (uint32) page_lsn, + (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + break; } } diff --git a/src/dir.c b/src/dir.c index bd0c09303..d44728dcc 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1693,7 +1693,6 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ char database_dir[MAXPGPATH]; char database_map_path[MAXPGPATH]; -// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); join_path_components(database_dir, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, database_dir, DATABASE_MAP); @@ -1717,7 +1716,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ file->path = pgut_strdup(DATABASE_MAP); file->crc = pgFileGetCRC(database_map_path, true, false); - file->write_size = file->read_size; + file->write_size = file->size; file->uncompressed_size = file->read_size; parray_append(backup_files_list, file); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 33b2373ef..eb12e9fae 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -86,7 +86,10 @@ extern const char *PROGRAM_EMAIL; #endif /* stdio buffer size */ -#define STDIO_BUFSIZE 65536 +#define STDIO_BUFSIZE 65536 + +/* retry attempts */ +#define PAGE_READ_ATTEMPTS 100 /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ @@ -170,7 +173,8 @@ typedef struct pgFile bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ volatile pg_atomic_flag lock;/* lock for synchronization of parallel threads */ - datapagemap_t pagemap; /* bitmap of pages updated since previous backup */ + datapagemap_t pagemap; /* bitmap of pages updated since previous backup + may take up to 16kB per file */ bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, * i.e. datafiles without _ptrack */ } pgFile; @@ -526,9 +530,9 @@ typedef struct BackupPageHeader /* Special value for compressed_size field */ #define PageIsOk 0 +#define SkipCurrentPage -1 #define PageIsTruncated -2 -#define SkipCurrentPage -3 -#define PageIsCorrupted -4 /* used by checkdb */ +#define PageIsCorrupted -3 /* used by checkdb */ /* @@ -722,6 +726,18 @@ extern void help_command(char *command); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); extern int do_validate_all(void); +extern int validate_one_page(Page page, BlockNumber absolute_blkno, + XLogRecPtr stop_lsn, XLogRecPtr *page_lsn, + uint32 checksum_version); + +/* return codes for validate_one_page */ +/* TODO: use enum */ +#define PAGE_IS_VALID (-1) +#define PAGE_IS_NOT_FOUND (-2) +#define PAGE_IS_ZEROED (-3) +#define PAGE_HEADER_IS_INVALID (-4) +#define PAGE_CHECKSUM_MISMATCH (-5) +#define PAGE_LSN_FROM_FUTURE (-6) /* in catalog.c */ extern pgBackup *read_backup(const char *instance_name, time_t timestamp); @@ -945,4 +961,19 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +/* FIO */ +extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, + int calg, int clevel, uint32 checksum_version, + datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); + +/* return codes for fio_send_pages */ +#define WRITE_FAILED (-1) +#define REMOTE_ERROR (-2) +#define PAGE_CORRUPTION (-3) +#define SEND_OK (-4) + +extern void get_header_errormsg(Page page, char **errormsg); +extern void get_checksum_errormsg(Page page, char **errormsg, + BlockNumber absolute_blkno); + #endif /* PG_PROBACKUP_H */ diff --git a/src/utils/file.c b/src/utils/file.c index 0031f2c26..68a81322e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -14,7 +14,6 @@ #define PRINTF_BUF_SIZE 1024 #define FILE_PERMISSIONS 0600 -#define PAGE_READ_ATTEMPTS 100 static __thread unsigned long fio_fdset = 0; static __thread void* fio_stdin_buffer; @@ -27,11 +26,12 @@ fio_location MyLocation; typedef struct { BlockNumber nblocks; - BlockNumber segBlockNum; + BlockNumber segmentno; XLogRecPtr horizonLsn; uint32 checksumVersion; int calg; int clevel; + int bitmapsize; } fio_send_request; @@ -114,6 +114,7 @@ fio_safestat(const char *path, struct stat *buf) #define stat(x, y) fio_safestat(x, y) +/* TODO: use real pread on Linux */ static ssize_t pread(int fd, void* buf, size_t size, off_t off) { off_t rc = lseek(fd, off, SEEK_SET); @@ -551,13 +552,14 @@ int fio_pread(FILE* f, void* buf, off_t offs) if (hdr.size != 0) IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + /* TODO: error handling */ + return hdr.arg; } else { - /* For local file, opened by fopen, we should use stdio operations */ - int rc; - rc = fseek(f, offs, SEEK_SET); + /* For local file, opened by fopen, we should use stdio functions */ + int rc = fseek(f, offs, SEEK_SET); if (rc < 0) return rc; @@ -1307,8 +1309,24 @@ static void fio_send_file(int out, char const* path) } } -int fio_send_pages(FILE* in, FILE* out, pgFile *file, - XLogRecPtr horizonLsn, BlockNumber* nBlocksSkipped, int calg, int clevel) +/* + * Return number of actually(!) readed blocks, attempts or + * half-readed block are not counted. + * Return values in case of error: + * REMOTE_ERROR + * PAGE_CORRUPTION + * WRITE_FAILED + * + * If none of the above, this function return number of blocks + * readed by remote agent. + * + * In case of DELTA mode horizonLsn must be a valid lsn, + * otherwise it should be set to InvalidXLogRecPtr. + */ +int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, + int calg, int clevel, uint32 checksum_version, + datapagemap_t *pagemap, BlockNumber* err_blknum, + char **errormsg) { struct { fio_header hdr; @@ -1319,144 +1337,240 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, Assert(fio_is_remote_file(in)); - req.hdr.cop = FIO_SEND_PAGES; - req.hdr.size = sizeof(fio_send_request); + /* send message with header + + 8bytes 20bytes var + ------------------------------------------------------ + | fio_header | fio_send_request | BITMAP(if any) | + ------------------------------------------------------ + */ + req.hdr.handle = fio_fileno(in) & ~FIO_PIPE_MARKER; + if (pagemap) + { + req.hdr.cop = FIO_SEND_PAGES_PAGEMAP; + req.hdr.size = sizeof(fio_send_request) + pagemap->bitmapsize; + req.arg.bitmapsize = pagemap->bitmapsize; + + /* TODO: add optimization for the case of pagemap + * containing small number of blocks with big serial numbers: + * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 + */ + } + else + { + req.hdr.cop = FIO_SEND_PAGES; + req.hdr.size = sizeof(fio_send_request); + } + req.arg.nblocks = file->size/BLCKSZ; - req.arg.segBlockNum = file->segno * RELSEG_SIZE; + req.arg.segmentno = file->segno * RELSEG_SIZE; req.arg.horizonLsn = horizonLsn; - req.arg.checksumVersion = current.checksum_version; + req.arg.checksumVersion = checksum_version; req.arg.calg = calg; req.arg.clevel = clevel; - file->compress_alg = calg; + file->compress_alg = calg; /* TODO: wtf? why here? */ + +//<----- +// datapagemap_iterator_t *iter; +// BlockNumber blkno; +// iter = datapagemap_iterate(pagemap); +// while (datapagemap_next(iter, &blkno)) +// elog(INFO, "block %u", blkno); +// pg_free(iter); +//<----- IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + if (pagemap) + /* now send pagemap itself */ + IO_CHECK(fio_write_all(fio_stdout, pagemap->bitmap, pagemap->bitmapsize), pagemap->bitmapsize); + while (true) { fio_header hdr; char buf[BLCKSZ + sizeof(BackupPageHeader)]; IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_PAGE); - if ((int)hdr.arg < 0) /* read error */ + if (interrupted) + elog(ERROR, "Interrupted during page reading"); + + if (hdr.cop == FIO_ERROR) { - return (int)hdr.arg; + errno = hdr.arg; + *err_blknum = hdr.size; + return REMOTE_ERROR; } + else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) + { + *err_blknum = hdr.arg; - blknum = hdr.arg; - if (hdr.size == 0) /* end of segment */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + strncpy(*errormsg, buf, hdr.size); + } + return PAGE_CORRUPTION; + } + else if (hdr.cop == FIO_SEND_FILE_EOF) + { + /* n_blocks_read reported by EOF */ + n_blocks_read = hdr.size; break; + } + else if (hdr.cop == FIO_PAGE) + { + blknum = hdr.arg; - Assert(hdr.size <= sizeof(buf)); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + Assert(hdr.size <= sizeof(buf)); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - if (fio_fwrite(out, buf, hdr.size) != hdr.size) - { - int errno_tmp = errno; - fio_fclose(out); - elog(ERROR, "File: %s, cannot write backup at block %u: %s", - file->path, blknum, strerror(errno_tmp)); - } - file->write_size += hdr.size; - n_blocks_read++; - - if (((BackupPageHeader*)buf)->compressed_size == PageIsTruncated) - { - blknum += 1; - break; + if (fio_fwrite(out, buf, hdr.size) != hdr.size) + { + fio_fclose(out); + *err_blknum = blknum; + return WRITE_FAILED; + } + file->write_size += hdr.size; + file->uncompressed_size += BLCKSZ; } + else + elog(ERROR, "Remote agent returned message of unknown type"); } - *nBlocksSkipped = blknum - n_blocks_read; - return blknum; + + return n_blocks_read; } -static void fio_send_pages_impl(int fd, int out, fio_send_request* req) +static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) { - BlockNumber blknum; + BlockNumber blknum = 0; + BlockNumber n_blocks_read = 0; + XLogRecPtr page_lsn = 0; char read_buffer[BLCKSZ+1]; fio_header hdr; + fio_send_request *req = (fio_send_request*) buf; + + /* parse buffer */ + datapagemap_t *map = NULL; + datapagemap_iterator_t *iter = NULL; + + if (with_pagemap) + { + map = pgut_malloc(sizeof(datapagemap_t)); + map->bitmapsize = req->bitmapsize; + map->bitmap = (char*) buf + sizeof(fio_send_request); + + /* get first block */ + iter = datapagemap_iterate(map); + datapagemap_next(iter, &blknum); + } hdr.cop = FIO_PAGE; read_buffer[BLCKSZ] = 1; /* barrier */ - for (blknum = 0; blknum < req->nblocks; blknum++) + while (blknum < req->nblocks) { + int rc = 0; int retry_attempts = PAGE_READ_ATTEMPTS; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - while (true) - { - ssize_t rc = pread(fd, read_buffer, BLCKSZ, blknum*BLCKSZ); + /* TODO: handle signals on the agent */ + if (interrupted) + elog(ERROR, "Interrupted during remote page reading"); - if (rc <= 0) + /* read page, check header and validate checksumms */ + /* TODO: libpq connection on the agent, so we can do ptrack + * magic right here. + */ + for (;;) + { + ssize_t read_len = pread(fd, read_buffer, BLCKSZ, blknum*BLCKSZ); + page_lsn = InvalidXLogRecPtr; + + /* report eof */ + if (read_len == 0) + goto eof; + /* report error */ + else if (read_len < 0) { - if (rc < 0) - { - hdr.arg = -errno; - hdr.size = 0; - Assert((int)hdr.arg < 0); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } - else - { - BackupPageHeader bph; - bph.block = blknum; - bph.compressed_size = PageIsTruncated; - hdr.arg = blknum; - hdr.size = sizeof(bph); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, &bph, sizeof(bph)), sizeof(bph)); - } - return; + /* TODO: better to report exact error message, not errno */ + hdr.cop = FIO_ERROR; + hdr.arg = errno; + hdr.size = blknum; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + goto cleanup; } - else if (rc == BLCKSZ) + else if (read_len == BLCKSZ) { - if (!parse_page((Page)read_buffer, &page_lsn)) - { - int i; - for (i = 0; read_buffer[i] == 0; i++); + rc = validate_one_page(read_buffer, req->segmentno + blknum, + InvalidXLogRecPtr, &page_lsn, req->checksumVersion); - /* Page is zeroed. No need to check header and checksum. */ - if (i == BLCKSZ) - break; - } - else if (!req->checksumVersion - || pg_checksum_page(read_buffer, req->segBlockNum + blknum) == ((PageHeader)read_buffer)->pd_checksum) - { + /* TODO: optimize copy of zeroed page */ + if (rc == PAGE_IS_ZEROED) + break; + else if (rc == PAGE_IS_VALID) break; - } } +// else /* readed less than BLKSZ bytes, retry */ + /* File is either has insane header or invalid checksum, + * retry. If retry attempts are exhausted, report corruption. + */ if (--retry_attempts == 0) { - hdr.size = 0; - hdr.arg = PAGE_CHECKSUM_MISMATCH; + char *errormsg = NULL; + hdr.cop = FIO_SEND_FILE_CORRUPTION; + hdr.arg = blknum; + + /* Construct the error message */ + if (rc == PAGE_HEADER_IS_INVALID) + get_header_errormsg(read_buffer, &errormsg); + else if (rc == PAGE_CHECKSUM_MISMATCH) + get_checksum_errormsg(read_buffer, &errormsg, + req->segmentno + blknum); + + /* if error message is not empty, set payload size to its length */ + hdr.size = errormsg ? strlen(errormsg) + 1 : 0; + + /* send header */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - return; + + /* send error message if any */ + if (errormsg) + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + + pg_free(errormsg); + goto cleanup; } } + + n_blocks_read++; + /* - * horizonLsn is not 0 for delta backup. + * horizonLsn is not 0 only in case of delta backup. * As far as unsigned number are always greater or equal than zero, * there is no sense to add more checks. */ - if (page_lsn >= req->horizonLsn || page_lsn == InvalidXLogRecPtr) + if ((req->horizonLsn == InvalidXLogRecPtr) || + (page_lsn == InvalidXLogRecPtr) || /* zeroed page */ + (req->horizonLsn > 0 && page_lsn >= req->horizonLsn)) /* delta */ { char write_buffer[BLCKSZ*2]; BackupPageHeader* bph = (BackupPageHeader*)write_buffer; - const char *errormsg = NULL; + /* compress page */ hdr.arg = bph->block = blknum; hdr.size = sizeof(BackupPageHeader); - bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), sizeof(write_buffer) - sizeof(BackupPageHeader), + bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), + sizeof(write_buffer) - sizeof(BackupPageHeader), read_buffer, BLCKSZ, req->calg, req->clevel, - &errormsg); + NULL); + if (bph->compressed_size <= 0 || bph->compressed_size >= BLCKSZ) { /* Do not compress page */ @@ -1468,10 +1582,29 @@ static void fio_send_pages_impl(int fd, int out, fio_send_request* req) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); } + + /* next block */ + if (with_pagemap) + { + /* exit if pagemap is exhausted */ + if (!datapagemap_next(iter, &blknum)) + break; + } + else + blknum++; } - hdr.size = 0; - hdr.arg = blknum; + +eof: + /* We are done, send eof */ + hdr.cop = FIO_SEND_FILE_EOF; + hdr.arg = 0; + hdr.size = n_blocks_read; /* TODO: report number of backed up blocks */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + +cleanup: + pg_free(map); + pg_free(iter); + return; } /* Execute commands at remote host */ @@ -1615,10 +1748,14 @@ void fio_communicate(int in, int out) break; case FIO_SEND_PAGES: Assert(hdr.size == sizeof(fio_send_request)); - fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf); + fio_send_pages_impl(fd[hdr.handle], out, buf, false); + break; + case FIO_SEND_PAGES_PAGEMAP: + // buf contain fio_send_request header and bitmap. + fio_send_pages_impl(fd[hdr.handle], out, buf, true); break; case FIO_SYNC: - /* open file and fsync it */ + /* open file and fsync it */ tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); if (tmp_fd < 0) hdr.arg = errno; diff --git a/src/utils/file.h b/src/utils/file.h index 848a018e6..a831a4c1e 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -33,10 +33,16 @@ typedef enum FIO_OPENDIR, FIO_READDIR, FIO_CLOSEDIR, - FIO_SEND_PAGES, FIO_PAGE, FIO_WRITE_COMPRESSED, FIO_GET_CRC32, + /* used in fio_send_pages */ + FIO_SEND_PAGES, + FIO_SEND_PAGES_PAGEMAP, + FIO_ERROR, + FIO_SEND_FILE_EOF, + FIO_SEND_FILE_CORRUPTION, + /* messages for closing connection */ FIO_DISCONNECT, FIO_DISCONNECTED, } fio_operations; @@ -51,7 +57,6 @@ typedef enum #define FIO_FDMAX 64 #define FIO_PIPE_MARKER 0x40000000 -#define PAGE_CHECKSUM_MISMATCH (-256) #define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) #define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) @@ -85,10 +90,6 @@ extern int fio_fclose(FILE* f); extern int fio_ffstat(FILE* f, struct stat* st); extern void fio_error(int rc, int size, char const* file, int line); -struct pgFile; -extern int fio_send_pages(FILE* in, FILE* out, struct pgFile *file, XLogRecPtr horizonLsn, - BlockNumber* nBlocksSkipped, int calg, int clevel); - extern int fio_open(char const* name, int mode, fio_location location); extern ssize_t fio_write(int fd, void const* buf, size_t size); extern ssize_t fio_read(int fd, void* buf, size_t size); diff --git a/tests/backup.py b/tests/backup.py index e1d451120..398bb5f14 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -488,6 +488,7 @@ def test_backup_detect_corruption(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.slow_start() if self.ptrack and node.major_version > 11: @@ -499,18 +500,29 @@ def test_backup_detect_corruption(self): "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") + "from generate_series(0,10000) i") heap_path = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").rstrip() + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + node.stop() - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + heap_fullpath = os.path.join(node.data_dir, heap_path) + + with open(heap_fullpath, "rb+", 0) as f: f.seek(9000) f.write(b"bla") f.flush() @@ -518,6 +530,10 @@ def test_backup_detect_corruption(self): node.slow_start() + # self.backup_node( + # backup_dir, 'node', node, + # backup_type="full", options=["-j", "4", "--stream"]) + try: self.backup_node( backup_dir, 'node', node, @@ -525,11 +541,360 @@ def test_backup_detect_corruption(self): # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because tablespace mapping is incorrect" + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - if self.ptrack: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: page verification failed, ' + 'calculated checksum' in e.message and + 'ERROR: query failed: ERROR: ' + 'invalid page in block 1 of relation' in e.message and + 'ERROR: Data files transferring failed' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_detect_invalid_block_header(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.ptrack and node.major_version > 11: + node.safe_psql( + "postgres", + "create extension ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + + node.stop() + + heap_fullpath = os.path.join(node.data_dir, heap_path) + with open(heap_fullpath, "rb+", 0) as f: + f.seek(8193) + f.write(b"blahblahblahblah") + f.flush() + f.close + + node.slow_start() + +# self.backup_node( +# backup_dir, 'node', node, +# backup_type="full", options=["-j", "4", "--stream"]) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: page verification failed, ' + 'calculated checksum' in e.message and + 'ERROR: query failed: ERROR: ' + 'invalid page in block 1 of relation' in e.message and + 'ERROR: Data files transferring failed' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_detect_missing_permissions(self): + """make node, corrupt some page, check that backup failed""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.ptrack and node.major_version > 11: + node.safe_psql( + "postgres", + "create extension ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + + node.stop() + + heap_fullpath = os.path.join(node.data_dir, heap_path) + with open(heap_fullpath, "rb+", 0) as f: + f.seek(8193) + f.write(b"blahblahblahblah") + f.flush() + f.close + + node.slow_start() + +# self.backup_node( +# backup_dir, 'node', node, +# backup_type="full", options=["-j", "4", "--stream"]) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: self.assertTrue( 'WARNING: page verification failed, ' 'calculated checksum' in e.message and @@ -538,24 +903,6 @@ def test_backup_detect_corruption(self): 'ERROR: Data files transferring failed' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - else: - if self.remote: - self.assertTrue( - "ERROR: Failed to read file" in e.message and - "data file checksum mismatch" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'WARNING: Corruption detected in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Data file corruption', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2355,52 +2702,3 @@ def test_issue_132_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_streaming_timeout(self): - """ - Illustrate the problem of loosing exact error - message because our WAL streaming engine is "borrowed" - from pg_receivexlog - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_sender_timeout': '5s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=LOG']) - - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - - sleep(10) - gdb.continue_execution_until_error() - gdb._execute('detach') - sleep(2) - - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_content = f.read() - - self.assertIn( - 'could not receive data from WAL stream', - log_content) - - self.assertIn( - 'ERROR: Problem in receivexlog', - log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/compatibility.py b/tests/compatibility.py index 0b75d1a0e..b26c8de1d 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -765,3 +765,83 @@ def test_page_vacuum_truncate(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_page_vacuum_truncate_compression(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node',node, old_binary=True, options=['--compress']) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/false_positive.py b/tests/false_positive.py index 571509db3..fc9ee4b62 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -294,3 +294,257 @@ def test_ptrack_concurrent_get_and_clear_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_pg_10_waldir(self): + """ + test group access for PG >= 11 + """ + if self.pg_config_version < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL >= 10 for this test') + + fname = self.id().split('.')[3] + wal_dir = os.path.join( + os.path.join(self.tmp_path, module_name, fname), 'wal_dir') + shutil.rmtree(wal_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--waldir={0}'.format(wal_dir)]) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # restore backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + # compare pgdata permissions + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertTrue( + os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), + 'pg_wal should be symlink') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_time_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + target_time = node.safe_psql( + "postgres", + "select now()").rstrip() + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-time={0}'.format(target_time)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_lsn_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--log-level-console=LOG'], gdb=True) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + self.switch_wal_segment(node) + + target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-lsn={0}'.format(target_lsn)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_streaming_timeout(self): + """ + Illustrate the problem of loosing exact error + message because our WAL streaming engine is "borrowed" + from pg_receivexlog + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_sender_timeout': '5s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=LOG']) + + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + sleep(10) + gdb.continue_execution_until_error() + gdb._execute('detach') + sleep(2) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_content = f.read() + + self.assertIn( + 'could not receive data from WAL stream', + log_content) + + self.assertIn( + 'ERROR: Problem in receivexlog', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_all_empty_catalog(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup_dir is empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: This backup catalog contains no backup instances', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/page.py b/tests/page.py index e815068d2..9dbb9e3f0 100644 --- a/tests/page.py +++ b/tests/page.py @@ -391,13 +391,12 @@ def test_page_multiple_segments(self): # PGBENCH STUFF pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) pgbench.wait() - node.safe_psql("postgres", "checkpoint") # GET LOGICAL CONTENT FROM NODE result = node.safe_psql("postgres", "select * from pgbench_accounts") # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -464,18 +463,15 @@ def test_page_delete(self): "postgres", "create table t_heap tablespace somedata as select i as id," " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) + " from generate_series(0,100) i") node.safe_psql( "postgres", - "delete from t_heap" - ) + "delete from t_heap") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") # PAGE BACKUP self.backup_node( @@ -485,8 +481,7 @@ def test_page_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1122,8 +1117,7 @@ def test_page_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') - ) + base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() self.restore_node( diff --git a/tests/pgpro560.py b/tests/pgpro560.py index de8302f56..53c7914a2 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -43,7 +43,7 @@ def test_pgpro560_control_file_loss(self): "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'ERROR: could not open file' in e.message and + 'ERROR: Could not open file' in e.message and 'pg_control' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/restore.py b/tests/restore.py index 54b6d44e4..2aec7a92e 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1915,8 +1915,7 @@ def test_restore_target_latest_archive(self): node.slow_start() # Take FULL - self.backup_node( - backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) if self.get_version(node) >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') @@ -1925,27 +1924,28 @@ def test_restore_target_latest_archive(self): # restore node.cleanup() - self.restore_node( - backup_dir, 'node', node) + self.restore_node(backup_dir, 'node', node) - # with open(recovery_conf, 'r') as f: - # print(f.read()) + # hash_1 = hashlib.md5( + # open(recovery_conf, 'rb').read()).hexdigest() - hash_1 = hashlib.md5( - open(recovery_conf, 'rb').read()).hexdigest() + with open(recovery_conf, 'r') as f: + content_1 = f.read() # restore node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--recovery-target=latest']) - # with open(recovery_conf, 'r') as f: - # print(f.read()) + self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) + + # hash_2 = hashlib.md5( + # open(recovery_conf, 'rb').read()).hexdigest() + + with open(recovery_conf, 'r') as f: + content_2 = f.read() - hash_2 = hashlib.md5( - open(recovery_conf, 'rb').read()).hexdigest() + self.assertEqual(content_1, content_2) - self.assertEqual(hash_1, hash_2) + # self.assertEqual(hash_1, hash_2) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2231,55 +2231,6 @@ def test_pg_11_group_access(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_pg_10_waldir(self): - """ - test group access for PG >= 11 - """ - if self.pg_config_version < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') - - fname = self.id().split('.')[3] - wal_dir = os.path.join( - os.path.join(self.tmp_path, module_name, fname), 'wal_dir') - shutil.rmtree(wal_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=[ - '--data-checksums', - '--waldir={0}'.format(wal_dir)]) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - # restore backup - node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - # compare pgdata permissions - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.assertTrue( - os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), - 'pg_wal should be symlink') - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_concurrent_drop_table(self): """""" diff --git a/tests/validate.py b/tests/validate.py index 9a6a48f16..6b76e64c1 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -13,36 +13,6 @@ class ValidateTest(ProbackupTest, unittest.TestCase): - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_validate_all_empty_catalog(self): - """ - """ - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because backup_dir is empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: This backup catalog contains no backup instances', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_basic_validate_nullified_heap_page_backup(self): @@ -3538,130 +3508,6 @@ def test_validate_target_lsn(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_recovery_target_time_backup_victim(self): - """ - Check that for validation to recovery target - probackup chooses valid backup - https://github.com/postgrespro/pg_probackup/issues/104 - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - target_time = node.safe_psql( - "postgres", - "select now()").rstrip() - - node.safe_psql( - "postgres", - "create table t_heap1 as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100) i") - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-time={0}'.format(target_time)]) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_recovery_target_lsn_backup_victim(self): - """ - Check that for validation to recovery target - probackup chooses valid backup - https://github.com/postgrespro/pg_probackup/issues/104 - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - node.safe_psql( - "postgres", - "create table t_heap1 as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100) i") - - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--log-level-console=LOG'], gdb=True) - - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - self.switch_wal_segment(node) - - target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] - - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-lsn={0}'.format(target_lsn)]) - - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_empty_and_mangled_database_map(self): """ From 2fe9ec22ed165a59d5fa08e5470c2ce101c0a554 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Mar 2020 21:40:58 +0300 Subject: [PATCH 1177/2107] [Issue #149] Documentation update --- doc/pgprobackup.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2eba912b9..1819b53ab 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3611,6 +3611,7 @@ pg_probackup restore -B backup_dir --instance cmdline] [--restore-command=cmdline] [--primary-conninfo=primary_conninfo] +[-S | --primary-slot-name=slotname] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -3663,6 +3664,19 @@ pg_probackup restore -B backup_dir --instance + + + + + + Sets the + primary_slot_name + parameter to the specified value. + This option will be ignored unless the flag if specified. + + + + From 48e4427f6b9f318682b8459e67418eb957d574bc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Mar 2020 21:42:25 +0300 Subject: [PATCH 1178/2107] [Issue #149] add option -S | --primary-slot-name --- src/pg_probackup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 43553b5c2..0fc7067bc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -172,6 +172,7 @@ static ConfigOption cmd_options[] = { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, + { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, From c50b0b16f05fafa659ef6baf0b8a2759e55ecb30 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Mar 2020 21:42:50 +0300 Subject: [PATCH 1179/2107] [Issue #149] test coverage --- tests/restore.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index 2aec7a92e..b8f7b11da 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3378,3 +3378,48 @@ def test_restore_primary_conninfo(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_primary_slot_info(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + node.safe_psql( + "SELECT pg_create_physical_replication_slot('master_slot')") + + self.restore_node( + backup_dir, 'node', replica, + options=['-R', '--primary-slot-name=master_slot']) + + self.set_auto_conf(replica, {'port': replica.port}) + self.set_auto_conf(replica, {'hot_standby': 'on'}) + + if self.get_version(node) >= self.version_to_num('12.0'): + standby_signal = os.path.join(replica.data_dir, 'standby.signal') + self.assertTrue( + os.path.isfile(standby_signal), + "File '{0}' do not exists".format(standby_signal)) + + replica.slow_start(replica=True) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 3a2a56a30047baceb2443c69a1c4bdd94c3c45f7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Mar 2020 11:59:37 +0300 Subject: [PATCH 1180/2107] [Issue #149] documentation update --- doc/pgprobackup.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1819b53ab..0c04b8e2c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3609,7 +3609,6 @@ pg_probackup restore -B backup_dir --instance cmdline] -[--restore-command=cmdline] [--primary-conninfo=primary_conninfo] [-S | --primary-slot-name=slotname] [recovery_target_options] [logging_options] [remote_options] From 925b7d588e775fb1a139e0fc8b054e6cc425d954 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Mar 2020 12:00:14 +0300 Subject: [PATCH 1181/2107] [Issue #149] help update --- src/help.c | 53 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/help.c b/src/help.c index 14ede6dd6..2bc50d512 100644 --- a/src/help.c +++ b/src/help.c @@ -153,6 +153,8 @@ help_pg_probackup(void) printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-as-replica] [--force]\n")); + printf(_(" [--primary-conninfo=primary_conninfo]\n")); + printf(_(" [-S | --primary-slot-name=slotname]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); @@ -382,20 +384,20 @@ help_restore(void) { printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); + printf(_(" [--progress] [--force] [--no-sync]\n")); + printf(_(" [--no-validate] [--skip-block-validation]\n")); + printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); + printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica] [--force]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); - printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--skip-external-dirs]\n")); printf(_(" [--restore-command=cmdline]\n")); - printf(_(" [--no-sync]\n")); - printf(_(" [--db-include dbname | --db-exclude dbname]\n")); + printf(_(" [-R | --restore-as-replica]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -410,6 +412,22 @@ help_restore(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); + printf(_(" --force ignore invalid status of the restored backup\n")); + printf(_(" --no-sync do not sync restored files to disk\n")); + printf(_(" --no-validate disable backup validation during restore\n")); + printf(_(" --skip-block-validation set to validate only file-level checksum\n")); + + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); + printf(_(" --skip-external-dirs do not restore all external directories\n")); + + printf(_("\n Partial restore options:\n")); + printf(_(" --db-include dbname restore only specified databases\n")); + printf(_(" --db-exclude dbname do not restore specified databases\n")); + + printf(_("\n Recovery options:\n")); printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); printf(_(" --recovery-target-xid=xid transaction ID up to which recovery will proceed\n")); printf(_(" --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); @@ -424,24 +442,15 @@ help_restore(void) printf(_(" --recovery-target-action=pause|promote|shutdown\n")); printf(_(" action the server should take once the recovery target is reached\n")); printf(_(" (default: pause)\n")); + printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); + printf(_("\n Standby options:\n")); printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); printf(_(" to ease setting up a standby server\n")); - printf(_(" --force ignore invalid status of the restored backup\n")); - printf(_(" --no-validate disable backup validation during restore\n")); - printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); - printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); - printf(_(" --skip-external-dirs do not restore all external directories\n")); - printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); - printf(_(" --no-sync do not sync restored files to disk\n")); - - printf(_("\n Partial restore options:\n")); - printf(_(" --db-include dbname restore only specified databases\n")); - printf(_(" --db-exclude dbname do not restore specified databases\n")); + printf(_(" --primary-conninfo=primary_conninfo\n")); + printf(_(" connection string to be used for establishing connection\n")); + printf(_(" with the primary server\n")); + printf(_(" -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); From 4a247e8311d0a8aa020e5e45fe70e3ec921420c4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Mar 2020 19:02:15 +0300 Subject: [PATCH 1182/2107] [Issue #182] do not set remote parameters into config when running add-instance via ssh --- src/init.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/init.c b/src/init.c index bfce95737..52cadc25f 100644 --- a/src/init.c +++ b/src/init.c @@ -103,6 +103,23 @@ do_add_instance(InstanceConfig *instance) SOURCE_FILE); config_set_opt(instance_options, &instance->xlog_seg_size, SOURCE_FILE); + + /* Kludge: do not save remote options into config */ + config_set_opt(instance_options, &instance_config.remote.host, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.proto, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.port, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.path, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.user, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.ssh_options, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.remote.ssh_config, + SOURCE_DEFAULT); + /* pgdata was set through command line */ do_set_config(true); From e124234b1d7ed443758dadf59401ba2b778a7611 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 15 Mar 2020 19:02:35 +0300 Subject: [PATCH 1183/2107] [Issue #182] test coverage --- tests/remote.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tests/remote.py b/tests/remote.py index f2d503a0e..4d46447f0 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -24,22 +24,27 @@ def test_remote_sanity(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', - node, options=['--remote-proto=ssh', '--stream'], no_remote=True) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remote-host option is missing." - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "Insert correct error", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + output = self.backup_node( + backup_dir, 'node', node, + options=['--stream'], no_remote=True, return_id=False) + self.assertIn('remote: false', output) + + # try: + # self.backup_node( + # backup_dir, 'node', + # node, options=['--remote-proto=ssh', '--stream'], no_remote=True) + # # we should die here because exception is what we expect to happen + # self.assertEqual( + # 1, 0, + # "Expecting Error because remote-host option is missing." + # "\n Output: {0} \n CMD: {1}".format( + # repr(self.output), self.cmd)) + # except ProbackupException as e: + # self.assertIn( + # "Insert correct error", + # e.message, + # "\n Unexpected Error Message: {0}\n CMD: {1}".format( + # repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) From a365520a1ca66ad89b271193a4be8330ec3de5f7 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Mon, 16 Mar 2020 19:44:35 +0300 Subject: [PATCH 1184/2107] Removed --error-state option and added --status option for delete command --- src/delete.c | 13 ++++++++++--- src/pg_probackup.c | 17 +++++++++-------- src/pg_probackup.h | 3 ++- src/util.c | 40 +++++++++++++++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/delete.c b/src/delete.c index ed9d56a3b..696301808 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1026,13 +1026,20 @@ do_delete_instance(void) /* Delete all error backup files of given instance. */ int -do_delete_error(void) +do_delete_status(char* status) { parray *backup_list; parray *xlog_files_list; int i; int rc; char instance_config_path[MAXPGPATH]; + + BackupStatus status_for_delete; + + status_for_delete = str2status(status); + + if (status_for_delete == BACKUP_STATUS_INVALID) + elog(ERROR, "Unknown '%s' value for --status option", status); /* Delete all error backups. */ @@ -1041,7 +1048,7 @@ do_delete_error(void) for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (backup->status == BACKUP_STATUS_ERROR){ + if (backup->status == status_for_delete){ /* elog(INFO, "Delete error backup '%s' ", base36enc(backup->backup_id)); */ catalog_lock_backup_list(backup_list, i, i); delete_backup_files(backup); @@ -1053,6 +1060,6 @@ do_delete_error(void) parray_free(backup_list); - elog(INFO, "Error backups from instance '%s' successfully deleted", instance_name); + elog(INFO, "Backups with status '%s' from instance '%s' successfully deleted", status, instance_name); return 0; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ed06ac49e..8c75d66d3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -117,7 +117,7 @@ bool delete_expired = false; bool merge_expired = false; bool force = false; bool dry_run = false; -bool delete_error = false; +static char *delete_status = NULL; /* compression options */ bool compress_shortcut = false; @@ -201,7 +201,8 @@ static ConfigOption cmd_options[] = /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, - { 'b', 172, "error-state", &delete_error, SOURCE_CMD_STRICT }, + { 's', 172, "status", &delete_status, SOURCE_CMD_STRICT }, + /* TODO not implemented yet */ { 'b', 147, "force", &force, SOURCE_CMD_STRICT }, /* compression options */ @@ -787,14 +788,14 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); if (merge_expired && backup_id_string) elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together"); - if (delete_error && backup_id_string) - elog(ERROR, "You cannot specify --error-state and (-i, --backup-id) options together"); - if (!delete_expired && !merge_expired && !delete_wal && !delete_error && !backup_id_string) + if (delete_status && backup_id_string) + elog(ERROR, "You cannot specify --status and (-i, --backup-id) options together"); + if (!delete_expired && !merge_expired && !delete_wal && delete_status==NULL && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: " - "--delete-expired |--delete-wal |--merge-expired |--error-state |(-i, --backup-id)"); + "--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)"); if (!backup_id_string) - if(delete_error) - do_delete_error(); + if (delete_status) + do_delete_status(delete_status); else return do_retention(); else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 58b538d43..2497fbd7d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -706,7 +706,7 @@ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); extern int do_retention(void); extern int do_delete_instance(void); -extern int do_delete_error(void); +extern int do_delete_status(char *status); /* in fetch.c */ extern char *slurpFile(const char *datadir, @@ -915,6 +915,7 @@ extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_loc extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); +extern BackupStatus str2status(const char *status); extern const char *base36enc(long unsigned int value); extern char *base36enc_dup(long unsigned int value); extern long unsigned int base36dec(const char *text); diff --git a/src/util.c b/src/util.c index 871ad1051..a6656980d 100644 --- a/src/util.c +++ b/src/util.c @@ -458,11 +458,37 @@ parse_program_version(const char *program_version) return result; } +static const char *statusName[] = +{ + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "MERGED", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" +}; const char * status2str(BackupStatus status) { - static const char *statusName[] = + if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) + return "UNKNOWN"; + + return statusName[status]; +} + + + +BackupStatus +str2status(const char *status) +{ + +/* static const char *statusName[] = { "UNKNOWN", "OK", @@ -475,9 +501,13 @@ status2str(BackupStatus status) "DONE", "ORPHAN", "CORRUPT" - }; - if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) - return "UNKNOWN"; + };*/ - return statusName[status]; + + for (int i = 0; i <= BACKUP_STATUS_CORRUPT; i++) + { + if (pg_strcasecmp(status, statusName[i]) == 0) return i; + } + + return 0; } From ffdf86d24832ae4c01df3cc3be3f79619dafaf48 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 17 Mar 2020 13:09:34 +0300 Subject: [PATCH 1185/2107] Readme: update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 155ae74e3..9ef122a24 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ The utility is compatible with: As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. +* Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to to do periodical full backups. * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery * Verification: on-demand verification of PostgreSQL instance with the `checkdb` command. * Retention: managing WAL archive and backups in accordance with retention policy. You can configure retention policy based on recovery time or the number of backups to keep, as well as specify `time to live` (TTL) for a particular backup. Expired backups can be merged or deleted. From 98c22cae46e7357b91e67e5fc0522e67be512914 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 17 Mar 2020 21:48:43 +0300 Subject: [PATCH 1186/2107] Updated documentation and test for --status option of delete command --- doc/pgprobackup.xml | 6 +++--- src/util.c | 16 ---------------- tests/delete.py | 2 +- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 5e9de6eca..a7171919f 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3229,10 +3229,10 @@ pg_probackup delete -B backup_dir --instance - To delete backups with the ERROR status, use the flag: + To delete all backups with some status, use the : -pg_probackup delete -B backup_dir --instance instance_name --error-state +pg_probackup delete -B backup_dir --instance instance_name --status=ERROR @@ -3869,7 +3869,7 @@ pg_probackup merge -B backup_dir --instance backup_dir --instance instance_name [--help] [-j num_threads] [--progress] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] -[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --error-state} +[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status= } [--dry-run] [logging_options] diff --git a/src/util.c b/src/util.c index a6656980d..4c4ebc53c 100644 --- a/src/util.c +++ b/src/util.c @@ -488,22 +488,6 @@ BackupStatus str2status(const char *status) { -/* static const char *statusName[] = - { - "UNKNOWN", - "OK", - "ERROR", - "RUNNING", - "MERGING", - "MERGED", - "DELETING", - "DELETED", - "DONE", - "ORPHAN", - "CORRUPT" - };*/ - - for (int i = 0; i <= BACKUP_STATUS_CORRUPT; i++) { if (pg_strcasecmp(status, statusName[i]) == 0) return i; diff --git a/tests/delete.py b/tests/delete.py index b4f8db6e6..794502800 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -835,7 +835,7 @@ def test_delete_error_backups(self): self.assertEqual(len(show_backups), 4) # delete error backups - self.delete_pb(backup_dir, 'node', options=['--error-state']) + self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) print(self.show_pb(backup_dir, as_text=True, as_json=False)) From 803c607b467a89a970d2a8d87ad2b7735d557fa3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Mar 2020 21:40:40 +0300 Subject: [PATCH 1187/2107] [Issue #187] Do not wait for WAL segment, previous to WAL segment with START LSN, when running backup in archive mode --- src/backup.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/backup.c b/src/backup.c index 183f1b2a3..a9151fdf1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1093,7 +1093,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, PQclear(res); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE && + if ((!stream_wal || current.backup_mode == BACKUP_MODE_DIFF_PAGE) && !backup->from_replica && !(nodeInfo->server_version < 90600 && !nodeInfo->is_superuser)) @@ -1105,17 +1105,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, */ pg_switch_wal(conn); - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) - /* In PAGE mode wait for current segment... */ + /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal) + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false); - /* - * Do not wait start_lsn for stream backup. - * Because WAL streaming will start after pg_start_backup() in stream - * mode. - */ - else if (!stream_wal) - /* ...for others wait for previous segment */ - wait_wal_lsn(backup->start_lsn, true, backup->tli, true, true, ERROR, false); } /* From 0cef487f438bb30ed663db0bd4c534f8200e7d8a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 21 Mar 2020 01:37:05 +0300 Subject: [PATCH 1188/2107] [Issue #187] test fix --- tests/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index 16f9aa8be..7d62198df 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -263,7 +263,7 @@ def test_pgpro434_3(self): log_content) else: self.assertIn( - "ERROR: WAL segment 000000010000000000000002 could not be archived in 60 seconds", + "ERROR: WAL segment 000000010000000000000003 could not be archived in 60 seconds", log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') From 2cf0c991f38bffcfc78bbbacbce7585759c0987c Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Mon, 23 Mar 2020 16:41:29 +0300 Subject: [PATCH 1189/2107] Added --note option for set-backup command. Fixed --note option for backup command. Added char *note field to pgSetBackupParams structure --- src/backup.c | 4 ++-- src/catalog.c | 8 ++++++-- src/pg_probackup.c | 5 +++-- src/pg_probackup.h | 7 ++++++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index a2fa44795..9aaeb617c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -750,7 +750,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) */ int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync, char *note) + pgSetBackupParams *set_backup_params, bool no_sync) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -767,7 +767,7 @@ do_backup(time_t start_time, bool no_validate, current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; - current.note = note; + current.note = set_backup_params ? set_backup_params->note : NULL; StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); diff --git a/src/catalog.c b/src/catalog.c index 87e72a698..aca686b39 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1412,9 +1412,11 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) /* Pin comes from expire-time */ else if (set_backup_params->expire_time > 0) target_backup->expire_time = set_backup_params->expire_time; - else + else if (!set_backup_params->note) return false; + if (set_backup_params->note) target_backup->note = set_backup_params->note; + /* Update backup.control */ write_backup(target_backup); @@ -1426,9 +1428,11 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), expire_timestamp); } - else + else if (set_backup_params->ttl == 0) elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time)); + if (set_backup_params->note) + elog(INFO, "Saved note for backup %s", base36enc(target_backup->start_time)); return true; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index dbeda715e..dc348fb29 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -726,11 +726,12 @@ main(int argc, char *argv[]) expire_time_string); } - if (expire_time > 0 || ttl >= 0) + if (expire_time > 0 || ttl >= 0 || backup_note) { set_backup_params = pgut_new(pgSetBackupParams); set_backup_params->ttl = ttl; set_backup_params->expire_time = expire_time; + set_backup_params->note = backup_note; } } @@ -770,7 +771,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, no_validate, set_backup_params, no_sync, backup_note); + return do_backup(start_time, no_validate, set_backup_params, no_sync); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1e33eb2da..c9820a2a8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -438,6 +438,7 @@ typedef struct pgSetBackupParams time_t expire_time; /* Point in time before which backup * must be pinned. */ + char *note; } pgSetBackupParams; typedef struct @@ -644,7 +645,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync, char *note); + pgSetBackupParams *set_backup_params, bool no_sync); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -947,4 +948,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +#ifdef WIN32 +#define setbuffer(stream, buf, size) setvbuf(stream, buf, buf ? _IOFBF : _IONBF, size); +#endif + #endif /* PG_PROBACKUP_H */ From 943548c8573cbec65b500db84f9c5b7e5fe3a0ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 23 Mar 2020 23:59:44 +0300 Subject: [PATCH 1190/2107] use setvbuf instead of setbuffer --- src/data.c | 8 ++++---- src/merge.c | 2 +- src/restore.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/data.c b/src/data.c index f9c83e5d9..2e1d034ac 100644 --- a/src/data.c +++ b/src/data.c @@ -623,7 +623,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, } if (!fio_is_remote_file(in)) - setbuffer(in, in_buffer, STDIO_BUFSIZE); + setvbuf(in, in_buffer, _IOFBF, STDIO_BUFSIZE); /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); @@ -631,7 +631,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, elog(ERROR, "Cannot open backup file \"%s\": %s", to_fullpath, strerror(errno)); - setbuffer(out, out_buffer, STDIO_BUFSIZE); + setvbuf(out, out_buffer, _IOFBF, STDIO_BUFSIZE); /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) @@ -874,7 +874,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuffer(in, buffer, STDIO_BUFSIZE); + setvbuf(in, buffer, _IOFBF, STDIO_BUFSIZE); /* * Restore the file. @@ -1164,7 +1164,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setbuffer(in, buffer, STDIO_BUFSIZE); + setvbuf(in, buffer, _IOFBF, STDIO_BUFSIZE); /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); diff --git a/src/merge.c b/src/merge.c index 295d4230c..06da8f4bd 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1132,7 +1132,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, if (out == NULL) elog(ERROR, "Cannot open merge target file \"%s\": %s", to_fullpath_tmp1, strerror(errno)); - setbuffer(out, buffer, STDIO_BUFSIZE); + setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); diff --git a/src/restore.c b/src/restore.c index 4bd62e09a..177f9d5f3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -857,7 +857,7 @@ restore_files(void *arg) goto done; if (!fio_is_remote_file(out)) - setbuffer(out, buffer, STDIO_BUFSIZE); + setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) From 544f49afb9711d27e472909c799755f53c4c54a1 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Tue, 24 Mar 2020 20:08:04 +0300 Subject: [PATCH 1191/2107] [Issue #146] Documentation update --- doc/pgprobackup.xml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2eba912b9..cd32696f3 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3347,7 +3347,7 @@ pg_probackup set-config -B backup_dir --instance set-backup pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id -{--ttl=ttl | --expire-time=time} [--help] +{--ttl=ttl | --expire-time=time} [--note=note] [--help] Sets the provided backup-specific settings into the @@ -3415,7 +3415,7 @@ pg_probackup backup -B backup_dir -b bac [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] [--archive-timeout=timeout] [--external-dirs=external_directory_path] -[--no-sync] +[--no-sync] [--note=note] [connection_options] [compression_options] [remote_options] [retention_options] [pinning_options] [logging_options] @@ -3580,6 +3580,15 @@ pg_probackup backup -B backup_dir -b bac + + + + + Sets the text note for backup copy. + + + + From 9864f5816edeefeaa171eb02ed0840331c58a7b9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 24 Mar 2020 21:37:35 +0300 Subject: [PATCH 1192/2107] [Issue #120] Review --- doc/pgprobackup.xml | 2 +- src/delete.c | 61 +++++++++++++++++++++++++++------------------ src/pg_probackup.c | 10 +++++--- src/pg_probackup.h | 4 +-- src/util.c | 36 +++++++++++++------------- tests/delete.py | 5 ++-- 6 files changed, 66 insertions(+), 52 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index ee05f9748..105fda4d9 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3239,7 +3239,7 @@ pg_probackup delete -B backup_dir --instance - To delete all backups with some status, use the : + To delete all backups with specific status, use the : pg_probackup delete -B backup_dir --instance instance_name --status=ERROR diff --git a/src/delete.c b/src/delete.c index 696301808..f92b9e6d1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -123,7 +123,7 @@ do_delete(time_t backup_id) * which FULL backup should be keeped for redundancy obligation(only valid do), * but if invalid backup is not guarded by retention - it is removed */ -int do_retention(void) +void do_retention(void) { parray *backup_list = NULL; parray *to_keep_list = parray_new(); @@ -154,7 +154,7 @@ int do_retention(void) /* Retention is disabled but we still can cleanup wal */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) - return 0; + return; } else /* At least one retention policy is active */ @@ -196,9 +196,6 @@ int do_retention(void) parray_free(backup_list); parray_free(to_keep_list); parray_free(to_purge_list); - - return 0; - } /* Evaluate every backup by retention policies and populate purge and keep lists. @@ -1024,42 +1021,58 @@ do_delete_instance(void) return 0; } -/* Delete all error backup files of given instance. */ -int -do_delete_status(char* status) +/* Delete all backups of given status in instance */ +void +do_delete_status(InstanceConfig *instance_config, const char *status) { parray *backup_list; - parray *xlog_files_list; int i; - int rc; - char instance_config_path[MAXPGPATH]; - - BackupStatus status_for_delete; + const char *pretty_status; + int n_deleted = 0; - status_for_delete = str2status(status); + BackupStatus status_for_delete = str2status(status); if (status_for_delete == BACKUP_STATUS_INVALID) - elog(ERROR, "Unknown '%s' value for --status option", status); + elog(ERROR, "Unknown value for '--status' option: '%s'", status); + + /* + * User may have provided status string in lower case, but + * we should print backup statuses consistently with show command, + * so convert it. + */ + pretty_status = status2str(status_for_delete); + backup_list = catalog_get_backup_list(instance_config->name, INVALID_BACKUP_ID); - /* Delete all error backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + if (parray_num(backup_list) == 0) + { + elog(WARNING, "Instance '%s' has no backups", instance_config->name); + return; + } + + elog(INFO, "Deleting all backups with status '%s'", pretty_status); + /* Delete all backups with specified status */ for (i = 0; i < parray_num(backup_list); i++) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (backup->status == status_for_delete){ - /* elog(INFO, "Delete error backup '%s' ", base36enc(backup->backup_id)); */ - catalog_lock_backup_list(backup_list, i, i); + + if (backup->status == status_for_delete) + { + lock_backup(backup); delete_backup_files(backup); + n_deleted++; } } + if (n_deleted > 0) + elog(INFO, "Successfully deleted all backups with status '%s' from instance '%s'", + pretty_status, instance_config->name); + else + elog(WARNING, "Instance '%s' has no backups with status '%s'", + instance_config->name, pretty_status); + /* Cleanup */ parray_walk(backup_list, pgBackupFree); parray_free(backup_list); - - - elog(INFO, "Backups with status '%s' from instance '%s' successfully deleted", status, instance_name); - return 0; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f86819a37..814c8ab95 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -806,14 +806,16 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together"); if (delete_status && backup_id_string) elog(ERROR, "You cannot specify --status and (-i, --backup-id) options together"); - if (!delete_expired && !merge_expired && !delete_wal && delete_status==NULL && !backup_id_string) + if (!delete_expired && !merge_expired && !delete_wal && delete_status == NULL && !backup_id_string) elog(ERROR, "You must specify at least one of the delete options: " "--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)"); if (!backup_id_string) - if (delete_status) - do_delete_status(delete_status); + { + if (delete_status) + do_delete_status(&instance_config, delete_status); else - return do_retention(); + do_retention(); + } else do_delete(current.backup_id); break; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3d70d81e3..cdc1bad1a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -709,9 +709,9 @@ extern int do_show(const char *instance_name, time_t requested_backup_id, bool s /* in delete.c */ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); -extern int do_retention(void); +extern void do_retention(void); extern int do_delete_instance(void); -extern int do_delete_status(char *status); +extern void do_delete_status(InstanceConfig *instance_config, const char *status); /* in fetch.c */ extern char *slurpFile(const char *datadir, diff --git a/src/util.c b/src/util.c index bf372a3b3..776c2ec72 100644 --- a/src/util.c +++ b/src/util.c @@ -18,6 +18,21 @@ #include +static const char *statusName[] = +{ + "UNKNOWN", + "OK", + "ERROR", + "RUNNING", + "MERGING", + "MERGED", + "DELETING", + "DELETED", + "DONE", + "ORPHAN", + "CORRUPT" +}; + const char * base36enc(long unsigned int value) { @@ -458,20 +473,6 @@ parse_program_version(const char *program_version) return result; } -static const char *statusName[] = -{ - "UNKNOWN", - "OK", - "ERROR", - "RUNNING", - "MERGING", - "MERGED", - "DELETING", - "DELETED", - "DONE", - "ORPHAN", - "CORRUPT" -}; const char * status2str(BackupStatus status) @@ -482,16 +483,15 @@ status2str(BackupStatus status) return statusName[status]; } - - BackupStatus str2status(const char *status) { + BackupStatus i; - for (int i = 0; i <= BACKUP_STATUS_CORRUPT; i++) + for (i = BACKUP_STATUS_INVALID; i <= BACKUP_STATUS_CORRUPT; i++) { if (pg_strcasecmp(status, statusName[i]) == 0) return i; } - return 0; + return BACKUP_STATUS_INVALID; } diff --git a/tests/delete.py b/tests/delete.py index 794502800..0f5be6bea 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -845,6 +845,5 @@ def test_delete_error_backups(self): self.assertEqual(show_backups[0]['status'], "OK") self.assertEqual(show_backups[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - + # Clean after yourself + self.del_test_dir(module_name, fname) From 7054d42861052da1732682cd39da96d13e27b9fe Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 24 Mar 2020 21:51:14 +0300 Subject: [PATCH 1193/2107] [Issue #120] report the number of deleted backups --- src/delete.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index f92b9e6d1..6fc26ab65 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1066,7 +1066,8 @@ do_delete_status(InstanceConfig *instance_config, const char *status) } if (n_deleted > 0) - elog(INFO, "Successfully deleted all backups with status '%s' from instance '%s'", + elog(INFO, "Successfully deleted %i %s with status '%s' from instance '%s'", + n_deleted, n_deleted == 1 ? "backup" : "backups", pretty_status, instance_config->name); else elog(WARNING, "Instance '%s' has no backups with status '%s'", From b3ac7469b2c783413dd174abe50d856616438cf4 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 25 Mar 2020 00:16:53 +0300 Subject: [PATCH 1194/2107] [Issue #146] Test update for backup note --- tests/backup.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e1d451120..37971ea96 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2404,3 +2404,43 @@ def test_streaming_timeout(self): # Clean after yourself self.del_test_dir(module_name, fname) + + + def test_note(self): + + fname = self.id().split('.')[3] + + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream', '--log-level-file=LOG', '--note=test_note']) + + show_backups = self.show_pb(backup_dir, 'node') + # self.assertEqual(len(show_backups), 1) + # print(self.show_pb(backup_dir, as_text=True, as_json=True)) + + self.assertEqual(show_backups[0]['note'], "test_note") + + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + + + + + + + + + + From a19607394407fa4e18217549fa91021f602b5057 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 25 Mar 2020 00:19:46 +0300 Subject: [PATCH 1195/2107] [Issue #174] archive-push improvements: batching, multi-thread support, checksum computation on remote agent, use O_EXCL flag, --no-sync flag support --- Makefile | 6 +- doc/pgprobackup.xml | 82 ++- src/archive.c | 1079 +++++++++++++++++++++++-------- src/data.c | 4 +- src/dir.c | 66 ++ src/help.c | 23 +- src/pg_probackup.c | 22 +- src/pg_probackup.h | 15 +- src/show.c | 28 +- src/utils/file.c | 49 +- src/utils/file.h | 2 +- src/utils/remote.c | 2 +- tests/archive.py | 188 +++++- tests/helpers/ptrack_helpers.py | 17 +- 14 files changed, 1239 insertions(+), 344 deletions(-) diff --git a/Makefile b/Makefile index 37cceb56e..41502286b 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,9 @@ OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ - src/xlogreader.c + src/xlogreader.c src/instr_time.h -INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h +INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h src/instr_time.h ifdef USE_PGXS PG_CONFIG = pg_config @@ -60,6 +60,8 @@ all: checksrcdir $(INCLUDES); $(PROGRAM): $(OBJS) +src/instr_time.h: $(top_srcdir)/src/include/portability/instr_time.h + rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@ src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 0c04b8e2c..02324b380 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -131,7 +131,6 @@ doc/src/sgml/pgprobackup.sgml backup_dir instance_name - wal_file_path wal_file_name option @@ -786,7 +785,7 @@ ALTER ROLE backup WITH REPLICATION; parameter, as follows: -archive_command = 'install_dir/pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' +archive_command = 'install_dir/pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-name=%f [remote_options]' @@ -3934,9 +3933,12 @@ pg_probackup delete -B backup_dir --instance archive-push pg_probackup archive-push -B backup_dir --instance instance_name ---wal-file-path=wal_file_path --wal-file-name=wal_file_name -[--help] [--compress] [--compress-algorithm=compression_algorithm] -[--compress-level=compression_level] [--overwrite] +--wal-file-name=wal_file_name +[--help] [--no-sync] [--compress] [--no-ready-rename] [--overwrite] +[-j num_threads] [--batch-size=batch_size] +[--archive-timeout=timeout] +[--compress-algorithm=compression_algorithm] +[--compress-level=compression_level] [remote_options] [logging_options] @@ -3961,13 +3963,24 @@ pg_probackup archive-push -B backup_dir --instance --overwrite flag. - The files are copied to a temporary file with the - .part suffix. After the copy is - done, atomic rename is performed. This algorithm ensures that a - failed archive-push will not stall continuous archiving and - that concurrent archiving from multiple sources into a single - WAL archive have no risk of archive corruption. WAL segments copied to - the archive are synced to disk. + Every file is copied to a temporary file with the + .part suffix. If the temporary file already + exists, pg_probackup will wait + seconds before discarding it. + After the copy is done, atomic rename is performed. + This algorithm ensures that a failed archive-push + will not stall continuous archiving and that concurrent archiving from + multiple sources into a single WAL archive have no risk of archive + corruption. + + + To speed up archiving, especially in remote mode, archive-push + can be run on multiple threads using option. + Files can also be copied in batches using option . + + + WAL segments copied to the archive are synced to disk unless + flag is used. You can use archive-push in the @@ -4073,7 +4086,8 @@ pg_probackup archive-get -B backup_dir --instance Sets the number of parallel threads for backup, restore, merge, - validate, and checkdb processes. + validate, checkdb and + archive-push processes. @@ -4732,6 +4746,48 @@ pg_probackup archive-get -B backup_dir --instance + + + + + + Sets the maximum number of files to be copied into archive by signle + archive-push process. + + + + + + + + + Sets the timeout for considering existing .part file to be stale. By default pg_probackup waits 300 seconds. + + + + + + + + + Do not rename status files in archive_status directory. + This option should be used only if archive_command + contain multiple commands. + + + + + + + + + Do not sync copied WAL files to disk. You can use this flag to speed + up archiving process. Using this flag can result in WAL archive + corruption in case of operating system or hardware crash. + + + + diff --git a/src/archive.c b/src/archive.c index b6a227ed9..ab3471b4e 100644 --- a/src/archive.c +++ b/src/archive.c @@ -8,50 +8,118 @@ *------------------------------------------------------------------------- */ -#include "pg_probackup.h" - #include +#include "pg_probackup.h" +#include "utils/thread.h" +#include "instr_time.h" -static void push_wal_file(const char *from_path, const char *to_path, - bool is_compress, bool overwrite, int compress_level); +static int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int thread_num, uint32 archive_timeout); +#ifdef HAVE_LIBZ +static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int compress_level, int thread_num, uint32 archive_timeout); +#endif +static void *push_files(void *arg); static void get_wal_file(const char *from_path, const char *to_path); #ifdef HAVE_LIBZ static const char *get_gz_error(gzFile gzf, int errnum); #endif -static bool fileEqualCRC(const char *path1, const char *path2, - bool path2_is_compressed); static void copy_file_attributes(const char *from_path, fio_location from_location, const char *to_path, fio_location to_location, bool unlink_on_error); +typedef struct +{ + const char *first_filename; + const char *pg_xlog_dir; + const char *archive_dir; + const char *archive_status_dir; + bool overwrite; + bool compress; + bool no_sync; + bool no_ready_rename; + uint32 archive_timeout; + + CompressAlg compress_alg; + int compress_level; + int thread_num; + + parray *files; + + uint32 n_pushed; + uint32 n_skipped; + + /* + * Return value from the thread. + * 0 means there is no error, + * 1 - there is an error. + * 2 - no error, but nothing to push + */ + int ret; +} archive_push_arg; + +typedef struct WALSegno +{ + char name[MAXFNAMELEN]; + volatile pg_atomic_flag lock; +} WALSegno; + +static int push_file(WALSegno *xlogfile, const char *archive_status_dir, + const char *pg_xlog_dir, const char *archive_dir, + bool overwrite, bool no_sync, uint32 archive_timeout, + bool no_ready_rename, bool is_compress, + int compress_level, int thread_num); + +static parray *setup_push_filelist(const char *archive_status_dir, + const char *first_file, int batch_size); + /* + * At this point, we already done one roundtrip to archive server + * to get instance config. + * * pg_probackup specific archive command for archive backups - * set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup - * --wal-file-path %p --wal-file-name %f', to move backups into arclog_path. - * Where archlog_path is $BACKUP_PATH/wal/system_id. - * Currently it just copies wal files to the new location. + * set archive_command to + * 'pg_probackup archive-push -B /home/anastasia/backup --wal-file-name %f', + * to move backups into arclog_path. + * Where archlog_path is $BACKUP_PATH/wal/instance_name */ -int -do_archive_push(InstanceConfig *instance, - char *wal_file_path, char *wal_file_name, bool overwrite) +void +do_archive_push(InstanceConfig *instance, char *wal_file_path, + char *wal_file_name, int batch_size, bool overwrite, + bool no_sync, bool no_ready_rename) { - char backup_wal_file_path[MAXPGPATH]; - char absolute_wal_file_path[MAXPGPATH]; + uint64 i; char current_dir[MAXPGPATH]; + char pg_xlog_dir[MAXPGPATH]; + char archive_status_dir[MAXPGPATH]; uint64 system_id; bool is_compress = false; - if (wal_file_name == NULL && wal_file_path == NULL) - elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + /* arrays with meta info for multi threaded backup */ + pthread_t *threads; + archive_push_arg *threads_args; + bool push_isok = true; - if (wal_file_name == NULL) - elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + /* reporting */ + uint32 n_total_pushed = 0; + uint32 n_total_skipped = 0; + uint32 n_total_failed = 0; + pid_t my_pid; + instr_time start_time, end_time; + double push_time; + char pretty_time_str[20]; - if (wal_file_path == NULL) - elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + /* files to push in multi-thread mode */ + parray *batch_files = NULL; + int n_threads; - canonicalize_path(wal_file_path); + my_pid = getpid(); + + if (wal_file_name == NULL) + elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); if (!getcwd(current_dir, sizeof(current_dir))) elog(ERROR, "getcwd() error"); @@ -60,204 +128,669 @@ do_archive_push(InstanceConfig *instance, system_id = get_system_identifier(current_dir); if (instance->pgdata == NULL) - elog(ERROR, "cannot read pg_probackup.conf for this instance"); + elog(ERROR, "Cannot read pg_probackup.conf for this instance"); - if(system_id != instance->system_identifier) + if (system_id != instance->system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instance->name, instance->system_identifier, - system_id); - - /* Create 'archlog_path' directory. Do nothing if it already exists. */ - fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + wal_file_name, instance->name, instance->system_identifier, system_id); - join_path_components(absolute_wal_file_path, current_dir, wal_file_path); - join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); + if (instance->compress_alg == PGLZ_COMPRESS) + elog(ERROR, "Cannot use pglz for WAL compression"); - elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path); + join_path_components(pg_xlog_dir, current_dir, XLOGDIR); + join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); - if (instance->compress_alg == PGLZ_COMPRESS) - elog(ERROR, "pglz compression is not supported"); + /* Create 'archlog_path' directory. Do nothing if it already exists. */ + //fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); #ifdef HAVE_LIBZ if (instance->compress_alg == ZLIB_COMPRESS) - is_compress = IsXLogFileName(wal_file_name); + is_compress = true; #endif - push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress, - overwrite, instance->compress_level); - elog(INFO, "pg_probackup archive-push completed successfully"); + /* Setup filelist and locks */ + batch_files = setup_push_filelist(archive_status_dir, wal_file_name, batch_size); + + n_threads = num_threads; + if (num_threads > parray_num(batch_files)) + n_threads = parray_num(batch_files); + + elog(INFO, "PID [%d]: pg_probackup push file %s into archive, " + "threads: %i/%i, batch: %lu/%i, compression: %s", + my_pid, wal_file_name, n_threads, num_threads, + parray_num(batch_files), batch_size, + is_compress ? "zlib" : "none"); + + num_threads = n_threads; + + /* Single-thread push + * We don`t want to start multi-thread push, if number of threads in equal to 1, + * or the number of files ready to push is small. + * Multithreading in remote mode isn`t cheap, + * establishing ssh connection can take 100-200ms, so running and terminating + * one thread using generic multithread approach can take + * almost as much time as copying itself. + * TODO: maybe we should be more conservative and force single thread + * push if batch_files array is small. + */ + if (num_threads == 1 || (parray_num(batch_files) == 1)) + { + INSTR_TIME_SET_CURRENT(start_time); + for (i = 0; i < parray_num(batch_files); i++) + { + int rc; + WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); + + rc = push_file(xlogfile, archive_status_dir, + pg_xlog_dir, instance->arclog_path, + overwrite, no_sync, + instance->archive_timeout, + no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false, + is_compress && IsXLogFileName(xlogfile->name) ? true : false, + instance->compress_level, 1); + if (rc == 0) + n_total_pushed++; + else + n_total_skipped++; + } - return 0; + push_isok = true; + goto push_done; + } + + /* init thread args with its own segno */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (archive_push_arg *) palloc(sizeof(archive_push_arg) * num_threads); + + for (i = 0; i < num_threads; i++) + { + archive_push_arg *arg = &(threads_args[i]); + + arg->first_filename = wal_file_name; + arg->archive_dir = instance->arclog_path; + arg->pg_xlog_dir = pg_xlog_dir; + arg->archive_status_dir = archive_status_dir; + arg->overwrite = overwrite; + arg->compress = is_compress; + arg->no_sync = no_sync; + arg->no_ready_rename = no_ready_rename; + arg->archive_timeout = instance->archive_timeout; + + arg->compress_alg = instance->compress_alg; + arg->compress_level = instance->compress_level; + + arg->files = batch_files; + arg->n_pushed = 0; + arg->n_skipped = 0; + + arg->thread_num = i+1; + /* By default there are some error */ + arg->ret = 1; + } + + /* Run threads */ + INSTR_TIME_SET_CURRENT(start_time); + for (i = 0; i < num_threads; i++) + { + archive_push_arg *arg = &(threads_args[i]); + pthread_create(&threads[i], NULL, push_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + if (threads_args[i].ret == 1) + { + push_isok = false; + n_total_failed++; + } + + n_total_pushed += threads_args[i].n_pushed; + n_total_skipped += threads_args[i].n_skipped; + } + + /* Note, that we are leaking memory here, + * because pushing into archive is a very + * time-sensetive operation, so we skip freeing stuff. + */ + +push_done: + fio_disconnect(); + /* calculate elapsed time */ + INSTR_TIME_SET_CURRENT(end_time); + INSTR_TIME_SUBTRACT(end_time, start_time); + push_time = INSTR_TIME_GET_DOUBLE(end_time); + pretty_time_interval(push_time, pretty_time_str, 20); + + if (push_isok) + /* report number of files pushed into archive */ + elog(INFO, "PID [%d]: pg_probackup archive-push completed successfully, " + "pushed: %u, skipped: %u, time elapsed: %s", + my_pid, n_total_pushed, n_total_skipped, pretty_time_str); + else + elog(ERROR, "PID [%d]: pg_probackup archive-push failed, " + "pushed: %i, skipped: %u, failed: %u, time elapsed: %s", + my_pid, n_total_pushed, n_total_skipped, n_total_failed, + pretty_time_str); } +/* ------------- INTERNAL FUNCTIONS ---------- */ /* - * pg_probackup specific restore command. - * Move files from arclog_path to pgdata/wal_file_path. + * Copy files from pg_wal to archive catalog with possible compression. */ -int -do_archive_get(InstanceConfig *instance, - char *wal_file_path, char *wal_file_name) +static void * +push_files(void *arg) { - char backup_wal_file_path[MAXPGPATH]; - char absolute_wal_file_path[MAXPGPATH]; - char current_dir[MAXPGPATH]; + int i; + int rc; + archive_push_arg *args = (archive_push_arg *) arg; - if (wal_file_name == NULL && wal_file_path == NULL) - elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + for (i = 0; i < parray_num(args->files); i++) + { + bool no_ready_rename = args->no_ready_rename; + WALSegno *xlogfile = (WALSegno *) parray_get(args->files, i); - if (wal_file_name == NULL) - elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + if (!pg_atomic_test_set_flag(&xlogfile->lock)) + continue; - if (wal_file_path == NULL) - elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + /* Do not rename ready file of the first file, + * we do this to avoid flooding PostgreSQL log with + * warnings about ready file been missing. + */ + if (strcmp(args->first_filename, xlogfile->name) == 0) + no_ready_rename = true; + + rc = push_file(xlogfile, args->archive_status_dir, + args->pg_xlog_dir, args->archive_dir, + args->overwrite, args->no_sync, + args->archive_timeout, no_ready_rename, + /* do not compress .backup, .partial and .history files */ + args->compress && IsXLogFileName(xlogfile->name) ? true : false, + args->compress_level, args->thread_num); + + if (rc == 0) + args->n_pushed++; + else + args->n_skipped++; + } - canonicalize_path(wal_file_path); + /* close ssh connection */ + fio_disconnect(); - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); + args->ret = 0; + return NULL; +} - join_path_components(absolute_wal_file_path, current_dir, wal_file_path); - join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); +int +push_file(WALSegno *xlogfile, const char *archive_status_dir, + const char *pg_xlog_dir, const char *archive_dir, + bool overwrite, bool no_sync, uint32 archive_timeout, + bool no_ready_rename, bool is_compress, + int compress_level, int thread_num) +{ + int rc; + char wal_file_dummy[MAXPGPATH]; - elog(INFO, "pg_probackup archive-get from %s to %s", - backup_wal_file_path, absolute_wal_file_path); - get_wal_file(backup_wal_file_path, absolute_wal_file_path); - elog(INFO, "pg_probackup archive-get completed successfully"); + join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); - return 0; + elog(LOG, "Thread [%d]: pushing file \"%s\"", thread_num, xlogfile->name); + + /* If compression is not required, then just copy it as is */ + if (!is_compress) + rc = push_file_internal_uncompressed(xlogfile->name, pg_xlog_dir, + archive_dir, overwrite, no_sync, + thread_num, archive_timeout); +#ifdef HAVE_LIBZ + else + rc = push_file_internal_gz(xlogfile->name, pg_xlog_dir, archive_dir, + overwrite, no_sync, compress_level, + thread_num, archive_timeout); +#endif + + /* take '--no-ready-rename' flag into account */ + if (!no_ready_rename) + { + char wal_file_ready[MAXPGPATH]; + char wal_file_done[MAXPGPATH]; + + snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready"); + snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done"); + + canonicalize_path(wal_file_ready); + canonicalize_path(wal_file_done); + /* It is ok to rename status file in archive_status directory */ + elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", thread_num, + wal_file_ready, wal_file_done); + + /* do not error out, if rename failed */ + if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) + elog(WARNING, "Thread [%d]: Cannot rename ready file \"%s\" to \"%s\": %s", + thread_num, wal_file_ready, wal_file_done, strerror(errno)); + } + + return rc; } -/* ------------- INTERNAL FUNCTIONS ---------- */ /* - * Copy WAL segment from pgdata to archive catalog with possible compression. + * Copy non WAL file, such as .backup or .history file, into WAL archive. + * Such files are not compressed. + * Returns: + * 0 - file was successfully pushed + * 1 - push was skipped because file already exists in the archive and + * has the same checksum */ -void -push_wal_file(const char *from_path, const char *to_path, bool is_compress, - bool overwrite, int compress_level) +int +push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int thread_num, uint32 archive_timeout) { FILE *in = NULL; int out = -1; - char buf[XLOG_BLCKSZ]; - const char *to_path_p; - char to_path_temp[MAXPGPATH]; - int errno_temp; + char buf[STDIO_BUFSIZE]; +// char buf[XLOG_BLCKSZ]; + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; /* partial handling */ struct stat st; + char to_fullpath_part[MAXPGPATH]; int partial_try_count = 0; int partial_file_size = 0; - bool partial_file_exists = false; + bool partial_is_stale = true; -#ifdef HAVE_LIBZ - char gz_to_path[MAXPGPATH]; - gzFile gz_out = NULL; + /* from path */ + join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); + canonicalize_path(from_fullpath); + /* to path */ + join_path_components(to_fullpath, archive_dir, wal_file_name); + canonicalize_path(to_fullpath); + + /* Open source file for read */ + in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); + if (in == NULL) + elog(ERROR, "Thread [%d]: Cannot open source file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); - if (is_compress) + /* open destination partial file for write */ + snprintf(to_fullpath_part, sizeof(to_fullpath_part), "%s.part", to_fullpath); + + /* Grab lock by creating temp file in exclusive mode */ + out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) { - snprintf(gz_to_path, sizeof(gz_to_path), "%s.gz", to_path); - to_path_p = gz_to_path; + if (errno != EEXIST) + elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + /* Already existing destination temp file is not an error condition */ } else -#endif - to_path_p = to_path; + goto part_opened; - /* open file for read */ - in = fio_fopen(from_path, PG_BINARY_R, FIO_DB_HOST); - if (in == NULL) - elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_path, - strerror(errno)); + /* + * Partial file already exists, it could have happened due to: + * 1. failed archive-push + * 2. concurrent archiving + * + * For ARCHIVE_TIMEOUT period we will try to create partial file + * and look for the size of already existing partial file, to + * determine if it is changing or not. + * If after ARCHIVE_TIMEOUT we still failed to create partial + * file, we will make a decision about discarding + * already existing partial file. + */ - /* Check if possible to skip copying */ - if (fileExists(to_path_p, FIO_BACKUP_HOST)) + while (partial_try_count < archive_timeout) { - if (fileEqualCRC(from_path, to_path_p, is_compress)) - return; - /* Do not copy and do not rise error. Just quit as normal. */ - else if (!overwrite) - elog(ERROR, "WAL segment \"%s\" already exists.", to_path_p); + if (fio_stat(to_fullpath_part, &st, false, FIO_BACKUP_HOST) < 0) + { + if (errno == ENOENT) + { + //part file is gone, lets try to grab it + out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) + { + if (errno != EEXIST) + elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + else + /* Successfully created partial file */ + break; + } + else + elog(ERROR, "Thread [%d]: Cannot stat temp WAL file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + + /* first round */ + if (!partial_try_count) + { + elog(LOG, "Thread [%d]: Temp WAL file already exists, " + "waiting on it %u seconds: \"%s\"", + thread_num, archive_timeout, to_fullpath_part); + partial_file_size = st.st_size; + } + + /* file size is changing */ + if (st.st_size > partial_file_size) + partial_is_stale = false; + + sleep(1); + partial_try_count++; } + /* The possible exit conditions: + * 1. File is grabbed + * 2. File is not grabbed, and it is not stale + * 2. File is not grabbed, and it is stale. + */ - /* open backup file for write */ -#ifdef HAVE_LIBZ - if (is_compress) + /* + * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, + * then exit with error. + */ + if (out < 0) { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", gz_to_path); + if (!partial_is_stale) + elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\" in %i seconds", + thread_num, to_fullpath_part, archive_timeout); + + /* Partial segment is considered stale, so reuse it */ + elog(LOG, "Thread [%d]: Reusing stale temp WAL file \"%s\"", + thread_num, to_fullpath_part); + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + + out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + if (out < 0) + elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + +part_opened: + elog(VERBOSE, "Thread [%d]: Temp WAL file successfully created: \"%s\"", + thread_num, to_fullpath_part); + /* Check if possible to skip copying */ + if (fileExists(to_fullpath, FIO_BACKUP_HOST)) + { + pg_crc32 crc32_src; + pg_crc32 crc32_dst; + + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + crc32_dst = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (gz_out == NULL) + if (crc32_src == crc32_dst) { - partial_file_exists = true; - elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " + "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); + /* cleanup */ + fio_fclose(in); + fio_close(out); + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + return 1; + } + else + { + if (overwrite) + elog(LOG, "Thread [%d]: WAL file already exists in archive with " + "different checksum, overwriting: \"%s\"", thread_num, to_fullpath); + else + { + /* Overwriting is forbidden, + * so we must unlink partial file and exit with error. + */ + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: WAL file already exists in archive with " + "different checksum: \"%s\"", thread_num, to_fullpath); + } } } - else -#endif + + /* copy content */ + for (;;) { - snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path); + ssize_t read_len = 0; - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) + read_len = fio_fread(in, buf, sizeof(buf)); + + if (read_len < 0) { - partial_file_exists = true; - elog(WARNING, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot read source file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); } + + if (read_len > 0) + { + if (fio_write(out, buf, read_len) != read_len) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot write to destination temp file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + } + + if (read_len == 0) + break; + } + + /* close source file */ + if (fio_fclose(in)) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot close source WAL file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); + } + + /* close temp file */ + if (fio_close(out) != 0) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot close temp WAL file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + + /* sync temp file to disk */ + if (!no_sync) + { + if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Thread [%d]: Failed to sync file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); + } + + elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", + thread_num, to_fullpath_part, to_fullpath); + + //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); + + /* Rename temp file to destination file */ + if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot rename file \"%s\" to \"%s\": %s", + thread_num, to_fullpath_part, to_fullpath, strerror(errno)); } - /* Partial file is already exists, it could have happened due to failed archive-push, - * in this case partial file can be discarded, or due to concurrent archiving. + return 0; +} + +#ifdef HAVE_LIBZ +/* + * Push WAL segment into archive and apply streaming compression to it. + * Returns: + * 0 - file was successfully pushed + * 1 - push was skipped because file already exists in the archive and + * has the same checksum + */ +int +push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int compress_level, int thread_num, uint32 archive_timeout) +{ + FILE *in = NULL; + gzFile out = NULL; + int errno_temp; + char buf[STDIO_BUFSIZE]; + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + char to_fullpath_gz[MAXPGPATH]; + + /* partial handling */ + struct stat st; + + char to_fullpath_gz_part[MAXPGPATH]; + int partial_try_count = 0; + int partial_file_size = 0; + bool partial_is_stale = true; + + /* from path */ + join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); + canonicalize_path(from_fullpath); + /* to path */ + join_path_components(to_fullpath, archive_dir, wal_file_name); + canonicalize_path(to_fullpath); + + /* destination file with .gz suffix */ + snprintf(to_fullpath_gz, sizeof(to_fullpath_gz), "%s.gz", to_fullpath); + /* destination temp file */ + snprintf(to_fullpath_gz_part, sizeof(to_fullpath_gz_part), "%s.part", to_fullpath_gz); + + /* Open source file for read */ + in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); + if (in == NULL) + elog(ERROR, "Thread [%d]: Cannot open source WAL file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); + + /* Grab lock by creating temp file in exclusive mode */ + out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + if (out == NULL) + { + if (errno != EEXIST) + elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno)); + /* Already existing destination temp file is not an error condition */ + } + else + goto part_opened; + + /* + * Partial file already exists, it could have happened due to: + * 1. failed archive-push + * 2. concurrent archiving * - * Our main goal here is to try to handle partial file to prevent stalling of - * continious archiving. - * To ensure that ecncountered partial file is actually a stale "orphaned" file, - * check its size every second. - * If the size has not changed in PARTIAL_WAL_TIMER seconds, we can consider - * the file stale and reuse it. - * If file size is changing, it means that another archiver works at the same - * directory with the same files. Such partial files cannot be reused. + * For ARCHIVE_TIMEOUT period we will try to create partial file + * and look for the size of already existing partial file, to + * determine if it is changing or not. + * If after ARCHIVE_TIMEOUT we still failed to create partial + * file, we will make a decision about discarding + * already existing partial file. */ - if (partial_file_exists) + + while (partial_try_count < archive_timeout) { - while (partial_try_count < PARTIAL_WAL_TIMER) + if (fio_stat(to_fullpath_gz_part, &st, false, FIO_BACKUP_HOST) < 0) { + if (errno == ENOENT) + { + //part file is gone, lets try to grab it + out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + if (out == NULL) + { + if (errno != EEXIST) + elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno)); + } + else + /* Successfully created partial file */ + break; + } + else + elog(ERROR, "Thread [%d]: Cannot stat temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno)); + } - if (fio_stat(to_path_temp, &st, false, FIO_BACKUP_HOST) < 0) - /* It is ok if partial is gone, we can safely error out */ - elog(ERROR, "Cannot stat destination temporary WAL file \"%s\": %s", to_path_temp, - strerror(errno)); + /* first round */ + if (!partial_try_count) + { + elog(LOG, "Thread [%d]: Temp WAL file already exists, " + "waiting on it %u seconds: \"%s\"", + thread_num, archive_timeout, to_fullpath_gz_part); + partial_file_size = st.st_size; + } - /* first round */ - if (!partial_try_count) - partial_file_size = st.st_size; + /* file size is changing */ + if (st.st_size > partial_file_size) + partial_is_stale = false; - /* file size is changing */ - if (st.st_size > partial_file_size) - elog(ERROR, "Destination temporary WAL file \"%s\" is not stale", to_path_temp); + sleep(1); + partial_try_count++; + } + /* The possible exit conditions: + * 1. File is grabbed + * 2. File is not grabbed, and it is not stale + * 2. File is not grabbed, and it is stale. + */ - sleep(1); - partial_try_count++; - } + /* + * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, + * then exit with error. + */ + if (out == NULL) + { + if (!partial_is_stale) + elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\" in %i seconds", + thread_num, to_fullpath_gz_part, archive_timeout); /* Partial segment is considered stale, so reuse it */ - elog(WARNING, "Reusing stale destination temporary WAL file \"%s\"", to_path_temp); - fio_unlink(to_path_temp, FIO_BACKUP_HOST); + elog(LOG, "Thread [%d]: Reusing stale temp WAL file \"%s\"", + thread_num, to_fullpath_gz_part); + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + + out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + if (out == NULL) + elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno)); + } -#ifdef HAVE_LIBZ - if (is_compress) +part_opened: + elog(VERBOSE, "Thread [%d]: Temp WAL file successfully created: \"%s\"", + thread_num, to_fullpath_gz_part); + /* Check if possible to skip copying, + */ + if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) + { + pg_crc32 crc32_src; + pg_crc32 crc32_dst; + + /* what if one of them goes missing */ + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_DB_HOST, true); + + if (crc32_src == crc32_dst) { - gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (gz_out == NULL) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " + "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); + /* cleanup */ + fio_fclose(in); + fio_gzclose(out); + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + return 1; } else -#endif { - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + if (overwrite) + elog(LOG, "Thread [%d]: WAL file already exists in archive with " + "different checksum, overwriting: \"%s\"", thread_num, to_fullpath_gz); + else + { + /* Overwriting is forbidden, + * so we must unlink partial file and exit with error. + */ + fio_fclose(in); + fio_gzclose(out); + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: WAL file already exists in archive with " + "different checksum: \"%s\"", thread_num, to_fullpath_gz); + } } } @@ -270,36 +803,19 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, if (read_len < 0) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, - "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot read from source file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); } if (read_len > 0) { -#ifdef HAVE_LIBZ - if (is_compress) + if (fio_gzwrite(out, buf, read_len) != read_len) { - if (fio_gzwrite(gz_out, buf, read_len) != read_len) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to compressed WAL file \"%s\": %s", - to_path_temp, get_gz_error(gz_out, errno_temp)); - } - } - else -#endif - { - if (fio_write(out, buf, read_len) != read_len) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write to WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); - } + errno_temp = errno; + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot write to compressed temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, get_gz_error(out, errno_temp)); } } @@ -307,52 +823,83 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress, break; } -#ifdef HAVE_LIBZ - if (is_compress) + /* close source file */ + if (fio_fclose(in)) { - if (fio_gzclose(gz_out) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - to_path_temp, get_gz_error(gz_out, errno_temp)); - } + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot close source WAL file \"%s\": %s", + thread_num, from_fullpath, strerror(errno)); } - else -#endif + + /* close temp file */ + if (fio_gzclose(out) != 0) { - if (fio_flush(out) != 0 || fio_close(out) != 0) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); - } + errno_temp = errno; + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot close compressed temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno_temp)); } - if (fio_fclose(in)) + /* sync temp file to disk */ + if (!no_sync) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot close source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); + if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Thread [%d]: Failed to sync file \"%s\": %s", + thread_num, to_fullpath_gz_part, strerror(errno)); } - /* update file permission. */ - copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); + elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", + thread_num, to_fullpath_gz_part, to_fullpath_gz); + + //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); - if (fio_rename(to_path_temp, to_path_p, FIO_BACKUP_HOST) < 0) + /* Rename temp file to destination file */ + if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", - to_path_temp, to_path_p, strerror(errno_temp)); + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot rename file \"%s\" to \"%s\": %s", + thread_num, to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); } -#ifdef HAVE_LIBZ - if (is_compress) - elog(INFO, "WAL file compressed to \"%s\"", gz_to_path); + return 0; +} #endif + +/* + * pg_probackup specific restore command. + * Move files from arclog_path to pgdata/wal_file_path. + */ +int +do_archive_get(InstanceConfig *instance, + char *wal_file_path, char *wal_file_name) +{ + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL && wal_file_path == NULL) + elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + + if (wal_file_name == NULL) + elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + + if (wal_file_path == NULL) + elog(ERROR, "required parameter not specified: --wal-file-path %%p"); + + canonicalize_path(wal_file_path); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); + + elog(INFO, "pg_probackup archive-get from %s to %s", + backup_wal_file_path, absolute_wal_file_path); + get_wal_file(backup_wal_file_path, absolute_wal_file_path); + elog(INFO, "pg_probackup archive-get completed successfully"); + + return 0; } /* @@ -545,64 +1092,6 @@ get_gz_error(gzFile gzf, int errnum) } #endif -/* - * compare CRC of two WAL files. - * If necessary, decompress WAL file from path2 - */ -static bool -fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed) -{ - pg_crc32 crc1; - pg_crc32 crc2; - - /* Get checksum of backup file */ -#ifdef HAVE_LIBZ - if (path2_is_compressed) - { - char buf [1024]; - gzFile gz_in = NULL; - - INIT_FILE_CRC32(true, crc2); - gz_in = fio_gzopen(path2, PG_BINARY_R, Z_DEFAULT_COMPRESSION, FIO_BACKUP_HOST); - if (gz_in == NULL) - /* File cannot be read */ - elog(ERROR, - "Cannot compare WAL file \"%s\" with compressed \"%s\"", - path1, path2); - - for (;;) - { - int read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len <= 0 && !fio_gzeof(gz_in)) - { - /* An error occurred while reading the file */ - elog(WARNING, - "Cannot compare WAL file \"%s\" with compressed \"%s\": %d", - path1, path2, read_len); - return false; - } - COMP_FILE_CRC32(true, crc2, buf, read_len); - if (fio_gzeof(gz_in) || read_len == 0) - break; - } - FIN_FILE_CRC32(true, crc2); - - if (fio_gzclose(gz_in) != 0) - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - path2, get_gz_error(gz_in, errno)); - } - else -#endif - { - crc2 = fio_get_crc32(path2, FIO_BACKUP_HOST); - } - - /* Get checksum of original file */ - crc1 = fio_get_crc32(path1, FIO_DB_HOST); - - return EQ_CRC32C(crc1, crc2); -} - /* Copy file attributes */ static void copy_file_attributes(const char *from_path, fio_location from_location, @@ -627,3 +1116,65 @@ copy_file_attributes(const char *from_path, fio_location from_location, to_path, strerror(errno)); } } + +/* Look for files with '.ready' suffix in archive_status directory + * and pack such files into batch sized array. + */ +parray * +setup_push_filelist(const char *archive_status_dir, const char *first_file, + int batch_size) +{ + int i; + WALSegno *xlogfile = NULL; + parray *status_files = NULL; + parray *batch_files = parray_new(); + + /* guarantee that first filename is in batch list */ + xlogfile = palloc(sizeof(WALSegno)); + pg_atomic_init_flag(&xlogfile->lock); + strncpy(xlogfile->name, first_file, MAXFNAMELEN); + parray_append(batch_files, xlogfile); + + if (batch_size < 2) + return batch_files; + + /* get list of files from archive_status */ + status_files = parray_new(); + dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); + parray_qsort(status_files, pgFileComparePath); + + for (i = 0; i < parray_num(status_files); i++) + { + int result = 0; + char filename[MAXFNAMELEN]; + char suffix[MAXFNAMELEN]; + pgFile *file = (pgFile *) parray_get(status_files, i); + + result = sscanf(file->name, "%[^.]%s", (char *) &filename, (char *) &suffix); + + if (result != 2) + continue; + + if (strcmp(suffix, ".ready") != 0) + continue; + + /* first filename already in batch list */ + if (strcmp(filename, first_file) == 0) + continue; + + xlogfile = palloc(sizeof(WALSegno)); + pg_atomic_init_flag(&xlogfile->lock); + + strncpy(xlogfile->name, filename, MAXFNAMELEN); + parray_append(batch_files, xlogfile); + + if (parray_num(batch_files) >= batch_size) + break; + } + + /* cleanup */ + parray_walk(status_files, pgFileFree); + parray_free(status_files); + + return batch_files; +} diff --git a/src/data.c b/src/data.c index 2e1d034ac..3bc168caa 100644 --- a/src/data.c +++ b/src/data.c @@ -803,7 +803,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, file->mtime <= parent_backup_time) { - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST); + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) @@ -1069,7 +1069,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, break; if (read_len < 0) - elog(ERROR, "Cannot read backup mode file \"%s\": %s", + elog(ERROR, "Cannot read backup file \"%s\": %s", from_fullpath, strerror(errno)); if (fio_fwrite(out, buf, read_len) != read_len) diff --git a/src/dir.c b/src/dir.c index d44728dcc..5b8fcf8d3 100644 --- a/src/dir.c +++ b/src/dir.c @@ -315,6 +315,72 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) return crc; } +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + */ +pg_crc32 +pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +{ + gzFile fp; + pg_crc32 crc = 0; + char buf[STDIO_BUFSIZE]; + int len = 0; + int err; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = gzopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = gzread(fp, &buf, sizeof(buf)); + + if (len <= 0) + { + /* we either run into eof or error */ + if (gzeof(fp)) + break; + else + { + const char *err_str = NULL; + + err_str = gzerror(fp, &err); + elog(ERROR, "Cannot read from compressed file %s", err_str); + } + } + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + } + + FIN_FILE_CRC32(use_crc32c, crc); + gzclose(fp); + + return crc; +} + void pgFileFree(void *file) { diff --git a/src/help.c b/src/help.c index 2bc50d512..c006ebb2b 100644 --- a/src/help.c +++ b/src/help.c @@ -214,10 +214,11 @@ help_pg_probackup(void) printf(_(" [--help]\n")); printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [--overwrite]\n")); - printf(_(" [--compress]\n")); + printf(_(" [-j num-threads] [--batch-size batch_size]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + printf(_(" [--no-ready-rename] [--no-sync]\n")); + printf(_(" [--overwrite] [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -868,10 +869,11 @@ static void help_archive_push(void) { printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [--overwrite]\n")); - printf(_(" [--compress]\n")); + printf(_(" [-j num-threads] [--batch-size batch_size]\n")); + printf(_(" [--archive-timeout=timeout]\n")); + printf(_(" [--no-ready-rename] [--no-sync]\n")); + printf(_(" [--overwrite] [--compress]\n")); printf(_(" [--compress-algorithm=compress-algorithm]\n")); printf(_(" [--compress-level=compress-level]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -880,10 +882,13 @@ help_archive_push(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance to delete\n")); - printf(_(" --wal-file-path=wal-file-path\n")); - printf(_(" relative path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" name of the WAL file to retrieve from the server\n")); + printf(_(" name of the file to copy into WAL archive\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --batch-size=NUM number of files to be copied\n")); + printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n")); + printf(_(" --no-ready-rename do not rename '.ready' files in 'archive_status' directory\n")); + printf(_(" --no-sync do not sync WAL file to disk\n")); printf(_(" --overwrite overwrite archived WAL file\n")); printf(_("\n Compression options:\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0fc7067bc..f73952116 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -125,9 +125,11 @@ bool compress_shortcut = false; char *instance_name; /* archive push options */ +int batch_size = 1; static char *wal_file_path; static char *wal_file_name; -static bool file_overwrite = false; +static bool file_overwrite = false; +static bool no_ready_rename = false; /* show options */ ShowFormat show_format = SHOW_PLAIN; @@ -172,7 +174,6 @@ static ConfigOption cmd_options[] = { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, - { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, @@ -189,13 +190,14 @@ static ConfigOption cmd_options[] = { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, - { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 'b', 143, "no-validate", &no_validate, SOURCE_CMD_STRICT }, { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, + { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, + { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -216,9 +218,11 @@ static ConfigOption cmd_options[] = { 's', 150, "wal-file-path", &wal_file_path, SOURCE_CMD_STRICT }, { 's', 151, "wal-file-name", &wal_file_name, SOURCE_CMD_STRICT }, { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, + { 'b', 153, "no-ready-rename", &no_ready_rename, SOURCE_CMD_STRICT }, + { 'i', 162, "batch-size", &batch_size, SOURCE_CMD_STRICT }, /* show options */ - { 'f', 153, "format", opt_show_format, SOURCE_CMD_STRICT }, - { 'b', 161, "archive", &show_archive, SOURCE_CMD_STRICT }, + { 'f', 163, "format", opt_show_format, SOURCE_CMD_STRICT }, + { 'b', 164, "archive", &show_archive, SOURCE_CMD_STRICT }, /* set-backup options */ { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, @@ -746,14 +750,18 @@ main(int argc, char *argv[]) if (num_threads < 1) num_threads = 1; + if (batch_size < 1) + batch_size = 1; + compress_init(); /* do actual operation */ switch (backup_subcmd) { case ARCHIVE_PUSH_CMD: - return do_archive_push(&instance_config, wal_file_path, - wal_file_name, file_overwrite); + do_archive_push(&instance_config, wal_file_path, wal_file_name, + batch_size, file_overwrite, no_sync, no_ready_rename); + break; case ARCHIVE_GET_CMD: return do_archive_get(&instance_config, wal_file_path, wal_file_name); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 601c6ecaa..8d6ad40de 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -67,7 +67,6 @@ extern const char *PROGRAM_EMAIL; #define DATABASE_MAP "database_map" /* Timeout defaults */ -#define PARTIAL_WAL_TIMER 60 #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 @@ -573,6 +572,9 @@ typedef struct BackupPageHeader #define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg + +#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) #else #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo) @@ -589,6 +591,9 @@ typedef struct BackupPageHeader #define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg + +#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ + XLogFromFileName(fname, tli, logSegNo) #endif #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) @@ -692,8 +697,9 @@ extern int do_init(void); extern int do_add_instance(InstanceConfig *instance); /* in archive.c */ -extern int do_archive_push(InstanceConfig *instance, char *wal_file_path, - char *wal_file_name, bool overwrite); +extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, + char *wal_file_name, int batch_size, bool overwrite, + bool no_sync, bool no_ready_rename); extern int do_archive_get(InstanceConfig *instance, char *wal_file_path, char *wal_file_name); @@ -846,6 +852,7 @@ extern void pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c); +extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool missing_ok, bool use_crc32c); extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileComparePath(const void *f1, const void *f2); @@ -939,7 +946,7 @@ extern int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t CompressAlg alg, const char **errormsg); extern void pretty_size(int64 size, char *buf, size_t len); -extern void pretty_time_interval(int64 num_seconds, char *buf, size_t len); +extern void pretty_time_interval(double time, char *buf, size_t len); extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); extern void check_system_identifiers(PGconn *conn, char *pgdata); diff --git a/src/show.c b/src/show.c index f2af77255..fd69ff529 100644 --- a/src/show.c +++ b/src/show.c @@ -191,14 +191,18 @@ pretty_size(int64 size, char *buf, size_t len) } void -pretty_time_interval(int64 num_seconds, char *buf, size_t len) +pretty_time_interval(double time, char *buf, size_t len) { - int seconds = 0; - int minutes = 0; - int hours = 0; - int days = 0; + int num_seconds = 0; + int milliseconds = 0; + int seconds = 0; + int minutes = 0; + int hours = 0; + int days = 0; - if (num_seconds <= 0) + num_seconds = (int) time; + + if (time <= 0) { strncpy(buf, "0", len); return; @@ -214,6 +218,7 @@ pretty_time_interval(int64 num_seconds, char *buf, size_t len) num_seconds %= 60; seconds = num_seconds; + milliseconds = (int)((time - (int) time) * 1000.0); if (days > 0) { @@ -233,7 +238,16 @@ pretty_time_interval(int64 num_seconds, char *buf, size_t len) return; } - snprintf(buf, len, "%ds", seconds); + if (seconds > 0) + { + if (milliseconds > 0) + snprintf(buf, len, "%ds:%dms", seconds, milliseconds); + else + snprintf(buf, len, "%ds", seconds); + return; + } + + snprintf(buf, len, "%dms", milliseconds); return; } diff --git a/src/utils/file.c b/src/utils/file.c index 68a81322e..7cce80099 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -340,7 +340,10 @@ int fio_open(char const* path, int mode, fio_location location) hdr.cop = FIO_OPEN; hdr.handle = i; hdr.size = strlen(path) + 1; - hdr.arg = mode & ~O_EXCL; + hdr.arg = mode; +// hdr.arg = mode & ~O_EXCL; +// elog(INFO, "PATH: %s MODE: %i, %i", path, mode, O_EXCL); +// elog(INFO, "MODE: %i", hdr.arg); fio_fdset |= 1 << i; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -490,6 +493,7 @@ int fio_close(int fd) fio_fdset &= ~(1 << hdr.handle); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + /* Note, that file is closed without waiting for confirmation */ return 0; } @@ -865,6 +869,8 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); + //TODO: wait for confirmation. + return 0; } else @@ -916,7 +922,7 @@ int fio_sync(char const* path, fio_location location) } /* Get crc32 of file */ -pg_crc32 fio_get_crc32(const char *file_path, fio_location location) +pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress) { if (fio_is_remote(location)) { @@ -926,6 +932,10 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location) hdr.cop = FIO_GET_CRC32; hdr.handle = -1; hdr.size = path_len; + hdr.arg = 0; + + if (decompress) + hdr.arg = 1; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); @@ -934,7 +944,12 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location) return crc; } else - return pgFileGetCRC(file_path, true, true); + { + if (decompress) + return pgFileGetCRCgz(file_path, true, true); + else + return pgFileGetCRC(file_path, true, true); + } } /* Remove file */ @@ -1027,6 +1042,7 @@ typedef struct fioGZFile Bytef buf[ZLIB_BUFFER_SIZE]; } fioGZFile; +/* On error returns NULL and errno should be checked */ gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location) { @@ -1037,6 +1053,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) memset(&gz->strm, 0, sizeof(gz->strm)); gz->eof = 0; gz->errnum = Z_OK; + /* check if file opened for writing */ if (strcmp(mode, PG_BINARY_W) == 0) /* compress */ { gz->strm.next_out = gz->buf; @@ -1049,14 +1066,12 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) if (rc == Z_OK) { gz->compress = 1; - if (fio_access(path, F_OK, location) == 0) + gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); + if (gz->fd < 0) { - elog(LOG, "File %s exists", path); free(gz); - errno = EEXIST; return NULL; } - gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); } } else @@ -1069,21 +1084,27 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) { gz->compress = 0; gz->fd = fio_open(path, O_RDONLY | PG_BINARY, location); + if (gz->fd < 0) + { + free(gz); + return NULL; + } } } if (rc != Z_OK) { - free(gz); - return NULL; + elog(ERROR, "zlib internal error when opening file %s: %s", + path, gz->strm.msg); } return (gzFile)((size_t)gz + FIO_GZ_REMOTE_MARKER); } else { gzFile file; + /* check if file opened for writing */ if (strcmp(mode, PG_BINARY_W) == 0) { - int fd = open(path, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FILE_PERMISSIONS); + int fd = open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, FILE_PERMISSIONS); if (fd < 0) return NULL; file = gzdopen(fd, mode); @@ -1143,7 +1164,8 @@ fio_gzread(gzFile f, void *buf, unsigned size) { gz->strm.next_in = gz->buf; } - rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in, gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); + rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in, + gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); if (rc > 0) { gz->strm.avail_in += rc; @@ -1772,7 +1794,10 @@ void fio_communicate(int in, int out) break; case FIO_GET_CRC32: /* calculate crc32 for a file */ - crc = pgFileGetCRC(buf, true, true); + if (hdr.arg == 1) + crc = pgFileGetCRCgz(buf, true, true); + else + crc = pgFileGetCRC(buf, true, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_DISCONNECT: diff --git a/src/utils/file.h b/src/utils/file.h index a831a4c1e..58c4e44a5 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -100,7 +100,7 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location); +extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, fio_location location); diff --git a/src/utils/remote.c b/src/utils/remote.c index 574d38091..7f8068d6b 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -220,7 +220,7 @@ bool launch_agent(void) return false; } else { #endif - elog(LOG, "Spawn agent %d version %s", child_pid, PROGRAM_VERSION); + elog(LOG, "Start SSH client process, pid %d", child_pid); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); SYS_CHECK(close(errfd[1])); diff --git a/tests/archive.py b/tests/archive.py index 7d62198df..1190c1b86 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -281,7 +281,7 @@ def test_pgpro434_3(self): # @unittest.skip("skip") def test_pgpro434_4(self): """ - Check pg_stop_backup_timeout, needed backup_timeout + Check pg_stop_backup_timeout, libpq-timeout requested. Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ fname = self.id().split('.')[3] @@ -398,15 +398,11 @@ def test_archive_push_file_exists(self): log_content) self.assertIn( - 'INFO: pg_probackup archive-push from', - log_content) - - self.assertIn( - 'ERROR: WAL segment ', + 'pg_probackup push file', log_content) self.assertIn( - 'already exists.', + 'WAL file already exists in archive with different checksum', log_content) self.assertNotIn( @@ -448,8 +444,7 @@ def test_archive_push_file_exists_overwrite(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) + pg_options={'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -487,9 +482,13 @@ def test_archive_push_file_exists_overwrite(self): self.assertIn( 'DETAIL: The failed archive command was:', log_content) self.assertIn( - 'INFO: pg_probackup archive-push from', log_content) + 'pg_probackup push file', log_content) + self.assertNotIn( + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log_content) self.assertIn( - '{0}" already exists.'.format(filename), log_content) + 'WAL file already exists in archive with ' + 'different checksum', log_content) self.assertNotIn( 'pg_probackup archive-push completed successfully', log_content) @@ -497,7 +496,7 @@ def test_archive_push_file_exists_overwrite(self): self.set_archiving(backup_dir, 'node', node, overwrite=True) node.reload() self.switch_wal_segment(node) - sleep(2) + sleep(5) with open(log_file, 'r') as f: log_content = f.read() @@ -505,6 +504,10 @@ def test_archive_push_file_exists_overwrite(self): 'pg_probackup archive-push completed successfully' in log_content, 'Expecting messages about successfull execution archive_command') + self.assertIn( + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log_content) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -520,7 +523,9 @@ def test_archive_push_partial_file_exists(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.set_archiving( + backup_dir, 'node', node, + log_level='verbose', archive_timeout=60) node.slow_start() @@ -579,12 +584,9 @@ def test_archive_push_partial_file_exists(self): log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: log_content = f.read() - self.assertIn( - 'Cannot open destination temporary WAL file', - log_content) self.assertIn( - 'Reusing stale destination temporary WAL file', + 'Reusing stale temp WAL file', log_content) # Clean after yourself @@ -602,7 +604,7 @@ def test_archive_push_partial_file_exists_not_stale(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, archive_timeout=60) node.slow_start() @@ -905,8 +907,8 @@ def test_basic_master_and_replica_concurrent_archiving(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'archive_timeout': '10s'} - ) + 'archive_timeout': '10s'}) + replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() @@ -923,6 +925,8 @@ def test_basic_master_and_replica_concurrent_archiving(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") + master.pgbench_init(scale=5) + # TAKE FULL ARCHIVE BACKUP FROM MASTER self.backup_node(backup_dir, 'master', master) # GET LOGICAL CONTENT FROM MASTER @@ -1718,7 +1722,7 @@ def test_hexadecimal_timeline(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, log_level='verbose') node.slow_start() backup_id = self.backup_node(backup_dir, 'node', node) @@ -1734,6 +1738,8 @@ def test_hexadecimal_timeline(self): node.slow_start() node.pgbench_init(scale=2) + sleep(5) + show = self.show_archive(backup_dir) timelines = show[0]['timelines'] @@ -1761,6 +1767,146 @@ def test_hexadecimal_timeline(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archiving_and_slots(self): + """ + Check that archiving don`t break slot + guarantee. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'checkpoint_timeout': '30s', + 'max_wal_size': '64MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, log_level='verbose') + node.slow_start() + + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " + # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" + + self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '--create-slot', '--slot', 'archive_slot', '--if-not-exists' + ]) + + node.pgbench_init(scale=10) + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node'), + '--no-loop', '--slot', 'archive_slot', + '-Z', '1' + ], asynchronous=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + sleep(2) + + pg_receivexlog.kill() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + exit(1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_archive_push_sanity(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_mode': 'on', + 'archive_command': 'exit 1'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=50) + node.stop() + + self.set_archiving(backup_dir, 'node', node) + os.remove(os.path.join(node.logs_dir, 'postgresql.log')) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + # print(postgres_log_content) + # make sure that .backup file is not compressed + self.assertNotIn('.backup.gz', postgres_log_content) + self.assertNotIn('WARNING', postgres_log_content) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, + data_dir=replica.data_dir, options=['-R']) + + #self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + self.set_auto_conf(replica, {'archive_mode': 'always'}) + self.set_auto_conf(replica, {'hot_standby': 'on'}) + replica.slow_start(replica=True) + + self.wait_until_replica_catch_with_master(node, replica) + + node.pgbench_init(scale=5) + + replica.promote() + replica.pgbench_init(scale=10) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + replica_log_content = f.read() + + # make sure that .partial file is not compressed + self.assertNotIn('.partial.gz', replica_log_content) + # make sure that .history file is not compressed + self.assertNotIn('.history.gz', replica_log_content) + self.assertNotIn('WARNING', replica_log_content) + + output = self.show_archive( + backup_dir, 'node', as_json=False, as_text=True, + options=['--log-level-console=VERBOSE']) + + self.assertNotIn('WARNING', output) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# TODO test with multiple not archived segments. + # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. # so write WAL validation code accordingly diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 595c8f152..d7ddfa037 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1131,7 +1131,8 @@ def get_recovery_conf(self, node): def set_archiving( self, backup_dir, instance, node, replica=False, - overwrite=False, compress=False, old_binary=False): + overwrite=False, compress=False, old_binary=False, + log_level=False, archive_timeout=False): # parse postgresql.auto.conf options = {} @@ -1161,12 +1162,26 @@ def set_archiving( if overwrite: options['archive_command'] += '--overwrite ' + options['archive_command'] += '--log-level-console=verbose ' + options['archive_command'] += '-j 5 ' + options['archive_command'] += '--batch-size 10 ' + options['archive_command'] += '--no-sync ' + + if archive_timeout: + options['archive_command'] += '--archive-timeout={0} '.format( + archive_timeout) + if os.name == 'posix': options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' elif os.name == 'nt': options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' + if log_level: + options['archive_command'] += ' --log-level-console={0}'.format(log_level) + options['archive_command'] += ' --log-level-file={0} '.format(log_level) + + self.set_auto_conf(node, options) def get_restore_command(self, backup_dir, instance, node): From 4a94fdaa33f8ee9781ab77e71024d5bc91df0b45 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 3 Apr 2020 18:08:53 +0300 Subject: [PATCH 1196/2107] [Issue #143] Multi-timeline incremental chain --- doc/pgprobackup.xml | 15 +- src/backup.c | 72 ++++- src/catalog.c | 165 +++++++++++- src/parsexlog.c | 208 ++++++++++++--- src/pg_probackup.h | 20 +- src/restore.c | 4 +- tests/archive.py | 2 +- tests/backup.py | 53 ++-- tests/page.py | 89 ++++++- tests/replica.py | 621 ++++++++++++++++++++++++++++++++++++++++---- tests/retention.py | 23 +- 11 files changed, 1107 insertions(+), 165 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 02324b380..02c144744 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -426,14 +426,6 @@ doc/src/sgml/pgprobackup.sgml or libc/libicu versions. - - - All backups in the incremental chain must belong to the same - timeline. For example, if you have taken incremental backups on a - standby server that gets promoted, you have to take another FULL - backup. - - @@ -753,9 +745,10 @@ ALTER ROLE backup WITH REPLICATION; Setting up Continuous WAL Archiving Making backups in PAGE backup mode, performing - PITR - and making backups with - ARCHIVE WAL delivery mode + PITR, + making backups with + ARCHIVE WAL delivery mode and + running incremental backup after timeline switch require continuous WAL archiving to be enabled. To set up continuous diff --git a/src/backup.c b/src/backup.c index a9151fdf1..bcc74ed30 100644 --- a/src/backup.c +++ b/src/backup.c @@ -153,6 +153,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) PGconn *master_conn = NULL; PGconn *pg_startbackup_conn = NULL; + /* used for multitimeline incremental backup */ + parray *tli_list = NULL; + + /* for fancy reporting */ time_t start_time, end_time; char pretty_time[20]; @@ -181,17 +185,43 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) current.backup_mode == BACKUP_MODE_DIFF_PTRACK || current.backup_mode == BACKUP_MODE_DIFF_DELTA) { - char prev_backup_filelist_path[MAXPGPATH]; - /* get list of backups already taken */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) - elog(ERROR, "Valid backup on current timeline %X is not found. " - "Create new FULL backup before an incremental one.", + { + /* try to setup multi-timeline backup chain */ + elog(WARNING, "Valid backup on current timeline %u is not found, " + "try to look up on previous timelines", current.tli); + tli_list = catalog_get_timelines(&instance_config); + + if (parray_num(tli_list) == 0) + elog(WARNING, "Cannot find valid backup on previous timelines, " + "WAL archive is not available"); + else + { + prev_backup = get_multi_timeline_parent(backup_list, tli_list, current.tli, + current.start_time, &instance_config); + + if (prev_backup == NULL) + elog(WARNING, "Cannot find valid backup on previous timelines"); + } + + /* failed to find suitable parent, error out */ + if (!prev_backup) + elog(ERROR, "Create new full backup before an incremental one"); + } + } + + if (prev_backup) + { + char prev_backup_filelist_path[MAXPGPATH]; + + elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); + join_path_components(prev_backup_filelist_path, prev_backup->root_dir, DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ @@ -378,8 +408,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - elog(INFO, "Compiling pagemap of changed blocks"); + bool pagemap_isok = true; + time(&start_time); + elog(INFO, "Extracting pagemap of changed blocks"); if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) { @@ -388,8 +420,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) * reading WAL segments present in archives up to the point * where this backup has started. */ - extractPageMap(arclog_path, current.tli, instance_config.xlog_seg_size, - prev_backup->start_lsn, current.start_lsn); + pagemap_isok = extractPageMap(arclog_path, instance_config.xlog_seg_size, + prev_backup->start_lsn, prev_backup->tli, + current.start_lsn, current.tli, tli_list); } else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -407,8 +440,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) } time(&end_time); - elog(INFO, "Pagemap compiled, time elapsed %.0f sec", - difftime(end_time, start_time)); + + /* TODO: add ms precision */ + if (pagemap_isok) + elog(INFO, "Pagemap successfully extracted, time elapsed %.0f sec", + difftime(end_time, start_time)); + else + elog(ERROR, "Pagemap extraction failed, time elasped: %.0f sec", + difftime(end_time, start_time)); } /* @@ -667,6 +706,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time); } + /* be paranoid about instance been from the past */ + if (current.backup_mode != BACKUP_MODE_FULL && + current.stop_lsn < prev_backup->stop_lsn) + elog(ERROR, "Current backup STOP LSN %X/%X is lower than STOP LSN %X/%X of previous backup %s. " + "It may indicate that we are trying to backup PostgreSQL instance from the past.", + (uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn), + (uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn), + base36enc(prev_backup->stop_lsn)); + /* clean external directories list */ if (external_dirs) free_dir_list(external_dirs); @@ -678,6 +726,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) parray_free(backup_list); } + if (tli_list) + { + parray_walk(tli_list, timelineInfoFree); + parray_free(tli_list); + } + parray_walk(backup_files_list, pgFileFree); parray_free(backup_files_list); backup_files_list = NULL; diff --git a/src/catalog.c b/src/catalog.c index 5221ccab7..972e78bfe 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -42,6 +42,24 @@ timelineInfoNew(TimeLineID tli) return tlinfo; } +/* free timelineInfo object */ +void +timelineInfoFree(void *tliInfo) +{ + timelineInfo *tli = (timelineInfo *) tliInfo; + + parray_walk(tli->xlog_filelist, pgFileFree); + parray_free(tli->xlog_filelist); + + if (tli->backups) + { + parray_walk(tli->backups, pgBackupFree); + parray_free(tli->backups); + } + + pfree(tliInfo); +} + /* Iterate over locked backups and delete locks files */ static void unlink_lock_atexit(void) @@ -621,11 +639,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current * anomalies. */ if (is_parent(full_backup->start_time, backup, true)) - { - elog(INFO, "Parent backup: %s", - base36enc(backup->start_time)); return backup; - } } } /* skip yourself */ @@ -641,6 +655,149 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current return NULL; } +/* + * For multi-timeline chain, look up suitable parent for incremental backup. + * Multi-timeline chain has full backup and one or more descendants located + * on different timelines. + */ +pgBackup * +get_multi_timeline_parent(parray *backup_list, parray *tli_list, + TimeLineID current_tli, time_t current_start_time, + InstanceConfig *instance) +{ + int i; + timelineInfo *my_tlinfo = NULL; + timelineInfo *tmp_tlinfo = NULL; + pgBackup *ancestor_backup = NULL; + + /* there are no timelines in the archive */ + if (parray_num(tli_list) == 0) + return NULL; + + /* look for current timelineInfo */ + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + + if (tlinfo->tli == current_tli) + { + my_tlinfo = tlinfo; + break; + } + } + + if (my_tlinfo == NULL) + return NULL; + + /* Locate tlinfo of suitable full backup. + * Consider this example: + * t3 s2-------X <-! We are here + * / + * t2 s1----D---*----E---> + * / + * t1--A--B--*---C-------> + * + * A, E - full backups + * B, C, D - incremental backups + * + * We must find A. + */ + tmp_tlinfo = my_tlinfo; + while (tmp_tlinfo->parent_link) + { + /* if timeline has backups, iterate over them */ + if (tmp_tlinfo->parent_link->backups) + { + for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++) + { + pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i); + + if (backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) && + backup->stop_lsn <= tmp_tlinfo->switchpoint) + { + ancestor_backup = backup; + break; + } + } + } + + if (ancestor_backup) + break; + + tmp_tlinfo = tmp_tlinfo->parent_link; + } + + /* failed to find valid FULL backup on parent timelines */ + if (!ancestor_backup) + return NULL; + else + elog(LOG, "Latest valid full backup: %s, tli: %i", + base36enc(ancestor_backup->start_time), ancestor_backup->tli); + + /* At this point we found suitable full backup, + * now we must find his latest child, suitable to be + * parent of current incremental backup. + * Consider this example: + * t3 s2-------X <-! We are here + * / + * t2 s1----D---*----E---> + * / + * t1--A--B--*---C-------> + * + * A, E - full backups + * B, C, D - incremental backups + * + * We found A, now we must find D. + */ + + /* Optimistically, look on current timeline for valid incremental backup, child of ancestor */ + if (my_tlinfo->backups) + { + for (i = 0; i < parray_num(my_tlinfo->backups); i++) + { + pgBackup *tmp_backup = NULL; + pgBackup *backup = (pgBackup *) parray_get(my_tlinfo->backups, i); + + /* found suitable parent */ + if (scan_parent_chain(backup, &tmp_backup) == 2 && + is_parent(ancestor_backup->start_time, backup, false)) + return backup; + } + } + + /* Iterate over parent timelines and look for a valid backup, child of ancestor */ + tmp_tlinfo = my_tlinfo; + while (tmp_tlinfo->parent_link) + { + + /* if timeline has backups, iterate over them */ + if (tmp_tlinfo->parent_link->backups) + { + for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++) + { + pgBackup *tmp_backup = NULL; + pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i); + + /* We are not interested in backups + * located outside of our timeline history + */ + if (backup->stop_lsn > tmp_tlinfo->switchpoint) + continue; + + if (scan_parent_chain(backup, &tmp_backup) == 2 && + is_parent(ancestor_backup->start_time, backup, true)) + return backup; + } + } + + tmp_tlinfo = tmp_tlinfo->parent_link; + } + + return NULL; +} + /* create backup directory in $BACKUP_PATH */ int pgBackupCreateDir(pgBackup *backup) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7c465db85..cda3bf33a 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -138,6 +138,9 @@ typedef struct */ bool got_target; + /* Should we read record, located at endpoint position */ + bool inclusive_endpoint; + /* * Return value from the thread. * 0 means there is no error, 1 - there is an error. @@ -162,7 +165,8 @@ static bool RunXLogThreads(const char *archivedir, XLogRecPtr startpoint, XLogRecPtr endpoint, bool consistent_read, xlog_record_function process_record, - XLogRecTarget *last_rec); + XLogRecTarget *last_rec, + bool inclusive_endpoint); //static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg); static bool SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg); @@ -231,18 +235,118 @@ static XLogRecPtr wal_target_lsn = InvalidXLogRecPtr; * Pagemap extracting is processed using threads. Each thread reads single WAL * file. */ -void -extractPageMap(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint) +bool +extractPageMap(const char *archivedir, uint32 wal_seg_size, + XLogRecPtr startpoint, TimeLineID start_tli, + XLogRecPtr endpoint, TimeLineID end_tli, + parray *tli_list) { - bool extract_isok = true; - - extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, - InvalidXLogRecPtr, tli, wal_seg_size, - startpoint, endpoint, false, extractPageInfo, - NULL); - if (!extract_isok) - elog(ERROR, "Pagemap compiling failed"); + bool extract_isok = false; + + if (start_tli == end_tli) + /* easy case */ + extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, end_tli, wal_seg_size, + startpoint, endpoint, false, extractPageInfo, + NULL, true); + else + { + /* We have to process WAL located on several different xlog intervals, + * located on different timelines. + * + * Consider this example: + * t3 C-----X + * / + * t1 -A----*-------> + * + * A - prev backup START_LSN + * B - switchpoint for t2, available as t2->switchpoint + * C - switch for t3, available as t3->switchpoint + * X - current backup START_LSN + * + * Intervals to be parsed: + * - [A,B) on t1 + * - [B,C) on t2 + * - [C,X] on t3 + */ + int i; + parray *interval_list = parray_new(); + timelineInfo *end_tlinfo = NULL; + timelineInfo *tmp_tlinfo = NULL; + XLogRecPtr prev_switchpoint = InvalidXLogRecPtr; + lsnInterval *wal_interval = NULL; + + /* We must find TLI information about final timeline (t3 in example) */ + for (i = 0; i < parray_num(tli_list); i++) + { + tmp_tlinfo = parray_get(tli_list, i); + + if (tmp_tlinfo->tli == end_tli) + { + end_tlinfo = tmp_tlinfo; + break; + } + } + + /* Iterate over timelines backward, + * starting with end_tli and ending with start_tli. + * For every timeline calculate LSN-interval that must be parsed. + */ + + tmp_tlinfo = end_tlinfo; + while (tmp_tlinfo) + { + wal_interval = pgut_malloc(sizeof(lsnInterval)); + wal_interval->tli = tmp_tlinfo->tli; + + if (tmp_tlinfo->tli == end_tli) + { + wal_interval->begin_lsn = tmp_tlinfo->switchpoint; + wal_interval->end_lsn = endpoint; + } + else if (tmp_tlinfo->tli == start_tli) + { + wal_interval->begin_lsn = startpoint; + wal_interval->end_lsn = prev_switchpoint; + } + else + { + wal_interval->begin_lsn = tmp_tlinfo->switchpoint; + wal_interval->end_lsn = prev_switchpoint; + } + + prev_switchpoint = tmp_tlinfo->switchpoint; + tmp_tlinfo = tmp_tlinfo->parent_link; + + parray_append(interval_list, wal_interval); + } + + for (i = parray_num(interval_list) - 1; i >= 0; i--) + { + bool inclusive_endpoint = false; + wal_interval = parray_get(interval_list, i); + + /* In case of replica promotion, endpoints of intermediate + * timelines can be unreachable. + */ + if (wal_interval->tli == end_tli) + inclusive_endpoint = true; + + extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, + InvalidXLogRecPtr, wal_interval->tli, wal_seg_size, + wal_interval->begin_lsn, wal_interval->end_lsn, + false, extractPageInfo, NULL, inclusive_endpoint); + if (!extract_isok) + break; + + pg_free(wal_interval); + } + pg_free(interval_list); + } + + return extract_isok; } /* @@ -262,7 +366,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, got_endpoint = RunXLogThreads(archivedir, 0, InvalidTransactionId, InvalidXLogRecPtr, tli, xlog_seg_size, backup->start_lsn, backup->stop_lsn, - false, NULL, NULL); + false, NULL, NULL, true); if (!got_endpoint) { @@ -373,7 +477,7 @@ validate_wal(pgBackup *backup, const char *archivedir, all_wal = all_wal || RunXLogThreads(archivedir, target_time, target_xid, target_lsn, tli, wal_seg_size, backup->stop_lsn, - InvalidXLogRecPtr, true, validateXLogRecord, &last_rec); + InvalidXLogRecPtr, true, validateXLogRecord, &last_rec, true); if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), timestamptz_to_time_t(last_rec.rec_time)); @@ -753,11 +857,26 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (!reader_data->xlogexists) { char xlogfname[MAXFNAMELEN]; + char partial_file[MAXPGPATH]; GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, xlogfname); + snprintf(partial_file, MAXPGPATH, "%s/%s.partial", wal_archivedir, + xlogfname); + snprintf(reader_data->gz_xlogpath, sizeof(reader_data->gz_xlogpath), + "%s.gz", reader_data->xlogpath); + + /* If segment do not exists, but the same + * segment with '.partial' suffix does, use it instead */ + if (!fileExists(reader_data->xlogpath, FIO_BACKUP_HOST) && + fileExists(partial_file, FIO_BACKUP_HOST)) + { + snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s.partial", wal_archivedir, + xlogfname); + strncpy(reader_data->xlogpath, partial_file, MAXPGPATH); + } if (fileExists(reader_data->xlogpath, FIO_BACKUP_HOST)) { @@ -778,29 +897,23 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, } #ifdef HAVE_LIBZ /* Try to open compressed WAL segment */ - else + else if (fileExists(reader_data->gz_xlogpath, FIO_BACKUP_HOST)) { - snprintf(reader_data->gz_xlogpath, sizeof(reader_data->gz_xlogpath), - "%s.gz", reader_data->xlogpath); - if (fileExists(reader_data->gz_xlogpath, FIO_BACKUP_HOST)) - { - elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", - reader_data->thread_num, reader_data->gz_xlogpath); + elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", + reader_data->thread_num, reader_data->gz_xlogpath); - reader_data->xlogexists = true; - reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, + reader_data->xlogexists = true; + reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, "rb", -1, FIO_BACKUP_HOST); - if (reader_data->gz_xlogfile == NULL) - { - elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - strerror(errno)); - return -1; - } + if (reader_data->gz_xlogfile == NULL) + { + elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", + reader_data->thread_num, reader_data->gz_xlogpath, + strerror(errno)); + return -1; } } #endif - /* Exit without error if WAL segment doesn't exist */ if (!reader_data->xlogexists) return -1; @@ -923,7 +1036,7 @@ RunXLogThreads(const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, uint32 segment_size, XLogRecPtr startpoint, XLogRecPtr endpoint, bool consistent_read, xlog_record_function process_record, - XLogRecTarget *last_rec) + XLogRecTarget *last_rec, bool inclusive_endpoint) { pthread_t *threads; xlog_thread_arg *thread_args; @@ -932,17 +1045,27 @@ RunXLogThreads(const char *archivedir, time_t target_time, XLogSegNo endSegNo = 0; bool result = true; - if (!XRecOffIsValid(startpoint)) + if (!XRecOffIsValid(startpoint) && + !XRecOffIsNull(startpoint)) + { elog(ERROR, "Invalid startpoint value %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); + } if (!XLogRecPtrIsInvalid(endpoint)) { - if (!XRecOffIsValid(endpoint)) + if (XRecOffIsNull(endpoint)) + { + GetXLogSegNo(endpoint, endSegNo, segment_size); + endSegNo--; + } + else if (!XRecOffIsValid(endpoint)) + { elog(ERROR, "Invalid endpoint value %X/%X", (uint32) (endpoint >> 32), (uint32) (endpoint)); - - GetXLogSegNo(endpoint, endSegNo, segment_size); + } + else + GetXLogSegNo(endpoint, endSegNo, segment_size); } /* Initialize static variables for workers */ @@ -977,6 +1100,7 @@ RunXLogThreads(const char *archivedir, time_t target_time, arg->startpoint = startpoint; arg->endpoint = endpoint; arg->endSegNo = endSegNo; + arg->inclusive_endpoint = inclusive_endpoint; arg->got_target = false; /* By default there is some error */ arg->ret = 1; @@ -1192,6 +1316,18 @@ XLogThreadWorker(void *arg) reader_data->thread_num, (uint32) (errptr >> 32), (uint32) (errptr)); + /* In we failed to read record located at endpoint position, + * and endpoint is not inclusive, do not consider this as an error. + */ + if (!thread_arg->inclusive_endpoint && + errptr == thread_arg->endpoint) + { + elog(LOG, "Thread [%d]: Endpoint %X/%X is not inclusive, switch to the next timeline", + reader_data->thread_num, + (uint32) (thread_arg->endpoint >> 32), (uint32) (thread_arg->endpoint)); + break; + } + /* * If we don't have all WAL files from prev backup start_lsn to current * start_lsn, we won't be able to build page map and PAGE backup will diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8d6ad40de..47e0f46a9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -474,7 +474,7 @@ struct timelineInfo { TimeLineID tli; /* this timeline */ TimeLineID parent_tli; /* parent timeline. 0 if none */ timelineInfo *parent_link; /* link to parent timeline */ - XLogRecPtr switchpoint; /* if this timeline has a parent + XLogRecPtr switchpoint; /* if this timeline has a parent, then * switchpoint contains switchpoint LSN, * otherwise 0 */ XLogSegNo begin_segno; /* first present segment in this timeline */ @@ -500,6 +500,13 @@ typedef struct xlogInterval XLogSegNo end_segno; } xlogInterval; +typedef struct lsnInterval +{ + TimeLineID tli; + XLogRecPtr begin_lsn; + XLogRecPtr end_lsn; +} lsnInterval; + typedef enum xlogFileType { SEGMENT, @@ -763,6 +770,10 @@ extern void catalog_lock_backup_list(parray *backup_list, int from_idx, extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); +extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list, + TimeLineID current_tli, time_t current_start_time, + InstanceConfig *instance); +extern void timelineInfoFree(void *tliInfo); extern parray *catalog_get_timelines(InstanceConfig *instance); extern void do_set_backup(const char *instance_name, time_t backup_id, pgSetBackupParams *set_backup_params); @@ -898,9 +909,10 @@ extern bool create_empty_file(fio_location from_location, const char *to_root, extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ -extern void extractPageMap(const char *archivedir, - TimeLineID tli, uint32 seg_size, - XLogRecPtr startpoint, XLogRecPtr endpoint); +extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, + XLogRecPtr startpoint, TimeLineID start_tli, + XLogRecPtr endpoint, TimeLineID end_tli, + parray *tli_list); extern void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, diff --git a/src/restore.c b/src/restore.c index 177f9d5f3..f13b4c8cf 100644 --- a/src/restore.c +++ b/src/restore.c @@ -403,7 +403,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ validate_wal(dest_backup, arclog_path, rt->target_time, rt->target_xid, rt->target_lsn, - base_full_backup->tli, instance_config.xlog_seg_size); + dest_backup->tli, instance_config.xlog_seg_size); } /* Orphanize every OK descendant of corrupted backup */ else @@ -1326,7 +1326,7 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); if (backup->tli == timeline->tli && (XLogRecPtrIsInvalid(timeline->end) || - backup->stop_lsn < timeline->end)) + backup->stop_lsn <= timeline->end)) return true; } return false; diff --git a/tests/archive.py b/tests/archive.py index 1190c1b86..95f5c178c 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1761,7 +1761,7 @@ def test_hexadecimal_timeline(self): tli13['closest-backup-id']) self.assertEqual( - '0000000D000000000000001B', + '0000000D000000000000001C', tli13['max-segno']) # Clean after yourself diff --git a/tests/backup.py b/tests/backup.py index 398bb5f14..ef418c970 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -228,10 +228,9 @@ def test_incremental_backup_corrupt_full(self): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -2294,10 +2293,9 @@ def test_parent_choosing_2(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'ERROR: Valid backup on current timeline 1 is not found. ' - 'Create new FULL backup before an incremental one.', - e.message, + self.assertTrue( + 'WARNING: Valid backup on current timeline 1 is not found' in e.message and + 'ERROR: Create new full backup before an incremental one' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -2324,10 +2322,13 @@ def test_backup_with_less_privileges_role(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '1h'}) + 'archive_mode': 'always', + 'checkpoint_timeout': '60s', + 'wal_level': 'logical'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) self.set_archiving(backup_dir, 'node', node) node.slow_start() @@ -2447,12 +2448,15 @@ def test_backup_with_less_privileges_role(self): self.restore_node(backup_dir, 'node', replica) self.set_replica(node, replica) self.add_instance(backup_dir, 'replica', replica) + self.set_config( + backup_dir, 'replica', + options=['--archive-timeout=120s', '--log-level-console=LOG']) self.set_archiving(backup_dir, 'replica', replica, replica=True) self.set_auto_conf(replica, {'hot_standby': 'on'}) # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) + # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] + # gdb_checkpointer = self.gdb_attach(bgwriter_pid) copy_tree( os.path.join(backup_dir, 'wal', 'node'), @@ -2460,21 +2464,22 @@ def test_backup_with_less_privileges_role(self): replica.slow_start(replica=True) - self.switch_wal_segment(node) - self.switch_wal_segment(node) + # self.switch_wal_segment(node) + # self.switch_wal_segment(node) - # FULL backup from replica self.backup_node( backup_dir, 'replica', replica, - datname='backupdb', options=['--stream', '-U', 'backup', '--archive-timeout=30s']) - -# self.switch_wal_segment(node) + datname='backupdb', options=['-U', 'backup']) + # stream full backup from replica self.backup_node( - backup_dir, 'replica', replica, datname='backupdb', - options=['-U', 'backup', '--archive-timeout=300s']) + backup_dir, 'replica', replica, + datname='backupdb', options=['--stream', '-U', 'backup']) + +# self.switch_wal_segment(node) # PAGE backup from replica + self.switch_wal_segment(node) self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) @@ -2484,20 +2489,22 @@ def test_backup_with_less_privileges_role(self): datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA backup from replica + self.switch_wal_segment(node) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK backup from replica if self.ptrack: + self.switch_wal_segment(node) self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + backup_dir, 'replica', replica, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', + backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) # Clean after yourself diff --git a/tests/page.py b/tests/page.py index 9dbb9e3f0..68dd266c7 100644 --- a/tests/page.py +++ b/tests/page.py @@ -874,7 +874,6 @@ def test_page_backup_with_corrupted_wal_segment(self): 'INFO: Wait for WAL segment' in e.message and 'to be archived' in e.message and 'Could not read WAL record at' in e.message and - 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -899,7 +898,6 @@ def test_page_backup_with_corrupted_wal_segment(self): 'INFO: Wait for WAL segment' in e.message and 'to be archived' in e.message and 'Could not read WAL record at' in e.message and - 'incorrect resource manager data checksum in record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -942,8 +940,10 @@ def test_page_backup_with_alien_wal_segment(self): self.set_archiving(backup_dir, 'alien_node', alien_node) alien_node.slow_start() - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'alien_node', alien_node) + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + self.backup_node( + backup_dir, 'alien_node', alien_node, options=['--stream']) # make some wals node.safe_psql( @@ -996,8 +996,6 @@ def test_page_backup_with_alien_wal_segment(self): 'INFO: Wait for WAL segment' in e.message and 'to be archived' in e.message and 'Could not read WAL record at' in e.message and - 'WAL file is from different database system: WAL file database system identifier is' in e.message and - 'pg_control database system identifier is' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1181,6 +1179,85 @@ def test_page_create_db(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multi_timeline_page(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t12 /---P--> + ... + t3 /----> + t2 /----> + t1 -F-----D-> + + P must have F as parent + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create timelines + for i in range(2, 12): + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=['--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + pgbench = node.pgbench(options=['-T', '3', '-c', '1', '--no-vacuum']) + pgbench.wait() + + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=VERBOSE']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + self.restore_node(backup_dir, 'node', node) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + show = self.show_archive(backup_dir) + + timelines = show[0]['timelines'] + + # self.assertEqual() + self.assertEqual( + self.show_pb(backup_dir, 'node', page_id)['parent-backup-id'], + full_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) + @unittest.skip("skip") # @unittest.expectedFailure def test_page_pg_resetxlog(self): diff --git a/tests/replica.py b/tests/replica.py index 7b37b6861..a767f68e1 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -571,30 +571,25 @@ def test_replica_stop_lsn_null_offset(self): 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) master.slow_start() # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] gdb_checkpointer = self.gdb_attach(bgwriter_pid) - self.backup_node(backup_dir, 'master', master) + self.backup_node(backup_dir, 'node', master) # Create replica replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.restore_node(backup_dir, 'node', replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) + self.set_archiving(backup_dir, 'node', replica, replica=True) replica.slow_start(replica=True) @@ -602,7 +597,7 @@ def test_replica_stop_lsn_null_offset(self): self.switch_wal_segment(master) output = self.backup_node( - backup_dir, 'replica', replica, + backup_dir, 'node', replica, replica.data_dir, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -611,24 +606,24 @@ def test_replica_stop_lsn_null_offset(self): return_id=False) self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/4000000', output) self.assertIn( - 'WARNING: WAL segment 000000010000000000000003 could not be streamed in 30 seconds', + 'WARNING: WAL segment 000000010000000000000004 could not be streamed in 30 seconds', output) self.assertIn( - 'WARNING: Failed to get next WAL record after 0/3000000, looking for previous WAL record', + 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', output) self.assertIn( - 'LOG: Looking for LSN 0/3000000 in segment: 000000010000000000000002', + 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', output) self.assertIn( - 'has endpoint 0/3000000 which is ' - 'equal or greater than requested LSN 0/3000000', + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN 0/4000000', output) self.assertIn( @@ -719,19 +714,19 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content = f.read() self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/4000000', log_content) self.assertIn( - 'LOG: Looking for segment: 000000010000000000000003', + 'LOG: Looking for segment: 000000010000000000000004', log_content) self.assertIn( - 'LOG: First record in WAL segment "000000010000000000000003": 0/3000028', + 'LOG: First record in WAL segment "000000010000000000000004": 0/4000028', log_content) self.assertIn( - 'LOG: current.stop_lsn: 0/3000028', + 'LOG: current.stop_lsn: 0/4000028', log_content) # Clean after yourself @@ -757,31 +752,26 @@ def test_archive_replica_null_offset(self): 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) master.slow_start() - self.backup_node(backup_dir, 'master', master) + self.backup_node(backup_dir, 'node', master) # Create replica replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.restore_node(backup_dir, 'node', replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] gdb_checkpointer = self.gdb_attach(bgwriter_pid) - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) self.switch_wal_segment(master) @@ -789,7 +779,7 @@ def test_archive_replica_null_offset(self): # take backup from replica output = self.backup_node( - backup_dir, 'replica', replica, + backup_dir, 'node', replica, replica.data_dir, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -797,24 +787,24 @@ def test_archive_replica_null_offset(self): return_id=False) self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/3000000', + 'LOG: Null offset in stop_backup_lsn value 0/4000000', output) self.assertIn( - 'WARNING: WAL segment 000000010000000000000003 could not be archived in 30 seconds', + 'WARNING: WAL segment 000000010000000000000004 could not be archived in 30 seconds', output) self.assertIn( - 'WARNING: Failed to get next WAL record after 0/3000000, looking for previous WAL record', + 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', output) self.assertIn( - 'LOG: Looking for LSN 0/3000000 in segment: 000000010000000000000002', + 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', output) self.assertIn( - 'has endpoint 0/3000000 which is ' - 'equal or greater than requested LSN 0/3000000', + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN 0/4000000', output) self.assertIn( @@ -846,44 +836,39 @@ def test_archive_replica_not_null_offset(self): 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) master.slow_start() - self.backup_node(backup_dir, 'master', master) + self.backup_node(backup_dir, 'node', master) # Create replica replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.restore_node(backup_dir, 'node', replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) + self.set_archiving(backup_dir, 'node', replica, replica=True) replica.slow_start(replica=True) # take backup from replica self.backup_node( - backup_dir, 'replica', replica, + backup_dir, 'node', replica, replica.data_dir, options=[ '--archive-timeout=30', - '--log-level-console=verbose', + '--log-level-console=LOG', '--no-validate'], return_id=False) try: self.backup_node( - backup_dir, 'replica', replica, + backup_dir, 'node', replica, replica.data_dir, options=[ '--archive-timeout=30', - '--log-level-console=verbose', + '--log-level-console=LOG', '--no-validate']) # we should die here because exception is what we expect to happen self.assertEqual( @@ -893,19 +878,19 @@ def test_archive_replica_not_null_offset(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'LOG: Looking for LSN 0/3000060 in segment: 000000010000000000000003', + 'LOG: Looking for LSN 0/4000060 in segment: 000000010000000000000004', e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) self.assertIn( - 'INFO: Wait for LSN 0/3000060 in archived WAL segment', + 'INFO: Wait for LSN 0/4000060 in archived WAL segment', e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: WAL segment 000000010000000000000003 could not be archived in 30 seconds', + 'ERROR: WAL segment 000000010000000000000004 could not be archived in 30 seconds', e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -1016,7 +1001,7 @@ def test_replica_toast(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_replica_promote_1(self): """ """ @@ -1037,7 +1022,7 @@ def test_replica_promote_1(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) - # set replica True, so archive_mode 'always' is used. + # set replica True, so archive_mode 'always' is used. self.set_archiving(backup_dir, 'master', master, replica=True) master.slow_start() @@ -1091,6 +1076,528 @@ def test_replica_promote_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_replica_promote_2(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + # set replica True, so archive_mode 'always' is used. + self.set_archiving( + backup_dir, 'master', master, replica=True) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,1) i') + + self.wait_until_replica_catch_with_master(master, replica) + + replica.promote() + + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + +# replica.safe_psql( +# 'postgres', +# 'create table t2()') +# +# replica.safe_psql( +# 'postgres', +# 'CHECKPOINT') + + self.backup_node( + backup_dir, 'master', replica, data_dir=replica.data_dir, + backup_type='page') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_promote_3(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + + full_id = self.backup_node( + backup_dir, 'replica', + replica, options=['--stream']) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + replica.promote() + + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + + # failing, because without archving, it is impossible to + # take multi-timeline backup. + try: + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of timeline switch " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Cannot find valid backup on previous timelines, ' + 'WAL archive is not available' in e.message and + 'ERROR: Create new full backup before an incremental one' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_promote_archive_delta(self): + """ + t3 /---D3--> + t2 /-------> + t1 --F---D1--D2-- + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'archive_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + self.set_config( + backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_archiving(backup_dir, 'node', node1) + + node1.slow_start() + + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + + # Create replica + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + self.restore_node(backup_dir, 'node', node2, node2.data_dir) + + # Settings for Replica + self.set_replica(node1, node2) + self.set_auto_conf(node2, {'port': node2.port}) + self.set_archiving(backup_dir, 'node', node2, replica=True) + + node2.slow_start(replica=True) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + # delta backup on replica on timeline 1 + delta1_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, + 'delta', options=['--stream']) + + # delta backup on replica on timeline 1 + delta2_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, 'delta') + + self.change_backup_status( + backup_dir, 'node', delta2_id, 'ERROR') + + # node2 is now master + node2.promote() + node2.safe_psql('postgres', 'CHECKPOINT') + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t3 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + + # node1 is now replica + node1.cleanup() + # kludge "backup_id=delta1_id" + self.restore_node( + backup_dir, 'node', node1, node1.data_dir, + backup_id=delta1_id, + options=[ + '--recovery-target-timeline=2', + '--recovery-target=latest']) + + # Settings for Replica + self.set_replica(node2, node1) + self.set_auto_conf(node1, {'port': node1.port}) + self.set_archiving(backup_dir, 'node', node1, replica=True) + + node1.slow_start(replica=True) + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t4 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,30) i') + self.wait_until_replica_catch_with_master(node2, node1) + + # node1 is back to be a master + node1.promote() + node1.safe_psql('postgres', 'CHECKPOINT') + + # delta backup on timeline 3 + self.backup_node( + backup_dir, 'node', node1, node1.data_dir, 'delta', + options=['--archive-timeout=60']) + + pgdata = self.pgdata_content(node1.data_dir) + + node1.cleanup() + self.restore_node(backup_dir, 'node', node1, node1.data_dir) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_replica_promote_archive_page(self): + """ + t3 /---P3--> + t2 /-------> + t1 --F---P1--P2-- + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'archive_timeout': '30s', + 'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + self.set_archiving(backup_dir, 'node', node1) + self.set_config( + backup_dir, 'node', options=['--archive-timeout=60s']) + + node1.slow_start() + + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + + # Create replica + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + self.restore_node(backup_dir, 'node', node2, node2.data_dir) + + # Settings for Replica + self.set_replica(node1, node2) + self.set_auto_conf(node2, {'port': node2.port}) + self.set_archiving(backup_dir, 'node', node2, replica=True) + + node2.slow_start(replica=True) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + # page backup on replica on timeline 1 + page1_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, + 'page', options=['--stream']) + + # page backup on replica on timeline 1 + page2_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, 'page') + + self.change_backup_status( + backup_dir, 'node', page2_id, 'ERROR') + + # node2 is now master + node2.promote() + node2.safe_psql('postgres', 'CHECKPOINT') + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t3 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + + # node1 is now replica + node1.cleanup() + # kludge "backup_id=page1_id" + self.restore_node( + backup_dir, 'node', node1, node1.data_dir, + backup_id=page1_id, + options=[ + '--recovery-target-timeline=2', + '--recovery-target=latest']) + + # Settings for Replica + self.set_replica(node2, node1) + self.set_auto_conf(node1, {'port': node1.port}) + self.set_archiving(backup_dir, 'node', node1, replica=True) + + node1.slow_start(replica=True) + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t4 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,30) i') + self.wait_until_replica_catch_with_master(node2, node1) + + # node1 is back to be a master + node1.promote() + node1.safe_psql('postgres', 'CHECKPOINT') + + # delta3_id = self.backup_node( + # backup_dir, 'node', node2, node2.data_dir, 'delta') + # page backup on timeline 3 + page3_id = self.backup_node( + backup_dir, 'node', node1, node1.data_dir, 'page', + options=['--archive-timeout=60']) + + pgdata = self.pgdata_content(node1.data_dir) + + node1.cleanup() + self.restore_node(backup_dir, 'node', node1, node1.data_dir) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_choosing(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + + full_id = self.backup_node( + backup_dir, 'replica', + replica, options=['--stream']) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + replica.promote() + replica.safe_psql('postgres', 'CHECKPOINT') + + # failing, because without archving, it is impossible to + # take multi-timeline backup. + try: + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of timeline switch " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Cannot find valid backup on previous timelines, ' + 'WAL archive is not available' in e.message and + 'ERROR: Create new full backup before an incremental one' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_instance_from_the_past(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=10) + self.backup_node(backup_dir, 'node', node, options=['--stream']) + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=full_id) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance is from the past " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Current START LSN' in e.message and + 'is lower than START LSN' in e.message and + 'It may indicate that we are trying to backup ' + 'PostgreSQL instance from the past' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) diff --git a/tests/retention.py b/tests/retention.py index ef17d3e04..4885187f9 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1712,10 +1712,9 @@ def test_wal_purge_victim(self): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -2675,7 +2674,7 @@ def test_wal_depth_2(self): self.assertIn( 'LOG: Archive backup {0} to stay consistent protect from ' 'purge WAL interval between 000000010000000000000004 ' - 'and 000000010000000000000004 on timeline 1'.format(B1), output) + 'and 000000010000000000000005 on timeline 1'.format(B1), output) start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] self.assertIn( @@ -2684,13 +2683,13 @@ def test_wal_depth_2(self): self.assertIn( 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' - 'from purge WAL interval between 000000020000000000000005 and ' - '000000020000000000000008 on timeline 2', output) + 'from purge WAL interval between 000000020000000000000006 and ' + '000000020000000000000009 on timeline 2', output) self.assertIn( 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' 'from purge WAL interval between 000000010000000000000004 and ' - '000000010000000000000005 on timeline 1', output) + '000000010000000000000006 on timeline 1', output) show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) @@ -2745,19 +2744,19 @@ def test_wal_depth_2(self): self.assertEqual( show_tli1_after['lost-segments'][0]['begin-segno'], - '000000010000000000000006') + '000000010000000000000007') self.assertEqual( show_tli1_after['lost-segments'][0]['end-segno'], - '000000010000000000000009') + '00000001000000000000000A') self.assertEqual( show_tli2_after['lost-segments'][0]['begin-segno'], - '000000020000000000000009') + '00000002000000000000000A') self.assertEqual( show_tli2_after['lost-segments'][0]['end-segno'], - '000000020000000000000009') + '00000002000000000000000A') self.validate_pb(backup_dir, 'node') From 4d51c7f549bf5949739d3c2579dbd991e2131c22 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 3 Apr 2020 20:03:01 +0300 Subject: [PATCH 1197/2107] [Issue #143] review feedback implementation --- src/catalog.c | 17 +++++++++-------- src/parsexlog.c | 22 +++++++++++----------- src/pg_probackup.h | 5 +++++ src/restore.c | 4 ++-- src/validate.c | 6 +++--- tests/page.py | 2 +- tests/validate.py | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 972e78bfe..f53c9b38d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -615,7 +615,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current switch (scan_parent_chain(backup, &tmp_backup)) { /* broken chain */ - case 0: + case ChainIsBroken: invalid_backup_id = base36enc_dup(tmp_backup->parent_backup); elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", @@ -624,7 +624,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current continue; /* chain is intact, but at least one parent is invalid */ - case 1: + case ChainIsInvalid: invalid_backup_id = base36enc_dup(tmp_backup->start_time); elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", @@ -633,7 +633,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current continue; /* chain is ok */ - case 2: + case ChainIsOk: /* Yes, we could call is_parent() earlier - after choosing the ancestor, * but this way we have an opportunity to detect and report all possible * anomalies. @@ -755,13 +755,14 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, /* Optimistically, look on current timeline for valid incremental backup, child of ancestor */ if (my_tlinfo->backups) { + /* backups are sorted in descending order and we need latest valid */ for (i = 0; i < parray_num(my_tlinfo->backups); i++) { pgBackup *tmp_backup = NULL; pgBackup *backup = (pgBackup *) parray_get(my_tlinfo->backups, i); /* found suitable parent */ - if (scan_parent_chain(backup, &tmp_backup) == 2 && + if (scan_parent_chain(backup, &tmp_backup) == ChainIsOk && is_parent(ancestor_backup->start_time, backup, false)) return backup; } @@ -786,7 +787,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, if (backup->stop_lsn > tmp_tlinfo->switchpoint) continue; - if (scan_parent_chain(backup, &tmp_backup) == 2 && + if (scan_parent_chain(backup, &tmp_backup) == ChainIsOk && is_parent(ancestor_backup->start_time, backup, true)) return backup; } @@ -2382,18 +2383,18 @@ scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) { /* Set oldest child backup in chain */ *result_backup = target_backup; - return 0; + return ChainIsBroken; } /* chain is ok, but some backups are invalid */ if (invalid_backup) { *result_backup = invalid_backup; - return 1; + return ChainIsInvalid; } *result_backup = target_backup; - return 2; + return ChainIsOk; } /* diff --git a/src/parsexlog.c b/src/parsexlog.c index cda3bf33a..11f2725ec 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -859,23 +859,23 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, char xlogfname[MAXFNAMELEN]; char partial_file[MAXPGPATH]; - GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, - wal_seg_size); - snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, - xlogfname); - snprintf(partial_file, MAXPGPATH, "%s/%s.partial", wal_archivedir, - xlogfname); - snprintf(reader_data->gz_xlogpath, sizeof(reader_data->gz_xlogpath), - "%s.gz", reader_data->xlogpath); + GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); + + snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, xlogfname); + snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath); + + /* We fall back to using .partial segment in case if we are running + * multi-timeline incremental backup right after standby promotion. + * TODO: it should be explicitly enabled. + */ + snprintf(partial_file, MAXPGPATH, "%s.partial", reader_data->xlogpath); /* If segment do not exists, but the same * segment with '.partial' suffix does, use it instead */ if (!fileExists(reader_data->xlogpath, FIO_BACKUP_HOST) && fileExists(partial_file, FIO_BACKUP_HOST)) { - snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s.partial", wal_archivedir, - xlogfname); - strncpy(reader_data->xlogpath, partial_file, MAXPGPATH); + snprintf(reader_data->xlogpath, MAXPGPATH, "%s", partial_file); } if (fileExists(reader_data->xlogpath, FIO_BACKUP_HOST)) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 47e0f46a9..a845cd987 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -800,6 +800,11 @@ extern int pgBackupCompareIdEqual(const void *l, const void *r); extern pgBackup* find_parent_full_backup(pgBackup *current_backup); extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); +/* return codes for scan_parent_chain */ +#define ChainIsBroken 0 +#define ChainIsInvalid 1 +#define ChainIsOk 2 + extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern bool is_prolific(parray *backup_list, pgBackup *target_backup); extern bool in_backup_list(parray *backup_list, pgBackup *target_backup); diff --git a/src/restore.c b/src/restore.c index f13b4c8cf..cfc5aacae 100644 --- a/src/restore.c +++ b/src/restore.c @@ -251,7 +251,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, result = scan_parent_chain(dest_backup, &tmp_backup); - if (result == 0) + if (result == ChainIsBroken) { /* chain is broken, determine missing backup ID * and orphinize all his descendants @@ -290,7 +290,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* No point in doing futher */ elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time)); } - else if (result == 1) + else if (result == ChainIsInvalid) { /* chain is intact, but at least one parent is invalid */ set_orphan_status(backups, tmp_backup); diff --git a/src/validate.c b/src/validate.c index 1129f0481..209cf8f4d 100644 --- a/src/validate.c +++ b/src/validate.c @@ -479,7 +479,7 @@ do_validate_instance(void) result = scan_parent_chain(current_backup, &tmp_backup); /* chain is broken */ - if (result == 0) + if (result == ChainIsBroken) { char *parent_backup_id; /* determine missing backup ID */ @@ -505,7 +505,7 @@ do_validate_instance(void) continue; } /* chain is whole, but at least one parent is invalid */ - else if (result == 1) + else if (result == ChainIsInvalid) { /* Oldest corrupt backup has a chance for revalidation */ if (current_backup->start_time != tmp_backup->start_time) @@ -630,7 +630,7 @@ do_validate_instance(void) */ result = scan_parent_chain(backup, &tmp_backup); - if (result == 1) + if (result == ChainIsInvalid) { /* revalidation make sense only if oldest invalid backup is current_backup */ diff --git a/tests/page.py b/tests/page.py index 68dd266c7..c38f98a4a 100644 --- a/tests/page.py +++ b/tests/page.py @@ -819,7 +819,7 @@ def test_page_backup_with_corrupted_wal_segment(self): self.backup_node(backup_dir, 'node', node) # make some wals - node.pgbench_init(scale=4) + node.pgbench_init(scale=10) # delete last wal segment wals_dir = os.path.join(backup_dir, 'wal', 'node') diff --git a/tests/validate.py b/tests/validate.py index 6b76e64c1..1dfdfe9e8 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1786,7 +1786,7 @@ def test_pgpro561(self): self.assertTrue( 'LOG: archive command failed with exit code 1' in log_content and 'DETAIL: The failed archive command was:' in log_content and - 'INFO: pg_probackup archive-push from' in log_content, + 'WAL file already exists in archive with different checksum' in log_content, 'Expecting error messages about failed archive_command' ) self.assertFalse( From accfc34e710f5b7741159157acd8cafb94e15ce8 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Sun, 5 Apr 2020 00:20:43 +0300 Subject: [PATCH 1198/2107] Added support --dry-run for --status option of delete command. Added check descendant status for deleted backups. --- src/delete.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/src/delete.c b/src/delete.c index 6fc26ab65..c52b2b7aa 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1021,16 +1021,57 @@ do_delete_instance(void) return 0; } +/* checks that backup childs has status like parent */ +bool checkChilds(parray *backup_list, time_t backup_id) +{ + int i; + pgBackup *target_backup = NULL; + pgBackup *backup; + /* search target bakcup */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *)parray_get(backup_list, i); + if (backup->start_time == backup_id) + { + target_backup = backup; + break; + } + } + if (target_backup == NULL) return false; + + /* check childs */ + for (i = 0; i < parray_num(backup_list); i++) + { + backup = (pgBackup *)parray_get(backup_list, i); + /* check if backup is descendant of delete target */ + if (is_parent(target_backup->start_time, backup, false)) + { + if (backup->status != target_backup->status){ + elog(INFO, "Skip deleting the backup %s because the backup has children with a different status", + base36enc(target_backup->start_time)); + return false; + } + /* recursive call */ + if (!checkChilds(backup_list, backup->start_time)) return false; + } + } + return true; +} + /* Delete all backups of given status in instance */ void do_delete_status(InstanceConfig *instance_config, const char *status) { - parray *backup_list; + parray *backup_list, *delete_list;; int i; const char *pretty_status; - int n_deleted = 0; + int n_deleted = 0, n_found = 0; + size_t size_to_delete = 0; + char size_to_delete_pretty[20]; + pgBackup *backup; BackupStatus status_for_delete = str2status(status); + delete_list = parray_new(); if (status_for_delete == BACKUP_STATUS_INVALID) elog(ERROR, "Unknown value for '--status' option: '%s'", status); @@ -1052,28 +1093,63 @@ do_delete_status(InstanceConfig *instance_config, const char *status) elog(INFO, "Deleting all backups with status '%s'", pretty_status); - /* Delete all backups with specified status */ + /* Selects backups for deleting to delete_list array. Will delete all backups with specified status */ for (i = 0; i < parray_num(backup_list); i++) { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + backup = (pgBackup *) parray_get(backup_list, i); if (backup->status == status_for_delete) { - lock_backup(backup); - delete_backup_files(backup); + + n_found++; + if (!checkChilds(backup_list, backup->start_time)) continue; + + elog(dry_run ? INFO: LOG, "Backup %s %s be deleted", + base36enc(backup->start_time), dry_run ? "can" : "will"); + size_to_delete += backup->data_bytes; + if (backup->stream) + size_to_delete += backup->wal_bytes; + if (!dry_run){ + parray_append(delete_list, backup); + } n_deleted++; + } } - if (n_deleted > 0) + /* Inform about data size to free */ + if (size_to_delete >= 0) + { + pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); + elog(INFO, "Resident data size to free by delete of %i backups: %s", + n_deleted, size_to_delete_pretty); + } + + /* delete selected backups */ + if (!dry_run) + { + for (i = 0; i < parray_num(delete_list); i++) + { + backup = (pgBackup *)parray_get(delete_list, i); + if (lock_backup(backup)) + { + delete_backup_files(backup); + } + else n_deleted--; + } elog(INFO, "Successfully deleted %i %s with status '%s' from instance '%s'", n_deleted, n_deleted == 1 ? "backup" : "backups", pretty_status, instance_config->name); - else + + } + + + if (n_found == 0) elog(WARNING, "Instance '%s' has no backups with status '%s'", instance_config->name, pretty_status); /* Cleanup */ + parray_free(delete_list); parray_walk(backup_list, pgBackupFree); parray_free(backup_list); } From ad419565cdca8affebf151a6a42d2c29ab75b67a Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Sun, 5 Apr 2020 15:32:30 +0300 Subject: [PATCH 1199/2107] Added deletion all childs with any status when deleting backup with specific status (--status) --- src/delete.c | 93 ++++++++++++++++++++++++++-------------------------- src/help.c | 5 +-- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/delete.c b/src/delete.c index c52b2b7aa..004fd239b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1021,41 +1021,36 @@ do_delete_instance(void) return 0; } -/* checks that backup childs has status like parent */ -bool checkChilds(parray *backup_list, time_t backup_id) +/* checks that parray contains element */ +bool parray_contains(parray *array, void *elem) { int i; - pgBackup *target_backup = NULL; - pgBackup *backup; - /* search target bakcup */ - for (i = 0; i < parray_num(backup_list); i++) + for (i = 0; i < parray_num(array); i++) { - backup = (pgBackup *)parray_get(backup_list, i); - if (backup->start_time == backup_id) - { - target_backup = backup; - break; - } + if (parray_get(array, i) == elem) return true; } - if (target_backup == NULL) return false; + return false; +} + - /* check childs */ +void append_childs(parray *backup_list, pgBackup *target_backup, parray *delete_list) +{ + int i; + pgBackup *backup; for (i = 0; i < parray_num(backup_list); i++) { backup = (pgBackup *)parray_get(backup_list, i); + if (backup == target_backup) continue; /* check if backup is descendant of delete target */ if (is_parent(target_backup->start_time, backup, false)) { - if (backup->status != target_backup->status){ - elog(INFO, "Skip deleting the backup %s because the backup has children with a different status", - base36enc(target_backup->start_time)); - return false; - } + if (!parray_contains(delete_list, backup)) + parray_append(delete_list, backup); /* recursive call */ - if (!checkChilds(backup_list, backup->start_time)) return false; + append_childs(backup_list, backup, delete_list); } } - return true; + } /* Delete all backups of given status in instance */ @@ -1093,30 +1088,42 @@ do_delete_status(InstanceConfig *instance_config, const char *status) elog(INFO, "Deleting all backups with status '%s'", pretty_status); - /* Selects backups for deleting to delete_list array. Will delete all backups with specified status */ + /* Selects backups for deleting to delete_list array. Will delete all backups with specified status and childs*/ for (i = 0; i < parray_num(backup_list); i++) { backup = (pgBackup *) parray_get(backup_list, i); if (backup->status == status_for_delete) { - n_found++; - if (!checkChilds(backup_list, backup->start_time)) continue; + if (parray_contains(delete_list, backup)) continue; + parray_append(delete_list, backup); + append_childs(backup_list, backup, delete_list); + } + } + /* delete and calculate free size from delete_list */ + for (i = 0; i < parray_num(delete_list); i++) + { + backup = (pgBackup *)parray_get(delete_list, i); + elog(dry_run ? INFO : LOG, "Backup %s with status %s %s be deleted", + base36enc(backup->start_time), status2str(backup->status), dry_run ? "can" : "will"); - elog(dry_run ? INFO: LOG, "Backup %s %s be deleted", - base36enc(backup->start_time), dry_run ? "can" : "will"); - size_to_delete += backup->data_bytes; - if (backup->stream) - size_to_delete += backup->wal_bytes; - if (!dry_run){ - parray_append(delete_list, backup); - } + size_to_delete += backup->data_bytes; + if (backup->stream) + size_to_delete += backup->wal_bytes; + + + if (!dry_run && lock_backup(backup)) + { + if (interrupted) + elog(ERROR, "interrupted during delete backup"); + + delete_backup_files(backup); n_deleted++; } - } + } /* Inform about data size to free */ if (size_to_delete >= 0) { @@ -1127,27 +1134,19 @@ do_delete_status(InstanceConfig *instance_config, const char *status) /* delete selected backups */ if (!dry_run) - { - for (i = 0; i < parray_num(delete_list); i++) - { - backup = (pgBackup *)parray_get(delete_list, i); - if (lock_backup(backup)) - { - delete_backup_files(backup); - } - else n_deleted--; - } - elog(INFO, "Successfully deleted %i %s with status '%s' from instance '%s'", + elog(INFO, "Successfully deleted %i %s from instance '%s'", n_deleted, n_deleted == 1 ? "backup" : "backups", - pretty_status, instance_config->name); - - } + instance_config->name); if (n_found == 0) elog(WARNING, "Instance '%s' has no backups with status '%s'", instance_config->name, pretty_status); + /* Clean WAL segments */ + if (delete_wal) + do_retention_wal(dry_run); + /* Cleanup */ parray_free(delete_list); parray_walk(backup_list, pgBackupFree); diff --git a/src/help.c b/src/help.c index 2bc50d512..065f399d0 100644 --- a/src/help.c +++ b/src/help.c @@ -193,7 +193,7 @@ help_pg_probackup(void) printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--delete-wal] [-i backup-id | --delete-expired | --merge-expired]\n")); + printf(_(" [--delete-wal] [-i backup-id | --delete-expired | --merge-expired | --status= ]\n")); printf(_(" [--dry-run]\n")); printf(_(" [--help]\n")); @@ -635,7 +635,8 @@ help_delete(void) printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); - + printf(_(" --status=backups_status delete all backups with specific status\n")); + printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); From 510c9477538bed76cb748241a6035d9b8875e791 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 20:59:10 +1000 Subject: [PATCH 1200/2107] compare parent backup version with current binary version while searching for valid ancestor --- src/backup.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backup.c b/src/backup.c index bcc74ed30..4b511ca42 100644 --- a/src/backup.c +++ b/src/backup.c @@ -218,6 +218,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (prev_backup) { + if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); + char prev_backup_filelist_path[MAXPGPATH]; elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); From 3d836b2215ecd95bb8f29683da2d605e91d2f186 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 20:59:56 +1000 Subject: [PATCH 1201/2107] add probackup_version attribute to tests --- tests/helpers/ptrack_helpers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d7ddfa037..0f0134908 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -246,6 +246,18 @@ def __init__(self, *args, **kwargs): print('pg_probackup binary is not found') exit(1) + self.probackup_version = None + + try: + self.probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8')) + + self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) + if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( From cc8ba20965a2d95393930b3fefecf031f414e254 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 21:00:12 +1000 Subject: [PATCH 1202/2107] add test test_parent_backup_made_by_newer_version --- tests/backup.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index ef418c970..fc0793331 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2709,3 +2709,56 @@ def test_issue_132_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_backup_made_by_newer_version(self): + """incremental backup with parent made by newer version""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + control_file = os.path.join( + backup_dir, "backups", "node", backup_id, + "backup.control") + + version = self.probackup_version + fake_new_version = str(int(version.split('.')[0]) + 1) + '.0.0' + + with open(control_file, 'r') as f: + data = f.read(); + + data = data.replace(version, fake_new_version) + + with open(control_file, 'w') as f: + f.write(data); + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "if parent made by newer version.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) From 615dff5229553d9a4f4f49922bb52c742708ba11 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 1 Apr 2020 15:48:51 +0300 Subject: [PATCH 1203/2107] Travis CI integration --- .gitignore | 11 ++++++ .travis.yml | 45 ++++++++++++++++++++-- README.md | 2 + travis/Dockerfile.in | 24 ++++++++++++ travis/docker-compose.yml | 2 + travis/make_dockerfile.sh | 25 ++++++++++++ travis/run_tests.sh | 80 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 travis/Dockerfile.in create mode 100644 travis/docker-compose.yml create mode 100755 travis/make_dockerfile.sh create mode 100755 travis/run_tests.sh diff --git a/.gitignore b/.gitignore index 2f492c3c0..f537d80ae 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,14 @@ # Doc files /doc/*html + +# Docker files +/docker-compose.yml +/Dockerfile +/Dockerfile.in +/run_tests.sh +/make_dockerfile.sh +/backup_restore.sh + +# Misc +.python-version diff --git a/.travis.yml b/.travis.yml index 35b49ec5b..fff56a8fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,46 @@ -sudo: required +os: linux + +dist: bionic + +language: c services: -- docker + - docker + +before_install: + - cp travis/* . + +install: + - ./make_dockerfile.sh + - docker-compose build script: -- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh + - docker-compose run tests + # - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + # - docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh + +notifications: + email: + on_success: change + on_failure: always + +# Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON +env: + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE + - PG_VERSION=10 PG_BRANCH=REL_10_STABLE + - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE + +jobs: + allow_failures: + - if: env(MODE) IN (archive, backup, delta, locking, merge, page, replica, retention, restore) diff --git a/README.md b/README.md index 9ef122a24..ecd0560b9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) + # pg_probackup `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in new file mode 100644 index 000000000..68f5ffe22 --- /dev/null +++ b/travis/Dockerfile.in @@ -0,0 +1,24 @@ +FROM ololobus/postgres-dev:stretch + +USER root +RUN apt-get update +RUN apt-get -yq install python python-pip python-virtualenv + +# Environment +ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} +ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin + +# Make directories +RUN mkdir -p /pg/testdir + +COPY run_tests.sh /run.sh +RUN chmod 755 /run.sh + +COPY . /pg/testdir +WORKDIR /pg/testdir + +# Grant privileges +RUN chown -R postgres:postgres /pg/testdir + +USER postgres +ENTRYPOINT MODE=${MODE} /run.sh diff --git a/travis/docker-compose.yml b/travis/docker-compose.yml new file mode 100644 index 000000000..471ab779f --- /dev/null +++ b/travis/docker-compose.yml @@ -0,0 +1,2 @@ +tests: + build: . diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh new file mode 100755 index 000000000..3e6938bd9 --- /dev/null +++ b/travis/make_dockerfile.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh + +if [ -z ${PG_VERSION+x} ]; then + echo PG_VERSION is not set! + exit 1 +fi + +if [ -z ${PG_BRANCH+x} ]; then + echo PG_BRANCH is not set! + exit 1 +fi + +if [ -z ${MODE+x} ]; then + MODE=basic +fi + +echo PG_VERSION=${PG_VERSION} +echo PG_BRANCH=${PG_BRANCH} +echo MODE=${MODE} + +sed \ + -e 's/${PG_VERSION}/'${PG_VERSION}/g \ + -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ + -e 's/${MODE}/'${MODE}/g \ +Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh new file mode 100755 index 000000000..95cb646c5 --- /dev/null +++ b/travis/run_tests.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# +# Copyright (c) 2019-2020, Postgres Professional +# + + +PG_SRC=$PWD/postgres + +# # Here PG_VERSION is provided by postgres:X-alpine docker image +# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2 +# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c - + +# mkdir $PG_SRC + +# tar \ +# --extract \ +# --file postgresql.tar.bz2 \ +# --directory $PG_SRC \ +# --strip-components 1 + +# Clone Postgres +echo "############### Getting Postgres sources:" +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 + +# Compile and install Postgres +echo "############### Compiling Postgres:" +cd postgres # Go to postgres dir +./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests +make -s -j$(nproc) install +make -s -j$(nproc) -C contrib/ install + +# Override default Postgres instance +export PATH=$PGHOME/bin:$PATH +export LD_LIBRARY_PATH=$PGHOME/lib +export PG_CONFIG=$(which pg_config) + +# Get amcheck if missing +if [ ! -d "contrib/amcheck" ]; then + echo "############### Getting missing amcheck:" + git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck + make USE_PGXS=1 -C contrib/amcheck install +fi + +# Get back to testdir +cd .. + +# Show pg_config path (just in case) +echo "############### pg_config path:" +which pg_config + +# Show pg_config just in case +echo "############### pg_config:" +pg_config + +# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) +echo "############### Compiling and installing pg_probackup:" +# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install +make USE_PGXS=1 top_srcdir=$PG_SRC install + +# Setup python environment +echo "############### Setting up python env:" +virtualenv pyenv +source pyenv/bin/activate +pip install testgres==1.8.2 + +echo "############### Testing:" +if [ "$MODE" = "basic" ]; then + export PG_PROBACKUP_TEST_BASIC=ON + python -m unittest -v tests + python -m unittest -v tests.init +else + python -m unittest -v tests.$MODE +fi + +# Generate *.gcov files +# gcov src/*.c src/*.h + +# Send coverage stats to Codecov +# bash <(curl -s https://codecov.io/bash) From a0cb12cec397c042ea83a8fa4f9a2e692c779a8d Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 6 Apr 2020 18:11:02 +0300 Subject: [PATCH 1204/2107] Run travis tests for 9.5 as well --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fff56a8fc..8428ead25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,7 @@ env: - PG_VERSION=11 PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE + - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE jobs: allow_failures: From 597021d4b319a07cc8aa84f9b167544ae8ace1f2 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 6 Apr 2020 18:57:04 +0300 Subject: [PATCH 1205/2107] Remove page backup tests from allow_failures list on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8428ead25..fc7ecc059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,4 +44,4 @@ env: jobs: allow_failures: - - if: env(MODE) IN (archive, backup, delta, locking, merge, page, replica, retention, restore) + - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) From 8405c4e57e07db766f046347858e50b3f3ceb7f4 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Tue, 7 Apr 2020 16:37:12 +0300 Subject: [PATCH 1206/2107] DOC: review retention-related pbk docs - PBCKP-95 --- doc/pgprobackup.xml | 159 +++++++++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 67 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 02c144744..0a328da9b 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1475,7 +1475,7 @@ pg_probackup checkdb [-B backup_dir [--instance pg_probackup to determine the required connection options. However, if -B and - --instance options are ommitted, you have to provide + --instance options are omitted, you have to provide connection options and data_dir via environment variables or command-line options. @@ -2364,7 +2364,8 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod expire-time — the point in time - when a pinned backup can be removed by retention purge. + when a pinned backup can be removed in accordance with retention + policy. This attribute is only available for pinned backups. @@ -2808,17 +2809,19 @@ pg_probackup show -B backup_dir [--instance Configuring Retention Policy - With pg_probackup, you can set retention policies for backups - and WAL archive. All policies can be combined together in any - way. + With pg_probackup, you can configure + retention policy to remove redundant backups, clean up unneeded + WAL files, as well as pin specific backups to ensure they are + kept for the specified time, as explained in the sections below. + All these actions can be combined together in any way. + - Backup Retention Policy + Removing Redundant Backups By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, - you can configure retention policy and periodically clean up - redundant backup copies accordingly. + you can configure retention policy to remove redundant backup copies. To configure retention policy, set one or more of the @@ -2841,56 +2844,51 @@ pg_probackup show -B backup_dir [--instance the number of days from the current moment. For example, if retention-window=7, pg_probackup must - delete all backup copies that are older than seven days, with - all the corresponding WAL files. + keep at least one backup copy that is older than seven days, with + all the corresponding WAL files, and all the backups that follow. If both and - options are set, - pg_probackup keeps backup copies that satisfy at least one - condition. For example, if you set - --retention-redundancy=2 and - --retention-window=7, pg_probackup purges - the backup catalog to keep only two full backup copies and all - backups that are newer than seven days: + options are set, both these + conditions have to be taken into account when purging the backup + catalog. For example, if you set --retention-redundancy=2 + and --retention-window=7, + pg_probackup has to keep two full backup + copies, as well as all the backups required to ensure recoverability + for the last seven days: pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 + + + To clean up the backup catalog in accordance with retention policy, + you have to run the command with + retention flags, as shown + below, or use the command with + these flags to process the outdated backup copies right when the new + backup is created. + + - To clean up the backup catalog in accordance with retention - policy, run: + For example, to remove all backup copies that no longer satisfy the + defined retention policy, run the following command with the + --delete-expired flag: pg_probackup delete -B backup_dir --instance instance_name --delete-expired - - pg_probackup deletes all backup copies that do not conform to - the defined retention policy. - If you would like to also remove the WAL files that are no - longer required for any of the backups, add the + longer required for any of the backups, you should also specify the flag: pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal - - - Alternatively, you can use the - , - , - flags and the - and - options together - with the command to - remove and merge the outdated backup copies once the new - backup is created. - - + - You can set or override the current retention policy by + You can also set or override the current retention policy by specifying and options directly when running delete or backup @@ -2911,6 +2909,7 @@ pg_probackup delete -B backup_dir --instance or commands. + Suppose you have backed up the node instance in the backup_dir directory, @@ -2963,9 +2962,10 @@ BACKUP INSTANCE 'node' The Time field for the merged backup displays the time required for the merge. + - Backup Pinning + Pinning Backups If you need to keep certain backups longer than the established retention policy allows, you can pin them @@ -3004,8 +3004,8 @@ pg_probackup show -B backup_dir --instance - If the backup is pinned, the expire-time - attribute displays its expiration time: + If the backup is pinned, it has the expire-time + attribute that displays its expiration time: ... recovery-time = '2017-05-16 12:57:31' @@ -3015,34 +3015,65 @@ data-bytes = 22288792 - Only pinned backups have the expire-time - attribute in the backup metadata. + You can unpin the backup by setting the option to zero: + +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 + + A pinned incremental backup implicitly pins all - its parent backups. + its parent backups. If you unpin such a backup later, + its implicitly pinned parents will also be automatically unpinned. - - You can unpin the backup by setting the - option to zero using the - command. For example: - - -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 - - WAL Archive Retention Policy + Configuring WAL Archive Retention Policy + + When continuous + WAL archiving is enabled, archived WAL segments can take a lot + of disk space. Even if you delete old backup copies from time to time, + the --delete-wal flag can + purge only those WAL segments that do not apply to any of the + remaining backups in the backup catalog. However, if point-in-time + recovery is critical only for the most recent backups, you can + configure WAL archive retention policy to keep WAL archive of + limited depth and win back some more disk space. + + + + To configure WAL archive retention policy, you have to run the + command with the + --wal-depth option that specifies the number + of backups that can be used for PITR. + This setting applies to all the timelines, so you should be able to perform + PITR for the same number of backups on each timeline, if available. + Pinned backups are + not included into this count: if one of the latest backups + is pinned, pg_probackup ensures that + PITR is possible for one extra backup. + + - By default, pg_probackup purges - only redundant WAL segments that cannot be applied to any of the - backups in the backup catalog. To save disk space, - you can configure WAL archive retention policy, which allows to - keep WAL of limited depth measured in backups per timeline. + To remove WAL segments that do not satisfy the defined WAL archive + retention policy, you simply have to run the + or command with the --delete-wal + flag. For archive backups, WAL segments between Start LSN + and Stop LSN are always kept intact, so such backups + remain valid regardless of the --wal-depth setting + and can still be restored, if required. + + + You can also use the option + with the and + commands to override the previously defined WAL archive retention + policy and purge old WAL segments on the fly. + + Suppose you have backed up the node instance in the backup_dir directory and @@ -3096,8 +3127,8 @@ ARCHIVE INSTANCE 'node' If you would like, for example, to keep only those WAL - segments that can be applied to the last valid backup, use the - option: + segments that can be applied to the latest valid backup, set the + option to 1: pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 @@ -3123,12 +3154,6 @@ ARCHIVE INSTANCE 'node' =============================================================================================================================== 1 0 0/0 000000010000000000000048 000000010000000000000049 1 72kB 228.00 7 OK - - - Pinned backups are - ignored for the purpose of WAL Archive Retention Policy fulfilment. - - @@ -4130,7 +4155,7 @@ pg_probackup archive-get -B backup_dir --instance immediate value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the - / option is omitted. + / option is omitted. This is the default behavior for STREAM backups. From 3b2d152a9cf692779f84c92afb3ec144524bf120 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Tue, 7 Apr 2020 16:57:45 +0300 Subject: [PATCH 1207/2107] DOC: fix link format for consistency --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 0a328da9b..f4ce9f4d3 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3686,7 +3686,7 @@ pg_probackup restore -B backup_dir --instance Sets the - primary_slot_name + primary_slot_name parameter to the specified value. This option will be ignored unless the flag if specified. From 84ad9d422e0aec8ef22da110626ced536a130afa Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Wed, 8 Apr 2020 16:53:01 +0300 Subject: [PATCH 1208/2107] DOC: fix grammar in new content --- doc/pgprobackup.xml | 83 +++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index f4ce9f4d3..431a60fe7 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2239,7 +2239,7 @@ BACKUP INSTANCE 'node' MERGED — the backup data files were successfully merged, but its metadata is in the process - of been updated. Only full backup can have this status. + of being updated. Only full backups can have this status. @@ -3169,16 +3169,16 @@ ARCHIVE INSTANCE 'node' pg_probackup merge -B backup_dir --instance instance_name -i backup_id - This command merges the specified incremental backup to its - parent full backup, together with all incremental backups - between them. If the specified backup ID belong to the full backup, - then it will be merged with the closest incremental backup. - Once the merge is complete, the incremental - backups are removed as redundant. Thus, the merge operation is - virtually equivalent to retaking a full backup and removing all - the outdated backups, but it allows to save much time, - especially for large data volumes, as well as I/O and network traffic - if you are using pg_probackup in the + This command merges backups that belong to a common incremental backup + chain. If you specify a full backup, it will be merged with its first + incremental backup. If you specify an incremental backup, it will be + merged to its parent full backup, together with all incremental backups + between them. Once the merge is complete, the full backup takes in all + the merged data, and the incremental backups are removed as redundant. + Thus, the merge operation is virtually equivalent to retaking a full + backup and removing all the outdated backups, but it allows to save much + time, especially for large data volumes, as well as I/O and network + traffic if you are using pg_probackup in the remote mode. @@ -3192,8 +3192,10 @@ pg_probackup show -B backup_dir --instance If the merge is still in progress, the backup status is - displayed as MERGING or, at the final stage, - MERGED. The merge is idempotent, so you can + displayed as MERGING. For full backups, + it can also be shown as MERGED while the + metadata is being updated at the final stage of the merge. + The merge is idempotent, so you can restart the merge if it was interrupted. @@ -3591,9 +3593,11 @@ pg_probackup backup -B backup_dir -b bac Do not sync backed up files to disk. You can use this flag to speed - up backup process. Using this flag can result in data + up the backup process. Using this flag can result in data corruption in case of operating system or hardware crash. - Corruption can be detected by backup validation. + If you use this option, it is recommended to run the + command once the backup is complete + to detect possible issues. @@ -3627,7 +3631,7 @@ pg_probackup restore -B backup_dir --instance cmdline] [--primary-conninfo=primary_conninfo] -[-S | --primary-slot-name=slotname] +[-S | --primary-slot-name=slot_name] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -3672,7 +3676,7 @@ pg_probackup restore -B backup_dir --instance primary_conninfo parameter to the specified value. - This option will be ignored unless the flag if specified. + This option will be ignored unless the flag is specified. Example: --primary-conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' @@ -3688,7 +3692,7 @@ pg_probackup restore -B backup_dir --instance primary_slot_name parameter to the specified value. - This option will be ignored unless the flag if specified. + This option will be ignored unless the flag is specified. @@ -3785,6 +3789,8 @@ pg_probackup restore -B backup_dir --instance + command again. @@ -3912,12 +3918,12 @@ pg_probackup merge -B backup_dir --instance logging_options] - Merges the specified incremental backup to its parent full - backup, together with all incremental backups between them, if - any. If the specified backup ID belong to the full backup, - then it will be merged with the closest incremental backup. - As a result, the full backup takes in all the merged - data, and the incremental backups are removed as redundant. + Merges backups that belong to a common incremental backup + chain. If you specify a full backup, it will be merged with its first + incremental backup. If you specify an incremental backup, it will be + merged to its parent full backup, together with all incremental backups + between them. Once the merge is complete, the full backup takes in all + the merged data, and the incremental backups are removed as redundant. For details, see the section @@ -3981,24 +3987,25 @@ pg_probackup archive-push -B backup_dir --instance --overwrite flag. - Every file is copied to a temporary file with the + Each file is copied to a temporary file with the .part suffix. If the temporary file already exists, pg_probackup will wait seconds before discarding it. After the copy is done, atomic rename is performed. This algorithm ensures that a failed archive-push will not stall continuous archiving and that concurrent archiving from - multiple sources into a single WAL archive have no risk of archive + multiple sources into a single WAL archive has no risk of archive corruption. - To speed up archiving, especially in remote mode, archive-push - can be run on multiple threads using option. - Files can also be copied in batches using option . + To speed up archiving, you can specify the option + to run archive-push on multiple threads. + If you provide the option, WAL files + will be copied in batches of the specified size. WAL segments copied to the archive are synced to disk unless - flag is used. + the flag is used. You can use archive-push in the @@ -4104,7 +4111,7 @@ pg_probackup archive-get -B backup_dir --instance Sets the number of parallel threads for backup, restore, merge, - validate, checkdb and + validate, checkdb, and archive-push processes. @@ -4766,11 +4773,11 @@ pg_probackup archive-get -B backup_dir --instance - + - Sets the maximum number of files to be copied into archive by signle - archive-push process. + Sets the maximum number of files that can be copied into the archive + by a single archive-push process. @@ -4779,7 +4786,9 @@ pg_probackup archive-get -B backup_dir --instance - Sets the timeout for considering existing .part file to be stale. By default pg_probackup waits 300 seconds. + Sets the timeout for considering existing .part + files to be stale. By default, pg_probackup + waits 300 seconds. @@ -4788,9 +4797,9 @@ pg_probackup archive-get -B backup_dir --instance - Do not rename status files in archive_status directory. + Do not rename status files in the archive_status directory. This option should be used only if archive_command - contain multiple commands. + contains multiple commands. From 96344e4be27c84e22a29a80cbfe33d285521d0c5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 8 Apr 2020 23:54:41 +0300 Subject: [PATCH 1209/2107] Readme: add reference to ptrack extension --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ecd0560b9..5ef0ff995 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* Postgres Pro Standard 9.6, 10, 11 -* Postgres Pro Enterprise 9.6, 10, 11 +* vanilla PostgreSQL 12 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 9.6, 10, 11, 12 +* Postgres Pro Enterprise 9.6, 10, 11, 12 ## Limitations From b5aee7df032c281e72a58698259ecffbbbc08b13 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 9 Apr 2020 01:22:45 +0300 Subject: [PATCH 1210/2107] tests: fixes for ptrack --- tests/ptrack.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index b9c86e312..98fa3ead3 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3,10 +3,10 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack from datetime import datetime, timedelta import subprocess -from testgres import QueryException +from testgres import QueryException, StartNodeException import shutil import sys -import time +from time import sleep from threading import Thread @@ -3848,7 +3848,7 @@ def test_ptrack_zero_changes(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): fname = self.id().split('.')[3] node = self.make_simple_node( @@ -4016,14 +4016,17 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) + ptrack_map = os.path.join(node.data_dir, 'global', 'ptrack.map') + ptrack_map_mmap = os.path.join(node.data_dir, 'global', 'ptrack.map.mmap') + # Let`s do index corruption. ptrack.map, ptrack.map.mmap - with open(os.path.join(node.data_dir, 'global', 'ptrack.map'), "rb+", 0) as f: + with open(ptrack_map, "rb+", 0) as f: f.seek(42) f.write(b"blablahblahs") f.flush() f.close - with open(os.path.join(node.data_dir, 'global', 'ptrack.map.mmap'), "rb+", 0) as f: + with open(ptrack_map_mmap, "rb+", 0) as f: f.seek(42) f.write(b"blablahblahs") f.flush() @@ -4031,13 +4034,97 @@ def test_corrupt_ptrack_map(self): # os.remove(os.path.join(node.logs_dir, node.pg_log_name)) + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack.map is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) + + self.set_auto_conf(node, {'ptrack_map_size': '0'}) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance ptrack is disabled" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + self.set_auto_conf(node, {'ptrack_map_size': '32'}) + node.slow_start() + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack map is from future" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + sleep(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) + node.cleanup() self.restore_node(backup_dir, 'node', node) From 9679cae18bc7f287c7624d360340485743d11c70 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 9 Apr 2020 12:05:55 +0300 Subject: [PATCH 1211/2107] tests: some more ptrack fixes --- tests/ptrack.py | 54 ++++++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 98fa3ead3..e805cacf3 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -210,46 +210,36 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - if self.ptrack: - fnames = [] - if node.major_version < 12: - fnames += [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' - ] - else: - # TODO why backup works without these grants ? -# fnames += [ -# 'pg_ptrack_get_pagemapset(pg_lsn)', -# 'pg_ptrack_control_lsn()', -# 'pg_ptrack_get_block(oid, oid, oid, bigint)' -# ] - - node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack") - - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - - node.safe_psql( - "backupdb", - "GRANT USAGE ON SCHEMA ptrack TO backup") + if node.major_version < 12: + fnames = [ + 'pg_catalog.oideq(oid, oid)', + 'pg_catalog.ptrack_version()', + 'pg_catalog.pg_ptrack_clear()', + 'pg_catalog.pg_ptrack_control_lsn()', + 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', + 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', + 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' + ] for fname in fnames: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) + else: + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack") node.safe_psql( "backupdb", - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") + + node.safe_psql( + "backupdb", + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") if ProbackupTest.enterprise: node.safe_psql( From acd5afad81b2b19fed69ac459d66a163c623badd Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Thu, 9 Apr 2020 16:28:30 +0300 Subject: [PATCH 1212/2107] DOC: fix commands in the how-to section --- doc/pgprobackup.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 431a60fe7..7d6ad109a 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -5252,7 +5252,7 @@ INFO: Backup PZ7YK2 completed Let's take a look at the backup catalog: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' +[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' BACKUP INSTANCE 'pg-11' ================================================================================================================================== @@ -5343,7 +5343,7 @@ remote-host = postgres_host Let's take a look at the backup catalog: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' +[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' ==================================================================================================================================== Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status From 439c27c7e7229d4cc7ea4efb9e4575856a6c2909 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 10 Apr 2020 16:04:54 +0300 Subject: [PATCH 1213/2107] [Issue #180] test coverage --- tests/archive.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index 95f5c178c..8d96dcb19 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1767,7 +1767,7 @@ def test_hexadecimal_timeline(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") # @unittest.expectedFailure def test_archiving_and_slots(self): """ @@ -1905,6 +1905,101 @@ def test_archive_push_sanity(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_partial_handling(self): + """check that archive-get delivers .partial and .gz.partial files""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, replica.data_dir, options=['-R']) + self.set_auto_conf(replica, {'port': replica.port}) + self.set_replica(node, replica) + + self.add_instance(backup_dir, 'replica', replica) + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + + replica.slow_start(replica=True) + + node.safe_psql('postgres', 'CHECKPOINT') + + if self.get_version(replica) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + cmdline = [ + pg_receivexlog_path, '-p', str(replica.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'replica')] + + if self.archive_compress and node.major_version >= 10: + cmdline += ['-Z', '1'] + + pg_receivexlog = self.run_binary(cmdline, asynchronous=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000000) i") + + # FULL + self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(1000000,2000000) i") + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'replica', node_restored, + node_restored.data_dir, options=['--recovery-target=latest', '--recovery-target-action=promote']) + self.set_auto_conf(node_restored, {'port': node_restored.port}) + self.set_auto_conf(node_restored, {'hot_standby': 'off'}) + + node_restored.slow_start() + + result = node.safe_psql( + "postgres", + "select sum(id) from t_heap") + + result_new = node_restored.safe_psql( + "postgres", + "select sum(id) from t_heap") + + self.assertEqual(result, result_new) + + # Clean after yourself + pg_receivexlog.kill() + self.del_test_dir(module_name, fname) + # TODO test with multiple not archived segments. # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. From 703d311b4b9faa6188be5a9bac09e164c927b3aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 17 Apr 2020 03:40:23 +0300 Subject: [PATCH 1214/2107] [Issue #185] archive-get refactoring: multi-threading, batching, prefetchin, retrying and handling of '.partial' files --- doc/pgprobackup.xml | 51 +- src/archive.c | 1001 ++++++++++++++++++++++++------- src/backup.c | 2 +- src/help.c | 12 +- src/parsexlog.c | 50 +- src/pg_probackup.c | 45 +- src/pg_probackup.h | 27 +- src/utils/file.c | 348 ++++++++++- src/utils/file.h | 2 + tests/archive.py | 344 ++++++++++- tests/helpers/ptrack_helpers.py | 30 +- 11 files changed, 1631 insertions(+), 281 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7d6ad109a..c6aca459b 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3957,7 +3957,7 @@ pg_probackup delete -B backup_dir --instance archive-push pg_probackup archive-push -B backup_dir --instance instance_name ---wal-file-name=wal_file_name +--wal-file-name=wal_file_name [--wal-file-path=wal_file_path] [--help] [--no-sync] [--compress] [--no-ready-rename] [--overwrite] [-j num_threads] [--batch-size=batch_size] [--archive-timeout=timeout] @@ -3973,12 +3973,10 @@ pg_probackup archive-push -B backup_dir --instance Refuse to push WAL segment segment_name into archive. Instance parameters - mismatch. For each WAL file moved to the backup catalog, you - will see the following message in the PostgreSQL log file: - pg_probackup archive-push completed successfully. + mismatch. - If the files to be copied already exist in the backup catalog, + If the files to be copied already exists in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns a successful execution code. Otherwise, archive-push @@ -4025,6 +4023,8 @@ pg_probackup archive-push -B backup_dir --instance archive-get pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name +[-j num_threads] [--batch-size=batch_size] +[--prefetch-dir=prefetch_dir_path] [--no-validate-wal] [--help] [remote_options] [logging_options] @@ -4035,6 +4035,17 @@ pg_probackup archive-get -B backup_dir --instance + + + To speed up recovery, you can specify the option + to run archive-get on multiple threads. + If you provide the option, WAL segments + will be copied in batches of the specified size. + + + + For details, see section Archiving Options. + @@ -4777,7 +4788,8 @@ pg_probackup archive-get -B backup_dir --instance Sets the maximum number of files that can be copied into the archive - by a single archive-push process. + by a single archive-push process, or from + the archive by a single archive-get process. @@ -4789,6 +4801,7 @@ pg_probackup archive-get -B backup_dir --instance .part files to be stale. By default, pg_probackup waits 300 seconds. + This option can be used only with command. @@ -4800,6 +4813,7 @@ pg_probackup archive-get -B backup_dir --instance archive_status directory. This option should be used only if archive_command contains multiple commands. + This option can be used only with command. @@ -4811,6 +4825,31 @@ pg_probackup archive-get -B backup_dir --instance command. + + + + + + + + + Directory used to store prefetched WAL segments if option is used. + Directory must be located on the same filesystem and on the same mountpoint the + PGDATA/pg_wal is located. + By default files are stored in PGDATA/pg_wal/pbk_prefetch directory. + This option can be used only with command. + + + + + + + + + Do not validate prefetched WAL file before using it. + Use this option if you want to increase the speed of recovery. + This option can be used only with command. diff --git a/src/archive.c b/src/archive.c index ab3471b4e..8866ce66e 100644 --- a/src/archive.c +++ b/src/archive.c @@ -22,14 +22,31 @@ static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_ int compress_level, int thread_num, uint32 archive_timeout); #endif static void *push_files(void *arg); -static void get_wal_file(const char *from_path, const char *to_path); +static void *get_files(void *arg); +static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, + bool prefetch_mode, int thread_num); +static int get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, + bool is_decompress, int thread_num); #ifdef HAVE_LIBZ static const char *get_gz_error(gzFile gzf, int errnum); #endif -static void copy_file_attributes(const char *from_path, - fio_location from_location, - const char *to_path, fio_location to_location, - bool unlink_on_error); +//static void copy_file_attributes(const char *from_path, +// fio_location from_location, +// const char *to_path, fio_location to_location, +// bool unlink_on_error); + +static bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size); +static uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, TimeLineID tli, + XLogSegNo first_segno, int num_threads, bool inclusive, int batch_size, + uint32 wal_seg_size); +static bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, + const char *prefetch_dir, const char *absolute_wal_file_path, + uint32 wal_seg_size, bool parse_wal, int thread_num); + +static uint32 maintain_prefetch(const char *prefetch_dir, XLogSegNo first_segno, uint32 wal_seg_size); + +static bool prefetch_stop = false; +static uint32 xlog_seg_size; typedef struct { @@ -61,6 +78,15 @@ typedef struct int ret; } archive_push_arg; +typedef struct +{ + const char *prefetch_dir; + const char *archive_dir; + int thread_num; + parray *files; + uint32 n_fetched; +} archive_get_arg; + typedef struct WALSegno { char name[MAXFNAMELEN]; @@ -156,7 +182,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, if (num_threads > parray_num(batch_files)) n_threads = parray_num(batch_files); - elog(INFO, "PID [%d]: pg_probackup push file %s into archive, " + elog(INFO, "PID [%d]: pg_probackup archive-push WAL file: %s, " "threads: %i/%i, batch: %lu/%i, compression: %s", my_pid, wal_file_name, n_threads, num_threads, parray_num(batch_files), batch_size, @@ -865,316 +891,865 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, } #endif +#ifdef HAVE_LIBZ +/* + * Show error during work with compressed file + */ +static const char * +get_gz_error(gzFile gzf, int errnum) +{ + int gz_errnum; + const char *errmsg; + + errmsg = fio_gzerror(gzf, &gz_errnum); + if (gz_errnum == Z_ERRNO) + return strerror(errnum); + else + return errmsg; +} +#endif + +/* Copy file attributes */ +//static void +//copy_file_attributes(const char *from_path, fio_location from_location, +// const char *to_path, fio_location to_location, +// bool unlink_on_error) +//{ +// struct stat st; +// +// if (fio_stat(from_path, &st, true, from_location) == -1) +// { +// if (unlink_on_error) +// fio_unlink(to_path, to_location); +// elog(ERROR, "Cannot stat file \"%s\": %s", +// from_path, strerror(errno)); +// } +// +// if (fio_chmod(to_path, st.st_mode, to_location) == -1) +// { +// if (unlink_on_error) +// fio_unlink(to_path, to_location); +// elog(ERROR, "Cannot change mode of file \"%s\": %s", +// to_path, strerror(errno)); +// } +//} + +/* Look for files with '.ready' suffix in archive_status directory + * and pack such files into batch sized array. + */ +parray * +setup_push_filelist(const char *archive_status_dir, const char *first_file, + int batch_size) +{ + int i; + WALSegno *xlogfile = NULL; + parray *status_files = NULL; + parray *batch_files = parray_new(); + + /* guarantee that first filename is in batch list */ + xlogfile = palloc(sizeof(WALSegno)); + pg_atomic_init_flag(&xlogfile->lock); + snprintf(xlogfile->name, MAXFNAMELEN, "%s", first_file); + parray_append(batch_files, xlogfile); + + if (batch_size < 2) + return batch_files; + + /* get list of files from archive_status */ + status_files = parray_new(); + dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); + parray_qsort(status_files, pgFileComparePath); + + for (i = 0; i < parray_num(status_files); i++) + { + int result = 0; + char filename[MAXFNAMELEN]; + char suffix[MAXFNAMELEN]; + pgFile *file = (pgFile *) parray_get(status_files, i); + + result = sscanf(file->name, "%[^.]%s", (char *) &filename, (char *) &suffix); + + if (result != 2) + continue; + + if (strcmp(suffix, ".ready") != 0) + continue; + + /* first filename already in batch list */ + if (strcmp(filename, first_file) == 0) + continue; + + xlogfile = palloc(sizeof(WALSegno)); + pg_atomic_init_flag(&xlogfile->lock); + + snprintf(xlogfile->name, MAXFNAMELEN, "%s", filename); + parray_append(batch_files, xlogfile); + + if (parray_num(batch_files) >= batch_size) + break; + } + + /* cleanup */ + parray_walk(status_files, pgFileFree); + parray_free(status_files); + + return batch_files; +} + /* * pg_probackup specific restore command. * Move files from arclog_path to pgdata/wal_file_path. + * + * The problem with archive-get: we must be very careful about + * erroring out, because postgres will interpretent our negative exit code + * as the fact, that requested file is missing and may take irreversible actions. + * So if file copying has failed we must retry several times before bailing out. + * + * TODO: add support of -D option. + * TOTHINK: what can be done about ssh connection been broken? + * TOTHINk: do we need our own rmtree function ? + * TOTHINk: so sort of async prefetch ? + */ -int -do_archive_get(InstanceConfig *instance, - char *wal_file_path, char *wal_file_name) +void +do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, + char *wal_file_path, char *wal_file_name, int batch_size, + bool validate_wal) { - char backup_wal_file_path[MAXPGPATH]; - char absolute_wal_file_path[MAXPGPATH]; - char current_dir[MAXPGPATH]; + int fail_count = 0; + char backup_wal_file_path[MAXPGPATH]; + char absolute_wal_file_path[MAXPGPATH]; + char current_dir[MAXPGPATH]; + char prefetch_dir[MAXPGPATH]; + char pg_xlog_dir[MAXPGPATH]; + char prefetched_file[MAXPGPATH]; - if (wal_file_name == NULL && wal_file_path == NULL) - elog(ERROR, "required parameters are not specified: --wal-file-name %%f --wal-file-path %%p"); + /* reporting */ + pid_t my_pid = getpid(); + uint32 n_fetched = 0; + int n_actual_threads = num_threads; + uint32 n_files_in_prefetch = 0; + + /* time reporting */ + instr_time start_time, end_time; + double get_time; + char pretty_time_str[20]; if (wal_file_name == NULL) - elog(ERROR, "required parameter not specified: --wal-file-name %%f"); + elog(ERROR, "PID [%d]: Required parameter not specified: --wal-file-name %%f", my_pid); if (wal_file_path == NULL) - elog(ERROR, "required parameter not specified: --wal-file-path %%p"); - - canonicalize_path(wal_file_path); + elog(ERROR, "PID [%d]: Required parameter not specified: --wal_file_path %%p", my_pid); if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); + elog(ERROR, "PID [%d]: getcwd() error", my_pid); + + /* path to PGDATA/pg_wal directory */ + join_path_components(pg_xlog_dir, current_dir, XLOGDIR); + /* destination full filepath, usually it is PGDATA/pg_wal/RECOVERYXLOG */ join_path_components(absolute_wal_file_path, current_dir, wal_file_path); + + /* full filepath to WAL file in archive directory. + * backup_path/wal/instance_name/000000010000000000000001 */ join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); - elog(INFO, "pg_probackup archive-get from %s to %s", - backup_wal_file_path, absolute_wal_file_path); - get_wal_file(backup_wal_file_path, absolute_wal_file_path); - elog(INFO, "pg_probackup archive-get completed successfully"); + INSTR_TIME_SET_CURRENT(start_time); + if (num_threads > batch_size) + n_actual_threads = batch_size; + elog(INFO, "PID [%d]: pg_probackup archive-get WAL file: %s, remote: %s, threads: %i/%i, batch: %i", + my_pid, wal_file_name, IsSshProtocol() ? "ssh" : "none", n_actual_threads, num_threads, batch_size); - return 0; + num_threads = n_actual_threads; + + elog(VERBOSE, "Obtaining XLOG_SEG_SIZE from pg_control file"); + instance->xlog_seg_size = get_xlog_seg_size(current_dir); + + /* Prefetch optimization kicks in only if simple XLOG segments is requested + * and batching is enabled. + * + * We check that file do exists in prefetch directory, then we validate it and + * rename to destination path. + * If file do not exists, then we run prefetch and rename it. + */ + if (IsXLogFileName(wal_file_name) && batch_size > 1) + { + XLogSegNo segno; + TimeLineID tli; + + GetXLogFromFileName(wal_file_name, &tli, &segno, instance->xlog_seg_size); + + if (prefetch_dir_arg) + /* use provided prefetch directory */ + snprintf(prefetch_dir, sizeof(prefetch_dir), "%s", prefetch_dir_arg); + else + /* use default path */ + join_path_components(prefetch_dir, pg_xlog_dir, "pbk_prefetch"); + + /* Construct path to WAL file in prefetch directory. + * current_dir/pg_wal/pbk_prefech/000000010000000000000001 + */ + join_path_components(prefetched_file, prefetch_dir, wal_file_name); + + /* check if file is available in prefetch directory */ + if (access(prefetched_file, F_OK) == 0) + { + /* Prefetched WAL segment is available, before using it, we must validate it. + * But for validation to work properly(because of contrecord), we must be sure + * that next WAL segment is also available in prefetch directory. + * If next segment do not exists in prefetch directory, we must provide it from + * archive. If it is NOT available in the archive, then file in prefetch directory + * cannot be trusted. In this case we discard all prefetched files and + * copy requested file directly from archive. + */ + if (!next_wal_segment_exists(tli, segno, prefetch_dir, instance->xlog_seg_size)) + n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + tli, segno, num_threads, false, batch_size, + instance->xlog_seg_size); + + n_files_in_prefetch = maintain_prefetch(prefetch_dir, segno, instance->xlog_seg_size); + + if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, + absolute_wal_file_path, instance->xlog_seg_size, + validate_wal, 0)) + { + n_files_in_prefetch--; + elog(INFO, "PID [%d]: pg_probackup archive-get used prefetched WAL segment %s, prefetch state: %u/%u", + my_pid, wal_file_name, n_files_in_prefetch, batch_size); + goto get_done; + } + else + { + /* discard prefetch */ +// n_fetched = 0; + rmtree(prefetch_dir, false); + } + } + else + { + /* Do prefetch maintenance here */ + + mkdir(prefetch_dir, DIR_PERMISSION); /* In case prefetch directory do not exists yet */ + + /* We`ve failed to satisfy current request from prefetch directory, + * therefore we can discard its content, since it may be corrupted or + * contain stale files. + * + * UPDATE: we should not discard prefetch easily, because failing to satisfy + * request for WAL may come from this recovery behavior: + * https://www.postgresql.org/message-id/flat/16159-f5a34a3a04dc67e0%40postgresql.org + */ +// rmtree(prefetch_dir, false); + + /* prefetch files */ + n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + tli, segno, num_threads, true, batch_size, + instance->xlog_seg_size); + + n_files_in_prefetch = maintain_prefetch(prefetch_dir, segno, instance->xlog_seg_size); + + if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, absolute_wal_file_path, + instance->xlog_seg_size, validate_wal, 0)) + { + n_files_in_prefetch--; + elog(INFO, "PID [%d]: pg_probackup archive-get copied WAL file %s, prefetch state: %u/%u", + my_pid, wal_file_name, n_files_in_prefetch, batch_size); + goto get_done; + } +// else +// { +// /* yet again failed to satisfy request from prefetch */ +// n_fetched = 0; +// rmtree(prefetch_dir, false); +// } + } + } + + /* we use it to extend partial file later */ + xlog_seg_size = instance->xlog_seg_size; + + /* Either prefetch didn`t cut it, or batch mode is disabled or + * the requested file is not WAL segment. + * Copy file from the archive directly. + * Retry several times before bailing out. + * + * TODO: + * files copied from archive directly are not validated, which is not ok. + * TOTHINK: + * Current WAL validation cannot be applied to partial files. + */ + + while (fail_count < 3) + { + if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false, 0)) + { + fail_count = 0; + elog(INFO, "PID [%d]: pg_probackup archive-get copied WAL file %s", + my_pid, wal_file_name); + n_fetched++; + break; + } + else + fail_count++; + + elog(LOG, "PID [%d]: Failed to get WAL file %s, retry %i/3", + 0, wal_file_name, fail_count); + } + + /* TODO/TOTHINK: + * If requested file is corrupted, we have no way to warn PostgreSQL about it. + * We either can: + * 1. feed to recovery and let PostgreSQL sort it out. Currently we do this. + * 2. error out. + * + * Also note, that we can detect corruption only if prefetch mode is used. + * TODO: if corruption or network problem encountered, kill yourself + * with SIGTERN to prevent recovery from starting up database. + */ + +get_done: + INSTR_TIME_SET_CURRENT(end_time); + INSTR_TIME_SUBTRACT(end_time, start_time); + get_time = INSTR_TIME_GET_DOUBLE(end_time); + pretty_time_interval(get_time, pretty_time_str, 20); + + if (fail_count == 0) + elog(INFO, "PID [%d]: pg_probackup archive-get completed successfully, fetched: %i/%i, time elapsed: %s", + my_pid, n_fetched, batch_size, pretty_time_str); + else + elog(ERROR, "PID [%d]: pg_probackup archive-get failed to deliver WAL file: %s, time elapsed: %s", + my_pid, wal_file_name, pretty_time_str); } /* - * Copy WAL segment from archive catalog to pgdata with possible decompression. + * Copy batch_size of regular WAL segments into prefetch directory, + * starting with first_file. + * + * inclusive - should we copy first_file or not. */ -void -get_wal_file(const char *from_path, const char *to_path) +uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, + TimeLineID tli, XLogSegNo first_segno, int num_threads, + bool inclusive, int batch_size, uint32 wal_seg_size) { - FILE *in = NULL; - int out; - char buf[XLOG_BLCKSZ]; - const char *from_path_p = from_path; - char to_path_temp[MAXPGPATH]; - int errno_temp; - bool is_decompress = false; + int i; + XLogSegNo segno; + parray *batch_files = parray_new(); + int n_total_fetched = 0; -#ifdef HAVE_LIBZ - char gz_from_path[MAXPGPATH]; - gzFile gz_in = NULL; -#endif + if (!inclusive) + first_segno++; - /* First check source file for existance */ - if (fio_access(from_path, F_OK, FIO_BACKUP_HOST) != 0) + for (segno = first_segno; segno < (first_segno + batch_size); segno++) { -#ifdef HAVE_LIBZ - /* - * Maybe we need to decompress the file. Check it with .gz - * extension. - */ - snprintf(gz_from_path, sizeof(gz_from_path), "%s.gz", from_path); - if (fio_access(gz_from_path, F_OK, FIO_BACKUP_HOST) == 0) - { - /* Found compressed file */ - is_decompress = true; - from_path_p = gz_from_path; - } -#endif - /* Didn't find compressed file */ - if (!is_decompress) - elog(ERROR, "Source WAL file \"%s\" doesn't exist", - from_path); + WALSegno *xlogfile = palloc(sizeof(WALSegno)); + pg_atomic_init_flag(&xlogfile->lock); + + /* construct filename for WAL segment */ + GetXLogFileName(xlogfile->name, tli, segno, wal_seg_size); + + parray_append(batch_files, xlogfile); + } - /* open file for read */ - if (!is_decompress) + /* copy segments */ + if (num_threads == 1) { - in = fio_fopen(from_path, PG_BINARY_R, FIO_BACKUP_HOST); - if (in == NULL) - elog(ERROR, "Cannot open source WAL file \"%s\": %s", - from_path, strerror(errno)); + for (i = 0; i < parray_num(batch_files); i++) + { + char to_fullpath[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); + + join_path_components(to_fullpath, prefetch_dir, xlogfile->name); + join_path_components(from_fullpath, archive_dir, xlogfile->name); + + /* It is ok, maybe requested batch is greater than the number of available + * files in the archive + */ + if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true, 0)) + { + elog(LOG, "Thread [%d]: Failed to prefetch WAL segment %s", 0, xlogfile->name); + break; + } + + n_total_fetched++; + } } -#ifdef HAVE_LIBZ else { - gz_in = fio_gzopen(gz_from_path, PG_BINARY_R, Z_DEFAULT_COMPRESSION, - FIO_BACKUP_HOST); - if (gz_in == NULL) - elog(ERROR, "Cannot open compressed WAL file \"%s\": %s", - gz_from_path, strerror(errno)); - } -#endif + /* arrays with meta info for multi threaded archive-get */ + pthread_t *threads; + archive_get_arg *threads_args; - /* open backup file for write */ - snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", to_path); + /* init thread args */ + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + threads_args = (archive_get_arg *) palloc(sizeof(archive_get_arg) * num_threads); - out = fio_open(to_path_temp, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_DB_HOST); - if (out < 0) - elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s", - to_path_temp, strerror(errno)); + for (i = 0; i < num_threads; i++) + { + archive_get_arg *arg = &(threads_args[i]); - /* copy content */ - for (;;) + arg->prefetch_dir = prefetch_dir; + arg->archive_dir = archive_dir; + + arg->thread_num = i+1; + arg->files = batch_files; + } + + /* Run threads */ + for (i = 0; i < num_threads; i++) + { + archive_get_arg *arg = &(threads_args[i]); + pthread_create(&threads[i], NULL, get_files, arg); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + n_total_fetched += threads_args[i].n_fetched; + } + } + /* TODO: free batch_files */ + return n_total_fetched; +} + +/* + * Copy files from archive catalog to pg_wal. + */ +static void * +get_files(void *arg) +{ + int i; + char to_fullpath[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + archive_get_arg *args = (archive_get_arg *) arg; + + for (i = 0; i < parray_num(args->files); i++) { - int read_len = 0; + WALSegno *xlogfile = (WALSegno *) parray_get(args->files, i); -#ifdef HAVE_LIBZ - if (is_decompress) + if (prefetch_stop) + break; + + if (!pg_atomic_test_set_flag(&xlogfile->lock)) + continue; + + join_path_components(from_fullpath, args->archive_dir, xlogfile->name); + join_path_components(to_fullpath, args->prefetch_dir, xlogfile->name); + + if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true, args->thread_num)) { - read_len = fio_gzread(gz_in, buf, sizeof(buf)); - if (read_len <= 0 && !fio_gzeof(gz_in)) - { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot read compressed WAL file \"%s\": %s", - gz_from_path, get_gz_error(gz_in, errno_temp)); - } + /* It is ok, maybe requested batch is greater than the number of available + * files in the archive + */ + elog(LOG, "Thread [%d]: Failed to prefetch WAL segment %s", + args->thread_num, xlogfile->name); + prefetch_stop = true; + break; } - else + + args->n_fetched++; + } + + /* close ssh connection */ + fio_disconnect(); + + return NULL; +} + +/* + * Copy WAL segment from archive catalog to pgdata with possible decompression. + * When running in prefetch mode, we should not error out. + */ +bool +get_wal_file(const char *filename, const char *from_fullpath, + const char *to_fullpath, bool prefetch_mode, int thread_num) +{ + int rc = FILE_MISSING; + FILE *out; + char from_fullpath_gz[MAXPGPATH]; + bool src_partial = false; + + snprintf(from_fullpath_gz, sizeof(from_fullpath_gz), "%s.gz", from_fullpath); + + /* open destination file */ + out = fopen(to_fullpath, PG_BINARY_W); + if (!out) + { + elog(WARNING, "Thread [%d]: Failed to open file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + return false; + } + + if (chmod(to_fullpath, FILE_PERMISSION) == -1) + { + elog(WARNING, "Thread [%d]: Cannot change mode of file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + fclose(out); + unlink(to_fullpath); + return false; + } + + /* disable buffering for output file */ + setvbuf(out, NULL, _IONBF, BUFSIZ); + + /* In prefetch mode, we do look only for full WAL segments + * In non-prefetch mode, do look up '.partial' and '.gz.partial' + * segments. + */ + if (fio_is_remote(FIO_BACKUP_HOST)) + { + /* get file via ssh */ +#ifdef HAVE_LIBZ + /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ + if (IsXLogFileName(filename)) + rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, thread_num); + if (rc == FILE_MISSING) #endif + /* ... failing that, use uncompressed */ + rc = fio_send_file(from_fullpath, to_fullpath, out, thread_num); + + /* When not in prefetch mode, try to use partial file */ + if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) { - read_len = fio_fread(in, buf, sizeof(buf)); - if (read_len < 0) + char from_partial[MAXPGPATH]; + +#ifdef HAVE_LIBZ + /* '.gz.partial' goes first ... */ + snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); + rc = fio_send_file_gz(from_partial, to_fullpath, out, thread_num); + if (rc == FILE_MISSING) +#endif { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); + /* ... failing that, use '.partial' */ + snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); + rc = fio_send_file(from_partial, to_fullpath, out, thread_num); } + + if (rc == SEND_OK) + src_partial = true; } + } + else + { + /* get file locally */ +#ifdef HAVE_LIBZ + /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ + if (IsXLogFileName(filename)) + rc = get_wal_file_internal(from_fullpath_gz, to_fullpath, out, true, thread_num); + if (rc == FILE_MISSING) +#endif + /* ... failing that, use uncompressed */ + rc = get_wal_file_internal(from_fullpath, to_fullpath, out, false, thread_num); - if (read_len > 0) + /* When not in prefetch mode, try to use partial file */ + if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) { - if (fio_write(out, buf, read_len) != read_len) + char from_partial[MAXPGPATH]; + +#ifdef HAVE_LIBZ + /* '.gz.partial' goes first ... */ + snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); + rc = get_wal_file_internal(from_partial, to_fullpath, + out, true, thread_num); + if (rc == FILE_MISSING) +#endif { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot write to WAL file \"%s\": %s", to_path_temp, - strerror(errno_temp)); + /* ... failing that, use '.partial' */ + snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); + rc = get_wal_file_internal(from_partial, to_fullpath, out, + false, thread_num); } + + if (rc == SEND_OK) + src_partial = true; } + } - /* Check for EOF */ -#ifdef HAVE_LIBZ - if (is_decompress) + if (!prefetch_mode && (rc == FILE_MISSING)) + elog(LOG, "Thread [%d]: Target WAL file is missing: %s", + thread_num, filename); + + if (rc < 0) + { + fclose(out); + unlink(to_fullpath); + return false; + } + + /* If partial file was used as source, then it is very likely that destination + * file is not equal to XLOG_SEG_SIZE - that is the way pg_receivexlog works. + * We must manually extent it up to XLOG_SEG_SIZE. + */ + if (src_partial) + { + + if (fflush(out) != 0) { - if (fio_gzeof(gz_in) || read_len == 0) - break; + elog(WARNING, "Thread [%d]: Cannot flush file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + fclose(out); + unlink(to_fullpath); + return false; } - else -#endif + + if (ftruncate(fileno(out), xlog_seg_size) != 0) { - if (/* feof(in) || */ read_len == 0) - break; + elog(WARNING, "Thread [%d]: Cannot extend file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + fclose(out); + unlink(to_fullpath); + return false; } } - if (fio_close(out) != 0) + if (fclose(out) != 0) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot write WAL file \"%s\": %s", - to_path_temp, strerror(errno_temp)); + elog(WARNING, "Thread [%d]: Cannot close file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + unlink(to_fullpath); + return false; } + elog(LOG, "Thread [%d]: WAL file successfully %s: %s", + thread_num, prefetch_mode ? "prefetched" : "copied", filename); + return true; +} + +/* + * Copy local WAL segment with possible decompression. + * Return codes: + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + * WRITE_FAILED (-4) + * ZLIB_ERROR (-5) + */ +int +get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, + bool is_decompress, int thread_num) +{ #ifdef HAVE_LIBZ - if (is_decompress) + gzFile gz_in = NULL; +#endif + FILE *in = NULL; + char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ + int exit_code = 0; + + elog(VERBOSE, "Thread [%d]: Attempting to %s WAL file '%s'", + thread_num, is_decompress ? "open compressed" : "open", from_path); + + /* open source file for read */ + if (!is_decompress) { - if (fio_gzclose(gz_in) != 0) + in = fopen(from_path, PG_BINARY_R); + if (in == NULL) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot close compressed WAL file \"%s\": %s", - gz_from_path, get_gz_error(gz_in, errno_temp)); + if (errno == ENOENT) + exit_code = FILE_MISSING; + else + { + elog(WARNING, "Thread [%d]: Cannot open source WAL file \"%s\": %s", + thread_num, from_path, strerror(errno)); + exit_code = OPEN_FAILED; + } + goto cleanup; } } +#ifdef HAVE_LIBZ else -#endif { - if (fio_fclose(in)) + gz_in = gzopen(from_path, PG_BINARY_R); + if (gz_in == NULL) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot close source WAL file \"%s\": %s", - from_path, strerror(errno_temp)); + if (errno == ENOENT) + exit_code = FILE_MISSING; + else + { + elog(WARNING, "Thread [%d]: Cannot open compressed WAL file \"%s\": %s", + thread_num, from_path, strerror(errno)); + exit_code = OPEN_FAILED; + } + + goto cleanup; } } +#endif - /* update file permission. */ - copy_file_attributes(from_path_p, FIO_BACKUP_HOST, to_path_temp, FIO_DB_HOST, true); - - if (fio_rename(to_path_temp, to_path, FIO_DB_HOST) < 0) + /* copy content */ + for (;;) { - errno_temp = errno; - fio_unlink(to_path_temp, FIO_DB_HOST); - elog(ERROR, "Cannot rename WAL file \"%s\" to \"%s\": %s", - to_path_temp, to_path, strerror(errno_temp)); + int read_len = 0; + +#ifdef HAVE_LIBZ + if (is_decompress) + { + read_len = gzread(gz_in, buf, OUT_BUF_SIZE); + + if (read_len <= 0) + { + if (gzeof(gz_in)) + break; + else + { + elog(WARNING, "Thread [%d]: Cannot read compressed WAL file \"%s\": %s", + thread_num, from_path, get_gz_error(gz_in, errno)); + exit_code = READ_FAILED; + break; + } + } + } + else +#endif + { + read_len = fread(buf, 1, OUT_BUF_SIZE, in); + + if (read_len < 0 || ferror(in)) + { + elog(WARNING, "Thread [%d]: Cannot read source WAL file \"%s\": %s", + thread_num, from_path, strerror(errno)); + exit_code = READ_FAILED; + break; + } + else if (read_len == 0 && feof(in)) + break; + } + + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + { + elog(WARNING, "Thread [%d]: Cannot write to WAL file '%s': %s", + thread_num, to_path, strerror(errno)); + exit_code = WRITE_FAILED; + break; + } + } } +cleanup: #ifdef HAVE_LIBZ - if (is_decompress) - elog(INFO, "WAL file decompressed from \"%s\"", gz_from_path); + if (gz_in) + gzclose(gz_in); #endif + if (in) + fclose(in); + + pg_free(buf); + return exit_code; } -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf, int errnum) +bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size) { - int gz_errnum; - const char *errmsg; + char next_wal_filename[MAXFNAMELEN]; + char next_wal_fullpath[MAXPGPATH]; - errmsg = fio_gzerror(gzf, &gz_errnum); - if (gz_errnum == Z_ERRNO) - return strerror(errnum); - else - return errmsg; + GetXLogFileName(next_wal_filename, tli, segno + 1, wal_seg_size); + + join_path_components(next_wal_fullpath, prefetch_dir, next_wal_filename); + + if (access(next_wal_fullpath, F_OK) == 0) + return true; + + return false; } -#endif -/* Copy file attributes */ -static void -copy_file_attributes(const char *from_path, fio_location from_location, - const char *to_path, fio_location to_location, - bool unlink_on_error) +/* Try to use content of prefetch directory to satisfy request for WAL segment + * If file is found, then validate it and rename. + * If requested file do not exists or validation has failed, then + * caller must copy WAL file directly from archive. + */ +bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, + const char *prefetch_dir, const char *absolute_wal_file_path, + uint32 wal_seg_size, bool parse_wal, int thread_num) { - struct stat st; + char prefetched_file[MAXPGPATH]; + + join_path_components(prefetched_file, prefetch_dir, wal_file_name); + + /* If prefetched file do not exists, then nothing can be done */ + if (access(prefetched_file, F_OK) != 0) + return false; + + /* If the next WAL segment do not exists in prefetch directory, + * then current segment cannot be validated, therefore cannot be used + * to satisfy recovery request. + */ + if (parse_wal && !next_wal_segment_exists(tli, segno, prefetch_dir, wal_seg_size)) + return false; - if (fio_stat(from_path, &st, true, from_location) == -1) + if (parse_wal && !validate_wal_segment(tli, segno, prefetch_dir, wal_seg_size)) { - if (unlink_on_error) - fio_unlink(to_path, to_location); - elog(ERROR, "Cannot stat file \"%s\": %s", - from_path, strerror(errno)); + /* prefetched WAL segment is not looking good */ + elog(LOG, "Thread [%d]: Prefetched WAL segment %s is invalid, cannot use it", + thread_num, wal_file_name); + unlink(prefetched_file); + return false; } - if (fio_chmod(to_path, st.st_mode, to_location) == -1) + /* file is available in prefetch directory */ + if (rename(prefetched_file, absolute_wal_file_path) == 0) + return true; + else { - if (unlink_on_error) - fio_unlink(to_path, to_location); - elog(ERROR, "Cannot change mode of file \"%s\": %s", - to_path, strerror(errno)); + elog(WARNING, "Thread [%d]: Cannot rename file '%s' to '%s': %s", + thread_num, prefetched_file, absolute_wal_file_path, strerror(errno)); + unlink(prefetched_file); } + + return false; } -/* Look for files with '.ready' suffix in archive_status directory - * and pack such files into batch sized array. +/* + * Maintain prefetch directory: drop redundant files + * Return number of files in prefetch directory. */ -parray * -setup_push_filelist(const char *archive_status_dir, const char *first_file, - int batch_size) +uint32 maintain_prefetch(const char *prefetch_dir, XLogSegNo first_segno, uint32 wal_seg_size) { - int i; - WALSegno *xlogfile = NULL; - parray *status_files = NULL; - parray *batch_files = parray_new(); + DIR *dir; + struct dirent *dir_ent; + uint32 n_files = 0; - /* guarantee that first filename is in batch list */ - xlogfile = palloc(sizeof(WALSegno)); - pg_atomic_init_flag(&xlogfile->lock); - strncpy(xlogfile->name, first_file, MAXFNAMELEN); - parray_append(batch_files, xlogfile); + XLogSegNo segno; + TimeLineID tli; - if (batch_size < 2) - return batch_files; + char fullpath[MAXPGPATH]; - /* get list of files from archive_status */ - status_files = parray_new(); - dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); - parray_qsort(status_files, pgFileComparePath); - - for (i = 0; i < parray_num(status_files); i++) + dir = opendir(prefetch_dir); + if (dir == NULL) { - int result = 0; - char filename[MAXFNAMELEN]; - char suffix[MAXFNAMELEN]; - pgFile *file = (pgFile *) parray_get(status_files, i); + if (errno != ENOENT) + elog(WARNING, "Cannot open directory \"%s\": %s", prefetch_dir, strerror(errno)); - result = sscanf(file->name, "%[^.]%s", (char *) &filename, (char *) &suffix); - - if (result != 2) - continue; + return n_files; + } - if (strcmp(suffix, ".ready") != 0) + while ((dir_ent = readdir(dir))) + { + /* Skip entries point current dir or parent dir */ + if (strcmp(dir_ent->d_name, ".") == 0 || + strcmp(dir_ent->d_name, "..") == 0) continue; - /* first filename already in batch list */ - if (strcmp(filename, first_file) == 0) - continue; + if (IsXLogFileName(dir_ent->d_name)) + { - xlogfile = palloc(sizeof(WALSegno)); - pg_atomic_init_flag(&xlogfile->lock); + GetXLogFromFileName(dir_ent->d_name, &tli, &segno, wal_seg_size); - strncpy(xlogfile->name, filename, MAXFNAMELEN); - parray_append(batch_files, xlogfile); + /* potentially useful segment, keep it */ + if (segno >= first_segno) + { + n_files++; + continue; + } + } - if (parray_num(batch_files) >= batch_size) - break; + join_path_components(fullpath, prefetch_dir, dir_ent->d_name); + unlink(fullpath); } - /* cleanup */ - parray_walk(status_files, pgFileFree); - parray_free(status_files); + closedir(dir); - return batch_files; + return n_files; } diff --git a/src/backup.c b/src/backup.c index bcc74ed30..5d29781b8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -443,7 +443,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* TODO: add ms precision */ if (pagemap_isok) - elog(INFO, "Pagemap successfully extracted, time elapsed %.0f sec", + elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", difftime(end_time, start_time)); else elog(ERROR, "Pagemap extraction failed, time elasped: %.0f sec", diff --git a/src/help.c b/src/help.c index c006ebb2b..964650091 100644 --- a/src/help.c +++ b/src/help.c @@ -215,7 +215,7 @@ help_pg_probackup(void) printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [-j num-threads] [--batch-size batch_size]\n")); + printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); printf(_(" [--overwrite] [--compress]\n")); @@ -229,6 +229,8 @@ help_pg_probackup(void) printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); + printf(_(" [--no-validate-wal]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -870,7 +872,7 @@ help_archive_push(void) { printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [-j num-threads] [--batch-size batch_size]\n")); + printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); printf(_(" [--overwrite] [--compress]\n")); @@ -916,6 +918,8 @@ help_archive_get(void) printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); + printf(_(" [--no-validate-wal]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); @@ -926,6 +930,10 @@ help_archive_get(void) printf(_(" relative destination path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the WAL file to retrieve from the archive\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --batch-size=NUM number of files to be prefetched\n")); + printf(_(" --prefetch-dir=path location of the store area for prefetched WAL files\n")); + printf(_(" --no-validate-wal skip validation of prefetched WAL file before using it\n")); printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); diff --git a/src/parsexlog.c b/src/parsexlog.c index 11f2725ec..e8ea27ceb 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -325,12 +325,15 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, for (i = parray_num(interval_list) - 1; i >= 0; i--) { - bool inclusive_endpoint = false; + bool inclusive_endpoint; wal_interval = parray_get(interval_list, i); /* In case of replica promotion, endpoints of intermediate * timelines can be unreachable. */ + inclusive_endpoint = false; + + /* ... but not the end timeline */ if (wal_interval->tli == end_tli) inclusive_endpoint = true; @@ -453,7 +456,7 @@ validate_wal(pgBackup *backup, const char *archivedir, * If recovery target is provided, ensure that archive files exist in * archive directory. */ - if (dir_is_empty(archivedir, FIO_BACKUP_HOST)) + if (dir_is_empty(archivedir, FIO_LOCAL_HOST)) elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); /* @@ -872,20 +875,20 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, /* If segment do not exists, but the same * segment with '.partial' suffix does, use it instead */ - if (!fileExists(reader_data->xlogpath, FIO_BACKUP_HOST) && - fileExists(partial_file, FIO_BACKUP_HOST)) + if (!fileExists(reader_data->xlogpath, FIO_LOCAL_HOST) && + fileExists(partial_file, FIO_LOCAL_HOST)) { snprintf(reader_data->xlogpath, MAXPGPATH, "%s", partial_file); } - if (fileExists(reader_data->xlogpath, FIO_BACKUP_HOST)) + if (fileExists(reader_data->xlogpath, FIO_LOCAL_HOST)) { elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", reader_data->thread_num, reader_data->xlogpath); reader_data->xlogexists = true; reader_data->xlogfile = fio_open(reader_data->xlogpath, - O_RDONLY | PG_BINARY, FIO_BACKUP_HOST); + O_RDONLY | PG_BINARY, FIO_LOCAL_HOST); if (reader_data->xlogfile < 0) { @@ -897,14 +900,14 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, } #ifdef HAVE_LIBZ /* Try to open compressed WAL segment */ - else if (fileExists(reader_data->gz_xlogpath, FIO_BACKUP_HOST)) + else if (fileExists(reader_data->gz_xlogpath, FIO_LOCAL_HOST)) { elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", reader_data->thread_num, reader_data->gz_xlogpath); reader_data->xlogexists = true; reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, - "rb", -1, FIO_BACKUP_HOST); + "rb", -1, FIO_LOCAL_HOST); if (reader_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", @@ -1045,15 +1048,13 @@ RunXLogThreads(const char *archivedir, time_t target_time, XLogSegNo endSegNo = 0; bool result = true; - if (!XRecOffIsValid(startpoint) && - !XRecOffIsNull(startpoint)) - { + if (!XRecOffIsValid(startpoint) && !XRecOffIsNull(startpoint)) elog(ERROR, "Invalid startpoint value %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); - } if (!XLogRecPtrIsInvalid(endpoint)) { +// if (XRecOffIsNull(endpoint) && !inclusive_endpoint) if (XRecOffIsNull(endpoint)) { GetXLogSegNo(endpoint, endSegNo, segment_size); @@ -1719,3 +1720,28 @@ getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) return false; } +bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size) +{ + XLogRecPtr startpoint; + XLogRecPtr endpoint; + + bool rc; + int tmp_num_threads = num_threads; + num_threads = 1; + + /* calculate startpoint and endpoint */ + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); + GetXLogRecPtr(segno+1, 0, wal_seg_size, endpoint); + + /* disable multi-threading */ + num_threads = 1; + + rc = RunXLogThreads(prefetch_dir, 0, InvalidTransactionId, + InvalidXLogRecPtr, tli, wal_seg_size, + startpoint, endpoint, false, NULL, NULL, true); + + num_threads = tmp_num_threads; + + return rc; +} + diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f73952116..7055f7e29 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -131,6 +131,10 @@ static char *wal_file_name; static bool file_overwrite = false; static bool no_ready_rename = false; +/* archive get options */ +static char *prefetch_dir; +bool no_validate_wal = false; + /* show options */ ShowFormat show_format = SHOW_PLAIN; bool show_archive = false; @@ -220,9 +224,12 @@ static ConfigOption cmd_options[] = { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, { 'b', 153, "no-ready-rename", &no_ready_rename, SOURCE_CMD_STRICT }, { 'i', 162, "batch-size", &batch_size, SOURCE_CMD_STRICT }, + /* archive-get options */ + { 's', 163, "prefetch-dir", &prefetch_dir, SOURCE_CMD_STRICT }, + { 'b', 164, "no-validate-wal", &no_validate_wal, SOURCE_CMD_STRICT }, /* show options */ - { 'f', 163, "format", opt_show_format, SOURCE_CMD_STRICT }, - { 'b', 164, "archive", &show_archive, SOURCE_CMD_STRICT }, + { 'f', 165, "format", opt_show_format, SOURCE_CMD_STRICT }, + { 'b', 166, "archive", &show_archive, SOURCE_CMD_STRICT }, /* set-backup options */ { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, @@ -266,9 +273,6 @@ main(int argc, char *argv[]) { char *command = NULL, *command_name; - /* Check if backup_path is directory. */ - struct stat stat_buf; - int rc; PROGRAM_NAME_FULL = argv[0]; @@ -448,11 +452,6 @@ main(int argc, char *argv[]) /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); - - /* Ensure that backup_path is a path to a directory */ - rc = stat(backup_path, &stat_buf); - if (rc != -1 && !S_ISDIR(stat_buf.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); } /* Ensure that backup_path is an absolute path */ @@ -504,12 +503,16 @@ main(int argc, char *argv[]) /* * Ensure that requested backup instance exists. - * for all commands except init, which doesn't take this parameter - * and add-instance which creates new instance. + * for all commands except init, which doesn't take this parameter, + * add-instance which creates new instance + * and archive-get, which just do not require it at this point */ - if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD) + if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && + backup_subcmd != ARCHIVE_GET_CMD) { - if (fio_access(backup_instance_path, F_OK, FIO_BACKUP_HOST) != 0) + struct stat st; + + if (fio_stat(backup_instance_path, &st, true, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Failed to access directory \"%s\": %s", backup_instance_path, strerror(errno)); @@ -518,6 +521,12 @@ main(int argc, char *argv[]) elog(ERROR, "Instance '%s' does not exist in this backup catalog", instance_name); } + else + { + /* Ensure that backup_path is a path to a directory */ + if (!S_ISDIR(st.st_mode)) + elog(ERROR, "-B, --backup-path must be a path to directory"); + } } } @@ -533,7 +542,8 @@ main(int argc, char *argv[]) config_get_opt_env(instance_options); /* Read options from configuration file */ - if (backup_subcmd != ADD_INSTANCE_CMD) + if (backup_subcmd != ADD_INSTANCE_CMD && + backup_subcmd != ARCHIVE_GET_CMD) { join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); @@ -763,8 +773,9 @@ main(int argc, char *argv[]) batch_size, file_overwrite, no_sync, no_ready_rename); break; case ARCHIVE_GET_CMD: - return do_archive_get(&instance_config, - wal_file_path, wal_file_name); + do_archive_get(&instance_config, prefetch_dir, + wal_file_path, wal_file_name, batch_size, !no_validate_wal); + break; case ADD_INSTANCE_CMD: return do_add_instance(&instance_config); case DELETE_INSTANCE_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a845cd987..9d037db54 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -707,8 +707,8 @@ extern int do_add_instance(InstanceConfig *instance); extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename); -extern int do_archive_get(InstanceConfig *instance, char *wal_file_path, - char *wal_file_name); +extern void do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, + char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ extern void do_show_config(void); @@ -922,6 +922,8 @@ extern void validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, uint32 seg_size); +extern bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, + const char *prefetch_dir, uint32 wal_seg_size); extern bool read_recovery_info(const char *archivedir, TimeLineID tli, uint32 seg_size, XLogRecPtr start_lsn, XLogRecPtr stop_lsn, @@ -990,12 +992,23 @@ extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); - /* return codes for fio_send_pages */ -#define WRITE_FAILED (-1) -#define REMOTE_ERROR (-2) -#define PAGE_CORRUPTION (-3) -#define SEND_OK (-4) +#define OUT_BUF_SIZE (1024 * 1024) +extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); +extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); + +/* return codes for fio_send_pages() and fio_send_file() */ +#define SEND_OK (0) +#define FILE_MISSING (-1) +#define OPEN_FAILED (-2) +#define READ_FAILED (-3) +#define WRITE_FAILED (-4) +#define ZLIB_ERROR (-5) +#define REMOTE_ERROR (-6) +#define PAGE_CORRUPTION (-8) + +/* Check if specified location is local for current node */ +extern bool fio_is_remote(fio_location location); extern void get_header_errormsg(Page page, char **errormsg); extern void get_checksum_errormsg(Page page, char **errormsg, diff --git a/src/utils/file.c b/src/utils/file.c index 7cce80099..b53f5ccf7 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -14,6 +14,7 @@ #define PRINTF_BUF_SIZE 1024 #define FILE_PERMISSIONS 0600 +#define CHUNK_SIZE 1024 * 128 static __thread unsigned long fio_fdset = 0; static __thread void* fio_stdin_buffer; @@ -136,7 +137,7 @@ static int remove_file_or_dir(char const* path) #endif /* Check if specified location is local for current node */ -static bool fio_is_remote(fio_location location) +bool fio_is_remote(fio_location location) { bool is_remote = MyLocation != FIO_LOCAL_HOST && location != FIO_LOCAL_HOST @@ -1026,7 +1027,6 @@ int fio_chmod(char const* path, int mode, fio_location location) #ifdef HAVE_LIBZ - #define ZLIB_BUFFER_SIZE (64*1024) #define MAX_WBITS 15 /* 32K LZ77 window */ #define DEF_MEM_LEVEL 8 @@ -1304,8 +1304,10 @@ z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) #endif -/* Send file content */ -static void fio_send_file(int out, char const* path) +/* Send file content + * Note: it should not be used for large files. + */ +static void fio_load_file(int out, char const* path) { int fd = open(path, O_RDONLY); fio_header hdr; @@ -1462,7 +1464,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, file->uncompressed_size += BLCKSZ; } else - elog(ERROR, "Remote agent returned message of unknown type"); + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); } return n_blocks_read; @@ -1629,6 +1631,337 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) return; } +/* Receive chunks of compressed data, decompress them and write to + * destination file. + * Return codes: + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + * WRITE_FAILED (-4) + * ZLIB_ERROR (-5) + * REMOTE_ERROR (-6) + */ +int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num) +{ + fio_header hdr; + int exit_code = SEND_OK; + char *in_buf = pgut_malloc(CHUNK_SIZE); /* buffer for compressed data */ + char *out_buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer for decompressed data */ + size_t path_len = strlen(from_fullpath) + 1; + /* decompressor */ + z_stream *strm = NULL; + + hdr.cop = FIO_SEND_FILE; + hdr.size = path_len; + + elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", + thread_num, from_fullpath); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); + + for (;;) + { + fio_header hdr; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_SEND_FILE_EOF) + { + break; + } + else if (hdr.cop == FIO_ERROR) + { + /* handle error, reported by the agent */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + elog(WARNING, "Thread [%d]: %s", thread_num, in_buf); + } + exit_code = hdr.arg; + goto cleanup; + } + else if (hdr.cop == FIO_PAGE) + { + int rc; + Assert(hdr.size <= CHUNK_SIZE); + IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + + /* We have received a chunk of compressed data, lets decompress it */ + if (strm == NULL) + { + /* Initialize decompressor */ + strm = pgut_malloc(sizeof(z_stream)); + memset(strm, 0, sizeof(z_stream)); + + /* The fields next_in, avail_in initialized before init */ + strm->next_in = (Bytef *)in_buf; + strm->avail_in = hdr.size; + + rc = inflateInit2(strm, 15 + 16); + + if (rc != Z_OK) + { + elog(WARNING, "Thread [%d]: Failed to initialize decompression stream for file '%s': %i: %s", + thread_num, from_fullpath, rc, strm->msg); + exit_code = ZLIB_ERROR; + goto cleanup; + } + } + else + { + strm->next_in = (Bytef *)in_buf; + strm->avail_in = hdr.size; + } + + strm->next_out = (Bytef *)out_buf; /* output buffer */ + strm->avail_out = OUT_BUF_SIZE; /* free space in output buffer */ + + /* + * From zlib documentation: + * The application must update next_in and avail_in when avail_in + * has dropped to zero. It must update next_out and avail_out when + * avail_out has dropped to zero. + */ + while (strm->avail_in != 0) /* while there is data in input buffer, decompress it */ + { + /* decompress until there is no data to decompress, + * or buffer with uncompressed data is full + */ + rc = inflate(strm, Z_NO_FLUSH); + if (rc == Z_STREAM_END) + /* end of stream */ + break; + else if (rc != Z_OK) + { + /* got an error */ + elog(WARNING, "Thread [%d]: Decompression failed for file '%s': %i: %s", + thread_num, from_fullpath, rc, strm->msg); + exit_code = ZLIB_ERROR; + goto cleanup; + } + + if (strm->avail_out == 0) + { + /* Output buffer is full, write it out */ + if (fwrite(out_buf, 1, OUT_BUF_SIZE, out) != OUT_BUF_SIZE) + { + elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + exit_code = WRITE_FAILED; + goto cleanup; + } + + strm->next_out = (Bytef *)out_buf; /* output buffer */ + strm->avail_out = OUT_BUF_SIZE; + } + } + + /* write out leftovers if any */ + if (strm->avail_out != OUT_BUF_SIZE) + { + int len = OUT_BUF_SIZE - strm->avail_out; + + if (fwrite(out_buf, 1, len, out) != len) + { + elog(WARNING, "Thread [%d]: Cannot write to file: %s", + thread_num, strerror(errno)); + exit_code = WRITE_FAILED; + goto cleanup; + } + } + } + else + { + elog(WARNING, "Thread [%d]: Remote agent returned message of unexpected type: %i", + thread_num, hdr.cop); + exit_code = REMOTE_ERROR; + break; + } + } + +cleanup: + if (exit_code < OPEN_FAILED) + fio_disconnect(); /* discard possible pending data in pipe */ + + if (strm) + { + inflateEnd(strm); + pg_free(strm); + } + + pg_free(in_buf); + pg_free(out_buf); + return exit_code; +} + +/* Receive chunks of data and write them to destination file. + * Return codes: + * SEND_OK (0) + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAIL (-3) + * WRITE_FAIL (-4) + */ +int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num) +{ + fio_header hdr; + int exit_code = SEND_OK; + size_t path_len = strlen(from_fullpath) + 1; + char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */ + + hdr.cop = FIO_SEND_FILE; + hdr.size = path_len; + + elog(VERBOSE, "Thread [%d]: Attempting to open remote WAL file '%s'", + thread_num, from_fullpath); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); + + for (;;) + { + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_SEND_FILE_EOF) + { + break; + } + else if (hdr.cop == FIO_ERROR) + { + /* handle error, reported by the agent */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + elog(WARNING, "Thread [%d]: %s", thread_num, buf); + } + exit_code = hdr.arg; + break; + } + else if (hdr.cop == FIO_PAGE) + { + Assert(hdr.size <= CHUNK_SIZE); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + /* We have received a chunk of data data, lets write it out */ + if (fwrite(buf, 1, hdr.size, out) != hdr.size) + { + elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + exit_code = WRITE_FAILED; + break; + } + } + else + { + /* TODO: fio_disconnect may get assert fail when running after this */ + elog(WARNING, "Thread [%d]: Remote agent returned message of unexpected type: %i", + thread_num, hdr.cop); + exit_code = REMOTE_ERROR; + break; + } + } + + if (exit_code < OPEN_FAILED) + fio_disconnect(); /* discard possible pending data in pipe */ + + pg_free(buf); + return exit_code; +} + +/* Send file content + * On error we return FIO_ERROR message with following codes + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + * + */ +static void fio_send_file_impl(int out, char const* path) +{ + FILE *fp; + fio_header hdr; + char *buf = pgut_malloc(CHUNK_SIZE); + ssize_t read_len = 0; + char *errormsg = NULL; + + /* open source file for read */ + /* TODO: check that file is regular file */ + fp = fopen(path, PG_BINARY_R); + if (!fp) + { + hdr.cop = FIO_ERROR; + + /* do not send exact wording of ENOENT error message + * because it is a very common error in our case, so + * error code is enough. + */ + if (errno == ENOENT) + { + hdr.arg = FILE_MISSING; + hdr.size = 0; + } + else + { + hdr.arg = OPEN_FAILED; + errormsg = pgut_malloc(MAXPGPATH); + /* Construct the error message */ + snprintf(errormsg, MAXPGPATH, "Cannot open source file '%s': %s", path, strerror(errno)); + hdr.size = strlen(errormsg) + 1; + } + + /* send header and message */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (errormsg) + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + + goto cleanup; + } + + /* copy content */ + for (;;) + { + read_len = fread(buf, 1, CHUNK_SIZE, fp); + + /* report error */ + if (read_len < 0 || (read_len == 0 && !feof(fp))) + { + hdr.cop = FIO_ERROR; + errormsg = pgut_malloc(MAXPGPATH); + hdr.arg = READ_FAILED; + /* Construct the error message */ + snprintf(errormsg, MAXPGPATH, "Cannot read source file '%s': %s", path, strerror(errno)); + hdr.size = strlen(errormsg) + 1; + /* send header and message */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + + goto cleanup; + } + + else if (read_len == 0) + break; + else + { + /* send chunk */ + hdr.cop = FIO_PAGE; + hdr.size = read_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, read_len), read_len); + } + } + + /* we are done, send eof */ + hdr.cop = FIO_SEND_FILE_EOF; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + +cleanup: + if (fp) + fclose(fp); + pg_free(buf); + pg_free(errormsg); + return; +} + /* Execute commands at remote host */ void fio_communicate(int in, int out) { @@ -1665,7 +1998,7 @@ void fio_communicate(int in, int out) } switch (hdr.cop) { case FIO_LOAD: /* Send file content */ - fio_send_file(out, buf); + fio_load_file(out, buf); break; case FIO_OPENDIR: /* Open directory for traversal */ dir[hdr.handle] = opendir(buf); @@ -1776,6 +2109,9 @@ void fio_communicate(int in, int out) // buf contain fio_send_request header and bitmap. fio_send_pages_impl(fd[hdr.handle], out, buf, true); break; + case FIO_SEND_FILE: + fio_send_file_impl(out, buf); + break; case FIO_SYNC: /* open file and fsync it */ tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); diff --git a/src/utils/file.h b/src/utils/file.h index 58c4e44a5..6dcdfcb1e 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -40,6 +40,8 @@ typedef enum FIO_SEND_PAGES, FIO_SEND_PAGES_PAGEMAP, FIO_ERROR, + FIO_SEND_FILE, +// FIO_CHUNK, FIO_SEND_FILE_EOF, FIO_SEND_FILE_CORRUPTION, /* messages for closing connection */ diff --git a/tests/archive.py b/tests/archive.py index 8d96dcb19..59496b2f0 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -398,7 +398,7 @@ def test_archive_push_file_exists(self): log_content) self.assertIn( - 'pg_probackup push file', + 'pg_probackup archive-push WAL file', log_content) self.assertIn( @@ -482,7 +482,7 @@ def test_archive_push_file_exists_overwrite(self): self.assertIn( 'DETAIL: The failed archive command was:', log_content) self.assertIn( - 'pg_probackup push file', log_content) + 'pg_probackup archive-push WAL file', log_content) self.assertNotIn( 'WAL file already exists in archive with ' 'different checksum, overwriting', log_content) @@ -941,11 +941,11 @@ def test_basic_master_and_replica_concurrent_archiving(self): pgdata_replica = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata_master, pgdata_replica) - self.set_replica(master, replica, synchronous=True) + self.set_replica(master, replica, synchronous=False) # ADD INSTANCE REPLICA # self.add_instance(backup_dir, 'replica', replica) # SET ARCHIVING FOR REPLICA - # self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_archiving(backup_dir, 'master', replica, replica=True) replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA @@ -977,6 +977,18 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.assertEqual( 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + master.pgbench_init(scale=50) + + sleep(10) + + replica.promote() + + master.pgbench_init(scale=10) + replica.pgbench_init(scale=10) + + + exit(1) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -1984,6 +1996,11 @@ def test_archive_pg_receivexlog_partial_handling(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) self.set_auto_conf(node_restored, {'hot_standby': 'off'}) + # it will set node_restored as warm standby. +# with open(os.path.join(node_restored.data_dir, "standby.signal"), 'w') as f: +# f.flush() +# f.close() + node_restored.slow_start() result = node.safe_psql( @@ -2000,7 +2017,326 @@ def test_archive_pg_receivexlog_partial_handling(self): pg_receivexlog.kill() self.del_test_dir(module_name, fname) + def test_multi_timeline_recovery_prefetching(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=50) + + target_xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + node.pgbench_init(scale=20) + + node.stop() + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-action=promote']) + + node.slow_start() + + node.pgbench_init(scale=20) + + target_xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ +# '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', +# '--recovery-target-action=promote', + '--no-validate']) + node.slow_start() + + node.pgbench_init(scale=20) + result = node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + node.stop() + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ +# '--recovery-target-xid=100500', + '--recovery-target-timeline=3', +# '--recovery-target-action=promote', + '--no-validate']) + os.remove(os.path.join(node.logs_dir, 'postgresql.log')) + + restore_command = self.get_restore_command(backup_dir, 'node', node) + restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' + + if node.major_version >= 12: + node.append_conf( + 'probackup_recovery.conf', "restore_command = '{0}'".format(restore_command)) + else: + node.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + node.slow_start() + + result_new = node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + + self.assertEqual(result, result_new) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + # check that requesting of non-existing segment do not + # throwns aways prefetch + self.assertIn( + 'pg_probackup archive-get failed to ' + 'deliver WAL file: 000000030000000000000006', + postgres_log_content) + + self.assertIn( + 'pg_probackup archive-get failed to ' + 'deliver WAL file: 000000020000000000000006', + postgres_log_content) + + self.assertIn( + 'pg_probackup archive-get used prefetched ' + 'WAL segment 000000010000000000000006, prefetch state: 5/10', + postgres_log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_archive_get_batching_sanity(self): + """ + Make sure that batching works. + .gz file is corrupted and uncompressed is not, check that both + corruption detected and uncompressed file is used. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=50) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, replica.data_dir) + self.set_replica(node, replica, log_shipping=True) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + else: + replica.append_conf('recovery.conf', "restore_command = 'exit 1'") + + replica.slow_start(replica=True) + + # at this point replica is consistent + restore_command = self.get_restore_command(backup_dir, 'node', replica) + + restore_command += ' -j 2 --batch-size=10' + + print(restore_command) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + replica.restart() + + sleep(5) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'pg_probackup archive-get completed successfully, fetched: 10/10', + postgres_log_content) + self.assertIn('used prefetched WAL segment', postgres_log_content) + self.assertIn('prefetch state: 9/10', postgres_log_content) + self.assertIn('prefetch state: 8/10', postgres_log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_archive_get_prefetch_corruption(self): + """ + Make sure that WAL corruption is detected. + And --prefetch-dir is honored. + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_keep_segments': '200'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=50) + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, replica.data_dir) + self.set_replica(node, replica, log_shipping=True) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + else: + replica.append_conf('recovery.conf', "restore_command = 'exit 1'") + + replica.slow_start(replica=True) + + # at this point replica is consistent + restore_command = self.get_restore_command(backup_dir, 'node', replica) + + restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' + #restore_command += ' --batch-size=2 --log-level-console=VERBOSE' + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + replica.restart() + + sleep(5) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'pg_probackup archive-get completed successfully, fetched: 10/10', + postgres_log_content) + self.assertIn('used prefetched WAL segment', postgres_log_content) + self.assertIn('prefetch state: 9/10', postgres_log_content) + self.assertIn('prefetch state: 8/10', postgres_log_content) + + replica.stop() + + # generate WAL, copy it into prefetch directory, then corrupt + # some segment + node.pgbench_init(scale=20) + sleep(10) + + # now copy WAL files into prefetch directory and corrupt some of them + archive_dir = os.path.join(backup_dir, 'wal', 'node') + files = os.listdir(archive_dir) + files.sort() + + for filename in [files[-4], files[-3], files[-2], files[-1]]: + src_file = os.path.join(archive_dir, filename) + + if node.major_version >= 10: + wal_dir = 'pg_wal' + else: + wal_dir = 'pg_xlog' + + if filename.endswith('.gz'): + dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename[:-3]) + with gzip.open(src_file, 'rb') as f_in, open(dst_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + else: + dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) + shutil.copyfile(src_file, dst_file) + + print(dst_file) + + # corrupt file + if files[-2].endswith('.gz'): + filename = files[-2][:-3] + else: + filename = files[-2] + + prefetched_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) + + with open(prefetched_file, "rb+", 0) as f: + f.seek(8192*2) + f.write(b"SURIKEN") + f.flush() + f.close + + # enable restore_command + restore_command = self.get_restore_command(backup_dir, 'node', replica) + restore_command += ' --batch-size=2 --log-level-console=VERBOSE' + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) + replica.slow_start(replica=True) + + sleep(10) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), + postgres_log_content) + + self.assertIn( + 'LOG: restored log file "{0}" from archive'.format(filename), + postgres_log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # TODO test with multiple not archived segments. +# TODO corrupted file in archive. # important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. # so write WAL validation code accordingly diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d7ddfa037..14109ba57 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -339,7 +339,7 @@ def make_simple_node( options['wal_level'] = 'logical' options['hot_standby'] = 'off' - options['log_line_prefix'] = '"%t [%p]: [%l-1] "' + options['log_line_prefix'] = '%t [%p]: [%l-1] ' options['log_statement'] = 'none' options['log_duration'] = 'on' options['log_min_duration_statement'] = 0 @@ -1259,7 +1259,8 @@ def set_auto_conf(self, node, options, config='postgresql.auto.conf'): def set_replica( self, master, replica, replica_name='replica', - synchronous=False + synchronous=False, + log_shipping=False ): self.set_auto_conf( @@ -1279,19 +1280,22 @@ def set_replica( if os.stat(probackup_recovery_path).st_size > 0: config = 'probackup_recovery.conf' - self.set_auto_conf( - replica, - {'primary_conninfo': 'user={0} port={1} application_name={2} ' - ' sslmode=prefer sslcompression=1'.format( - self.user, master.port, replica_name)}, - config) + if not log_shipping: + self.set_auto_conf( + replica, + {'primary_conninfo': 'user={0} port={1} application_name={2} ' + ' sslmode=prefer sslcompression=1'.format( + self.user, master.port, replica_name)}, + config) else: replica.append_conf('recovery.conf', 'standby_mode = on') - replica.append_conf( - 'recovery.conf', - "primary_conninfo = 'user={0} port={1} application_name={2}" - " sslmode=prefer sslcompression=1'".format( - self.user, master.port, replica_name)) + + if not log_shipping: + replica.append_conf( + 'recovery.conf', + "primary_conninfo = 'user={0} port={1} application_name={2}" + " sslmode=prefer sslcompression=1'".format( + self.user, master.port, replica_name)) if synchronous: self.set_auto_conf( From 2fdab7c387edf2bc1475b9ebfa8629070f691d6b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 Apr 2020 13:53:55 +0300 Subject: [PATCH 1215/2107] [Issue #120] fix tests.delete.DeleteTest.test_delete_error_backups --- tests/delete.py | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/delete.py b/tests/delete.py index 0f5be6bea..8ebd7d13a 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -825,25 +825,51 @@ def test_delete_error_backups(self): # Take PAGE BACKUP backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_c = self.backup_node(backup_dir, 'node', node, backup_type="page") + + backup_id_d = self.backup_node(backup_dir, 'node', node, backup_type="page") + + # full backup mode + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_e = self.backup_node(backup_dir, 'node', node, backup_type="page") + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Change status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_c, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_e, 'ERROR') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.show_pb(backup_dir, as_text=True, as_json=False)) show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 4) + self.assertEqual(len(show_backups), 10) # delete error backups - self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) + output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR', '--dry-run']) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 10) + + self.assertIn( + "Deleting all backups with status 'ERROR' in dry run mode", + output) + + self.assertIn( + "INFO: Backup {0} with status OK can be deleted".format(backup_id_d), + output) - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.show_pb(backup_dir, as_text=True, as_json=False)) show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 2) + output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) + print(output) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) - self.assertEqual(show_backups[0]['status'], "OK") - self.assertEqual(show_backups[1]['status'], "OK") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['status'], "OK") + self.assertEqual(show_backups[2]['status'], "OK") + self.assertEqual(show_backups[3]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) + # Clean after yourself + self.del_test_dir(module_name, fname) From 4ee66be4c90e7058a0f0d0aca4aaca62f0202623 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 Apr 2020 13:57:05 +0300 Subject: [PATCH 1216/2107] [Issue #120] minor improvements --- src/catalog.c | 20 +++++++++++ src/delete.c | 82 +++++++++++++++------------------------------- src/help.c | 4 +-- src/pg_probackup.h | 2 +- src/utils/parray.c | 13 ++++++++ src/utils/parray.h | 1 + 6 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index f53c9b38d..ca723083a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2447,3 +2447,23 @@ get_backup_index_number(parray *backup_list, pgBackup *backup) elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time)); return -1; } + +/* On backup_list lookup children of target_backup and append them to append_list */ +void +append_children(parray *backup_list, pgBackup *target_backup, parray *append_list) +{ + int i; + + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + + /* check if backup is descendant of target backup */ + if (is_parent(target_backup->start_time, backup, false)) + { + /* if backup is already in the list, then skip it */ + if (!parray_contains(append_list, backup)) + parray_append(append_list, backup); + } + } +} diff --git a/src/delete.c b/src/delete.c index 004fd239b..85d273479 100644 --- a/src/delete.c +++ b/src/delete.c @@ -1021,48 +1021,16 @@ do_delete_instance(void) return 0; } -/* checks that parray contains element */ -bool parray_contains(parray *array, void *elem) -{ - int i; - for (i = 0; i < parray_num(array); i++) - { - if (parray_get(array, i) == elem) return true; - } - return false; -} - - -void append_childs(parray *backup_list, pgBackup *target_backup, parray *delete_list) -{ - int i; - pgBackup *backup; - for (i = 0; i < parray_num(backup_list); i++) - { - backup = (pgBackup *)parray_get(backup_list, i); - if (backup == target_backup) continue; - /* check if backup is descendant of delete target */ - if (is_parent(target_backup->start_time, backup, false)) - { - if (!parray_contains(delete_list, backup)) - parray_append(delete_list, backup); - /* recursive call */ - append_childs(backup_list, backup, delete_list); - } - } - -} - /* Delete all backups of given status in instance */ void do_delete_status(InstanceConfig *instance_config, const char *status) { - parray *backup_list, *delete_list;; - int i; - const char *pretty_status; - int n_deleted = 0, n_found = 0; - size_t size_to_delete = 0; - char size_to_delete_pretty[20]; + int i; + parray *backup_list, *delete_list; + const char *pretty_status; + int n_deleted = 0, n_found = 0; + size_t size_to_delete = 0; + char size_to_delete_pretty[20]; pgBackup *backup; BackupStatus status_for_delete = str2status(status); @@ -1086,9 +1054,12 @@ do_delete_status(InstanceConfig *instance_config, const char *status) return; } - elog(INFO, "Deleting all backups with status '%s'", pretty_status); + if (dry_run) + elog(INFO, "Deleting all backups with status '%s' in dry run mode", pretty_status); + else + elog(INFO, "Deleting all backups with status '%s'", pretty_status); - /* Selects backups for deleting to delete_list array. Will delete all backups with specified status and childs*/ + /* Selects backups with specified status and their children into delete_list array. */ for (i = 0; i < parray_num(backup_list); i++) { backup = (pgBackup *) parray_get(backup_list, i); @@ -1096,34 +1067,36 @@ do_delete_status(InstanceConfig *instance_config, const char *status) if (backup->status == status_for_delete) { n_found++; - if (parray_contains(delete_list, backup)) continue; + + /* incremental backup can be already in delete_list due to append_children() */ + if (parray_contains(delete_list, backup)) + continue; parray_append(delete_list, backup); - append_childs(backup_list, backup, delete_list); + + append_children(backup_list, backup, delete_list); } } + + parray_qsort(delete_list, pgBackupCompareIdDesc); + /* delete and calculate free size from delete_list */ for (i = 0; i < parray_num(delete_list); i++) { backup = (pgBackup *)parray_get(delete_list, i); - elog(dry_run ? INFO : LOG, "Backup %s with status %s %s be deleted", + + elog(INFO, "Backup %s with status %s %s be deleted", base36enc(backup->start_time), status2str(backup->status), dry_run ? "can" : "will"); size_to_delete += backup->data_bytes; if (backup->stream) size_to_delete += backup->wal_bytes; - if (!dry_run && lock_backup(backup)) - { - if (interrupted) - elog(ERROR, "interrupted during delete backup"); - delete_backup_files(backup); - n_deleted++; - - } + n_deleted++; } + /* Inform about data size to free */ if (size_to_delete >= 0) { @@ -1133,7 +1106,7 @@ do_delete_status(InstanceConfig *instance_config, const char *status) } /* delete selected backups */ - if (!dry_run) + if (!dry_run && n_deleted > 0) elog(INFO, "Successfully deleted %i %s from instance '%s'", n_deleted, n_deleted == 1 ? "backup" : "backups", instance_config->name); @@ -1143,9 +1116,8 @@ do_delete_status(InstanceConfig *instance_config, const char *status) elog(WARNING, "Instance '%s' has no backups with status '%s'", instance_config->name, pretty_status); - /* Clean WAL segments */ - if (delete_wal) - do_retention_wal(dry_run); + // we don`t do WAL purge here, because it is impossible to correctly handle + // dry-run case. /* Cleanup */ parray_free(delete_list); diff --git a/src/help.c b/src/help.c index 2f06e89b3..68d7c337d 100644 --- a/src/help.c +++ b/src/help.c @@ -638,8 +638,8 @@ help_delete(void) printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); printf(_(" --dry-run perform a trial run without any changes\n")); - printf(_(" --status=backups_status delete all backups with specific status\n")); - + printf(_(" --status=backup_status delete all backups with specified status\n")); + printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 049ddee5a..bbf556a0c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -808,8 +808,8 @@ extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern bool is_prolific(parray *backup_list, pgBackup *target_backup); -extern bool in_backup_list(parray *backup_list, pgBackup *target_backup); extern int get_backup_index_number(parray *backup_list, pgBackup *backup); +extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list); extern bool launch_agent(void); extern void launch_ssh(char* argv[]); extern void wait_ssh(void); diff --git a/src/utils/parray.c b/src/utils/parray.c index 23c227b19..31148ee9a 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -197,3 +197,16 @@ parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const { return bsearch(&key, array->data, array->used, sizeof(void *), compare); } + +/* checks that parray contains element */ +bool parray_contains(parray *array, void *elem) +{ + int i; + + for (i = 0; i < parray_num(array); i++) + { + if (parray_get(array, i) == elem) + return true; + } + return false; +} diff --git a/src/utils/parray.h b/src/utils/parray.h index 833a6961b..85d7383f3 100644 --- a/src/utils/parray.h +++ b/src/utils/parray.h @@ -30,6 +30,7 @@ extern size_t parray_num(const parray *array); extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); extern void parray_walk(parray *array, void (*action)(void *)); +extern bool parray_contains(parray *array, void *elem); #endif /* PARRAY_H */ From 0b449d7e0b9db8e0668d34f37e4085e96a6389fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 Apr 2020 14:16:12 +0300 Subject: [PATCH 1217/2107] [Issue #120] Documentation update --- doc/pgprobackup.xml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index e24d31fef..0efbf2c0d 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3262,7 +3262,11 @@ pg_probackup delete -B backup_dir --instance pg_probackup delete -B backup_dir --instance instance_name --status=ERROR - + + + + Deleting backups by status ignores established retention policies. + @@ -3942,10 +3946,9 @@ pg_probackup merge -B backup_dir --instance pg_probackup delete -B backup_dir --instance instance_name [--help] [-j num_threads] [--progress] -[--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] -[--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status= } -[--dry-run] -[logging_options] +[--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--delete-wal] +{-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status=backup_status} +[--dry-run] [logging_options] Deletes backup with specified backup_id From aa56f1d485006d5126ad7cb61156d483c05eab03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 Apr 2020 14:20:02 +0300 Subject: [PATCH 1218/2107] [Issue #120] update help --- src/help.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/help.c b/src/help.c index 68d7c337d..a048afba0 100644 --- a/src/help.c +++ b/src/help.c @@ -193,7 +193,8 @@ help_pg_probackup(void) printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--delete-wal] [-i backup-id | --delete-expired | --merge-expired | --status= ]\n")); + printf(_(" [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n")); + printf(_(" [--delete-wal]\n")); printf(_(" [--dry-run]\n")); printf(_(" [--help]\n")); From 30fbea7d97e94d666b1afec5cf4e0adcd5222126 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 18 Apr 2020 22:53:05 +0300 Subject: [PATCH 1219/2107] bugfix: "note" and "expire-time" backup metainformation fields must be preserved during merge --- src/merge.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/merge.c b/src/merge.c index 06da8f4bd..bb2c0d698 100644 --- a/src/merge.c +++ b/src/merge.c @@ -702,6 +702,15 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->compress_alg = dest_backup->compress_alg; full_backup->compress_level = dest_backup->compress_level; + /* If incremental backup is pinned, + * then result FULL backup must also be pinned. + */ + if (dest_backup->expire_time) + full_backup->expire_time = dest_backup->expire_time; + + if (dest_backup->note) + full_backup->note = pgut_strdup(dest_backup->note); + /* FULL backup must inherit wal mode. */ full_backup->stream = dest_backup->stream; From 2c1c19a42b58bcc31f34450963ddf86fec104aa5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 19 Apr 2020 02:49:35 +0300 Subject: [PATCH 1220/2107] [Issue #146] Refactoring --- src/backup.c | 12 ++++----- src/catalog.c | 65 ++++++++++++++++++++++++++++++++++++---------- src/pg_probackup.c | 13 ++++++---- src/pg_probackup.h | 12 +++++---- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/src/backup.c b/src/backup.c index 66d5de50e..258e3c002 100644 --- a/src/backup.c +++ b/src/backup.c @@ -156,7 +156,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* used for multitimeline incremental backup */ parray *tli_list = NULL; - /* for fancy reporting */ time_t start_time, end_time; char pretty_time[20]; @@ -820,9 +819,7 @@ do_backup(time_t start_time, bool no_validate, /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; current.start_time = start_time; - - current.note = set_backup_params ? set_backup_params->note : NULL; - + StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); @@ -905,6 +902,10 @@ do_backup(time_t start_time, bool no_validate, if (instance_config.master_conn_opt.pghost == NULL) elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); + /* add note to backup if requested */ + if (set_backup_params && set_backup_params->note) + add_note(¤t, set_backup_params->note); + /* backup data */ do_backup_instance(backup_conn, &nodeInfo, no_sync); pgut_atexit_pop(backup_cleanup, NULL); @@ -937,8 +938,7 @@ do_backup(time_t start_time, bool no_validate, (set_backup_params->ttl > 0 || set_backup_params->expire_time > 0)) { - if (!pin_backup(¤t, set_backup_params)) - elog(ERROR, "Failed to pin the backup %s", base36enc(current.backup_id)); + pin_backup(¤t, set_backup_params); } if (!no_validate) diff --git a/src/catalog.c b/src/catalog.c index a1323d18f..f1faaaec2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1534,18 +1534,23 @@ do_set_backup(const char *instance_name, time_t backup_id, target_backup = (pgBackup *) parray_get(backup_list, 0); - if (!pin_backup(target_backup, set_backup_params)) - elog(ERROR, "Failed to pin the backup %s", base36enc(backup_id)); + /* Pin or unpin backup if requested */ + if (set_backup_params->ttl >= 0 || set_backup_params->expire_time > 0) + pin_backup(target_backup, set_backup_params); + + if (set_backup_params->note) + add_note(target_backup, set_backup_params->note); } /* * Set 'expire-time' attribute based on set_backup_params, or unpin backup * if ttl is equal to zero. */ -bool +void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) { + /* sanity, backup must have positive recovery-time */ if (target_backup->recovery_time <= 0) elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'", base36enc(target_backup->backup_id)); @@ -1563,17 +1568,16 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) { elog(WARNING, "Backup %s is not pinned, nothing to unpin", base36enc(target_backup->start_time)); - return false; + return; } target_backup->expire_time = 0; } /* Pin comes from expire-time */ else if (set_backup_params->expire_time > 0) target_backup->expire_time = set_backup_params->expire_time; - else if (!set_backup_params->note) - return false; - - if (set_backup_params->note) target_backup->note = set_backup_params->note; + else + /* nothing to do */ + return; /* Update backup.control */ write_backup(target_backup); @@ -1586,12 +1590,47 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), expire_timestamp); } - else if (set_backup_params->ttl == 0) + else elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time)); - if (set_backup_params->note) - elog(INFO, "Saved note for backup %s", base36enc(target_backup->start_time)); - return true; + return; +} + +/* + * Add note to backup metadata or unset already existing note. + * It is a job of the caller to make sure that note is not NULL. + */ +void +add_note(pgBackup *target_backup, char *note) +{ + + char *note_string; + + /* unset note */ + if (pg_strcasecmp(note, "none") == 0) + { + target_backup->note = NULL; + elog(INFO, "Removing note from backup %s", + base36enc(target_backup->start_time)); + } + else + { + /* Currently we do not allow string with newlines as note, + * because it will break parsing of backup.control. + * So if user provides string like this "aaa\nbbbbb", + * we save only "aaa" + * Example: tests.set_backup.SetBackupTest.test_add_note_newlines + */ + note_string = pgut_malloc(MAX_NOTE_SIZE); + sscanf(note, "%[^\n]", note_string); + + target_backup->note = note_string; + elog(INFO, "Adding note to backup %s: '%s'", + base36enc(target_backup->start_time), target_backup->note); + } + + /* Update backup.control */ + write_backup(target_backup); } /* @@ -1688,7 +1727,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); if (backup->note) - fio_fprintf(out, "note = %s\n", backup->note); + fio_fprintf(out, "note = '%s'\n", backup->note); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 157ed5f26..e7c3dce68 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -75,10 +75,10 @@ char *replication_slot = NULL; bool temp_slot = false; /* backup options */ -bool backup_logs = false; -bool smooth_checkpoint; -char *remote_agent; -static char *backup_note = NULL; +bool backup_logs = false; +bool smooth_checkpoint; +char *remote_agent; +static char *backup_note = NULL; /* restore options */ static char *target_time = NULL; static char *target_xid = NULL; @@ -183,7 +183,7 @@ static ConfigOption cmd_options[] = { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, - { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, + { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -753,6 +753,9 @@ main(int argc, char *argv[]) set_backup_params->ttl = ttl; set_backup_params->expire_time = expire_time; set_backup_params->note = backup_note; + + if (backup_note && strlen(backup_note) > MAX_NOTE_SIZE) + elog(ERROR, "Backup note cannot exceed %u bytes", MAX_NOTE_SIZE); } } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8ae815241..275956f8b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -90,6 +90,8 @@ extern const char *PROGRAM_EMAIL; /* retry attempts */ #define PAGE_READ_ATTEMPTS 100 +#define MAX_NOTE_SIZE 1024 + /* Check if an XLogRecPtr value is pointed to 0 offset */ #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) @@ -391,7 +393,6 @@ struct pgBackup parray *files; /* list of files belonging to this backup * must be populated explicitly */ char *note; - }; /* Recovery target for restore and validate subcommands */ @@ -435,14 +436,14 @@ typedef struct pgRestoreParams /* Options needed for set-backup command */ typedef struct pgSetBackupParams { - int64 ttl; /* amount of time backup must be pinned + int64 ttl; /* amount of time backup must be pinned * -1 - do nothing * 0 - disable pinning */ - time_t expire_time; /* Point in time before which backup + time_t expire_time; /* Point in time until backup * must be pinned. */ - char *note; + char *note; } pgSetBackupParams; typedef struct @@ -781,8 +782,9 @@ extern void timelineInfoFree(void *tliInfo); extern parray *catalog_get_timelines(InstanceConfig *instance); extern void do_set_backup(const char *instance_name, time_t backup_id, pgSetBackupParams *set_backup_params); -extern bool pin_backup(pgBackup *target_backup, +extern void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params); +extern void add_note(pgBackup *target_backup, char *note); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list); From 70143fca17ca8480caa024dc784a17b2992291b3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 19 Apr 2020 02:50:49 +0300 Subject: [PATCH 1221/2107] [Issue #146] tests update --- tests/backup.py | 10 +- tests/merge.py | 61 +++++++- tests/retention.py | 329 -------------------------------------------- tests/set_backup.py | 87 ++++++++++++ 4 files changed, 153 insertions(+), 334 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 9a44ed70f..e04930091 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2726,7 +2726,7 @@ def test_note_sanity(self): node.slow_start() # FULL backup - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, options=['--stream', '--log-level-file=LOG', '--note=test_note']) @@ -2736,5 +2736,13 @@ def test_note_sanity(self): self.assertEqual(show_backups[0]['note'], "test_note") + self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + + self.assertNotIn( + 'note', + backup_meta) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/merge.py b/tests/merge.py index 8f6ccbb57..2bc096685 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -22,8 +22,7 @@ def test_basic_merge_full_page(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=["--data-checksums"] - ) + initdb_params=["--data-checksums"]) self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) @@ -1981,8 +1980,7 @@ def test_merge_backup_from_future(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2384,5 +2382,60 @@ def test_idempotent_merge(self): self.del_test_dir(module_name, fname) + def test_merge_correct_inheritance(self): + """ + Make sure that backup metainformation fields + 'note' and 'expire-time' are correctly inherited + during merge + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.set_backup( + backup_dir, 'node', page_id, options=['--note=hello', '--ttl=20d']) + + page_meta = self.show_pb(backup_dir, 'node', page_id) + + self.merge_backup(backup_dir, 'node', page_id) + + print(self.show_pb(backup_dir, 'node', page_id)) + + self.assertEqual( + page_meta['note'], + self.show_pb(backup_dir, 'node', page_id)['note']) + + self.assertEqual( + page_meta['expire-time'], + self.show_pb(backup_dir, 'node', page_id)['expire-time']) + + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels diff --git a/tests/retention.py b/tests/retention.py index 4885187f9..e797d3c60 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1840,335 +1840,6 @@ def test_failed_merge_redundancy_retention(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure - @unittest.skip("skip") - def test_wal_depth(self): - """ - ARCHIVE replica: - - t6 |----------------------> - t5 | |------> - | | - t4 | |----|------> - | | - t3 | |--B1--|/|--B2-|/|-B3--> - | | - t2 |--A1-----|--A2---> - t1 ---------Y1--Y2-| - - ARCHIVE master: - t1 -Z1--Z2--> - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - - master.slow_start() - - # FULL - master.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - self.backup_node(backup_dir, 'master', master) - - # PAGE - master.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - - self.backup_node( - backup_dir, 'master', master, backup_type='page') - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - self.set_replica(master, replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - - # Check data correctness on replica - replica.slow_start(replica=True) - - # FULL backup replica - Y1 = self.backup_node( - backup_dir, 'replica', replica, - options=['--stream', '--archive-timeout=60s']) - - master.pgbench_init(scale=5) - - # PAGE backup replica - Y2 = self.backup_node( - backup_dir, 'replica', replica, - backup_type='page', options=['--stream', '--archive-timeout=60s']) - - # create timeline t2 - replica.promote() - - # do checkpoint to increment timeline ID in pg_control - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - - # FULL backup replica - A1 = self.backup_node( - backup_dir, 'replica', replica) - - replica.pgbench_init(scale=5) - - replica.safe_psql( - 'postgres', - "CREATE TABLE t1 (a text)") - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - # DELTA backup replica - A2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='delta') - - # create timeline t3 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - replica.slow_start() - - B1 = self.backup_node( - backup_dir, 'replica', replica) - - replica.pgbench_init(scale=2) - - B2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') - - replica.pgbench_init(scale=2) - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - B3 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') - - replica.pgbench_init(scale=2) - - # create timeline t4 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=3', - '--recovery-target-action=promote']) - - replica.slow_start() - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't2 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,6) i') - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't3 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,10) i') - - # create timeline t5 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=4', - '--recovery-target-action=promote']) - - replica.slow_start() - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't4 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,6) i') - - # create timeline t6 - replica.cleanup() - - self.restore_node( - backup_dir, 'replica', replica, backup_id=A1, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - replica.slow_start() - - replica.pgbench_init(scale=2) - - show = self.show_archive(backup_dir, as_text=True) - show = self.show_archive(backup_dir) - - for instance in show: - if instance['instance'] == 'replica': - replica_timelines = instance['timelines'] - - if instance['instance'] == 'master': - master_timelines = instance['timelines'] - - # check that all timelines are ok - for timeline in replica_timelines: - self.assertTrue(timeline['status'], 'OK') - - # check that all timelines are ok - for timeline in master_timelines: - self.assertTrue(timeline['status'], 'OK') - - # create holes in t3 - wals_dir = os.path.join(backup_dir, 'wal', 'replica') - wals = [ - f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) - and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') - ] - wals.sort() - - # check that t3 is ok - self.show_archive(backup_dir) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - # check that t3 is not OK - show = self.show_archive(backup_dir) - - show = self.show_archive(backup_dir) - - for instance in show: - if instance['instance'] == 'replica': - replica_timelines = instance['timelines'] - - # sanity - for timeline in replica_timelines: - if timeline['tli'] == 1: - timeline_1 = timeline - continue - - if timeline['tli'] == 2: - timeline_2 = timeline - continue - - if timeline['tli'] == 3: - timeline_3 = timeline - continue - - if timeline['tli'] == 4: - timeline_4 = timeline - continue - - if timeline['tli'] == 5: - timeline_5 = timeline - continue - - if timeline['tli'] == 6: - timeline_6 = timeline - continue - - self.assertEqual(timeline_6['status'], "OK") - self.assertEqual(timeline_5['status'], "OK") - self.assertEqual(timeline_4['status'], "OK") - self.assertEqual(timeline_3['status'], "DEGRADED") - self.assertEqual(timeline_2['status'], "OK") - self.assertEqual(timeline_1['status'], "OK") - - self.assertEqual(len(timeline_3['lost-segments']), 2) - self.assertEqual(timeline_3['lost-segments'][0]['begin-segno'], '0000000000000012') - self.assertEqual(timeline_3['lost-segments'][0]['end-segno'], '0000000000000013') - self.assertEqual(timeline_3['lost-segments'][1]['begin-segno'], '0000000000000017') - self.assertEqual(timeline_3['lost-segments'][1]['end-segno'], '0000000000000017') - - self.assertEqual(len(timeline_6['backups']), 0) - self.assertEqual(len(timeline_5['backups']), 0) - self.assertEqual(len(timeline_4['backups']), 0) - self.assertEqual(len(timeline_3['backups']), 3) - self.assertEqual(len(timeline_2['backups']), 2) - self.assertEqual(len(timeline_1['backups']), 2) - - # check closest backup correctness - self.assertEqual(timeline_6['closest-backup-id'], A1) - self.assertEqual(timeline_5['closest-backup-id'], B2) - self.assertEqual(timeline_4['closest-backup-id'], B2) - self.assertEqual(timeline_3['closest-backup-id'], A1) - self.assertEqual(timeline_2['closest-backup-id'], Y2) - - # check parent tli correctness - self.assertEqual(timeline_6['parent-tli'], 2) - self.assertEqual(timeline_5['parent-tli'], 4) - self.assertEqual(timeline_4['parent-tli'], 3) - self.assertEqual(timeline_3['parent-tli'], 2) - self.assertEqual(timeline_2['parent-tli'], 1) - self.assertEqual(timeline_1['parent-tli'], 0) - - output = self.delete_pb( - backup_dir, 'replica', - options=['--delete-wal', '--log-level-console=verbose']) - - self.validate_pb(backup_dir, 'node') - - self.del_test_dir(module_name, fname) - def test_wal_depth_1(self): """ |-------------B5----------> WAL timeline3 diff --git a/tests/set_backup.py b/tests/set_backup.py index b8d97ad5a..861de7564 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -383,5 +383,92 @@ def test_wal_retention_and_pinning_1(self): self.validate_pb(backup_dir) + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_add_note_newlines(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format('hello\nhello')]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], "hello") + + self.set_backup(backup_dir, 'node', backup_id, options=['--note=hello\nhello']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], "hello") + + self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertNotIn('note', backup_meta) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_add_big_note(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + +# note = node.safe_psql( +# "postgres", +# "SELECT repeat('hello', 400)").rstrip() # TODO: investigate + + note = node.safe_psql( + "postgres", + "SELECT repeat('hello', 210)").rstrip() + + # FULL + try: + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format(note)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because note is too large " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup note cannot exceed 1024 bytes", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + note = node.safe_psql( + "postgres", + "SELECT repeat('hello', 200)").rstrip() + + backup_id = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format(note)]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], note) + # Clean after yourself self.del_test_dir(module_name, fname) \ No newline at end of file From 90b8fe6a8c16737baf8db7b9d61133480d162cd5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 19 Apr 2020 03:08:04 +0300 Subject: [PATCH 1222/2107] [Issue #146] help update --- src/help.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/help.c b/src/help.c index a048afba0..153fe6aab 100644 --- a/src/help.c +++ b/src/help.c @@ -106,7 +106,8 @@ help_pg_probackup(void) printf(_(" [--help]\n")); printf(_("\n %s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id [--ttl] [--expire-time]\n")); + printf(_(" -i backup-id [--ttl=interval] [--expire-time=timestamp]\n")); + printf(_(" [--note=text]\n")); printf(_(" [--help]\n")); printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -140,7 +141,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); - printf(_(" [--ttl] [--expire-time]\n")); + printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n")); printf(_(" [--help]\n")); @@ -285,7 +286,7 @@ help_backup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); - printf(_(" [--ttl] [--expire-time]\n\n")); + printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -304,6 +305,8 @@ help_backup(void) printf(_(" backup some directories not from pgdata \n")); printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); printf(_(" --no-sync do not sync backed up files to disk\n")); + printf(_(" --note=text add note to backup\n")); + printf(_(" (example: --note='backup before app update to v13.1')\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -341,8 +344,9 @@ help_backup(void) printf(_(" --dry-run perform a trial run without any changes\n")); printf(_("\n Pinning options:\n")); - printf(_(" --ttl=ttl pin backup for specified amount of time; 0 unpin\n")); + printf(_(" --ttl=interval pin backup for specified amount of time; 0 unpin\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); + printf(_(" (example: --ttl=20d)\n")); printf(_(" --expire-time=time pin backup until specified time stamp\n")); printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); @@ -710,12 +714,15 @@ help_set_backup(void) { printf(_("\n%s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id\n")); - printf(_(" [--ttl] [--expire-time]\n\n")); + printf(_(" [--ttl=interval] [--expire-time=time] [--note=text]\n\n")); - printf(_(" --ttl=ttl pin backup for specified amount of time; 0 unpin\n")); + printf(_(" --ttl=interval pin backup for specified amount of time; 0 unpin\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); + printf(_(" (example: --ttl=20d)\n")); printf(_(" --expire-time=time pin backup until specified time stamp\n")); printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); + printf(_(" --note=text add note to backup; 'none' to remove note\n")); + printf(_(" (example: --note='backup before app update to v13.1')\n")); } static void From 4e0d1dd495e648d6ee1281d0d31907bbe75794d2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 19 Apr 2020 03:08:47 +0300 Subject: [PATCH 1223/2107] [Issue #146] documentation update --- doc/pgprobackup.xml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 04d53567f..a94a23c37 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2402,6 +2402,11 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod password is not included. + + + note — text note attached to backup. + + @@ -3377,15 +3382,28 @@ pg_probackup set-config -B backup_dir --instance set-backup pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id -{--ttl=ttl | --expire-time=time} [--note=note] [--help] +{--ttl=ttl | --expire-time=time} +[--note=backup_note] [--help] Sets the provided backup-specific settings into the backup.control configuration file, or modifies the previously defined values. + + + + + Sets the text note for backup copy. + If backup_note contain newline characters, + then only substring before first newline character will be saved. + Max size of text note is 1 KB. + The 'none' value removes current note. + + + - For all available settings, see the section + For all available pinning settings, see the section Pinning Options. @@ -3445,7 +3463,7 @@ pg_probackup backup -B backup_dir -b bac [--no-validate] [--skip-block-validation] [-w --no-password] [-W --password] [--archive-timeout=timeout] [--external-dirs=external_directory_path] -[--no-sync] [--note=note] +[--no-sync] [--note=backup_note] [connection_options] [compression_options] [remote_options] [retention_options] [pinning_options] [logging_options] @@ -3613,10 +3631,14 @@ pg_probackup backup -B backup_dir -b bac - + Sets the text note for backup copy. + If backup_note contain newline characters, + then only substring before first newline character will be saved. + Max size of text note is 1 KB. + The 'none' value removes current note. From 23532e8bac86b3d948754b02140d8253fd649ce6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 20 Apr 2020 14:11:20 +0300 Subject: [PATCH 1224/2107] [Issue #183] test coverage --- tests/helpers/ptrack_helpers.py | 5 +- tests/locking.py | 98 +++++++++++++++++++++++++++++++++ tests/merge.py | 50 +++++++++++++++++ 3 files changed, 151 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 14109ba57..99e1eb7b4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -870,7 +870,8 @@ def merge_backup( def restore_node( self, backup_dir, instance, node=False, - data_dir=None, backup_id=None, old_binary=False, options=[] + data_dir=None, backup_id=None, old_binary=False, options=[], + gdb=False ): if data_dir is None: @@ -895,7 +896,7 @@ def restore_node( if not old_binary: cmd_list += ['--no-sync'] - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, gdb=gdb, old_binary=old_binary) def show_pb( self, backup_dir, instance=None, backup_id=None, diff --git a/tests/locking.py b/tests/locking.py index 5044575ba..2da2415ea 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -440,3 +440,101 @@ def test_locking_concurrent_validate_and_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_locking_concurren_restore_and_delete(self): + """ + make node, take full backup, launch restore + and stop it in the middle, delete full backup. + Expect it to fail. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_id = self.backup_node(backup_dir, 'node', node) + + node.cleanup() + gdb = self.restore_node(backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('create_data_directories') + gdb.run_until_break() + + # This PAGE backup is expected to be successfull + try: + self.delete_pb(backup_dir, 'node', full_id) + self.assertEqual( + 1, 0, + "Expecting Error because backup is locked\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Cannot lock backup {0} directory".format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_backup_directory_name(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_id_1 = self.backup_node(backup_dir, 'node', node) + page_id_1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + full_id_2 = self.backup_node(backup_dir, 'node', node) + page_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + old_path = os.path.join(backup_dir, 'backups', 'node', full_id_1) + new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') + + os.rename(old_path, new_path) + + # This PAGE backup is expected to be successfull + self.show_pb(backup_dir, 'node', full_id_1) + + self.validate_pb(backup_dir) + self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', full_id_1) + + self.restore_node(backup_dir, 'node', node, backup_id=full_id_1) + + self.delete_pb(backup_dir, 'node', full_id_1) + + old_path = os.path.join(backup_dir, 'backups', 'node', full_id_2) + new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') + + self.set_backup( + backup_dir, 'node', full_id_2, options=['--note=hello']) + + self.merge_backup(backup_dir, 'node', page_id_2, options=["-j", "4"]) + + self.assertNotIn( + 'note', + self.show_pb(backup_dir, 'node', page_id_2)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/merge.py b/tests/merge.py index 2bc096685..0ddec258d 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2437,5 +2437,55 @@ def test_merge_correct_inheritance(self): self.del_test_dir(module_name, fname) + def test_merge_correct_inheritance_1(self): + """ + Make sure that backup metainformation fields + 'note' and 'expire-time' are correctly inherited + during merge + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note=hello', '--ttl=20d']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.merge_backup(backup_dir, 'node', page_id) + + self.assertNotIn( + 'note', + self.show_pb(backup_dir, 'node', page_id)) + + self.assertNotIn( + 'expire-time', + self.show_pb(backup_dir, 'node', page_id)) + + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From 295b8a1aa5a34a625d2979a3afb4a7181d6a9f65 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 20 Apr 2020 14:20:58 +0300 Subject: [PATCH 1225/2107] [Issue #183] If backup deletion failed to lock backup due to "out of space" condition, then treat backup as locked and carry on. --- src/backup.c | 18 +++++++------- src/catalog.c | 60 ++++++++++++++++++++++++++++------------------ src/delete.c | 12 +++++----- src/merge.c | 28 +++++++++++++++------- src/parsexlog.c | 10 ++++---- src/pg_probackup.h | 10 ++++---- src/restore.c | 8 +++---- src/show.c | 21 ++++++++++++++-- src/validate.c | 25 +++++++++---------- 9 files changed, 119 insertions(+), 73 deletions(-) diff --git a/src/backup.c b/src/backup.c index 258e3c002..1ecd7a229 100644 --- a/src/backup.c +++ b/src/backup.c @@ -230,7 +230,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) prev_backup_start_lsn = prev_backup->start_lsn; current.parent_backup = prev_backup->start_time; - write_backup(¤t); + write_backup(¤t, true); } /* @@ -287,7 +287,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) base36enc(prev_backup->start_time)); /* Update running backup meta with START LSN */ - write_backup(¤t); + write_backup(¤t, true); pgBackupGetPath(¤t, database_path, lengthof(database_path), DATABASE_DIR); @@ -496,7 +496,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* write initial backup_content.control file and update backup.control */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, external_dirs); - write_backup(¤t); + write_backup(¤t, true); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -661,7 +661,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, external_dirs); /* update backup control file to update size info */ - write_backup(¤t); + write_backup(¤t, true); /* Sync all copied files unless '--no-sync' flag is used */ if (no_sync) @@ -840,10 +840,10 @@ do_backup(time_t start_time, bool no_validate, /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) elog(ERROR, "Cannot create backup directory"); - if (!lock_backup(¤t)) + if (!lock_backup(¤t, true)) elog(ERROR, "Cannot lock backup %s directory", base36enc(current.start_time)); - write_backup(¤t); + write_backup(¤t, true); /* set the error processing function for the backup process */ pgut_atexit_push(backup_cleanup, NULL); @@ -931,7 +931,7 @@ do_backup(time_t start_time, bool no_validate, /* Backup is done. Update backup status */ current.end_time = time(NULL); current.status = BACKUP_STATUS_DONE; - write_backup(¤t); + write_backup(¤t, true); /* Pin backup if requested */ if (set_backup_params && @@ -2020,7 +2020,7 @@ backup_cleanup(bool fatal, void *userdata) base36enc(current.start_time)); current.end_time = time(NULL); current.status = BACKUP_STATUS_ERROR; - write_backup(¤t); + write_backup(¤t, true); } } @@ -2065,7 +2065,7 @@ backup_files(void *arg) write_backup_filelist(¤t, arguments->files_list, arguments->from_root, arguments->external_dirs); /* update backup control file to update size info */ - write_backup(¤t); + write_backup(¤t, true); } } diff --git a/src/catalog.c b/src/catalog.c index f1faaaec2..118ba6697 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -89,14 +89,11 @@ unlink_lock_atexit(void) * If no backup matches, return NULL. */ pgBackup * -read_backup(const char *instance_name, time_t timestamp) +read_backup(const char *root_dir) { - pgBackup tmp; char conf_path[MAXPGPATH]; - tmp.start_time = timestamp; - pgBackupGetPathInInstance(instance_name, &tmp, conf_path, - lengthof(conf_path), BACKUP_CONTROL_FILE, NULL); + join_path_components(conf_path, root_dir, BACKUP_CONTROL_FILE); return readBackupControlFile(conf_path); } @@ -109,11 +106,11 @@ read_backup(const char *instance_name, time_t timestamp) */ void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name) + const char *instance_name, bool strict) { pgBackup *tmp; - tmp = read_backup(instance_name, backup->start_time); + tmp = read_backup(backup->root_dir); if (!tmp) { /* @@ -125,7 +122,9 @@ write_backup_status(pgBackup *backup, BackupStatus status, backup->status = status; tmp->status = backup->status; - write_backup(tmp); + tmp->root_dir = pgut_strdup(backup->root_dir); + + write_backup(tmp, strict); pgBackupFree(tmp); } @@ -134,7 +133,7 @@ write_backup_status(pgBackup *backup, BackupStatus status, * Create exclusive lockfile in the backup's directory. */ bool -lock_backup(pgBackup *backup) +lock_backup(pgBackup *backup, bool strict) { char lock_file[MAXPGPATH]; int fd; @@ -280,6 +279,14 @@ lock_backup(pgBackup *backup) fio_unlink(lock_file, FIO_BACKUP_HOST); /* if write didn't set errno, assume problem is no disk space */ errno = save_errno ? save_errno : ENOSPC; + + /* In lax mode if we failed to grab lock because of 'out of space error', + * then treat backup as locked. + * Only delete command should be run in lax mode. + */ + if (!strict && errno == ENOSPC) + return true; + elog(ERROR, "Could not write lock file \"%s\": %s", lock_file, strerror(errno)); } @@ -536,7 +543,7 @@ get_backup_filelist(pgBackup *backup) parray *files = NULL; char backup_filelist_path[MAXPGPATH]; - pgBackupGetPath(backup, backup_filelist_path, lengthof(backup_filelist_path), DATABASE_FILE_LIST); + join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST); /* redundant sanity? */ @@ -550,7 +557,7 @@ get_backup_filelist(pgBackup *backup) * Lock list of backups. Function goes in backward direction. */ void -catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) +catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict) { int start_idx, end_idx; @@ -565,7 +572,7 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx) for (i = start_idx; i >= end_idx; i--) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (!lock_backup(backup)) + if (!lock_backup(backup, strict)) elog(ERROR, "Cannot lock backup %s directory", base36enc(backup->start_time)); } @@ -837,7 +844,7 @@ pgBackupCreateDir(pgBackup *backup) /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { - pgBackupGetPath(backup, path, lengthof(path), parray_get(subdirs, i)); + join_path_components(path, backup->root_dir, parray_get(subdirs, i)); fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); } @@ -1580,7 +1587,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) return; /* Update backup.control */ - write_backup(target_backup); + write_backup(target_backup, true); if (set_backup_params->ttl > 0 || set_backup_params->expire_time > 0) { @@ -1630,7 +1637,7 @@ add_note(pgBackup *target_backup, char *note) } /* Update backup.control */ - write_backup(target_backup); + write_backup(target_backup, true); } /* @@ -1735,14 +1742,15 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) * Save the backup content into BACKUP_CONTROL_FILE. */ void -write_backup(pgBackup *backup) +write_backup(pgBackup *backup, bool strict) { - FILE *fp = NULL; - char path[MAXPGPATH]; - char path_temp[MAXPGPATH]; - int errno_temp; + FILE *fp = NULL; + char path[MAXPGPATH]; + char path_temp[MAXPGPATH]; + int errno_temp; + char buf[4096]; - pgBackupGetPath(backup, path, lengthof(path), BACKUP_CONTROL_FILE); + join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); fp = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); @@ -1750,12 +1758,18 @@ write_backup(pgBackup *backup) elog(ERROR, "Cannot open configuration file \"%s\": %s", path_temp, strerror(errno)); + setvbuf(fp, buf, _IOFBF, sizeof(buf)); + pgBackupWriteControl(fp, backup); - if (fio_fflush(fp) || fio_fclose(fp)) + if (fio_fclose(fp)) { errno_temp = errno; fio_unlink(path_temp, FIO_BACKUP_HOST); + + if (!strict && errno_temp == ENOSPC) + return; + elog(ERROR, "Cannot write configuration file \"%s\": %s", path_temp, strerror(errno_temp)); } @@ -1788,7 +1802,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int64 uncompressed_size_on_disk = 0; int64 wal_size_on_disk = 0; - pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); out = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); diff --git a/src/delete.c b/src/delete.c index 85d273479..7ecdaf1dd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -89,7 +89,7 @@ do_delete(time_t backup_id) if (!dry_run) { /* Lock marked for delete backups */ - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0); + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false); /* Delete backups from the end of list */ for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) @@ -510,7 +510,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l parray_rm(to_purge_list, full_backup, pgBackupCompareId); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); /* Consider this extreme case */ // PAGEa1 PAGEb1 both valid @@ -627,7 +627,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) continue; /* Actual purge */ - if (!lock_backup(delete_backup)) + if (!lock_backup(delete_backup, false)) { /* If the backup still is used, do not interrupt and go to the next */ elog(WARNING, "Cannot lock backup %s directory, skip purging", @@ -746,7 +746,7 @@ delete_backup_files(pgBackup *backup) * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which * the error occurs before deleting all backup files. */ - write_backup_status(backup, BACKUP_STATUS_DELETING, instance_name); + write_backup_status(backup, BACKUP_STATUS_DELETING, instance_name, false); /* list files to be deleted */ files = parray_new(); @@ -968,7 +968,7 @@ do_delete_instance(void) /* Delete all backups. */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); - catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1); + catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true); for (i = 0; i < parray_num(backup_list); i++) { @@ -1091,7 +1091,7 @@ do_delete_status(InstanceConfig *instance_config, const char *status) if (backup->stream) size_to_delete += backup->wal_bytes; - if (!dry_run && lock_backup(backup)) + if (!dry_run && lock_backup(backup, false)) delete_backup_files(backup); n_deleted++; diff --git a/src/merge.c b/src/merge.c index bb2c0d698..b5927f719 100644 --- a/src/merge.c +++ b/src/merge.c @@ -397,7 +397,7 @@ do_merge(time_t backup_id) parray_append(merge_list, full_backup); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); /* do actual merge */ merge_chain(merge_list, full_backup, dest_backup); @@ -583,10 +583,10 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) */ backup->merge_dest_backup = dest_backup->start_time; backup->status = BACKUP_STATUS_MERGING; - write_backup(backup); + write_backup(backup, true); } else - write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name); + write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name, true); } /* Create directories */ @@ -704,9 +704,13 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* If incremental backup is pinned, * then result FULL backup must also be pinned. + * And reverse, if FULL backup was pinned and dest was not, + * then pinning is no more. */ - if (dest_backup->expire_time) - full_backup->expire_time = dest_backup->expire_time; + full_backup->expire_time = dest_backup->expire_time; + + pg_free(full_backup->note); + full_backup->note = NULL; if (dest_backup->note) full_backup->note = pgut_strdup(dest_backup->note); @@ -724,7 +728,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) parray_qsort(result_filelist, pgFileCompareRelPathWithExternal); write_backup_filelist(full_backup, result_filelist, full_database_dir, NULL); - write_backup(full_backup); + write_backup(full_backup, true); /* Delete FULL backup files, that do not exists in destination backup * Both arrays must be sorted in in reversed order to delete from leaf @@ -760,7 +764,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * Files are merged into FULL backup. It is time to remove incremental chain. */ full_backup->status = BACKUP_STATUS_MERGED; - write_backup(full_backup); + write_backup(full_backup, true); merge_delete: for (i = parray_num(parent_chain) - 2; i >= 0; i--) @@ -787,6 +791,10 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (rename(full_backup->root_dir, dest_backup->root_dir) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", full_backup->root_dir, dest_backup->root_dir, strerror(errno)); + + /* update root_dir after rename */ + pg_free(full_backup->root_dir); + full_backup->root_dir = pgut_strdup(dest_backup->root_dir); } else { @@ -804,6 +812,10 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (rename(full_backup->root_dir, destination_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", full_backup->root_dir, destination_path, strerror(errno)); + + /* update root_dir after rename */ + pg_free(full_backup->root_dir); + full_backup->root_dir = pgut_strdup(destination_path); } /* If we crash here, it will produce full backup in MERGED @@ -821,7 +833,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->status = BACKUP_STATUS_OK; full_backup->start_time = full_backup->merge_dest_backup; full_backup->merge_dest_backup = INVALID_BACKUP_ID; - write_backup(full_backup); + write_backup(full_backup, true); /* Critical section end */ /* Cleanup */ diff --git a/src/parsexlog.c b/src/parsexlog.c index e8ea27ceb..d55f42573 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -377,7 +377,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", @@ -404,7 +404,6 @@ validate_wal(pgBackup *backup, const char *archivedir, char last_timestamp[100], target_timestamp[100]; bool all_wal = false; - char backup_xlog_path[MAXPGPATH]; /* We need free() this later */ backup_id = base36enc(backup->start_time); @@ -425,8 +424,11 @@ validate_wal(pgBackup *backup, const char *archivedir, */ if (backup->stream) { - pgBackupGetPath2(backup, backup_xlog_path, lengthof(backup_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); + char backup_database_dir[MAXPGPATH]; + char backup_xlog_path[MAXPGPATH]; + + join_path_components(backup_database_dir, backup->root_dir, DATABASE_DIR); + join_path_components(backup_xlog_path, backup_database_dir, PG_XLOG_DIR); validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, wal_seg_size); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 275956f8b..01768acc8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -759,19 +759,19 @@ extern int validate_one_page(Page page, BlockNumber absolute_blkno, #define PAGE_LSN_FROM_FUTURE (-6) /* in catalog.c */ -extern pgBackup *read_backup(const char *instance_name, time_t timestamp); -extern void write_backup(pgBackup *backup); +extern pgBackup *read_backup(const char *root_dir); +extern void write_backup(pgBackup *backup, bool strict); extern void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name); + const char *instance_name, bool strict); extern void write_backup_data_bytes(pgBackup *backup); -extern bool lock_backup(pgBackup *backup); +extern bool lock_backup(pgBackup *backup, bool strict); extern const char *pgBackupGetBackupMode(pgBackup *backup); extern parray *catalog_get_instance_list(void); extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, - int to_idx); + int to_idx, bool strict); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); diff --git a/src/restore.c b/src/restore.c index cfc5aacae..49ff04b3f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -71,7 +71,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", @@ -274,7 +274,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(backup->start_time), missing_backup_id); @@ -361,7 +361,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, tmp_backup = (pgBackup *) parray_get(parent_chain, i); /* Do not interrupt, validate the next backup */ - if (!lock_backup(tmp_backup)) + if (!lock_backup(tmp_backup, true)) { if (params->is_restore) elog(ERROR, "Cannot lock backup %s directory", @@ -523,7 +523,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - if (!lock_backup(backup)) + if (!lock_backup(backup, true)) elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); if (backup->status != BACKUP_STATUS_OK && diff --git a/src/show.c b/src/show.c index fd4b460e4..f242bee4e 100644 --- a/src/show.c +++ b/src/show.c @@ -444,9 +444,25 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) static int show_backup(const char *instance_name, time_t requested_backup_id) { + int i; pgBackup *backup; + parray *backups; + + backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + + /* Find requested backup */ + for (i = 0; i < parray_num(backups); i++) + { + pgBackup *tmp_backup = (pgBackup *) parray_get(backups, i); + + /* found target */ + if (tmp_backup->start_time == requested_backup_id) + { + backup = tmp_backup; + break; + } + } - backup = read_backup(instance_name, requested_backup_id); if (backup == NULL) { // TODO for 3.0: we should ERROR out here. @@ -463,7 +479,8 @@ show_backup(const char *instance_name, time_t requested_backup_id) elog(ERROR, "Invalid show format %d", (int) show_format); /* cleanup */ - pgBackupFree(backup); + parray_walk(backups, pgBackupFree); + parray_free(backups); return 0; } diff --git a/src/validate.c b/src/validate.c index 209cf8f4d..36f71ffae 100644 --- a/src/validate.c +++ b/src/validate.c @@ -69,7 +69,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", base36enc(backup->start_time), status2str(backup->status)); - write_backup_status(backup, BACKUP_STATUS_ERROR, instance_name); + write_backup_status(backup, BACKUP_STATUS_ERROR, instance_name, true); corrupted_backup_found = true; return; } @@ -108,9 +108,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->backup_mode != BACKUP_MODE_DIFF_DELTA) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); - pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR); - pgBackupGetPath(backup, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + join_path_components(base_path, backup->root_dir, DATABASE_DIR); + join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); + join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST); // if (params && params->partial_db_list) @@ -174,7 +174,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) if (corrupted) backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK, instance_name); + BACKUP_STATUS_OK, instance_name, true); if (corrupted) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); @@ -189,7 +189,8 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { char path[MAXPGPATH]; - pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + //pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); + join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); if (pgFileSize(path) >= (BLCKSZ*500)) { @@ -198,7 +199,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) "https://github.com/postgrespro/pg_probackup/issues/132", base36enc(backup->start_time)); backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); } } @@ -491,7 +492,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(current_backup->start_time), parent_backup_id); @@ -515,7 +516,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(current_backup->start_time), backup_id, status2str(tmp_backup->status)); @@ -546,7 +547,7 @@ do_validate_instance(void) base_full_backup = current_backup; /* Do not interrupt, validate the next backup */ - if (!lock_backup(current_backup)) + if (!lock_backup(current_backup, true)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(current_backup->start_time)); @@ -588,7 +589,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(backup->start_time), @@ -641,7 +642,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_ORPHAN) { /* Do not interrupt, validate the next backup */ - if (!lock_backup(backup)) + if (!lock_backup(backup, true)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(backup->start_time)); From 7adb0b6e4f53eb03036c60644ac743fa0278d6d0 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 20:59:10 +1000 Subject: [PATCH 1226/2107] compare parent backup version with current binary version while searching for valid ancestor --- src/backup.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backup.c b/src/backup.c index 1ecd7a229..e16466403 100644 --- a/src/backup.c +++ b/src/backup.c @@ -217,6 +217,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (prev_backup) { + if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); + char prev_backup_filelist_path[MAXPGPATH]; elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); From 3fd92c629a32058bf06bf016ab932190d1c6c66c Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 20:59:56 +1000 Subject: [PATCH 1227/2107] add probackup_version attribute to tests --- tests/helpers/ptrack_helpers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 99e1eb7b4..23a0cefe9 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -246,6 +246,18 @@ def __init__(self, *args, **kwargs): print('pg_probackup binary is not found') exit(1) + self.probackup_version = None + + try: + self.probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8')) + + self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) + if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( From 83cee2d01c5133dfb135019d7b6942100879840a Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 6 Apr 2020 21:00:12 +1000 Subject: [PATCH 1228/2107] add test test_parent_backup_made_by_newer_version --- tests/backup.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e04930091..c7b1291de 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2746,3 +2746,56 @@ def test_note_sanity(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_parent_backup_made_by_newer_version(self): + """incremental backup with parent made by newer version""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + control_file = os.path.join( + backup_dir, "backups", "node", backup_id, + "backup.control") + + version = self.probackup_version + fake_new_version = str(int(version.split('.')[0]) + 1) + '.0.0' + + with open(control_file, 'r') as f: + data = f.read(); + + data = data.replace(version, fake_new_version) + + with open(control_file, 'w') as f: + f.write(data); + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "if parent made by newer version.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, fname) From aa27b53f9d488bebc92eb91b2a1bd3d93044665e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 20 Apr 2020 18:49:46 +0300 Subject: [PATCH 1229/2107] [Issue #177] added test tests.archive.ArchiveTest.test_archive_show_partial_files_handling --- tests/archive.py | 151 +++++++++++++++++++++++++++++--- tests/helpers/ptrack_helpers.py | 4 +- 2 files changed, 140 insertions(+), 15 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 59496b2f0..3f48a9f94 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -593,7 +593,7 @@ def test_archive_push_partial_file_exists(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_archive_push_partial_file_exists_not_stale(self): + def test_archive_push_part_file_exists_not_stale(self): """Archive-push if .part file exists and it is not stale""" fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -896,8 +896,8 @@ def test_basic_master_and_replica_concurrent_archiving(self): """ make node 'master 'with archiving, take archive backup and turn it into replica, - set replica with archiving, make archive backup from replica, - make archive backup from master + set replica with archiving, + make sure that archiving on both node is working. """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -959,13 +959,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): "from generate_series(0,10000) i") # TAKE FULL ARCHIVE BACKUP FROM REPLICA - backup_id = self.backup_node( - backup_dir, 'master', replica, - options=[ - '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + backup_id = self.backup_node(backup_dir, 'master', replica) self.validate_pb(backup_dir, 'master') self.assertEqual( @@ -977,7 +971,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.assertEqual( 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - master.pgbench_init(scale=50) + master.pgbench_init(scale=10) sleep(10) @@ -986,8 +980,8 @@ def test_basic_master_and_replica_concurrent_archiving(self): master.pgbench_init(scale=10) replica.pgbench_init(scale=10) - - exit(1) + self.backup_node(backup_dir, 'master', master) + self.backup_node(backup_dir, 'master', replica) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1472,6 +1466,10 @@ def test_archive_catalog_1(self): """ double segment - compressed and not """ + if not self.archive_compress: + return self.fail( + 'You need to enable ARCHIVE_COMPRESSION for this test to run') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1524,6 +1522,10 @@ def test_archive_catalog_2(self): """ double segment - compressed and not """ + if not self.archive_compress: + return self.fail( + 'You need to enable ARCHIVE_COMPRESSION for this test to run') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2335,6 +2337,129 @@ def test_archive_get_prefetch_corruption(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_archive_show_partial_files_handling(self): + """ + check that files with '.part', '.part.gz', '.partial' and '.partial.gz' + siffixes are handled correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=False) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + + # .part file + node.safe_psql( + "postgres", + "create table t1()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.part'.format(filename))) + + # .gz.part file + node.safe_psql( + "postgres", + "create table t2()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.gz.part'.format(filename))) + + # .partial file + node.safe_psql( + "postgres", + "create table t3()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.partial'.format(filename))) + + # .gz.partial file + node.safe_psql( + "postgres", + "create table t4()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.gz.partial'.format(filename))) + + self.show_archive(backup_dir, 'node', options=['--log-level-file=VERBOSE']) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log'), 'r') as f: + log_content = f.read() + + self.assertNotIn( + 'WARNING', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # TODO test with multiple not archived segments. # TODO corrupted file in archive. diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 99e1eb7b4..58adfcbb1 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1132,7 +1132,7 @@ def get_recovery_conf(self, node): def set_archiving( self, backup_dir, instance, node, replica=False, - overwrite=False, compress=False, old_binary=False, + overwrite=False, compress=True, old_binary=False, log_level=False, archive_timeout=False): # parse postgresql.auto.conf @@ -1157,7 +1157,7 @@ def set_archiving( options['archive_command'] += '--remote-proto=ssh ' options['archive_command'] += '--remote-host=localhost ' - if self.archive_compress or compress: + if self.archive_compress and compress: options['archive_command'] += '--compress ' if overwrite: From 33472260f4a961af84a614969c5078b27d7f0b5f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 20 Apr 2020 18:56:19 +0300 Subject: [PATCH 1230/2107] [Issue #177] treat files with '.gz.partial', '.part' and '.gz.part' siffixes as WAL segments --- src/catalog.c | 27 +++++++++++++++++++++++++-- src/delete.c | 2 ++ src/pg_probackup.h | 28 ++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 118ba6697..05fe48143 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -853,7 +853,8 @@ pgBackupCreateDir(pgBackup *backup) } /* - * Create list of timelines + * Create list of timelines. + * TODO: '.partial' and '.part' segno information should be added to tlinfo. */ parray * catalog_get_timelines(InstanceConfig *instance) @@ -933,7 +934,8 @@ catalog_get_timelines(InstanceConfig *instance) continue; } /* partial WAL segment */ - else if (IsPartialXLogFileName(file->name)) + else if (IsPartialXLogFileName(file->name) || + IsPartialCompressXLogFileName(file->name)) { elog(VERBOSE, "partial WAL file \"%s\"", file->name); @@ -952,6 +954,27 @@ catalog_get_timelines(InstanceConfig *instance) parray_append(tlinfo->xlog_filelist, wal_file); continue; } + /* temp WAL segment */ + else if (IsTempXLogFileName(file->name) || + IsTempCompressXLogFileName(file->name)) + { + elog(VERBOSE, "temp WAL file \"%s\"", file->name); + + if (!tlinfo || tlinfo->tli != tli) + { + tlinfo = timelineInfoNew(tli); + parray_append(timelineinfos, tlinfo); + } + + /* append file to xlog file list */ + wal_file = palloc(sizeof(xlogFile)); + wal_file->file = *file; + wal_file->segno = segno; + wal_file->type = TEMP_SEGMENT; + wal_file->keep = false; + parray_append(tlinfo->xlog_filelist, wal_file); + continue; + } /* we only expect compressed wal files with .gz suffix */ else if (strcmp(suffix, "gz") != 0) { diff --git a/src/delete.c b/src/delete.c index 7ecdaf1dd..be7dba9d4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -942,6 +942,8 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { if (wal_file->type == SEGMENT) elog(VERBOSE, "Removed WAL segment \"%s\"", wal_file->file.path); + else if (wal_file->type == TEMP_SEGMENT) + elog(VERBOSE, "Removed temp WAL segment \"%s\"", wal_file->file.path); else if (wal_file->type == PARTIAL_SEGMENT) elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_file->file.path); else if (wal_file->type == BACKUP_HISTORY_FILE) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 01768acc8..96c79982a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -46,7 +46,7 @@ extern const char *PROGRAM_URL; extern const char *PROGRAM_EMAIL; /* Directory/File names */ -#define DATABASE_DIR "database" +#define DATABASE_DIR "database" #define BACKUPS_DIR "backups" #if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" @@ -90,6 +90,7 @@ extern const char *PROGRAM_EMAIL; /* retry attempts */ #define PAGE_READ_ATTEMPTS 100 +/* max size of note, that can be added to backup */ #define MAX_NOTE_SIZE 1024 /* Check if an XLogRecPtr value is pointed to 0 offset */ @@ -514,18 +515,18 @@ typedef struct lsnInterval typedef enum xlogFileType { SEGMENT, + TEMP_SEGMENT, PARTIAL_SEGMENT, BACKUP_HISTORY_FILE } xlogFileType; typedef struct xlogFile { - pgFile file; - XLogSegNo segno; + pgFile file; + XLogSegNo segno; xlogFileType type; - bool keep; /* Used to prevent removal of WAL segments - * required by ARCHIVE backups. - */ + bool keep; /* Used to prevent removal of WAL segments + * required by ARCHIVE backups. */ } xlogFile; @@ -607,6 +608,21 @@ typedef struct BackupPageHeader XLogFromFileName(fname, tli, logSegNo) #endif +#define IsPartialCompressXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0) + +#define IsTempXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".part") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0) + +#define IsTempCompressXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".gz.part") == 0) + #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) /* directory options */ From 726d8916e20fdec496ac3f588be73d5059e13849 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 Apr 2020 02:09:33 +0300 Subject: [PATCH 1231/2107] [Issue #192] Handle 0 offset STOP LSN on master --- src/backup.c | 15 ++++++++++++--- src/parsexlog.c | 32 +++++++++++++++++++++----------- src/pg_probackup.h | 2 +- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1ecd7a229..7ded50f90 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1740,8 +1740,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!XRecOffIsValid(stop_backup_lsn_tmp)) { - /* It is ok for replica to return STOP LSN with NullXRecOff */ - if (backup->from_replica && XRecOffIsNull(stop_backup_lsn_tmp)) + /* It is ok for replica to return STOP LSN with NullXRecOff + * UPD: Apparently it is ok even for master. + */ + if (XRecOffIsNull(stop_backup_lsn_tmp)) { char *xlog_path, stream_xlog_path[MAXPGPATH]; @@ -1798,13 +1800,20 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* Get the first record in segment with current stop_lsn */ lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size); + instance_config.xlog_seg_size, + instance_config.archive_timeout); /* Check that returned LSN is valid and greater than stop_lsn */ if (XLogRecPtrIsInvalid(lsn_tmp) || !XRecOffIsValid(lsn_tmp) || lsn_tmp < stop_backup_lsn_tmp) { + /* Backup from master should error out here */ + if (!backup->from_replica) + elog(ERROR, "Failed to get next WAL record after %X/%X", + (uint32) (stop_backup_lsn_tmp >> 32), + (uint32) (stop_backup_lsn_tmp)); + /* No luck, falling back to looking up for previous record */ elog(WARNING, "Failed to get next WAL record after %X/%X, " "looking for previous WAL record", diff --git a/src/parsexlog.c b/src/parsexlog.c index d55f42573..6c6691703 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -631,13 +631,14 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, */ XLogRecPtr get_first_record_lsn(const char *archivedir, XLogSegNo segno, - TimeLineID tli, uint32 wal_seg_size) + TimeLineID tli, uint32 wal_seg_size, int timeout) { XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr record = InvalidXLogRecPtr; - XLogRecPtr startpoint; - char wal_segment[MAXFNAMELEN]; + XLogReaderData reader_data; + XLogRecPtr record = InvalidXLogRecPtr; + XLogRecPtr startpoint; + char wal_segment[MAXFNAMELEN]; + int attempts = 0; if (segno <= 1) elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); @@ -653,13 +654,22 @@ get_first_record_lsn(const char *archivedir, XLogSegNo segno, /* Set startpoint to 0 in segno */ GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - record = XLogFindNextRecord(xlogreader, startpoint); + while (attempts <= timeout) + { + record = XLogFindNextRecord(xlogreader, startpoint); - if (XLogRecPtrIsInvalid(record)) - record = InvalidXLogRecPtr; - else - elog(LOG, "First record in WAL segment \"%s\": %X/%X", wal_segment, - (uint32) (record >> 32), (uint32) (record)); + if (XLogRecPtrIsInvalid(record)) + record = InvalidXLogRecPtr; + else + { + elog(LOG, "First record in WAL segment \"%s\": %X/%X", wal_segment, + (uint32) (record >> 32), (uint32) (record)); + break; + } + + attempts++; + sleep(1); + } /* cleanup */ CleanupXLogPageRead(xlogreader); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 96c79982a..152966be4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -957,7 +957,7 @@ extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_ bool seek_prev_segment, uint32 seg_size); extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - TimeLineID tli, uint32 wal_seg_size); + TimeLineID tli, uint32 wal_seg_size, int timeout); /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); From f7313c1ade0d9efbcc679a8c1643dc14fb83ee15 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 21 Apr 2020 02:17:35 +0300 Subject: [PATCH 1232/2107] minor fix --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index b4493ea47..0e4b002de 100644 --- a/src/backup.c +++ b/src/backup.c @@ -217,14 +217,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (prev_backup) { + char prev_backup_filelist_path[MAXPGPATH]; + if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); - char prev_backup_filelist_path[MAXPGPATH]; - elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); join_path_components(prev_backup_filelist_path, prev_backup->root_dir, From bebdef6c167df7463d0387214b216ed01b59de25 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 01:17:06 +0300 Subject: [PATCH 1233/2107] tests: fixes --- tests/archive.py | 12 +++++++++++- tests/backup.py | 1 + tests/helpers/ptrack_helpers.py | 2 +- tests/page.py | 8 ++++++-- tests/replica.py | 2 ++ tests/set_backup.py | 1 + 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 3f48a9f94..17e90ddcb 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1157,6 +1157,11 @@ def test_archive_catalog(self): 'checkpoint_timeout': '30s', 'autovacuum': 'off'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) self.set_archiving(backup_dir, 'master', master) @@ -2145,6 +2150,11 @@ def test_archive_get_batching_sanity(self): initdb_params=['--data-checksums'], pg_options={'autovacuum': 'off'}) + if self.get_version(node) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2321,7 +2331,7 @@ def test_archive_get_prefetch_corruption(self): os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) replica.slow_start(replica=True) - sleep(10) + sleep(60) with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: postgres_log_content = f.read() diff --git a/tests/backup.py b/tests/backup.py index c7b1291de..7c388cceb 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2718,6 +2718,7 @@ def test_note_sanity(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 53d4188b8..2f95417d0 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -340,7 +340,7 @@ def make_simple_node( # set major version with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version = str(f.read().rstrip()) + node.major_version = float(str(f.read().rstrip())) # Sane default parameters options = {} diff --git a/tests/page.py b/tests/page.py index c38f98a4a..d84003d6a 100644 --- a/tests/page.py +++ b/tests/page.py @@ -517,7 +517,8 @@ def test_page_delete_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + set_replication=True, + initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'autovacuum': 'off' @@ -925,10 +926,13 @@ def test_page_backup_with_alien_wal_segment(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) alien_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'alien_node')) + base_dir=os.path.join(module_name, fname, 'alien_node'), + set_replication=True, + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/replica.py b/tests/replica.py index a767f68e1..62a56d8c0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1333,6 +1333,8 @@ def test_replica_promote_archive_delta(self): node1.promote() node1.safe_psql('postgres', 'CHECKPOINT') + sleep(5) + # delta backup on timeline 3 self.backup_node( backup_dir, 'node', node1, node1.data_dir, 'delta', diff --git a/tests/set_backup.py b/tests/set_backup.py index 861de7564..f73b522c7 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -426,6 +426,7 @@ def test_add_big_note(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 058a3bd401782aa6b26292b287464214a2089dd3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 18:35:19 +0300 Subject: [PATCH 1234/2107] bugfix: page, received from shared buffer, should be validated --- src/data.c | 60 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/src/data.c b/src/data.c index 3bc168caa..19082ac40 100644 --- a/src/data.c +++ b/src/data.c @@ -418,43 +418,55 @@ prepare_page(ConnectionArgs *conn_arg, && (ptrack_version_num >= 15 && ptrack_version_num < 20)) || !page_is_valid) { + int rc = 0; size_t page_size = 0; Page ptrack_page = NULL; - ptrack_page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, + page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size, ptrack_version_num, ptrack_schema); if (ptrack_page == NULL) - { /* This block was truncated.*/ return PageIsTruncated; - } - else if (page_size != BLCKSZ) - { - free(ptrack_page); + + if (page_size != BLCKSZ) elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", from_fullpath, absolute_blknum, BLCKSZ, page_size); - } - else + + /* + * We need to copy the page that was successfully + * retrieved from ptrack into our output "page" parameter. + */ + memcpy(page, ptrack_page, BLCKSZ); + pg_free(ptrack_page); + + /* UPD: It apprears that is possible to get zeroed page or page with invalid header + * from shared buffer. + * Note, that getting page with wrong checksumm from shared buffer is + * acceptable. + */ + rc = validate_one_page(page, absolute_blknum, + InvalidXLogRecPtr, &page_lsn, + checksum_version); + + /* It is ok to get zeroed page */ + if (rc == PAGE_IS_ZEROED) + return PageIsOk; + + /* Getting page with invalid header from shared buffers is unacceptable */ + if (PAGE_HEADER_IS_INVALID) { - /* - * We need to copy the page that was successfully - * retrieved from ptrack into our output "page" parameter. - * We must set checksum here, because it is outdated - * in the block recieved from shared buffers. - */ - memcpy(page, ptrack_page, BLCKSZ); - free(ptrack_page); - if (checksum_version) - ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + char *errormsg = NULL; + get_header_errormsg(page, &errormsg); + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, blknum, errormsg); } - /* get lsn from page, provided by pg_ptrack_get_block() */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA && - file->exists_in_prev && - !parse_page(page, &page_lsn)) - elog(ERROR, "Cannot parse page after pg_ptrack_get_block. " - "Possible risk of a memory corruption"); + /* We must set checksum here, because it is outdated + * in the block recieved from shared buffers. + */ + if (checksum_version) + ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* From 29897c65e0113a5435464e19bc20bbf82c4d5aa7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 19:05:20 +0300 Subject: [PATCH 1235/2107] fix destination page for ptrack_get_block --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 19082ac40..244270c2d 100644 --- a/src/data.c +++ b/src/data.c @@ -421,7 +421,7 @@ prepare_page(ConnectionArgs *conn_arg, int rc = 0; size_t page_size = 0; Page ptrack_page = NULL; - page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, + ptrack_page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, file->relOid, absolute_blknum, &page_size, ptrack_version_num, ptrack_schema); From f86c33a72fa02b2ed9cf234446f2ff6c55f0f151 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 19:09:02 +0300 Subject: [PATCH 1236/2107] use relative block number when reporting corruption --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 244270c2d..57dbb7646 100644 --- a/src/data.c +++ b/src/data.c @@ -431,7 +431,7 @@ prepare_page(ConnectionArgs *conn_arg, if (page_size != BLCKSZ) elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", - from_fullpath, absolute_blknum, BLCKSZ, page_size); + from_fullpath, blknum, BLCKSZ, page_size); /* * We need to copy the page that was successfully From 9f449016a56e2d840c6a92d36181bf2b0d1a73f7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 20:14:41 +0300 Subject: [PATCH 1237/2107] minor fix --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 57dbb7646..ce8beffd7 100644 --- a/src/data.c +++ b/src/data.c @@ -454,7 +454,7 @@ prepare_page(ConnectionArgs *conn_arg, return PageIsOk; /* Getting page with invalid header from shared buffers is unacceptable */ - if (PAGE_HEADER_IS_INVALID) + if (rc == PAGE_HEADER_IS_INVALID) { char *errormsg = NULL; get_header_errormsg(page, &errormsg); From e4d1db83cb85412ba702c289b18aa9070d1e218c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 23:35:57 +0300 Subject: [PATCH 1238/2107] tests: some fixes for ptrack --- tests/ptrack.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index e805cacf3..470db3dcf 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -344,10 +344,10 @@ def test_ptrack_disable(self): node.slow_start() # ENABLE PTRACK - if node.major_version < 11: - node.safe_psql('postgres', "alter system set ptrack_enable to on") - else: + if node.major_version >= 12: node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") + else: + node.safe_psql('postgres', "alter system set ptrack_enable to on") node.stop() node.slow_start() @@ -3966,6 +3966,10 @@ def test_ptrack_pg_resetxlog(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_ptrack_map(self): + + if self.pg_config_version < self.version_to_num('12.0'): + return unittest.skip('You need PostgreSQL >= 12 for this test') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), From 54f63319be04bc005d3b9348973f705f418b95de Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 22 Apr 2020 23:36:20 +0300 Subject: [PATCH 1239/2107] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f537d80ae..0258e4e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ /src/xlogreader.c /src/walmethods.c /src/walmethods.h +/src/instr_time.h # Doc files /doc/*html From a9aab29a04b4c6fe80d3fb9fd1bfcab3ad5c1d7a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 23 Apr 2020 22:56:56 +0300 Subject: [PATCH 1240/2107] improvement: use dynamically allocated buffers for stdio buffering --- src/archive.c | 3 + src/data.c | 171 +++++++++++++++++++++++++++++++++-------------- src/dir.c | 43 +++++++----- src/restore.c | 23 +++++-- src/utils/file.c | 15 +++-- 5 files changed, 177 insertions(+), 78 deletions(-) diff --git a/src/archive.c b/src/archive.c index 8866ce66e..7df3837d1 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1556,6 +1556,9 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, } goto cleanup; } + + /* disable stdio buffering */ + setvbuf(out, NULL, _IONBF, BUFSIZ); } #ifdef HAVE_LIBZ else diff --git a/src/data.c b/src/data.c index ce8beffd7..c49b82836 100644 --- a/src/data.c +++ b/src/data.c @@ -566,8 +566,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, datapagemap_iterator_t *iter = NULL; /* stdio buffers */ - char in_buffer[STDIO_BUFSIZE]; - char out_buffer[STDIO_BUFSIZE]; + char *in_buf = NULL; + char *out_buf = NULL; /* sanity */ if (file->size % BLCKSZ != 0) @@ -634,17 +634,12 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, from_fullpath, strerror(errno)); } - if (!fio_is_remote_file(in)) - setvbuf(in, in_buffer, _IOFBF, STDIO_BUFSIZE); - /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open backup file \"%s\": %s", to_fullpath, strerror(errno)); - setvbuf(out, out_buffer, _IOFBF, STDIO_BUFSIZE); - /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, @@ -667,6 +662,24 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, else use_pagemap = true; + if (!fio_is_remote_file(in)) + { + /* enable stdio buffering for local input file, + * unless the pagemap is involved, which + * imply a lot of random access. + */ + if (use_pagemap) + setvbuf(in, NULL, _IONBF, BUFSIZ); + else + { + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + } + } + + /* enable stdio buffering for output file */ + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Remote mode */ if (fio_is_remote_file(in)) @@ -789,6 +802,9 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath, strerror(errno)); } + + pg_free(in_buf); + pg_free(out_buf); } /* @@ -837,18 +853,18 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) { - int i; + int i; size_t total_write_len = 0; - char buffer[STDIO_BUFSIZE]; + char *in_buf; for (i = parray_num(parent_chain) - 1; i >= 0; i--) { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; + char from_root[MAXPGPATH]; + char from_fullpath[MAXPGPATH]; + FILE *in = NULL; - pgFile **res_file = NULL; - pgFile *tmp_file = NULL; + pgFile **res_file = NULL; + pgFile *tmp_file = NULL; pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -886,7 +902,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setvbuf(in, buffer, _IOFBF, STDIO_BUFSIZE); + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* * Restore the file. @@ -902,6 +919,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, strerror(errno)); } + pg_free(in_buf); + return total_write_len; } @@ -912,6 +931,21 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers BackupPageHeader header; BlockNumber blknum = 0; size_t write_len = 0; + off_t cur_pos = 0; + + /* + * We rely on stdio buffering of input and output. + * For buffering to be efficient, we try to minimize the + * number of lseek syscalls, because it forces buffer flush. + * For that, we track current write position in + * output file and issue fseek only when offset of block to be + * written not equal to current write position, which happens + * a lot when blocks from incremental backup are restored, + * but should never happen in case of blocks from FULL backup. + */ + if (fio_fseek(out, cur_pos) < 0) + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); for (;;) { @@ -928,23 +962,24 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); + if (ferror(in)) + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", + blknum, from_fullpath, strerror(errno)); + if (read_len != sizeof(header)) { - int errno_tmp = errno; if (read_len == 0 && feof(in)) break; /* EOF found */ - else if (read_len != 0 && feof(in)) + + if (read_len != 0 && feof(in)) elog(ERROR, "Odd size page found at block %u of \"%s\"", blknum, from_fullpath); - else - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno_tmp)); } /* Consider empty blockm. wtf empty block ? */ if (header.block == 0 && header.compressed_size == 0) { - elog(VERBOSE, "Skip empty block of \"%s\"", from_fullpath); + elog(WARNING, "Skip empty block of \"%s\"", from_fullpath); continue; } @@ -1019,14 +1054,19 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers is_compressed = true; } - write_pos = blknum * BLCKSZ; - /* * Seek and write the restored page. + * When restoring file from FULL backup, pages are written sequentially, + * so there is no need to issue fseek for every page. */ - if (fio_fseek(out, write_pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); + write_pos = blknum * BLCKSZ; + + if (cur_pos != write_pos) + { + if (fio_fseek(out, blknum * BLCKSZ) < 0) + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } /* If page is compressed and restore is in remote mode, send compressed * page to the remote side. @@ -1048,6 +1088,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } write_len += BLCKSZ; + cur_pos = write_pos + BLCKSZ; /* update current write position */ } elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); @@ -1063,8 +1104,8 @@ void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath) { - ssize_t read_len = 0; - char buf[STDIO_BUFSIZE]; /* 64kB buffer */ + size_t read_len = 0; + char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ /* copy content */ for (;;) @@ -1075,20 +1116,25 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during non-data file restore"); - read_len = fread(buf, 1, sizeof(buf), in); - - if (read_len == 0 && feof(in)) - break; + read_len = fread(buf, 1, STDIO_BUFSIZE, in); - if (read_len < 0) + if (ferror(in)) elog(ERROR, "Cannot read backup file \"%s\": %s", - from_fullpath, strerror(errno)); + from_fullpath, strerror(errno)); - if (fio_fwrite(out, buf, read_len) != read_len) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno)); + if (read_len > 0) + { + if (fio_fwrite(out, buf, read_len) != read_len) + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, + strerror(errno)); + } + + if (feof(in)) + break; } + pg_free(buf); + elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } @@ -1103,7 +1149,6 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *tmp_file = NULL; pgBackup *tmp_backup = NULL; - char buffer[STDIO_BUFSIZE]; /* Check if full copy of destination file is available in destination backup */ if (dest_file->write_size > 0) @@ -1176,7 +1221,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - setvbuf(in, buffer, _IOFBF, STDIO_BUFSIZE); + /* disable stdio buffering for non-data files */ + setvbuf(in, NULL, _IONBF, BUFSIZ); /* do actual work */ restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); @@ -1192,6 +1238,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, * Copy file to backup. * We do not apply compression to these files, because * it is either small control file or already compressed cfs file. + * TODO: optimize remote copying */ void backup_non_data_file_internal(const char *from_fullpath, @@ -1199,10 +1246,10 @@ backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok) { - FILE *in; - FILE *out; - ssize_t read_len = 0; - char buf[STDIO_BUFSIZE]; /* 64kB buffer */ + FILE *in; + FILE *out; + ssize_t read_len = 0; + char *buf; pg_crc32 crc; INIT_FILE_CRC32(true, crc); @@ -1247,18 +1294,26 @@ backup_non_data_file_internal(const char *from_fullpath, elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); + /* disable stdio buffering for local input/output files */ + if (!fio_is_remote_file(in)) + setvbuf(in, NULL, _IONBF, BUFSIZ); + setvbuf(out, NULL, _IONBF, BUFSIZ); + + /* allocate 64kB buffer */ + buf = pgut_malloc(STDIO_BUFSIZE); + /* copy content and calc CRC */ for (;;) { - read_len = fio_fread(in, buf, sizeof(buf)); - - if (read_len == 0) - break; + read_len = fio_fread(in, buf, STDIO_BUFSIZE); if (read_len < 0) elog(ERROR, "Cannot read from source file \"%s\": %s", from_fullpath, strerror(errno)); + if (read_len == 0) + break; + if (fwrite(buf, 1, read_len, out) != read_len) elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); @@ -1267,6 +1322,19 @@ backup_non_data_file_internal(const char *from_fullpath, COMP_FILE_CRC32(true, crc, buf, read_len); file->read_size += read_len; + +// if (read_len < STDIO_BUFSIZE) +// { +// if (!fio_is_remote_file(in)) +// { +// if (ferror(in)) +// elog(ERROR, "Cannot read from source file \"%s\": %s", +// from_fullpath, strerror(errno)); +// +// if (feof(in)) +// break; +// } +// } } file->write_size = (int64) file->read_size; @@ -1280,6 +1348,7 @@ backup_non_data_file_internal(const char *from_fullpath, if (fclose(out)) elog(ERROR, "Cannot write \"%s\": %s", to_fullpath, strerror(errno)); fio_fclose(in); + pg_free(buf); } /* @@ -1478,9 +1547,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, /* read BackupPageHeader */ read_len = fread(&header, 1, sizeof(header), in); + + if (ferror(in)) + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", + blknum, file->path, strerror(errno)); + if (read_len != sizeof(header)) { - int errno_tmp = errno; if (read_len == 0 && feof(in)) break; /* EOF found */ else if (read_len != 0 && feof(in)) @@ -1489,7 +1562,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, blknum, file->path); else elog(WARNING, "Cannot read header of block %u of \"%s\": %s", - blknum, file->path, strerror(errno_tmp)); + blknum, file->path, strerror(errno)); return false; } diff --git a/src/dir.c b/src/dir.c index 5b8fcf8d3..5b5cd2b88 100644 --- a/src/dir.c +++ b/src/dir.c @@ -265,7 +265,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) { FILE *fp; pg_crc32 crc = 0; - char buf[STDIO_BUFSIZE]; + char *buf; size_t len = 0; INIT_FILE_CRC32(use_crc32c, crc); @@ -287,30 +287,31 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) file_path, strerror(errno)); } + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + /* calc CRC of file */ for (;;) { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); - len = fread(&buf, 1, sizeof(buf), fp); - - if (len == 0) - { - /* we either run into eof or error */ - if (feof(fp)) - break; + len = fread(buf, 1, STDIO_BUFSIZE, fp); - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - } + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); /* update CRC */ COMP_FILE_CRC32(use_crc32c, crc, buf, len); + + if (feof(fp)) + break; } FIN_FILE_CRC32(use_crc32c, crc); fclose(fp); + pg_free(buf); return crc; } @@ -324,11 +325,11 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) { - gzFile fp; - pg_crc32 crc = 0; - char buf[STDIO_BUFSIZE]; - int len = 0; - int err; + gzFile fp; + pg_crc32 crc = 0; + int len = 0; + int err; + char *buf; INIT_FILE_CRC32(use_crc32c, crc); @@ -349,13 +350,15 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) file_path, strerror(errno)); } + buf = pgut_malloc(STDIO_BUFSIZE); + /* calc CRC of file */ for (;;) { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); - len = gzread(fp, &buf, sizeof(buf)); + len = gzread(fp, buf, STDIO_BUFSIZE); if (len <= 0) { @@ -377,6 +380,7 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) FIN_FILE_CRC32(use_crc32c, crc); gzclose(fp); + pg_free(buf); return crc; } @@ -1505,11 +1509,16 @@ dir_read_file_list(const char *root, const char *external_prefix, FILE *fp; parray *files; char buf[MAXPGPATH * 2]; + char stdio_buf[STDIO_BUFSIZE]; fp = fio_open_stream(file_txt, location); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", file_txt, strerror(errno)); + /* enable stdio buffering for local file */ + if (!fio_is_remote(location)) + setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); + files = parray_new(); while (fgets(buf, lengthof(buf), fp)) diff --git a/src/restore.c b/src/restore.c index 49ff04b3f..767e9a572 100644 --- a/src/restore.c +++ b/src/restore.c @@ -756,10 +756,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, static void * restore_files(void *arg) { - int i; - char to_fullpath[MAXPGPATH]; - FILE *out = NULL; - char buffer[STDIO_BUFSIZE]; + int i; + char to_fullpath[MAXPGPATH]; + FILE *out = NULL; + char *out_buf = pgut_malloc(STDIO_BUFSIZE); restore_files_arg *arguments = (restore_files_arg *) arg; @@ -856,18 +856,25 @@ restore_files(void *arg) if (dest_file->write_size == 0) goto done; - if (!fio_is_remote_file(out)) - setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); - /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) + { + /* enable stdio buffering for local destination file */ + if (!fio_is_remote_file(out)) + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file, out, to_fullpath); + } else + { + /* disable stdio buffering for local destination file */ + if (!fio_is_remote_file(out)) + setvbuf(out, NULL, _IONBF, BUFSIZ); /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, arguments->dest_backup, dest_file, out, to_fullpath); + } done: /* close file */ @@ -876,6 +883,8 @@ restore_files(void *arg) strerror(errno)); } + free(out_buf); + /* ssh connection to longer needed */ fio_disconnect(); diff --git a/src/utils/file.c b/src/utils/file.c index b53f5ccf7..bb05f3793 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1470,6 +1470,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, return n_blocks_read; } +/* TODO: read file using large buffer */ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) { BlockNumber blknum = 0; @@ -1881,7 +1882,7 @@ static void fio_send_file_impl(int out, char const* path) FILE *fp; fio_header hdr; char *buf = pgut_malloc(CHUNK_SIZE); - ssize_t read_len = 0; + size_t read_len = 0; char *errormsg = NULL; /* open source file for read */ @@ -1917,13 +1918,16 @@ static void fio_send_file_impl(int out, char const* path) goto cleanup; } + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + /* copy content */ for (;;) { read_len = fread(buf, 1, CHUNK_SIZE, fp); /* report error */ - if (read_len < 0 || (read_len == 0 && !feof(fp))) + if (ferror(fp)) { hdr.cop = FIO_ERROR; errormsg = pgut_malloc(MAXPGPATH); @@ -1938,9 +1942,7 @@ static void fio_send_file_impl(int out, char const* path) goto cleanup; } - else if (read_len == 0) - break; - else + if (read_len > 0) { /* send chunk */ hdr.cop = FIO_PAGE; @@ -1948,6 +1950,9 @@ static void fio_send_file_impl(int out, char const* path) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, read_len), read_len); } + + if (feof(fp)) + break; } /* we are done, send eof */ From b7f8283d48088ef6e09928092949ec384e5360cd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 Apr 2020 17:37:16 +0300 Subject: [PATCH 1241/2107] use dynamically allocated buffers in archive-push and archive-get --- src/archive.c | 74 +++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/src/archive.c b/src/archive.c index 7df3837d1..08d7825dd 100644 --- a/src/archive.c +++ b/src/archive.c @@ -414,8 +414,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d { FILE *in = NULL; int out = -1; - char buf[STDIO_BUFSIZE]; -// char buf[XLOG_BLCKSZ]; + char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; /* partial handling */ @@ -433,11 +432,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d canonicalize_path(to_fullpath); /* Open source file for read */ - in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); + in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) elog(ERROR, "Thread [%d]: Cannot open source file \"%s\": %s", thread_num, from_fullpath, strerror(errno)); + /* disable stdio buffering for input file */ + setvbuf(in, NULL, _IONBF, BUFSIZ); + /* open destination partial file for write */ snprintf(to_fullpath_part, sizeof(to_fullpath_part), "%s.part", to_fullpath); @@ -542,14 +544,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d pg_crc32 crc32_dst; crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false); if (crc32_src == crc32_dst) { elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); /* cleanup */ - fio_fclose(in); + fclose(in); fio_close(out); fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); return 1; @@ -574,11 +576,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* copy content */ for (;;) { - ssize_t read_len = 0; + size_t read_len = 0; - read_len = fio_fread(in, buf, sizeof(buf)); + read_len = fread(buf, 1, OUT_BUF_SIZE, in); - if (read_len < 0) + if (ferror(in)) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); elog(ERROR, "Thread [%d]: Cannot read source file \"%s\": %s", @@ -595,17 +597,12 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d } } - if (read_len == 0) + if (feof(in)) break; } /* close source file */ - if (fio_fclose(in)) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot close source WAL file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); - } + fclose(in); /* close temp file */ if (fio_close(out) != 0) @@ -636,6 +633,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d thread_num, to_fullpath_part, to_fullpath, strerror(errno)); } + pg_free(buf); return 0; } @@ -654,8 +652,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, { FILE *in = NULL; gzFile out = NULL; - int errno_temp; - char buf[STDIO_BUFSIZE]; + char *buf = pgut_malloc(OUT_BUF_SIZE); char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; char to_fullpath_gz[MAXPGPATH]; @@ -681,11 +678,14 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, snprintf(to_fullpath_gz_part, sizeof(to_fullpath_gz_part), "%s.part", to_fullpath_gz); /* Open source file for read */ - in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); + in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) elog(ERROR, "Thread [%d]: Cannot open source WAL file \"%s\": %s", thread_num, from_fullpath, strerror(errno)); + /* disable stdio buffering for input file */ + setvbuf(in, NULL, _IONBF, BUFSIZ); + /* Grab lock by creating temp file in exclusive mode */ out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (out == NULL) @@ -787,16 +787,16 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, pg_crc32 crc32_src; pg_crc32 crc32_dst; - /* what if one of them goes missing */ + /* TODO: what if one of them goes missing? */ crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_DB_HOST, true); + crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true); if (crc32_src == crc32_dst) { elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); /* cleanup */ - fio_fclose(in); + fclose(in); fio_gzclose(out); fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); return 1; @@ -811,8 +811,6 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Overwriting is forbidden, * so we must unlink partial file and exit with error. */ - fio_fclose(in); - fio_gzclose(out); fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); elog(ERROR, "Thread [%d]: WAL file already exists in archive with " "different checksum: \"%s\"", thread_num, to_fullpath_gz); @@ -823,11 +821,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* copy content */ for (;;) { - ssize_t read_len = 0; + size_t read_len = 0; - read_len = fio_fread(in, buf, sizeof(buf)); + read_len = fread(buf, 1, OUT_BUF_SIZE, in); - if (read_len < 0) + if (ferror(in)) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); elog(ERROR, "Thread [%d]: Cannot read from source file \"%s\": %s", @@ -838,32 +836,25 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, { if (fio_gzwrite(out, buf, read_len) != read_len) { - errno_temp = errno; fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); elog(ERROR, "Thread [%d]: Cannot write to compressed temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, get_gz_error(out, errno_temp)); + thread_num, to_fullpath_gz_part, get_gz_error(out, errno)); } } - if (read_len == 0) + if (feof(in)) break; } /* close source file */ - if (fio_fclose(in)) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot close source WAL file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); - } + fclose(in); /* close temp file */ if (fio_gzclose(out) != 0) { - errno_temp = errno; fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); elog(ERROR, "Thread [%d]: Cannot close compressed temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno_temp)); + thread_num, to_fullpath_gz_part, strerror(errno)); } /* sync temp file to disk */ @@ -887,6 +878,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, thread_num, to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); } + pg_free(buf); + return 0; } #endif @@ -1518,7 +1511,7 @@ get_wal_file(const char *filename, const char *from_fullpath, } /* - * Copy local WAL segment with possible decompression. + * Copy WAL segment with possible decompression from local archive. * Return codes: * FILE_MISSING (-1) * OPEN_FAILED (-2) @@ -1608,14 +1601,15 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, { read_len = fread(buf, 1, OUT_BUF_SIZE, in); - if (read_len < 0 || ferror(in)) + if (ferror(in)) { elog(WARNING, "Thread [%d]: Cannot read source WAL file \"%s\": %s", thread_num, from_path, strerror(errno)); exit_code = READ_FAILED; break; } - else if (read_len == 0 && feof(in)) + + if (read_len == 0 && feof(in)) break; } From 5d0a3be53628b9f929ff9cafa7ba13fadf31a36d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 Apr 2020 20:18:14 +0300 Subject: [PATCH 1242/2107] improvement: ignore hidden files when taking a backup --- src/dir.c | 10 +++++++++- src/pg_probackup.c | 2 +- src/restore.c | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index 5b5cd2b88..926a57b53 100644 --- a/src/dir.c +++ b/src/dir.c @@ -867,13 +867,21 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, continue; } + /* skip hidden files and directories */ + if (file->name[0] == '.') + { + elog(WARNING, "Skip hidden file: '%s'", file->path); + pgFileFree(file); + continue; + } + /* * Add only files, directories and links. Skip sockets and other * unexpected file formats. */ if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) { - elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + elog(WARNING, "Skip '%s': unexpected file format", file->path); pgFileFree(file); continue; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e7c3dce68..0f81b772f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -507,7 +507,7 @@ main(int argc, char *argv[]) /* * Ensure that requested backup instance exists. * for all commands except init, which doesn't take this parameter, - * add-instance which creates new instance + * add-instance, which creates new instance, * and archive-get, which just do not require it at this point */ if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && diff --git a/src/restore.c b/src/restore.c index 767e9a572..caecaea27 100644 --- a/src/restore.c +++ b/src/restore.c @@ -779,7 +779,7 @@ restore_files(void *arg) elog(ERROR, "Interrupted during restore"); if (progress) - elog(INFO, "Progress: (%d/%lu). Process file %s ", + elog(INFO, "Progress: (%d/%lu). Restore file \"%s\"", i + 1, (unsigned long) parray_num(arguments->dest_files), dest_file->rel_path); From f7159f3333381f12a5125ff7589670bd084af262 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 24 Apr 2020 20:28:33 +0300 Subject: [PATCH 1243/2107] tests: various fixes --- tests/archive.py | 2 +- tests/backup.py | 3 ++- tests/set_backup.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 17e90ddcb..c8ca97efe 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2279,7 +2279,7 @@ def test_archive_get_prefetch_corruption(self): # generate WAL, copy it into prefetch directory, then corrupt # some segment node.pgbench_init(scale=20) - sleep(10) + sleep(20) # now copy WAL files into prefetch directory and corrupt some of them archive_dir = os.path.join(backup_dir, 'wal', 'node') diff --git a/tests/backup.py b/tests/backup.py index 7c388cceb..f77f43f61 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1348,7 +1348,8 @@ def test_drop_rel_during_backup_page(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/set_backup.py b/tests/set_backup.py index f73b522c7..db039c92d 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -392,6 +392,7 @@ def test_add_note_newlines(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 95825afcfe3675626ab7a6c43bf855079e2aa382 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Apr 2020 02:05:46 +0300 Subject: [PATCH 1244/2107] [Issue #197] setting session parameters via PGOPTIONS envvar breaks parsing of backup.control --- src/utils/pgut.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a050a5128..b7599816f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -187,8 +187,10 @@ pgut_get_conninfo_string(PGconn *conn) (option->val != NULL && option->val[0] == '\0')) continue; - /* do not print password into the file */ - if (strcmp(option->keyword, "password") == 0) + /* do not print password, passfile and options into the file */ + if (strcmp(option->keyword, "password") == 0 || + strcmp(option->keyword, "passfile") == 0 || + strcmp(option->keyword, "options") == 0) continue; if (!firstkeyword) From 04dacea1e7040fe804e6dfafc6e70bd5f39ca220 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 01:24:54 +0300 Subject: [PATCH 1245/2107] various bugfixes and improvements for archiving and buffering --- src/archive.c | 22 ++++++++-------------- src/data.c | 4 ++-- src/merge.c | 19 ++++++++++++++----- src/pg_probackup.h | 2 +- src/restore.c | 4 ++-- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/archive.c b/src/archive.c index 08d7825dd..a063b07ed 100644 --- a/src/archive.c +++ b/src/archive.c @@ -587,14 +587,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d thread_num, from_fullpath, strerror(errno)); } - if (read_len > 0) + if (read_len > 0 && fio_write(out, buf, read_len) != read_len) { - if (fio_write(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot write to destination temp file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); - } + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot write to destination temp file \"%s\": %s", + thread_num, to_fullpath_part, strerror(errno)); } if (feof(in)) @@ -832,14 +829,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, thread_num, from_fullpath, strerror(errno)); } - if (read_len > 0) + if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) { - if (fio_gzwrite(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot write to compressed temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, get_gz_error(out, errno)); - } + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Thread [%d]: Cannot write to compressed temp WAL file \"%s\": %s", + thread_num, to_fullpath_gz_part, get_gz_error(out, errno)); } if (feof(in)) diff --git a/src/data.c b/src/data.c index c49b82836..aa3c0a6e1 100644 --- a/src/data.c +++ b/src/data.c @@ -855,7 +855,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char { int i; size_t total_write_len = 0; - char *in_buf; + char *in_buf = pgut_malloc(STDIO_BUFSIZE); for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -902,7 +902,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - in_buf = pgut_malloc(STDIO_BUFSIZE); + /* set stdio buffering for input data file */ setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* diff --git a/src/merge.c b/src/merge.c index b5927f719..811df7752 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1132,11 +1132,11 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, const char *full_database_dir) { - FILE *out = NULL; - char to_fullpath[MAXPGPATH]; - char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ - char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ - char buffer[STDIO_BUFSIZE]; + FILE *out = NULL; + char *buffer = pgut_malloc(STDIO_BUFSIZE); + char to_fullpath[MAXPGPATH]; + char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ + char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ /* The next possible optimization is copying "as is" the file * from intermediate incremental backup, that didn`t changed in @@ -1158,6 +1158,15 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); fclose(out); + pg_free(buffer); + + /* tmp_file->size is greedy, even if there is single 8KB block in file, + * that was overwritten twice during restore_data_file, we would assume that its size is + * 16KB. + * TODO: maybe we should just trust dest_file->n_blocks? + * No, we can`t, because current binary can be used to merge + * 2 backups of old versions, were n_blocks is missing. + */ backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 152966be4..e3007a3a2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1016,7 +1016,7 @@ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonL int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); /* return codes for fio_send_pages */ -#define OUT_BUF_SIZE (1024 * 1024) +#define OUT_BUF_SIZE (512 * 1024) extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); diff --git a/src/restore.c b/src/restore.c index caecaea27..9e770281a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -859,7 +859,7 @@ restore_files(void *arg) /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) { - /* enable stdio buffering for local destination file */ + /* enable stdio buffering for local destination non-data file */ if (!fio_is_remote_file(out)) setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ @@ -868,7 +868,7 @@ restore_files(void *arg) } else { - /* disable stdio buffering for local destination file */ + /* disable stdio buffering for local destination data file */ if (!fio_is_remote_file(out)) setvbuf(out, NULL, _IONBF, BUFSIZ); /* Destination file is non-data file */ From 29275494565a4fb3eb918943bf02f95f01164b3c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 01:26:56 +0300 Subject: [PATCH 1246/2107] tests: fixes for 2.3.0 --- tests/archive.py | 1 - tests/compatibility.py | 143 +++++++++++++++++++++++++++++--- tests/helpers/ptrack_helpers.py | 3 +- tests/ptrack.py | 15 ++-- tests/replica.py | 3 + tests/restore.py | 2 +- 6 files changed, 148 insertions(+), 19 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index c8ca97efe..ec14254f7 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1200,7 +1200,6 @@ def test_archive_catalog(self): os.path.join(backup_dir, 'wal', 'master'), os.path.join(backup_dir, 'wal', 'replica')) - # Check data correctness on replica replica.slow_start(replica=True) # FULL backup replica diff --git a/tests/compatibility.py b/tests/compatibility.py index b26c8de1d..a3f440447 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -3,6 +3,7 @@ import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from sys import exit +import shutil module_name = 'compatibility' @@ -627,24 +628,23 @@ def test_backward_compatibility_merge_1(self): self.set_archiving(backup_dir, 'node', node, old_binary=True) node.slow_start() - node.pgbench_init(scale=1) + node.pgbench_init(scale=20) # FULL backup with OLD binary self.backup_node( - backup_dir, 'node', node, - old_binary=True) + backup_dir, 'node', node, old_binary=True) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"]) + options=["-c", "1", "-T", "10", "--no-vacuum"]) pgbench.wait() pgbench.stdout.close() # PAGE1 backup with OLD binary - backup_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, - backup_type='page', old_binary=True) + backup_type='page', old_binary=True, options=['--log-level-file=LOG']) node.safe_psql( 'postgres', @@ -655,20 +655,20 @@ def test_backward_compatibility_merge_1(self): 'VACUUM pgbench_accounts') # PAGE2 backup with OLD binary - backup_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, - backup_type='page', old_binary=True) + backup_type='page', old_binary=True, options=['--log-level-file=LOG']) # PAGE3 backup with OLD binary backup_id = self.backup_node( backup_dir, 'node', node, - backup_type='page', old_binary=True) + backup_type='page', old_binary=True, options=['--log-level-file=LOG']) pgdata = self.pgdata_content(node.data_dir) # merge chain created by old binary with new binary output = self.merge_backup( - backup_dir, "node", backup_id) + backup_dir, "node", backup_id, options=['--log-level-file=LOG']) # check that in-place is disabled self.assertIn( @@ -689,6 +689,129 @@ def test_backward_compatibility_merge_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_2(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=50) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + page1 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE from pgbench_accounts where ctid > '(10,1)'") + + # PAGE2 backup with OLD binary + page2 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata2 = self.pgdata_content(node.data_dir) + + # PAGE3 backup with OLD binary + page3 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE4 backup with NEW binary + page4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + pgdata4 = self.pgdata_content(node.data_dir) + + # merge backups one by one and check data correctness + # merge PAGE1 + self.merge_backup( + backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + + # check data correctness for PAGE1 + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, backup_id=page1, + options=['--log-level-file=VERBOSE']) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + # merge PAGE2 + self.merge_backup(backup_dir, "node", page2) + + # check data correctness for PAGE2 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + # merge PAGE3 + self.show_pb(backup_dir, 'node', page3) + self.merge_backup(backup_dir, "node", page3) + self.show_pb(backup_dir, 'node', page3) + + # check data correctness for PAGE3 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # merge PAGE4 + self.merge_backup(backup_dir, "node", page4) + + # check data correctness for PAGE4 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata4, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_page_vacuum_truncate(self): """ diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 2f95417d0..09a85c2ca 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -340,7 +340,8 @@ def make_simple_node( # set major version with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version = float(str(f.read().rstrip())) + node.major_version_str = str(f.read().rstrip()) + node.major_version = float(node.major_version_str) # Sane default parameters options = {} diff --git a/tests/ptrack.py b/tests/ptrack.py index 470db3dcf..309a3c442 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -388,7 +388,9 @@ def test_ptrack_uncommitted_xact(self): set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], - pg_options={'wal_level': 'replica'}) + pg_options={ + 'wal_level': 'replica', + 'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -421,19 +423,20 @@ def test_ptrack_uncommitted_xact(self): node_restored.cleanup() self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + backup_dir, 'node', node_restored, + node_restored.data_dir, options=["-j", "4"]) - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( + pgdata_restored = self.pgdata_content( node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) self.set_auto_conf( node_restored, {'port': node_restored.port}) node_restored.slow_start() + # Physical comparison + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py index 62a56d8c0..85e104f06 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1455,6 +1455,9 @@ def test_replica_promote_archive_page(self): # node1 is back to be a master node1.promote() node1.safe_psql('postgres', 'CHECKPOINT') + self.switch_wal_segment(node1) + + sleep(5) # delta3_id = self.backup_node( # backup_dir, 'node', node2, node2.data_dir, 'delta') diff --git a/tests/restore.py b/tests/restore.py index b8f7b11da..e7a3c1a76 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2462,7 +2462,7 @@ def test_partial_restore_exclude_tablespace(self): node.slow_start() cat_version = node.get_control_data()["Catalog version number"] - version_specific_dir = 'PG_' + str(node.major_version) + '_' + cat_version + version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version # PG_10_201707211 # pg_tblspc/33172/PG_9.5_201510051/16386/ From 985ddf898af5a91fc1277f4974a2281f1d9023b6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 01:38:00 +0300 Subject: [PATCH 1247/2107] fix Windows build --- gen_probackup_project.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index cf82543d0..b78f4699e 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -207,6 +207,7 @@ sub build_pgprobackup $probackup->AddIncludeDir("$pgsrc/src/interfaces/libpq"); $probackup->AddIncludeDir("$pgsrc/src"); $probackup->AddIncludeDir("$pgsrc/src/port"); + $probackup->AddIncludeDir("$pgsrc/src/include/portability"); $probackup->AddIncludeDir("$currpath"); $probackup->AddIncludeDir("$currpath/src"); From 853d68522f3620d27a1a1b580e668ca3d4951838 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 03:09:57 +0300 Subject: [PATCH 1248/2107] tests: disable autostart --- tests/helpers/ptrack_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 09a85c2ca..52a1ffda5 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -358,6 +358,7 @@ def make_simple_node( options['log_min_duration_statement'] = 0 options['log_connections'] = 'on' options['log_disconnections'] = 'on' + options['restart_after_crash'] = 'off' # Allow replication in pg_hba.conf if set_replication: From 1c81e7569930723aa8167e7de7e43fcfe010e3ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 14:27:36 +0300 Subject: [PATCH 1249/2107] minor improvements --- src/data.c | 5 ++++- tests/expected/option_help.out | 21 +++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/data.c b/src/data.c index aa3c0a6e1..7628c37e5 100644 --- a/src/data.c +++ b/src/data.c @@ -821,8 +821,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, { /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) - return copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, + { + copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, to_fullpath, FIO_BACKUP_HOST, file); + return; + } /* * If non-data file exists in previous backup diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index da7ebcd17..aaf663536 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -32,7 +32,8 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--help] pg_probackup set-backup -B backup-path --instance=instance_name - -i backup-id [--ttl] [--expire-time] + -i backup-id [--ttl=interval] [--expire-time=timestamp] + [--note=text] [--help] pg_probackup show-config -B backup-path --instance=instance_name @@ -45,6 +46,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--backup-pg-log] [-j num-threads] [--progress] [--no-validate] [--skip-block-validation] [--external-dirs=external-directories-paths] + [--no-sync] [--log-level-console=log-level-console] [--log-level-file=log-level-file] [--log-filename=log-filename] @@ -65,7 +67,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] - [--ttl] [--expire-time] + [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] pg_probackup restore -B backup-path --instance=instance_name @@ -77,10 +79,13 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] [--restore-as-replica] [--force] + [--primary-conninfo=primary_conninfo] + [-S | --primary-slot-name=slotname] [--no-validate] [--skip-block-validation] [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [--restore-command=cmdline] + [--no-sync] [--db-include | --db-exclude] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] @@ -114,7 +119,8 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] [--wal-depth=wal-depth] - [--delete-wal] [-i backup-id | --delete-expired | --merge-expired] + [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] + [--delete-wal] [--dry-run] [--help] @@ -135,10 +141,11 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--help] pg_probackup archive-push -B backup-path --instance=instance_name - --wal-file-path=wal-file-path --wal-file-name=wal-file-name - [--overwrite] - [--compress] + [-j num-threads] [--batch-size=batch_size] + [--archive-timeout=timeout] + [--no-ready-rename] [--no-sync] + [--overwrite] [--compress] [--compress-algorithm=compress-algorithm] [--compress-level=compress-level] [--remote-proto] [--remote-host] @@ -149,6 +156,8 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup archive-get -B backup-path --instance=instance_name --wal-file-path=wal-file-path --wal-file-name=wal-file-name + [-j num-threads] [--batch-size=batch_size] + [--no-validate-wal] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] From 99fdd3b919c9e4a7fe2ab738552e76dd7feed5fb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 29 Apr 2020 14:28:55 +0300 Subject: [PATCH 1250/2107] Version 2.3.0 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e3007a3a2..676731685 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.2.8" -#define AGENT_PROTOCOL_VERSION 20208 +#define PROGRAM_VERSION "2.3.0" +#define AGENT_PROTOCOL_VERSION 20300 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index dd345e045..bf837e9d2 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.2.7 \ No newline at end of file +pg_probackup 2.3.0 \ No newline at end of file From c97abd81e40b84eb29f43bfe875a3414241749f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 30 Apr 2020 19:09:02 +0300 Subject: [PATCH 1251/2107] Readme: update current release version --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ef0ff995..7ca124ea2 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,11 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. -* Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. +* Remote backup via ssh on Windows currently is not supported. ## Current release -[2.2.7](https://github.com/postgrespro/pg_probackup/releases/tag/2.2.7) +[2.3.0](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.0) ## Documentation From e8346d5d2971edf78cb1b6948380025f64d0e608 Mon Sep 17 00:00:00 2001 From: Liudmila Mantrova Date: Thu, 30 Apr 2020 19:28:18 +0300 Subject: [PATCH 1252/2107] DOC: fix xml markup --- doc/pgprobackup.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a94a23c37..49ab51964 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3390,6 +3390,7 @@ pg_probackup set-backup -B backup_dir --instance backup.control configuration file, or modifies the previously defined values. + @@ -3402,6 +3403,7 @@ pg_probackup set-backup -B backup_dir --instance + For all available pinning settings, see the section Pinning Options. From 3e0c871f6b741ec85793e549137aed50c8c7fa81 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 00:52:25 +0300 Subject: [PATCH 1253/2107] keep compiler happy --- src/show.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/show.c b/src/show.c index f242bee4e..edd020a60 100644 --- a/src/show.c +++ b/src/show.c @@ -445,7 +445,7 @@ static int show_backup(const char *instance_name, time_t requested_backup_id) { int i; - pgBackup *backup; + pgBackup *backup = NULL; parray *backups; backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); From 0ac5920a7c2ea394f2f5bc10f160611d99065c3a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 02:44:36 +0300 Subject: [PATCH 1254/2107] bugfix: multi-timeline pagemap --- src/parsexlog.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 6c6691703..885907c00 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -276,7 +276,6 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, timelineInfo *end_tlinfo = NULL; timelineInfo *tmp_tlinfo = NULL; XLogRecPtr prev_switchpoint = InvalidXLogRecPtr; - lsnInterval *wal_interval = NULL; /* We must find TLI information about final timeline (t3 in example) */ for (i = 0; i < parray_num(tli_list); i++) @@ -298,7 +297,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, tmp_tlinfo = end_tlinfo; while (tmp_tlinfo) { - wal_interval = pgut_malloc(sizeof(lsnInterval)); + lsnInterval *wal_interval = pgut_malloc(sizeof(lsnInterval)); wal_interval->tli = tmp_tlinfo->tli; if (tmp_tlinfo->tli == end_tli) @@ -326,7 +325,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, for (i = parray_num(interval_list) - 1; i >= 0; i--) { bool inclusive_endpoint; - wal_interval = parray_get(interval_list, i); + lsnInterval *tmp_interval = (lsnInterval *) parray_get(interval_list, i); /* In case of replica promotion, endpoints of intermediate * timelines can be unreachable. @@ -334,17 +333,17 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, inclusive_endpoint = false; /* ... but not the end timeline */ - if (wal_interval->tli == end_tli) + if (tmp_interval->tli == end_tli) inclusive_endpoint = true; extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, - InvalidXLogRecPtr, wal_interval->tli, wal_seg_size, - wal_interval->begin_lsn, wal_interval->end_lsn, + InvalidXLogRecPtr, tmp_interval->tli, wal_seg_size, + tmp_interval->begin_lsn, tmp_interval->end_lsn, false, extractPageInfo, NULL, inclusive_endpoint); if (!extract_isok) break; - pg_free(wal_interval); + pg_free(tmp_interval); } pg_free(interval_list); } @@ -1064,6 +1063,12 @@ RunXLogThreads(const char *archivedir, time_t target_time, elog(ERROR, "Invalid startpoint value %X/%X", (uint32) (startpoint >> 32), (uint32) (startpoint)); + if (process_record) + elog(LOG, "Extracting pagemap from tli %i on range from %X/%X to %X/%X", + tli, + (uint32) (startpoint >> 32), (uint32) (startpoint), + (uint32) (endpoint >> 32), (uint32) (endpoint)); + if (!XLogRecPtrIsInvalid(endpoint)) { // if (XRecOffIsNull(endpoint) && !inclusive_endpoint) From 686552825e09d0ad63a3a836d852e0d831d3d303 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 02:55:41 +0300 Subject: [PATCH 1255/2107] additional fix --- src/parsexlog.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index 885907c00..3b68f45ae 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -320,6 +320,9 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, tmp_tlinfo = tmp_tlinfo->parent_link; parray_append(interval_list, wal_interval); + + if (tmp_tlinfo->tli == start_tli) + break; } for (i = parray_num(interval_list) - 1; i >= 0; i--) From d3ed73ba318b26965928f271ea238fe3679cc7eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 03:19:24 +0300 Subject: [PATCH 1256/2107] minor fix --- src/parsexlog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 3b68f45ae..f16285ee3 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -272,7 +272,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, * - [C,X] on t3 */ int i; - parray *interval_list = parray_new(); + parray *interval_list = parray_new(); timelineInfo *end_tlinfo = NULL; timelineInfo *tmp_tlinfo = NULL; XLogRecPtr prev_switchpoint = InvalidXLogRecPtr; @@ -327,7 +327,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, for (i = parray_num(interval_list) - 1; i >= 0; i--) { - bool inclusive_endpoint; + bool inclusive_endpoint; lsnInterval *tmp_interval = (lsnInterval *) parray_get(interval_list, i); /* In case of replica promotion, endpoints of intermediate From 2e417720bd234632df8b2b1975f21064df82af86 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 03:27:30 +0300 Subject: [PATCH 1257/2107] bugfix: multi-timeline pagemap, another take --- src/parsexlog.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index f16285ee3..d39662e4a 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -316,13 +316,13 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size, wal_interval->end_lsn = prev_switchpoint; } - prev_switchpoint = tmp_tlinfo->switchpoint; - tmp_tlinfo = tmp_tlinfo->parent_link; - parray_append(interval_list, wal_interval); if (tmp_tlinfo->tli == start_tli) break; + + prev_switchpoint = tmp_tlinfo->switchpoint; + tmp_tlinfo = tmp_tlinfo->parent_link; } for (i = parray_num(interval_list) - 1; i >= 0; i--) From 0f0aa42d155dbd478517b9cb3c407f5e0253a7ed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 1 May 2020 14:13:08 +0300 Subject: [PATCH 1258/2107] tests: update for tests.page.PageTest.test_multi_timeline_page --- tests/page.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/page.py b/tests/page.py index d84003d6a..7e95f9c4b 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1210,10 +1210,12 @@ def test_multi_timeline_page(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=50) + node.safe_psql("postgres", "create extension pageinspect") + + node.pgbench_init(scale=20) full_id = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() self.backup_node(backup_dir, 'node', node, backup_type='delta') @@ -1231,23 +1233,43 @@ def test_multi_timeline_page(self): pgbench.wait() # create timelines - for i in range(2, 12): + for i in range(2, 6): node.cleanup() self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=['--recovery-target-timeline={0}'.format(i)]) node.slow_start() - pgbench = node.pgbench(options=['-T', '3', '-c', '1', '--no-vacuum']) + node.safe_psql("postgres", "CHECKPOINT") + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() + if i % 2 == 0: + self.backup_node(backup_dir, 'node', node, backup_type='page') + page_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--log-level-file=VERBOSE']) pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - self.restore_node(backup_dir, 'node', node) - pgdata_restored = self.pgdata_content(node.data_dir) + +# result = node.safe_psql( +# "postgres", "select * from pgbench_accounts") + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + +# result_new = node_restored.safe_psql( +# "postgres", "select * from pgbench_accounts") + +# self.assertEqual(result, result_new) + self.compare_pgdata(pgdata, pgdata_restored) show = self.show_archive(backup_dir) From 6247ba4f535ef0ea31bc492302e7a59891dcfab5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 May 2020 14:20:50 +0300 Subject: [PATCH 1259/2107] tests: fixes --- tests/page.py | 141 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 128 insertions(+), 13 deletions(-) diff --git a/tests/page.py b/tests/page.py index 7e95f9c4b..5545f3c0c 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1212,6 +1212,15 @@ def test_multi_timeline_page(self): node.safe_psql("postgres", "create extension pageinspect") + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + node.pgbench_init(scale=20) full_id = self.backup_node(backup_dir, 'node', node) @@ -1233,16 +1242,18 @@ def test_multi_timeline_page(self): pgbench.wait() # create timelines - for i in range(2, 6): + for i in range(2, 7): node.cleanup() self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + backup_dir, 'node', node, options=['--recovery-target-timeline={0}'.format(i)]) node.slow_start() - node.safe_psql("postgres", "CHECKPOINT") + + # at this point there is i+1 timeline pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() + # create backup at 2, 4 and 6 timeline if i % 2 == 0: self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -1252,8 +1263,8 @@ def test_multi_timeline_page(self): pgdata = self.pgdata_content(node.data_dir) -# result = node.safe_psql( -# "postgres", "select * from pgbench_accounts") + result = node.safe_psql( + "postgres", "select * from pgbench_accounts") node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -1265,21 +1276,125 @@ def test_multi_timeline_page(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() -# result_new = node_restored.safe_psql( -# "postgres", "select * from pgbench_accounts") + result_new = node_restored.safe_psql( + "postgres", "select * from pgbench_accounts") -# self.assertEqual(result, result_new) + self.assertEqual(result, result_new) self.compare_pgdata(pgdata, pgdata_restored) - show = self.show_archive(backup_dir) + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node.port)]) - timelines = show[0]['timelines'] + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node_restored.port)]) + + backup_list = self.show_pb(backup_dir, 'node') - # self.assertEqual() self.assertEqual( - self.show_pb(backup_dir, 'node', page_id)['parent-backup-id'], - full_id) + backup_list[2]['parent-backup-id'], + backup_list[0]['id']) + + self.assertEqual( + backup_list[3]['parent-backup-id'], + backup_list[2]['id']) + + self.assertEqual( + backup_list[4]['parent-backup-id'], + backup_list[3]['id']) + + self.assertEqual( + backup_list[5]['parent-backup-id'], + backup_list[4]['id']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multitimeline_page_1(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t2 /----> + t1 -F--P---D-> + + P must have F as parent + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql("postgres", "create extension pageinspect") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + node.pgbench_init(scale=20) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '20', '-c', '1']) + pgbench.wait() + + page1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + page1 = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page1, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + print(self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-console=LOG'], return_id=False)) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From 96090b252273f0073b2867ae18ad32b701bbc676 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 May 2020 14:43:50 +0300 Subject: [PATCH 1260/2107] tests: some more fixes --- tests/page.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/page.py b/tests/page.py index 5545f3c0c..a73328e1c 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1238,7 +1238,7 @@ def test_multi_timeline_page(self): node.slow_start() - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # create timelines @@ -1246,11 +1246,14 @@ def test_multi_timeline_page(self): node.cleanup() self.restore_node( backup_dir, 'node', node, - options=['--recovery-target-timeline={0}'.format(i)]) + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote', + '--recovery-target-timeline={0}'.format(i)]) node.slow_start() # at this point there is i+1 timeline - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) pgbench.wait() # create backup at 2, 4 and 6 timeline @@ -1302,18 +1305,22 @@ def test_multi_timeline_page(self): self.assertEqual( backup_list[2]['parent-backup-id'], backup_list[0]['id']) + self.assertEqual(backup_list[2]['current-tli'], 3) self.assertEqual( backup_list[3]['parent-backup-id'], backup_list[2]['id']) + self.assertEqual(backup_list[3]['current-tli'], 5) self.assertEqual( backup_list[4]['parent-backup-id'], backup_list[3]['id']) + self.assertEqual(backup_list[4]['current-tli'], 7) self.assertEqual( backup_list[5]['parent-backup-id'], backup_list[4]['id']) + self.assertEqual(backup_list[5]['current-tli'], 7) # Clean after yourself self.del_test_dir(module_name, fname) From c0f33fc68796763e40a720b3a5ad52847a21b042 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 May 2020 21:29:01 +0300 Subject: [PATCH 1261/2107] bugfix: full backup, produced by merge of multi-timeline chain, now correctly inherits timeline ID of destination backup --- src/merge.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/merge.c b/src/merge.c index 811df7752..3057a3a5e 100644 --- a/src/merge.c +++ b/src/merge.c @@ -690,6 +690,8 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->stop_lsn = dest_backup->stop_lsn; full_backup->recovery_time = dest_backup->recovery_time; full_backup->recovery_xid = dest_backup->recovery_xid; + full_backup->tli = dest_backup->tli; + full_backup->from_replica = dest_backup->from_replica; pfree(full_backup->external_dir_str); full_backup->external_dir_str = pgut_strdup(dest_backup->external_dir_str); From 60fba1722121606ea6e40819076065c3dc5fc473 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 May 2020 21:29:35 +0300 Subject: [PATCH 1262/2107] tests: added tests.merge.MergeTest.test_multi_timeline_merge --- tests/merge.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 0ddec258d..61bb27c1d 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2487,5 +2487,124 @@ def test_merge_correct_inheritance_1(self): self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multi_timeline_merge(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t12 /---P--> + ... + t3 /----> + t2 /----> + t1 -F-----D-> + + P must have F as parent + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql("postgres", "create extension pageinspect") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + node.pgbench_init(scale=20) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create timelines + for i in range(2, 7): + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote', + '--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + + # at this point there is i+1 timeline + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create backup at 2, 4 and 6 timeline + if i % 2 == 0: + self.backup_node(backup_dir, 'node', node, backup_type='page') + + page_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + self.merge_backup(backup_dir, 'node', page_id) + + result = node.safe_psql( + "postgres", "select * from pgbench_accounts") + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from pgbench_accounts") + + self.assertEqual(result, result_new) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node.port)]) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node_restored.port)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From 53f9d283d7ebbecb90b248099588c3a72eda3daa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 3 May 2020 00:02:11 +0300 Subject: [PATCH 1263/2107] tests: some more fixes for 2.3.1 --- tests/compatibility.py | 20 +++++--------------- tests/option.py | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index a3f440447..65114c78c 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -631,8 +631,7 @@ def test_backward_compatibility_merge_1(self): node.pgbench_init(scale=20) # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) + self.backup_node(backup_dir, 'node', node, old_binary=True) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -643,8 +642,7 @@ def test_backward_compatibility_merge_1(self): # PAGE1 backup with OLD binary self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--log-level-file=LOG']) + backup_dir, 'node', node, backup_type='page', old_binary=True) node.safe_psql( 'postgres', @@ -655,20 +653,13 @@ def test_backward_compatibility_merge_1(self): 'VACUUM pgbench_accounts') # PAGE2 backup with OLD binary - self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--log-level-file=LOG']) - - # PAGE3 backup with OLD binary backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--log-level-file=LOG']) + backup_dir, 'node', node, backup_type='page', old_binary=True) pgdata = self.pgdata_content(node.data_dir) # merge chain created by old binary with new binary - output = self.merge_backup( - backup_dir, "node", backup_id, options=['--log-level-file=LOG']) + output = self.merge_backup(backup_dir, "node", backup_id) # check that in-place is disabled self.assertIn( @@ -680,8 +671,7 @@ def test_backward_compatibility_merge_1(self): base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.restore_node(backup_dir, 'node', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/option.py b/tests/option.py index 70d8fdd61..023a0c2c6 100644 --- a/tests/option.py +++ b/tests/option.py @@ -94,7 +94,7 @@ def test_options_4(self): except ProbackupException as e: self.assertIn( 'ERROR: You must specify at least one of the delete options: ' - '--delete-expired |--delete-wal |--merge-expired |(-i, --backup-id)', + '--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From 5bdfd0fe5cd1a3bbcf426e90e727578dd90f5454 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 May 2020 17:40:19 +0300 Subject: [PATCH 1264/2107] Version 2.3.1 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 676731685..6a4fc6dab 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.0" -#define AGENT_PROTOCOL_VERSION 20300 +#define PROGRAM_VERSION "2.3.1" +#define AGENT_PROTOCOL_VERSION 20301 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index bf837e9d2..ba90ec994 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.0 \ No newline at end of file +pg_probackup 2.3.1 \ No newline at end of file From 3ea7b02ef586c3c92bbaaf70146924e0b5337457 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 3 May 2020 23:03:55 +0300 Subject: [PATCH 1265/2107] Readme: update for 2.3.1 --- README.md | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7ca124ea2..e5d59afe9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The utility is compatible with: As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. -* Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to to do periodical full backups. +* Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to do periodical full backups. * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery * Verification: on-demand verification of PostgreSQL instance with the `checkdb` command. * Retention: managing WAL archive and backups in accordance with retention policy. You can configure retention policy based on recovery time or the number of backups to keep, as well as specify `time to live` (TTL) for a particular backup. Expired backups can be merged or deleted. @@ -52,11 +52,12 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.0](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.0) +[2.3.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.1) ## Documentation -Documentation can be found at [github](https://postgrespro.github.io/pg_probackup). +Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](h +ttps://postgrespro.com/docs/postgrespro/current/app-pgprobackup) ## Installation and Setup ### Windows Installation @@ -66,28 +67,28 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #### pg_probackup for vanilla PostgreSQL ```shell #DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list -sudo wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update +sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{12,11,10,9.6,9.5} sudo apt-get install pg-probackup-{12,11,10,9.6,9.5}-dbg #DEB-SRC Packages -sudo echo "deb-src [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ +sudo echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update sudo apt-get source pg-probackup-{12,11,10,9.6,9.5} #RPM Centos Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm yum install pg_probackup-{12,11,10,9.6,9.5} yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM RHEL Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm yum install pg_probackup-{12,11,10,9.6,9.5} yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM Oracle Linux Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm yum install pg_probackup-{12,11,10,9.6,9.5} yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo @@ -95,19 +96,19 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} #RPM ALT Linux p7 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux p8 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux p9 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo @@ -116,40 +117,40 @@ sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #### pg_probackup for PostgresPro Standart and Enterprise ```shell #DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list -sudo wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update +sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list +sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg #RPM Centos Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM RHEL Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM Oracle Linux Packages -rpm -ivh http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm +rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm yum install pg_probackup-{std,ent}-{11,10,9.6} yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM ALT Linux p7 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM ALT Linux p8 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo #RPM ALT Linux p9 -sudo echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo ``` From 596d08b9e2e7d053e04361b8cab40901522d3a94 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 3 May 2020 23:05:50 +0300 Subject: [PATCH 1266/2107] fix for Readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index e5d59afe9..8829e236e 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Documentation -Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](h -ttps://postgrespro.com/docs/postgrespro/current/app-pgprobackup) +Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup) ## Installation and Setup ### Windows Installation From 46c7f921a9dff0faad8172061b553b868d59f5c2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 May 2020 02:14:32 +0300 Subject: [PATCH 1267/2107] [Issue #201] WIP --- src/data.c | 28 ++++++++++++++++++------ src/pg_probackup.h | 9 +++++++- src/util.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/data.c b/src/data.c index 7628c37e5..469a294bd 100644 --- a/src/data.c +++ b/src/data.c @@ -860,7 +860,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); - for (i = parray_num(parent_chain) - 1; i >= 0; i--) +// for (i = parray_num(parent_chain) - 1; i >= 0; i--) + for (i = 0; i < parray_num(parent_chain); i++) { char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; @@ -916,11 +917,14 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char */ total_write_len += restore_data_file_internal(in, out, tmp_file, parse_program_version(backup->program_version), - from_fullpath, to_fullpath, dest_file->n_blocks); + from_fullpath, to_fullpath, dest_file->n_blocks, + &(dest_file)->pagemap); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, strerror(errno)); + +// datapagemap_print_debug(&(dest_file)->pagemap); } pg_free(in_buf); @@ -929,7 +933,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks) + const char *from_fullpath, const char *to_fullpath, int nblocks, + datapagemap_t *map) { BackupPageHeader header; BlockNumber blknum = 0; @@ -987,9 +992,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } /* sanity? */ - if (header.block < blknum) - elog(ERROR, "Backup is broken at block %u of \"%s\"", - blknum, from_fullpath); +// if (header.block < blknum) +// elog(ERROR, "Backup is broken at block %u of \"%s\"", +// blknum, from_fullpath); blknum = header.block; @@ -1037,6 +1042,15 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); + /* if this page was already restored, then skip it */ + if (datapagemap_is_set(map, blknum)) + { + elog(WARNING, "Skipping block %u because is was already restored", blknum); + /* TODO: check error */ + fseek(in, compressed_size, SEEK_CUR); + continue; + } + /* read a page from file */ read_len = fread(page.data, 1, MAXALIGN(compressed_size), in); @@ -1092,6 +1106,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers write_len += BLCKSZ; cur_pos = write_pos + BLCKSZ; /* update current write position */ + + datapagemap_add(map, blknum); } elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6a4fc6dab..a126bf4e3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -925,7 +925,8 @@ extern void backup_non_data_file_internal(const char *from_fullpath, extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks); + const char *from_fullpath, const char *to_fullpath, int nblocks, + datapagemap_t *map); extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath); extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, @@ -1037,4 +1038,10 @@ extern void get_header_errormsg(Page page, char **errormsg); extern void get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno); +extern bool +datapagemap_is_set(datapagemap_t *map, BlockNumber blkno); + +extern void +datapagemap_print_debug(datapagemap_t *map); + #endif /* PG_PROBACKUP_H */ diff --git a/src/util.c b/src/util.c index 776c2ec72..bf9c6e33a 100644 --- a/src/util.c +++ b/src/util.c @@ -495,3 +495,57 @@ str2status(const char *status) return BACKUP_STATUS_INVALID; } + +bool +datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) +{ + int offset; + int bitno; + + offset = blkno / 8; + bitno = blkno % 8; + + /* enlarge or create bitmap if needed */ + if (map->bitmapsize <= offset) + { + int oldsize = map->bitmapsize; + int newsize; + + /* + * The minimum to hold the new bit is offset + 1. But add some + * headroom, so that we don't need to repeatedly enlarge the bitmap in + * the common case that blocks are modified in order, from beginning + * of a relation to the end. + */ + newsize = offset + 1; + newsize += 10; + + map->bitmap = pg_realloc(map->bitmap, newsize); + + /* zero out the newly allocated region */ + memset(&map->bitmap[oldsize], 0, newsize - oldsize); + + map->bitmapsize = newsize; + } + + //datapagemap_print(map); + + /* check the bit */ + return map->bitmap[offset] & (1 << bitno); +} + +/* + * A debugging aid. Prints out the contents of the page map. + */ +void +datapagemap_print_debug(datapagemap_t *map) +{ + datapagemap_iterator_t *iter; + BlockNumber blocknum; + + iter = datapagemap_iterate(map); + while (datapagemap_next(iter, &blocknum)) + elog(INFO, " block %u", blocknum); + + pg_free(iter); +} From 26a834b44c7cde63378ea93b0ae87e14049cf035 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 May 2020 02:30:13 +0300 Subject: [PATCH 1268/2107] [Issue #201] fix alignment when skipping block during restore --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 469a294bd..45b159fcd 100644 --- a/src/data.c +++ b/src/data.c @@ -1047,7 +1047,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers { elog(WARNING, "Skipping block %u because is was already restored", blknum); /* TODO: check error */ - fseek(in, compressed_size, SEEK_CUR); + fseek(in, MAXALIGN(compressed_size), SEEK_CUR); continue; } From b949231b9390edbcae79494338539f4802e8e1e1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 7 May 2020 16:21:26 +0300 Subject: [PATCH 1269/2107] [Issue #201] check fseek return code --- src/data.c | 9 +++++---- src/restore.c | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/data.c b/src/data.c index 45b159fcd..733aec052 100644 --- a/src/data.c +++ b/src/data.c @@ -1042,12 +1042,13 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); - /* if this page was already restored, then skip it */ + /* if this page is marked as already restored, then skip it */ if (datapagemap_is_set(map, blknum)) { - elog(WARNING, "Skipping block %u because is was already restored", blknum); - /* TODO: check error */ - fseek(in, MAXALIGN(compressed_size), SEEK_CUR); + /* skip to the next page */ + if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) + elog(ERROR, "Cannot seek block %u of '%s': %s", + blknum, from_fullpath, strerror(errno)); continue; } diff --git a/src/restore.c b/src/restore.c index 9e770281a..ecd99022b 100644 --- a/src/restore.c +++ b/src/restore.c @@ -876,6 +876,9 @@ restore_files(void *arg) arguments->dest_backup, dest_file, out, to_fullpath); } + /* free pagemap used for restore optimization */ + pg_free(dest_file->pagemap.bitmap); + done: /* close file */ if (fio_fclose(out) != 0) From b0cbba8417e9e9355ff161311c112a2de134417c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 8 May 2020 20:24:13 +0300 Subject: [PATCH 1270/2107] [Issue #203] avoid backup stalling because of frequent metadata flushing --- src/backup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0e4b002de..b9bc9836e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2072,15 +2072,15 @@ backup_files(void *arg) if (arguments->thread_num == 1) { - /* update backup_content.control every 10 seconds */ - if ((difftime(time(NULL), prev_time)) > 10) + /* update backup_content.control every 60 seconds */ + if ((difftime(time(NULL), prev_time)) > 60) { - prev_time = time(NULL); - write_backup_filelist(¤t, arguments->files_list, arguments->from_root, arguments->external_dirs); /* update backup control file to update size info */ write_backup(¤t, true); + + prev_time = time(NULL); } } From e368eb738029036122ea9aba735148e0697871e7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 9 May 2020 01:06:34 +0300 Subject: [PATCH 1271/2107] [Issue #203] Refactoring of write_backup_filelist() --- src/backup.c | 12 +++-- src/catalog.c | 119 +++++++++++++++++---------------------------- src/merge.c | 2 +- src/pg_probackup.h | 2 +- 4 files changed, 56 insertions(+), 79 deletions(-) diff --git a/src/backup.c b/src/backup.c index b9bc9836e..510b1e44c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -375,6 +375,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (file->external_dir_num != 0) continue; + if (S_ISDIR(file->mode)) + { + current.pgdata_bytes += 4096; + continue; + } + current.pgdata_bytes += file->size; } @@ -501,7 +507,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* write initial backup_content.control file and update backup.control */ write_backup_filelist(¤t, backup_files_list, - instance_config.pgdata, external_dirs); + instance_config.pgdata, external_dirs, true); write_backup(¤t, true); /* init thread args with own file lists */ @@ -665,7 +671,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* Print the list of files to backup catalog */ write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, - external_dirs); + external_dirs, true); /* update backup control file to update size info */ write_backup(¤t, true); @@ -2076,7 +2082,7 @@ backup_files(void *arg) if ((difftime(time(NULL), prev_time)) > 60) { write_backup_filelist(¤t, arguments->files_list, arguments->from_root, - arguments->external_dirs); + arguments->external_dirs, false); /* update backup control file to update size info */ write_backup(¤t, true); diff --git a/src/catalog.c b/src/catalog.c index 05fe48143..6316b238d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1770,40 +1770,39 @@ write_backup(pgBackup *backup, bool strict) FILE *fp = NULL; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; - int errno_temp; char buf[4096]; join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - fp = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fopen(path_temp, PG_BINARY_W); if (fp == NULL) - elog(ERROR, "Cannot open configuration file \"%s\": %s", + elog(ERROR, "Cannot open control file \"%s\": %s", path_temp, strerror(errno)); + if (chmod(path_temp, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, + strerror(errno)); + setvbuf(fp, buf, _IOFBF, sizeof(buf)); pgBackupWriteControl(fp, backup); - if (fio_fclose(fp)) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); + if (fflush(fp) != 0) + elog(ERROR, "Cannot flush control file \"%s\": %s", + path_temp, strerror(errno)); - if (!strict && errno_temp == ENOSPC) - return; + if (fsync(fileno(fp)) < 0) + elog(ERROR, "Cannot sync control file \"%s\": %s", + path_temp, strerror(errno)); - elog(ERROR, "Cannot write configuration file \"%s\": %s", - path_temp, strerror(errno_temp)); - } + if (fclose(fp) != 0) + elog(ERROR, "Cannot close control file \"%s\": %s", + path_temp, strerror(errno)); - if (fio_rename(path_temp, path, FIO_BACKUP_HOST) < 0) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno_temp)); - } + if (rename(path_temp, path) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno)); } /* @@ -1811,16 +1810,14 @@ write_backup(pgBackup *backup, bool strict) */ void write_backup_filelist(pgBackup *backup, parray *files, const char *root, - parray *external_list) + parray *external_list, bool sync) { FILE *out; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; - int errno_temp; size_t i = 0; #define BUFFERSZ 1024*1024 char *buf; - size_t write_len = 0; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; int64 wal_size_on_disk = 0; @@ -1828,22 +1825,23 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - out = fio_fopen(path_temp, PG_BINARY_W, FIO_BACKUP_HOST); + out = fopen(path_temp, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, strerror(errno)); + if (chmod(path_temp, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, + strerror(errno)); + buf = pgut_malloc(BUFFERSZ); + setvbuf(out, buf, _IOFBF, BUFFERSZ); /* print each file in the list */ - while(i < parray_num(files)) + for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); - char *path = file->path; /* for streamed WAL files */ - char line[BLCKSZ]; - int len = 0; - - i++; + pgFile *file = (pgFile *) parray_get(files, i); + char *path = file->path; /* for streamed WAL files */ if (S_ISDIR(file->mode)) { @@ -1875,7 +1873,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, (file->external_dir_num && external_list)) path = file->rel_path; - len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " @@ -1889,59 +1887,32 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, file->dbOid); if (file->is_datafile) - len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); + fprintf(out, ",\"segno\":\"%d\"", file->segno); if (file->linked) - len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); + fprintf(out, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) - len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); - - len += sprintf(line+len, "}\n"); + fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); - if (write_len + len >= BUFFERSZ) - { - /* write buffer to file */ - if (fio_fwrite(out, buf, write_len) != write_len) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write file list \"%s\": %s", - path_temp, strerror(errno)); - } - /* reset write_len */ - write_len = 0; - } - - memcpy(buf+write_len, line, len); - write_len += len; + fprintf(out, "}\n"); } - /* write what is left in the buffer to file */ - if (write_len > 0) - if (fio_fwrite(out, buf, write_len) != write_len) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write file list \"%s\": %s", - path_temp, strerror(errno)); - } + if (fflush(out) != 0) + elog(ERROR, "Cannot flush file list \"%s\": %s", + path_temp, strerror(errno)); - if (fio_fflush(out) || fio_fclose(out)) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write file list \"%s\": %s", + if (sync && fsync(fileno(out)) < 0) + elog(ERROR, "Cannot sync file list \"%s\": %s", path_temp, strerror(errno)); - } - if (fio_rename(path_temp, path, FIO_BACKUP_HOST) < 0) - { - errno_temp = errno; - fio_unlink(path_temp, FIO_BACKUP_HOST); - elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno_temp)); - } + if (fclose(out) != 0) + elog(ERROR, "Cannot close file list \"%s\": %s", + path_temp, strerror(errno)); + + if (rename(path_temp, path) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + path_temp, path, strerror(errno)); /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; diff --git a/src/merge.c b/src/merge.c index 3057a3a5e..69c17b66c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -729,7 +729,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) parray_qsort(result_filelist, pgFileCompareRelPathWithExternal); - write_backup_filelist(full_backup, result_filelist, full_database_dir, NULL); + write_backup_filelist(full_backup, result_filelist, full_database_dir, NULL, true); write_backup(full_backup, true); /* Delete FULL backup files, that do not exists in destination backup diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6a4fc6dab..f93d76cb0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -803,7 +803,7 @@ extern void pin_backup(pgBackup *target_backup, extern void add_note(pgBackup *target_backup, char *note); extern void pgBackupWriteControl(FILE *out, pgBackup *backup); extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, parray *external_list); + const char *root, parray *external_list, bool sync); extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); From 9b0922c6b72469e131740798d357fa3a75f7e968 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 May 2020 15:06:07 +0300 Subject: [PATCH 1272/2107] [Issue #203] Add new pgBackup attribute 'content-crc', containing crc of backup_content.control file, to detect corruption --- src/backup.c | 6 +---- src/catalog.c | 60 ++++++++++++++++++++++++++++++---------------- src/data.c | 36 ++++++++++++++-------------- src/dir.c | 26 ++++++++++++++++---- src/merge.c | 6 +---- src/pg_probackup.h | 8 ++++--- src/restore.c | 11 +++------ src/show.c | 7 +++++- src/validate.c | 53 ++++++++++++++++++++++++---------------- 9 files changed, 127 insertions(+), 86 deletions(-) diff --git a/src/backup.c b/src/backup.c index 510b1e44c..96b47916d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -217,8 +217,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (prev_backup) { - char prev_backup_filelist_path[MAXPGPATH]; - if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " @@ -227,10 +225,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); - join_path_components(prev_backup_filelist_path, prev_backup->root_dir, - DATABASE_FILE_LIST); /* Files of previous backup needed by DELTA backup */ - prev_backup_filelist = dir_read_file_list(NULL, NULL, prev_backup_filelist_path, FIO_BACKUP_HOST); + prev_backup_filelist = get_backup_filelist(prev_backup, true); /* If lsn is not NULL, only pages with higher lsn will be copied. */ prev_backup_start_lsn = prev_backup->start_lsn; diff --git a/src/catalog.c b/src/catalog.c index 6316b238d..aedd01fbb 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -538,17 +538,17 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) * TODO this function only used once. Is it really needed? */ parray * -get_backup_filelist(pgBackup *backup) +get_backup_filelist(pgBackup *backup, bool strict) { parray *files = NULL; char backup_filelist_path[MAXPGPATH]; join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST); + files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc); /* redundant sanity? */ if (!files) - elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time)); + elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time)); return files; } @@ -1759,6 +1759,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) if (backup->note) fio_fprintf(out, "note = '%s'\n", backup->note); + if (backup->content_crc != 0) + fio_fprintf(out, "content-crc = %u\n", backup->content_crc); + } /* @@ -1813,8 +1816,8 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync) { FILE *out; - char path[MAXPGPATH]; - char path_temp[MAXPGPATH]; + char control_path[MAXPGPATH]; + char control_path_temp[MAXPGPATH]; size_t i = 0; #define BUFFERSZ 1024*1024 char *buf; @@ -1822,24 +1825,29 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int64 uncompressed_size_on_disk = 0; int64 wal_size_on_disk = 0; - join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); + join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST); + snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path); - out = fopen(path_temp, PG_BINARY_W); + out = fopen(control_path_temp, PG_BINARY_W); if (out == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", path_temp, + elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp, strerror(errno)); - if (chmod(path_temp, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, + if (chmod(control_path_temp, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp, strerror(errno)); buf = pgut_malloc(BUFFERSZ); setvbuf(out, buf, _IOFBF, BUFFERSZ); + if (sync) + INIT_FILE_CRC32(true, backup->content_crc); + /* print each file in the list */ for (i = 0; i < parray_num(files); i++) { + int len = 0; + char line[BLCKSZ]; pgFile *file = (pgFile *) parray_get(files, i); char *path = file->path; /* for streamed WAL files */ @@ -1873,7 +1881,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, (file->external_dir_num && external_list)) path = file->rel_path; - fprintf(out, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " + len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " @@ -1887,32 +1895,40 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, file->dbOid); if (file->is_datafile) - fprintf(out, ",\"segno\":\"%d\"", file->segno); + len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); if (file->linked) - fprintf(out, ",\"linked\":\"%s\"", file->linked); + len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks != BLOCKNUM_INVALID) - fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks); + len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); + + sprintf(line+len, "}\n"); - fprintf(out, "}\n"); + if (sync) + COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line)); + + fprintf(out, "%s", line); } + if (sync) + FIN_FILE_CRC32(true, backup->content_crc); + if (fflush(out) != 0) elog(ERROR, "Cannot flush file list \"%s\": %s", - path_temp, strerror(errno)); + control_path_temp, strerror(errno)); if (sync && fsync(fileno(out)) < 0) elog(ERROR, "Cannot sync file list \"%s\": %s", - path_temp, strerror(errno)); + control_path_temp, strerror(errno)); if (fclose(out) != 0) elog(ERROR, "Cannot close file list \"%s\": %s", - path_temp, strerror(errno)); + control_path_temp, strerror(errno)); - if (rename(path_temp, path) < 0) + if (rename(control_path_temp, control_path) < 0) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno)); + control_path_temp, control_path, strerror(errno)); /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; @@ -1975,6 +1991,7 @@ readBackupControlFile(const char *path) {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, {'s', 0, "note", &backup->note, SOURCE_FILE_STRICT}, + {'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT}, {0} }; @@ -2242,6 +2259,7 @@ pgBackupInit(pgBackup *backup) backup->root_dir = NULL; backup->files = NULL; backup->note = NULL; + backup->content_crc = 0; } diff --git a/src/data.c b/src/data.c index 7628c37e5..1270de4f7 100644 --- a/src/data.c +++ b/src/data.c @@ -1462,7 +1462,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, */ if (errno == ENOENT) { - elog(LOG, "File \"%s\" is not found", file->path); + elog(LOG, "File \"%s\" is not found", from_fullpath); return true; } @@ -1472,7 +1472,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, } if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", file->path, file->size); + elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); /* * Compute expected number of blocks in the file. @@ -1508,8 +1508,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, - uint32 backup_version) +check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version) { size_t read_len = 0; bool is_valid = true; @@ -1517,19 +1517,19 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, pg_crc32 crc; bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - elog(VERBOSE, "Validate relation blocks for file \"%s\"", file->path); + elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); - in = fopen(file->path, PG_BINARY_R); + in = fopen(fullpath, PG_BINARY_R); if (in == NULL) { if (errno == ENOENT) { - elog(WARNING, "File \"%s\" is not found", file->path); + elog(WARNING, "File \"%s\" is not found", fullpath); return false; } elog(ERROR, "Cannot open file \"%s\": %s", - file->path, strerror(errno)); + fullpath, strerror(errno)); } /* calc CRC of backup file */ @@ -1553,7 +1553,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (ferror(in)) elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, fullpath, strerror(errno)); if (read_len != sizeof(header)) { @@ -1562,10 +1562,10 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, else if (read_len != 0 && feof(in)) elog(WARNING, "Odd size page found at block %u of \"%s\"", - blknum, file->path); + blknum, fullpath); else elog(WARNING, "Cannot read header of block %u of \"%s\": %s", - blknum, file->path, strerror(errno)); + blknum, fullpath, strerror(errno)); return false; } @@ -1573,14 +1573,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (header.block == 0 && header.compressed_size == 0) { - elog(VERBOSE, "Skip empty block of \"%s\"", file->path); + elog(VERBOSE, "Skip empty block of \"%s\"", fullpath); continue; } if (header.block < blknum) { elog(WARNING, "Backup is broken at block %u of \"%s\"", - blknum, file->path); + blknum, fullpath); return false; } @@ -1589,7 +1589,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (header.compressed_size == PageIsTruncated) { elog(LOG, "Block %u of \"%s\" is truncated", - blknum, file->path); + blknum, fullpath); continue; } @@ -1600,7 +1600,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (read_len != MAXALIGN(header.compressed_size)) { elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, file->path, read_len, header.compressed_size); + blknum, fullpath, read_len, header.compressed_size); return false; } @@ -1620,7 +1620,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, &errormsg); if (uncompressed_size < 0 && errormsg != NULL) elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, file->path, errormsg); + blknum, fullpath, errormsg); if (uncompressed_size != BLCKSZ) { @@ -1630,7 +1630,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, continue; } elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - file->path, uncompressed_size); + fullpath, uncompressed_size); return false; } @@ -1676,7 +1676,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version, if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", - file->path, crc, file->crc); + fullpath, crc, file->crc); is_valid = false; } diff --git a/src/dir.c b/src/dir.c index 926a57b53..33b140721 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1512,12 +1512,13 @@ get_control_value(const char *str, const char *name, */ parray * dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location) + const char *file_txt, fio_location location, pg_crc32 expected_crc) { - FILE *fp; - parray *files; - char buf[MAXPGPATH * 2]; - char stdio_buf[STDIO_BUFSIZE]; + FILE *fp; + parray *files; + char buf[BLCKSZ]; + char stdio_buf[STDIO_BUFSIZE]; + pg_crc32 content_crc = 0; fp = fio_open_stream(file_txt, location); if (fp == NULL) @@ -1529,6 +1530,8 @@ dir_read_file_list(const char *root, const char *external_prefix, files = parray_new(); + INIT_FILE_CRC32(true, content_crc); + while (fgets(buf, lengthof(buf), fp)) { char path[MAXPGPATH]; @@ -1546,6 +1549,8 @@ dir_read_file_list(const char *root, const char *external_prefix, dbOid; /* used for partial restore */ pgFile *file; + COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + get_control_value(buf, "path", path, NULL, true); get_control_value(buf, "size", NULL, &write_size, true); get_control_value(buf, "mode", NULL, &mode, true); @@ -1598,10 +1603,21 @@ dir_read_file_list(const char *root, const char *external_prefix, parray_append(files, file); } + FIN_FILE_CRC32(true, content_crc); + if (ferror(fp)) elog(ERROR, "Failed to read from file: \"%s\"", file_txt); fio_close_stream(fp); + + if (expected_crc != 0 && + expected_crc != content_crc) + { + elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", + file_txt, content_crc, expected_crc); + return NULL; + } + return files; } diff --git a/src/merge.c b/src/merge.c index 69c17b66c..dd30ef5d9 100644 --- a/src/merge.c +++ b/src/merge.c @@ -565,13 +565,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) */ for (i = parray_num(parent_chain) - 1; i >= 0; i--) { - char control_file[MAXPGPATH]; - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); - backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - + backup->files = get_backup_filelist(backup, true); parray_qsort(backup->files, pgFileCompareRelPathWithExternal); /* Set MERGING status for every member of the chain */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f93d76cb0..2608419ce 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -394,6 +394,8 @@ struct pgBackup parray *files; /* list of files belonging to this backup * must be populated explicitly */ char *note; + + pg_crc32 content_crc; }; /* Recovery target for restore and validate subcommands */ @@ -708,7 +710,7 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); -extern parray *get_backup_filelist(pgBackup *backup); +extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI); /* in merge.c */ @@ -867,7 +869,7 @@ extern void db_map_entry_free(void *map); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *external_prefix, parray *external_list); extern parray *dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location); + const char *file_txt, fio_location location, pg_crc32 expected_crc); extern parray *make_external_directory_list(const char *colon_separated_dirs, bool remap); extern void free_dir_list(parray *list); @@ -933,7 +935,7 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, extern bool create_empty_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); -extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn, +extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, diff --git a/src/restore.c b/src/restore.c index 9e770281a..749f35572 100644 --- a/src/restore.c +++ b/src/restore.c @@ -494,7 +494,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, const char *pgdata_path, bool no_sync) { int i; - char control_file[MAXPGPATH]; char timestamp[100]; parray *dest_files = NULL; parray *external_dirs = NULL; @@ -515,8 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); elog(INFO, "Restoring the database from backup at %s", timestamp); - join_path_components(control_file, dest_backup->root_dir, DATABASE_FILE_LIST); - dest_files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); + dest_files = get_backup_filelist(dest_backup, true); /* Lock backup chain and make sanity checks */ for (i = parray_num(parent_chain) - 1; i >= 0; i--) @@ -550,10 +548,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, /* populate backup filelist */ if (backup->start_time != dest_backup->start_time) - { - join_path_components(control_file, backup->root_dir, DATABASE_FILE_LIST); - backup->files = dir_read_file_list(NULL, NULL, control_file, FIO_BACKUP_HOST); - } + backup->files = get_backup_filelist(backup, true); else backup->files = dest_files; @@ -1496,7 +1491,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, char database_map_path[MAXPGPATH]; parray *files = NULL; - files = get_backup_filelist(backup); + files = get_backup_filelist(backup, true); /* look for 'database_map' file in backup_content.control */ for (i = 0; i < parray_num(files); i++) diff --git a/src/show.c b/src/show.c index edd020a60..81a16ad64 100644 --- a/src/show.c +++ b/src/show.c @@ -430,11 +430,16 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add_value(buf, "status", status2str(backup->status), json_level, true); - if (backup->note){ + if (backup->note) json_add_value(buf, "note", backup->note, json_level, true); + if (backup->content_crc != 0) + { + json_add_key(buf, "content-crc", json_level); + appendPQExpBuffer(buf, "%u", backup->content_crc); } + json_add(buf, JT_END_OBJECT, &json_level); } diff --git a/src/validate.c b/src/validate.c index 36f71ffae..75c6e33e8 100644 --- a/src/validate.c +++ b/src/validate.c @@ -31,6 +31,7 @@ typedef struct uint32 backup_version; BackupMode backup_mode; parray *dbOid_exclude_list; + const char *external_prefix; /* * Return value from the thread. @@ -48,8 +49,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { char base_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; - char path[MAXPGPATH]; - parray *files; + parray *files = NULL; bool corrupted = false; bool validation_isok = true; /* arrays with meta info for multi threaded validate */ @@ -110,8 +110,15 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) join_path_components(base_path, backup->root_dir, DATABASE_DIR); join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); - join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST); + files = get_backup_filelist(backup, false); + + if (!files) + { + elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time)); + backup->status = BACKUP_STATUS_CORRUPT; + write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + return; + } // if (params && params->partial_db_list) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, @@ -142,6 +149,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) arg->stop_lsn = backup->stop_lsn; arg->checksum_version = backup->checksum_version; arg->backup_version = parse_program_version(backup->program_version); + arg->external_prefix = external_prefix; // arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -223,6 +231,7 @@ pgBackupValidateFiles(void *arg) { struct stat st; pgFile *file = (pgFile *) parray_get(arguments->files, i); + char file_fullpath[MAXPGPATH]; if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during validate"); @@ -244,14 +253,6 @@ pgBackupValidateFiles(void *arg) // continue; //} - /* - * Currently we don't compute checksums for - * cfs_compressed data files, so skip them. - * TODO: investigate - */ - if (file->is_cfs) - continue; - if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -282,14 +283,24 @@ pgBackupValidateFiles(void *arg) if (file->write_size == 0) continue; + if (file->external_dir_num) + { + char temp[MAXPGPATH]; + + makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num); + join_path_components(file_fullpath, temp, file->rel_path); + } + else + join_path_components(file_fullpath, arguments->base_path, file->rel_path); + /* TODO: it is redundant to check file existence using stat */ - if (stat(file->path, &st) == -1) + if (stat(file_fullpath, &st) == -1) { if (errno == ENOENT) - elog(WARNING, "Backup file \"%s\" is not found", file->path); + elog(WARNING, "Backup file \"%s\" is not found", file_fullpath); else elog(WARNING, "Cannot stat backup file \"%s\": %s", - file->path, strerror(errno)); + file_fullpath, strerror(errno)); arguments->corrupted = true; break; } @@ -297,7 +308,7 @@ pgBackupValidateFiles(void *arg) if (file->write_size != st.st_size) { elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file->path, (unsigned long) st.st_size, file->write_size); + file_fullpath, (unsigned long) st.st_size, file->write_size); arguments->corrupted = true; break; } @@ -305,8 +316,10 @@ pgBackupValidateFiles(void *arg) /* * If option skip-block-validation is set, compute only file-level CRC for * datafiles, otherwise check them block by block. + * Currently we don't compute checksums for + * cfs_compressed data files, so skip block validation for them. */ - if (!file->is_datafile || skip_block_validation) + if (!file->is_datafile || skip_block_validation || file->is_cfs) { /* * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we @@ -326,14 +339,14 @@ pgBackupValidateFiles(void *arg) !file->external_dir_num) crc = get_pgcontrol_checksum(arguments->base_path); else - crc = pgFileGetCRC(file->path, + crc = pgFileGetCRC(file_fullpath, arguments->backup_version <= 20021 || arguments->backup_version >= 20025, false); if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file->path, crc, file->crc); + file_fullpath, crc, file->crc); arguments->corrupted = true; } } @@ -344,7 +357,7 @@ pgBackupValidateFiles(void *arg) * check page headers, checksums (if enabled) * and compute checksum of the file */ - if (!check_file_pages(file, arguments->stop_lsn, + if (!check_file_pages(file, file_fullpath, arguments->stop_lsn, arguments->checksum_version, arguments->backup_version)) arguments->corrupted = true; From 73efc0bec8bce94d8a1934b74b675266cfdd861a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 May 2020 15:25:29 +0300 Subject: [PATCH 1273/2107] [Issue #203] test coverage --- tests/archive.py | 4 ++++ tests/backup.py | 4 +++- tests/config.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/merge.py | 8 +++---- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index ec14254f7..5508f54b8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -977,6 +977,10 @@ def test_basic_master_and_replica_concurrent_archiving(self): replica.promote() + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + master.pgbench_init(scale=10) replica.pgbench_init(scale=10) diff --git a/tests/backup.py b/tests/backup.py index f77f43f61..74149da93 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2511,7 +2511,7 @@ def test_backup_with_less_privileges_role(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_issue_132(self): """ https://github.com/postgrespro/pg_probackup/issues/132 @@ -2546,6 +2546,8 @@ def test_issue_132(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + exit(1) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/config.py b/tests/config.py index 4a382e13a..505b88a82 100644 --- a/tests/config.py +++ b/tests/config.py @@ -3,6 +3,7 @@ import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from sys import exit +from shutil import copyfile module_name = 'config' @@ -51,3 +52,61 @@ def test_remove_instance_config(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_corrupt_backup_content(self): + """corrupt backup_content.control""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full1_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'create table t1()') + + fulle2_id = self.backup_node(backup_dir, 'node', node) + + fulle1_conf_file = os.path.join( + backup_dir, 'backups','node', full1_id, 'backup_content.control') + + fulle2_conf_file = os.path.join( + backup_dir, 'backups','node', fulle2_id, 'backup_content.control') + + copyfile(fulle2_conf_file, fulle1_conf_file) + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because pg_probackup.conf is missing. " + ".\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Invalid CRC of backup control file '{0}':".format(fulle1_conf_file), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Failed to get file list for backup {0}".format(full1_id), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Backup {0} file list is corrupted".format(full1_id), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) diff --git a/tests/merge.py b/tests/merge.py index 61bb27c1d..9d567547b 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1530,8 +1530,8 @@ def test_crash_after_opening_backup_control_2(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('fio_fwrite') - gdb.continue_execution_until_break(2) + gdb.set_breakpoint('sprintf') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1621,8 +1621,8 @@ def test_losing_file_after_failed_merge(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('fio_fwrite') - gdb.continue_execution_until_break(2) + gdb.set_breakpoint('sprintf') + gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') From e373754988b7118bfc9f8e5cba4bef9064604b19 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 May 2020 17:33:45 +0300 Subject: [PATCH 1274/2107] [Issue #203] documentation update --- doc/pgprobackup.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 49ab51964..34e8f37da 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2407,6 +2407,12 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod note — text note attached to backup. + + + content-crc — CRC32 checksum of backup_content.control file. + It is used to detect corruption of backup metainformation. + + From 7d0add5ad65fd453352205b5c4c73d597743c4f5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 May 2020 21:23:25 +0300 Subject: [PATCH 1275/2107] tests: fixes --- tests/config.py | 5 +++++ tests/merge.py | 7 +++---- tests/restore.py | 7 +++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/config.py b/tests/config.py index 505b88a82..b41382204 100644 --- a/tests/config.py +++ b/tests/config.py @@ -110,3 +110,8 @@ def test_corrupt_backup_content(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) + + self.show_pb(backup_dir, 'node', full1_id)['status'] + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], 'CORRUPT') + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], 'OK') diff --git a/tests/merge.py b/tests/merge.py index 9d567547b..e40ed05c8 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1991,7 +1991,7 @@ def test_merge_backup_from_future(self): # Take FULL self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=3) + node.pgbench_init(scale=5) # Take PAGE from future backup_id = self.backup_node( @@ -2011,11 +2011,10 @@ def test_merge_backup_from_future(self): os.path.join(backup_dir, 'backups', 'node', backup_id), os.path.join(backup_dir, 'backups', 'node', new_id)) - pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) result = node.safe_psql( diff --git a/tests/restore.py b/tests/restore.py index e7a3c1a76..6ab3da335 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1747,7 +1747,7 @@ def test_restore_backup_from_future(self): # Take FULL self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=3) + node.pgbench_init(scale=5) # pgbench = node.pgbench(options=['-T', '20', '-c', '2']) # pgbench.wait() @@ -1769,11 +1769,10 @@ def test_restore_backup_from_future(self): os.path.join(backup_dir, 'backups', 'node', backup_id), os.path.join(backup_dir, 'backups', 'node', new_id)) - pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.cleanup() From 77e99179afb698b14c3f90ccbb6cb8053f509051 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 11 May 2020 21:24:51 +0300 Subject: [PATCH 1276/2107] Version 2.3.2 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2608419ce..83b2cbe2a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.1" -#define AGENT_PROTOCOL_VERSION 20301 +#define PROGRAM_VERSION "2.3.2" +#define AGENT_PROTOCOL_VERSION 20302 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index ba90ec994..716674977 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.1 \ No newline at end of file +pg_probackup 2.3.2 \ No newline at end of file From f39ef5fa7fd933f7692fd5077d403e72d80d7120 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 May 2020 17:26:54 +0300 Subject: [PATCH 1277/2107] Readme: update for 2.3.2 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8829e236e..8cada0944 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.1) +[2.3.2](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.2) ## Documentation @@ -60,7 +60,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/latest). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.1). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 497f5b2e9e8528894f6563603eb50a7e90e0e4db Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 May 2020 19:53:44 +0300 Subject: [PATCH 1278/2107] [Issue #204] Fix agent version check --- src/pg_probackup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0f81b772f..b065a4854 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -349,7 +349,7 @@ main(int argc, char *argv[]) if (strcmp(remote_agent, PROGRAM_VERSION) != 0) { uint32 agent_version = parse_program_version(remote_agent); - elog(agent_version < AGENT_PROTOCOL_VERSION ? ERROR : WARNING, + elog(agent_version != AGENT_PROTOCOL_VERSION ? ERROR : WARNING, "Agent version %s doesn't match master pg_probackup version %s", PROGRAM_VERSION, remote_agent); } @@ -528,7 +528,8 @@ main(int argc, char *argv[]) { /* Ensure that backup_path is a path to a directory */ if (!S_ISDIR(st.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); + elog(ERROR, "-B, --backup-path must be a path to directory. Inode: %lu. ST_MODE: %u", + st.st_ino, st.st_mode); } } } From 2c711bd57bf79c46e9c8c438804984928a26a103 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 12 May 2020 22:14:29 +0300 Subject: [PATCH 1279/2107] [Issue #204] Make version check of remote agent more robust --- src/fetch.c | 2 +- src/pg_probackup.c | 13 ++--------- src/utils/file.c | 57 ++++++++++++---------------------------------- src/utils/file.h | 4 +++- src/utils/remote.c | 38 +++++++++++++++++++++++-------- 5 files changed, 49 insertions(+), 65 deletions(-) diff --git a/src/fetch.c b/src/fetch.c index 6a9c2db51..bef30dac6 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -44,7 +44,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi fullpath, strerror(errno)); } - if (fio_fstat(fd, &statbuf) < 0) + if (fio_stat(fullpath, &statbuf, true, location) < 0) { if (safe) return NULL; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b065a4854..66208d53a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -343,16 +343,8 @@ main(int argc, char *argv[]) else if (strcmp(argv[1], "ssh") == 0) launch_ssh(argv); #endif - else if (strcmp(argv[1], "agent") == 0 && argc > 2) + else if (strcmp(argv[1], "agent") == 0) { - remote_agent = argv[2]; - if (strcmp(remote_agent, PROGRAM_VERSION) != 0) - { - uint32 agent_version = parse_program_version(remote_agent); - elog(agent_version != AGENT_PROTOCOL_VERSION ? ERROR : WARNING, - "Agent version %s doesn't match master pg_probackup version %s", - PROGRAM_VERSION, remote_agent); - } fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; } @@ -528,8 +520,7 @@ main(int argc, char *argv[]) { /* Ensure that backup_path is a path to a directory */ if (!S_ISDIR(st.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory. Inode: %lu. ST_MODE: %u", - st.st_ino, st.st_mode); + elog(ERROR, "-B, --backup-path must be a path to directory"); } } } diff --git a/src/utils/file.c b/src/utils/file.c index bb05f3793..3242c5d3c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -185,6 +185,19 @@ static ssize_t fio_write_all(int fd, void const* buf, size_t size) return offs; } +/* Get version of remote agent */ +int fio_get_agent_version(void) +{ + fio_header hdr; + hdr.cop = FIO_AGENT_VERSION; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + return hdr.arg; +} + /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ FILE* fio_open_stream(char const* path, fio_location location) { @@ -728,44 +741,6 @@ ssize_t fio_read(int fd, void* buf, size_t size) } } -/* Get information about stdio file */ -int fio_ffstat(FILE* f, struct stat* st) -{ - return fio_is_remote_file(f) - ? fio_fstat(fio_fileno(f), st) - : fio_fstat(fileno(f), st); -} - -/* Get information about file descriptor */ -int fio_fstat(int fd, struct stat* st) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_FSTAT; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_FSTAT); - IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - return 0; - } - else - { - return fstat(fd, st); - } -} - /* Get information about file */ int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) { @@ -2065,11 +2040,9 @@ void fio_communicate(int in, int out) if (hdr.size != 0) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; - case FIO_FSTAT: /* Get information about opened file */ - hdr.size = sizeof(st); - hdr.arg = fstat(fd[hdr.handle], &st) < 0 ? errno : 0; + case FIO_AGENT_VERSION: + hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); diff --git a/src/utils/file.h b/src/utils/file.h index 6dcdfcb1e..6aea02122 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -27,7 +27,6 @@ typedef enum FIO_READ, FIO_LOAD, FIO_STAT, - FIO_FSTAT, FIO_SEND, FIO_ACCESS, FIO_OPENDIR, @@ -47,6 +46,8 @@ typedef enum /* messages for closing connection */ FIO_DISCONNECT, FIO_DISCONNECTED, + /* message for compatibility check */ + FIO_AGENT_VERSION } fio_operations; typedef enum @@ -79,6 +80,7 @@ extern fio_location MyLocation; extern void fio_redirect(int in, int out, int err); extern void fio_communicate(int in, int out); +extern int fio_get_agent_version(void); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg); diff --git a/src/utils/remote.c b/src/utils/remote.c index 7f8068d6b..f590a82b4 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -111,6 +111,7 @@ bool launch_agent(void) int outfd[2]; int infd[2]; int errfd[2]; + int agent_version; ssh_argc = 0; #ifdef WIN32 @@ -163,24 +164,24 @@ bool launch_agent(void) } } if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent", + instance_config.remote.path, probackup); else - snprintf(cmd, sizeof(cmd), "%s\\%s agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "%s\\%s agent", + instance_config.remote.path, probackup); #else if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent", + instance_config.remote.path, probackup); else - snprintf(cmd, sizeof(cmd), "%s/%s agent %s", - instance_config.remote.path, probackup, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "%s/%s agent", + instance_config.remote.path, probackup); #endif } else { if (needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s\" agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "\"%s\" agent", PROGRAM_NAME_FULL); else - snprintf(cmd, sizeof(cmd), "%s agent %s", PROGRAM_NAME_FULL, PROGRAM_VERSION); + snprintf(cmd, sizeof(cmd), "%s agent", PROGRAM_NAME_FULL); } #ifdef WIN32 @@ -228,5 +229,22 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } + + /* Make sure that remote agent has the same version + * TODO: we must also check PG version and fork edition + */ + agent_version = fio_get_agent_version(); + if (agent_version != AGENT_PROTOCOL_VERSION) + { + char agent_version_str[1024]; + sprintf(agent_version_str, "%d.%d.%d", + agent_version / 10000, + (agent_version / 100) % 100, + agent_version % 100); + + elog(ERROR, "Remote agent version %s does not match local program version %s", + agent_version_str, PROGRAM_VERSION); + } + return true; } From 6a1931faf2fc9537f67dc344cd1db5dc34c68058 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 May 2020 02:15:26 +0300 Subject: [PATCH 1280/2107] Readne: update "Limitations" section --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8cada0944..bd0e27cba 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `pg_probackup` currently has the following limitations: * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. * Remote backup via ssh on Windows currently is not supported. +* When running remote operations via ssh, remote and local pg_probackup versions must be the same. ## Current release From cddc778024d24648f365c7feb1ff9777b7d69f56 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 May 2020 17:10:47 +0300 Subject: [PATCH 1281/2107] [Issue #203] test coverage --- tests/backup.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 74149da93..8b44aedd8 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2803,3 +2803,44 @@ def test_parent_backup_made_by_newer_version(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_issue_203(self): + """ + https://github.com/postgrespro/pg_probackup/issues/203 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + for i in range(1200000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() + + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', + node_restored, data_dir=node_restored.data_dir) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 04d1ec93fc40c29b62aa63966f0fa0a550c33ffa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 13 May 2020 23:51:01 +0300 Subject: [PATCH 1282/2107] tests: added tests.backup.BackupTest.test_drop_rel_during_full_backup --- tests/backup.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 8b44aedd8..86c9e2fe8 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1273,6 +1273,89 @@ def test_tablespace_handling_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_drop_rel_during_full_backup(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + for i in range(1, 512): + node.safe_psql( + "postgres", + "create table t_heap_{0} as select i" + " as id from generate_series(0,100) i".format(i)) + + node.safe_psql( + "postgres", + "VACUUM") + + node.pgbench_init(scale=10) + + relative_path_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + + relative_path_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").rstrip() + + absolute_path_1 = os.path.join(node.data_dir, relative_path_1) + absolute_path_2 = os.path.join(node.data_dir, relative_path_2) + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=LOG', '--log-level-console=LOG', '--progress'], + gdb=True) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + for i in range(1, 512): + node.safe_psql( + "postgres", + "drop table t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "CHECKPOINT") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + # log_content = f.read() + # self.assertTrue( + # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" @@ -1288,6 +1371,8 @@ def test_drop_rel_during_backup_delta(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + node.pgbench_init(scale=10) + node.safe_psql( "postgres", "create table t_heap as select i" From 9570db096e8fba8adf7b49ec8ee56bf40bf52af6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 02:05:06 +0300 Subject: [PATCH 1283/2107] [Issue #203] Fix --- src/backup.c | 6 +++--- src/utils/file.c | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 96b47916d..dc4b578bd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -352,9 +352,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) dir_list_file(backup_files_list, parray_get(external_dirs, i), false, true, false, i+1, FIO_DB_HOST); - /* close ssh session in main thread */ - fio_disconnect(); - /* Sanity check for backup_files_list, thank you, Windows: * https://github.com/postgrespro/pg_probackup/issues/48 */ @@ -529,6 +526,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) arg->ret = 1; } + /* close ssh session in main thread */ + fio_disconnect(); + /* Run threads */ thread_interrupted = false; elog(INFO, "Start transferring data files"); diff --git a/src/utils/file.c b/src/utils/file.c index 3242c5d3c..15e758911 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -368,6 +368,7 @@ int fio_open(char const* path, int mode, fio_location location) if (hdr.arg != 0) { errno = hdr.arg; + fio_fdset &= ~(1 << hdr.handle); return -1; } fd = i | FIO_PIPE_MARKER; From 200e47b3946c241f6328cad7496f15d3664eb5a9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 12:41:11 +0300 Subject: [PATCH 1284/2107] [Issue #203] error out if FIO_FDMAX is exceeded in fio_open --- src/backup.c | 6 +++--- src/utils/file.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index dc4b578bd..96b47916d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -352,6 +352,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) dir_list_file(backup_files_list, parray_get(external_dirs, i), false, true, false, i+1, FIO_DB_HOST); + /* close ssh session in main thread */ + fio_disconnect(); + /* Sanity check for backup_files_list, thank you, Windows: * https://github.com/postgrespro/pg_probackup/issues/48 */ @@ -526,9 +529,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) arg->ret = 1; } - /* close ssh session in main thread */ - fio_disconnect(); - /* Run threads */ thread_interrupted = false; elog(INFO, "Start transferring data files"); diff --git a/src/utils/file.c b/src/utils/file.c index 15e758911..38ea60b43 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -348,9 +348,9 @@ int fio_open(char const* path, int mode, fio_location location) mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); - if (i == FIO_FDMAX) { - return -1; - } + if (i == FIO_FDMAX) + elog(ERROR, "FIO_FDMAX is exceeded, probably too many remote files has been opened"); + hdr.cop = FIO_OPEN; hdr.handle = i; hdr.size = strlen(path) + 1; From da98b27453ee5daded0922ad3d4a573bd09cca76 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 12:46:34 +0300 Subject: [PATCH 1285/2107] [Issue #203] error out if FIO_FDMAX is exceeded in fio_opendir --- src/utils/file.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 38ea60b43..e4b979916 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -262,7 +262,8 @@ DIR* fio_opendir(char const* path, fio_location location) mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) { - return NULL; + elog(ERROR, "FIO_FDMAX is exceeded in fio_opendir, " + "probably too many remote directories has been opened"); } hdr.cop = FIO_OPENDIR; hdr.handle = i; @@ -277,6 +278,7 @@ DIR* fio_opendir(char const* path, fio_location location) if (hdr.arg != 0) { errno = hdr.arg; + fio_fdset &= ~(1 << hdr.handle); return NULL; } dir = (DIR*)(size_t)(i + 1); @@ -349,7 +351,8 @@ int fio_open(char const* path, int mode, fio_location location) mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) - elog(ERROR, "FIO_FDMAX is exceeded, probably too many remote files has been opened"); + elog(ERROR, "FIO_FDMAX is exceeded in fio_open, " + "probably too many remote files has been opened"); hdr.cop = FIO_OPEN; hdr.handle = i; From 7bc8f490812e53e35a9c5368a0b9975abb87252b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 12:54:17 +0300 Subject: [PATCH 1286/2107] tests: added tests.backup.BackupTest.test_drop_db_during_full_backup --- tests/backup.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 86c9e2fe8..9d07bd1a7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1285,10 +1285,8 @@ def test_drop_rel_during_full_backup(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) node.slow_start() - for i in range(1, 512): node.safe_psql( "postgres", @@ -1356,6 +1354,74 @@ def test_drop_rel_during_full_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_drop_db_during_full_backup(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 100): + node.safe_psql( + "postgres", + "create database t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "VACUUM") + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=[ + '--stream', '--log-level-file=LOG', + '--log-level-console=LOG', '--progress']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + for i in range(1, 100): + node.safe_psql( + "postgres", + "drop database t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "CHECKPOINT") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + # log_content = f.read() + # self.assertTrue( + # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" From b0e90be2c2dce2312872a7fd88fe0a13a8895e65 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 13:27:17 +0300 Subject: [PATCH 1287/2107] tests: minor fixes --- tests/backup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 9d07bd1a7..176aba619 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1354,7 +1354,7 @@ def test_drop_rel_during_full_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_drop_db_during_full_backup(self): """""" fname = self.id().split('.')[3] @@ -1368,7 +1368,7 @@ def test_drop_db_during_full_backup(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - for i in range(1, 100): + for i in range(1, 2): node.safe_psql( "postgres", "create database t_heap_{0}".format(i)) @@ -1388,7 +1388,7 @@ def test_drop_db_during_full_backup(self): gdb.run_until_break() # REMOVE file - for i in range(1, 100): + for i in range(1, 2): node.safe_psql( "postgres", "drop database t_heap_{0}".format(i)) From 612e928d1bbb6d10aeb3d3b6ad1a5b676c85081b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 15:33:24 +0300 Subject: [PATCH 1288/2107] [Issue #203] change error message wording --- src/utils/file.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index e4b979916..e19754080 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -262,8 +262,8 @@ DIR* fio_opendir(char const* path, fio_location location) mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) { - elog(ERROR, "FIO_FDMAX is exceeded in fio_opendir, " - "probably too many remote directories has been opened"); + elog(ERROR, "Descriptor pool for remote files is exhausted, " + "probably too many remote directories are opened"); } hdr.cop = FIO_OPENDIR; hdr.handle = i; @@ -351,8 +351,8 @@ int fio_open(char const* path, int mode, fio_location location) mask = fio_fdset; for (i = 0; (mask & 1) != 0; i++, mask >>= 1); if (i == FIO_FDMAX) - elog(ERROR, "FIO_FDMAX is exceeded in fio_open, " - "probably too many remote files has been opened"); + elog(ERROR, "Descriptor pool for remote files is exhausted, " + "probably too many remote files are opened"); hdr.cop = FIO_OPEN; hdr.handle = i; From 884dd528a5e6d03ff84d98bf2ce9a0062b98d373 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 17:43:39 +0300 Subject: [PATCH 1289/2107] [Issue #204] Enforce "no-forward-compatibility" policy on remote agent --- src/pg_probackup.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 66208d53a..3235f1562 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -345,6 +345,16 @@ main(int argc, char *argv[]) #endif else if (strcmp(argv[1], "agent") == 0) { + /* 'No forward compatibility' sanity: + * /old/binary -> ssh execute -> /newer/binary agent version_num + * If we are executed as an agent for older binary, then exit with error + */ + if (argc > 2) + { + elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " + "is launched as an agent for pg_probackup binary with version '%s'", + PROGRAM_VERSION, argv[2]); + } fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; } From 6a736c2db6402d776fe0fb9e038adc6c2bb0fe37 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 14 May 2020 18:00:29 +0300 Subject: [PATCH 1290/2107] Version 2.3.3 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 83b2cbe2a..93af776c9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.2" -#define AGENT_PROTOCOL_VERSION 20302 +#define PROGRAM_VERSION "2.3.3" +#define AGENT_PROTOCOL_VERSION 20303 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 716674977..ce2332584 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.2 \ No newline at end of file +pg_probackup 2.3.3 \ No newline at end of file From 74e92823b06278a0d20deb3f5f8364dc7b71fcef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 May 2020 18:14:50 +0300 Subject: [PATCH 1291/2107] Readme: update version to 2.3.3 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bd0e27cba..e908f273d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.2](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.2) +[2.3.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.3) ## Documentation @@ -61,7 +61,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.1). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.3). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 7d8dd3652e5306ef36cccec5425a2cdbe1fdf4ef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 15 May 2020 20:36:26 +0300 Subject: [PATCH 1292/2107] [Issue #206] Improve 'Setting up STREAM Backups' documentation section by, add note about '.pgpass' file --- doc/pgprobackup.xml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 34e8f37da..58d427494 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -740,6 +740,16 @@ ALTER ROLE backup WITH REPLICATION; DELTA, and PTRACK backups in the STREAM WAL mode. + + + If you are planning to rely on + .pgpass + for authentication when running backup in STREAM mode, + then .pgpass must contain credentials for replication database, + used to establish connection via replication protocol. Example: + pghost:5432:replication:backup_user:my_strong_password + + Setting up Continuous WAL Archiving @@ -4714,7 +4724,7 @@ pg_probackup archive-get -B backup_dir --instance - Forces a password prompt. + Forces a password prompt. (Deprecated) From 6ed7350af4d76c4bc414093b34bf7f48a8d1d4dc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 May 2020 01:29:04 +0300 Subject: [PATCH 1293/2107] [Issue #153] Optimize memory consumption: get rid of pgFile.path and pgFile.forkname, use pgFile.rel_path substring for pgFile.name --- src/archive.c | 2 +- src/backup.c | 97 +++++++++++-------------- src/catalog.c | 21 +++--- src/checkdb.c | 2 +- src/delete.c | 40 ++++------- src/dir.c | 171 ++++++++++++--------------------------------- src/merge.c | 4 +- src/pg_probackup.h | 26 ++++--- src/ptrack.c | 13 ++-- src/validate.c | 4 +- 10 files changed, 135 insertions(+), 245 deletions(-) diff --git a/src/archive.c b/src/archive.c index a063b07ed..cbe572298 100644 --- a/src/archive.c +++ b/src/archive.c @@ -945,7 +945,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, /* get list of files from archive_status */ status_files = parray_new(); dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); - parray_qsort(status_files, pgFileComparePath); + parray_qsort(status_files, pgFileCompareName); for (i = 0; i < parray_num(status_files); i++) { diff --git a/src/backup.c b/src/backup.c index 96b47916d..0d84d82a5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -149,7 +149,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) parray *external_dirs = NULL; parray *database_map = NULL; - pgFile *pg_control = NULL; PGconn *master_conn = NULL; PGconn *pg_startbackup_conn = NULL; @@ -394,7 +393,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) * Sorted array is used at least in parse_filelist_filenames(), * extractPageMap(), make_pagemap_from_ptrack(). */ - parray_qsort(backup_files_list, pgFileComparePath); + parray_qsort(backup_files_list, pgFileCompareRelPathWithExternal); /* Extract information about files in backup_list parsing their names:*/ parse_filelist_filenames(backup_files_list, instance_config.pgdata); @@ -468,26 +467,18 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (S_ISDIR(file->mode)) { char dirpath[MAXPGPATH]; - char *dir_name; - - if (file->external_dir_num) - dir_name = GetRelativePath(file->path, - parray_get(external_dirs, - file->external_dir_num - 1)); - else - dir_name = GetRelativePath(file->path, instance_config.pgdata); - - elog(VERBOSE, "Create directory \"%s\"", dir_name); if (file->external_dir_num) { char temp[MAXPGPATH]; snprintf(temp, MAXPGPATH, "%s%d", external_prefix, file->external_dir_num); - join_path_components(dirpath, temp, dir_name); + join_path_components(dirpath, temp, file->rel_path); } else - join_path_components(dirpath, database_path, dir_name); + join_path_components(dirpath, database_path, file->rel_path); + + elog(VERBOSE, "Create directory '%s'", dirpath); fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } @@ -590,16 +581,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) */ if (current.from_replica && !exclusive_backup) { - char pg_control_path[MAXPGPATH]; - - snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s", - instance_config.pgdata, XLOG_CONTROL_FILE); + pgFile *pg_control = NULL; for (i = 0; i < parray_num(backup_files_list); i++) { pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); - if (strcmp(tmp_file->path, pg_control_path) == 0) + if (tmp_file->external_dir_num == 0 && + (strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0)) { pg_control = tmp_file; break; @@ -608,7 +597,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (!pg_control) elog(ERROR, "Failed to find file \"%s\" in backup filelist.", - pg_control_path); + XLOG_CONTROL_FILE); set_min_recovery_point(pg_control, database_path, current.stop_lsn); } @@ -636,21 +625,23 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) join_path_components(wal_full_path, pg_xlog_path, file->rel_path); - if (S_ISREG(file->mode)) - { - file->crc = pgFileGetCRC(wal_full_path, true, false); - file->write_size = file->size; - } - /* Remove file path root prefix*/ - if (strstr(file->path, database_path) == file->path) - { - char *ptr = file->path; + if (!S_ISREG(file->mode)) + continue; - file->path = pgut_strdup(GetRelativePath(ptr, database_path)); - file->rel_path = pgut_strdup(file->path); - free(ptr); - } + file->crc = pgFileGetCRC(wal_full_path, true, false); + file->write_size = file->size; + + /* overwrite rel_path, because now it is relative to + * /backup_dir/backups/instance_name/backup_id/database/pg_xlog/ + */ + pg_free(file->rel_path); + + file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path)); + file->name = last_dir_separator(file->rel_path); + + /* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */ } + /* Add xlog files into the list of backed up files */ parray_concat(backup_files_list, xlog_files_list); parray_free(xlog_files_list); @@ -1884,8 +1875,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, file->write_size = file->size; file->uncompressed_size = file->size; - free(file->path); - file->path = strdup(PG_BACKUP_LABEL_FILE); parray_append(backup_files_list, file); } } @@ -1932,8 +1921,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, file->crc = pgFileGetCRC(tablespace_map, true, false); file->write_size = file->size; } - free(file->path); - file->path = strdup(PG_TABLESPACE_MAP_FILE); + parray_append(backup_files_list, file); } } @@ -2204,13 +2192,13 @@ parse_filelist_filenames(parray *files, const char *root) while (i < parray_num(files)) { pgFile *file = (pgFile *) parray_get(files, i); - char *relative; +// char *relative; int sscanf_result; - relative = GetRelativePath(file->path, root); +// relative = GetRelativePath(file->rel_path, root); if (S_ISREG(file->mode) && - path_is_prefix_of_path(PG_TBLSPC_DIR, relative)) + path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) { /* * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY @@ -2225,21 +2213,21 @@ parse_filelist_filenames(parray *files, const char *root) * Check that the file is located under * TABLESPACE_VERSION_DIRECTORY */ - sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u", + sscanf_result = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s/%u", &tblspcOid, tmp_rel_path, &dbOid); /* Yes, it is */ if (sscanf_result == 2 && strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY, strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) - set_cfs_datafiles(files, root, relative, i); + set_cfs_datafiles(files, root, file->rel_path, i); } } if (S_ISREG(file->mode) && file->tblspcOid != 0 && file->name && file->name[0]) { - if (strcmp(file->forkName, "init") == 0) + if (file->forkName == INIT) { /* * Do not backup files of unlogged relations. @@ -2290,7 +2278,6 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) int p; pgFile *prev_file; char *cfs_tblspc_path; - char *relative_prev_file; cfs_tblspc_path = strdup(relative); if(!cfs_tblspc_path) @@ -2302,22 +2289,21 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) for (p = (int) i; p >= 0; p--) { prev_file = (pgFile *) parray_get(files, (size_t) p); - relative_prev_file = GetRelativePath(prev_file->path, root); - elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file); + elog(VERBOSE, "Checking file in cfs tablespace %s", prev_file->rel_path); - if (strstr(relative_prev_file, cfs_tblspc_path) != NULL) + if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) { if (S_ISREG(prev_file->mode) && prev_file->is_datafile) { elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", - relative_prev_file, prev_file->name); + prev_file->rel_path, prev_file->name); prev_file->is_cfs = true; } } else { - elog(VERBOSE, "Breaking on %s", relative_prev_file); + elog(VERBOSE, "Breaking on %s", prev_file->rel_path); break; } } @@ -2331,7 +2317,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) { - char *path; +// char *path; char *rel_path; BlockNumber blkno_inseg; int segno; @@ -2343,16 +2329,15 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) rel_path = relpathperm(rnode, forknum); if (segno > 0) - path = psprintf("%s/%s.%u", instance_config.pgdata, rel_path, segno); + f.rel_path = psprintf("%s.%u", rel_path, segno); else - path = psprintf("%s/%s", instance_config.pgdata, rel_path); + f.rel_path = rel_path; - pg_free(rel_path); + f.external_dir_num = 0; - f.path = path; /* backup_files_list should be sorted before */ file_item = (pgFile **) parray_bsearch(backup_files_list, &f, - pgFileComparePath); + pgFileCompareRelPathWithExternal); /* * If we don't have any record of this file in the file map, it means @@ -2372,7 +2357,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pthread_mutex_unlock(&backup_pagemap_mutex); } - pg_free(path); + pg_free(rel_path); } /* diff --git a/src/catalog.c b/src/catalog.c index aedd01fbb..d9c774228 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -471,6 +471,9 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) backup->root_dir = pgut_strdup(data_path); + backup->database_dir = pgut_malloc(MAXPGPATH); + join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); + /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID @@ -841,6 +844,9 @@ pgBackupCreateDir(pgBackup *backup) fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); backup->root_dir = pgut_strdup(path); + backup->database_dir = pgut_malloc(MAXPGPATH); + join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); + /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { @@ -873,7 +879,7 @@ catalog_get_timelines(InstanceConfig *instance) /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); - parray_qsort(xlog_files_list, pgFileComparePath); + parray_qsort(xlog_files_list, pgFileCompareName); timelineinfos = parray_new(); tlinfo = NULL; @@ -1849,7 +1855,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, int len = 0; char line[BLCKSZ]; pgFile *file = (pgFile *) parray_get(files, i); - char *path = file->path; /* for streamed WAL files */ if (S_ISDIR(file->mode)) { @@ -1873,20 +1878,12 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, } } - /* for files from PGDATA and external files use rel_path - * streamed WAL files has rel_path relative not to "database/" - * but to "database/pg_wal", so for them use path. - */ - if ((root && strstr(path, root) == path) || - (file->external_dir_num && external_list)) - path = file->rel_path; - len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " "\"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " "\"dbOid\":\"%u\"", - path, file->write_size, file->mode, + file->rel_path, file->write_size, file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, @@ -2257,6 +2254,7 @@ pgBackupInit(pgBackup *backup) backup->server_version[0] = '\0'; backup->external_dir_str = NULL; backup->root_dir = NULL; + backup->database_dir = NULL; backup->files = NULL; backup->note = NULL; backup->content_crc = 0; @@ -2272,6 +2270,7 @@ pgBackupFree(void *backup) pfree(b->primary_conninfo); pfree(b->external_dir_str); pfree(b->root_dir); + pfree(b->database_dir); pfree(b->note); pfree(backup); } diff --git a/src/checkdb.c b/src/checkdb.c index 8ae4255cf..0650b50a0 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -217,7 +217,7 @@ do_block_validation(char *pgdata, uint32 checksum_version) * 1 - create 'base' * 2 - create 'base/1' */ - parray_qsort(files_list, pgFileComparePath); + parray_qsort(files_list, pgFileCompareRelPathWithExternal); /* Extract information about files in pgdata parsing their names:*/ parse_filelist_filenames(files_list, pgdata); diff --git a/src/delete.c b/src/delete.c index be7dba9d4..5da60da90 100644 --- a/src/delete.c +++ b/src/delete.c @@ -753,7 +753,7 @@ delete_backup_files(pgBackup *backup) dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST); /* delete leaf node first */ - parray_qsort(files, pgFileComparePathDesc); + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); num_files = parray_num(files); for (i = 0; i < num_files; i++) { @@ -923,31 +923,35 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, */ if (purge_all || wal_file->segno < OldestToKeepSegNo) { + char wal_fullpath[MAXPGPATH]; + + join_path_components(wal_fullpath, instance_config.arclog_path, wal_file->file.name); + /* save segment from purging */ if (instance_config.wal_depth >= 0 && wal_file->keep) { - elog(VERBOSE, "Retain WAL segment \"%s\"", wal_file->file.path); + elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath); continue; } /* unlink segment */ - if (fio_unlink(wal_file->file.path, FIO_BACKUP_HOST) < 0) + if (fio_unlink(wal_fullpath, FIO_BACKUP_HOST) < 0) { /* Missing file is not considered as error condition */ if (errno != ENOENT) elog(ERROR, "Could not remove file \"%s\": %s", - wal_file->file.path, strerror(errno)); + wal_fullpath, strerror(errno)); } else { if (wal_file->type == SEGMENT) - elog(VERBOSE, "Removed WAL segment \"%s\"", wal_file->file.path); + elog(VERBOSE, "Removed WAL segment \"%s\"", wal_fullpath); else if (wal_file->type == TEMP_SEGMENT) - elog(VERBOSE, "Removed temp WAL segment \"%s\"", wal_file->file.path); + elog(VERBOSE, "Removed temp WAL segment \"%s\"", wal_fullpath); else if (wal_file->type == PARTIAL_SEGMENT) - elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_file->file.path); + elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_fullpath); else if (wal_file->type == BACKUP_HISTORY_FILE) - elog(VERBOSE, "Removed backup history file \"%s\"", wal_file->file.path); + elog(VERBOSE, "Removed backup history file \"%s\"", wal_fullpath); } wal_deleted = true; @@ -961,7 +965,6 @@ int do_delete_instance(void) { parray *backup_list; - parray *xlog_files_list; int i; int rc; char instance_config_path[MAXPGPATH]; @@ -983,24 +986,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - xlog_files_list = parray_new(); - dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); - - for (i = 0; i < parray_num(xlog_files_list); i++) - { - pgFile *wal_file = (pgFile *) parray_get(xlog_files_list, i); - if (S_ISREG(wal_file->mode)) - { - rc = unlink(wal_file->path); - if (rc != 0) - elog(WARNING, "Failed to remove file \"%s\": %s", - wal_file->path, strerror(errno)); - } - } - - /* Cleanup */ - parray_walk(xlog_files_list, pgFileFree); - parray_free(xlog_files_list); + rmtree(arclog_path, false); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); diff --git a/src/dir.c b/src/dir.c index 33b140721..13c8d0cf8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -124,8 +124,8 @@ typedef struct TablespaceCreatedList static int pgCompareString(const void *str1, const void *str2); static char dir_check_file(pgFile *file); -static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, +static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, + bool exclude, bool follow_symlink, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -178,7 +178,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, strerror(errno)); } - file = pgFileInit(path, rel_path); + file = pgFileInit(rel_path); file->size = st.st_size; file->mode = st.st_mode; file->mtime = st.st_mtime; @@ -188,7 +188,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, } pgFile * -pgFileInit(const char *path, const char *rel_path) +pgFileInit(const char *rel_path) { pgFile *file; char *file_name = NULL; @@ -196,20 +196,14 @@ pgFileInit(const char *path, const char *rel_path) file = (pgFile *) pgut_malloc(sizeof(pgFile)); MemSet(file, 0, sizeof(pgFile)); - file->forkName = pgut_malloc(MAXPGPATH); - file->forkName[0] = '\0'; - - file->path = pgut_strdup(path); - canonicalize_path(file->path); - file->rel_path = pgut_strdup(rel_path); canonicalize_path(file->rel_path); /* Get file name from the path */ - file_name = pgut_strdup(last_dir_separator(file->path)); + file_name = last_dir_separator(file->rel_path); if (file_name == NULL) - file->name = file->path; + file->name = file->rel_path; else { file_name++; @@ -395,25 +389,10 @@ pgFileFree(void *file) file_ptr = (pgFile *) file; - if (file_ptr->linked) - free(file_ptr->linked); - - if (file_ptr->forkName) - free(file_ptr->forkName); - - pfree(file_ptr->path); + pfree(file_ptr->linked); pfree(file_ptr->rel_path); - pfree(file); -} - -/* Compare two pgFile with their path in ascending order of ASCII code. */ -int -pgFileComparePath(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - return strcmp(f1p->path, f2p->path); + pfree(file); } /* Compare two pgFile with their path in ascending order of ASCII code. */ @@ -436,30 +415,6 @@ pgFileCompareName(const void *f1, const void *f2) return strcmp(f1p->name, f2p->name); } -/* - * Compare two pgFile with their path and external_dir_num - * in ascending order of ASCII code. - */ -int -pgFileComparePathWithExternal(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - int res; - - res = strcmp(f1p->path, f2p->path); - if (!res) - { - if (f1p->external_dir_num > f2p->external_dir_num) - return 1; - else if (f1p->external_dir_num < f2p->external_dir_num) - return -1; - else - return 0; - } - return res; -} - /* * Compare two pgFile with their relative path and external_dir_num in ascending * order of ASСII code. @@ -484,23 +439,6 @@ pgFileCompareRelPathWithExternal(const void *f1, const void *f2) return res; } -/* Compare two pgFile with their path in descending order of ASCII code. */ -int -pgFileComparePathDesc(const void *f1, const void *f2) -{ - return -pgFileComparePath(f1, f2); -} - -/* - * Compare two pgFile with their path and external_dir_num - * in descending order of ASCII code. - */ -int -pgFileComparePathWithExternalDesc(const void *f1, const void *f2) -{ - return -pgFileComparePathWithExternal(f1, f2); -} - /* * Compare two pgFile with their rel_path and external_dir_num * in descending order of ASCII code. @@ -593,15 +531,15 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink { if (external_dir_num > 0) elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", - file->path); + root); else - elog(WARNING, "Skip \"%s\": unexpected file format", file->path); + elog(WARNING, "Skip \"%s\": unexpected file format", root); return; } if (add_root) parray_append(files, file); - dir_list_file_internal(files, file, exclude, follow_symlink, + dir_list_file_internal(files, file, root, exclude, follow_symlink, external_dir_num, location); if (!add_root) @@ -661,7 +599,7 @@ dir_check_file(pgFile *file) * If the directory name is in the exclude list, do not list the * contents. */ - else if (S_ISDIR(file->mode) && !in_tablespace) + else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0) { /* * If the item in the exclude list starts with '/', compare to @@ -670,20 +608,10 @@ dir_check_file(pgFile *file) */ for (i = 0; pgdata_exclude_dir[i]; i++) { - /* Full-path exclude*/ - if (pgdata_exclude_dir[i][0] == '/') - { - if (strcmp(file->path, pgdata_exclude_dir[i]) == 0) - { - elog(VERBOSE, "Excluding directory content: %s", - file->name); - return CHECK_EXCLUDE_FALSE; - } - } - else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) + /* relative path exclude */ + if (strcmp(file->rel_path, pgdata_exclude_dir[i]) == 0) { - elog(VERBOSE, "Excluding directory content: %s", - file->name); + elog(VERBOSE, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; } } @@ -780,10 +708,23 @@ dir_check_file(pgFile *file) if (fork_name) { /* Auxiliary fork of the relfile */ - sscanf(file->name, "%u_%s", &(file->relOid), file->forkName); + if (strcmp(fork_name, "vm") == 0) + file->forkName = VM; + + else if (strcmp(fork_name, "fsm") == 0) + file->forkName = FSM; + + else if (strcmp(fork_name, "cfm") == 0) + file->forkName = CFM; + + else if (strcmp(fork_name, "init") == 0) + file->forkName = INIT; + + else if (strcmp(fork_name, "ptrack") == 0) + file->forkName = PTRACK; /* Do not backup ptrack files */ - if (strcmp(file->forkName, "ptrack") == 0) + if (file->forkName == PTRACK) return CHECK_FALSE; } else @@ -820,18 +761,18 @@ dir_check_file(pgFile *file) * pgdata_exclude_dir. */ static void -dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, +dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, + bool exclude, bool follow_symlink, int external_dir_num, fio_location location) { DIR *dir; struct dirent *dent; if (!S_ISDIR(parent->mode)) - elog(ERROR, "\"%s\" is not a directory", parent->path); + elog(ERROR, "\"%s\" is not a directory", parent_dir); /* Open directory and list contents */ - dir = fio_opendir(parent->path, location); + dir = fio_opendir(parent_dir, location); if (dir == NULL) { if (errno == ENOENT) @@ -840,7 +781,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, return; } elog(ERROR, "Cannot open directory \"%s\": %s", - parent->path, strerror(errno)); + parent_dir, strerror(errno)); } errno = 0; @@ -851,7 +792,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, char rel_child[MAXPGPATH]; char check_res; - join_path_components(child, parent->path, dent->d_name); + join_path_components(child, parent_dir, dent->d_name); join_path_components(rel_child, parent->rel_path, dent->d_name); file = pgFileNew(child, rel_child, follow_symlink, external_dir_num, @@ -870,7 +811,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, /* skip hidden files and directories */ if (file->name[0] == '.') { - elog(WARNING, "Skip hidden file: '%s'", file->path); + elog(WARNING, "Skip hidden file: '%s'", child); pgFileFree(file); continue; } @@ -881,7 +822,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, */ if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) { - elog(WARNING, "Skip '%s': unexpected file format", file->path); + elog(WARNING, "Skip '%s': unexpected file format", child); pgFileFree(file); continue; } @@ -910,7 +851,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, * recursively. */ if (S_ISDIR(file->mode)) - dir_list_file_internal(files, file, exclude, follow_symlink, + dir_list_file_internal(files, file, child, exclude, follow_symlink, external_dir_num, location); } @@ -919,7 +860,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, int errno_tmp = errno; fio_closedir(dir); elog(ERROR, "Cannot read directory \"%s\": %s", - parent->path, strerror(errno_tmp)); + parent_dir, strerror(errno_tmp)); } fio_closedir(dir); } @@ -1201,16 +1142,10 @@ read_tablespace_map(parray *files, const char *backup_dir) file = pgut_new(pgFile); memset(file, 0, sizeof(pgFile)); - file->path = pgut_malloc(strlen(link_name) + 1); - strcpy(file->path, link_name); - - file->name = file->path; - - file->linked = pgut_malloc(strlen(path) + 1); - strcpy(file->linked, path); - + /* follow the convention for pgFileFree */ + file->name = pgut_strdup(link_name); + file->linked = pgut_strdup(path); canonicalize_path(file->linked); - canonicalize_path(file->path); parray_append(files, file); } @@ -1535,7 +1470,6 @@ dir_read_file_list(const char *root, const char *external_prefix, while (fgets(buf, lengthof(buf), fp)) { char path[MAXPGPATH]; - char filepath[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; int64 write_size, @@ -1561,20 +1495,7 @@ dir_read_file_list(const char *root, const char *external_prefix, get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); get_control_value(buf, "dbOid", NULL, &dbOid, false); - if (external_dir_num && external_prefix) - { - char temp[MAXPGPATH]; - - makeExternalDirPathByNum(temp, external_prefix, external_dir_num); - join_path_components(filepath, temp, path); - } - else if (root) - join_path_components(filepath, root, path); - else - strcpy(filepath, path); - - file = pgFileInit(filepath, path); - + file = pgFileInit(path); file->write_size = (int64) write_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; @@ -1811,12 +1732,10 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ /* Add metadata to backup_content.control */ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); - pfree(file->path); - file->path = pgut_strdup(DATABASE_MAP); file->crc = pgFileGetCRC(database_map_path, true, false); - file->write_size = file->size; file->uncompressed_size = file->read_size; + parray_append(backup_files_list, file); } diff --git a/src/merge.c b/src/merge.c index dd30ef5d9..50f786a06 100644 --- a/src/merge.c +++ b/src/merge.c @@ -889,7 +889,7 @@ merge_files(void *arg) if (!pg_atomic_test_set_flag(&dest_file->lock)) continue; - tmp_file = pgFileInit(dest_file->rel_path, dest_file->rel_path); + tmp_file = pgFileInit(dest_file->rel_path); tmp_file->mode = dest_file->mode; tmp_file->is_datafile = dest_file->is_datafile; tmp_file->is_cfs = dest_file->is_cfs; @@ -1056,7 +1056,7 @@ remove_dir_with_files(const char *path) char full_path[MAXPGPATH]; dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST); - parray_qsort(files, pgFileComparePathDesc); + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); for (i = 0; i < parray_num(files); i++) { pgFile *file = (pgFile *) parray_get(files, i); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 93af776c9..bc24d3e69 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -118,6 +118,15 @@ typedef enum CompressAlg ZLIB_COMPRESS, } CompressAlg; +typedef enum ForkName +{ + VM, + FSM, + CFM, + INIT, + PTRACK +} ForkName; + #define INIT_FILE_CRC32(use_crc32c, crc) \ do { \ if (use_crc32c) \ @@ -154,21 +163,20 @@ typedef struct pgFile int64 write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ - int64 uncompressed_size; /* size of the backed-up file before compression + size_t uncompressed_size; /* size of the backed-up file before compression * and adding block headers. */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ - char *path; /* absolute path of the file */ char *rel_path; /* relative path of the file */ Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ Oid dbOid; /* dbOid extracted from path, if applicable */ Oid relOid; /* relOid extracted from path, if applicable */ - char *forkName; /* forkName extracted from path, if applicable */ + ForkName forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ - int n_blocks; /* size of the file in blocks, readed during DELTA backup */ + BlockNumber n_blocks; /* size of the data file in blocks */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; int external_dir_num; /* Number of external directory. 0 if not external */ @@ -391,6 +399,8 @@ struct pgBackup * separated by ':' */ char *root_dir; /* Full path for root backup directory: backup_path/instance_name/backup_id */ + char *database_dir; /* Full path to directory with data files: + backup_path/instance_name/backup_id/database */ parray *files; /* list of files belonging to this backup * must be populated explicitly */ char *note; @@ -886,7 +896,7 @@ extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location); -extern pgFile *pgFileInit(const char *path, const char *rel_path); +extern pgFile *pgFileInit(const char *rel_path); extern void pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); @@ -894,14 +904,10 @@ extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c); extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool missing_ok, bool use_crc32c); -extern int pgFileCompareName(const void *f1, const void *f2); -extern int pgFileComparePath(const void *f1, const void *f2); extern int pgFileMapComparePath(const void *f1, const void *f2); -extern int pgFileComparePathWithExternal(const void *f1, const void *f2); +extern int pgFileCompareName(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); -extern int pgFileComparePathDesc(const void *f1, const void *f2); -extern int pgFileComparePathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); extern int pgCompareOid(const void *f1, const void *f2); diff --git a/src/ptrack.c b/src/ptrack.c index ee39d23b8..d5905b516 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -24,7 +24,7 @@ /* * Given a list of files in the instance to backup, build a pagemap for each * data file that has ptrack. Result is saved in the pagemap field of pgFile. - * NOTE we rely on the fact that provided parray is sorted by file->path. + * NOTE we rely on the fact that provided parray is sorted by file->rel_path. */ void make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) @@ -46,11 +46,6 @@ make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) */ if (file->is_database) { - char *filename = strrchr(file->path, '/'); - - Assert(filename != NULL); - filename++; - /* * The function pg_ptrack_get_and_clear_db returns true * if there was a ptrack_init file. @@ -71,7 +66,7 @@ make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) file->dbOid == dbOid_with_ptrack_init) { /* ignore ptrack if ptrack_init exists */ - elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->path); + elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->rel_path); file->pagemap_isabsent = true; continue; } @@ -106,7 +101,7 @@ make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) */ if (start_addr > ptrack_nonparsed_size) { - elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); file->pagemap_isabsent = true; } else @@ -137,7 +132,7 @@ make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) * i.e. CREATE DATABASE * - target relation was deleted. */ - elog(VERBOSE, "Ptrack is missing for file: %s", file->path); + elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); file->pagemap_isabsent = true; } } diff --git a/src/validate.c b/src/validate.c index 75c6e33e8..b8ba8efc5 100644 --- a/src/validate.c +++ b/src/validate.c @@ -258,7 +258,7 @@ pgBackupValidateFiles(void *arg) if (progress) elog(INFO, "Progress: (%d/%d). Validate file \"%s\"", - i + 1, num_files, file->path); + i + 1, num_files, file->rel_path); /* * Skip files which has no data, because they @@ -271,7 +271,7 @@ pgBackupValidateFiles(void *arg) { /* It is illegal for file in FULL backup to have BYTES_INVALID */ elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.", - file->path); + file->rel_path); arguments->corrupted = true; break; } From 50e1f77faaa1e9ddf758c20e6c5c9bcf1da583e3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 18 May 2020 01:40:37 +0300 Subject: [PATCH 1294/2107] tests: minor improvemnt for "merge" module --- tests/merge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index e40ed05c8..60c2285f5 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -261,7 +261,7 @@ def test_merge_compressed_and_uncompressed_backups(self): node.slow_start() # Fill with data - node.pgbench_init(scale=5) + node.pgbench_init(scale=10) # Do compressed FULL backup self.backup_node(backup_dir, "node", node, options=[ @@ -272,7 +272,7 @@ def test_merge_compressed_and_uncompressed_backups(self): self.assertEqual(show_backup["backup-mode"], "FULL") # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed DELTA backup @@ -281,7 +281,7 @@ def test_merge_compressed_and_uncompressed_backups(self): options=['--compress', '--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do uncompressed PAGE backup From 8b46ef5d4b92acd18e21fec44e4292fd21aaea6f Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Mon, 18 May 2020 19:50:58 +1000 Subject: [PATCH 1295/2107] Do not validate backup, if binary is compiled with different PG version --- src/validate.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/validate.c b/src/validate.c index 75c6e33e8..89a032336 100644 --- a/src/validate.c +++ b/src/validate.c @@ -120,6 +120,13 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) return; } + if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) + { + elog(ERROR, "Backup was made with server version %s, but pg_probackup compiled " + "with server version %s.", + backup->server_version, PG_MAJORVERSION); + } + // if (params && params->partial_db_list) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // params->partial_restore_type); From 9bf20803b2585ae7ff43c94e305eb8371951563b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 May 2020 19:27:01 +0300 Subject: [PATCH 1296/2107] [Issue #205] Avoid opening remote files via fio_open when copying files during backup --- src/archive.c | 18 ++- src/backup.c | 22 +-- src/catalog.c | 4 + src/data.c | 337 +++++++++++++++++++++++++-------------------- src/pg_probackup.h | 17 ++- src/utils/file.c | 264 +++++++++++++++++++++++------------ src/utils/file.h | 1 - 7 files changed, 394 insertions(+), 269 deletions(-) diff --git a/src/archive.c b/src/archive.c index cbe572298..1c325e2e1 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1387,15 +1387,16 @@ get_wal_file(const char *filename, const char *from_fullpath, */ if (fio_is_remote(FIO_BACKUP_HOST)) { + char *errmsg = NULL; /* get file via ssh */ #ifdef HAVE_LIBZ /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, thread_num); + rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, NULL, &errmsg); if (rc == FILE_MISSING) #endif /* ... failing that, use uncompressed */ - rc = fio_send_file(from_fullpath, to_fullpath, out, thread_num); + rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg); /* When not in prefetch mode, try to use partial file */ if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) @@ -1405,18 +1406,27 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* '.gz.partial' goes first ... */ snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, to_fullpath, out, thread_num); + rc = fio_send_file_gz(from_partial, to_fullpath, out, NULL, &errmsg); if (rc == FILE_MISSING) #endif { /* ... failing that, use '.partial' */ snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = fio_send_file(from_partial, to_fullpath, out, thread_num); + rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg); } if (rc == SEND_OK) src_partial = true; } + + if (rc == WRITE_FAILED) + elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", + thread_num, to_fullpath, strerror(errno)); + + if (errmsg) + elog(WARNING, "Thread [%d]: %s", thread_num, errmsg); + + pg_free(errmsg); } else { diff --git a/src/backup.c b/src/backup.c index 0d84d82a5..fc67d99a6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -550,19 +550,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) elog(ERROR, "Data files transferring failed, time elapsed: %s", pretty_time); - /* Remove disappeared during backup files from backup_list */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); - - if (tmp_file->write_size == FILE_NOT_FOUND) - { - pgFileFree(tmp_file); - parray_remove(backup_files_list, i); - i--; - } - } - /* clean previous backup file list */ if (prev_backup_filelist) { @@ -2150,6 +2137,12 @@ backup_files(void *arg) current.backup_mode, current.parent_backup, true); } + /* No point in storing empty, missing or not changed files */ + if (file->write_size <= 0) + unlink(to_fullpath); +// elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath, +// strerror(errno)); + if (file->write_size == FILE_NOT_FOUND) continue; @@ -2192,11 +2185,8 @@ parse_filelist_filenames(parray *files, const char *root) while (i < parray_num(files)) { pgFile *file = (pgFile *) parray_get(files, i); -// char *relative; int sscanf_result; -// relative = GetRelativePath(file->rel_path, root); - if (S_ISREG(file->mode) && path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) { diff --git a/src/catalog.c b/src/catalog.c index d9c774228..599bb4d28 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1856,6 +1856,10 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char line[BLCKSZ]; pgFile *file = (pgFile *) parray_get(files, i); + /* Ignore disappeared file */ + if (file->write_size == FILE_NOT_FOUND) + continue; + if (S_ISDIR(file->mode)) { backup_size_on_disk += 4096; diff --git a/src/data.c b/src/data.c index 1270de4f7..fd50ace2b 100644 --- a/src/data.c +++ b/src/data.c @@ -199,44 +199,44 @@ void get_header_errormsg(Page page, char **errormsg) { PageHeader phdr = (PageHeader) page; - *errormsg = pgut_malloc(MAXPGPATH); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); if (PageGetPageSize(phdr) != BLCKSZ) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "page size %lu is not equal to block size %u", PageGetPageSize(phdr), BLCKSZ); else if (phdr->pd_lower < SizeOfPageHeaderData) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_lower %i is less than page header size %lu", phdr->pd_lower, SizeOfPageHeaderData); else if (phdr->pd_lower > phdr->pd_upper) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_lower %u is greater than pd_upper %u", phdr->pd_lower, phdr->pd_upper); else if (phdr->pd_upper > phdr->pd_special) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_upper %u is greater than pd_special %u", phdr->pd_upper, phdr->pd_special); else if (phdr->pd_special > BLCKSZ) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_special %u is greater than block size %u", phdr->pd_special, BLCKSZ); else if (phdr->pd_special != MAXALIGN(phdr->pd_special)) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_special %i is misaligned, expected %lu", phdr->pd_special, MAXALIGN(phdr->pd_special)); else if (phdr->pd_flags & ~PD_VALID_FLAG_BITS) - snprintf(*errormsg, MAXPGPATH, "page header invalid, " + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "pd_flags mask contain illegal bits"); else - snprintf(*errormsg, MAXPGPATH, "page header invalid"); + snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid"); } /* We know that checksumms are mismatched, store specific @@ -246,9 +246,9 @@ void get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) { PageHeader phdr = (PageHeader) page; - *errormsg = pgut_malloc(MAXPGPATH); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, MAXPGPATH, + snprintf(*errormsg, ERRMSG_MAX_LEN, "page verification failed, " "calculated checksum %u but expected %u", phdr->pd_checksum, @@ -555,8 +555,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok) { - FILE *in; - FILE *out; + FILE *in = NULL; + FILE *out = NULL; BlockNumber blknum = 0; BlockNumber nblocks = 0; /* number of blocks in source file */ BlockNumber n_blocks_skipped = 0; @@ -571,7 +571,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* sanity */ if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); + elog(WARNING, "File: '%s', invalid file size %zu", from_fullpath, file->size); /* * Compute expected number of blocks in the file. @@ -607,42 +607,15 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); - /* open source file for read */ - in = fio_fopen(from_fullpath, PG_BINARY_R, FIO_DB_HOST); - if (in == NULL) - { - FIN_FILE_CRC32(true, file->crc); - - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - return; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - - /* In all other cases throw an error */ - elog(ERROR, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - } - /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", + elog(ERROR, "Cannot open backup file '%s': %s", to_fullpath, strerror(errno)); /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + elog(ERROR, "Cannot change mode of '%s': %s", to_fullpath, strerror(errno)); /* @@ -662,33 +635,17 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, else use_pagemap = true; - if (!fio_is_remote_file(in)) - { - /* enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - if (use_pagemap) - setvbuf(in, NULL, _IONBF, BUFSIZ); - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - } - /* enable stdio buffering for output file */ out_buf = pgut_malloc(STDIO_BUFSIZE); setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Remote mode */ - if (fio_is_remote_file(in)) + if (fio_is_remote(FIO_DB_HOST)) { char *errmsg = NULL; BlockNumber err_blknum = 0; - /* TODO: retrying via ptrack should be implemented on the agent */ - int rc = fio_send_pages(in, out, file, + int rc = fio_send_pages(out, from_fullpath, file, /* send prev backup START_LSN */ backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, @@ -699,23 +656,41 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, &err_blknum, &errmsg); /* check for errors */ - if (rc == REMOTE_ERROR) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - err_blknum, from_fullpath, strerror(errno)); + if (rc == FILE_MISSING) + { + elog(LOG, "File '%s' is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write block %u of '%s': %s", + err_blknum, to_fullpath, strerror(errno)); else if (rc == PAGE_CORRUPTION) { if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + elog(ERROR, "Corruption detected in file '%s', block %u: %s", from_fullpath, err_blknum, errmsg); else - elog(ERROR, "Corruption detected in file \"%s\", block %u", + elog(ERROR, "Corruption detected in file '%s', block %u", from_fullpath, err_blknum); } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); + /* OPEN_FAILED and READ_FAILED */ + else if (rc == OPEN_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Failed to open for reading remote file '%s'", from_fullpath); + } + else if (rc == READ_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Failed to read from remote file '%s'", from_fullpath); + } file->read_size = rc * BLCKSZ; pg_free(errmsg); @@ -724,10 +699,47 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* Local mode */ else { + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + { + if (missing_ok) + { + elog(LOG, "File '%s' is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + else + elog(ERROR, "File '%s' is not found", from_fullpath); + } + + /* In all other cases throw an error */ + elog(ERROR, "Cannot open file '%s': %s", + from_fullpath, strerror(errno)); + } + + /* Enable stdio buffering for local input file, + * unless the pagemap is involved, which + * imply a lot of random access. + */ + if (use_pagemap) { iter = datapagemap_iterate(&file->pagemap); datapagemap_next(iter, &blknum); /* set first block */ + + setvbuf(in, NULL, _IONBF, BUFSIZ); + } + else + { + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); } while (blknum < nblocks) @@ -776,14 +788,6 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, backup_mode == BACKUP_MODE_DIFF_DELTA) file->n_blocks = file->read_size / BLCKSZ; - if (fclose(out)) - elog(ERROR, "Cannot close the backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - fio_fclose(in); - - FIN_FILE_CRC32(true, file->crc); - /* Determine that file didn`t changed in case of incremental backup */ if (backup_mode != BACKUP_MODE_FULL && file->exists_in_prev && @@ -793,15 +797,19 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, file->write_size = BYTES_INVALID; } - /* - * No point in storing empty files. - */ - if (file->write_size <= 0) - { - if (unlink(to_fullpath) == -1) - elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath, - strerror(errno)); - } +cleanup: + /* finish CRC calculation */ + FIN_FILE_CRC32(true, file->crc); + + /* close local input file */ + if (in && fclose(in)) + elog(ERROR, "Cannot close the source file '%s': %s", + to_fullpath, strerror(errno)); + + /* close local output file */ + if (out && fclose(out)) + elog(ERROR, "Cannot close the backup file '%s': %s", + to_fullpath, strerror(errno)); pg_free(in_buf); pg_free(out_buf); @@ -1249,108 +1257,131 @@ backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok) { - FILE *in; - FILE *out; + FILE *in = NULL; + FILE *out = NULL; ssize_t read_len = 0; - char *buf; - pg_crc32 crc; + char *buf = NULL; - INIT_FILE_CRC32(true, crc); + INIT_FILE_CRC32(true, file->crc); /* reset size summary */ file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - /* open source file for read */ - in = fio_fopen(from_fullpath, PG_BINARY_R, from_location); - if (in == NULL) + /* open backup file for write */ + out = fopen(to_fullpath, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination file '%s': %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath, file->mode) == -1) + elog(ERROR, "Cannot change mode of '%s': %s", to_fullpath, + strerror(errno)); + + /* backup remote file */ + if (fio_is_remote(FIO_DB_HOST)) { - FIN_FILE_CRC32(true, crc); - file->crc = crc; + char *errmsg = NULL; + int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg); - /* maybe deleted, it's not error in case of backup */ - if (errno == ENOENT) + /* handle errors */ + if (rc == FILE_MISSING) { + /* maybe deleted, it's not error in case of backup */ if (missing_ok) { - elog(LOG, "File \"%s\" is not found", from_fullpath); + elog(LOG, "File '%s' is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; - return; + goto cleanup; } else - elog(ERROR, "File \"%s\" is not found", from_fullpath); + elog(ERROR, "File '%s' is not found", from_fullpath); + } + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write to '%s': %s", to_fullpath, strerror(errno)); + else if (rc != SEND_OK) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot access remote file '%s'", from_fullpath); } - elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, - strerror(errno)); + pg_free(errmsg); } + /* backup local file */ + else + { + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* maybe deleted, it's not error in case of backup */ + if (errno == ENOENT) + { + if (missing_ok) + { + elog(LOG, "File '%s' is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + else + elog(ERROR, "File '%s' is not found", from_fullpath); + } - /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + elog(ERROR, "Cannot open file '%s': %s", from_fullpath, + strerror(errno)); + } - /* disable stdio buffering for local input/output files */ - if (!fio_is_remote_file(in)) + /* disable stdio buffering for local input/output files to avoid triple buffering */ setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); + setvbuf(out, NULL, _IONBF, BUFSIZ); - /* allocate 64kB buffer */ - buf = pgut_malloc(STDIO_BUFSIZE); + /* allocate 64kB buffer */ + buf = pgut_malloc(CHUNK_SIZE); - /* copy content and calc CRC */ - for (;;) - { - read_len = fio_fread(in, buf, STDIO_BUFSIZE); + /* copy content and calc CRC */ + for (;;) + { + read_len = fread(buf, 1, CHUNK_SIZE, in); - if (read_len < 0) - elog(ERROR, "Cannot read from source file \"%s\": %s", - from_fullpath, strerror(errno)); + if (ferror(in)) + elog(ERROR, "Cannot read from file '%s': %s", + from_fullpath, strerror(errno)); - if (read_len == 0) - break; + if (read_len > 0) + { + if (fwrite(buf, 1, read_len, out) != read_len) + elog(ERROR, "Cannot write to file '%s': %s", to_fullpath, + strerror(errno)); - if (fwrite(buf, 1, read_len, out) != read_len) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno)); + /* update CRC */ + COMP_FILE_CRC32(true, file->crc, buf, read_len); + file->read_size += read_len; + } - /* update CRC */ - COMP_FILE_CRC32(true, crc, buf, read_len); - - file->read_size += read_len; - -// if (read_len < STDIO_BUFSIZE) -// { -// if (!fio_is_remote_file(in)) -// { -// if (ferror(in)) -// elog(ERROR, "Cannot read from source file \"%s\": %s", -// from_fullpath, strerror(errno)); -// -// if (feof(in)) -// break; -// } -// } + if (feof(in)) + break; + } } file->write_size = (int64) file->read_size; if (file->write_size > 0) file->uncompressed_size = file->write_size; + +cleanup: /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(true, crc); - file->crc = crc; + FIN_FILE_CRC32(true, file->crc); + + if (in && fclose(in)) + elog(ERROR, "Cannot close the file '%s': %s", from_fullpath, strerror(errno)); + + if (out && fclose(out)) + elog(ERROR, "Cannot close the file '%s': %s", to_fullpath, strerror(errno)); - if (fclose(out)) - elog(ERROR, "Cannot write \"%s\": %s", to_fullpath, strerror(errno)); - fio_fclose(in); pg_free(buf); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc24d3e69..b64ac7313 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -87,6 +87,10 @@ extern const char *PROGRAM_EMAIL; /* stdio buffer size */ #define STDIO_BUFSIZE 65536 +#define ERRMSG_MAX_LEN 2048 +#define CHUNK_SIZE (128 * 1024) +#define OUT_BUF_SIZE (512 * 1024) + /* retry attempts */ #define PAGE_READ_ATTEMPTS 100 @@ -168,9 +172,9 @@ typedef struct pgFile */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ + char *rel_path; /* relative path of the file */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ - char *rel_path; /* relative path of the file */ Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ Oid dbOid; /* dbOid extracted from path, if applicable */ Oid relOid; /* relOid extracted from path, if applicable */ @@ -178,7 +182,7 @@ typedef struct pgFile int segno; /* Segment number for ptrack */ BlockNumber n_blocks; /* size of the data file in blocks */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ - bool is_database; + bool is_database; /* Flag used strictly by ptrack 1.x backup */ int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -1020,13 +1024,14 @@ extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); /* FIO */ -extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, +extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); /* return codes for fio_send_pages */ -#define OUT_BUF_SIZE (512 * 1024) -extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); -extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); +extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, + pgFile *file, char **errormsg); +extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, + pgFile *file, char **errormsg); /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) diff --git a/src/utils/file.c b/src/utils/file.c index e19754080..356e09df3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -14,7 +14,6 @@ #define PRINTF_BUF_SIZE 1024 #define FILE_PERMISSIONS 0600 -#define CHUNK_SIZE 1024 * 128 static __thread unsigned long fio_fdset = 0; static __thread void* fio_stdin_buffer; @@ -33,6 +32,7 @@ typedef struct int calg; int clevel; int bitmapsize; + int path_len; } fio_send_request; @@ -1316,9 +1316,11 @@ static void fio_load_file(int out, char const* path) * Return number of actually(!) readed blocks, attempts or * half-readed block are not counted. * Return values in case of error: - * REMOTE_ERROR - * PAGE_CORRUPTION - * WRITE_FAILED + * FILE_MISSING + * OPEN_FAILED + * READ_ERROR + * PAGE_CORRUPTION + * WRITE_FAILED * * If none of the above, this function return number of blocks * readed by remote agent. @@ -1326,7 +1328,7 @@ static void fio_load_file(int out, char const* path) * In case of DELTA mode horizonLsn must be a valid lsn, * otherwise it should be set to InvalidXLogRecPtr. */ -int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, +int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg) @@ -1338,22 +1340,19 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, BlockNumber n_blocks_read = 0; BlockNumber blknum = 0; - Assert(fio_is_remote_file(in)); - /* send message with header - 8bytes 20bytes var - ------------------------------------------------------ - | fio_header | fio_send_request | BITMAP(if any) | - ------------------------------------------------------ + 8bytes 24bytes var var + -------------------------------------------------------------- + | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | + -------------------------------------------------------------- */ - req.hdr.handle = fio_fileno(in) & ~FIO_PIPE_MARKER; + req.hdr.cop = FIO_SEND_PAGES; if (pagemap) { - req.hdr.cop = FIO_SEND_PAGES_PAGEMAP; - req.hdr.size = sizeof(fio_send_request) + pagemap->bitmapsize; + req.hdr.size = sizeof(fio_send_request) + pagemap->bitmapsize + strlen(from_fullpath) + 1; req.arg.bitmapsize = pagemap->bitmapsize; /* TODO: add optimization for the case of pagemap @@ -1363,8 +1362,8 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, } else { - req.hdr.cop = FIO_SEND_PAGES; - req.hdr.size = sizeof(fio_send_request); + req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; + req.arg.bitmapsize = 0; } req.arg.nblocks = file->size/BLCKSZ; @@ -1373,6 +1372,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, req.arg.checksumVersion = checksum_version; req.arg.calg = calg; req.arg.clevel = clevel; + req.arg.path_len = strlen(from_fullpath) + 1; file->compress_alg = calg; /* TODO: wtf? why here? */ @@ -1385,10 +1385,14 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, // pg_free(iter); //<----- + /* send header */ IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + /* send file path */ + IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); + + /* send pagemap if any */ if (pagemap) - /* now send pagemap itself */ IO_CHECK(fio_write_all(fio_stdout, pagemap->bitmap, pagemap->bitmapsize), pagemap->bitmapsize); while (true) @@ -1402,9 +1406,15 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, if (hdr.cop == FIO_ERROR) { - errno = hdr.arg; - *err_blknum = hdr.size; - return REMOTE_ERROR; + /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); + } + + return hdr.arg; } else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) { @@ -1414,7 +1424,7 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, { IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); *errormsg = pgut_malloc(hdr.size); - strncpy(*errormsg, buf, hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); } return PAGE_CORRUPTION; } @@ -1449,66 +1459,134 @@ int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, return n_blocks_read; } -/* TODO: read file using large buffer */ -static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) +/* TODO: read file using large buffer + * Return codes: + * FIO_ERROR: + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + + * FIO_SEND_FILE_CORRUPTION + * FIO_SEND_FILE_EOF + */ +static void fio_send_pages_impl(int out, char* buf) { - BlockNumber blknum = 0; - BlockNumber n_blocks_read = 0; - XLogRecPtr page_lsn = 0; - char read_buffer[BLCKSZ+1]; - fio_header hdr; + FILE *in = NULL; + BlockNumber blknum = 0; + BlockNumber n_blocks_read = 0; + XLogRecPtr page_lsn = 0; + char read_buffer[BLCKSZ+1]; + char in_buf[STDIO_BUFSIZE]; + fio_header hdr; fio_send_request *req = (fio_send_request*) buf; - + char *from_fullpath = (char*) buf + sizeof(fio_send_request); + int current_pos = 0; + bool with_pagemap = req->bitmapsize > 0 ? true : false; + /* error reporting */ + char *errormsg = NULL; /* parse buffer */ datapagemap_t *map = NULL; datapagemap_iterator_t *iter = NULL; + /* open source file */ + in = fopen(from_fullpath, PG_BINARY_R); + if (!in) + { + hdr.cop = FIO_ERROR; + + /* do not send exact wording of ENOENT error message + * because it is a very common error in our case, so + * error code is enough. + */ + if (errno == ENOENT) + { + hdr.arg = FILE_MISSING; + hdr.size = 0; + } + else + { + hdr.arg = OPEN_FAILED; + errormsg = pgut_malloc(ERRMSG_MAX_LEN); + /* Construct the error message */ + snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open source file '%s': %s", + from_fullpath, strerror(errno)); + hdr.size = strlen(errormsg) + 1; + } + + /* send header and message */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (errormsg) + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); + + goto cleanup; + } + if (with_pagemap) { map = pgut_malloc(sizeof(datapagemap_t)); map->bitmapsize = req->bitmapsize; - map->bitmap = (char*) buf + sizeof(fio_send_request); + map->bitmap = (char*) buf + sizeof(fio_send_request) + req->path_len; /* get first block */ iter = datapagemap_iterate(map); datapagemap_next(iter, &blknum); + + setvbuf(in, NULL, _IONBF, BUFSIZ); } + else + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - hdr.cop = FIO_PAGE; + /* TODO: what is this barrier for? */ read_buffer[BLCKSZ] = 1; /* barrier */ while (blknum < req->nblocks) { - int rc = 0; - int retry_attempts = PAGE_READ_ATTEMPTS; + int rc = 0; + size_t read_len = 0; + int retry_attempts = PAGE_READ_ATTEMPTS; /* TODO: handle signals on the agent */ if (interrupted) elog(ERROR, "Interrupted during remote page reading"); /* read page, check header and validate checksumms */ - /* TODO: libpq connection on the agent, so we can do ptrack - * magic right here. - */ for (;;) { - ssize_t read_len = pread(fd, read_buffer, BLCKSZ, blknum*BLCKSZ); + /* Optimize stdio buffer usage, fseek only when current position + * does not match the position of requested block. + */ + if (current_pos != blknum*BLCKSZ) + { + current_pos = blknum*BLCKSZ; + if (fseek(in, current_pos, SEEK_SET) != 0) + elog(ERROR, "fseek to position %u is failed on remote file '%s': %s", + current_pos, from_fullpath, strerror(errno)); + } + + read_len = fread(read_buffer, 1, BLCKSZ, in); page_lsn = InvalidXLogRecPtr; - /* report eof */ - if (read_len == 0) - goto eof; + current_pos += BLCKSZ; + /* report error */ - else if (read_len < 0) + if (ferror(in)) { - /* TODO: better to report exact error message, not errno */ hdr.cop = FIO_ERROR; - hdr.arg = errno; - hdr.size = blknum; + hdr.arg = READ_FAILED; + + errormsg = pgut_malloc(ERRMSG_MAX_LEN); + /* Construct the error message */ + snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read block %u of '%s': %s", + blknum, from_fullpath, strerror(errno)); + hdr.size = strlen(errormsg) + 1; + + /* send header and message */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); goto cleanup; } - else if (read_len == BLCKSZ) + + if (read_len == BLCKSZ) { rc = validate_one_page(read_buffer, req->segmentno + blknum, InvalidXLogRecPtr, &page_lsn, req->checksumVersion); @@ -1519,6 +1597,9 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) else if (rc == PAGE_IS_VALID) break; } + + if (feof(in)) + goto eof; // else /* readed less than BLKSZ bytes, retry */ /* File is either has insane header or invalid checksum, @@ -1526,7 +1607,6 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) */ if (--retry_attempts == 0) { - char *errormsg = NULL; hdr.cop = FIO_SEND_FILE_CORRUPTION; hdr.arg = blknum; @@ -1547,7 +1627,6 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) if (errormsg) IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - pg_free(errormsg); goto cleanup; } } @@ -1567,6 +1646,7 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) BackupPageHeader* bph = (BackupPageHeader*)write_buffer; /* compress page */ + hdr.cop = FIO_PAGE; hdr.arg = bph->block = blknum; hdr.size = sizeof(BackupPageHeader); @@ -1608,6 +1688,9 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) cleanup: pg_free(map); pg_free(iter); + pg_free(errormsg); + if (in) + fclose(in); return; } @@ -1621,7 +1704,8 @@ static void fio_send_pages_impl(int fd, int out, char* buf, bool with_pagemap) * ZLIB_ERROR (-5) * REMOTE_ERROR (-6) */ -int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num) +int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, + pgFile *file, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; @@ -1634,8 +1718,8 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o hdr.cop = FIO_SEND_FILE; hdr.size = path_len; - elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", - thread_num, from_fullpath); +// elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", +// thread_num, from_fullpath); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); @@ -1655,7 +1739,8 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o if (hdr.size > 0) { IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); - elog(WARNING, "Thread [%d]: %s", thread_num, in_buf); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", in_buf); } exit_code = hdr.arg; goto cleanup; @@ -1681,8 +1766,10 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o if (rc != Z_OK) { - elog(WARNING, "Thread [%d]: Failed to initialize decompression stream for file '%s': %i: %s", - thread_num, from_fullpath, rc, strm->msg); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, + "Failed to initialize decompression stream for file '%s': %i: %s", + from_fullpath, rc, strm->msg); exit_code = ZLIB_ERROR; goto cleanup; } @@ -1714,8 +1801,10 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o else if (rc != Z_OK) { /* got an error */ - elog(WARNING, "Thread [%d]: Decompression failed for file '%s': %i: %s", - thread_num, from_fullpath, rc, strm->msg); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, + "Decompression failed for file '%s': %i: %s", + from_fullpath, rc, strm->msg); exit_code = ZLIB_ERROR; goto cleanup; } @@ -1725,8 +1814,6 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o /* Output buffer is full, write it out */ if (fwrite(out_buf, 1, OUT_BUF_SIZE, out) != OUT_BUF_SIZE) { - elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", - thread_num, to_fullpath, strerror(errno)); exit_code = WRITE_FAILED; goto cleanup; } @@ -1743,20 +1830,13 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o if (fwrite(out_buf, 1, len, out) != len) { - elog(WARNING, "Thread [%d]: Cannot write to file: %s", - thread_num, strerror(errno)); exit_code = WRITE_FAILED; goto cleanup; } } } else - { - elog(WARNING, "Thread [%d]: Remote agent returned message of unexpected type: %i", - thread_num, hdr.cop); - exit_code = REMOTE_ERROR; - break; - } + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); } cleanup: @@ -1779,10 +1859,14 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o * SEND_OK (0) * FILE_MISSING (-1) * OPEN_FAILED (-2) - * READ_FAIL (-3) - * WRITE_FAIL (-4) + * READ_FAILED (-3) + * WRITE_FAILED (-4) + * + * OPEN_FAILED and READ_FAIL should also set errormsg. + * If pgFile is not NULL then we must calculate crc and read_size for it. */ -int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num) +int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, + pgFile *file, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; @@ -1792,8 +1876,8 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, hdr.cop = FIO_SEND_FILE; hdr.size = path_len; - elog(VERBOSE, "Thread [%d]: Attempting to open remote WAL file '%s'", - thread_num, from_fullpath); +// elog(VERBOSE, "Thread [%d]: Attempting to open remote WAL file '%s'", +// thread_num, from_fullpath); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); @@ -1813,7 +1897,8 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, if (hdr.size > 0) { IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - elog(WARNING, "Thread [%d]: %s", thread_num, buf); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); } exit_code = hdr.arg; break; @@ -1826,19 +1911,20 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, /* We have received a chunk of data data, lets write it out */ if (fwrite(buf, 1, hdr.size, out) != hdr.size) { - elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", - thread_num, to_fullpath, strerror(errno)); exit_code = WRITE_FAILED; break; } + + if (file) + { + file->read_size += hdr.size; + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + } } else { /* TODO: fio_disconnect may get assert fail when running after this */ - elog(WARNING, "Thread [%d]: Remote agent returned message of unexpected type: %i", - thread_num, hdr.cop); - exit_code = REMOTE_ERROR; - break; + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); } } @@ -1851,9 +1937,13 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, /* Send file content * On error we return FIO_ERROR message with following codes - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) + * FIO_ERROR: + * FILE_MISSING (-1) + * OPEN_FAILED (-2) + * READ_FAILED (-3) + * + * FIO_PAGE + * FIO_SEND_FILE_EOF * */ static void fio_send_file_impl(int out, char const* path) @@ -1883,9 +1973,9 @@ static void fio_send_file_impl(int out, char const* path) else { hdr.arg = OPEN_FAILED; - errormsg = pgut_malloc(MAXPGPATH); + errormsg = pgut_malloc(ERRMSG_MAX_LEN); /* Construct the error message */ - snprintf(errormsg, MAXPGPATH, "Cannot open source file '%s': %s", path, strerror(errno)); + snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file '%s': %s", path, strerror(errno)); hdr.size = strlen(errormsg) + 1; } @@ -1909,10 +1999,10 @@ static void fio_send_file_impl(int out, char const* path) if (ferror(fp)) { hdr.cop = FIO_ERROR; - errormsg = pgut_malloc(MAXPGPATH); + errormsg = pgut_malloc(ERRMSG_MAX_LEN); hdr.arg = READ_FAILED; /* Construct the error message */ - snprintf(errormsg, MAXPGPATH, "Cannot read source file '%s': %s", path, strerror(errno)); + snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read from file '%s': %s", path, strerror(errno)); hdr.size = strlen(errormsg) + 1; /* send header and message */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -2084,12 +2174,8 @@ void fio_communicate(int in, int out) SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; case FIO_SEND_PAGES: - Assert(hdr.size == sizeof(fio_send_request)); - fio_send_pages_impl(fd[hdr.handle], out, buf, false); - break; - case FIO_SEND_PAGES_PAGEMAP: // buf contain fio_send_request header and bitmap. - fio_send_pages_impl(fd[hdr.handle], out, buf, true); + fio_send_pages_impl(out, buf); break; case FIO_SEND_FILE: fio_send_file_impl(out, buf); diff --git a/src/utils/file.h b/src/utils/file.h index 6aea02122..5171dacd5 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -37,7 +37,6 @@ typedef enum FIO_GET_CRC32, /* used in fio_send_pages */ FIO_SEND_PAGES, - FIO_SEND_PAGES_PAGEMAP, FIO_ERROR, FIO_SEND_FILE, // FIO_CHUNK, From 1475a5dc55040d0fa6d4b7b37583ae673022fd2f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 19 May 2020 19:29:05 +0300 Subject: [PATCH 1297/2107] [Issue #205] fix some tests --- tests/backup.py | 48 ++++++++++++++++++++++++------------------------ tests/merge.py | 6 +++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 176aba619..814bf684f 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -545,8 +545,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( + "ERROR: Corruption detected in file '{0}', block 1: " + "page verification failed, calculated checksum".format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -566,8 +566,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( + "ERROR: Corruption detected in file '{0}', block 1: " + "page verification failed, calculated checksum".format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -587,8 +587,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( + "ERROR: Corruption detected in file '{0}', block 1: " + "page verification failed, calculated checksum".format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -691,8 +691,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -711,8 +711,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -731,8 +731,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -834,8 +834,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -854,8 +854,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -874,8 +874,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), + "ERROR: Corruption detected in file '{0}', block 1: " + "page header invalid, pd_lower".format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1478,8 +1478,8 @@ def test_drop_rel_during_backup_delta(self): with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() self.assertTrue( - 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + "LOG: File '{0}' is not found".format(absolute_path) in log_content, + "File '{0}' should be deleted but it`s not".format(absolute_path)) node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) @@ -2015,7 +2015,7 @@ def test_basic_missing_file_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Cannot open file', + 'ERROR: Cannot open source file', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -2459,7 +2459,7 @@ def test_parent_choosing_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_backup_with_less_privileges_role(self): """ check permissions correctness from documentation: @@ -2973,13 +2973,13 @@ def test_issue_203(self): node.slow_start() with node.connect("postgres") as conn: - for i in range(1200000): + for i in range(1000000): conn.execute( "CREATE TABLE t_{0} as select 1".format(i)) conn.commit() full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_dir, 'node', node, options=['--stream', '-j2']) pgdata = self.pgdata_content(node.data_dir) diff --git a/tests/merge.py b/tests/merge.py index 60c2285f5..909876b40 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -413,7 +413,7 @@ def test_merge_compressed_and_uncompressed_backups_2(self): node.slow_start() # Fill with data - node.pgbench_init(scale=5) + node.pgbench_init(scale=20) # Do uncompressed FULL backup self.backup_node(backup_dir, "node", node) @@ -423,7 +423,7 @@ def test_merge_compressed_and_uncompressed_backups_2(self): self.assertEqual(show_backup["backup-mode"], "FULL") # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed DELTA backup @@ -432,7 +432,7 @@ def test_merge_compressed_and_uncompressed_backups_2(self): options=['--compress-algorithm=zlib', '--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do uncompressed PAGE backup From 8d1b5df07c2b8b0d281661589a893feb1d5b3d1b Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 15 Apr 2020 19:17:21 +0300 Subject: [PATCH 1298/2107] Correct expected error message in the test_ptrack_pg_resetxlog --- tests/ptrack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 309a3c442..6ca8d93c6 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3841,7 +3841,7 @@ def test_ptrack_zero_changes(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - @unittest.expectedFailure + # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): fname = self.id().split('.')[3] node = self.make_simple_node( @@ -3946,7 +3946,7 @@ def test_ptrack_pg_resetxlog(self): ) except ProbackupException as e: self.assertIn( - 'Insert error message', + 'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) From ad34a1d53cfb36597c7c0761d2641d03b1109fca Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 22 Apr 2020 19:30:39 +0300 Subject: [PATCH 1299/2107] Teach pg_probackup to work with ptrack 2.1 --- doc/pgprobackup.xml | 10 +++++----- src/backup.c | 2 +- src/ptrack.c | 4 +++- tests/helpers/ptrack_helpers.py | 3 ++- tests/ptrack.py | 13 +++++++------ 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 58d427494..a2161f0c6 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1168,12 +1168,12 @@ CREATE EXTENSION ptrack; - To enable tracking page updates, set ptrack_map_size + To enable tracking page updates, set ptrack.map_size parameter to a positive integer and restart the server. For optimal performance, it is recommended to set - ptrack_map_size to + ptrack.map_size to N / 1024, where N is the size of the PostgreSQL cluster, in MB. If you set this @@ -1181,7 +1181,7 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack_map_size to a higher value + Setting ptrack.map_size to a higher value does not affect PTRACK operation. The maximum allowed value is 1024. @@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - If you change the ptrack_map_size parameter value, + If you change the ptrack.map_size parameter value, the previously created PTRACK map file is cleared, and tracking newly changed blocks starts from scratch. Thus, you have to retake a full backup before taking incremental PTRACK backups after - changing ptrack_map_size. + changing ptrack.map_size. diff --git a/src/backup.c b/src/backup.c index 0d84d82a5..f312517c5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -435,7 +435,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num == 20) + if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, nodeInfo->ptrack_schema, prev_backup_start_lsn); diff --git a/src/ptrack.c b/src/ptrack.c index d5905b516..cb0acf7b2 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -199,6 +199,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = 17; else if (strcmp(ptrack_version_str, "2.0") == 0) nodeInfo->ptrack_version_num = 20; + else if (strcmp(ptrack_version_str, "2.1") == 0) + nodeInfo->ptrack_version_num = 21; else elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", ptrack_version_str); @@ -567,7 +569,7 @@ pg_ptrack_enable2(PGconn *backup_conn) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) { diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 52a1ffda5..eec57a3ba 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -366,7 +366,8 @@ def make_simple_node( if ptrack_enable: if node.major_version > 11: - options['ptrack_map_size'] = '128MB' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' else: options['ptrack_enable'] = 'on' diff --git a/tests/ptrack.py b/tests/ptrack.py index 6ca8d93c6..4678708bf 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -269,7 +269,8 @@ def test_ptrack_enable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '30s', + 'shared_preload_libraries': 'ptrack'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -336,16 +337,16 @@ def test_ptrack_disable(self): # DISABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to 0") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") else: node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.stop() node.slow_start() # ENABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") else: node.safe_psql('postgres', "alter system set ptrack_enable to on") node.stop() @@ -4054,7 +4055,7 @@ def test_corrupt_ptrack_map(self): 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) - self.set_auto_conf(node, {'ptrack_map_size': '0'}) + self.set_auto_conf(node, {'ptrack.map_size': '0'}) node.slow_start() @@ -4082,7 +4083,7 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) - self.set_auto_conf(node, {'ptrack_map_size': '32'}) + self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) node.slow_start() From 01dc9cf062dacafa7488a45dd73ccc05d30e9032 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 29 Apr 2020 20:55:39 +0300 Subject: [PATCH 1300/2107] Adopt ptrack 2.1 public API changes --- src/backup.c | 14 ++++----- src/pg_probackup.h | 10 ++++--- src/ptrack.c | 75 ++++++++++++++++++++++++---------------------- tests/ptrack.py | 2 +- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/backup.c b/src/backup.c index f312517c5..b421ab58a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -437,8 +437,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) */ if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - prev_backup_start_lsn); + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); else if (nodeInfo->ptrack_version_num == 15 || nodeInfo->ptrack_version_num == 16 || nodeInfo->ptrack_version_num == 17) @@ -875,15 +876,10 @@ do_backup(time_t start_time, bool no_validate, #endif get_ptrack_version(backup_conn, &nodeInfo); -// elog(WARNING, "ptrack_version_num %d", ptrack_version_num); + // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - { - if (nodeInfo.ptrack_version_num >= 20) - nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn); - else - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn); - } + nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc24d3e69..3ce98c4a4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1005,11 +1005,12 @@ extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, - const char *ptrack_schema, XLogRecPtr lsn); + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn); extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn); -extern bool pg_ptrack_enable2(PGconn *backup_conn); +extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, @@ -1017,7 +1018,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, size_t *result_size, PGconn *backup_conn); extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn); /* FIO */ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, diff --git a/src/ptrack.c b/src/ptrack.c index cb0acf7b2..77df7f6fe 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -212,19 +212,29 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn) +pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; + bool result = false; - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + if (ptrack_version_num < 20) { - PQclear(res_db); - return false; + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; + } + else if (ptrack_version_num == 20) + { + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + } + else + { + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; } + PQclear(res_db); - return true; + return result; } @@ -454,7 +464,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) { char query[128]; - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + res = pgut_execute(backup_conn, query, 0, NULL); } @@ -521,7 +535,10 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, @@ -561,30 +578,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments, * ---------------------------- */ -/* - * Check if ptrack is enabled in target instance - */ -bool -pg_ptrack_enable2(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - return true; -} - /* * Fetch a list of changed files with their ptrack maps. */ parray * -pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn) +pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn) { PGresult *res; char lsn_buf[17 + 1]; @@ -599,8 +598,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); + else + sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); res = pgut_execute(backup_conn, query, 1, (const char **) params); pfree(params[0]); @@ -642,16 +645,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec */ void make_pagemap_from_ptrack_2(parray *files, - PGconn *backup_conn, - const char *ptrack_schema, - XLogRecPtr lsn) + PGconn *backup_conn, + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn) { parray *filemaps; int file_i = 0; page_map_entry *dummy_map = NULL; /* Receive all available ptrack bitmaps at once */ - filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn); + filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, + ptrack_version_num, lsn); if (filemaps != NULL) parray_qsort(filemaps, pgFileMapComparePath); diff --git a/tests/ptrack.py b/tests/ptrack.py index 4678708bf..9f9dbac58 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4052,7 +4052,7 @@ def test_corrupt_ptrack_map(self): log_content = f.read() self.assertIn( - 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) self.set_auto_conf(node, {'ptrack.map_size': '0'}) From 0da843c073f5b64bb38ab6382edf091811df500b Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 6 May 2020 15:58:16 +0300 Subject: [PATCH 1301/2107] Skip test_ptrack_get_block for ptrack 2.* --- tests/ptrack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 9f9dbac58..db158081b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -631,6 +631,7 @@ def test_ptrack_get_block(self): node.slow_start() if node.major_version >= 12: + self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") node.safe_psql( "postgres", "CREATE EXTENSION ptrack") From 3880d60921ded90b2370e40fdf911c8d01454c93 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Thu, 7 May 2020 14:47:20 +0300 Subject: [PATCH 1302/2107] Change a way of setting up python environment --- travis/Dockerfile.in | 6 +++++- travis/run_tests.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index 68f5ffe22..a3c858ee2 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,7 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv +RUN apt-get -yq install python python-pip + +# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +# RUN python2 get-pip.py +RUN python2 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 95cb646c5..1bb3a6fde 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -60,7 +60,7 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -virtualenv pyenv +python2 -m virtualenv pyenv source pyenv/bin/activate pip install testgres==1.8.2 From d5b15e1d7c6ef5812afbf23cfc18718f97833998 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 8 May 2020 17:42:22 +0300 Subject: [PATCH 1303/2107] Check for ptrack.map_size = -1 for ptrack >= 2.1 --- src/ptrack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ptrack.c b/src/ptrack.c index 77df7f6fe..f8e77cc12 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -230,7 +230,8 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) else { res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; } PQclear(res_db); From f3d22bfe90b5246d8fbb5547c5a925e0601cf160 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 9 May 2020 15:19:36 +0300 Subject: [PATCH 1304/2107] Use pg_ptrack_get_block() only with ptrack <2.0.0 --- src/data.c | 17 ++++++++--------- src/ptrack.c | 5 +++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index 1270de4f7..83e95c86a 100644 --- a/src/data.c +++ b/src/data.c @@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg, } } - /* Get page via ptrack interface from PostgreSQL shared buffer. - * We do this in following cases: - * 1. PTRACK backup of 1.x versions - * 2. During backup, regardless of backup mode, of PostgreSQL instance - * with ptrack support we encountered invalid page. + /* + * Get page via ptrack interface from PostgreSQL shared buffer. + * We do this only in the cases of PTRACK 1.x versions backup */ - if ((backup_mode == BACKUP_MODE_DIFF_PTRACK + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && (ptrack_version_num >= 15 && ptrack_version_num < 20)) - || !page_is_valid) { int rc = 0; size_t page_size = 0; @@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg, memcpy(page, ptrack_page, BLCKSZ); pg_free(ptrack_page); - /* UPD: It apprears that is possible to get zeroed page or page with invalid header + /* + * UPD: It apprears that is possible to get zeroed page or page with invalid header * from shared buffer. * Note, that getting page with wrong checksumm from shared buffer is * acceptable. @@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg, from_fullpath, blknum, errormsg); } - /* We must set checksum here, because it is outdated + /* + * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ if (checksum_version) diff --git a/src/ptrack.c b/src/ptrack.c index f8e77cc12..3f2591137 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -521,7 +521,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (arguments->cancel_conn == NULL) arguments->cancel_conn = PQgetCancel(arguments->conn); - //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); if (ptrack_version_num < 20) res = pgut_execute_parallel(arguments->conn, @@ -539,7 +539,8 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (ptrack_version_num == 20) sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); else - sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); + // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, From 19129a7e646d81f19395b37d51aa73cab41b64b6 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sun, 10 May 2020 14:30:29 +0300 Subject: [PATCH 1305/2107] Fix (?) test_ptrack_uncommitted_xact --- tests/ptrack.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index db158081b..7d304d52e 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -413,7 +413,11 @@ def test_ptrack_uncommitted_xact(self): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) + # TODO: what's the point in taking pgdata content, then taking + # backup, and the trying to compare those two? Backup issues a + # checkpoint, so it will modify pgdata with close to 100% chance. + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -427,8 +431,9 @@ def test_ptrack_uncommitted_xact(self): backup_dir, 'node', node_restored, node_restored.data_dir, options=["-j", "4"]) - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.set_auto_conf( node_restored, {'port': node_restored.port}) @@ -436,7 +441,8 @@ def test_ptrack_uncommitted_xact(self): node_restored.slow_start() # Physical comparison - self.compare_pgdata(pgdata, pgdata_restored) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From f1606282042d8aff66dacb9c2d471eb2b588120e Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Tue, 12 May 2020 17:47:22 +0300 Subject: [PATCH 1306/2107] Add latest release badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e908f273d..ad098872f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) +[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) # pg_probackup From bdff1a749262fe944e4533cd2fd7617226a538a8 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Tue, 12 May 2020 17:47:22 +0300 Subject: [PATCH 1307/2107] Add latest release badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e908f273d..ad098872f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) +[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) # pg_probackup From 81a13b8c03c17bea896403ab38553f529176b587 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 20 May 2020 13:39:11 +0300 Subject: [PATCH 1308/2107] [Issue #205] Do the listing of remote directory on remote agent --- src/backup.c | 20 ++++- src/catalog.c | 1 + src/pg_probackup.h | 3 + src/utils/file.c | 182 +++++++++++++++++++++++++++++++++++++++++++++ src/utils/file.h | 3 +- 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index f67309387..467dc55a9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -331,8 +331,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) backup_files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, 0, FIO_DB_HOST); + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(backup_files_list, instance_config.pgdata, + true, true, false, 0); + else + dir_list_file(backup_files_list, instance_config.pgdata, + true, true, false, 0, FIO_LOCAL_HOST); /* * Get database_map (name to oid) for use in partial restore feature. @@ -345,11 +349,19 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) * from external directory option */ if (external_dirs) + { for (i = 0; i < parray_num(external_dirs); i++) + { /* External dirs numeration starts with 1. * 0 value is not external dir */ - dir_list_file(backup_files_list, parray_get(external_dirs, i), - false, true, false, i+1, FIO_DB_HOST); + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(backup_files_list, parray_get(external_dirs, i), + false, true, false, i+1); + else + dir_list_file(backup_files_list, parray_get(external_dirs, i), + false, true, false, i+1, FIO_LOCAL_HOST); + } + } /* close ssh session in main thread */ fio_disconnect(); diff --git a/src/catalog.c b/src/catalog.c index 599bb4d28..be08b1164 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1772,6 +1772,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* * Save the backup content into BACKUP_CONTROL_FILE. + * TODO: honor the strict flag */ void write_backup(pgBackup *backup, bool strict) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 288cd7efd..75730acfe 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1035,6 +1035,9 @@ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, pgFile *file, char **errormsg); +extern void fio_list_dir(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, int external_dir_num); + /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) diff --git a/src/utils/file.c b/src/utils/file.c index 356e09df3..1b8a8c14d 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -36,6 +36,33 @@ typedef struct } fio_send_request; +typedef struct +{ + char path[MAXPGPATH]; + bool exclude; + bool follow_symlink; + bool add_root; + int external_dir_num; +} fio_list_dir_request; + +typedef struct +{ + mode_t mode; + size_t size; + time_t mtime; + bool is_datafile; + bool is_database; + + Oid tblspcOid; + Oid dbOid; + Oid relOid; + ForkName forkName; + int segno; + int external_dir_num; + int linked_len; +} fio_pgFile; + + /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -2036,6 +2063,157 @@ static void fio_send_file_impl(int out, char const* path) return; } +/* Compile the array of files located on remote machine in directory root */ +void fio_list_dir(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, int external_dir_num) +{ + fio_header hdr; + fio_list_dir_request req; + char *buf = pgut_malloc(CHUNK_SIZE); + + /* Send to the agent message with parameters for directory listing */ + snprintf(req.path, MAXPGPATH, "%s", root); + req.exclude = exclude; + req.follow_symlink = follow_symlink; + req.add_root = add_root; + req.external_dir_num = external_dir_num; + + hdr.cop = FIO_LIST_DIR; + hdr.size = sizeof(req); + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &req, hdr.size), hdr.size); + + for (;;) + { + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.cop == FIO_SEND_FILE_EOF) + { + /* the work is done */ + break; + } + else if (hdr.cop == FIO_SEND_FILE) + { + pgFile *file = NULL; + fio_pgFile fio_file; + + /* receive rel_path */ + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + file = pgFileInit(buf); + + /* receive metainformation */ + IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file)); + + file->mode = fio_file.mode; + file->size = fio_file.size; + file->mtime = fio_file.mtime; + file->is_datafile = fio_file.is_datafile; + file->is_database = fio_file.is_database; + file->tblspcOid = fio_file.tblspcOid; + file->dbOid = fio_file.dbOid; + file->relOid = fio_file.relOid; + file->forkName = fio_file.forkName; + file->segno = fio_file.segno; + file->external_dir_num = fio_file.external_dir_num; + + if (fio_file.linked_len > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, fio_file.linked_len), fio_file.linked_len); + + file->linked = pgut_malloc(fio_file.linked_len); + snprintf(file->linked, fio_file.linked_len, "%s", buf); + } + +// elog(INFO, "Received file: %s, mode: %u, size: %lu, mtime: %lu", +// file->rel_path, file->mode, file->size, file->mtime); + + parray_append(files, file); + } + else + { + /* TODO: fio_disconnect may get assert fail when running after this */ + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + } + } + + pg_free(buf); +} + + +/* + * To get the arrays of files we use the same function dir_list_file(), + * that is used for local backup. + * After that we iterate over arrays and for every file send at least + * two messages to main process: + * 1. rel_path + * 2. metainformation (size, mtime, etc) + * 3. link path (optional) + * + * TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages + */ +static void fio_list_dir_impl(int out, char* buf) +{ + int i; + fio_header hdr; + fio_list_dir_request *req = (fio_list_dir_request*) buf; + parray *file_files = parray_new(); + + /* + * Disable logging into console any messages with exception of ERROR messages, + * to avoid sending messages to main process, because it may screw his FIO message parsing. + */ + instance_config.logger.log_level_console = ERROR; + + dir_list_file(file_files, req->path, req->exclude, req->follow_symlink, + req->add_root, req->external_dir_num, FIO_LOCAL_HOST); + + /* send information about files to the main process */ + for (i = 0; i < parray_num(file_files); i++) + { + fio_pgFile fio_file; + pgFile *file = (pgFile *) parray_get(file_files, i); + + fio_file.mode = file->mode; + fio_file.size = file->size; + fio_file.mtime = file->mtime; + fio_file.is_datafile = file->is_datafile; + fio_file.is_database = file->is_database; + fio_file.tblspcOid = file->tblspcOid; + fio_file.dbOid = file->dbOid; + fio_file.relOid = file->relOid; + fio_file.forkName = file->forkName; + fio_file.segno = file->segno; + fio_file.external_dir_num = file->external_dir_num; + + if (file->linked) + fio_file.linked_len = strlen(file->linked) + 1; + else + fio_file.linked_len = 0; + + hdr.cop = FIO_SEND_FILE; + hdr.size = strlen(file->rel_path) + 1; + + /* send rel_path first */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, file->rel_path, hdr.size), hdr.size); + + /* now send file metainformation */ + IO_CHECK(fio_write_all(out, &fio_file, sizeof(fio_file)), sizeof(fio_file)); + + /* If file is a symlink, then send link path */ + if (file->linked) + IO_CHECK(fio_write_all(out, file->linked, fio_file.linked_len), fio_file.linked_len); + + pgFileFree(file); + } + + parray_free(file_files); + hdr.cop = FIO_SEND_FILE_EOF; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Execute commands at remote host */ void fio_communicate(int in, int out) { @@ -2173,6 +2351,10 @@ void fio_communicate(int in, int out) case FIO_TRUNCATE: /* Truncate file */ SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; + case FIO_LIST_DIR: + // buf contain fio_send_request header and bitmap. + fio_list_dir_impl(out, buf); + break; case FIO_SEND_PAGES: // buf contain fio_send_request header and bitmap. fio_send_pages_impl(out, buf); diff --git a/src/utils/file.h b/src/utils/file.h index 5171dacd5..baf2685e9 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -46,7 +46,8 @@ typedef enum FIO_DISCONNECT, FIO_DISCONNECTED, /* message for compatibility check */ - FIO_AGENT_VERSION + FIO_AGENT_VERSION, + FIO_LIST_DIR } fio_operations; typedef enum From 09a78d29dba0e24ade163a4d557c7094a270d95a Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmin Date: Thu, 21 May 2020 00:27:33 +1000 Subject: [PATCH 1309/2107] issue 188: add test --- tests/validate.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index 1dfdfe9e8..b7bc501d1 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3728,3 +3728,57 @@ def test_partial_validate_include(self): # 716 if (read_len != MAXALIGN(header.compressed_size)) # -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", # 718 blknum, file->path, read_len, header.compressed_size); + + + # @unittest.skip("skip") + def test_not_validate_diffenent_pg_version(self): + """Do not validate backup, if binary is compiled with different PG version""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + control_file = os.path.join( + backup_dir, "backups", "node", backup_id, + "backup.control") + + pg_version = node.major_version + + if pg_version.is_integer(): + pg_version = int(pg_version) + + fake_new_pg_version = pg_version + 1 + + with open(control_file, 'r') as f: + data = f.read(); + + data = data.replace(str(pg_version), str(fake_new_pg_version)) + + with open(control_file, 'w') as f: + f.write(data); + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because validation is forbidden if server version of backup " + "is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup was made with server version", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + From 70b9e152d28460e1f2a5e00f7a7565f8e3f6b2a6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 11:51:56 +0300 Subject: [PATCH 1310/2107] [Issue #205] now fio_dir_list honors the "--backup-pg-log" flag and correctly handles exclusive backups --- src/archive.c | 2 +- src/backup.c | 20 ++++++++++---------- src/catalog.c | 2 +- src/checkdb.c | 4 ++-- src/delete.c | 3 +-- src/dir.c | 13 ++++++++++++- src/merge.c | 2 +- src/pg_probackup.c | 13 +------------ src/pg_probackup.h | 10 +++++----- src/utils/file.c | 33 +++++++++++++++++++-------------- 10 files changed, 53 insertions(+), 49 deletions(-) diff --git a/src/archive.c b/src/archive.c index 1c325e2e1..1055948b6 100644 --- a/src/archive.c +++ b/src/archive.c @@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, /* get list of files from archive_status */ status_files = parray_new(); - dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); + dir_list_file(status_files, archive_status_dir, false, false, false, false, 0, FIO_DB_HOST); parray_qsort(status_files, pgFileCompareName); for (i = 0; i < parray_num(status_files); i++) diff --git a/src/backup.c b/src/backup.c index 467dc55a9..3dfdeee95 100644 --- a/src/backup.c +++ b/src/backup.c @@ -80,7 +80,7 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync); +static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn); @@ -129,7 +129,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) * Move files from 'pgdata' to a subdirectory in 'backup_path'. */ static void -do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) +do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) { int i; char database_path[MAXPGPATH]; @@ -333,10 +333,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* list files with the logical path. omit $PGDATA */ if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, 0); + true, true, false, backup_logs, 0); else dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, 0, FIO_LOCAL_HOST); + true, true, false, backup_logs, 0, FIO_LOCAL_HOST); /* * Get database_map (name to oid) for use in partial restore feature. @@ -356,10 +356,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) * 0 value is not external dir */ if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(backup_files_list, parray_get(external_dirs, i), - false, true, false, i+1); + false, true, false, false, i+1); else dir_list_file(backup_files_list, parray_get(external_dirs, i), - false, true, false, i+1, FIO_LOCAL_HOST); + false, true, false, false, i+1, FIO_LOCAL_HOST); } } @@ -615,7 +615,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0, + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, 0, FIO_BACKUP_HOST); /* TODO: Drop streamed WAL segments greater than stop_lsn */ @@ -801,8 +801,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) * Entry point of pg_probackup BACKUP subcommand. */ int -do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync) +do_backup(time_t start_time, pgSetBackupParams *set_backup_params, + bool no_validate, bool no_sync, bool backup_logs) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -901,7 +901,7 @@ do_backup(time_t start_time, bool no_validate, add_note(¤t, set_backup_params->note); /* backup data */ - do_backup_instance(backup_conn, &nodeInfo, no_sync); + do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ diff --git a/src/catalog.c b/src/catalog.c index be08b1164..167d9fd72 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -878,7 +878,7 @@ catalog_get_timelines(InstanceConfig *instance) /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST); parray_qsort(xlog_files_list, pgFileCompareName); timelineinfos = parray_new(); diff --git a/src/checkdb.c b/src/checkdb.c index 0650b50a0..f8847851c 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -207,8 +207,8 @@ do_block_validation(char *pgdata, uint32 checksum_version) files_list = parray_new(); /* list files with the logical path. omit $PGDATA */ - dir_list_file(files_list, pgdata, - true, true, false, 0, FIO_DB_HOST); + dir_list_file(files_list, pgdata, true, true, + false, false, 0, FIO_DB_HOST); /* * Sort pathname ascending. diff --git a/src/delete.c b/src/delete.c index 5da60da90..5f422c302 100644 --- a/src/delete.c +++ b/src/delete.c @@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); - dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST); + dir_list_file(files, backup->root_dir, false, false, true, false, 0, FIO_LOCAL_HOST); /* delete leaf node first */ parray_qsort(files, pgFileCompareRelPathWithExternalDesc); @@ -966,7 +966,6 @@ do_delete_instance(void) { parray *backup_list; int i; - int rc; char instance_config_path[MAXPGPATH]; diff --git a/src/dir.c b/src/dir.c index 13c8d0cf8..2110e0aab 100644 --- a/src/dir.c +++ b/src/dir.c @@ -513,7 +513,7 @@ db_map_entry_free(void *entry) */ void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, int external_dir_num, fio_location location) + bool add_root, bool backup_logs, int external_dir_num, fio_location location) { pgFile *file; @@ -527,6 +527,17 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink return; } + /* setup exclusion list for file search */ + if (!backup_logs) + { + int i; + + for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ + + /* Set 'pg_log' in first empty slot */ + pgdata_exclude_dir[i] = PG_LOG_DIR; + } + if (!S_ISDIR(file->mode)) { if (external_dir_num > 0) diff --git a/src/merge.c b/src/merge.c index 50f786a06..dd521c319 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1055,7 +1055,7 @@ remove_dir_with_files(const char *path) int i; char full_path[MAXPGPATH]; - dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST); + dir_list_file(files, path, false, false, true, false, 0, FIO_LOCAL_HOST); parray_qsort(files, pgFileCompareRelPathWithExternalDesc); for (i = 0; i < parray_num(files); i++) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3235f1562..9f66c38d5 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -668,17 +668,6 @@ main(int argc, char *argv[]) if (instance_config.conn_opt.pguser != NULL) dbuser = pstrdup(instance_config.conn_opt.pguser); - /* setup exclusion list for file search */ - if (!backup_logs) - { - int i; - - for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ - - /* Set 'pg_log' in first empty slot */ - pgdata_exclude_dir[i] = PG_LOG_DIR; - } - if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) { /* @@ -802,7 +791,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, no_validate, set_backup_params, no_sync); + return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 75730acfe..9c5903654 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -694,8 +694,8 @@ extern char** commands_args; extern const char *pgdata_exclude_dir[]; /* in backup.c */ -extern int do_backup(time_t start_time, bool no_validate, - pgSetBackupParams *set_backup_params, bool no_sync); +extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, + bool no_validate, bool no_sync, bool backup_logs); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -859,7 +859,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, + bool follow_symlink, bool add_root, bool backup_logs, int external_dir_num, fio_location location); extern void create_data_directories(parray *dest_files, @@ -1035,8 +1035,8 @@ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, pgFile *file, char **errormsg); -extern void fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, int external_dir_num); +extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, + bool add_root, bool backup_logs, int external_dir_num); /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) diff --git a/src/utils/file.c b/src/utils/file.c index 1b8a8c14d..052f11be3 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -42,23 +42,24 @@ typedef struct bool exclude; bool follow_symlink; bool add_root; + bool backup_logs; + bool exclusive_backup; int external_dir_num; } fio_list_dir_request; typedef struct { - mode_t mode; - size_t size; + mode_t mode; + size_t size; time_t mtime; - bool is_datafile; - bool is_database; - - Oid tblspcOid; - Oid dbOid; - Oid relOid; + bool is_datafile; + bool is_database; + Oid tblspcOid; + Oid dbOid; + Oid relOid; ForkName forkName; - int segno; - int external_dir_num; + int segno; + int external_dir_num; int linked_len; } fio_pgFile; @@ -2065,7 +2066,7 @@ static void fio_send_file_impl(int out, char const* path) /* Compile the array of files located on remote machine in directory root */ void fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, int external_dir_num) + bool follow_symlink, bool add_root, bool backup_logs, int external_dir_num) { fio_header hdr; fio_list_dir_request req; @@ -2076,6 +2077,8 @@ void fio_list_dir(parray *files, const char *root, bool exclude, req.exclude = exclude; req.follow_symlink = follow_symlink; req.add_root = add_root; + req.backup_logs = backup_logs; + req.exclusive_backup = exclusive_backup; req.external_dir_num = external_dir_num; hdr.cop = FIO_LIST_DIR; @@ -2162,12 +2165,15 @@ static void fio_list_dir_impl(int out, char* buf) /* * Disable logging into console any messages with exception of ERROR messages, - * to avoid sending messages to main process, because it may screw his FIO message parsing. + * because currently we have no mechanism to notify the main process + * about then message been sent. + * TODO: correctly send elog messages from agent to main process. */ instance_config.logger.log_level_console = ERROR; + exclusive_backup = req->exclusive_backup; dir_list_file(file_files, req->path, req->exclude, req->follow_symlink, - req->add_root, req->external_dir_num, FIO_LOCAL_HOST); + req->add_root, req->backup_logs, req->external_dir_num, FIO_LOCAL_HOST); /* send information about files to the main process */ for (i = 0; i < parray_num(file_files); i++) @@ -2352,7 +2358,6 @@ void fio_communicate(int in, int out) SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); break; case FIO_LIST_DIR: - // buf contain fio_send_request header and bitmap. fio_list_dir_impl(out, buf); break; case FIO_SEND_PAGES: From e95919228588c280f3566c371b3ad285c9ba5f93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 13:43:13 +0300 Subject: [PATCH 1311/2107] replace rmtree with pgut_rmtree --- src/archive.c | 2 +- src/delete.c | 2 +- src/pg_probackup.h | 2 + src/utils/pgut.c | 140 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/src/archive.c b/src/archive.c index cbe572298..c885184d2 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1109,7 +1109,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, { /* discard prefetch */ // n_fetched = 0; - rmtree(prefetch_dir, false); + pgut_rmtree(prefetch_dir, false, false); } } else diff --git a/src/delete.c b/src/delete.c index 5da60da90..b5e6fb68c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -986,7 +986,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - rmtree(arclog_path, false); + pgut_rmtree(arclog_path, false, true); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3ce98c4a4..a2d65b885 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1030,6 +1030,8 @@ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonL extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, int thread_num); +extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); + /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index b7599816f..6d996f47f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -43,6 +43,9 @@ static void on_interrupt(void); static void on_cleanup(void); static pqsigfunc oldhandler = NULL; +static char ** pgut_pgfnames(const char *path, bool strict); +static void pgut_pgfnames_cleanup(char **filenames); + void discard_response(PGconn *conn); void @@ -1062,3 +1065,140 @@ discard_response(PGconn *conn) PQclear(res); } while (res); } + +/* + * pgfnames + * + * return a list of the names of objects in the argument directory. Caller + * must call pgfnames_cleanup later to free the memory allocated by this + * function. + */ +char ** +pgut_pgfnames(const char *path, bool strict) +{ + DIR *dir; + struct dirent *file; + char **filenames; + int numnames = 0; + int fnsize = 200; /* enough for many small dbs */ + + dir = opendir(path); + if (dir == NULL) + { + elog(strict ? ERROR : WARNING, "could not open directory \"%s\": %m", path); + return NULL; + } + + filenames = (char **) palloc(fnsize * sizeof(char *)); + + while (errno = 0, (file = readdir(dir)) != NULL) + { + if (strcmp(file->d_name, ".") != 0 && strcmp(file->d_name, "..") != 0) + { + if (numnames + 1 >= fnsize) + { + fnsize *= 2; + filenames = (char **) repalloc(filenames, + fnsize * sizeof(char *)); + } + filenames[numnames++] = pstrdup(file->d_name); + } + } + + if (errno) + { + elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path); + return NULL; + } + + filenames[numnames] = NULL; + + if (closedir(dir)) + { + elog(strict ? ERROR : WARNING, "could not close directory \"%s\": %m", path); + return NULL; + } + + return filenames; +} + +/* + * pgfnames_cleanup + * + * deallocate memory used for filenames + */ +void +pgut_pgfnames_cleanup(char **filenames) +{ + char **fn; + + for (fn = filenames; *fn; fn++) + pfree(*fn); + + pfree(filenames); +} + +/* Shamelessly stolen from commom/rmtree.c */ +bool +pgut_rmtree(const char *path, bool rmtopdir, bool strict) +{ + bool result = true; + char pathbuf[MAXPGPATH]; + char **filenames; + char **filename; + struct stat statbuf; + + /* + * we copy all the names out of the directory before we start modifying + * it. + */ + filenames = pgut_pgfnames(path, strict); + + if (filenames == NULL) + return false; + + /* now we have the names we can start removing things */ + for (filename = filenames; *filename; filename++) + { + snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename); + + if (lstat(pathbuf, &statbuf) != 0) + { + elog(strict ? ERROR : WARNING, "could not stat file or directory \"%s\": %m", pathbuf); + result = false; + break; + } + + if (S_ISDIR(statbuf.st_mode)) + { + /* call ourselves recursively for a directory */ + if (!pgut_rmtree(pathbuf, true, strict)) + { + result = false; + break; + } + } + else + { + if (unlink(pathbuf) != 0) + { + elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", pathbuf); + result = false; + break; + } + } + } + + if (rmtopdir) + { + if (rmdir(path) != 0) + { + elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", path); + result = false; + } + } + + pgut_pgfnames_cleanup(filenames); + + return result; +} From a4e9eff91b58173cd404e3d295f3e9c2e82424c4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 13:44:22 +0300 Subject: [PATCH 1312/2107] tests: added tests.exclude.ExcludeTest.test_exclude_log_dir_1 and tests.exclude.ExcludeTest.test_exclude_log_dir --- tests/exclude.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/exclude.py b/tests/exclude.py index 0402c978d..c9efe22af 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -153,3 +153,85 @@ def test_exclude_unlogged_tables_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_log_dir(self): + """ + check that by default 'log' and 'pg_log' directories are not backed up + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').rstrip() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertFalse(os.path.exists(log_file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_exclude_log_dir_1(self): + """ + check that "--backup-pg-log" works correctly + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream', '--backup-pg-log']) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.exists(log_file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 61b476bb312110ec2621207d5d0114f8df8e7e27 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 15 Apr 2020 19:17:21 +0300 Subject: [PATCH 1313/2107] Correct expected error message in the test_ptrack_pg_resetxlog --- tests/ptrack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 309a3c442..6ca8d93c6 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3841,7 +3841,7 @@ def test_ptrack_zero_changes(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - @unittest.expectedFailure + # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): fname = self.id().split('.')[3] node = self.make_simple_node( @@ -3946,7 +3946,7 @@ def test_ptrack_pg_resetxlog(self): ) except ProbackupException as e: self.assertIn( - 'Insert error message', + 'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) From 583ffaaa307e0861b71ab560140bc43f65ef4fe1 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 22 Apr 2020 19:30:39 +0300 Subject: [PATCH 1314/2107] Teach pg_probackup to work with ptrack 2.1 --- doc/pgprobackup.xml | 10 +++++----- src/backup.c | 2 +- src/ptrack.c | 4 +++- tests/helpers/ptrack_helpers.py | 3 ++- tests/ptrack.py | 13 +++++++------ 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 58d427494..a2161f0c6 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1168,12 +1168,12 @@ CREATE EXTENSION ptrack; - To enable tracking page updates, set ptrack_map_size + To enable tracking page updates, set ptrack.map_size parameter to a positive integer and restart the server. For optimal performance, it is recommended to set - ptrack_map_size to + ptrack.map_size to N / 1024, where N is the size of the PostgreSQL cluster, in MB. If you set this @@ -1181,7 +1181,7 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack_map_size to a higher value + Setting ptrack.map_size to a higher value does not affect PTRACK operation. The maximum allowed value is 1024. @@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - If you change the ptrack_map_size parameter value, + If you change the ptrack.map_size parameter value, the previously created PTRACK map file is cleared, and tracking newly changed blocks starts from scratch. Thus, you have to retake a full backup before taking incremental PTRACK backups after - changing ptrack_map_size. + changing ptrack.map_size. diff --git a/src/backup.c b/src/backup.c index 96b47916d..4ec515a1f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -436,7 +436,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num == 20) + if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, nodeInfo->ptrack_schema, prev_backup_start_lsn); diff --git a/src/ptrack.c b/src/ptrack.c index ee39d23b8..a267f3f40 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -204,6 +204,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = 17; else if (strcmp(ptrack_version_str, "2.0") == 0) nodeInfo->ptrack_version_num = 20; + else if (strcmp(ptrack_version_str, "2.1") == 0) + nodeInfo->ptrack_version_num = 21; else elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", ptrack_version_str); @@ -572,7 +574,7 @@ pg_ptrack_enable2(PGconn *backup_conn) { PGresult *res_db; - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) { diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 52a1ffda5..eec57a3ba 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -366,7 +366,8 @@ def make_simple_node( if ptrack_enable: if node.major_version > 11: - options['ptrack_map_size'] = '128MB' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' else: options['ptrack_enable'] = 'on' diff --git a/tests/ptrack.py b/tests/ptrack.py index 6ca8d93c6..4678708bf 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -269,7 +269,8 @@ def test_ptrack_enable(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '30s', + 'shared_preload_libraries': 'ptrack'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -336,16 +337,16 @@ def test_ptrack_disable(self): # DISABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to 0") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") else: node.safe_psql('postgres', "alter system set ptrack_enable to off") - node.stop() node.slow_start() # ENABLE PTRACK if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack_map_size to '128MB'") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") else: node.safe_psql('postgres', "alter system set ptrack_enable to on") node.stop() @@ -4054,7 +4055,7 @@ def test_corrupt_ptrack_map(self): 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) - self.set_auto_conf(node, {'ptrack_map_size': '0'}) + self.set_auto_conf(node, {'ptrack.map_size': '0'}) node.slow_start() @@ -4082,7 +4083,7 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) - self.set_auto_conf(node, {'ptrack_map_size': '32'}) + self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) node.slow_start() From 667a80edfb9cf5278d6791478ab8035b0529b411 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 29 Apr 2020 20:55:39 +0300 Subject: [PATCH 1315/2107] Adopt ptrack 2.1 public API changes --- src/backup.c | 14 ++++----- src/pg_probackup.h | 10 ++++--- src/ptrack.c | 75 ++++++++++++++++++++++++---------------------- tests/ptrack.py | 2 +- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4ec515a1f..1bcae2c03 100644 --- a/src/backup.c +++ b/src/backup.c @@ -438,8 +438,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) */ if (nodeInfo->ptrack_version_num >= 20) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - prev_backup_start_lsn); + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); else if (nodeInfo->ptrack_version_num == 15 || nodeInfo->ptrack_version_num == 16 || nodeInfo->ptrack_version_num == 17) @@ -884,15 +885,10 @@ do_backup(time_t start_time, bool no_validate, #endif get_ptrack_version(backup_conn, &nodeInfo); -// elog(WARNING, "ptrack_version_num %d", ptrack_version_num); + // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - { - if (nodeInfo.ptrack_version_num >= 20) - nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn); - else - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn); - } + nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 93af776c9..79de41b86 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -999,11 +999,12 @@ extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, - const char *ptrack_schema, XLogRecPtr lsn); + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn); extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn); -extern bool pg_ptrack_enable2(PGconn *backup_conn); +extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, @@ -1011,7 +1012,8 @@ extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, size_t *result_size, PGconn *backup_conn); extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn); +extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn); /* FIO */ extern int fio_send_pages(FILE* in, FILE* out, pgFile *file, XLogRecPtr horizonLsn, diff --git a/src/ptrack.c b/src/ptrack.c index a267f3f40..035bad134 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -217,19 +217,29 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn) +pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; + bool result = false; - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0) + if (ptrack_version_num < 20) { - PQclear(res_db); - return false; + res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; + } + else if (ptrack_version_num == 20) + { + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + } + else + { + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; } + PQclear(res_db); - return true; + return result; } @@ -459,7 +469,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) { char query[128]; - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + res = pgut_execute(backup_conn, query, 0, NULL); } @@ -526,7 +540,10 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, @@ -566,30 +583,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments, * ---------------------------- */ -/* - * Check if ptrack is enabled in target instance - */ -bool -pg_ptrack_enable2(PGconn *backup_conn) -{ - PGresult *res_db; - - res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0) - { - PQclear(res_db); - return false; - } - PQclear(res_db); - return true; -} - /* * Fetch a list of changed files with their ptrack maps. */ parray * -pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn) +pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, + int ptrack_version_num, XLogRecPtr lsn) { PGresult *res; char lsn_buf[17 + 1]; @@ -604,8 +603,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); + if (ptrack_version_num == 20) + sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); + else + sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1", + ptrack_schema); res = pgut_execute(backup_conn, query, 1, (const char **) params); pfree(params[0]); @@ -647,16 +650,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec */ void make_pagemap_from_ptrack_2(parray *files, - PGconn *backup_conn, - const char *ptrack_schema, - XLogRecPtr lsn) + PGconn *backup_conn, + const char *ptrack_schema, + int ptrack_version_num, + XLogRecPtr lsn) { parray *filemaps; int file_i = 0; page_map_entry *dummy_map = NULL; /* Receive all available ptrack bitmaps at once */ - filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn); + filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, + ptrack_version_num, lsn); if (filemaps != NULL) parray_qsort(filemaps, pgFileMapComparePath); diff --git a/tests/ptrack.py b/tests/ptrack.py index 4678708bf..9f9dbac58 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4052,7 +4052,7 @@ def test_corrupt_ptrack_map(self): log_content = f.read() self.assertIn( - 'FATAL: incorrect checksum of file "{0}"'.format(ptrack_map), + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) self.set_auto_conf(node, {'ptrack.map_size': '0'}) From 75f8a1a5852bf4a48545b3bdf5c93b6615497788 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 6 May 2020 15:58:16 +0300 Subject: [PATCH 1316/2107] Skip test_ptrack_get_block for ptrack 2.* --- tests/ptrack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 9f9dbac58..db158081b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -631,6 +631,7 @@ def test_ptrack_get_block(self): node.slow_start() if node.major_version >= 12: + self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") node.safe_psql( "postgres", "CREATE EXTENSION ptrack") From aa336b4e5041a7d5d0ceb7c4e1978b66a859b315 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Thu, 7 May 2020 14:47:20 +0300 Subject: [PATCH 1317/2107] Change a way of setting up python environment --- travis/Dockerfile.in | 6 +++++- travis/run_tests.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index 68f5ffe22..a3c858ee2 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,7 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv +RUN apt-get -yq install python python-pip + +# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +# RUN python2 get-pip.py +RUN python2 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 95cb646c5..1bb3a6fde 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -60,7 +60,7 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -virtualenv pyenv +python2 -m virtualenv pyenv source pyenv/bin/activate pip install testgres==1.8.2 From 23497fb78d2a310e94a90f24af6921d210174d0c Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 8 May 2020 17:42:22 +0300 Subject: [PATCH 1318/2107] Check for ptrack.map_size = -1 for ptrack >= 2.1 --- src/ptrack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ptrack.c b/src/ptrack.c index 035bad134..06c543088 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -235,7 +235,8 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) else { res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; } PQclear(res_db); From 8a6b5af2972f98704c2f054feb9251ba0581c8e7 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 9 May 2020 15:19:36 +0300 Subject: [PATCH 1319/2107] Use pg_ptrack_get_block() only with ptrack <2.0.0 --- src/data.c | 17 ++++++++--------- src/ptrack.c | 5 +++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/data.c b/src/data.c index 1270de4f7..83e95c86a 100644 --- a/src/data.c +++ b/src/data.c @@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg, } } - /* Get page via ptrack interface from PostgreSQL shared buffer. - * We do this in following cases: - * 1. PTRACK backup of 1.x versions - * 2. During backup, regardless of backup mode, of PostgreSQL instance - * with ptrack support we encountered invalid page. + /* + * Get page via ptrack interface from PostgreSQL shared buffer. + * We do this only in the cases of PTRACK 1.x versions backup */ - if ((backup_mode == BACKUP_MODE_DIFF_PTRACK + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && (ptrack_version_num >= 15 && ptrack_version_num < 20)) - || !page_is_valid) { int rc = 0; size_t page_size = 0; @@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg, memcpy(page, ptrack_page, BLCKSZ); pg_free(ptrack_page); - /* UPD: It apprears that is possible to get zeroed page or page with invalid header + /* + * UPD: It apprears that is possible to get zeroed page or page with invalid header * from shared buffer. * Note, that getting page with wrong checksumm from shared buffer is * acceptable. @@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg, from_fullpath, blknum, errormsg); } - /* We must set checksum here, because it is outdated + /* + * We must set checksum here, because it is outdated * in the block recieved from shared buffers. */ if (checksum_version) diff --git a/src/ptrack.c b/src/ptrack.c index 06c543088..0cb59960b 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -526,7 +526,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (arguments->cancel_conn == NULL) arguments->cancel_conn = PQgetCancel(arguments->conn); - //elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); + // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); if (ptrack_version_num < 20) res = pgut_execute_parallel(arguments->conn, @@ -544,7 +544,8 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (ptrack_version_num == 20) sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); else - sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); + elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); + // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, From 0b1ae536be9dee9fe353a78587af2d2e6b6683ba Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sun, 10 May 2020 14:30:29 +0300 Subject: [PATCH 1320/2107] Fix (?) test_ptrack_uncommitted_xact --- tests/ptrack.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index db158081b..7d304d52e 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -413,7 +413,11 @@ def test_ptrack_uncommitted_xact(self): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) + # TODO: what's the point in taking pgdata content, then taking + # backup, and the trying to compare those two? Backup issues a + # checkpoint, so it will modify pgdata with close to 100% chance. + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -427,8 +431,9 @@ def test_ptrack_uncommitted_xact(self): backup_dir, 'node', node_restored, node_restored.data_dir, options=["-j", "4"]) - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) self.set_auto_conf( node_restored, {'port': node_restored.port}) @@ -436,7 +441,8 @@ def test_ptrack_uncommitted_xact(self): node_restored.slow_start() # Physical comparison - self.compare_pgdata(pgdata, pgdata_restored) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) From 4609347796105991b82d79f0ccc6cc706e8936a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 16:47:15 +0300 Subject: [PATCH 1321/2107] DOC: minor change of wording in archive-push and archive-get --- doc/pgprobackup.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a2161f0c6..91ec1253b 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4055,10 +4055,10 @@ pg_probackup archive-push -B backup_dir --instance - To speed up archiving, you can specify the option - to run archive-push on multiple threads. - If you provide the option, WAL files - will be copied in batches of the specified size. + To speed up archiving, you can specify the option + to copy WAL segments in batches of the specified size. + If option is used, then you can also specify + the option to copy the batch of WAL segments on multiple threads. WAL segments copied to the archive are synced to disk unless @@ -4096,10 +4096,10 @@ pg_probackup archive-get -B backup_dir --instance - To speed up recovery, you can specify the option - to run archive-get on multiple threads. - If you provide the option, WAL segments - will be copied in batches of the specified size. + To speed up recovery, you can specify the option + to copy WAL segments in batches of the specified size. + If option is used, then you can also specify + the option to copy the batch of WAL segments on multiple threads. From b799e6d9507a1eed513affa9fa226e59c4c9068e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 19:04:45 +0300 Subject: [PATCH 1322/2107] minor spelling fix --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 1bcae2c03..2e67c3e3f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -192,7 +192,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) { /* try to setup multi-timeline backup chain */ elog(WARNING, "Valid backup on current timeline %u is not found, " - "try to look up on previous timelines", + "trying to look up on previous timelines", current.tli); tli_list = catalog_get_timelines(&instance_config); From 600d146b19f9ab352fb391ae08079d676a8aa7ee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 19:05:20 +0300 Subject: [PATCH 1323/2107] tests: remove tests for page healing via pg_ptrack_get_block --- tests/backup.py | 133 ++++++---------------------------------- tests/delta.py | 157 ------------------------------------------------ 2 files changed, 20 insertions(+), 270 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 176aba619..694c9a448 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -147,10 +147,9 @@ def test_incremental_backup_without_full(self): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -165,10 +164,9 @@ def test_incremental_backup_without_full(self): "without valid full backup.\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: Valid backup on current timeline 1 is not found. " - "Create new FULL backup before an incremental one.", - e.message, + self.assertTrue( + "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -176,6 +174,10 @@ def test_incremental_backup_without_full(self): self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], + "ERROR") + # Clean after yourself self.del_test_dir(module_name, fname) @@ -315,10 +317,8 @@ def test_ptrack_threads_stream(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_1(self): + def test_page_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -380,98 +380,6 @@ def test_page_corruption_heal_via_ptrack_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack_2(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - node.stop() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", '--log-level-console=LOG']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of page " - "corruption in PostgreSQL instance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.remote: - self.assertTrue( - "WARNING: File" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertTrue( - "LOG: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', - "Backup Status should be ERROR") - - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_backup_detect_corruption(self): @@ -495,6 +403,10 @@ def test_backup_detect_corruption(self): "postgres", "create extension ptrack") + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + node.safe_psql( "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, " @@ -529,10 +441,6 @@ def test_backup_detect_corruption(self): node.slow_start() - # self.backup_node( - # backup_dir, 'node', node, - # backup_type="full", options=["-j", "4", "--stream"]) - try: self.backup_node( backup_dir, 'node', node, @@ -608,12 +516,11 @@ def test_backup_detect_corruption(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: page verification failed, ' - 'calculated checksum' in e.message and - 'ERROR: query failed: ERROR: ' - 'invalid page in block 1 of relation' in e.message and - 'ERROR: Data files transferring failed' in e.message, + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/delta.py b/tests/delta.py index 4447b9a81..fdbaf1271 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1050,163 +1050,6 @@ def test_delta_delete(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_delta_corruption_heal_via_ptrack(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", - options=["-j", "4", "--stream", '--log-level-file=verbose']) - - # open log file and check - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertIn('block 1, try to fetch via shared buffer', log_content) - self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) - f.close - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', - "Backup Status should be OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_page_corruption_heal_via_ptrack(self): - """make node, corrupt some page, check that backup failed""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - node.stop() - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream", "--log-level-console=LOG"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of page " - "corruption in PostgreSQL instance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.remote: - self.assertTrue( - "LOG: File" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertTrue( - "WARNING: File" in e.message and - "blknum" in e.message and - "have wrong checksum" in e.message and - "try to fetch via shared buffer" in e.message and - "WARNING: page verification failed, " - "calculated checksum" in e.message and - "ERROR: query failed: " - "ERROR: invalid page in block" in e.message and - "query was: SELECT pg_catalog.pg_ptrack_get_block" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'ERROR', - "Backup Status should be ERROR") - - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_nullified_heap_page_backup(self): """ make node, take full backup, nullify some heap block, From 2f7f677d8681ca6020417fbec10e539b2b066d26 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 20:40:55 +0300 Subject: [PATCH 1324/2107] replace quotes with double quotes --- src/data.c | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/data.c b/src/data.c index 1315ecdb7..25270231e 100644 --- a/src/data.c +++ b/src/data.c @@ -570,7 +570,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* sanity */ if (file->size % BLCKSZ != 0) - elog(WARNING, "File: '%s', invalid file size %zu", from_fullpath, file->size); + elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); /* * Compute expected number of blocks in the file. @@ -609,12 +609,12 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) - elog(ERROR, "Cannot open backup file '%s': %s", + elog(ERROR, "Cannot open backup file \"%s\": %s", to_fullpath, strerror(errno)); /* update file permission */ if (chmod(to_fullpath, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of '%s': %s", to_fullpath, + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); /* @@ -657,22 +657,22 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* check for errors */ if (rc == FILE_MISSING) { - elog(LOG, "File '%s' is not found", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of '%s': %s", + elog(ERROR, "Cannot write block %u of \"%s\": %s", err_blknum, to_fullpath, strerror(errno)); else if (rc == PAGE_CORRUPTION) { if (errmsg) - elog(ERROR, "Corruption detected in file '%s', block %u: %s", + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", from_fullpath, err_blknum, errmsg); else - elog(ERROR, "Corruption detected in file '%s', block %u", + elog(ERROR, "Corruption detected in file \"%s\", block %u", from_fullpath, err_blknum); } /* OPEN_FAILED and READ_FAILED */ @@ -681,14 +681,14 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, if (errmsg) elog(ERROR, "%s", errmsg); else - elog(ERROR, "Failed to open for reading remote file '%s'", from_fullpath); + elog(ERROR, "Failed to open for reading remote file \"%s\"", from_fullpath); } else if (rc == READ_FAILED) { if (errmsg) elog(ERROR, "%s", errmsg); else - elog(ERROR, "Failed to read from remote file '%s'", from_fullpath); + elog(ERROR, "Failed to read from remote file \"%s\"", from_fullpath); } file->read_size = rc * BLCKSZ; @@ -710,16 +710,16 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, { if (missing_ok) { - elog(LOG, "File '%s' is not found", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } else - elog(ERROR, "File '%s' is not found", from_fullpath); + elog(ERROR, "File \"%s\" is not found", from_fullpath); } /* In all other cases throw an error */ - elog(ERROR, "Cannot open file '%s': %s", + elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); } @@ -802,12 +802,12 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* close local input file */ if (in && fclose(in)) - elog(ERROR, "Cannot close the source file '%s': %s", + elog(ERROR, "Cannot close the source file \"%s\": %s", to_fullpath, strerror(errno)); /* close local output file */ if (out && fclose(out)) - elog(ERROR, "Cannot close the backup file '%s': %s", + elog(ERROR, "Cannot close the backup file \"%s\": %s", to_fullpath, strerror(errno)); pg_free(in_buf); @@ -1271,12 +1271,12 @@ backup_non_data_file_internal(const char *from_fullpath, /* open backup file for write */ out = fopen(to_fullpath, PG_BINARY_W); if (out == NULL) - elog(ERROR, "Cannot open destination file '%s': %s", + elog(ERROR, "Cannot open destination file \"%s\": %s", to_fullpath, strerror(errno)); /* update file permission */ if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of '%s': %s", to_fullpath, + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); /* backup remote file */ @@ -1291,21 +1291,21 @@ backup_non_data_file_internal(const char *from_fullpath, /* maybe deleted, it's not error in case of backup */ if (missing_ok) { - elog(LOG, "File '%s' is not found", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } else - elog(ERROR, "File '%s' is not found", from_fullpath); + elog(ERROR, "File \"%s\" is not found", from_fullpath); } else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write to '%s': %s", to_fullpath, strerror(errno)); + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); else if (rc != SEND_OK) { if (errmsg) elog(ERROR, "%s", errmsg); else - elog(ERROR, "Cannot access remote file '%s'", from_fullpath); + elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); } pg_free(errmsg); @@ -1322,15 +1322,15 @@ backup_non_data_file_internal(const char *from_fullpath, { if (missing_ok) { - elog(LOG, "File '%s' is not found", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } else - elog(ERROR, "File '%s' is not found", from_fullpath); + elog(ERROR, "File \"%s\" is not found", from_fullpath); } - elog(ERROR, "Cannot open file '%s': %s", from_fullpath, + elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); } @@ -1347,13 +1347,13 @@ backup_non_data_file_internal(const char *from_fullpath, read_len = fread(buf, 1, CHUNK_SIZE, in); if (ferror(in)) - elog(ERROR, "Cannot read from file '%s': %s", + elog(ERROR, "Cannot read from file \"%s\": %s", from_fullpath, strerror(errno)); if (read_len > 0) { if (fwrite(buf, 1, read_len, out) != read_len) - elog(ERROR, "Cannot write to file '%s': %s", to_fullpath, + elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath, strerror(errno)); /* update CRC */ @@ -1376,10 +1376,10 @@ backup_non_data_file_internal(const char *from_fullpath, FIN_FILE_CRC32(true, file->crc); if (in && fclose(in)) - elog(ERROR, "Cannot close the file '%s': %s", from_fullpath, strerror(errno)); + elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); if (out && fclose(out)) - elog(ERROR, "Cannot close the file '%s': %s", to_fullpath, strerror(errno)); + elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); pg_free(buf); } From ba29f9f6f70c1da1aa61519ee7d8b4e47aef3f4e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 May 2020 21:28:38 +0300 Subject: [PATCH 1325/2107] fix after merge --- src/backup.c | 2 +- src/delete.c | 1 - src/dir.c | 16 ++++++++-------- src/pg_probackup.h | 10 +++++----- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index 8d084fe41..72fd7c264 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2223,7 +2223,7 @@ parse_filelist_filenames(parray *files, const char *root) if (S_ISREG(file->mode) && file->tblspcOid != 0 && file->name && file->name[0]) { - if (file->forkName == INIT) + if (file->forkName == init) { /* * Do not backup files of unlogged relations. diff --git a/src/delete.c b/src/delete.c index b5e6fb68c..20185db5f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -966,7 +966,6 @@ do_delete_instance(void) { parray *backup_list; int i; - int rc; char instance_config_path[MAXPGPATH]; diff --git a/src/dir.c b/src/dir.c index 13c8d0cf8..ea8b56b5f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -709,22 +709,22 @@ dir_check_file(pgFile *file) { /* Auxiliary fork of the relfile */ if (strcmp(fork_name, "vm") == 0) - file->forkName = VM; + file->forkName = vm; else if (strcmp(fork_name, "fsm") == 0) - file->forkName = FSM; + file->forkName = fsm; else if (strcmp(fork_name, "cfm") == 0) - file->forkName = CFM; - - else if (strcmp(fork_name, "init") == 0) - file->forkName = INIT; + file->forkName = cfm; else if (strcmp(fork_name, "ptrack") == 0) - file->forkName = PTRACK; + file->forkName = ptrack; + + else if (strcmp(fork_name, "init") == 0) + file->forkName = init; /* Do not backup ptrack files */ - if (file->forkName == PTRACK) + if (file->forkName == ptrack) return CHECK_FALSE; } else diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1d636493f..5c8f43495 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -120,11 +120,11 @@ typedef enum CompressAlg typedef enum ForkName { - VM, - FSM, - CFM, - INIT, - PTRACK + vm, + fsm, + cfm, + init, + ptrack } ForkName; #define INIT_FILE_CRC32(use_crc32c, crc) \ From eea727db1755fef802f3f8675ccaabe170dc95bc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 May 2020 23:22:54 +0300 Subject: [PATCH 1326/2107] tests: fix tests.backup.BackupTest.test_page_detect_corruption --- tests/backup.py | 75 ++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index d99cfe106..be51dfefc 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -324,7 +324,6 @@ def test_page_detect_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - ptrack_enable=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -356,26 +355,32 @@ def test_page_detect_corruption(self): "postgres", "select pg_relation_filepath('t_heap')").rstrip() - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: f.seek(9000) f.write(b"bla") f.flush() f.close - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) - - # open log file and check - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertIn('block 1, try to fetch via shared buffer', log_content) - self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) - f.close + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) + self.assertEqual( + 1, 0, + "Expecting Error because data file is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Corruption detected in file "{0}", ' + 'block 1: page verification failed, calculated checksum'.format(path), + e.message) - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', - "Backup Status should be OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], + 'ERROR', + "Backup Status should be ERROR") # Clean after yourself self.del_test_dir(module_name, fname) @@ -453,8 +458,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -474,8 +479,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -495,8 +500,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -598,8 +603,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -618,8 +623,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -638,8 +643,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -741,8 +746,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -761,8 +766,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -781,8 +786,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1385,8 +1390,8 @@ def test_drop_rel_during_backup_delta(self): with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() self.assertTrue( - "LOG: File '{0}' is not found".format(absolute_path) in log_content, - "File '{0}' should be deleted but it`s not".format(absolute_path)) + 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) From 8aa5231c9a50c681ee4bf79945b138eb4db200b8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 01:07:29 +0300 Subject: [PATCH 1327/2107] [Issue #201] fix restore of files from versions older than 2.3.0 --- src/data.c | 37 ++++++++++++++++++++++++++++++------- src/merge.c | 15 +++++++++++---- src/pg_probackup.h | 2 +- src/restore.c | 16 +++++++++++++++- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/data.c b/src/data.c index 046ca191c..571c40a03 100644 --- a/src/data.c +++ b/src/data.c @@ -861,14 +861,25 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, * Apply changed blocks to destination file from every backup in parent chain. */ size_t -restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath) +restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, + const char *to_fullpath, bool use_bitmap) { - int i; size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); + int backup_seq = 0; + + // FULL -> INCR -> DEST + // 2 1 0 + if (use_bitmap) + /* start with dest backup */ + backup_seq = 0; + else + /* start with full backup */ + backup_seq = parray_num(parent_chain) - 1; // for (i = parray_num(parent_chain) - 1; i >= 0; i--) - for (i = 0; i < parray_num(parent_chain); i++) +// for (i = 0; i < parray_num(parent_chain); i++) + while (backup_seq >= 0 && backup_seq < parray_num(parent_chain)) { char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; @@ -877,7 +888,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char pgFile **res_file = NULL; pgFile *tmp_file = NULL; - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); + + if (use_bitmap) + backup_seq++; + else + backup_seq--; /* lookup file in intermediate backup */ res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); @@ -925,7 +941,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char total_write_len += restore_data_file_internal(in, out, tmp_file, parse_program_version(backup->program_version), from_fullpath, to_fullpath, dest_file->n_blocks, - &(dest_file)->pagemap); + use_bitmap ? &(dest_file)->pagemap : NULL); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -938,6 +954,11 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char return total_write_len; } +/* Restore block from "in" file to "out" file. + * If "nblocks" is greater than zero, then skip restoring blocks, + * whose position if greater than "nblocks". + * If map is NULL, then page bitmap cannot be used for restore optimization + */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, @@ -1050,7 +1071,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); /* if this page is marked as already restored, then skip it */ - if (datapagemap_is_set(map, blknum)) + if (map && datapagemap_is_set(map, blknum)) { /* skip to the next page */ if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) @@ -1115,7 +1136,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers write_len += BLCKSZ; cur_pos = write_pos + BLCKSZ; /* update current write position */ - datapagemap_add(map, blknum); + if (map) + datapagemap_add(map, blknum); } elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); @@ -1191,6 +1213,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, * full copy of destination file. * Full copy is latest possible destination file with size equal or * greater than zero. + * TODO: rewrite to use parent_link of dest backup. */ for (i = 1; i < parray_num(parent_chain); i++) { diff --git a/src/merge.c b/src/merge.c index dd521c319..bf7dc1fdf 100644 --- a/src/merge.c +++ b/src/merge.c @@ -28,6 +28,7 @@ typedef struct // size_t in_place_merge_bytes; bool compression_match; bool program_version_match; + bool use_bitmap; /* * Return value from the thread. @@ -47,7 +48,7 @@ get_external_index(const char *key, const parray *list); static void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, - pgFile *tmp_file, const char *to_root); + pgFile *tmp_file, const char *to_root, bool use_bitmap); static void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, @@ -439,6 +440,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) *dest_externals = NULL; parray *result_filelist = NULL; + bool use_bitmap = true; // size_t total_in_place_merge_bytes = 0; pthread_t *threads = NULL; @@ -601,6 +603,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (full_externals && dest_externals) reorder_external_dirs(full_backup, full_externals, dest_externals); + if (parse_program_version(dest_backup->program_version) < 20300) + use_bitmap = false; + /* Setup threads */ for (i = 0; i < parray_num(dest_backup->files); i++) { @@ -639,6 +644,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) arg->compression_match = compression_match; arg->program_version_match = program_version_match; + arg->use_bitmap = use_bitmap; /* By default there are some error */ arg->ret = 1; @@ -1028,7 +1034,8 @@ merge_files(void *arg) arguments->full_backup, arguments->dest_backup, dest_file, tmp_file, - arguments->full_database_dir); + arguments->full_database_dir, + arguments->use_bitmap); else merge_non_data_file(arguments->parent_chain, arguments->full_backup, @@ -1128,7 +1135,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir) + const char *full_database_dir, bool use_bitmap) { FILE *out = NULL; char *buffer = pgut_malloc(STDIO_BUFSIZE); @@ -1154,7 +1161,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); /* restore file into temp file */ - tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1); + tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap); fclose(out); pg_free(buffer); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c27c41392..c9a2a5ab4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -935,7 +935,7 @@ extern void backup_non_data_file_internal(const char *from_fullpath, bool missing_ok); extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, - FILE *out, const char *to_fullpath); + FILE *out, const char *to_fullpath, bool use_bitmap); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map); diff --git a/src/restore.c b/src/restore.c index bd339fbd1..10e52b1af 100644 --- a/src/restore.c +++ b/src/restore.c @@ -27,6 +27,7 @@ typedef struct bool skip_external_dirs; const char *to_root; size_t restored_bytes; + bool use_bitmap; /* * Return value from the thread. @@ -501,6 +502,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pthread_t *threads; restore_files_arg *threads_args; bool restore_isok = true; + bool use_bitmap = true; /* fancy reporting */ char pretty_dest_bytes[20]; @@ -560,6 +562,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } + /* If dest backup version is older than 2.3.0, then bitmap optimization + * is impossible to use, because bitmap restore rely on pgFile.n_blocks, + * which is not always available in old backups. + */ + if (parse_program_version(dest_backup->program_version) < 20300) + use_bitmap = false; + /* * Restore dest_backup internal directories. */ @@ -645,6 +654,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, arg->dbOid_exclude_list = dbOid_exclude_list; arg->skip_external_dirs = params->skip_external_dirs; arg->to_root = pgdata_path; + arg->use_bitmap = use_bitmap; threads_args[i].restored_bytes = 0; /* By default there are some error */ threads_args[i].ret = 1; @@ -859,7 +869,8 @@ restore_files(void *arg) setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ arguments->restored_bytes += restore_data_file(arguments->parent_chain, - dest_file, out, to_fullpath); + dest_file, out, to_fullpath, + arguments->use_bitmap); } else { @@ -976,6 +987,9 @@ create_recovery_conf(time_t backup_id, elog(ERROR, "cannot open file \"%s\": %s", path, strerror(errno)); + if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); + #if PG_VERSION_NUM >= 120000 fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n", PROGRAM_VERSION); From e802a2d6f4b16b9b7f4acd2868bde790c826ed7f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 02:09:42 +0300 Subject: [PATCH 1328/2107] minor fixes --- src/archive.c | 4 ++-- src/pg_probackup.h | 3 +-- src/utils/file.c | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/archive.c b/src/archive.c index a712c1021..8f9616f44 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1392,7 +1392,7 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg); if (rc == FILE_MISSING) #endif /* ... failing that, use uncompressed */ @@ -1406,7 +1406,7 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* '.gz.partial' goes first ... */ snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg); if (rc == FILE_MISSING) #endif { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e202c39b3..92d909b85 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1030,8 +1030,7 @@ extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XL int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); /* return codes for fio_send_pages */ -extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, - pgFile *file, char **errormsg); +extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, pgFile *file, char **errormsg); diff --git a/src/utils/file.c b/src/utils/file.c index 052f11be3..1e52c9c15 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1732,8 +1732,7 @@ static void fio_send_pages_impl(int out, char* buf) * ZLIB_ERROR (-5) * REMOTE_ERROR (-6) */ -int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, - pgFile *file, char **errormsg) +int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; From 4dc3ab8901f6805f576a60e9608e90293a6117cb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 May 2020 23:22:54 +0300 Subject: [PATCH 1329/2107] tests: fix tests.backup.BackupTest.test_page_detect_corruption --- tests/backup.py | 75 ++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index d99cfe106..be51dfefc 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -324,7 +324,6 @@ def test_page_detect_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - ptrack_enable=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -356,26 +355,32 @@ def test_page_detect_corruption(self): "postgres", "select pg_relation_filepath('t_heap')").rstrip() - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: f.seek(9000) f.write(b"bla") f.flush() f.close - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) - - # open log file and check - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertIn('block 1, try to fetch via shared buffer', log_content) - self.assertIn('SELECT pg_catalog.pg_ptrack_get_block', log_content) - f.close + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) + self.assertEqual( + 1, 0, + "Expecting Error because data file is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Corruption detected in file "{0}", ' + 'block 1: page verification failed, calculated checksum'.format(path), + e.message) - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'] == 'OK', - "Backup Status should be OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], + 'ERROR', + "Backup Status should be ERROR") # Clean after yourself self.del_test_dir(module_name, fname) @@ -453,8 +458,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -474,8 +479,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -495,8 +500,8 @@ def test_backup_detect_corruption(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page verification failed, calculated checksum".format( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -598,8 +603,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -618,8 +623,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -638,8 +643,8 @@ def test_backup_detect_invalid_block_header(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -741,8 +746,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -761,8 +766,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -781,8 +786,8 @@ def test_backup_detect_missing_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Corruption detected in file '{0}', block 1: " - "page header invalid, pd_lower".format(heap_fullpath), + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1385,8 +1390,8 @@ def test_drop_rel_during_backup_delta(self): with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() self.assertTrue( - "LOG: File '{0}' is not found".format(absolute_path) in log_content, - "File '{0}' should be deleted but it`s not".format(absolute_path)) + 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) From d4a8384cfaa720f6e7dfd2d287a8acc5e3fc7b06 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 03:45:24 +0300 Subject: [PATCH 1330/2107] [Issue #201] use parent link instead of backup list in restore_non_data_file() --- src/data.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/data.c b/src/data.c index 571c40a03..ced1c716c 100644 --- a/src/data.c +++ b/src/data.c @@ -1191,7 +1191,7 @@ size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath) { - int i; +// int i; char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; FILE *in = NULL; @@ -1213,14 +1213,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, * full copy of destination file. * Full copy is latest possible destination file with size equal or * greater than zero. - * TODO: rewrite to use parent_link of dest backup. */ - for (i = 1; i < parray_num(parent_chain); i++) + tmp_backup = dest_backup->parent_backup_link; + while (tmp_backup) { pgFile **res_file = NULL; - tmp_backup = (pgBackup *) parray_get(parent_chain, i); - /* lookup file in intermediate backup */ res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); tmp_file = (res_file) ? *res_file : NULL; @@ -1243,6 +1241,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* Full copy is found */ if (tmp_file->write_size > 0) break; + + tmp_backup = tmp_backup->parent_backup_link; } } @@ -1254,6 +1254,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (!tmp_file) elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); + if (tmp_file->write_size <= 0) + elog(ERROR, "Full copy of non-data file has invalid size. " + "Metadata corruption in backup %s in file: \"%s\"", + base36enc(tmp_backup->start_time), to_fullpath); + if (tmp_file->external_dir_num == 0) join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); else From d67cb67243dae7a93c383a996fa56e90893d5df8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 04:25:19 +0300 Subject: [PATCH 1331/2107] [Issue #201] do not employ page bitmap when restoring a single full backup --- src/restore.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/restore.c b/src/restore.c index 10e52b1af..50440f568 100644 --- a/src/restore.c +++ b/src/restore.c @@ -569,6 +569,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (parse_program_version(dest_backup->program_version) < 20300) use_bitmap = false; + /* There is no point in bitmap restore, when restoring a single FULL backup */ + if (parray_num(parent_chain) == 1) + use_bitmap = false; + /* * Restore dest_backup internal directories. */ From 721a15360b32b7cd862e65e85c4f16400a8c95b3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 13:48:16 +0300 Subject: [PATCH 1332/2107] tests: fix tests.retention.RetentionTest.test_window_merge_interleaved_incremental_chains_1 --- tests/retention.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index e797d3c60..7a2e80a55 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -629,15 +629,15 @@ def test_window_merge_interleaved_incremental_chains_1(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=3) + node.pgbench_init(scale=5) # Take FULL BACKUPs backup_id_a = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() backup_id_b = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() # Change FULL B backup status to ERROR @@ -648,7 +648,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): pgdata_a1 = self.pgdata_content(node.data_dir) - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() # PAGEa1 OK @@ -666,20 +666,20 @@ def test_window_merge_interleaved_incremental_chains_1(self): page_id_b1 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() page_id_b2 = self.backup_node( backup_dir, 'node', node, backup_type='page') - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='page') pgdata_b3 = self.pgdata_content(node.data_dir) - pgbench = node.pgbench(options=['-t', '10', '-c', '2']) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() # PAGEb3 OK From 10ca6356b79ed8b2d91b5891d2c2cbdf87931453 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 15:07:12 +0300 Subject: [PATCH 1333/2107] fix travis --- src/archive.c | 3 ++- src/delete.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/archive.c b/src/archive.c index 8f9616f44..ba77877d1 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1109,7 +1109,8 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, { /* discard prefetch */ // n_fetched = 0; - pgut_rmtree(prefetch_dir, false, false); +// pgut_rmtree(prefetch_dir, false, false); + rmtree(prefetch_dir, false); } } else diff --git a/src/delete.c b/src/delete.c index a6263b63a..5f422c302 100644 --- a/src/delete.c +++ b/src/delete.c @@ -985,7 +985,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - pgut_rmtree(arclog_path, false, true); + rmtree(arclog_path, false); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); From 50dbee34ecdff4a95e85f7d01a213ee9d691a2da Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 23 May 2020 16:47:30 +0300 Subject: [PATCH 1334/2107] Revert "fix travis" This reverts commit 10ca6356b79ed8b2d91b5891d2c2cbdf87931453. --- src/archive.c | 3 +-- src/delete.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/archive.c b/src/archive.c index ba77877d1..8f9616f44 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1109,8 +1109,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, { /* discard prefetch */ // n_fetched = 0; -// pgut_rmtree(prefetch_dir, false, false); - rmtree(prefetch_dir, false); + pgut_rmtree(prefetch_dir, false, false); } } else diff --git a/src/delete.c b/src/delete.c index 5f422c302..a6263b63a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -985,7 +985,7 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - rmtree(arclog_path, false); + pgut_rmtree(arclog_path, false, true); /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); From b696669681a9cdc289ebd7d89dffc6a82f621951 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 May 2020 14:50:01 +0300 Subject: [PATCH 1335/2107] [Issue #66] Incremental restore, initial commit --- src/data.c | 126 +++++++++++++++++++++++++++++++++++++++++-- src/dir.c | 32 +++++++++++ src/merge.c | 2 +- src/pg_probackup.c | 3 ++ src/pg_probackup.h | 21 ++++++-- src/restore.c | 132 ++++++++++++++++++++++++++++++++++++++++----- src/utils/file.c | 71 ++++++++++++++++++++++-- src/utils/file.h | 2 +- 8 files changed, 362 insertions(+), 27 deletions(-) diff --git a/src/data.c b/src/data.c index ced1c716c..1f2fdfcad 100644 --- a/src/data.c +++ b/src/data.c @@ -862,7 +862,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, */ size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, - const char *to_fullpath, bool use_bitmap) + const char *to_fullpath, bool use_bitmap, uint16 *checksum_map) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); @@ -941,7 +941,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, total_write_len += restore_data_file_internal(in, out, tmp_file, parse_program_version(backup->program_version), from_fullpath, to_fullpath, dest_file->n_blocks, - use_bitmap ? &(dest_file)->pagemap : NULL); + use_bitmap ? &(dest_file)->pagemap : NULL, + checksum_map, backup->checksum_version); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -962,7 +963,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map) + datapagemap_t *map, uint16 *checksum_map, int checksum_version) { BackupPageHeader header; BlockNumber blknum = 0; @@ -1100,6 +1101,43 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers is_compressed = true; } + /* Incremental restore */ + if (checksum_map && checksum_map[blknum] != 0) + { + uint16 page_crc = 0; + + if (is_compressed) + { + char uncompressed_buf[BLCKSZ]; + fio_decompress(uncompressed_buf, page.data, compressed_size, file->compress_alg); + + /* If checksumms are enabled, then we can trust checksumm in header */ + if (checksum_version) + page_crc = ((PageHeader) uncompressed_buf)->pd_checksum; + else + page_crc = pg_checksum_page(uncompressed_buf, file->segno * RELSEG_SIZE + blknum); + +// page_crc_1 = pg_checksum_page(uncompressed_buf, file->segno * RELSEG_SIZE + blknum); +// Assert(page_crc == page_crc_1); + } + else + { + /* if checksumms are enabled, then we can trust checksumm in header */ + if (checksum_version) + page_crc = ((PageHeader) page.data)->pd_checksum; + else + page_crc = pg_checksum_page(page.data, file->segno + blknum); + } + + /* the heart of incremental restore */ + if (page_crc == checksum_map[blknum]) + { + if (map) + datapagemap_add(map, blknum); + continue; + } + } + /* * Seek and write the restored page. * When restoring file from FULL backup, pages are written sequentially, @@ -1189,7 +1227,8 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath) + pgFile *dest_file, FILE *out, const char *to_fullpath, + bool already_exists) { // int i; char from_root[MAXPGPATH]; @@ -1259,6 +1298,20 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, "Metadata corruption in backup %s in file: \"%s\"", base36enc(tmp_backup->start_time), to_fullpath); + /* incremental restore */ + if (already_exists) + { + /* compare checksumms of remote and local files */ + pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + + if (file_crc == tmp_file->crc) + { + elog(VERBOSE, "Remote nondata file \"%s\" is unchanged, skip restore", + to_fullpath); + return 0; + } + } + if (tmp_file->external_dir_num == 0) join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); else @@ -1757,3 +1810,68 @@ check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return is_valid; } + +/* read local data file and construct map with block checksums */ +uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) +{ + uint16 *checksum_map = NULL; + FILE *in = NULL; + BlockNumber blknum = 0; + XLogRecPtr page_lsn = 0; + char read_buffer[BLCKSZ]; + char in_buf[STDIO_BUFSIZE]; + + /* truncate up to blocks */ + if (truncate(fullpath, n_blocks * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + n_blocks, fullpath, strerror(errno)); + + /* open file */ + in = fopen(fullpath, PG_BINARY_R); + if (!in) + elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + + /* initialize array of checksums */ + checksum_map = pgut_malloc(n_blocks * sizeof(uint16)); + memset(checksum_map, 0, n_blocks * sizeof(uint16)); + + for (blknum = 0; blknum < n_blocks; blknum++) + { + size_t read_len = fread(read_buffer, 1, BLCKSZ, in); + page_lsn = InvalidXLogRecPtr; + + /* report error */ + if (ferror(in)) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + blknum, fullpath, strerror(errno)); + + if (read_len == BLCKSZ) + { + int rc = validate_one_page(read_buffer, segmentno + blknum, + dest_stop_lsn, &page_lsn, checksum_version); + + if (rc == PAGE_IS_VALID) + { + if (checksum_version) + checksum_map[blknum] = ((PageHeader) read_buffer)->pd_checksum; + else + checksum_map[blknum] = pg_checksum_page(read_buffer, segmentno + blknum); + } + } + else + elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); + + if (feof(in)) + break; + + if (interrupted) + elog(ERROR, "Interrupted during page reading"); + } + + if (in) + fclose(in); + + return checksum_map; +} diff --git a/src/dir.c b/src/dir.c index e21ac9cb6..fdfe4d1b0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -248,6 +248,38 @@ pgFileDelete(pgFile *file, const char *full_path) } } +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +fio_pgFileDelete(pgFile *file, const char *full_path) +{ + if (S_ISDIR(file->mode)) + { + if (fio_unlink(full_path, FIO_DB_HOST) == -1) + { + if (errno == ENOENT) + return; + else if (errno == ENOTDIR) /* could be symbolic link */ + goto delete_file; + + elog(ERROR, "Cannot remove directory \"%s\": %s", + full_path, strerror(errno)); + } + return; + } + +delete_file: + if (fio_unlink(full_path, FIO_DB_HOST) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "Cannot remove file \"%s\": %s", full_path, + strerror(errno)); + } +} + /* * Read the local file to compute its CRC. * We cannot make decision about file decompression because diff --git a/src/merge.c b/src/merge.c index bf7dc1fdf..40339646b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1161,7 +1161,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); /* restore file into temp file */ - tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap); + tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap, NULL); fclose(out); pg_free(buffer); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9f66c38d5..e3ba92740 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -98,6 +98,7 @@ static pgRestoreParams *restore_params = NULL; time_t current_time = 0; bool restore_as_replica = false; bool no_validate = false; +bool incremental_restore = false; bool skip_block_validation = false; bool skip_external_dirs = false; @@ -203,6 +204,7 @@ static ConfigOption cmd_options[] = { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, + { 'b', 161, "incremental", &incremental_restore, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -703,6 +705,7 @@ main(int argc, char *argv[]) restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; restore_params->primary_conninfo = primary_conninfo; + restore_params->incremental = incremental_restore; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a0600c220..9c9b3e21f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -180,7 +180,7 @@ typedef struct pgFile Oid relOid; /* relOid extracted from path, if applicable */ ForkName forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ - BlockNumber n_blocks; /* size of the data file in blocks */ + int n_blocks; /* size of the data file in blocks */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; /* Flag used strictly by ptrack 1.x backup */ int external_dir_num; /* Number of external directory. 0 if not external */ @@ -441,6 +441,7 @@ typedef struct pgRestoreParams bool restore_as_replica; bool skip_external_dirs; bool skip_block_validation; //Start using it + bool incremental; const char *restore_command; const char *primary_slot_name; @@ -902,6 +903,7 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, fio_location location); extern pgFile *pgFileInit(const char *rel_path); extern void pgFileDelete(pgFile *file, const char *full_path); +extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); @@ -934,18 +936,22 @@ extern void backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok); -extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, - FILE *out, const char *to_fullpath, bool use_bitmap); +extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, + const char *to_fullpath, bool use_bitmap, uint16 *checksum_map); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map); + datapagemap_t *map, uint16 *checksum_map, int checksum_version); extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath); + pgFile *dest_file, FILE *out, const char *to_fullpath, + bool already_exists); extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, const char *from_fullpath, const char *to_fullpath); extern bool create_empty_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); +extern uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); + extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ @@ -1040,6 +1046,11 @@ extern void fio_list_dir(parray *files, const char *root, bool exclude, bool fol extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); +extern uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); + +extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg); + /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) diff --git a/src/restore.c b/src/restore.c index 50440f568..10a742c0c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -19,6 +19,7 @@ typedef struct { + parray *pgdata_files; parray *dest_files; pgBackup *dest_backup; parray *dest_external_dirs; @@ -28,6 +29,7 @@ typedef struct const char *to_root; size_t restored_bytes; bool use_bitmap; + bool incremental; /* * Return value from the thread. @@ -117,8 +119,24 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, "required parameter not specified: PGDATA (-D, --pgdata)"); /* Check if restore destination empty */ if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) - elog(ERROR, "restore destination is not empty: \"%s\"", - instance_config.pgdata); + { + // TODO: check that remote systemd id is the same as ours. + // TODO: check that remote system is NOT running, check pg_control and pid. + if (params->incremental) + { + elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", + instance_config.pgdata); + } + else + elog(ERROR, "Restore destination is not empty: \"%s\"", + instance_config.pgdata); + } + else + { + /* if remote directory is empty then disable incremental restore */ + if (params->incremental) + params->incremental = false; + } } if (instance_name == NULL) @@ -496,6 +514,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { int i; char timestamp[100]; + parray *pgdata_files = NULL; parray *dest_files = NULL; parray *external_dirs = NULL; /* arrays with meta info for multi threaded backup */ @@ -567,9 +586,15 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * which is not always available in old backups. */ if (parse_program_version(dest_backup->program_version) < 20300) + { use_bitmap = false; + if (params->incremental) + elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version"); + } + /* There is no point in bitmap restore, when restoring a single FULL backup */ + // TODO: if there is lsn-based incremental restore, then bitmap is mandatory if (parray_num(parent_chain) == 1) use_bitmap = false; @@ -627,6 +652,46 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_atomic_clear_flag(&file->lock); } + // Get list of files in destination directory and remove redundant files. + if (params->incremental) + { + pgdata_files = parray_new(); + + elog(INFO, "Extracting the content of destination directory for incremental restore"); + + /* TODO: external directorues */ + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, 0); + else + dir_list_file(pgdata_files, pgdata_path, + false, true, false, false, 0, FIO_LOCAL_HOST); + + parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc); + elog(INFO, "Destination directory content extracted, time elapsed:"); + + elog(INFO, "Removing redundant files"); + for (i = 0; i < parray_num(pgdata_files); i++) + { + pgFile *file = (pgFile *) parray_get(pgdata_files, i); + + /* if file does not exists in destination list, then we can safely unlink it */ + if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal) == NULL) + { + char full_file_path[MAXPGPATH]; + + join_path_components(full_file_path, pgdata_path, file->rel_path); + + fio_pgFileDelete(file, full_file_path); + elog(WARNING, "Deleted remote file \"%s\"", full_file_path); + } + } + + elog(INFO, "Redundant files are removed, time elapsed:"); + +// use_bitmap = true; + /* At this point PDATA do not contain files, that also do not exists in backup filelist */ + } + /* * Close ssh connection belonging to the main thread * to avoid the possibility of been killed for idleness @@ -652,6 +717,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, restore_files_arg *arg = &(threads_args[i]); arg->dest_files = dest_files; + arg->pgdata_files = pgdata_files; arg->dest_backup = dest_backup; arg->dest_external_dirs = external_dirs; arg->parent_chain = parent_chain; @@ -659,6 +725,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, arg->skip_external_dirs = params->skip_external_dirs; arg->to_root = pgdata_path; arg->use_bitmap = use_bitmap; + arg->incremental = params->incremental; threads_args[i].restored_bytes = 0; /* By default there are some error */ threads_args[i].ret = 1; @@ -750,6 +817,12 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (external_dirs != NULL) free_dir_list(external_dirs); + if (pgdata_files) + { + parray_walk(pgdata_files, pgFileFree); + parray_free(pgdata_files); + } + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -766,15 +839,20 @@ static void * restore_files(void *arg) { int i; + uint64 n_files; char to_fullpath[MAXPGPATH]; FILE *out = NULL; char *out_buf = pgut_malloc(STDIO_BUFSIZE); restore_files_arg *arguments = (restore_files_arg *) arg; + n_files = (unsigned long) parray_num(arguments->dest_files); + for (i = 0; i < parray_num(arguments->dest_files); i++) { - pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); + bool already_exists = false; + uint16 *checksum_map = NULL; /* it should take 512kB at most */ + pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ if (S_ISDIR(dest_file->mode)) @@ -789,8 +867,7 @@ restore_files(void *arg) if (progress) elog(INFO, "Progress: (%d/%lu). Restore file \"%s\"", - i + 1, (unsigned long) parray_num(arguments->dest_files), - dest_file->rel_path); + i + 1, n_files, dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) @@ -842,14 +919,40 @@ restore_files(void *arg) join_path_components(to_fullpath, external_path, dest_file->rel_path); } + if (arguments->incremental && + parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc)) + already_exists = true; + + /* + * Handle incremental restore case for data files. + * If file is already exists in pgdata, then + * we scan it block by block and get + * array of checksums for every page. + */ + if (already_exists && + dest_file->is_datafile && !dest_file->is_cfs && + dest_file->n_blocks > 0) + { + /* remote mode */ + if (fio_is_remote(FIO_DB_HOST)) + checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, + dest_file->n_blocks, arguments->dest_backup->stop_lsn, + dest_file->segno * RELSEG_SIZE); + /* local mode */ + else + checksum_map = get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, + dest_file->n_blocks, arguments->dest_backup->stop_lsn, + dest_file->segno * RELSEG_SIZE); + } + /* open destination file */ - out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); + if (already_exists) + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + else + out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); if (out == NULL) - { - int errno_tmp = errno; elog(ERROR, "Cannot open restore target file \"%s\": %s", - to_fullpath, strerror(errno_tmp)); - } + to_fullpath, strerror(errno)); /* update file permission */ if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) @@ -873,8 +976,8 @@ restore_files(void *arg) setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ arguments->restored_bytes += restore_data_file(arguments->parent_chain, - dest_file, out, to_fullpath, - arguments->use_bitmap); + dest_file, out, to_fullpath, + arguments->use_bitmap, checksum_map); } else { @@ -883,7 +986,8 @@ restore_files(void *arg) setvbuf(out, NULL, _IONBF, BUFSIZ); /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, - arguments->dest_backup, dest_file, out, to_fullpath); + arguments->dest_backup, dest_file, out, to_fullpath, + already_exists); } /* free pagemap used for restore optimization */ @@ -894,6 +998,8 @@ restore_files(void *arg) if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, strerror(errno)); + + pg_free(checksum_map); } free(out_buf); diff --git a/src/utils/file.c b/src/utils/file.c index 1e52c9c15..31887d4b6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -63,6 +63,14 @@ typedef struct int linked_len; } fio_pgFile; +typedef struct +{ + BlockNumber n_blocks; + BlockNumber segmentno; + XLogRecPtr stop_lsn; + uint32 checksumVersion; +} fio_checksum_map_request; + /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -423,7 +431,7 @@ fio_disconnect(void) hdr.size = 0; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_DISCONNECTED); + Assert(hdr.cop == FIO_SEND_FILE_EOF); SYS_CHECK(close(fio_stdin)); SYS_CHECK(close(fio_stdout)); fio_stdin = 0; @@ -678,7 +686,7 @@ ssize_t fio_write(int fd, void const* buf, size_t size) } } -static int32 +int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg) { const char *errormsg = NULL; @@ -2219,6 +2227,59 @@ static void fio_list_dir_impl(int out, char* buf) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } +uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) +{ + + fio_header hdr; + fio_checksum_map_request req_hdr; + uint16 *checksum_map = NULL; + size_t path_len = strlen(fullpath) + 1; + + req_hdr.n_blocks = n_blocks; + req_hdr.segmentno = segmentno; + req_hdr.stop_lsn = dest_stop_lsn; + req_hdr.checksumVersion = checksum_version; + + hdr.cop = FIO_GET_CHECKSUM_MAP; + hdr.size = sizeof(req_hdr) + path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + checksum_map = pgut_malloc(hdr.size * sizeof(uint16)); + IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(uint16)), hdr.size * sizeof(uint16)); + } + + return checksum_map; +} + +/* TODO: sent n_blocks to truncate file before reading */ +static void fio_get_checksum_map_impl(int out, char *buf) +{ + fio_header hdr; + uint16 *checksum_map = NULL; + char *fullpath = (char*) buf + sizeof(fio_checksum_map_request); + fio_checksum_map_request *req = (fio_checksum_map_request*) buf; + + checksum_map = get_checksum_map(fullpath, req->checksumVersion, + req->n_blocks, req->stop_lsn, req->segmentno); + hdr.size = req->n_blocks; + + /* send arrays of checksums to main process */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > 0) + IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(uint16)), hdr.size * sizeof(uint16)); + + pg_free(checksum_map); +} + /* Execute commands at remote host */ void fio_communicate(int in, int out) { @@ -2390,8 +2451,12 @@ void fio_communicate(int in, int out) crc = pgFileGetCRC(buf, true, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; + case FIO_GET_CHECKSUM_MAP: + /* calculate crc32 for a file */ + fio_get_checksum_map_impl(out, buf); + break; case FIO_DISCONNECT: - hdr.cop = FIO_DISCONNECTED; + hdr.cop = FIO_SEND_FILE_EOF; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; default: diff --git a/src/utils/file.h b/src/utils/file.h index baf2685e9..cf6176418 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -35,6 +35,7 @@ typedef enum FIO_PAGE, FIO_WRITE_COMPRESSED, FIO_GET_CRC32, + FIO_GET_CHECKSUM_MAP, /* used in fio_send_pages */ FIO_SEND_PAGES, FIO_ERROR, @@ -44,7 +45,6 @@ typedef enum FIO_SEND_FILE_CORRUPTION, /* messages for closing connection */ FIO_DISCONNECT, - FIO_DISCONNECTED, /* message for compatibility check */ FIO_AGENT_VERSION, FIO_LIST_DIR From b72089e11e81984b871efa53c851acf68c69cc4f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 May 2020 18:32:49 +0300 Subject: [PATCH 1336/2107] tests: added tests.compatibility.CompatibilityTest.test_hidden_files --- tests/compatibility.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index 65114c78c..5201c0d86 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -958,3 +958,33 @@ def test_page_vacuum_truncate_compression(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_hidden_files(self): + """ + old_version should be < 2.3.0 + Create hidden file in pgdata, take backup + with old binary, then try to delete backup + with new binary + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close() + + backup_id = self.backup_node( + backup_dir, 'node',node, old_binary=True, options=['--compress']) + + self.delete_pb(backup_dir, 'node', backup_id) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 205ca274488ae5aabee17515af0ba033504d7ce5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 May 2020 18:40:13 +0300 Subject: [PATCH 1337/2107] [Issue #217] test coverage added --- tests/compatibility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 5201c0d86..f2e208e44 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -965,7 +965,7 @@ def test_hidden_files(self): old_version should be < 2.3.0 Create hidden file in pgdata, take backup with old binary, then try to delete backup - with new binary + with new binary """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -982,7 +982,7 @@ def test_hidden_files(self): open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close() backup_id = self.backup_node( - backup_dir, 'node',node, old_binary=True, options=['--compress']) + backup_dir, 'node',node, old_binary=True, options=['--stream']) self.delete_pb(backup_dir, 'node', backup_id) From 79d26cdea4c6fd7d09add395466b1c317cb2c9b2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 May 2020 18:42:35 +0300 Subject: [PATCH 1338/2107] [Issue #217] delete of backup with hidden files works correctly now --- src/archive.c | 2 +- src/backup.c | 6 +++--- src/catalog.c | 2 +- src/checkdb.c | 2 +- src/delete.c | 4 ++-- src/dir.c | 12 ++++++------ src/merge.c | 2 +- src/pg_probackup.h | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/archive.c b/src/archive.c index a063b07ed..ec23f7412 100644 --- a/src/archive.c +++ b/src/archive.c @@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, /* get list of files from archive_status */ status_files = parray_new(); - dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST); + dir_list_file(status_files, archive_status_dir, false, false, false, true, 0, FIO_DB_HOST); parray_qsort(status_files, pgFileComparePath); for (i = 0; i < parray_num(status_files); i++) diff --git a/src/backup.c b/src/backup.c index 2e67c3e3f..19742090a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -333,7 +333,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* list files with the logical path. omit $PGDATA */ dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, 0, FIO_DB_HOST); + true, true, false, true, 0, FIO_DB_HOST); /* * Get database_map (name to oid) for use in partial restore feature. @@ -350,7 +350,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* External dirs numeration starts with 1. * 0 value is not external dir */ dir_list_file(backup_files_list, parray_get(external_dirs, i), - false, true, false, i+1, FIO_DB_HOST); + false, true, false, true, i+1, FIO_DB_HOST); /* close ssh session in main thread */ fio_disconnect(); @@ -627,7 +627,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* Scan backup PG_XLOG_DIR */ xlog_files_list = parray_new(); join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0, + dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, true, 0, FIO_BACKUP_HOST); /* TODO: Drop streamed WAL segments greater than stop_lsn */ diff --git a/src/catalog.c b/src/catalog.c index aedd01fbb..20b0fca32 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -872,7 +872,7 @@ catalog_get_timelines(InstanceConfig *instance) /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, arclog_path, false, false, false, true, 0, FIO_BACKUP_HOST); parray_qsort(xlog_files_list, pgFileComparePath); timelineinfos = parray_new(); diff --git a/src/checkdb.c b/src/checkdb.c index 8ae4255cf..51e284964 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -208,7 +208,7 @@ do_block_validation(char *pgdata, uint32 checksum_version) /* list files with the logical path. omit $PGDATA */ dir_list_file(files_list, pgdata, - true, true, false, 0, FIO_DB_HOST); + true, true, false, true, 0, FIO_DB_HOST); /* * Sort pathname ascending. diff --git a/src/delete.c b/src/delete.c index be7dba9d4..d8e4003ae 100644 --- a/src/delete.c +++ b/src/delete.c @@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup) /* list files to be deleted */ files = parray_new(); - dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST); + dir_list_file(files, backup->root_dir, false, true, true, false, 0, FIO_BACKUP_HOST); /* delete leaf node first */ parray_qsort(files, pgFileComparePathDesc); @@ -984,7 +984,7 @@ do_delete_instance(void) /* Delete all wal files. */ xlog_files_list = parray_new(); - dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST); for (i = 0; i < parray_num(xlog_files_list); i++) { diff --git a/src/dir.c b/src/dir.c index 33b140721..a7c42c2df 100644 --- a/src/dir.c +++ b/src/dir.c @@ -125,7 +125,7 @@ static int pgCompareString(const void *str1, const void *str2); static char dir_check_file(pgFile *file); static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, + bool follow_symlink, bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -575,7 +575,7 @@ db_map_entry_free(void *entry) */ void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, int external_dir_num, fio_location location) + bool add_root, bool skip_hidden, int external_dir_num, fio_location location) { pgFile *file; @@ -601,7 +601,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink if (add_root) parray_append(files, file); - dir_list_file_internal(files, file, exclude, follow_symlink, + dir_list_file_internal(files, file, exclude, follow_symlink, skip_hidden, external_dir_num, location); if (!add_root) @@ -821,7 +821,7 @@ dir_check_file(pgFile *file) */ static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude, - bool follow_symlink, + bool follow_symlink, bool skip_hidden, int external_dir_num, fio_location location) { DIR *dir; @@ -868,7 +868,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, } /* skip hidden files and directories */ - if (file->name[0] == '.') + if (skip_hidden && file->name[0] == '.') { elog(WARNING, "Skip hidden file: '%s'", file->path); pgFileFree(file); @@ -911,7 +911,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, file, exclude, follow_symlink, - external_dir_num, location); + skip_hidden, external_dir_num, location); } if (errno && errno != ENOENT) diff --git a/src/merge.c b/src/merge.c index dd30ef5d9..7a72ac93e 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1055,7 +1055,7 @@ remove_dir_with_files(const char *path) int i; char full_path[MAXPGPATH]; - dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST); + dir_list_file(files, path, true, true, true, false, 0, FIO_LOCAL_HOST); parray_qsort(files, pgFileComparePathDesc); for (i = 0; i < parray_num(files); i++) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 79de41b86..3cd965273 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -845,7 +845,7 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, + bool follow_symlink, bool add_root, bool skip_hidden, int external_dir_num, fio_location location); extern void create_data_directories(parray *dest_files, From a3b8823ecc2f5cf1ec4dc0d6a2d23c77e05bc31f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 25 May 2020 19:36:13 +0300 Subject: [PATCH 1339/2107] [Issue #212] Fix logging into file for local archive-push and archive-get --- src/pg_probackup.c | 15 ++++++++++++--- src/pg_probackup.h | 1 + src/utils/file.c | 9 +++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3235f1562..4b758d9d8 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -561,9 +561,18 @@ main(int argc, char *argv[]) setMyLocation(); } - /* disable logging into file for archive-push and archive-get */ - if (backup_subcmd == ARCHIVE_GET_CMD || - backup_subcmd == ARCHIVE_PUSH_CMD) + /* + * Disable logging into file for archive-push and archive-get. + * Note, that we should NOT use fio_is_remote() here, + * because it will launch ssh connection and we do not + * want it, because it will kill archive-get prefetch + * performance. + * + * TODO: make logging into file possible via ssh + */ + if (fio_is_remote_simple(FIO_BACKUP_HOST) && + (backup_subcmd == ARCHIVE_GET_CMD || + backup_subcmd == ARCHIVE_PUSH_CMD)) { instance_config.logger.log_level_file = LOG_OFF; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3cd965273..67b2d6a96 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1036,6 +1036,7 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL /* Check if specified location is local for current node */ extern bool fio_is_remote(fio_location location); +extern bool fio_is_remote_simple(fio_location location); extern void get_header_errormsg(Page page, char **errormsg); extern void get_checksum_errormsg(Page page, char **errormsg, diff --git a/src/utils/file.c b/src/utils/file.c index e19754080..36bdd6177 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -147,6 +147,15 @@ bool fio_is_remote(fio_location location) return is_remote; } +/* Check if specified location is local for current node */ +bool fio_is_remote_simple(fio_location location) +{ + bool is_remote = MyLocation != FIO_LOCAL_HOST + && location != FIO_LOCAL_HOST + && location != MyLocation; + return is_remote; +} + /* Try to read specified amount of bytes unless error or EOF are encountered */ static ssize_t fio_read_all(int fd, void* buf, size_t size) { From d6ea475e4b7040a551cf1e20facd82b1fee91042 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 May 2020 00:23:58 +0300 Subject: [PATCH 1340/2107] Version 2.3.4 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 67b2d6a96..3e9fd11c0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.3" -#define AGENT_PROTOCOL_VERSION 20303 +#define PROGRAM_VERSION "2.3.4" +#define AGENT_PROTOCOL_VERSION 20304 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index ce2332584..b271f8264 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.3 \ No newline at end of file +pg_probackup 2.3.4 \ No newline at end of file From 963f20f477e5f17faf5b31f87c683a9e832374da Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 May 2020 18:06:52 +0300 Subject: [PATCH 1341/2107] Readme: update version to 2.3.4 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad098872f..6f95b1dae 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.3) +[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4) ## Documentation From 79c8ea3dabd114bcc5d8dbe440cacaf632441e9d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 May 2020 01:31:55 +0300 Subject: [PATCH 1342/2107] fix fio_list_dir --- src/restore.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index 10a742c0c..d3062a962 100644 --- a/src/restore.c +++ b/src/restore.c @@ -661,10 +661,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, /* TODO: external directorues */ if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, 0); + fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); else dir_list_file(pgdata_files, pgdata_path, - false, true, false, false, 0, FIO_LOCAL_HOST); + false, true, false, false, true, 0, FIO_LOCAL_HOST); parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc); elog(INFO, "Destination directory content extracted, time elapsed:"); @@ -933,6 +933,7 @@ restore_files(void *arg) dest_file->is_datafile && !dest_file->is_cfs && dest_file->n_blocks > 0) { + elog(INFO, "HELLO"); /* remote mode */ if (fio_is_remote(FIO_DB_HOST)) checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, From 87252a82eb89c7e26dad69474e239d170e27297b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 May 2020 01:30:57 +0300 Subject: [PATCH 1343/2107] [Issue #66] add sanity checks for incremental restore --- src/catalog.c | 2 +- src/dir.c | 33 +++++++++--- src/merge.c | 2 +- src/pg_probackup.h | 5 +- src/restore.c | 122 +++++++++++++++++++++++++++++++++++++++------ src/util.c | 47 +++++++++++++++++ src/utils/file.c | 57 +++++++++++++++++++-- src/utils/file.h | 10 ++-- 8 files changed, 246 insertions(+), 32 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index edc5e68fe..7b3dfd861 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1902,7 +1902,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (file->linked) len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); - if (file->n_blocks != BLOCKNUM_INVALID) + if (file->n_blocks > 0) len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); sprintf(line+len, "}\n"); diff --git a/src/dir.c b/src/dir.c index 61876a41c..bf5500300 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1021,7 +1021,7 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) */ void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, - bool extract_tablespaces, fio_location location) + bool extract_tablespaces, bool incremental, fio_location location) { int i; parray *links = NULL; @@ -1126,7 +1126,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba fio_mkdir(linked_path, pg_tablespace_mode, location); /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, location) < 0) + if (fio_symlink(linked_path, to_path, incremental, location) < 0) elog(ERROR, "Could not create symbolic link \"%s\": %s", to_path, strerror(errno)); @@ -1203,13 +1203,18 @@ read_tablespace_map(parray *files, const char *backup_dir) /* * Check that all tablespace mapping entries have correct linked directory - * paths. Linked directories must be empty or do not exist. + * paths. Linked directories must be empty or do not exist, unless + * we are running incremental restore, then linked directories can be nonempty. * * If tablespace-mapping option is supplied, all OLDDIR entries must have * entries in tablespace_map file. + * + * When running incremental restore with tablespace remapping, then + * new tablespace directory MUST be empty, because there is no + * we can be sure, that files laying there belong to our instance. */ void -check_tablespace_mapping(pgBackup *backup) +check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty) { // char this_backup_path[MAXPGPATH]; parray *links; @@ -1236,6 +1241,18 @@ check_tablespace_mapping(pgBackup *backup) elog(ERROR, "--tablespace-mapping option's old directory " "doesn't have an entry in tablespace_map file: \"%s\"", cell->old_dir); + + /* For incremental restore, check that new directory is empty */ + if (incremental) + { + if (!is_absolute_path(cell->new_dir)) + elog(ERROR, "tablespace directory is not an absolute path: %s\n", + cell->new_dir); + + if (!dir_is_empty(cell->new_dir, FIO_DB_HOST)) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + cell->new_dir); + } } /* 2 - all linked directories must be empty */ @@ -1257,8 +1274,12 @@ check_tablespace_mapping(pgBackup *backup) linked_path); if (!dir_is_empty(linked_path, FIO_DB_HOST)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); + { + if (!incremental) + elog(ERROR, "restore tablespace destination is not empty: \"%s\"", + linked_path); + *tblspaces_are_empty = false; + } } free(tmp_file); diff --git a/src/merge.c b/src/merge.c index d4d155752..457fab15c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -589,7 +589,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* Create directories */ create_data_directories(dest_backup->files, full_database_dir, - dest_backup->root_dir, false, FIO_BACKUP_HOST); + dest_backup->root_dir, false, false, FIO_BACKUP_HOST); /* External directories stuff */ if (dest_backup->external_dir_str) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b719de586..594179569 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -867,12 +867,13 @@ extern void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, bool extract_tablespaces, + bool incremental, fio_location location); extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); -extern void check_tablespace_mapping(pgBackup *backup); +extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty); extern void check_external_dir_mapping(pgBackup *backup); extern char *get_external_remap(char *current_dir); @@ -951,6 +952,7 @@ extern bool create_empty_file(fio_location from_location, const char *to_root, extern uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); +extern pid_t check_postmaster(const char *pgdata); extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); @@ -1048,6 +1050,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); extern uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); +extern pid_t fio_check_postmaster(const char *pgdata); extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg); diff --git a/src/restore.c b/src/restore.c index d3062a962..5993a3962 100644 --- a/src/restore.c +++ b/src/restore.c @@ -49,6 +49,8 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync); +static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier, + bool lsn_based); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -111,6 +113,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, char *action = params->is_restore ? "Restore":"Validate"; parray *parent_chain = NULL; parray *dbOid_exclude_list = NULL; + bool pgdata_is_empty = true; + bool tblspaces_are_empty = true; if (params->is_restore) { @@ -126,16 +130,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", instance_config.pgdata); + + check_incremental_compatibility(instance_config.pgdata, + instance_config.system_identifier, + false); } else elog(ERROR, "Restore destination is not empty: \"%s\"", instance_config.pgdata); - } - else - { - /* if remote directory is empty then disable incremental restore */ - if (params->incremental) - params->incremental = false; + + /* if remote directory is empty then incremental restore may be disabled */ + pgdata_is_empty = true; } } @@ -340,7 +345,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - check_tablespace_mapping(dest_backup); + check_tablespace_mapping(dest_backup, params->incremental, &tblspaces_are_empty); + + if (pgdata_is_empty && tblspaces_are_empty) + params->incremental = false; /* no point in checking external directories if their restore is not requested */ if (!params->skip_external_dirs) @@ -602,7 +610,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * Restore dest_backup internal directories. */ create_data_directories(dest_files, instance_config.pgdata, - dest_backup->root_dir, true, FIO_DB_HOST); + dest_backup->root_dir, true, + params->incremental, FIO_DB_HOST); /* * Restore dest_backup external directories. @@ -660,16 +669,23 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Extracting the content of destination directory for incremental restore"); /* TODO: external directorues */ + time(&start_time); if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); else dir_list_file(pgdata_files, pgdata_path, false, true, false, false, true, 0, FIO_LOCAL_HOST); - parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc); - elog(INFO, "Destination directory content extracted, time elapsed:"); - elog(INFO, "Removing redundant files"); + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + + elog(INFO, "Destination directory content extracted, time elapsed: %s", + pretty_time); + + elog(INFO, "Removing redundant files in destination directory"); + time(&start_time); for (i = 0; i < parray_num(pgdata_files); i++) { pgFile *file = (pgFile *) parray_get(pgdata_files, i); @@ -686,7 +702,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } } - elog(INFO, "Redundant files are removed, time elapsed:"); + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + + elog(INFO, "Redundant files are removed, time elapsed: %s", pretty_time); // use_bitmap = true; /* At this point PDATA do not contain files, that also do not exists in backup filelist */ @@ -756,9 +776,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s", pretty_total_bytes, pretty_time); - elog(INFO, "Approximate restore efficiency ratio: %.f%% (%s/%s)", - ((float) dest_bytes / total_bytes) * 100, - pretty_dest_bytes, pretty_total_bytes); + elog(INFO, "Restore overwriting ratio (less is better): %.f%% (%s/%s)", + ((float) total_bytes / dest_bytes) * 100, + pretty_total_bytes, pretty_dest_bytes); } else elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s", @@ -921,7 +941,9 @@ restore_files(void *arg) if (arguments->incremental && parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc)) + { already_exists = true; + } /* * Handle incremental restore case for data files. @@ -933,7 +955,6 @@ restore_files(void *arg) dest_file->is_datafile && !dest_file->is_cfs && dest_file->n_blocks > 0) { - elog(INFO, "HELLO"); /* remote mode */ if (fio_is_remote(FIO_DB_HOST)) checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, @@ -1743,3 +1764,72 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, return dbOid_exclude_list; } + +/* check that instance has the same SYSTEM_ID, */ +void +check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool lsn_based) +{ + uint64 system_id_pgdata; + bool success = true; + pid_t pid; + char backup_label[MAXPGPATH]; + + /* slurp pg_control and check that system ID is the same */ + /* check that instance is not running */ + /* if lsn_based, check that there is no backup_label files is around AND + * get redo point lsn from destination pg_control. + + * It is really important to be sure that pg_control is in cohesion with + * data files content, because based on pg_control information we will + * choose a backup suitable for lsn based incremental restore. + */ + /* TODO: handle timeline discrepancies */ + + system_id_pgdata = get_system_identifier(pgdata); + + if (system_id_pgdata != instance_config.system_identifier) + { + elog(WARNING, "Backup catalog was initialized for system id %lu, " + "but destination directory system id is %lu", + system_identifier, system_id_pgdata); + success = false; + } + + /* check postmaster pid */ + if (fio_is_remote(FIO_DB_HOST)) + pid = fio_check_postmaster(pgdata); + else + pid = check_postmaster(pgdata); + + if (pid == 1) /* postmaster.pid is mangled */ + { + char pid_file[MAXPGPATH]; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); + elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", + pid_file); + success = false; + } + else if (pid > 1) + { + elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", + pid, pgdata); + success = false; + } + + if (lsn_based) + { + snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata); + if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) + { + elog(WARNING, "Destination directory contains \"backup_control\" file. " + "It does not mean that you should delete this file, only that " + "lsn-based incremental restore is dangerous to use in this case. " + "Consider to use checksum-based incremental restore"); + success = false; + } + } + + if (!success) + elog(ERROR, "Incremental restore is impossible"); +} diff --git a/src/util.c b/src/util.c index bf9c6e33a..abec59949 100644 --- a/src/util.c +++ b/src/util.c @@ -549,3 +549,50 @@ datapagemap_print_debug(datapagemap_t *map) pg_free(iter); } + +/* + * return pid of postmaster process running in given pgdata. + * return 0 if there is none. + */ +pid_t +check_postmaster(const char *pgdata) +{ + FILE *fp; + pid_t pid; + char pid_file[MAXPGPATH]; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); + + fp = fopen(pid_file, "r"); + if (fp == NULL) + { + /* No pid file, acceptable*/ + if (errno == ENOENT) + return 0; + else + elog(ERROR, "Cannot open file \"%s\": %s", + pid_file, strerror(errno)); + } + + if (fscanf(fp, "%i", &pid) != 1) + { + /* something is wrong with the file content */ + pid = 1; + } + + if (pid > 1) + { + if (kill(pid, 0) != 0) + { + /* process no longer exists */ + if (errno == ESRCH) + pid = 0; + else + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + pid, strerror(errno)); + } + } + + fclose(fp); + return pid; +} diff --git a/src/utils/file.c b/src/utils/file.c index b05ea649b..a3e71dd7c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -856,7 +856,7 @@ int fio_access(char const* path, int mode, fio_location location) } /* Create symbolic link */ -int fio_symlink(char const* target, char const* link_path, fio_location location) +int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) { if (fio_is_remote(location)) { @@ -866,6 +866,7 @@ int fio_symlink(char const* target, char const* link_path, fio_location location hdr.cop = FIO_SYMLINK; hdr.handle = -1; hdr.size = target_len + link_path_len; + hdr.arg = overwrite ? 1 : 0; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); @@ -875,10 +876,26 @@ int fio_symlink(char const* target, char const* link_path, fio_location location } else { + if (overwrite) + remove_file_or_dir(link_path); + return symlink(target, link_path); } } +static void fio_symlink_impl(int out, char *buf, bool overwrite) +{ + char *linked_path = buf; + char *link_path = buf + strlen(buf) + 1; + + if (overwrite) + remove_file_or_dir(link_path); + + if (symlink(linked_path, link_path)) + elog(ERROR, "Could not create symbolic link \"%s\": %s", + link_path, strerror(errno)); +} + /* Rename file */ int fio_rename(char const* old_path, char const* new_path, fio_location location) { @@ -1394,6 +1411,9 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt -------------------------------------------------------------- */ +// elog(WARNING, "Size: %lu", sizeof(fio_header)); +// elog(ERROR, "Size: %lu", MAXALIGN(sizeof(fio_header))); + req.hdr.cop = FIO_SEND_PAGES; if (pagemap) @@ -2243,7 +2263,6 @@ static void fio_list_dir_impl(int out, char* buf) uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { - fio_header hdr; fio_checksum_map_request req_hdr; uint16 *checksum_map = NULL; @@ -2293,6 +2312,34 @@ static void fio_get_checksum_map_impl(int out, char *buf) pg_free(checksum_map); } +pid_t fio_check_postmaster(const char *pgdata) +{ + fio_header hdr; + + hdr.cop = FIO_CHECK_POSTMASTER; + hdr.size = strlen(pgdata) + 1; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); + + /* receive result */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + return hdr.arg; +} + +static void fio_check_postmaster_impl(int out, char *buf) +{ + fio_header hdr; + pid_t postmaster_pid; + char *pgdata = (char*) buf; + + postmaster_pid = check_postmaster(pgdata); + + /* send arrays of checksums to main process */ + hdr.arg = postmaster_pid; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Execute commands at remote host */ void fio_communicate(int in, int out) { @@ -2411,7 +2458,7 @@ void fio_communicate(int in, int out) SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); break; case FIO_SYMLINK: /* Create symbolic link */ - SYS_CHECK(symlink(buf, buf + strlen(buf) + 1)); + fio_symlink_impl(out, buf, hdr.arg > 0 ? true : false); break; case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ SYS_CHECK(remove_file_or_dir(buf)); @@ -2468,6 +2515,10 @@ void fio_communicate(int in, int out) /* calculate crc32 for a file */ fio_get_checksum_map_impl(out, buf); break; + case FIO_CHECK_POSTMASTER: + /* calculate crc32 for a file */ + fio_check_postmaster_impl(out, buf); + break; case FIO_DISCONNECT: hdr.cop = FIO_SEND_FILE_EOF; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); diff --git a/src/utils/file.h b/src/utils/file.h index cf6176418..310c561bb 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -47,7 +47,8 @@ typedef enum FIO_DISCONNECT, /* message for compatibility check */ FIO_AGENT_VERSION, - FIO_LIST_DIR + FIO_LIST_DIR, + FIO_CHECK_POSTMASTER } fio_operations; typedef enum @@ -66,8 +67,9 @@ typedef enum typedef struct { - unsigned cop : 5; - unsigned handle : 7; +// fio_operations cop; + unsigned cop : 6; + unsigned handle : 6; unsigned size : 20; unsigned arg; } fio_header; @@ -107,7 +109,7 @@ extern int fio_sync(char const* path, fio_location location); extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); -extern int fio_symlink(char const* target, char const* link_path, fio_location location); +extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); extern int fio_unlink(char const* path, fio_location location); extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); From 87dc977baae3b584ad18dbbc4b273c2cf7e3a8b3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 28 May 2020 01:37:52 +0300 Subject: [PATCH 1344/2107] set version to 2.4.0 --- src/pg_probackup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 594179569..966c96e61 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -241,8 +241,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.4" -#define AGENT_PROTOCOL_VERSION 20304 +#define PROGRAM_VERSION "2.4.0" +#define AGENT_PROTOCOL_VERSION 20400 typedef struct ConnectionOptions From 45995f0b0d8b0a6f91a0563ff3266e528168c3ef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 30 May 2020 07:15:11 +0300 Subject: [PATCH 1345/2107] [Issue #66] Add lsn-based incremental restore --- src/data.c | 126 +++++++++++++++++++++++++----- src/merge.c | 8 +- src/pg_probackup.c | 12 ++- src/pg_probackup.h | 35 +++++++-- src/restore.c | 157 ++++++++++++++++++++++++++++---------- src/util.c | 36 ++++++++- src/utils/file.c | 186 +++++++++++++++++++++++++++++++++++---------- src/utils/file.h | 2 + 8 files changed, 453 insertions(+), 109 deletions(-) diff --git a/src/data.c b/src/data.c index 1f2fdfcad..9b751b05f 100644 --- a/src/data.c +++ b/src/data.c @@ -862,14 +862,19 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, */ size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, - const char *to_fullpath, bool use_bitmap, uint16 *checksum_map) + const char *to_fullpath, bool use_bitmap, PageState *checksum_map, + XLogRecPtr horizonLsn, datapagemap_t *lsn_map) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); int backup_seq = 0; - // FULL -> INCR -> DEST - // 2 1 0 + /* + * FULL -> INCR -> DEST + * 2 1 0 + * Restore of backups of older versions cannot be optimized with bitmap + * because of n_blocks + */ if (use_bitmap) /* start with dest backup */ backup_seq = 0; @@ -942,7 +947,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, parse_program_version(backup->program_version), from_fullpath, to_fullpath, dest_file->n_blocks, use_bitmap ? &(dest_file)->pagemap : NULL, - checksum_map, backup->checksum_version); + checksum_map, backup->checksum_version, + backup->start_lsn <= horizonLsn ? lsn_map : NULL); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -963,7 +969,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, uint16 *checksum_map, int checksum_version) + datapagemap_t *map, PageState *checksum_map, int checksum_version, + datapagemap_t *lsn_map) { BackupPageHeader header; BlockNumber blknum = 0; @@ -1071,6 +1078,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); + if (lsn_map && datapagemap_is_set(lsn_map, blknum)) + datapagemap_add(map, blknum); + /* if this page is marked as already restored, then skip it */ if (map && datapagemap_is_set(map, blknum)) { @@ -1101,10 +1111,14 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers is_compressed = true; } - /* Incremental restore */ - if (checksum_map && checksum_map[blknum] != 0) + /* Incremental restore + * TODO: move to another function + */ + if (checksum_map && checksum_map[blknum].checksum != 0) { uint16 page_crc = 0; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + PageHeader phdr; if (is_compressed) { @@ -1117,8 +1131,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers else page_crc = pg_checksum_page(uncompressed_buf, file->segno * RELSEG_SIZE + blknum); -// page_crc_1 = pg_checksum_page(uncompressed_buf, file->segno * RELSEG_SIZE + blknum); -// Assert(page_crc == page_crc_1); + phdr = (PageHeader) uncompressed_buf; } else { @@ -1127,10 +1140,19 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers page_crc = ((PageHeader) page.data)->pd_checksum; else page_crc = pg_checksum_page(page.data, file->segno + blknum); + + phdr = (PageHeader) page.data; } - /* the heart of incremental restore */ - if (page_crc == checksum_map[blknum]) + page_lsn = PageXLogRecPtrGet(phdr->pd_lsn); + + /* + * The heart of incremental restore + * If page in backup has the same checksum and lsn as + * page in backup, then page can be skipped. + */ + if (page_crc == checksum_map[blknum].checksum && + page_lsn == checksum_map[blknum].lsn) { if (map) datapagemap_add(map, blknum); @@ -1812,10 +1834,10 @@ check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } /* read local data file and construct map with block checksums */ -uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, +PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { - uint16 *checksum_map = NULL; + PageState *checksum_map = NULL; FILE *in = NULL; BlockNumber blknum = 0; XLogRecPtr page_lsn = 0; @@ -1834,8 +1856,8 @@ uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* initialize array of checksums */ - checksum_map = pgut_malloc(n_blocks * sizeof(uint16)); - memset(checksum_map, 0, n_blocks * sizeof(uint16)); + checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); + memset(checksum_map, 0, n_blocks * sizeof(PageState)); for (blknum = 0; blknum < n_blocks; blknum++) { @@ -1855,9 +1877,11 @@ uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, if (rc == PAGE_IS_VALID) { if (checksum_version) - checksum_map[blknum] = ((PageHeader) read_buffer)->pd_checksum; + checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; else - checksum_map[blknum] = pg_checksum_page(read_buffer, segmentno + blknum); + checksum_map[blknum].checksum = pg_checksum_page(read_buffer, segmentno + blknum); + + checksum_map[blknum].lsn = page_lsn; } } else @@ -1875,3 +1899,71 @@ uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, return checksum_map; } + +/* return bitmap of valid blocks, bitmap is empty, then NULL is returned */ +datapagemap_t * +get_lsn_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno) +{ + FILE *in = NULL; + BlockNumber blknum = 0; + XLogRecPtr page_lsn = 0; + char read_buffer[BLCKSZ]; + char in_buf[STDIO_BUFSIZE]; + datapagemap_t *lsn_map = NULL; + + /* truncate up to blocks */ + if (truncate(fullpath, n_blocks * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + n_blocks, fullpath, strerror(errno)); + + Assert(horizonLsn > 0); + + /* open file */ + in = fopen(fullpath, PG_BINARY_R); + if (!in) + elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + + lsn_map = pgut_malloc(sizeof(datapagemap_t)); + memset(lsn_map, 0, sizeof(datapagemap_t)); + + for (blknum = 0; blknum < n_blocks; blknum++) + { + size_t read_len = fread(read_buffer, 1, BLCKSZ, in); + page_lsn = InvalidXLogRecPtr; + + /* report error */ + if (ferror(in)) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + blknum, fullpath, strerror(errno)); + + if (read_len == BLCKSZ) + { + int rc = validate_one_page(read_buffer, segmentno + blknum, + horizonLsn, &page_lsn, checksum_version); + + if (rc == PAGE_IS_VALID) + datapagemap_add(lsn_map, blknum); + } + else + elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); + + if (feof(in)) + break; + + if (interrupted) + elog(ERROR, "Interrupted during page reading"); + } + + if (in) + fclose(in); + + if (lsn_map->bitmapsize == 0) + { + pg_free(lsn_map); + lsn_map = NULL; + } + + return lsn_map; +} diff --git a/src/merge.c b/src/merge.c index 457fab15c..fb2f39181 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1161,8 +1161,12 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); /* restore file into temp file */ - tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap, NULL); - fclose(out); + tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, + use_bitmap, NULL, InvalidXLogRecPtr, NULL); + if (fclose(out) != 0) + elog(ERROR, "Cannot close file \"%s\": %s", + to_fullpath_tmp1, strerror(errno)); + pg_free(buffer); /* tmp_file->size is greedy, even if there is single 8KB block in file, diff --git a/src/pg_probackup.c b/src/pg_probackup.c index fbc5c3d1c..cd5b440f3 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -98,7 +98,8 @@ static pgRestoreParams *restore_params = NULL; time_t current_time = 0; bool restore_as_replica = false; bool no_validate = false; -bool incremental_restore = false; +bool incremental = false; +bool incremental_lsn = false; bool skip_block_validation = false; bool skip_external_dirs = false; @@ -204,7 +205,8 @@ static ConfigOption cmd_options[] = { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, - { 'b', 161, "incremental", &incremental_restore, SOURCE_CMD_STRICT }, + { 'b', 161, "incremental", &incremental, SOURCE_CMD_STRICT }, + { 'b', 167, "incremental-lsn", &incremental_lsn, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -702,6 +704,9 @@ main(int argc, char *argv[]) if (replication_slot != NULL) restore_as_replica = true; + if (!incremental && incremental_lsn) + incremental = true; + /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); @@ -714,7 +719,8 @@ main(int argc, char *argv[]) restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; restore_params->primary_conninfo = primary_conninfo; - restore_params->incremental = incremental_restore; + restore_params->incremental = incremental; + restore_params->incremental_lsn = incremental_lsn; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 966c96e61..902123ee2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -101,6 +101,18 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +typedef struct RedoParams +{ + TimeLineID tli; + XLogRecPtr lsn; +} RedoParams; + +typedef struct PageState +{ + uint16 checksum; + XLogRecPtr lsn; +} PageState; + typedef struct db_map_entry { Oid dbOid; @@ -442,6 +454,8 @@ typedef struct pgRestoreParams bool skip_external_dirs; bool skip_block_validation; //Start using it bool incremental; + bool incremental_lsn; + XLogRecPtr horizonLsn; const char *restore_command; const char *primary_slot_name; @@ -938,10 +952,12 @@ extern void backup_non_data_file_internal(const char *from_fullpath, bool missing_ok); extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, - const char *to_fullpath, bool use_bitmap, uint16 *checksum_map); + const char *to_fullpath, bool use_bitmap, PageState *checksum_map, + XLogRecPtr horizonLsn, datapagemap_t *lsn_map); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, uint16 *checksum_map, int checksum_version); + datapagemap_t *map, PageState *checksum_map, int checksum_version, + datapagemap_t *lsn_map); extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath, bool already_exists); @@ -950,8 +966,10 @@ extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, extern bool create_empty_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); -extern uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version, +extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); +extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno); extern pid_t check_postmaster(const char *pgdata); extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, @@ -989,6 +1007,7 @@ extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); extern uint32 get_xlog_seg_size(char *pgdata_path); +extern void get_redo(const char *pgdata_path, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, @@ -1048,9 +1067,13 @@ extern void fio_list_dir(parray *files, const char *root, bool exclude, bool fol extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); -extern uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); -extern pid_t fio_check_postmaster(const char *pgdata); +extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, + XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); + +extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, + fio_location location); +extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg); diff --git a/src/restore.c b/src/restore.c index 5993a3962..b26e48eda 100644 --- a/src/restore.c +++ b/src/restore.c @@ -30,6 +30,8 @@ typedef struct size_t restored_bytes; bool use_bitmap; bool incremental; + bool incremental_lsn; + XLogRecPtr horizonLsn; /* * Return value from the thread. @@ -50,7 +52,7 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync); static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - bool lsn_based); + bool incremental_lsn); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -115,6 +117,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray *dbOid_exclude_list = NULL; bool pgdata_is_empty = true; bool tblspaces_are_empty = true; + XLogRecPtr horizonLsn = InvalidXLogRecPtr; if (params->is_restore) { @@ -124,8 +127,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Check if restore destination empty */ if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) { - // TODO: check that remote systemd id is the same as ours. - // TODO: check that remote system is NOT running, check pg_control and pid. + /* Check that remote system is NOT running and systemd id is the same as ours */ if (params->incremental) { elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", @@ -133,14 +135,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, check_incremental_compatibility(instance_config.pgdata, instance_config.system_identifier, - false); + params->incremental_lsn); } else elog(ERROR, "Restore destination is not empty: \"%s\"", instance_config.pgdata); - /* if remote directory is empty then incremental restore may be disabled */ - pgdata_is_empty = true; + /* if destination directory is empty, then incremental restore may be disabled */ + pgdata_is_empty = false; } } @@ -366,13 +368,72 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ tmp_backup = dest_backup; - while(tmp_backup->parent_backup_link) + while (tmp_backup) { parray_append(parent_chain, tmp_backup); tmp_backup = tmp_backup->parent_backup_link; } - parray_append(parent_chain, base_full_backup); + /* + * Determine horizon LSN + * Consider the following example: + * + * + * /----D----------F-> + * -A--B---C---*-------X-----> + * + * [A,F] - incremental chain + * X - the state of pgdata + * F - destination backup + * + * When running incremental-lsn restore, we get a bitmap of pages, + * whose LSN is less than C. + * So when restoring file, we can skip restore of pages coming from + * A, B and C. + * Pages from D and F cannot be skipped due to incremental restore. + * + */ + if (params->is_restore && params->incremental && params->incremental_lsn) + { + RedoParams redo; + get_redo(instance_config.pgdata, &redo); + + tmp_backup = dest_backup; + + while (tmp_backup) + { + if (tmp_backup->start_lsn < redo.lsn && + redo.tli == tmp_backup->tli) + { + horizonLsn = tmp_backup->start_lsn; + break; + } + + if (!tmp_backup->parent_backup_link) + break; + + tmp_backup = tmp_backup->parent_backup_link; + } + + if (XLogRecPtrIsInvalid(horizonLsn)) + elog(ERROR, "Cannot perform lsn-based incremental restore of backup chain %s, " + "because destination directory redo point %X/%X on tli %i is less than " + "START LSN %X/%X of oldest backup in chain", + base36enc(dest_backup->start_time), + (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, + (uint32) (tmp_backup->start_lsn >> 32), (uint32) tmp_backup->start_lsn); + else + elog(INFO, "Destination directory redo point %X/%X on tli %i is within reach of " + "backup %s with START LSN %X/%X, lsn-based incremental restore is possible", + (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, + base36enc(tmp_backup->start_time), + (uint32) (tmp_backup->start_lsn >> 32), (uint32) tmp_backup->start_lsn); + + elog(INFO, "Horizon LSN: %X/%X", + (uint32) (horizonLsn >> 32), (uint32) horizonLsn); + + params->horizonLsn = horizonLsn; + } /* for validation or restore with enabled validation */ if (!params->is_restore || !params->no_validate) @@ -601,10 +662,16 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version"); } - /* There is no point in bitmap restore, when restoring a single FULL backup */ - // TODO: if there is lsn-based incremental restore, then bitmap is mandatory - if (parray_num(parent_chain) == 1) - use_bitmap = false; + /* There is no point in bitmap restore, when restoring a single FULL backup, + * unless we are running incremental-lsn restore, then bitmap is mandatory. + */ + if (use_bitmap && parray_num(parent_chain) == 1) + { + if (params->incremental && params->incremental_lsn) + use_bitmap = true; + else + use_bitmap = false; + } /* * Restore dest_backup internal directories. @@ -661,7 +728,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_atomic_clear_flag(&file->lock); } - // Get list of files in destination directory and remove redundant files. + /* Get list of files in destination directory and remove redundant files */ if (params->incremental) { pgdata_files = parray_new(); @@ -698,7 +765,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(full_file_path, pgdata_path, file->rel_path); fio_pgFileDelete(file, full_file_path); - elog(WARNING, "Deleted remote file \"%s\"", full_file_path); + elog(WARNING, "Deleted file \"%s\"", full_file_path); } } @@ -706,10 +773,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); + /* At this point PDATA do not contain files, that do not exists in dest backup file list */ elog(INFO, "Redundant files are removed, time elapsed: %s", pretty_time); - -// use_bitmap = true; - /* At this point PDATA do not contain files, that also do not exists in backup filelist */ } /* @@ -746,6 +811,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, arg->to_root = pgdata_path; arg->use_bitmap = use_bitmap; arg->incremental = params->incremental; + arg->incremental_lsn = params->incremental_lsn; + arg->horizonLsn = params->horizonLsn; threads_args[i].restored_bytes = 0; /* By default there are some error */ threads_args[i].ret = 1; @@ -776,7 +843,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s", pretty_total_bytes, pretty_time); - elog(INFO, "Restore overwriting ratio (less is better): %.f%% (%s/%s)", + elog(INFO, "Restore incremental ratio (less is better): %.f%% (%s/%s)", ((float) total_bytes / dest_bytes) * 100, pretty_total_bytes, pretty_dest_bytes); } @@ -871,7 +938,8 @@ restore_files(void *arg) for (i = 0; i < parray_num(arguments->dest_files); i++) { bool already_exists = false; - uint16 *checksum_map = NULL; /* it should take 512kB at most */ + PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ + datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ @@ -955,16 +1023,19 @@ restore_files(void *arg) dest_file->is_datafile && !dest_file->is_cfs && dest_file->n_blocks > 0) { - /* remote mode */ - if (fio_is_remote(FIO_DB_HOST)) + if (arguments->incremental_lsn) + { + /* TODO: return lsn_map */ + lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version, + dest_file->n_blocks, arguments->horizonLsn, + dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + } + else + { checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->dest_backup->stop_lsn, - dest_file->segno * RELSEG_SIZE); - /* local mode */ - else - checksum_map = get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, - dest_file->n_blocks, arguments->dest_backup->stop_lsn, - dest_file->segno * RELSEG_SIZE); + dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + } } /* open destination file */ @@ -999,7 +1070,8 @@ restore_files(void *arg) /* Destination file is data file */ arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file, out, to_fullpath, - arguments->use_bitmap, checksum_map); + arguments->use_bitmap, checksum_map, + arguments->horizonLsn, lsn_map); } else { @@ -1021,6 +1093,8 @@ restore_files(void *arg) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, strerror(errno)); + if (lsn_map) + pg_free(lsn_map->bitmap); pg_free(checksum_map); } @@ -1765,9 +1839,11 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, return dbOid_exclude_list; } -/* check that instance has the same SYSTEM_ID, */ +/* Check that instance is suitable for incremental restore + * Depending on type of incremental restore requirements are differs. + */ void -check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool lsn_based) +check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool incremental_lsn) { uint64 system_id_pgdata; bool success = true; @@ -1783,7 +1859,6 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bo * data files content, because based on pg_control information we will * choose a backup suitable for lsn based incremental restore. */ - /* TODO: handle timeline discrepancies */ system_id_pgdata = get_system_identifier(pgdata); @@ -1796,10 +1871,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bo } /* check postmaster pid */ - if (fio_is_remote(FIO_DB_HOST)) - pid = fio_check_postmaster(pgdata); - else - pid = check_postmaster(pgdata); + pid = fio_check_postmaster(pgdata, FIO_DB_HOST); if (pid == 1) /* postmaster.pid is mangled */ { @@ -1810,22 +1882,27 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bo pid_file); success = false; } - else if (pid > 1) + else if (pid > 1) /* postmaster is up */ { elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", pid, pgdata); success = false; } - if (lsn_based) + /* + * TODO: maybe there should be some other signs, pointing to pg_control + * desynchronization with cluster state. + */ + if (incremental_lsn) { snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata); if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " - "It does not mean that you should delete this file, only that " - "lsn-based incremental restore is dangerous to use in this case. " - "Consider to use checksum-based incremental restore"); + "This does NOT mean that you should delete this file and retry, only that " + "lsn-based incremental restore can corrupt data, when applied to instance " + "with pg_control not synchronized with cluster state." + "Consider to use checksum-based incremental restore."); success = false; } } diff --git a/src/util.c b/src/util.c index abec59949..a6cf2e393 100644 --- a/src/util.c +++ b/src/util.c @@ -351,6 +351,37 @@ get_pgcontrol_checksum(const char *pgdata_path) return ControlFile.crc; } +void +get_redo(const char *pgdata_path, RedoParams *redo) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + redo->lsn = ControlFile.checkPointCopy.redo; + redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; + + if (ControlFile.minRecoveryPoint > 0 && + ControlFile.minRecoveryPoint < redo->lsn) + { + redo->lsn = ControlFile.minRecoveryPoint; + redo->tli = ControlFile.minRecoveryPointTLI; + } + + if (ControlFile.backupStartPoint > 0 && + ControlFile.backupStartPoint < redo->lsn) + { + redo->lsn = ControlFile.backupStartPoint; + redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; + } +} + /* * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint * 'as-is' is not to be trusted. @@ -551,8 +582,9 @@ datapagemap_print_debug(datapagemap_t *map) } /* - * return pid of postmaster process running in given pgdata. - * return 0 if there is none. + * Return pid of postmaster process running in given pgdata. + * Return 0 if there is none. + * Return 1 if postmaster.pid is mangled. */ pid_t check_postmaster(const char *pgdata) diff --git a/src/utils/file.c b/src/utils/file.c index a3e71dd7c..51f209ff6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -72,6 +72,14 @@ typedef struct uint32 checksumVersion; } fio_checksum_map_request; +typedef struct +{ + BlockNumber n_blocks; + BlockNumber segmentno; + XLogRecPtr horizonLsn; + uint32 checksumVersion; +} fio_lsn_map_request; + /* Convert FIO pseudo handle to index in file descriptor array */ #define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) @@ -99,6 +107,7 @@ void fio_error(int rc, int size, char const* file, int line) else { char buf[PRINTF_BUF_SIZE+1]; +// Assert(false); int err_size = read(fio_stderr, buf, PRINTF_BUF_SIZE); if (err_size > 0) { @@ -200,14 +209,16 @@ static ssize_t fio_read_all(int fd, void* buf, size_t size) while (offs < size) { ssize_t rc = read(fd, (char*)buf + offs, size - offs); - if (rc < 0) { - if (errno == EINTR) { + if (rc < 0) + { + if (errno == EINTR) continue; - } + elog(WARNING, "fio_read_all error: %s", strerror(errno)); return rc; - } else if (rc == 0) { - break; } + else if (rc == 0) + break; + offs += rc; } return offs; @@ -2260,43 +2271,53 @@ static void fio_list_dir_impl(int out, char* buf) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) +PageState * +fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, + XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) { - fio_header hdr; - fio_checksum_map_request req_hdr; - uint16 *checksum_map = NULL; - size_t path_len = strlen(fullpath) + 1; + if (fio_is_remote(location)) + { + fio_header hdr; + fio_checksum_map_request req_hdr; + PageState *checksum_map = NULL; + size_t path_len = strlen(fullpath) + 1; - req_hdr.n_blocks = n_blocks; - req_hdr.segmentno = segmentno; - req_hdr.stop_lsn = dest_stop_lsn; - req_hdr.checksumVersion = checksum_version; + req_hdr.n_blocks = n_blocks; + req_hdr.segmentno = segmentno; + req_hdr.stop_lsn = dest_stop_lsn; + req_hdr.checksumVersion = checksum_version; - hdr.cop = FIO_GET_CHECKSUM_MAP; - hdr.size = sizeof(req_hdr) + path_len; + hdr.cop = FIO_GET_CHECKSUM_MAP; + hdr.size = sizeof(req_hdr) + path_len; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > 0) - { - checksum_map = pgut_malloc(hdr.size * sizeof(uint16)); - IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(uint16)), hdr.size * sizeof(uint16)); + if (hdr.size > 0) + { + checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); + memset(checksum_map, 0, n_blocks * sizeof(PageState)); + IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); + } + + return checksum_map; } + else + { - return checksum_map; + return get_checksum_map(fullpath, checksum_version, + n_blocks, dest_stop_lsn, segmentno); + } } -/* TODO: sent n_blocks to truncate file before reading */ static void fio_get_checksum_map_impl(int out, char *buf) { fio_header hdr; - uint16 *checksum_map = NULL; + PageState *checksum_map = NULL; char *fullpath = (char*) buf + sizeof(fio_checksum_map_request); fio_checksum_map_request *req = (fio_checksum_map_request*) buf; @@ -2307,24 +2328,107 @@ static void fio_get_checksum_map_impl(int out, char *buf) /* send arrays of checksums to main process */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > 0) - IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(uint16)), hdr.size * sizeof(uint16)); + IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); pg_free(checksum_map); } -pid_t fio_check_postmaster(const char *pgdata) +datapagemap_t * +fio_get_lsn_map(const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, + fio_location location) { - fio_header hdr; + datapagemap_t* lsn_map = NULL; - hdr.cop = FIO_CHECK_POSTMASTER; - hdr.size = strlen(pgdata) + 1; + if (fio_is_remote(location)) + { + fio_header hdr; + fio_lsn_map_request req_hdr; + size_t path_len = strlen(fullpath) + 1; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); + req_hdr.n_blocks = n_blocks; + req_hdr.segmentno = segmentno; + req_hdr.horizonLsn = horizonLsn; + req_hdr.checksumVersion = checksum_version; - /* receive result */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - return hdr.arg; + hdr.cop = FIO_GET_LSN_MAP; + hdr.size = sizeof(req_hdr) + path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); + + /* receive data */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + lsn_map = pgut_malloc(sizeof(datapagemap_t)); + memset(lsn_map, 0, sizeof(datapagemap_t)); + + lsn_map->bitmap = pgut_malloc(hdr.size); + lsn_map->bitmapsize = hdr.size; + + IO_CHECK(fio_read_all(fio_stdin, lsn_map->bitmap, hdr.size), hdr.size); + } + } + else + { + lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, + horizonLsn, segmentno); + } + + return lsn_map; +} + +static void fio_get_lsn_map_impl(int out, char *buf) +{ + fio_header hdr; + datapagemap_t *lsn_map = NULL; + char *fullpath = (char*) buf + sizeof(fio_lsn_map_request); + fio_lsn_map_request *req = (fio_lsn_map_request*) buf; + + lsn_map = get_lsn_map(fullpath, req->checksumVersion, req->n_blocks, + req->horizonLsn, req->segmentno); + if (lsn_map) + hdr.size = lsn_map->bitmapsize; + else + hdr.size = 0; + + /* send arrays of checksums to main process */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > 0) + IO_CHECK(fio_write_all(out, lsn_map->bitmap, hdr.size), hdr.size); + + if (lsn_map) + { + pg_free(lsn_map->bitmap); + pg_free(lsn_map); + } +} + +/* + * Go to the remote host and get postmaster pid from file postmaster.pid + * and check that process is running, if process is running, return its pid number. + */ +pid_t fio_check_postmaster(const char *pgdata, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + + hdr.cop = FIO_CHECK_POSTMASTER; + hdr.size = strlen(pgdata) + 1; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); + + /* receive result */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + return hdr.arg; + } + else + return check_postmaster(pgdata); } static void fio_check_postmaster_impl(int out, char *buf) @@ -2515,6 +2619,10 @@ void fio_communicate(int in, int out) /* calculate crc32 for a file */ fio_get_checksum_map_impl(out, buf); break; + case FIO_GET_LSN_MAP: + /* calculate crc32 for a file */ + fio_get_lsn_map_impl(out, buf); + break; case FIO_CHECK_POSTMASTER: /* calculate crc32 for a file */ fio_check_postmaster_impl(out, buf); diff --git a/src/utils/file.h b/src/utils/file.h index 310c561bb..5a6415c81 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -35,7 +35,9 @@ typedef enum FIO_PAGE, FIO_WRITE_COMPRESSED, FIO_GET_CRC32, + /* used for incremental restore */ FIO_GET_CHECKSUM_MAP, + FIO_GET_LSN_MAP, /* used in fio_send_pages */ FIO_SEND_PAGES, FIO_ERROR, From d46fa5f82fe086f44436f26fecf0683877ec7b44 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 May 2020 05:55:09 +0300 Subject: [PATCH 1346/2107] [PBCKP-98] fix invalid stop lsn. Reported by Alexander Lakhin and Alex Ignatov --- src/backup.c | 136 ++++++++++++++++++++++++++------------------- src/parsexlog.c | 91 ++++++++++++++++++++++++++++++ src/pg_probackup.h | 2 + 3 files changed, 171 insertions(+), 58 deletions(-) diff --git a/src/backup.c b/src/backup.c index 19742090a..aff0dbe4a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -401,10 +401,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) if (current.backup_mode != BACKUP_MODE_FULL) { - elog(LOG, "current_tli:%X", current.tli); - elog(LOG, "prev_backup->start_lsn: %X/%X", + elog(LOG, "Current tli: %X", current.tli); + elog(LOG, "Parent start_lsn: %X/%X", (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); - elog(LOG, "current.start_lsn: %X/%X", + elog(LOG, "start_lsn: %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); } @@ -583,9 +583,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* Notify end of backup */ pg_stop_backup(¤t, pg_startbackup_conn, nodeInfo); - elog(LOG, "current.stop_lsn: %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); - /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. */ @@ -1742,65 +1739,66 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* Calculate LSN */ stop_backup_lsn_tmp = ((uint64) lsn_hi) << 32 | lsn_lo; + /* It is ok for replica to return invalid STOP LSN + * UPD: Apparently it is ok even for a master. + */ if (!XRecOffIsValid(stop_backup_lsn_tmp)) { - /* It is ok for replica to return STOP LSN with NullXRecOff - * UPD: Apparently it is ok even for master. - */ - if (XRecOffIsNull(stop_backup_lsn_tmp)) - { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + XLogSegNo segno = 0; + XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Null offset in stop_backup_lsn value %X/%X, trying to fix", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* + * Even though the value is invalid, it's expected postgres behaviour + * and we're trying to fix it below. + */ + elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", + (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + NullXRecOff, so emulate it for manual testing. - */ - //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* + * Note: even with gdb it is very hard to produce automated tests for + * contrecord + invalid LSN, so emulate it for manual testing. + */ + //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; + //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", + // (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); - if (stream_wal) - { - pgBackupGetPath2(backup, stream_xlog_path, - lengthof(stream_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = arclog_path; + if (stream_wal) + { + pgBackupGetPath2(backup, stream_xlog_path, + lengthof(stream_xlog_path), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = arclog_path; - GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); + GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return LSN with NullXRecOff. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid XRecOff on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of NullXRecOff - * on write-idle system. If that fails too, error out. - */ + /* + * Note, that there is no guarantee that corresponding WAL file even exists. + * Replica may return LSN from future and keep staying in present. + * Or it can return invalid LSN. + * + * That's bad, since we want to get real LSN to save it in backup label file + * and to use it in WAL validation. + * + * So we try to do the following: + * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and + * look for the first valid record in it. + * It solves the problem of occasional invalid LSN on write-busy system. + * 2. Failing that, look for record in previous segment with endpoint + * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN + * on write-idle system. If that fails too, error out. + */ + /* stop_lsn is pointing to a 0 byte of xlog segment */ + if (stop_backup_lsn_tmp % instance_config.xlog_seg_size == 0) + { /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, true, WARNING, stream_wal); + false, true, WARNING, stream_wal); /* Get the first record in segment with current stop_lsn */ lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, @@ -1836,17 +1834,39 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); } + } + /* stop lsn is aligned to xlog block size, just find next lsn */ + else if (stop_backup_lsn_tmp % XLOG_BLCKSZ == 0) + { + /* Wait for segment with current stop_lsn */ + wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + false, true, ERROR, stream_wal); + + /* Get the next closest record in segment with current stop_lsn */ + lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout, + stop_backup_lsn_tmp); - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record next to %X/%X", + (uint32) (stop_backup_lsn_tmp >> 32), + (uint32) (stop_backup_lsn_tmp)); } /* PostgreSQL returned something very illegal as STOP_LSN, error out */ else elog(ERROR, "Invalid stop_backup_lsn value %X/%X", (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + + /* Setting stop_backup_lsn will set stop point for streaming */ + stop_backup_lsn = lsn_tmp; + stop_lsn_exists = true; } + elog(LOG, "stop_lsn: %X/%X", + (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + /* Write backup_label and tablespace_map */ if (!exclusive_backup) { diff --git a/src/parsexlog.c b/src/parsexlog.c index d39662e4a..e9410c880 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -680,6 +680,97 @@ get_first_record_lsn(const char *archivedir, XLogSegNo segno, return record; } + +/* + * Get LSN of the record next after target lsn. + */ +XLogRecPtr +get_next_record_lsn(const char *archivedir, XLogSegNo segno, + TimeLineID tli, uint32 wal_seg_size, int timeout, + XLogRecPtr target) +{ + XLogReaderState *xlogreader; + XLogReaderData reader_data; + XLogRecPtr startpoint, found, res; + char wal_segment[MAXFNAMELEN]; + int attempts = 0; + + if (segno <= 1) + elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); + + GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); + + xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, + false, false, true); + if (xlogreader == NULL) + elog(ERROR, "Out of memory"); + xlogreader->system_identifier = instance_config.system_identifier; + + /* Set startpoint to 0 in segno */ + GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); + + found = XLogFindNextRecord(xlogreader, startpoint); + + if (XLogRecPtrIsInvalid(found)) + { + if (xlogreader->errormsg_buf[0] != '\0') + elog(WARNING, "Could not read WAL record at %X/%X: %s", + (uint32) (startpoint >> 32), (uint32) (startpoint), + xlogreader->errormsg_buf); + else + elog(WARNING, "Could not read WAL record at %X/%X", + (uint32) (startpoint >> 32), (uint32) (startpoint)); + PrintXLogCorruptionMsg(&reader_data, ERROR); + } + startpoint = found; + + while (attempts <= timeout) + { + XLogRecord *record; + char *errormsg; + + if (interrupted) + elog(ERROR, "Interrupted during WAL reading"); + + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = XLogRecPtrIsInvalid(startpoint) ? xlogreader->EndRecPtr : + startpoint; + + if (errormsg) + elog(WARNING, "Could not read WAL record at %X/%X: %s", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + elog(WARNING, "Could not read WAL record at %X/%X", + (uint32) (errptr >> 32), (uint32) (errptr)); + PrintXLogCorruptionMsg(&reader_data, ERROR); + } + + if (xlogreader->ReadRecPtr >= target) + { + elog(LOG, "Record %X/%X is next after target LSN %X/%X", + (uint32) (xlogreader->ReadRecPtr >> 32), (uint32) (xlogreader->ReadRecPtr), + (uint32) (target >> 32), (uint32) (target)); + res = xlogreader->ReadRecPtr; + break; + } + else + startpoint = InvalidXLogRecPtr; + } + + /* cleanup */ + CleanupXLogPageRead(xlogreader); + XLogReaderFree(xlogreader); + + return res; +} + + /* * Get LSN of a record prior to target_lsn. * If 'start_lsn' is in the segment with number 'segno' then start from 'start_lsn', diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3e9fd11c0..7c02d245f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -960,6 +960,8 @@ extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_ extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_lsn, TimeLineID tli, uint32 wal_seg_size, int timeout); +extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, TimeLineID tli, + uint32 wal_seg_size, int timeout, XLogRecPtr target); /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); From 13a16e23b54ceb658d9f2e26dc21b79bff9a30f0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 May 2020 06:31:04 +0300 Subject: [PATCH 1347/2107] tests: fixes for d46fa5f82fe086 --- tests/replica.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 85e104f06..4ec51eb6d 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -606,7 +606,7 @@ def test_replica_stop_lsn_null_offset(self): return_id=False) self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/4000000', + 'LOG: Invalid offset in stop_lsn value 0/4000000', output) self.assertIn( @@ -714,7 +714,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content = f.read() self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/4000000', + 'LOG: Invalid offset in stop_lsn value 0/4000000', log_content) self.assertIn( @@ -787,7 +787,7 @@ def test_archive_replica_null_offset(self): return_id=False) self.assertIn( - 'LOG: Null offset in stop_backup_lsn value 0/4000000', + 'LOG: Invalid offset in stop_lsn value 0/4000000', output) self.assertIn( From c5dc624a79da2198229b0c7ab3a34c94ffcba836 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 May 2020 14:42:06 +0300 Subject: [PATCH 1348/2107] cleanup the leftovers from page fixing via pg_ptrack_get_block() --- src/data.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/data.c b/src/data.c index 83e95c86a..6bdb6028b 100644 --- a/src/data.c +++ b/src/data.c @@ -353,17 +353,6 @@ prepare_page(ConnectionArgs *conn_arg, Assert(false); } } - - /* - * If ptrack support is available, use it to get invalid block - * instead of rereading it 99 times - */ - if (!page_is_valid && strict && ptrack_version_num > 0) - { - elog(WARNING, "File \"%s\", block %u, try to fetch via shared buffer", - from_fullpath, blknum); - break; - } } /* @@ -385,7 +374,7 @@ prepare_page(ConnectionArgs *conn_arg, /* Error out in case of merge or backup without ptrack support; * issue warning in case of checkdb or backup with ptrack support */ - if (!strict || (strict && ptrack_version_num > 0)) + if (!strict) elevel = WARNING; if (errormsg) @@ -396,16 +385,12 @@ prepare_page(ConnectionArgs *conn_arg, from_fullpath, blknum); pg_free(errormsg); + return PageIsCorrupted; } /* Checkdb not going futher */ if (!strict) - { - if (page_is_valid) - return PageIsOk; - else - return PageIsCorrupted; - } + return PageIsOk; } /* From 122f88ba56473ba9561961045c874ac07df6f139 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 May 2020 22:49:45 +0300 Subject: [PATCH 1349/2107] [PBCKP-101] fix accessing already freed memory --- src/restore.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/restore.c b/src/restore.c index 749f35572..5d50da01e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -99,7 +99,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { int i = 0; int j = 0; - parray *backups; + parray *backups = NULL; pgBackup *tmp_backup = NULL; pgBackup *current_backup = NULL; pgBackup *dest_backup = NULL; @@ -475,13 +475,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* ssh connection to longer needed */ fio_disconnect(); + elog(INFO, "%s of backup %s completed.", + action, base36enc(dest_backup->start_time)); + /* cleanup */ parray_walk(backups, pgBackupFree); parray_free(backups); parray_free(parent_chain); - elog(INFO, "%s of backup %s completed.", - action, base36enc(dest_backup->start_time)); return 0; } From 5699b137561d396a3ef88b1e937e855f26ab30d0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 31 May 2020 22:56:02 +0300 Subject: [PATCH 1350/2107] [PBCKP-98] initialize result variable in get_next_record_lsn --- src/parsexlog.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index e9410c880..5a33d3045 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -691,7 +691,8 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, { XLogReaderState *xlogreader; XLogReaderData reader_data; - XLogRecPtr startpoint, found, res; + XLogRecPtr startpoint, found; + XLogRecPtr res = InvalidXLogRecPtr; char wal_segment[MAXFNAMELEN]; int attempts = 0; From 6d98d46681db07a0f0f81cfa2daf6da350522dd5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Jun 2020 15:47:19 +0300 Subject: [PATCH 1351/2107] DOC: Fix dangling link --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 91ec1253b..73b2bf1b7 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4310,7 +4310,7 @@ pg_probackup archive-get -B backup_dir --instance --recovery-target-lsn or options. The default depends on the - recovery_target_inclusive + recovery_target_inclusive parameter. From 783f0163edb7abeb9d4c20d3968f59f8bde9642b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jun 2020 15:28:20 +0300 Subject: [PATCH 1352/2107] tests: fixes --- tests/backup.py | 2 +- tests/restore.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index be51dfefc..0a47c1337 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1927,7 +1927,7 @@ def test_basic_missing_file_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Cannot open source file', + 'ERROR: Cannot open file', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/restore.py b/tests/restore.py index 6ab3da335..0ffa59f7a 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -845,7 +845,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore destination is not empty:', + 'ERROR: Restore destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 683e491f1a302012cbde8a3bf0056bbb35e39578 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Jun 2020 15:31:47 +0300 Subject: [PATCH 1353/2107] [PBCKP-98]: fix tests --- tests/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index 4ec51eb6d..ab6eaf592 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -726,7 +726,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content) self.assertIn( - 'LOG: current.stop_lsn: 0/4000028', + 'LOG: stop_lsn: 0/4000028', log_content) # Clean after yourself From 703131e27984ffb7f64d1ce96d05ec2174196089 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 14:36:45 +0300 Subject: [PATCH 1354/2107] [Issue #188] Minor improvements --- src/validate.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/validate.c b/src/validate.c index 89a032336..3ef7cb6d7 100644 --- a/src/validate.c +++ b/src/validate.c @@ -58,13 +58,19 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) int i; // parray *dbOid_exclude_list = NULL; - /* Check backup version */ + /* Check backup program version */ if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + /* Check backup server version */ + if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) + elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " + "compiled with server version %s", + base36enc(backup->start_time), backup->server_version, PG_MAJORVERSION); + if (backup->status == BACKUP_STATUS_RUNNING) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", @@ -120,13 +126,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) return; } - if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) - { - elog(ERROR, "Backup was made with server version %s, but pg_probackup compiled " - "with server version %s.", - backup->server_version, PG_MAJORVERSION); - } - // if (params && params->partial_db_list) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // params->partial_restore_type); From 4e1ec276cbf4e43db40701b58198ff079a97fbea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 15:46:45 +0300 Subject: [PATCH 1355/2107] [Issue #188] minor fix for tests --- tests/validate.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index b7bc501d1..8e27c50cc 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3716,20 +3716,6 @@ def test_partial_validate_include(self): # Clean after yourself self.del_test_dir(module_name, fname) -# validate empty backup list -# page from future during validate -# page from future during backup - -# corrupt block, so file become unaligned: -# 712 Assert(header.compressed_size <= BLCKSZ); -# 713 -# 714 read_len = fread(compressed_page.data, 1, -# 715 MAXALIGN(header.compressed_size), in); -# 716 if (read_len != MAXALIGN(header.compressed_size)) -# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", -# 718 blknum, file->path, read_len, header.compressed_size); - - # @unittest.skip("skip") def test_not_validate_diffenent_pg_version(self): """Do not validate backup, if binary is compiled with different PG version""" @@ -3774,7 +3760,7 @@ def test_not_validate_diffenent_pg_version(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Backup was made with server version", + "ERROR: Backup {0} has server version".format(backup_id), e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -3782,3 +3768,16 @@ def test_not_validate_diffenent_pg_version(self): # Clean after yourself self.del_test_dir(module_name, fname) + +# validate empty backup list +# page from future during validate +# page from future during backup + +# corrupt block, so file become unaligned: +# 712 Assert(header.compressed_size <= BLCKSZ); +# 713 +# 714 read_len = fread(compressed_page.data, 1, +# 715 MAXALIGN(header.compressed_size), in); +# 716 if (read_len != MAXALIGN(header.compressed_size)) +# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", +# 718 blknum, file->path, read_len, header.compressed_size); \ No newline at end of file From 269ca37c87d7297d9969d0db19ba3f14b5186f20 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 16:46:18 +0300 Subject: [PATCH 1356/2107] [Issue #66] support external directories, handle timeline mismatch --- src/catalog.c | 2 +- src/data.c | 17 +++-- src/dir.c | 47 +++++++------- src/pg_probackup.h | 5 +- src/restore.c | 155 +++++++++++++++++++++++++++++++++++++-------- src/utils/file.c | 11 ++-- 6 files changed, 172 insertions(+), 65 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 7b3dfd861..a8a7dec4a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1048,7 +1048,7 @@ catalog_get_timelines(InstanceConfig *instance) TimeLineHistoryEntry *tln; sscanf(file->name, "%08X.history", &tli); - timelines = read_timeline_history(arclog_path, tli); + timelines = read_timeline_history(arclog_path, tli, true); if (!tlinfo || tlinfo->tli != tli) { diff --git a/src/data.c b/src/data.c index fd7a5143a..1af60bb91 100644 --- a/src/data.c +++ b/src/data.c @@ -933,7 +933,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, from_fullpath, to_fullpath, dest_file->n_blocks, use_bitmap ? &(dest_file)->pagemap : NULL, checksum_map, backup->checksum_version, - backup->start_lsn <= horizonLsn ? lsn_map : NULL); + /* shiftmap can be used only if backup state precedes the shift */ + backup->stop_lsn <= horizonLsn ? lsn_map : NULL); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -1026,7 +1027,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * special value PageIsTruncated is encountered. * It was inefficient. * - * Nowadays every backup type has n_blocks, so instead + * Nowadays every backup type has n_blocks, so instead of * writing and then truncating redundant data, writing * is not happening in the first place. * TODO: remove in 3.0.0 @@ -1097,7 +1098,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } /* Incremental restore - * TODO: move to another function + * TODO: move to separate function, + * TODO: move BackupPageHeader headers to some separate storage, + * so they can be accessed without reading through whole file. */ if (checksum_map && checksum_map[blknum].checksum != 0) { @@ -1110,7 +1113,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers char uncompressed_buf[BLCKSZ]; fio_decompress(uncompressed_buf, page.data, compressed_size, file->compress_alg); - /* If checksumms are enabled, then we can trust checksumm in header */ + /* If checksums are enabled, then we can trust checksum in header */ if (checksum_version) page_crc = ((PageHeader) uncompressed_buf)->pd_checksum; else @@ -1143,6 +1146,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers datapagemap_add(map, blknum); continue; } + +// elog(INFO, "Replace blknum %u in file %s", blknum, to_fullpath); } /* @@ -1313,7 +1318,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (file_crc == tmp_file->crc) { - elog(VERBOSE, "Remote nondata file \"%s\" is unchanged, skip restore", + elog(VERBOSE, "Remote nonedata file \"%s\" is unchanged, skip restore", to_fullpath); return 0; } @@ -1820,7 +1825,7 @@ check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* read local data file and construct map with block checksums */ PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { PageState *checksum_map = NULL; FILE *in = NULL; diff --git a/src/dir.c b/src/dir.c index bf5500300..f1e5adcb0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -123,11 +123,11 @@ typedef struct TablespaceCreatedList static int pgCompareString(const void *str1, const void *str2); -static char dir_check_file(pgFile *file); +static char dir_check_file(pgFile *file, bool backup_logs); static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool skip_hidden, - int external_dir_num, fio_location location); + bool exclude, bool follow_symlink, bool backup_logs, + bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); @@ -561,17 +561,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink return; } - /* setup exclusion list for file search */ - if (!backup_logs) - { - int i; - - for (i = 0; pgdata_exclude_dir[i]; i++); /* find first empty slot */ - - /* Set 'pg_log' in first empty slot */ - pgdata_exclude_dir[i] = PG_LOG_DIR; - } - if (!S_ISDIR(file->mode)) { if (external_dir_num > 0) @@ -585,7 +574,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink parray_append(files, file); dir_list_file_internal(files, file, root, exclude, follow_symlink, - skip_hidden, external_dir_num, location); + backup_logs, skip_hidden, external_dir_num, location); if (!add_root) pgFileFree(file); @@ -609,7 +598,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink * - datafiles */ static char -dir_check_file(pgFile *file) +dir_check_file(pgFile *file, bool backup_logs) { int i; int sscanf_res; @@ -623,7 +612,7 @@ dir_check_file(pgFile *file) if (!exclusive_backup) { for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) - if (strcmp(file->name, + if (strcmp(file->rel_path, pgdata_exclude_files_non_exclusive[i]) == 0) { /* Skip */ @@ -633,7 +622,7 @@ dir_check_file(pgFile *file) } for (i = 0; pgdata_exclude_files[i]; i++) - if (strcmp(file->name, pgdata_exclude_files[i]) == 0) + if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) { /* Skip */ elog(VERBOSE, "Excluding file: %s", file->name); @@ -660,6 +649,16 @@ dir_check_file(pgFile *file) return CHECK_EXCLUDE_FALSE; } } + + if (!backup_logs) + { + if (strcmp(file->rel_path, PG_LOG_DIR) == 0) + { + /* Skip */ + elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + return CHECK_EXCLUDE_FALSE; + } + } } /* @@ -807,8 +806,8 @@ dir_check_file(pgFile *file) */ static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool skip_hidden, - int external_dir_num, fio_location location) + bool exclude, bool follow_symlink, bool backup_logs, + bool skip_hidden, int external_dir_num, fio_location location) { DIR *dir; struct dirent *dent; @@ -874,7 +873,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, if (exclude) { - check_res = dir_check_file(file); + check_res = dir_check_file(file, backup_logs); if (check_res == CHECK_FALSE) { /* Skip */ @@ -897,7 +896,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, file, child, exclude, follow_symlink, - skip_hidden, external_dir_num, location); + backup_logs, skip_hidden, external_dir_num, location); } if (errno && errno != ENOENT) @@ -1288,7 +1287,7 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are } void -check_external_dir_mapping(pgBackup *backup) +check_external_dir_mapping(pgBackup *backup, bool incremental) { TablespaceListCell *cell; parray *external_dirs_to_restore; @@ -1341,7 +1340,7 @@ check_external_dir_mapping(pgBackup *backup) char *external_dir = (char *) parray_get(external_dirs_to_restore, i); - if (!dir_is_empty(external_dir, FIO_DB_HOST)) + if (!incremental && !dir_is_empty(external_dir, FIO_DB_HOST)) elog(ERROR, "External directory is not empty: \"%s\"", external_dir); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ef6f47605..1c8232c7b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -740,7 +740,8 @@ extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); extern parray *get_backup_filelist(pgBackup *backup, bool strict); -extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI); +extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); +extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); /* in merge.c */ extern void do_merge(time_t backup_id); @@ -888,7 +889,7 @@ extern void read_tablespace_map(parray *files, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty); -extern void check_external_dir_mapping(pgBackup *backup); +extern void check_external_dir_mapping(pgBackup *backup, bool incremental); extern char *get_external_remap(char *current_dir); extern void print_database_map(FILE *out, parray *database_list); diff --git a/src/restore.c b/src/restore.c index 4842b14d6..16616f214 100644 --- a/src/restore.c +++ b/src/restore.c @@ -223,7 +223,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, // elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ - timelines = read_timeline_history(arclog_path, rt->target_tli); + timelines = read_timeline_history(arclog_path, rt->target_tli, true); if (!satisfy_timeline(timelines, current_backup)) { @@ -354,7 +354,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* no point in checking external directories if their restore is not requested */ if (!params->skip_external_dirs) - check_external_dir_mapping(dest_backup); + check_external_dir_mapping(dest_backup, params->incremental); } /* At this point we are sure that parent chain is whole @@ -375,8 +375,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* - * Determine horizon LSN - * Consider the following example: + * Determine the shift-LSN + * Consider the example A: * * * /----D----------F-> @@ -385,51 +385,98 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * [A,F] - incremental chain * X - the state of pgdata * F - destination backup + * * - switch point * - * When running incremental-lsn restore, we get a bitmap of pages, - * whose LSN is less than C. + * When running incremental restore in shift mode, we get a bitmap of pages, + * whose LSN is less than shift-LSN (backup C stop_lsn). * So when restoring file, we can skip restore of pages coming from * A, B and C. * Pages from D and F cannot be skipped due to incremental restore. * + * Consider the example B: + * + * + * /----------X----> + * ----*---A---B---C--> + * + * [A,C] - incremental chain + * X - the state of pgdata + * C - destination backup + * * - switch point + * + * Incremental restore in shift mode IS NOT POSSIBLE in this case. + * We must be able to differentiate the scenario A and scenario B. + * */ if (params->is_restore && params->incremental && params->incremental_lsn) { RedoParams redo; + parray *timelines = NULL; get_redo(instance_config.pgdata, &redo); + timelines = read_timeline_history(arclog_path, redo.tli, false); + + if (!timelines) + elog(WARNING, "Failed to get history for redo timeline %i, " + "multi-timeline incremental restore in shift mode is impossible", redo.tli); + tmp_backup = dest_backup; while (tmp_backup) { - if (tmp_backup->start_lsn < redo.lsn && - redo.tli == tmp_backup->tli) + /* Candidate, whose stop_lsn if less than shift LSN, is found */ + if (tmp_backup->stop_lsn < redo.lsn) { - horizonLsn = tmp_backup->start_lsn; - break; - } + /* if candidate timeline is the same as redo TLI, + * then we are good to go. + */ + if (redo.tli == tmp_backup->tli) + { + elog(INFO, "Backup %s is chosen as shiftpoint", + base36enc(tmp_backup->start_time)); - if (!tmp_backup->parent_backup_link) - break; + horizonLsn = tmp_backup->stop_lsn; + break; + } + + if (!timelines) + { + elog(WARNING, "Redo timeline %i differs from target timeline %i, " + "in this case, to safely run incremental restore in shift mode, " + "the history file for timeline %i is mandatory", + redo.tli, tmp_backup->tli, redo.tli); + break; + } + + /* check whether the candidate tli is a part of redo TLI history */ + if (tliIsPartOfHistory(timelines, tmp_backup->tli)) + { + horizonLsn = tmp_backup->stop_lsn; + break; + } + else + elog(INFO, "Backup %s cannot be a shiftpoint, " + "because its tli %i is not in history of redo timeline %i", + base36enc(tmp_backup->start_time), tmp_backup->tli, redo.tli); + } tmp_backup = tmp_backup->parent_backup_link; } if (XLogRecPtrIsInvalid(horizonLsn)) - elog(ERROR, "Cannot perform lsn-based incremental restore of backup chain %s, " - "because destination directory redo point %X/%X on tli %i is less than " - "START LSN %X/%X of oldest backup in chain", + elog(ERROR, "Cannot perform incremental restore of backup chain %s in shift mode, " + "because destination directory redo point %X/%X on tli %i is out of reach", base36enc(dest_backup->start_time), - (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, - (uint32) (tmp_backup->start_lsn >> 32), (uint32) tmp_backup->start_lsn); + (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); else elog(INFO, "Destination directory redo point %X/%X on tli %i is within reach of " - "backup %s with START LSN %X/%X, lsn-based incremental restore is possible", + "backup %s with STOP LSN %X/%X on tli %i, incremental restore in shift mode is possible", (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, base36enc(tmp_backup->start_time), - (uint32) (tmp_backup->start_lsn >> 32), (uint32) tmp_backup->start_lsn); + (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, + tmp_backup->tli); - elog(INFO, "Horizon LSN: %X/%X", + elog(INFO, "shift LSN: %X/%X", (uint32) (horizonLsn >> 32), (uint32) horizonLsn); params->horizonLsn = horizonLsn; @@ -736,13 +783,34 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(INFO, "Extracting the content of destination directory for incremental restore"); - /* TODO: external directorues */ time(&start_time); if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); else dir_list_file(pgdata_files, pgdata_path, false, true, false, false, true, 0, FIO_LOCAL_HOST); + + /* get external dirs content */ + if (external_dirs) + { + for (i = 0; i < parray_num(external_dirs); i++) + { + char *external_path = parray_get(external_dirs, i); + parray *external_files = parray_new(); + + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(external_files, external_path, + false, true, false, false, true, i+1); + else + dir_list_file(external_files, external_path, + false, true, false, false, true, i+1, + FIO_LOCAL_HOST); + + parray_concat(pgdata_files, external_files); + parray_free(external_files); + } + } + parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc); time(&end_time); @@ -767,6 +835,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, fio_pgFileDelete(file, full_file_path); elog(WARNING, "Deleted file \"%s\"", full_file_path); + + /* shrink pgdata list */ + parray_remove(pgdata_files, i); + i--; } } @@ -1085,17 +1157,19 @@ restore_files(void *arg) already_exists); } - /* free pagemap used for restore optimization */ - pg_free(dest_file->pagemap.bitmap); - done: /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, strerror(errno)); + /* free pagemap used for restore optimization */ + pg_free(dest_file->pagemap.bitmap); + if (lsn_map) pg_free(lsn_map->bitmap); + + pg_free(lsn_map); pg_free(checksum_map); } @@ -1427,7 +1501,7 @@ pg12_recovery_config(pgBackup *backup, bool add_include) * based on readTimeLineHistory() in timeline.c */ parray * -read_timeline_history(const char *arclog_path, TimeLineID targetTLI) +read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict) { parray *result; char path[MAXPGPATH]; @@ -1451,8 +1525,11 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI) strerror(errno)); /* There is no history file for target timeline */ - elog(ERROR, "recovery target timeline %u does not exist", - targetTLI); + if (strict) + elog(ERROR, "recovery target timeline %u does not exist", + targetTLI); + else + return NULL; } } @@ -1559,6 +1636,28 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) } return false; } + +/* timelines represents a history of one particular timeline, + * we must determine whether a target tli is part of that history. + * + * /--------* + * ---------*--------------> + */ +bool +tliIsPartOfHistory(const parray *timelines, TimeLineID tli) +{ + int i; + + for (i = 0; i < parray_num(timelines); i++) + { + TimeLineHistoryEntry *timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + + if (tli == timeline->tli) + return true; + } + + return false; +} /* * Get recovery options in the string format, parse them * and fill up the pgRecoveryTarget structure. diff --git a/src/utils/file.c b/src/utils/file.c index 51f209ff6..6f599f219 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -213,7 +213,7 @@ static ssize_t fio_read_all(int fd, void* buf, size_t size) { if (errno == EINTR) continue; - elog(WARNING, "fio_read_all error: %s", strerror(errno)); + elog(ERROR, "fio_read_all error, fd %i: %s", fd, strerror(errno)); return rc; } else if (rc == 0) @@ -231,10 +231,13 @@ static ssize_t fio_write_all(int fd, void const* buf, size_t size) while (offs < size) { ssize_t rc = write(fd, (char*)buf + offs, size - offs); - if (rc <= 0) { - if (errno == EINTR) { + if (rc <= 0) + { + if (errno == EINTR) continue; - } + + elog(ERROR, "fio_write_all error, fd %i: %s", fd, strerror(errno)); + return rc; } offs += rc; From ae832a963efd84a3179765127f2e0070b7140a47 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 21:35:59 +0300 Subject: [PATCH 1357/2107] tests: added new module "incr_restore" --- tests/__init__.py | 3 +- tests/incr_restore.py | 920 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 922 insertions(+), 1 deletion(-) create mode 100644 tests/incr_restore.py diff --git a/tests/__init__.py b/tests/__init__.py index cd3088ee5..dbf84feea 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,7 @@ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external, config, checkdb, set_backup + locking, remote, external, config, checkdb, set_backup, incr_restore def load_tests(loader, tests, pattern): @@ -36,6 +36,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(external)) suite.addTests(loader.loadTestsFromModule(false_positive)) suite.addTests(loader.loadTestsFromModule(init)) + suite.addTests(loader.loadTestsFromModule(incr_restore)) suite.addTests(loader.loadTestsFromModule(locking)) suite.addTests(loader.loadTestsFromModule(logging)) suite.addTests(loader.loadTestsFromModule(merge)) diff --git a/tests/incr_restore.py b/tests/incr_restore.py new file mode 100644 index 000000000..02475c269 --- /dev/null +++ b/tests/incr_restore.py @@ -0,0 +1,920 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +from time import sleep +from datetime import datetime, timedelta +import hashlib +import shutil +import json +from testgres import QueryException + + +module_name = 'restore' + + +class IncrRestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_incr_restore(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + def test_checksum_corruption_detection(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + # corrupt block + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + tblspace = self.get_tblspace_path(node, 'tblspace') + some_directory = self.get_tblspace_path(node, 'some_directory') + + self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--incremental", + "-T{0}={1}".format(tblspace, some_directory)]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_1(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + tblspace = self.get_tblspace_path(node, 'tblspace') + some_directory = self.get_tblspace_path(node, 'some_directory') + + self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_2(self): + """ + If "--tablespace-mapping" option is used with incremental restore, + then new directory must be empty. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + tblspace = self.get_tblspace_path(node, 'tblspace') + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental']) + + self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental', '-T{0}={1}'.format(tblspace, tblspace)]) + + exit(1) + + pgdata = self.pgdata_content(node.data_dir) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_sanity(self): + """recovery to target timeline""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + try: + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because there is running postmaster " + "process in destination directory.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Postmaster with pid', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is impossible', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + + try: + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=["-j", "4", "--incremental"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because destination directory has wrong system id.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup catalog was initialized for system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is impossible', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_checksum_restore(self): + """ + /----C-----D + ------A----B---*--------X + + X - is instance, we want to return it to C state. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + # --A-----B--------X + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=[ + '--recovery-target-action=promote', + '--recovery-target-xid={0}'.format(xid)]) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + # /-- + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # /--C + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + # /--C------ + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) + pgbench.wait() + + # /--C------D + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node_1.data_dir) + + print(self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental"])) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + def test_incr_lsn_restore(self): + """ + /----C-----D + ------A----B---*--------X + + X - is instance, we want to return it to C state. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + # --A-----B--------X + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=[ + '--recovery-target-action=promote', + '--recovery-target-xid={0}'.format(xid)]) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + # /-- + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # /--C + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + # /--C------ + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) + pgbench.wait() + + # /--C------D + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node_1.data_dir) + + print(self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-lsn"])) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) + + exit(1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_shift_sanity(self): + """ + /----A-----B + F------*--------X + + X - is instance, we want to return it to state B. + fail is expected behaviour in case of shift restore. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=10) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='full') + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-lsn"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in shift mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Cannot perform incremental restore of ' + 'backup chain {0} in shift mode'.format(page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_checksum_sanity(self): + """ + /----A-----B + F------*--------X + + X - is instance, we want to return it to state B. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='full') + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + pgdata = self.pgdata_content(node_1.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + # @unittest.skip("skip") + def test_incr_checksum_corruption_detection(self): + """ + check that corrupted page got detected and replaced + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), +# initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='full') + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: + f.seek(22000) + f.write(b"bla") + f.flush() + f.close + + print(self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental"])) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_shift_corruption_detection(self): + """ + check that corrupted page got detected and replaced + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='full') + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: + f.seek(22000) + f.write(b"bla") + f.flush() + f.close + + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-lsn"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_multiple_external(self): + """check that cmdline has priority over config""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + node.pgbench_init(scale=20) + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=['-E{0}:{1}'.format(external_dir1, external_dir2)]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", '--incremental', '--log-level-console=VERBOSE'])) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_shift_restore_multiple_external(self): + """check that cmdline has priority over config""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + node.pgbench_init(scale=20) + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=['-E{0}:{1}'.format(external_dir1, external_dir2)]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", '--incremental-lsn'])) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + +# check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn +# incremental restore + partial restore. From 0a2e5bee1bdab79ed96382a806b918507ca053f0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 23:33:17 +0300 Subject: [PATCH 1358/2107] [Issue #219] Make add-instance idempotent --- src/init.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/init.c b/src/init.c index 52cadc25f..431ea3b70 100644 --- a/src/init.c +++ b/src/init.c @@ -67,20 +67,19 @@ do_add_instance(InstanceConfig *instance) /* Ensure that all root directories already exist */ if (access(backup_path, F_OK) != 0) - elog(ERROR, "%s directory does not exist.", backup_path); + elog(ERROR, "Directory does not exist: '%s'", backup_path); join_path_components(path, backup_path, BACKUPS_DIR); if (access(path, F_OK) != 0) - elog(ERROR, "%s directory does not exist.", path); + elog(ERROR, "Directory does not exist: '%s'", path); join_path_components(arclog_path_dir, backup_path, "wal"); if (access(arclog_path_dir, F_OK) != 0) - elog(ERROR, "%s directory does not exist.", arclog_path_dir); + elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir); - /* Create directory for data files of this specific instance */ if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "instance '%s' already exists", instance->backup_instance_path); - dir_create_dir(instance->backup_instance_path, DIR_PERMISSION); + elog(ERROR, "Instance '%s' backup directory already exists: '%s'", + instance->name, instance->backup_instance_path); /* * Create directory for wal files of this specific instance. @@ -88,7 +87,11 @@ do_add_instance(InstanceConfig *instance) * directory in data dir, we shouldn't have it in wal as well. */ if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "arclog_path '%s' already exists", instance->arclog_path); + elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'", + instance->name, instance->arclog_path); + + /* Create directory for data files of this specific instance */ + dir_create_dir(instance->backup_instance_path, DIR_PERMISSION); dir_create_dir(instance->arclog_path, DIR_PERMISSION); /* From 618fa3f7edebd277b2289cb702bb8bae467902a3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Jun 2020 23:33:38 +0300 Subject: [PATCH 1359/2107] [Issue #219] added test tests.init.InitTest.test_add_instance_idempotence --- tests/init.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/init.py b/tests/init.py index 059e24d95..f5715d249 100644 --- a/tests/init.py +++ b/tests/init.py @@ -1,6 +1,7 @@ import os import unittest from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException +import shutil module_name = 'init' @@ -104,3 +105,53 @@ def test_abs_path(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_add_instance_idempotence(self): + """ + https://github.com/postgrespro/pg_probackup/issues/219 + """ + fname = self.id().split(".")[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + self.init_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node) + shutil.rmtree(os.path.join(backup_dir, 'backups', 'node')) + + dir_backups = os.path.join(backup_dir, 'backups', 'node') + dir_wal = os.path.join(backup_dir, 'wal', 'node') + + try: + self.add_instance(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' WAL archive directory already exists: ", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.add_instance(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' WAL archive directory already exists: ", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From e31bd6c88e26e3e1980ba3ed0041fbdebd374a87 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 6 Jun 2020 15:02:26 +0300 Subject: [PATCH 1360/2107] tests: minor fix in "merge" module --- tests/merge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/merge.py b/tests/merge.py index 60c2285f5..00f7345c3 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -3,6 +3,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import QueryException import shutil from datetime import datetime, timedelta import time From 889465b0ecd079ea11f82e777a5a8c4cc3f19597 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jun 2020 16:51:13 +0300 Subject: [PATCH 1361/2107] tests: fixes for 2.3.5 --- tests/merge.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 00f7345c3..54b7e2884 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -176,10 +176,7 @@ def test_merge_compressed_backups_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } - ) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) @@ -187,7 +184,7 @@ def test_merge_compressed_backups_1(self): node.slow_start() # Fill with data - node.pgbench_init(scale=5) + node.pgbench_init(scale=10) # Do compressed FULL backup self.backup_node(backup_dir, "node", node, options=['--compress', '--stream']) @@ -197,7 +194,7 @@ def test_merge_compressed_backups_1(self): self.assertEqual(show_backup["backup-mode"], "FULL") # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed DELTA backup @@ -206,7 +203,7 @@ def test_merge_compressed_backups_1(self): backup_type="delta", options=['--compress', '--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed PAGE backup @@ -1479,10 +1476,12 @@ def test_crash_after_opening_backup_control_1(self): self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_crash_after_opening_backup_control_2(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files + TODO: rewrite """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1532,7 +1531,7 @@ def test_crash_after_opening_backup_control_2(self): gdb.run_until_break() gdb.set_breakpoint('sprintf') - gdb.continue_execution_until_break(20) + gdb.continue_execution_until_break(1) gdb._execute('signal SIGKILL') @@ -1569,10 +1568,12 @@ def test_crash_after_opening_backup_control_2(self): self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_losing_file_after_failed_merge(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files + TODO: rewrite """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From c926ae90ea0b09ec1ca94111e4df3265c55827f9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jun 2020 16:52:15 +0300 Subject: [PATCH 1362/2107] print correct stop_lsn after execution of pg_stop_backup() --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index aff0dbe4a..bdd9719d3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1865,7 +1865,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, } elog(LOG, "stop_lsn: %X/%X", - (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn)); + (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); /* Write backup_label and tablespace_map */ if (!exclusive_backup) From 7cf1d5bca2c3328b69bbac3e62c8f3c36abf3758 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Jun 2020 16:59:13 +0300 Subject: [PATCH 1363/2107] Version 2.3.5 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7c02d245f..3d773e4d8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -229,8 +229,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.3.4" -#define AGENT_PROTOCOL_VERSION 20304 +#define PROGRAM_VERSION "2.3.5" +#define AGENT_PROTOCOL_VERSION 20305 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index b271f8264..abc5c2424 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.4 \ No newline at end of file +pg_probackup 2.3.5 \ No newline at end of file From 7e43e359afe6f4a6154cb0711dae7c3e6001a8cf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jun 2020 14:53:14 +0300 Subject: [PATCH 1364/2107] Readme: update for 2.3.5 --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6f95b1dae..c73a13683 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4) +[2.3.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.5) ## Documentation @@ -96,19 +96,19 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #SRPM Packages yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} -#RPM ALT Linux p7 +#RPM ALT Linux 7 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo -#RPM ALT Linux p8 +#RPM ALT Linux 8 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo -#RPM ALT Linux p9 +#RPM ALT Linux 9 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} @@ -120,40 +120,40 @@ sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{11,10,9.6} -yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{12,11,10,9.6} +yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo -#RPM ALT Linux p7 +#RPM ALT Linux 7 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo -#RPM ALT Linux p8 +#RPM ALT Linux 8 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo -#RPM ALT Linux p9 +#RPM ALT Linux 9 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). From f7edee6bdb64f319df62e4600f1d0acaf1f2a147 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Jun 2020 16:47:24 +0300 Subject: [PATCH 1365/2107] Readme: update link to Windows installers --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c73a13683..813abbcc8 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.3). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.5). ### Linux Installation #### pg_probackup for vanilla PostgreSQL @@ -115,7 +115,7 @@ sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo ``` -#### pg_probackup for PostgresPro Standart and Enterprise +#### pg_probackup for PostgresPro Standard and Enterprise ```shell #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list From 734e4951f28e8bd24be51d2a79ace3ca02b0011d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 01:06:14 +0300 Subject: [PATCH 1366/2107] [Issue #225] ptrack LSN should be compared with parent Start LSN instead of parent Stop LSN --- src/backup.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index bdd9719d3..c19664942 100644 --- a/src/backup.c +++ b/src/backup.c @@ -237,19 +237,19 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync) /* * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not - * equal to stop_lsn of previous backup. + * equal to start_lsn of previous backup. */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); - if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) { - elog(ERROR, "LSN from ptrack_control %X/%X differs from STOP LSN of previous backup %X/%X.\n" + elog(ERROR, "LSN from ptrack_control %X/%X differs from Start LSN of previous backup %X/%X.\n" "Create new full backup before an incremental one.", (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (prev_backup->stop_lsn >> 32), - (uint32) (prev_backup->stop_lsn)); + (uint32) (prev_backup->start_lsn >> 32), + (uint32) (prev_backup->start_lsn)); } } From 8aff48f1024981a154a67dee5d65b58bcdcaebc9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 01:35:28 +0300 Subject: [PATCH 1367/2107] [Issue #225] fix tests --- tests/ptrack.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 7d304d52e..8d878030e 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -413,16 +413,9 @@ def test_ptrack_uncommitted_xact(self): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - # TODO: what's the point in taking pgdata content, then taking - # backup, and the trying to compare those two? Backup issues a - # checkpoint, so it will modify pgdata with close to 100% chance. if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream']) - node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -3954,7 +3947,7 @@ def test_ptrack_pg_resetxlog(self): ) except ProbackupException as e: self.assertIn( - 'ERROR: LSN from ptrack_control 0/0 differs from STOP LSN of previous backup', + 'ERROR: LSN from ptrack_control 0/0 differs from Start LSN of previous backup', e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) From 3475ef663114d68152ef00c6a8b3a18b4dbdf5f9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 13:25:22 +0300 Subject: [PATCH 1368/2107] [PBCKP-98] increase PAGE_READ_ATTEMPTS up to 300 --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3d773e4d8..3122293f5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -88,7 +88,7 @@ extern const char *PROGRAM_EMAIL; #define STDIO_BUFSIZE 65536 /* retry attempts */ -#define PAGE_READ_ATTEMPTS 100 +#define PAGE_READ_ATTEMPTS 300 /* max size of note, that can be added to backup */ #define MAX_NOTE_SIZE 1024 From 7bdc6820751bfaeefb363d3d639ad5d876bec490 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 15:00:40 +0300 Subject: [PATCH 1369/2107] [Issue #66] added "-I | --incremenal-mode" option --- src/pg_probackup.c | 36 +++++++++++++----- src/pg_probackup.h | 16 ++++++-- src/restore.c | 91 ++++++++++++++++++++++------------------------ 3 files changed, 83 insertions(+), 60 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index cd5b440f3..9bebc5926 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -98,8 +98,7 @@ static pgRestoreParams *restore_params = NULL; time_t current_time = 0; bool restore_as_replica = false; bool no_validate = false; -bool incremental = false; -bool incremental_lsn = false; +IncrRestoreMode incremental_mode = INCR_NONE; bool skip_block_validation = false; bool skip_external_dirs = false; @@ -152,6 +151,7 @@ static ProbackupSubcmd backup_subcmd = NO_CMD; static bool help_opt = false; +static void opt_incr_restore_mode(ConfigOption *opt, const char *arg); static void opt_backup_mode(ConfigOption *opt, const char *arg); static void opt_show_format(ConfigOption *opt, const char *arg); @@ -205,8 +205,7 @@ static ConfigOption cmd_options[] = { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, - { 'b', 161, "incremental", &incremental, SOURCE_CMD_STRICT }, - { 'b', 167, "incremental-lsn", &incremental_lsn, SOURCE_CMD_STRICT }, + { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -704,9 +703,6 @@ main(int argc, char *argv[]) if (replication_slot != NULL) restore_as_replica = true; - if (!incremental && incremental_lsn) - incremental = true; - /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); @@ -719,8 +715,7 @@ main(int argc, char *argv[]) restore_params->partial_db_list = NULL; restore_params->partial_restore_type = NONE; restore_params->primary_conninfo = primary_conninfo; - restore_params->incremental = incremental; - restore_params->incremental_lsn = incremental_lsn; + restore_params->incremental_mode = incremental_mode; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) @@ -878,6 +873,29 @@ main(int argc, char *argv[]) return 0; } +static void +opt_incr_restore_mode(ConfigOption *opt, const char *arg) +{ + if (pg_strcasecmp(arg, "none") == 0) + { + incremental_mode = INCR_NONE; + return; + } + else if (pg_strcasecmp(arg, "checksum") == 0) + { + incremental_mode = INCR_CHECKSUM; + return; + } + else if (pg_strcasecmp(arg, "lsn") == 0) + { + incremental_mode = INCR_LSN; + return; + } + + /* Backup mode is invalid, so leave with an error */ + elog(ERROR, "Invalid value for '--incremental-mode' option: '%s'", arg); +} + static void opt_backup_mode(ConfigOption *opt, const char *arg) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1c8232c7b..412843f70 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -119,6 +119,13 @@ typedef struct db_map_entry char *datname; } db_map_entry; +typedef enum IncrRestoreMode +{ + INCR_NONE, + INCR_CHECKSUM, + INCR_LSN +} IncrRestoreMode; + typedef enum PartialRestoreType { NONE, @@ -453,16 +460,17 @@ typedef struct pgRestoreParams bool restore_as_replica; bool skip_external_dirs; bool skip_block_validation; //Start using it - bool incremental; - bool incremental_lsn; - XLogRecPtr horizonLsn; const char *restore_command; const char *primary_slot_name; + const char *primary_conninfo; + + /* options for incremental restore */ + IncrRestoreMode incremental_mode; + XLogRecPtr shift_lsn; /* options for partial restore */ PartialRestoreType partial_restore_type; parray *partial_db_list; - const char *primary_conninfo; } pgRestoreParams; /* Options needed for set-backup command */ diff --git a/src/restore.c b/src/restore.c index 16616f214..95bce8348 100644 --- a/src/restore.c +++ b/src/restore.c @@ -29,9 +29,8 @@ typedef struct const char *to_root; size_t restored_bytes; bool use_bitmap; - bool incremental; - bool incremental_lsn; - XLogRecPtr horizonLsn; + IncrRestoreMode incremental_mode; + XLogRecPtr shift_lsn; /* used only in LSN incremental_mode */ /* * Return value from the thread. @@ -52,7 +51,7 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync); static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - bool incremental_lsn); + IncrRestoreMode incremental_mode); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -117,7 +116,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray *dbOid_exclude_list = NULL; bool pgdata_is_empty = true; bool tblspaces_are_empty = true; - XLogRecPtr horizonLsn = InvalidXLogRecPtr; + XLogRecPtr shift_lsn = InvalidXLogRecPtr; if (params->is_restore) { @@ -128,14 +127,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) { /* Check that remote system is NOT running and systemd id is the same as ours */ - if (params->incremental) + if (params->incremental_mode != INCR_NONE) { elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", instance_config.pgdata); check_incremental_compatibility(instance_config.pgdata, instance_config.system_identifier, - params->incremental_lsn); + params->incremental_mode); } else elog(ERROR, "Restore destination is not empty: \"%s\"", @@ -328,13 +327,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time)); } - /* - * We have found full backup by link, - * now we need to walk the list to find its index. - * - * TODO I think we should rewrite it someday to use double linked list - * and avoid relying on sort order anymore. - */ + /* We have found full backup */ base_full_backup = tmp_backup; } @@ -347,14 +340,18 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - check_tablespace_mapping(dest_backup, params->incremental, &tblspaces_are_empty); + check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty); if (pgdata_is_empty && tblspaces_are_empty) - params->incremental = false; + { + elog(INFO, "Destination directory and tablespace directories are empty, " + "disabled incremental restore"); + params->incremental_mode = INCR_NONE; + } /* no point in checking external directories if their restore is not requested */ if (!params->skip_external_dirs) - check_external_dir_mapping(dest_backup, params->incremental); + check_external_dir_mapping(dest_backup, params->incremental_mode != INCR_NONE); } /* At this point we are sure that parent chain is whole @@ -387,7 +384,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * F - destination backup * * - switch point * - * When running incremental restore in shift mode, we get a bitmap of pages, + * When running incremental restore in 'lsn' mode, we get a bitmap of pages, * whose LSN is less than shift-LSN (backup C stop_lsn). * So when restoring file, we can skip restore of pages coming from * A, B and C. @@ -408,7 +405,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * We must be able to differentiate the scenario A and scenario B. * */ - if (params->is_restore && params->incremental && params->incremental_lsn) + if (params->is_restore && params->incremental_mode == INCR_LSN) { RedoParams redo; parray *timelines = NULL; @@ -418,7 +415,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (!timelines) elog(WARNING, "Failed to get history for redo timeline %i, " - "multi-timeline incremental restore in shift mode is impossible", redo.tli); + "multi-timeline incremental restore in 'lsn' mode is impossible", redo.tli); tmp_backup = dest_backup; @@ -432,17 +429,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (redo.tli == tmp_backup->tli) { - elog(INFO, "Backup %s is chosen as shiftpoint", + elog(INFO, "Backup %s is chosen as shiftpoint, its Stop LSN will be used as shift LSN", base36enc(tmp_backup->start_time)); - horizonLsn = tmp_backup->stop_lsn; + shift_lsn = tmp_backup->stop_lsn; break; } if (!timelines) { elog(WARNING, "Redo timeline %i differs from target timeline %i, " - "in this case, to safely run incremental restore in shift mode, " + "in this case, to safely run incremental restore in 'lsn' mode, " "the history file for timeline %i is mandatory", redo.tli, tmp_backup->tli, redo.tli); break; @@ -451,7 +448,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* check whether the candidate tli is a part of redo TLI history */ if (tliIsPartOfHistory(timelines, tmp_backup->tli)) { - horizonLsn = tmp_backup->stop_lsn; + shift_lsn = tmp_backup->stop_lsn; break; } else @@ -463,23 +460,23 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, tmp_backup = tmp_backup->parent_backup_link; } - if (XLogRecPtrIsInvalid(horizonLsn)) - elog(ERROR, "Cannot perform incremental restore of backup chain %s in shift mode, " + if (XLogRecPtrIsInvalid(shift_lsn)) + elog(ERROR, "Cannot perform incremental restore of backup chain %s in 'lsn' mode, " "because destination directory redo point %X/%X on tli %i is out of reach", base36enc(dest_backup->start_time), (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); else elog(INFO, "Destination directory redo point %X/%X on tli %i is within reach of " - "backup %s with STOP LSN %X/%X on tli %i, incremental restore in shift mode is possible", + "backup %s with Stop LSN %X/%X on tli %i, incremental restore in 'lsn' mode is possible", (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, base36enc(tmp_backup->start_time), (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, tmp_backup->tli); elog(INFO, "shift LSN: %X/%X", - (uint32) (horizonLsn >> 32), (uint32) horizonLsn); + (uint32) (shift_lsn >> 32), (uint32) shift_lsn); - params->horizonLsn = horizonLsn; + params->shift_lsn = shift_lsn; } /* for validation or restore with enabled validation */ @@ -706,7 +703,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { use_bitmap = false; - if (params->incremental) + if (params->incremental_mode != INCR_NONE) elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version"); } @@ -715,7 +712,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, */ if (use_bitmap && parray_num(parent_chain) == 1) { - if (params->incremental && params->incremental_lsn) + if (params->incremental_mode == INCR_NONE) use_bitmap = true; else use_bitmap = false; @@ -726,7 +723,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, */ create_data_directories(dest_files, instance_config.pgdata, dest_backup->root_dir, true, - params->incremental, FIO_DB_HOST); + params->incremental_mode != INCR_NONE, + FIO_DB_HOST); /* * Restore dest_backup external directories. @@ -777,7 +775,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } /* Get list of files in destination directory and remove redundant files */ - if (params->incremental) + if (params->incremental_mode != INCR_NONE) { pgdata_files = parray_new(); @@ -883,9 +881,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, arg->skip_external_dirs = params->skip_external_dirs; arg->to_root = pgdata_path; arg->use_bitmap = use_bitmap; - arg->incremental = params->incremental; - arg->incremental_lsn = params->incremental_lsn; - arg->horizonLsn = params->horizonLsn; + arg->incremental_mode = params->incremental_mode; + arg->shift_lsn = params->shift_lsn; threads_args[i].restored_bytes = 0; /* By default there are some error */ threads_args[i].ret = 1; @@ -1080,7 +1077,7 @@ restore_files(void *arg) join_path_components(to_fullpath, external_path, dest_file->rel_path); } - if (arguments->incremental && + if (arguments->incremental_mode != INCR_NONE && parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc)) { already_exists = true; @@ -1096,14 +1093,13 @@ restore_files(void *arg) dest_file->is_datafile && !dest_file->is_cfs && dest_file->n_blocks > 0) { - if (arguments->incremental_lsn) + if (arguments->incremental_mode == INCR_LSN) { - /* TODO: return lsn_map */ lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version, - dest_file->n_blocks, arguments->horizonLsn, + dest_file->n_blocks, arguments->shift_lsn, dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); } - else + else if (arguments->incremental_mode == INCR_CHECKSUM) { checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->dest_backup->stop_lsn, @@ -1144,7 +1140,7 @@ restore_files(void *arg) arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file, out, to_fullpath, arguments->use_bitmap, checksum_map, - arguments->horizonLsn, lsn_map); + arguments->shift_lsn, lsn_map); } else { @@ -1943,7 +1939,8 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, * Depending on type of incremental restore requirements are differs. */ void -check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool incremental_lsn) +check_incremental_compatibility(const char *pgdata, uint64 system_identifier, + IncrRestoreMode incremental_mode) { uint64 system_id_pgdata; bool success = true; @@ -1993,16 +1990,16 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bo * TODO: maybe there should be some other signs, pointing to pg_control * desynchronization with cluster state. */ - if (incremental_lsn) + if (incremental_mode == INCR_LSN) { snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata); if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " "This does NOT mean that you should delete this file and retry, only that " - "lsn-based incremental restore can corrupt data, when applied to instance " - "with pg_control not synchronized with cluster state." - "Consider to use checksum-based incremental restore."); + "incremental restore in 'lsn' mode can produce incorrect result, when applied " + "to cluster with pg_control not synchronized with cluster state." + "Consider to use incremental restore in 'checksum' mode"); success = false; } } From 20796bcb62d6574fd03ad386735548fb926dc635 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 20:07:10 +0300 Subject: [PATCH 1370/2107] do not force the tablespace mapping target directory to be empty --- src/dir.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/dir.c b/src/dir.c index f1e5adcb0..0cddba142 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1242,16 +1242,16 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are cell->old_dir); /* For incremental restore, check that new directory is empty */ - if (incremental) - { - if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", - cell->new_dir); - - if (!dir_is_empty(cell->new_dir, FIO_DB_HOST)) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - cell->new_dir); - } +// if (incremental) +// { +// if (!is_absolute_path(cell->new_dir)) +// elog(ERROR, "tablespace directory is not an absolute path: %s\n", +// cell->new_dir); +// +// if (!dir_is_empty(cell->new_dir, FIO_DB_HOST)) +// elog(ERROR, "restore tablespace destination is not empty: \"%s\"", +// cell->new_dir); +// } } /* 2 - all linked directories must be empty */ From cf4d2b73ef15f541c18532ba3986ea64a6bfc79c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 20:07:46 +0300 Subject: [PATCH 1371/2107] tests: fixes for 'incr_restore' module --- tests/incr_restore.py | 53 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 02475c269..e0b95c21c 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -70,7 +70,8 @@ def test_basic_incr_restore(self): node.stop() self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental"]) + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -128,7 +129,7 @@ def test_checksum_corruption_detection(self): # corrupt block self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental"]) + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -138,7 +139,8 @@ def test_checksum_corruption_detection(self): # @unittest.skip("skip") def test_incr_restore_with_tablespace(self): - """recovery to target timeline""" + """ + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -154,6 +156,7 @@ def test_incr_restore_with_tablespace(self): tblspace = self.get_tblspace_path(node, 'tblspace') some_directory = self.get_tblspace_path(node, 'some_directory') + # stuff new destination with garbage self.restore_node(backup_dir, 'node', node, data_dir=some_directory) self.create_tblspace_in_node(node, 'tblspace') @@ -167,12 +170,14 @@ def test_incr_restore_with_tablespace(self): self.restore_node( backup_dir, 'node', node, options=[ - "-j", "4", "--incremental", + "-j", "4", "--incremental-mode=checksum", "-T{0}={1}".format(tblspace, some_directory)]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + exit(1) + # Clean after yourself self.del_test_dir(module_name, fname) @@ -226,7 +231,7 @@ def test_incr_restore_with_tablespace_1(self): self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental"]) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -257,6 +262,8 @@ def test_incr_restore_with_tablespace_2(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + node_1 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_1')) @@ -265,18 +272,14 @@ def test_incr_restore_with_tablespace_2(self): self.restore_node( backup_dir, 'node', node, data_dir=node_1.data_dir, - options=['--incremental']) + options=['--incremental-mode=checksum']) self.restore_node( backup_dir, 'node', node, data_dir=node_1.data_dir, - options=['--incremental', '-T{0}={1}'.format(tblspace, tblspace)]) + options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) - exit(1) - - pgdata = self.pgdata_content(node.data_dir) - - pgdata_restored = self.pgdata_content(node.data_dir) + pgdata_restored = self.pgdata_content(node_1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -301,7 +304,7 @@ def test_incr_restore_sanity(self): try: self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental"]) + options=["-j", "4", "--incremental-mode=checksum"]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -327,7 +330,7 @@ def test_incr_restore_sanity(self): try: self.restore_node( backup_dir, 'node', node_1, data_dir=node_1.data_dir, - options=["-j", "4", "--incremental"]) + options=["-j", "4", "--incremental-mode=checksum"]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -426,7 +429,7 @@ def test_incr_checksum_restore(self): print(self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental"])) + options=["-j", "4", "--incremental-mode=checksum"])) pgdata_restored = self.pgdata_content(node.data_dir) @@ -515,7 +518,7 @@ def test_incr_lsn_restore(self): pgdata = self.pgdata_content(node_1.data_dir) print(self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-lsn"])) + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"])) pgdata_restored = self.pgdata_content(node.data_dir) @@ -524,8 +527,6 @@ def test_incr_lsn_restore(self): self.compare_pgdata(pgdata, pgdata_restored) - exit(1) - # Clean after yourself self.del_test_dir(module_name, fname) @@ -580,7 +581,7 @@ def test_incr_shift_sanity(self): try: self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-lsn"]) + options=["-j", "4", "--incremental-mode=lsn"]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -589,8 +590,8 @@ def test_incr_shift_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Cannot perform incremental restore of ' - 'backup chain {0} in shift mode'.format(page_id), + "ERROR: Cannot perform incremental restore of " + "backup chain {0} in 'lsn' mode".format(page_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -648,7 +649,7 @@ def test_incr_checksum_sanity(self): self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental"]) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -707,7 +708,7 @@ def test_incr_checksum_corruption_detection(self): print(self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental"])) + options=["-j", "4", "--incremental-mode=checksum"])) pgdata_restored = self.pgdata_content(node.data_dir) @@ -765,7 +766,7 @@ def test_incr_shift_corruption_detection(self): self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-lsn"]) + options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -836,7 +837,7 @@ def test_incr_restore_multiple_external(self): print(self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental', '--log-level-console=VERBOSE'])) + options=["-j", "4", '--incremental-mode=checksum', '--log-level-console=VERBOSE'])) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -907,7 +908,7 @@ def test_incr_shift_restore_multiple_external(self): print(self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental-lsn'])) + options=["-j", "4", '--incremental-mode=lsn'])) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) From b2830e9dd0615718e75f94cd4f5474f64967dfd2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Jun 2020 21:32:00 +0300 Subject: [PATCH 1372/2107] [Issue #66] update help --- src/help.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/help.c b/src/help.c index 153fe6aab..30604d193 100644 --- a/src/help.c +++ b/src/help.c @@ -160,6 +160,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); + printf(_(" [--incremental-mode=none|checksum|lsn\n")); printf(_(" [--no-sync]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -397,6 +398,7 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [--incremental-mode=none|checksum|lsn\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -430,6 +432,9 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); + printf(_(" reuse already existing unchanged valid pages in PGDATA\n")); + printf(_(" (default: none)\n")); printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); From 62db4921058fc1610ee599e87e296a8ea41c8e99 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 00:01:45 +0300 Subject: [PATCH 1373/2107] [Issue #66] test fix --- tests/incr_restore.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index e0b95c21c..9f169b7f0 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -176,8 +176,6 @@ def test_incr_restore_with_tablespace(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - exit(1) - # Clean after yourself self.del_test_dir(module_name, fname) From ff6e11ec8244d34448d35a0e3681ae074f28fff0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 00:02:49 +0300 Subject: [PATCH 1374/2107] [Issue #66] rename horizonLsn to shift_lsn --- src/data.c | 10 +++++----- src/pg_probackup.h | 4 ++-- src/utils/file.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/data.c b/src/data.c index 1af60bb91..c19d64897 100644 --- a/src/data.c +++ b/src/data.c @@ -848,7 +848,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr horizonLsn, datapagemap_t *lsn_map) + XLogRecPtr shift_lsn, datapagemap_t *lsn_map) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); @@ -934,7 +934,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, use_bitmap ? &(dest_file)->pagemap : NULL, checksum_map, backup->checksum_version, /* shiftmap can be used only if backup state precedes the shift */ - backup->stop_lsn <= horizonLsn ? lsn_map : NULL); + backup->stop_lsn <= shift_lsn ? lsn_map : NULL); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -1893,7 +1893,7 @@ PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, /* return bitmap of valid blocks, bitmap is empty, then NULL is returned */ datapagemap_t * get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno) + int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) { FILE *in = NULL; BlockNumber blknum = 0; @@ -1907,7 +1907,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", n_blocks, fullpath, strerror(errno)); - Assert(horizonLsn > 0); + Assert(shift_lsn > 0); /* open file */ in = fopen(fullpath, PG_BINARY_R); @@ -1931,7 +1931,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, if (read_len == BLCKSZ) { int rc = validate_one_page(read_buffer, segmentno + blknum, - horizonLsn, &page_lsn, checksum_version); + shift_lsn, &page_lsn, checksum_version); if (rc == PAGE_IS_VALID) datapagemap_add(lsn_map, blknum); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 412843f70..db38d4e14 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -962,7 +962,7 @@ extern void backup_non_data_file_internal(const char *from_fullpath, extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr horizonLsn, datapagemap_t *lsn_map); + XLogRecPtr shift_lsn, datapagemap_t *lsn_map); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, @@ -978,7 +978,7 @@ extern bool create_empty_file(fio_location from_location, const char *to_root, extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno); + int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); extern pid_t check_postmaster(const char *pgdata); extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, diff --git a/src/utils/file.c b/src/utils/file.c index 6f599f219..06e1a9889 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -76,7 +76,7 @@ typedef struct { BlockNumber n_blocks; BlockNumber segmentno; - XLogRecPtr horizonLsn; + XLogRecPtr shift_lsn; uint32 checksumVersion; } fio_lsn_map_request; @@ -2338,7 +2338,7 @@ static void fio_get_checksum_map_impl(int out, char *buf) datapagemap_t * fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, + int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno, fio_location location) { datapagemap_t* lsn_map = NULL; @@ -2351,7 +2351,7 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version, req_hdr.n_blocks = n_blocks; req_hdr.segmentno = segmentno; - req_hdr.horizonLsn = horizonLsn; + req_hdr.shift_lsn = shift_lsn; req_hdr.checksumVersion = checksum_version; hdr.cop = FIO_GET_LSN_MAP; @@ -2378,7 +2378,7 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version, else { lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, - horizonLsn, segmentno); + shift_lsn, segmentno); } return lsn_map; @@ -2392,7 +2392,7 @@ static void fio_get_lsn_map_impl(int out, char *buf) fio_lsn_map_request *req = (fio_lsn_map_request*) buf; lsn_map = get_lsn_map(fullpath, req->checksumVersion, req->n_blocks, - req->horizonLsn, req->segmentno); + req->shift_lsn, req->segmentno); if (lsn_map) hdr.size = lsn_map->bitmapsize; else From 8a65fc884a8a81abd4bcbaca08e92fc7312e9f00 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 14:35:17 +0300 Subject: [PATCH 1375/2107] [Issue #66] fix segfault when restoring full backup, rewrite help --- src/data.c | 2 +- src/help.c | 2 +- src/restore.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index c19d64897..2a3c9015e 100644 --- a/src/data.c +++ b/src/data.c @@ -1135,7 +1135,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers page_lsn = PageXLogRecPtrGet(phdr->pd_lsn); /* - * The heart of incremental restore + * The heart of incremental restore in 'checksum' mode * If page in backup has the same checksum and lsn as * page in backup, then page can be skipped. */ diff --git a/src/help.c b/src/help.c index 30604d193..8c8a606fe 100644 --- a/src/help.c +++ b/src/help.c @@ -433,7 +433,7 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); - printf(_(" reuse already existing unchanged valid pages in PGDATA\n")); + printf(_(" reuse valid pages available in PGDATA if they have not changed\n")); printf(_(" (default: none)\n")); printf(_("\n Partial restore options:\n")); diff --git a/src/restore.c b/src/restore.c index 95bce8348..6584a0628 100644 --- a/src/restore.c +++ b/src/restore.c @@ -712,7 +712,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, */ if (use_bitmap && parray_num(parent_chain) == 1) { - if (params->incremental_mode == INCR_NONE) + if (params->incremental_mode == INCR_LSN) use_bitmap = true; else use_bitmap = false; From 9bc310af8dbe49a227382c8e42d2ec16033a5b5b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 14:36:13 +0300 Subject: [PATCH 1376/2107] [Issue #66] new tests --- tests/incr_restore.py | 201 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 196 insertions(+), 5 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 9f169b7f0..095cd8ba3 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -529,13 +529,13 @@ def test_incr_lsn_restore(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_incr_shift_sanity(self): + def test_incr_lsn_sanity(self): """ /----A-----B F------*--------X X - is instance, we want to return it to state B. - fail is expected behaviour in case of shift restore. + fail is expected behaviour in case of lsn restore. """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -583,7 +583,7 @@ def test_incr_shift_sanity(self): # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because incremental restore in shift mode is impossible\n " + "Expecting Error because incremental restore in lsn mode is impossible\n " "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: @@ -716,7 +716,7 @@ def test_incr_checksum_corruption_detection(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_incr_shift_corruption_detection(self): + def test_incr_lsn_corruption_detection(self): """ check that corrupted page got detected and replaced """ @@ -846,7 +846,7 @@ def test_incr_restore_multiple_external(self): # @unittest.skip("skip") # @unittest.expectedFailure - def test_incr_shift_restore_multiple_external(self): + def test_incr_lsn_restore_multiple_external(self): """check that cmdline has priority over config""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -915,5 +915,196 @@ def test_incr_shift_restore_multiple_external(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_restore_backward(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'wal_log_hints': 'on', 'hot_standby': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + node.pgbench_init(scale=2) + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + full_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + page_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["-j", "4"]) + + delta_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + "-j", "4", '--incremental-mode=lsn', + '--recovery-target=immediate', '--recovery-target-action=pause'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(full_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", '--incremental-mode=lsn', + '--recovery-target=immediate', '--recovery-target-action=pause']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in lsn mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "Cannot perform incremental restore of backup chain", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", '--incremental-mode=checksum', + '--recovery-target=immediate', '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(page_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, + options=[ + "-j", "4", '--incremental-mode=lsn', + '--recovery-target=immediate', '--recovery-target-action=pause'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(delta_pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_checksum_restore_backward(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'hot_standby': 'on'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + node.pgbench_init(scale=20) + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + full_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + page_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["-j", "4"]) + + delta_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + "-j", "4", '--incremental-mode=checksum', + '--recovery-target=immediate', '--recovery-target-action=pause'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(full_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", '--incremental-mode=checksum', + '--recovery-target=immediate', '--recovery-target-action=pause'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(page_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, + options=[ + "-j", "4", '--incremental-mode=checksum', + '--recovery-target=immediate', '--recovery-target-action=pause'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(delta_pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From d9b7dee30f1bed00ca7a2c53b7de7c197003f65e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 18:16:55 +0300 Subject: [PATCH 1377/2107] [Issue #66] nitpicking --- src/restore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index 6584a0628..e71ecd52d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -832,7 +832,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(full_file_path, pgdata_path, file->rel_path); fio_pgFileDelete(file, full_file_path); - elog(WARNING, "Deleted file \"%s\"", full_file_path); + elog(VERBOSE, "Deleted file \"%s\"", full_file_path); /* shrink pgdata list */ parray_remove(pgdata_files, i); @@ -1461,6 +1461,7 @@ pg12_recovery_config(pgBackup *backup, bool add_include) elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, strerror(errno)); + // TODO: check if include 'probackup_recovery.conf' already exists fio_fprintf(fp, "\n# created by pg_probackup restore of backup %s at '%s'\n", base36enc(backup->start_time), current_time_str); fio_fprintf(fp, "include '%s'\n", "probackup_recovery.conf"); From 2c39666265c28e33aaa4cfa1558791cf89aa8511 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Jun 2020 19:38:19 +0300 Subject: [PATCH 1378/2107] [Issue #66] add switchover tests --- tests/incr_restore.py | 148 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 095cd8ba3..3ec2885e9 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -284,7 +284,7 @@ def test_incr_restore_with_tablespace_2(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_incr_restore_sanity(self): + def test_basic_incr_restore_sanity(self): """recovery to target timeline""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -1106,5 +1106,151 @@ def test_incr_checksum_restore_backward(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_make_replica_via_incr_checksum_restore(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + master.pgbench_init(scale=20) + + self.backup_node(backup_dir, 'node', master) + + self.restore_node( + backup_dir, 'node', replica, options=['-R']) + + # Settings for Replica + self.set_replica(master, replica, synchronous=False) + + replica.slow_start(replica=True) + + pgbench = master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PROMOTIONS + replica.promote() + new_master = replica + + # old master is going a bit further + old_master = master + pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + old_master.stop() + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # take backup from new master + self.backup_node( + backup_dir, 'node', new_master, + data_dir=new_master.data_dir, backup_type='page') + + # restore old master as replica + print(self.restore_node( + backup_dir, 'node', old_master, data_dir=old_master.data_dir, + options=['-R', '--incremental-mode=checksum'])) + + self.set_replica(new_master, old_master, synchronous=True) + + old_master.slow_start(replica=True) + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_make_replica_via_incr_lsn_restore(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + master.pgbench_init(scale=20) + + self.backup_node(backup_dir, 'node', master) + + self.restore_node( + backup_dir, 'node', replica, options=['-R']) + + # Settings for Replica + self.set_replica(master, replica, synchronous=False) + + replica.slow_start(replica=True) + + pgbench = master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PROMOTIONS + replica.promote() + new_master = replica + + # old master is going a bit further + old_master = master + pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + old_master.stop() + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # take backup from new master + self.backup_node( + backup_dir, 'node', new_master, + data_dir=new_master.data_dir, backup_type='page') + + # restore old master as replica + print(self.restore_node( + backup_dir, 'node', old_master, data_dir=old_master.data_dir, + options=['-R', '--incremental-mode=lsn'])) + + self.set_replica(new_master, old_master, synchronous=True) + + old_master.slow_start(replica=True) + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # Clean after yourself + self.del_test_dir(module_name, fname) + # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From 5d904b62e390bc63a3922b3d8bf9f17e72c7cae1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 02:06:37 +0300 Subject: [PATCH 1379/2107] [Issue #226] during backup obtain current tli after pg_start_backup execution, not before --- src/backup.c | 70 +++++++++++++++++++--------------------------------- 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/src/backup.c b/src/backup.c index cdb145133..00896637e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -83,7 +83,7 @@ static void *backup_files(void *arg); static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn); + PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static int checkpoint_timeout(PGconn *backup_conn); @@ -149,9 +149,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool parray *external_dirs = NULL; parray *database_map = NULL; - PGconn *master_conn = NULL; - PGconn *pg_startbackup_conn = NULL; - /* used for multitimeline incremental backup */ parray *tli_list = NULL; @@ -168,6 +165,18 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool check_external_for_tablespaces(external_dirs, backup_conn); } + /* Clear ptrack files for not PTRACK backups */ + if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable) + pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); + + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + + /* Call pg_start_backup function in PostgreSQL connect */ + pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); + /* Obtain current timeline */ #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); @@ -175,6 +184,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool current.tli = get_current_timeline_from_control(false); #endif + /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal) + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); + /* * In incremental backup mode ensure that already-validated * backup on current timeline exists and get its filelist. @@ -252,29 +270,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } } - /* Clear ptrack files for FULL and PAGE backup */ - if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable) - pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); - - /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time); - strncat(label, " with pg_probackup", lengthof(label) - - strlen(" with pg_probackup")); - - /* Create connection to master server needed to call pg_start_backup */ - if (current.from_replica && exclusive_backup) - { - master_conn = pgut_connect(instance_config.master_conn_opt.pghost, - instance_config.master_conn_opt.pgport, - instance_config.master_conn_opt.pgdatabase, - instance_config.master_conn_opt.pguser); - pg_startbackup_conn = master_conn; - } - else - pg_startbackup_conn = backup_conn; - - pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn, pg_startbackup_conn); - /* For incremental backup check that start_lsn is not from the past * Though it will not save us if PostgreSQL instance is actually * restored STREAM backup. @@ -342,7 +337,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool * Get database_map (name to oid) for use in partial restore feature. * It's possible that we fail and database_map will be NULL. */ - database_map = get_database_map(pg_startbackup_conn); + database_map = get_database_map(backup_conn); /* * Append to backup list all files and directories @@ -571,7 +566,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } /* Notify end of backup */ - pg_stop_backup(¤t, pg_startbackup_conn, nodeInfo); + pg_stop_backup(¤t, backup_conn, nodeInfo); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -1101,19 +1096,15 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) */ static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *pg_startbackup_conn) + PGNodeInfo *nodeInfo, PGconn *conn) { PGresult *res; const char *params[2]; uint32 lsn_hi; uint32 lsn_lo; - PGconn *conn; params[0] = label; - /* For 9.5 replica we call pg_start_backup() on master */ - conn = pg_startbackup_conn; - /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; if (!exclusive_backup) @@ -1132,7 +1123,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; - pgut_atexit_push(backup_stopbackup_callback, pg_startbackup_conn); + pgut_atexit_push(backup_stopbackup_callback, conn); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1152,15 +1143,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, * (because in 9.5 only superuser can switch WAL) */ pg_switch_wal(conn); - - /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal) - /* - * Do not wait start_lsn for stream backup. - * Because WAL streaming will start after pg_start_backup() in stream - * mode. - */ - wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false); } /* From 4fd6ba14d0c946cfce21e9736f9d10fbe1330d2a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 02:07:29 +0300 Subject: [PATCH 1380/2107] [Issue #226] fix tests --- tests/archive.py | 29 +++++++++-------------------- tests/backup.py | 30 +++++++++++++----------------- tests/ptrack.py | 6 +++--- tests/replica.py | 41 +++-------------------------------------- tests/restore.py | 23 +++++------------------ 5 files changed, 33 insertions(+), 96 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 5508f54b8..3aba71011 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -765,10 +765,6 @@ def test_replica_archive(self): before = master.safe_psql("postgres", "SELECT * FROM t_heap") - master.safe_psql( - "postgres", - "CHECKPOINT") - self.wait_until_replica_catch_with_master(master, replica) backup_id = self.backup_node( @@ -864,10 +860,6 @@ def test_master_and_replica_parallel_archiving(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0, 60000) i") - master.psql( - "postgres", - "CHECKPOINT") - backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -977,10 +969,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): replica.promote() - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - master.pgbench_init(scale=10) replica.pgbench_init(scale=10) @@ -1221,11 +1209,6 @@ def test_archive_catalog(self): # create timeline t2 replica.promote() - # do checkpoint to increment timeline ID in pg_control - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - # FULL backup replica A1 = self.backup_node( backup_dir, 'replica', replica) @@ -1959,7 +1942,8 @@ def test_archive_pg_receivexlog_partial_handling(self): replica.slow_start(replica=True) - node.safe_psql('postgres', 'CHECKPOINT') + # FULL + self.backup_node(backup_dir, 'replica', replica, options=['--stream']) if self.get_version(replica) < 100000: pg_receivexlog_path = self.get_bin_path('pg_receivexlog') @@ -1981,14 +1965,18 @@ def test_archive_pg_receivexlog_partial_handling(self): 'Failed to start pg_receivexlog: {0}'.format( pg_receivexlog.communicate()[1])) + replica.safe_psql( + 'postgres', + 'CHECKPOINT') + node.safe_psql( "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1000000) i") - # FULL - self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + # PAGE + self.backup_node(backup_dir, 'replica', replica, backup_type='page') node.safe_psql( "postgres", @@ -2027,6 +2015,7 @@ def test_archive_pg_receivexlog_partial_handling(self): pg_receivexlog.kill() self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): """""" fname = self.id().split('.')[3] diff --git a/tests/backup.py b/tests/backup.py index be51dfefc..8f97577b4 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -324,6 +324,7 @@ def test_page_detect_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -663,12 +664,10 @@ def test_backup_detect_invalid_block_header(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: page verification failed, ' - 'calculated checksum' in e.message and - 'ERROR: query failed: ERROR: ' - 'invalid page in block 1 of relation' in e.message and - 'ERROR: Data files transferring failed' in e.message, + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -806,12 +805,10 @@ def test_backup_detect_missing_permissions(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: page verification failed, ' - 'calculated checksum' in e.message and - 'ERROR: query failed: ERROR: ' - 'invalid page in block 1 of relation' in e.message and - 'ERROR: Data files transferring failed' in e.message, + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1480,7 +1477,7 @@ def test_drop_rel_during_backup_ptrack(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - ptrack_enable=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1927,7 +1924,7 @@ def test_basic_missing_file_permissions(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: Cannot open source file', + 'ERROR: Cannot open file', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -2131,9 +2128,8 @@ def test_backup_with_least_privileges_role(self): "TO backup".format(fname)) else: fnames = [ - 'pg_catalog.pg_ptrack_get_pagemapset(pg_lsn)', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint)' + 'pg_catalog.ptrack_get_pagemapset(pg_lsn)', + 'pg_catalog.ptrack_init_lsn()' ] for fname in fnames: diff --git a/tests/ptrack.py b/tests/ptrack.py index 8d878030e..a709afb74 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3946,9 +3946,9 @@ def test_ptrack_pg_resetxlog(self): repr(self.output), self.cmd) ) except ProbackupException as e: - self.assertIn( - 'ERROR: LSN from ptrack_control 0/0 differs from Start LSN of previous backup', - e.message, + self.assertTrue( + 'ERROR: LSN from ptrack_control ' in e.message and + 'differs from Start LSN of previous backup' in e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) diff --git a/tests/replica.py b/tests/replica.py index ab6eaf592..65f6df2c3 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -207,10 +207,6 @@ def test_replica_archive_page_backup(self): before = master.safe_psql("postgres", "SELECT * FROM t_heap") - master.psql( - "postgres", - "CHECKPOINT") - self.wait_until_replica_catch_with_master(master, replica) backup_id = self.backup_node( @@ -383,10 +379,6 @@ def test_take_backup_from_delayed_replica(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,165000) i") - master.psql( - "postgres", - "CHECKPOINT") - master.psql( "postgres", "create table t_heap_1 as select i as id, md5(i::text) as text, " @@ -726,9 +718,11 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content) self.assertIn( - 'LOG: stop_lsn: 0/4000028', + 'LOG: stop_lsn: 0/4000000', log_content) + self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') + # Clean after yourself self.del_test_dir(module_name, fname) @@ -1118,18 +1112,6 @@ def test_replica_promote_2(self): replica.promote() - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - -# replica.safe_psql( -# 'postgres', -# 'create table t2()') -# -# replica.safe_psql( -# 'postgres', -# 'CHECKPOINT') - self.backup_node( backup_dir, 'master', replica, data_dir=replica.data_dir, backup_type='page') @@ -1176,10 +1158,6 @@ def test_replica_promote_3(self): self.add_instance(backup_dir, 'replica', replica) - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - full_id = self.backup_node( backup_dir, 'replica', replica, options=['--stream']) @@ -1191,20 +1169,12 @@ def test_replica_promote_3(self): 'FROM generate_series(0,20) i') self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - self.backup_node( backup_dir, 'replica', replica, backup_type='delta', options=['--stream']) replica.promote() - replica.safe_psql( - 'postgres', - 'CHECKPOINT') - # failing, because without archving, it is impossible to # take multi-timeline backup. try: @@ -1297,7 +1267,6 @@ def test_replica_promote_archive_delta(self): # node2 is now master node2.promote() - node2.safe_psql('postgres', 'CHECKPOINT') node2.safe_psql( 'postgres', @@ -1331,7 +1300,6 @@ def test_replica_promote_archive_delta(self): # node1 is back to be a master node1.promote() - node1.safe_psql('postgres', 'CHECKPOINT') sleep(5) @@ -1420,7 +1388,6 @@ def test_replica_promote_archive_page(self): # node2 is now master node2.promote() - node2.safe_psql('postgres', 'CHECKPOINT') node2.safe_psql( 'postgres', @@ -1454,7 +1421,6 @@ def test_replica_promote_archive_page(self): # node1 is back to be a master node1.promote() - node1.safe_psql('postgres', 'CHECKPOINT') self.switch_wal_segment(node1) sleep(5) @@ -1532,7 +1498,6 @@ def test_parent_choosing(self): backup_type='delta', options=['--stream']) replica.promote() - replica.safe_psql('postgres', 'CHECKPOINT') # failing, because without archving, it is impossible to # take multi-timeline backup. diff --git a/tests/restore.py b/tests/restore.py index 6ab3da335..889a3b190 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -980,9 +980,9 @@ def test_restore_with_tablespace_mapping_2(self): # Create tablespace table with node.connect("postgres") as con: - con.connection.autocommit = True - con.execute("CHECKPOINT") - con.connection.autocommit = False +# con.connection.autocommit = True +# con.execute("CHECKPOINT") +# con.connection.autocommit = False con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") con.execute( "INSERT INTO tbl1 SELECT * " @@ -1389,10 +1389,6 @@ def test_zags_block_corrupt_1(self): 'postgres', 'create extension pageinspect') - node.safe_psql( - 'postgres', - 'checkpoint') - node.safe_psql( 'postgres', 'insert into tbl select i from generate_series(0,100) as i') @@ -3025,6 +3021,7 @@ def test_missing_database_map(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], pg_options={'autovacuum': 'off'}) @@ -3276,17 +3273,7 @@ def test_stream_restore_command_option(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) - node.pgbench_init(scale=1) - - node.safe_psql( - 'postgres', - 'CHECKPOINT') - - node.pgbench_init(scale=1) - - node.safe_psql( - 'postgres', - 'CHECKPOINT') + node.pgbench_init(scale=5) node.safe_psql( 'postgres', From da082fb29b4dc208c1532733e4b4c32a9fec82a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 18:56:11 +0300 Subject: [PATCH 1381/2107] [Issue #66] update help --- src/help.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/help.c b/src/help.c index 8c8a606fe..b2c63135b 100644 --- a/src/help.c +++ b/src/help.c @@ -432,6 +432,8 @@ help_restore(void) printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + + printf(_("\n Incremental restore options:\n")); printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); printf(_(" reuse valid pages available in PGDATA if they have not changed\n")); printf(_(" (default: none)\n")); From 41d4013a81c616c8fd19a4aaa98cdc0224897876 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 18:57:47 +0300 Subject: [PATCH 1382/2107] [Issue #66] fix help --- src/help.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/help.c b/src/help.c index b2c63135b..2bcd6dbc5 100644 --- a/src/help.c +++ b/src/help.c @@ -160,7 +160,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); - printf(_(" [--incremental-mode=none|checksum|lsn\n")); + printf(_(" [--incremental-mode=none|checksum|lsn]\n")); printf(_(" [--no-sync]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -398,7 +398,7 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); - printf(_(" [--incremental-mode=none|checksum|lsn\n")); + printf(_(" [--incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); From 040b5a61f6dc5b26421555fbebed12a74291f61f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 19:04:30 +0300 Subject: [PATCH 1383/2107] minor fixes for restore help --- src/help.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/help.c b/src/help.c index 2bcd6dbc5..2b5bcd06e 100644 --- a/src/help.c +++ b/src/help.c @@ -153,15 +153,15 @@ help_pg_probackup(void) printf(_(" [--recovery-target=immediate|latest]\n")); printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-as-replica] [--force]\n")); + printf(_(" [--restore-command=cmdline]\n")); + printf(_(" [-R | --restore-as-replica] [--force]\n")); printf(_(" [--primary-conninfo=primary_conninfo]\n")); printf(_(" [-S | --primary-slot-name=slotname]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--skip-external-dirs] [--restore-command=cmdline]\n")); - printf(_(" [--incremental-mode=none|checksum|lsn]\n")); - printf(_(" [--no-sync]\n")); + printf(_(" [--skip-external-dirs] [--no-sync]\n")); + printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); @@ -398,7 +398,7 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); - printf(_(" [--incremental-mode=none|checksum|lsn]\n")); + printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -408,6 +408,8 @@ help_restore(void) printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); printf(_(" [--restore-command=cmdline]\n")); printf(_(" [-R | --restore-as-replica]\n")); + printf(_(" [--primary-conninfo=primary_conninfo]\n")); + printf(_(" [-S | --primary-slot-name=slotname]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); From 496605bdb50012788fbab74a0f1625df1a26a789 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Jun 2020 23:32:19 +0300 Subject: [PATCH 1384/2107] [Issue #66] Documentation --- doc/pgprobackup.xml | 139 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 73b2bf1b7..9bf740a05 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -196,6 +196,12 @@ doc/src/sgml/pgprobackup.sgml backups than by replaying WAL files. + + + Incremental restore: speed up restore from backup by reusing + valid unchanged pages available in PGDATA. + + Validation: automatic data consistency checks and on-demand @@ -1698,6 +1704,96 @@ pg_probackup restore -B backup_dir --instance + + Incremental Restore + + The speed of restore from backup can be significantly improved + by replacing only invalid and changed pages in already + existing PostgreSQL data directory using + incremental + restore options with the + command. + + + To restore the database cluster from a backup in incremental mode, + run the command with the following options: + + +pg_probackup restore -B backup_dir --instance instance_name -D data_dir -I incremental_mode + + + Where incremental_mode can take one of the + following values: + + + + + CHECKSUM — read all data files in the data directory, validate + header and checksum in every page and replace only invalid + pages and those with checksum and LSN not matching with + corresponding page in backup. This is the simplest, + the most fool-proof incremental mode. + + + + + LSN — read the pg_control in the + data directory to obtain redo LSN and redo TLI, which allows + to determine a point in history(shiftpoint), where data directory + state shifted from backup chain history. If shiftpoint is not within + reach of backup chain history, then restore is aborted. + If shiftpoint is within reach of backup chain history, then read + all data files in the data directory, validate header and checksum in + every page and replace only invalid pages and those with LSN greater + than shiftpoint. + This mode offer a greatest speed up, but rely on pg_control + been synched with state of data directory, + so it is not recommended to use in any situation, where this condition + is not met, for example, after pg_resetxlog execution, + after restore from backup without recovery been run, etc. + + + + + NONE — regular restore without any incremental optimizations. + + + + + + Suppose you want to return an old master as replica after switchover + using incremental restore in LSN mode: + + +============================================================================================================================================= + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +============================================================================================================================================= + node 12 QBRNBP 2020-06-11 17:40:58+03 DELTA ARCHIVE 16/15 40s 194MB 16MB 8.26 15/2C000028 15/2D000128 OK + node 12 QBRIDX 2020-06-11 15:51:42+03 PAGE ARCHIVE 15/15 11s 18MB 16MB 5.10 14/DC000028 14/DD0000B8 OK + node 12 QBRIAJ 2020-06-11 15:51:08+03 PAGE ARCHIVE 15/15 20s 141MB 96MB 6.22 14/D4BABFE0 14/DA9871D0 OK + node 12 QBRHT8 2020-06-11 15:45:56+03 FULL ARCHIVE 15/0 2m:11s 1371MB 416MB 10.93 14/9D000028 14/B782E9A0 OK + +pg_probackup restore -B /backup --instance node -R -I lsn +INFO: Running incremental restore into nonempty directory: "/var/lib/pgsql/12/data" +INFO: Destination directory redo point 15/2E000028 on tli 16 is within reach of backup QBRIDX with Stop LSN 14/DD0000B8 on tli 15, incremental restore in 'lsn' mode is possible +INFO: shift LSN: 14/DD0000B8 +INFO: Restoring the database from backup at 2020-06-11 15:39:08+03 +INFO: Extracting the content of destination directory for incremental restore +INFO: Destination directory content extracted, time elapsed: 1s +INFO: Removing redundant files in destination directory +INFO: Redundant files are removed, time elapsed: 1s +INFO: Start restoring backup files. PGDATA size: 15GB +INFO: Backup files are restored. Transfered bytes: 1693MB, time elapsed: 43s +INFO: Restore incremental ratio (less is better): 11% (1693MB/15GB) +INFO: Restore of backup QBRHT8 completed. + + + + Incremental restore is possible only for backups with + program_version equal or greater than 2.3.0. + + + Partial Restore @@ -1715,7 +1811,7 @@ pg_probackup restore -B backup_dir --instance pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name - + The option can be specified multiple times. For example, to restore only databases @@ -3863,7 +3959,7 @@ pg_probackup restore -B backup_dir --instance , remote WAL archive options, logging - options, partial + options, partial restore options, and common options can be used. @@ -5075,6 +5171,45 @@ pg_probackup archive-get -B backup_dir --instance + + Incremental Restore Options + + This section describes the options for incremental cluster restore. + These options can be used with the + command. + + + + + + + + + Specifies the incremental mode to be used. Possible values are: + + + + + CHECKSUM — replace only pages with mismatched checksum and LSN. + + + + + LSN — replace only pages with LSN greater than point of divergence. + + + + + NONE — regular restore. + + + + + + + + + Partial Restore Options From 75068254680d7866792fd6465668764450e1d708 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Jun 2020 16:30:31 +0300 Subject: [PATCH 1385/2107] [Issue #66] for LSN mode data_checksums must be enabled --- src/dir.c | 5 +++-- src/pg_probackup.h | 1 + src/restore.c | 4 ++++ src/util.c | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dir.c b/src/dir.c index 0cddba142..a24bda63d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1208,8 +1208,9 @@ read_tablespace_map(parray *files, const char *backup_dir) * If tablespace-mapping option is supplied, all OLDDIR entries must have * entries in tablespace_map file. * - * When running incremental restore with tablespace remapping, then - * new tablespace directory MUST be empty, because there is no + * + * TODO: maybe when running incremental restore with tablespace remapping, then + * new tablespace directory MUST be empty? because there is no * we can be sure, that files laying there belong to our instance. */ void diff --git a/src/pg_probackup.h b/src/pg_probackup.h index db38d4e14..a537a388f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -105,6 +105,7 @@ typedef struct RedoParams { TimeLineID tli; XLogRecPtr lsn; + uint32 checksum_version; } RedoParams; typedef struct PageState diff --git a/src/restore.c b/src/restore.c index e71ecd52d..d4907d34d 100644 --- a/src/restore.c +++ b/src/restore.c @@ -411,6 +411,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray *timelines = NULL; get_redo(instance_config.pgdata, &redo); + if (redo.checksum_version == 0) + elog(ERROR, "Incremental restore in 'lsn' mode require " + "data_checksums to be enabled in destination data directory"); + timelines = read_timeline_history(arclog_path, redo.tli, false); if (!timelines) diff --git a/src/util.c b/src/util.c index a6cf2e393..5ad751df2 100644 --- a/src/util.c +++ b/src/util.c @@ -380,6 +380,8 @@ get_redo(const char *pgdata_path, RedoParams *redo) redo->lsn = ControlFile.backupStartPoint; redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; } + + redo->checksum_version = ControlFile.data_checksum_version; } /* From 2cb432655d5291c060749d9759c74942d0e2bd9d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Jun 2020 16:31:04 +0300 Subject: [PATCH 1386/2107] [Issue #66] tests for hint bits --- tests/incr_restore.py | 242 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 3ec2885e9..859885b1a 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1252,5 +1252,247 @@ def test_make_replica_via_incr_lsn_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_checksum_long_xact(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, +# initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + + con.commit() + + node.safe_psql( + 'postgres', + 'select * from t1') + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=checksum'])) + + node.slow_start() + + self.assertEqual( + node.safe_psql( + 'postgres', + 'select count(*) from t1').rstrip(), + '1') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_long_xact_1(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, +# initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + + con.commit() + + # when does LSN gets stamped when checksum gets updated ? + node.safe_psql( + 'postgres', + 'select * from t1') + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + + node.stop() + + try: + print(self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=lsn'])) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in lsn mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Incremental restore in 'lsn' mode require data_checksums to be " + "enabled in destination data directory", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_long_xact_2(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'full_page_writes': 'off', + 'wal_log_hints': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + + print(node.safe_psql( + 'postgres', + "select * from page_header(get_raw_page('t1', 0))")) + + con.commit() + + # when does LSN gets stamped when checksum gets updated ? + node.safe_psql( + 'postgres', + 'select * from t1') + + print(node.safe_psql( + 'postgres', + "select * from page_header(get_raw_page('t1', 0))")) + + print("HELLO") + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + + print(node.safe_psql( + 'postgres', + "select * from page_header(get_raw_page('t1', 0))")) + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=lsn'])) + + node.slow_start() + + self.assertEqual( + node.safe_psql( + 'postgres', + 'select count(*) from t1').rstrip(), + '1') + + # Clean after yourself + self.del_test_dir(module_name, fname) # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From bfdabf375bad52868bca65a872599c9a85203e06 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Jun 2020 16:31:36 +0300 Subject: [PATCH 1387/2107] [Issue #66] documentation update --- doc/pgprobackup.xml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 9bf740a05..efbbf980e 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1746,10 +1746,19 @@ pg_probackup restore -B backup_dir --instance pg_control - been synched with state of data directory, - so it is not recommended to use in any situation, where this condition - is not met, for example, after pg_resetxlog execution, + This mode offer a far greater speed up compared to CHECKSUM, but rely + on two conditions to be met. First, + + data checksums parameter must be enabled in data directory (to avoid corruption + due to hint bits). This condition will be checked at the start of + incremental restore and the operation will be aborted if checksums are disabled. + Second, the pg_control file must be + synched with state of data directory. This condition cannot checked + at the start of restore, so it is a user responsibility to ensure + that pg_control contain valid information. Because of that is not + recommended to use LSN mode in any situation, where pg_control has + been tampered with: + after pg_resetxlog execution, after restore from backup without recovery been run, etc. From 483f9b64be74f16de439e7621f8715b01c91bd69 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Jun 2020 16:58:52 +0300 Subject: [PATCH 1388/2107] Change wording in elog message --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index d4907d34d..7c863c6c3 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1126,7 +1126,7 @@ restore_files(void *arg) strerror(errno)); if (!dest_file->is_datafile || dest_file->is_cfs) - elog(VERBOSE, "Restoring non-data file: \"%s\"", to_fullpath); + elog(VERBOSE, "Restoring nonedata file: \"%s\"", to_fullpath); else elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); From f9d309959809bf498cef6d882e406e2264e52096 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 16 Jun 2020 00:12:30 +0300 Subject: [PATCH 1389/2107] [Issue #228] WIP --- src/catalog.c | 3 + src/data.c | 604 +++++++++++++++++++++++++++++++++------------ src/dir.c | 7 + src/pg_probackup.h | 23 +- src/restore.c | 10 +- src/utils/file.c | 82 ++++-- src/utils/file.h | 1 + src/validate.c | 2 +- 8 files changed, 548 insertions(+), 184 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index a8a7dec4a..686b21e70 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1905,6 +1905,9 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (file->n_blocks > 0) len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); + if (file->n_headers > 0) + len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); + sprintf(line+len, "}\n"); if (sync) diff --git a/src/data.c b/src/data.c index 2a3c9015e..3615aed64 100644 --- a/src/data.c +++ b/src/data.c @@ -31,6 +31,10 @@ typedef union DataPage char data[BLCKSZ]; } DataPage; +static BackupPageHeader2* get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version); +static bool get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, + BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c); + #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ static int32 @@ -281,9 +285,9 @@ prepare_page(ConnectionArgs *conn_arg, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, - const char *from_fullpath) + const char *from_fullpath, + PageState *page_st) { - XLogRecPtr page_lsn = 0; int try_again = PAGE_READ_ATTEMPTS; bool page_is_valid = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; @@ -304,7 +308,6 @@ prepare_page(ConnectionArgs *conn_arg, { /* read the block */ int read_len = fio_pread(in, page, blknum * BLCKSZ); - page_lsn = 0; /* The block could have been truncated. It is fine. */ if (read_len == 0) @@ -324,8 +327,8 @@ prepare_page(ConnectionArgs *conn_arg, { /* We have BLCKSZ of raw data, validate it */ rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, &page_lsn, - checksum_version); + InvalidXLogRecPtr, page_st, + checksum_version); switch (rc) { case PAGE_IS_ZEROED: @@ -429,8 +432,8 @@ prepare_page(ConnectionArgs *conn_arg, * acceptable. */ rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, &page_lsn, - checksum_version); + InvalidXLogRecPtr, page_st, + checksum_version); /* It is ok to get zeroed page */ if (rc == PAGE_IS_ZEROED) @@ -450,7 +453,7 @@ prepare_page(ConnectionArgs *conn_arg, * in the block recieved from shared buffers. */ if (checksum_version) - ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + page_st->checksum = ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); } /* @@ -459,8 +462,8 @@ prepare_page(ConnectionArgs *conn_arg, */ if (backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev && - page_lsn && - page_lsn < prev_backup_start_lsn) + page_st->lsn > 0 && + page_st->lsn < prev_backup_start_lsn) { elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, from_fullpath); return SkipCurrentPage; @@ -469,47 +472,47 @@ prepare_page(ConnectionArgs *conn_arg, return PageIsOk; } -static void +static int compress_and_backup_page(pgFile *file, BlockNumber blknum, FILE *in, FILE *out, pg_crc32 *crc, int page_state, Page page, CompressAlg calg, int clevel, const char *from_fullpath, const char *to_fullpath) { - BackupPageHeader header; - size_t write_buffer_size = sizeof(header); - char write_buffer[BLCKSZ+sizeof(header)]; +// BackupPageHeader header; + int compressed_size; + size_t write_buffer_size = 0; + char write_buffer[BLCKSZ]; char compressed_page[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ const char *errormsg = NULL; - header.block = blknum; +// header.block = blknum; /* Compress the page */ - header.compressed_size = do_compress(compressed_page, sizeof(compressed_page), - page, BLCKSZ, calg, clevel, - &errormsg); + compressed_size = do_compress(compressed_page, sizeof(compressed_page), + page, BLCKSZ, calg, clevel, + &errormsg); /* Something went wrong and errormsg was assigned, throw a warning */ - if (header.compressed_size < 0 && errormsg != NULL) + if (compressed_size < 0 && errormsg != NULL) elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", blknum, from_fullpath, errormsg); file->compress_alg = calg; /* TODO: wtf? why here? */ /* The page was successfully compressed. */ - if (header.compressed_size > 0 && header.compressed_size < BLCKSZ) + if (compressed_size > 0 && compressed_size < BLCKSZ) { - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), - compressed_page, header.compressed_size); - write_buffer_size += MAXALIGN(header.compressed_size); +// memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer, compressed_page, compressed_size); + write_buffer_size = MAXALIGN(compressed_size); } /* Non-positive value means that compression failed. Write it as is. */ else { - header.compressed_size = BLCKSZ; - memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer + sizeof(header), page, BLCKSZ); - write_buffer_size += header.compressed_size; + compressed_size = BLCKSZ; +// memcpy(write_buffer, &header, sizeof(header)); + memcpy(write_buffer, page, BLCKSZ); + write_buffer_size = compressed_size; } /* Update CRC */ @@ -522,6 +525,8 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->write_size += write_buffer_size; file->uncompressed_size += BLCKSZ; + + return compressed_size; } /* @@ -544,11 +549,15 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, BlockNumber blknum = 0; BlockNumber nblocks = 0; /* number of blocks in source file */ BlockNumber n_blocks_skipped = 0; - int page_state; + int rc; char curr_page[BLCKSZ]; bool use_pagemap; datapagemap_iterator_t *iter = NULL; + /* headers */ + int hdr_num = -1; + BackupPageHeader2 *headers = NULL; + /* stdio buffers */ char *in_buf = NULL; char *out_buf = NULL; @@ -637,7 +646,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* send pagemap if any */ use_pagemap ? &file->pagemap : NULL, /* variables for error reporting */ - &err_blknum, &errmsg); + &err_blknum, &errmsg, &headers); /* check for errors */ if (rc == FILE_MISSING) @@ -728,23 +737,43 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, while (blknum < nblocks) { - page_state = prepare_page(conn_arg, file, prev_backup_start_lsn, + PageState page_st; + rc = prepare_page(conn_arg, file, prev_backup_start_lsn, blknum, in, backup_mode, curr_page, true, checksum_version, ptrack_version_num, ptrack_schema, - from_fullpath); + from_fullpath, &page_st); - if (page_state == PageIsTruncated) + if (rc == PageIsTruncated) break; /* TODO: remove */ - else if (page_state == SkipCurrentPage) + else if (rc == SkipCurrentPage) n_blocks_skipped++; - else if (page_state == PageIsOk) - compress_and_backup_page(file, blknum, in, out, &(file->crc), - page_state, curr_page, calg, clevel, - from_fullpath, to_fullpath); + else if (rc == PageIsOk) + { + hdr_num++; + + if (!headers) + headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); + else + headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); + + headers[hdr_num].block = blknum; + headers[hdr_num].lsn = page_st.lsn; + headers[hdr_num].checksum = page_st.checksum; + headers[hdr_num].pos = ftell(out); /* optimize */ + +// elog(INFO, "CRC: %u", headers[hdr_num].checksum); +// elog(INFO, "POS: %u", headers[hdr_num].pos); + + headers[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), + rc, curr_page, calg, clevel, + from_fullpath, to_fullpath); + + file->n_headers = hdr_num +1; + } /* TODO: handle PageIsCorrupted, currently it is done in prepare_page */ else Assert(false); @@ -797,6 +826,46 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, pg_free(in_buf); pg_free(out_buf); + + /* handle hdr */ + /* TODO: move in separate function */ + if (headers) + { + size_t hdr_size; + char to_fullpath_hdr[MAXPGPATH]; + + snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); + + out = fopen(to_fullpath_hdr, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open header file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + /* TODO: calculate checksums */ + /* TODO: save file to backup_content metainformation */ + hdr_size = file->n_headers * sizeof(BackupPageHeader2); + +// elog(INFO, "Size: %lu, aligh: %lu", hdr_size, MAXALIGN(hdr_size)); +// elog(INFO, "checksum: %u, lsn: %lu", headers[file->n_headers-1].checksum, headers[file->n_headers-1].lsn); +// elog(INFO, "POS: %u", headers[file->n_headers-1].pos); +// elog(INFO, "blknum: %u", headers[file->n_headers-1].block); +// elog(INFO, "size: %u", headers[file->n_headers-1].compressed_size); + + if (fwrite(headers, 1, hdr_size, out) != hdr_size) + elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + + if (fclose(out)) + elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + +// elog(INFO, "n_headers: %u", file->n_headers); + + pg_free(headers); + } } /* @@ -878,6 +947,9 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, pgFile **res_file = NULL; pgFile *tmp_file = NULL; + /* page headers */ + BackupPageHeader2 *headers = NULL; + pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); if (use_bitmap) @@ -922,6 +994,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* set stdio buffering for input data file */ setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + headers = get_data_file_headers(from_fullpath, tmp_file, parse_program_version(backup->program_version)); + /* * Restore the file. * Datafiles are backed up block by block and every block @@ -934,12 +1008,15 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, use_bitmap ? &(dest_file)->pagemap : NULL, checksum_map, backup->checksum_version, /* shiftmap can be used only if backup state precedes the shift */ - backup->stop_lsn <= shift_lsn ? lsn_map : NULL); + backup->stop_lsn <= shift_lsn ? lsn_map : NULL, + headers); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, strerror(errno)); + pg_free(headers); + // datapagemap_print_debug(&(dest_file)->pagemap); } pg_free(in_buf); @@ -956,12 +1033,13 @@ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map) + datapagemap_t *lsn_map, BackupPageHeader2 *headers) { - BackupPageHeader header; BlockNumber blknum = 0; + int n_hdr = -1; size_t write_len = 0; - off_t cur_pos = 0; + off_t cur_pos_out = 0; + off_t cur_pos_in = 0; /* * We rely on stdio buffering of input and output. @@ -973,7 +1051,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * a lot when blocks from incremental backup are restored, * but should never happen in case of blocks from FULL backup. */ - if (fio_fseek(out, cur_pos) < 0) + if (fio_fseek(out, cur_pos_out) < 0) elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); @@ -985,41 +1063,37 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers int32 compressed_size = 0; bool is_compressed = false; + /* incremental restore vars */ + uint16 page_crc = 0; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + /* check for interrupt */ if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file restore"); - /* read BackupPageHeader */ - read_len = fread(&header, 1, sizeof(header), in); - - if (ferror(in)) - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - - if (read_len != sizeof(header)) + /* newer backups have headers in separate storage */ + if (headers) { - if (read_len == 0 && feof(in)) - break; /* EOF found */ + n_hdr++; + if (n_hdr >= file->n_headers) + break; - if (read_len != 0 && feof(in)) - elog(ERROR, "Odd size page found at block %u of \"%s\"", - blknum, from_fullpath); + blknum = headers[n_hdr].block; + page_lsn = headers[n_hdr].lsn; + page_crc = headers[n_hdr].checksum; + compressed_size = headers[n_hdr].compressed_size; } - - /* Consider empty blockm. wtf empty block ? */ - if (header.block == 0 && header.compressed_size == 0) + else { - elog(WARNING, "Skip empty block of \"%s\"", from_fullpath); - continue; + if (get_compressed_page_meta(in, from_fullpath, &compressed_size, + &blknum, NULL, false)) + { + cur_pos_in += sizeof(BackupPageHeader); + } + else + break; } - /* sanity? */ -// if (header.block < blknum) -// elog(ERROR, "Backup is broken at block %u of \"%s\"", -// blknum, from_fullpath); - - blknum = header.block; - /* * Backupward compatibility kludge: in the good old days * n_blocks attribute was available only in DELTA backups. @@ -1032,8 +1106,6 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * is not happening in the first place. * TODO: remove in 3.0.0 */ - compressed_size = header.compressed_size; - if (compressed_size == PageIsTruncated) { /* @@ -1041,7 +1113,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * We need to truncate file to this length. */ - elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, header.block); + elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, blknum); /* To correctly truncate file, we must first flush STDIO buffers */ if (fio_fflush(out) != 0) @@ -1051,7 +1123,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (fio_fseek(out, 0) < 0) elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); - if (fio_ftruncate(out, header.block * BLCKSZ) != 0) + if (fio_ftruncate(out, blknum * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); break; @@ -1064,92 +1136,73 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); - if (lsn_map && datapagemap_is_set(lsn_map, blknum)) + /* incremental restore in LSN mode */ + if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) datapagemap_add(map, blknum); + if (map && checksum_map && checksum_map[blknum].checksum != 0) + { +// elog(INFO, "HDR CRC: %u, MAP CRC: %u", page_crc, checksum_map[blknum].checksum); + /* + * The heart of incremental restore in CHECKSUM mode + * If page in backup has the same checksum and lsn as + * page in backup, then page can be skipped. + */ + if (page_crc == checksum_map[blknum].checksum && + page_lsn == checksum_map[blknum].lsn) + { + datapagemap_add(map, blknum); + } + } + /* if this page is marked as already restored, then skip it */ if (map && datapagemap_is_set(map, blknum)) { - /* skip to the next page */ - if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) +// elog(INFO, "HAHAHA"); + /* skip to the next page for backup withot header file */ + if (!headers && fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) elog(ERROR, "Cannot seek block %u of '%s': %s", blknum, from_fullpath, strerror(errno)); continue; } + if (headers && + cur_pos_in != headers[n_hdr].pos) + { + elog(INFO, "Seek to %u", headers[n_hdr].pos); + if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) + elog(ERROR, "Cannot seek to offset %u of '%s': %s", + headers[n_hdr].pos, from_fullpath, strerror(errno)); + + cur_pos_in = headers[n_hdr].pos; + } + +// elog(INFO, "Cur_pos: %u", ftell(in)); + /* read a page from file */ read_len = fread(page.data, 1, MAXALIGN(compressed_size), in); +// elog(INFO, "Blocknum %u, zread: %u ", blknum, compressed_size); + if (read_len != MAXALIGN(compressed_size)) elog(ERROR, "Cannot read block %u of \"%s\", read %zu of %d", blknum, from_fullpath, read_len, compressed_size); + cur_pos_in += read_len; + /* * if page size is smaller than BLCKSZ, decompress the page. * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. * we have to check, whether it is compressed or not using * page_may_be_compressed() function. */ - if (header.compressed_size != BLCKSZ + if (compressed_size != BLCKSZ || page_may_be_compressed(page.data, file->compress_alg, backup_version)) { is_compressed = true; } - /* Incremental restore - * TODO: move to separate function, - * TODO: move BackupPageHeader headers to some separate storage, - * so they can be accessed without reading through whole file. - */ - if (checksum_map && checksum_map[blknum].checksum != 0) - { - uint16 page_crc = 0; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - PageHeader phdr; - - if (is_compressed) - { - char uncompressed_buf[BLCKSZ]; - fio_decompress(uncompressed_buf, page.data, compressed_size, file->compress_alg); - - /* If checksums are enabled, then we can trust checksum in header */ - if (checksum_version) - page_crc = ((PageHeader) uncompressed_buf)->pd_checksum; - else - page_crc = pg_checksum_page(uncompressed_buf, file->segno * RELSEG_SIZE + blknum); - - phdr = (PageHeader) uncompressed_buf; - } - else - { - /* if checksumms are enabled, then we can trust checksumm in header */ - if (checksum_version) - page_crc = ((PageHeader) page.data)->pd_checksum; - else - page_crc = pg_checksum_page(page.data, file->segno + blknum); - - phdr = (PageHeader) page.data; - } - - page_lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - /* - * The heart of incremental restore in 'checksum' mode - * If page in backup has the same checksum and lsn as - * page in backup, then page can be skipped. - */ - if (page_crc == checksum_map[blknum].checksum && - page_lsn == checksum_map[blknum].lsn) - { - if (map) - datapagemap_add(map, blknum); - continue; - } - -// elog(INFO, "Replace blknum %u in file %s", blknum, to_fullpath); - } - /* * Seek and write the restored page. * When restoring file from FULL backup, pages are written sequentially, @@ -1157,11 +1210,13 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers */ write_pos = blknum * BLCKSZ; - if (cur_pos != write_pos) + if (cur_pos_out != write_pos) { - if (fio_fseek(out, blknum * BLCKSZ) < 0) + if (fio_fseek(out, write_pos) < 0) elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); + + cur_pos_out = write_pos; } /* If page is compressed and restore is in remote mode, send compressed @@ -1184,7 +1239,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } write_len += BLCKSZ; - cur_pos = write_pos + BLCKSZ; /* update current write position */ + cur_pos_out += BLCKSZ; /* update current write position */ if (map) datapagemap_add(map, blknum); @@ -1532,15 +1587,18 @@ create_empty_file(fio_location from_location, const char *to_root, */ int validate_one_page(Page page, BlockNumber absolute_blkno, - XLogRecPtr stop_lsn, XLogRecPtr *page_lsn, + XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version) { + page_st->lsn = InvalidXLogRecPtr; + page_st->checksum = 0; + /* new level of paranoia */ if (page == NULL) return PAGE_IS_NOT_FOUND; /* check that page header is ok */ - if (!parse_page(page, page_lsn)) + if (!parse_page(page, &(page_st)->lsn)) { int i; /* Check if the page is zeroed. */ @@ -1555,10 +1613,12 @@ validate_one_page(Page page, BlockNumber absolute_blkno, } /* Verify checksum */ + page_st->checksum = pg_checksum_page(page, absolute_blkno); + if (checksum_version) { /* Checksums are enabled, so check them. */ - if (pg_checksum_page(page, absolute_blkno) != ((PageHeader) page)->pd_checksum) + if (page_st->checksum != ((PageHeader) page)->pd_checksum) return PAGE_CHECKSUM_MISMATCH; } @@ -1569,7 +1629,7 @@ validate_one_page(Page page, BlockNumber absolute_blkno, if (stop_lsn > 0) { /* Get lsn from page header. Ensure that page is from our time. */ - if (*page_lsn > stop_lsn) + if (page_st->lsn > stop_lsn) return PAGE_LSN_FROM_FUTURE; } @@ -1623,11 +1683,11 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, for (blknum = 0; blknum < nblocks; blknum++) { - + PageState page_st; page_state = prepare_page(NULL, file, InvalidXLogRecPtr, blknum, in, BACKUP_MODE_FULL, curr_page, false, checksum_version, - 0, NULL, from_fullpath); + 0, NULL, from_fullpath, &page_st); if (page_state == PageIsTruncated) break; @@ -1646,6 +1706,168 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, return is_valid; } +/* Valiate pages of datafile in backup one by one */ +bool +check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version) +{ + size_t read_len = 0; + bool is_valid = true; + FILE *in; + pg_crc32 crc; + bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; + BackupPageHeader2 *headers = NULL; + int n_hdr = -1; + off_t cur_pos = 0; + + elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); + + in = fopen(fullpath, PG_BINARY_R); + if (in == NULL) + { + if (errno == ENOENT) + { + elog(WARNING, "File \"%s\" is not found", fullpath); + return false; + } + + elog(ERROR, "Cannot open file \"%s\": %s", + fullpath, strerror(errno)); + } + + headers = get_data_file_headers(fullpath, file, backup_version); + + /* calc CRC of backup file */ + INIT_FILE_CRC32(use_crc32c, crc); + + /* read and validate pages one by one */ + while (true) + { + int rc = 0; + DataPage compressed_page; /* used as read buffer */ + int compressed_size = 0; + DataPage page; + BlockNumber blknum = 0; + PageState page_st; + + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during data file validation"); + + /* newer backups have headers in separate storage */ + if (headers) + { + n_hdr++; + if (n_hdr >= file->n_headers) + break; + + blknum = headers[n_hdr].block; + compressed_size = headers[n_hdr].compressed_size; + + if (cur_pos != headers[n_hdr].pos && + fio_fseek(in, headers[n_hdr].pos) < 0) + { + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, fullpath, strerror(errno)); + } + } + else if (!get_compressed_page_meta(in, fullpath, &compressed_size, + &blknum, &crc, use_crc32c)) + { + break; + } + + read_len = fread(compressed_page.data, 1, + MAXALIGN(compressed_size), in); + if (read_len != MAXALIGN(compressed_size)) + { + elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", + blknum, fullpath, read_len, compressed_size); + return false; + } + + cur_pos += MAXALIGN(compressed_size); + + COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + + if (compressed_size != BLCKSZ + || page_may_be_compressed(compressed_page.data, file->compress_alg, + backup_version)) + { + int32 uncompressed_size = 0; + const char *errormsg = NULL; + + uncompressed_size = do_decompress(page.data, BLCKSZ, + compressed_page.data, + compressed_size, + file->compress_alg, + &errormsg); + if (uncompressed_size < 0 && errormsg != NULL) + elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", + blknum, fullpath, errormsg); + + if (uncompressed_size != BLCKSZ) + { + if (compressed_size == BLCKSZ) + { + is_valid = false; + continue; + } + elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + fullpath, uncompressed_size); + return false; + } + + rc = validate_one_page(page.data, + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); + } + else + rc = validate_one_page(compressed_page.data, + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); + + switch (rc) + { + case PAGE_IS_NOT_FOUND: + elog(LOG, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); + break; + case PAGE_IS_ZEROED: + elog(LOG, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); + break; + case PAGE_HEADER_IS_INVALID: + elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); + is_valid = false; + break; + case PAGE_CHECKSUM_MISMATCH: + elog(WARNING, "File: %s blknum %u have wrong checksum: %u", file->rel_path, blknum, page_st.checksum); + is_valid = false; + break; + case PAGE_LSN_FROM_FUTURE: + elog(WARNING, "File: %s, block %u, checksum is %s. " + "Page is from future: pageLSN %X/%X stopLSN %X/%X", + file->rel_path, blknum, + checksum_version ? "correct" : "not enabled", + (uint32) (page_st.lsn >> 32), (uint32) page_st.lsn, + (uint32) (stop_lsn >> 32), (uint32) stop_lsn); + break; + } + } + + FIN_FILE_CRC32(use_crc32c, crc); + fclose(in); + + if (crc != file->crc) + { + elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", + fullpath, crc, file->crc); + is_valid = false; + } + + pg_free(headers); + + return is_valid; +} + /* Valiate pages of datafile in backup one by one */ bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, @@ -1774,14 +1996,14 @@ check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return false; } - rc = validate_one_page(page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_lsn, checksum_version); +// rc = validate_one_page(page.data, +// file->segno * RELSEG_SIZE + blknum, +// stop_lsn, &page_lsn, NULL, checksum_version); } else - rc = validate_one_page(compressed_page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_lsn, checksum_version); +// rc = validate_one_page(compressed_page.data, +// file->segno * RELSEG_SIZE + blknum, +// stop_lsn, &page_lsn, NULL, checksum_version); switch (rc) { @@ -1830,7 +2052,6 @@ PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, PageState *checksum_map = NULL; FILE *in = NULL; BlockNumber blknum = 0; - XLogRecPtr page_lsn = 0; char read_buffer[BLCKSZ]; char in_buf[STDIO_BUFSIZE]; @@ -1852,7 +2073,7 @@ PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, for (blknum = 0; blknum < n_blocks; blknum++) { size_t read_len = fread(read_buffer, 1, BLCKSZ, in); - page_lsn = InvalidXLogRecPtr; + PageState page_st; /* report error */ if (ferror(in)) @@ -1862,16 +2083,17 @@ PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, if (read_len == BLCKSZ) { int rc = validate_one_page(read_buffer, segmentno + blknum, - dest_stop_lsn, &page_lsn, checksum_version); + dest_stop_lsn, &page_st, + checksum_version); if (rc == PAGE_IS_VALID) { if (checksum_version) checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; else - checksum_map[blknum].checksum = pg_checksum_page(read_buffer, segmentno + blknum); + checksum_map[blknum].checksum = page_st.checksum; - checksum_map[blknum].lsn = page_lsn; + checksum_map[blknum].lsn = page_st.lsn; } } else @@ -1897,7 +2119,6 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, { FILE *in = NULL; BlockNumber blknum = 0; - XLogRecPtr page_lsn = 0; char read_buffer[BLCKSZ]; char in_buf[STDIO_BUFSIZE]; datapagemap_t *lsn_map = NULL; @@ -1921,7 +2142,10 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, for (blknum = 0; blknum < n_blocks; blknum++) { size_t read_len = fread(read_buffer, 1, BLCKSZ, in); - page_lsn = InvalidXLogRecPtr; +// page_lsn = InvalidXLogRecPtr; + PageState page_st; + +// page_st.lsn = InvalidXLogRecPtr /* report error */ if (ferror(in)) @@ -1931,7 +2155,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, if (read_len == BLCKSZ) { int rc = validate_one_page(read_buffer, segmentno + blknum, - shift_lsn, &page_lsn, checksum_version); + shift_lsn, &page_st, checksum_version); if (rc == PAGE_IS_VALID) datapagemap_add(lsn_map, blknum); @@ -1957,3 +2181,79 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } + +/* attempt to open header file, read content and return as + * array of headers. + */ +BackupPageHeader2* +get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) +{ + int len; + FILE *in = NULL; + char fullpath_hdr[MAXPGPATH]; + BackupPageHeader2 *headers = NULL; + + if (backup_version < 20400) + return NULL; + + snprintf(fullpath_hdr, MAXPGPATH, "%s_hdr", fullpath); + + in = fopen(fullpath_hdr, PG_BINARY_R); + + if (!in) + elog(ERROR, "Cannot open header file \"%s\": %s", fullpath_hdr, strerror(errno)); + + len = file->n_headers * sizeof(BackupPageHeader2); + headers = pgut_malloc(len); + + if (fread(headers, 1, len, in) != len) + elog(ERROR, "Cannot read header file \"%s\": %s", fullpath_hdr, strerror(errno)); + + if (fclose(in)) + elog(ERROR, "Cannot close header file \"%s\": %s", fullpath_hdr, strerror(errno)); + + return headers; +} + +/* */ +bool +get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, + BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c) +{ + + /* read BackupPageHeader */ + BackupPageHeader header; + size_t read_len = fread(&header, 1, sizeof(header), in); + + if (ferror(in)) + elog(ERROR, "Cannot read file \"%s\": %s", + fullpath, strerror(errno)); + + if (read_len != sizeof(header)) + { + if (read_len == 0 && feof(in)) + return false; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "Odd size page found at block %u of \"%s\"", + *blknum, fullpath); + else + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", + *blknum, fullpath, strerror(errno)); + } + + if (crc) + COMP_FILE_CRC32(use_crc32c, *crc, &header, read_len); + + if (header.block == 0 && header.compressed_size == 0) + elog(ERROR, "Empty block in file \"%s\"", fullpath); + + + *blknum = header.block; + *compressed_size = header.compressed_size; + + Assert(*compressed_size <= BLCKSZ); + return true; + +} + diff --git a/src/dir.c b/src/dir.c index a24bda63d..7b367a34e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -214,6 +214,9 @@ pgFileInit(const char *rel_path) /* Number of blocks readed during backup */ file->n_blocks = BLOCKNUM_INVALID; + /* Number of blocks backed up during backup */ + file->n_headers = 0; + return file; } @@ -1546,6 +1549,7 @@ dir_read_file_list(const char *root, const char *external_prefix, crc, segno, n_blocks, + n_headers, dbOid; /* used for partial restore */ pgFile *file; @@ -1587,6 +1591,9 @@ dir_read_file_list(const char *root, const char *external_prefix, if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) file->n_blocks = (int) n_blocks; + if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) + file->n_headers = (int) n_headers; + parray_append(files, file); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a537a388f..fcaf39577 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -200,7 +200,8 @@ typedef struct pgFile Oid relOid; /* relOid extracted from path, if applicable */ ForkName forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ - int n_blocks; /* size of the data file in blocks */ + int n_blocks; /* number of blocks in the data file in data directory */ + int n_headers; /* number of blocks in the data file in backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; /* Flag used strictly by ptrack 1.x backup */ int external_dir_num; /* Number of external directory. 0 if not external */ @@ -580,6 +581,16 @@ typedef struct BackupPageHeader int32 compressed_size; } BackupPageHeader; +/* 4MB for 1GB file */ +typedef struct BackupPageHeader2 +{ + int32 block; /* block number */ + int32 pos; + int32 compressed_size; + XLogRecPtr lsn; + uint16 checksum; +} BackupPageHeader2; + /* Special value for compressed_size field */ #define PageIsOk 0 #define SkipCurrentPage -1 @@ -803,7 +814,7 @@ extern void help_command(char *command); extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); extern int do_validate_all(void); extern int validate_one_page(Page page, BlockNumber absolute_blkno, - XLogRecPtr stop_lsn, XLogRecPtr *page_lsn, + XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); /* return codes for validate_one_page */ @@ -967,7 +978,7 @@ extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *o extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map); + datapagemap_t *lsn_map, BackupPageHeader2 *headers); extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath, bool already_exists); @@ -982,7 +993,7 @@ extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); extern pid_t check_postmaster(const char *pgdata); -extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, +extern bool check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, @@ -1067,8 +1078,8 @@ extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack /* FIO */ extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, - int calg, int clevel, uint32 checksum_version, - datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg); + int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, + BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, diff --git a/src/restore.c b/src/restore.c index 7c863c6c3..5995a1e21 100644 --- a/src/restore.c +++ b/src/restore.c @@ -342,7 +342,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty); - if (pgdata_is_empty && tblspaces_are_empty) + if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty) { elog(INFO, "Destination directory and tablespace directories are empty, " "disabled incremental restore"); @@ -703,7 +703,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * is impossible to use, because bitmap restore rely on pgFile.n_blocks, * which is not always available in old backups. */ - if (parse_program_version(dest_backup->program_version) < 20300) + if (parse_program_version(dest_backup->program_version) < 20400) { use_bitmap = false; @@ -716,10 +716,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, */ if (use_bitmap && parray_num(parent_chain) == 1) { - if (params->incremental_mode == INCR_LSN) - use_bitmap = true; - else + if (params->incremental_mode == INCR_NONE) use_bitmap = false; + else + use_bitmap = true; } /* diff --git a/src/utils/file.c b/src/utils/file.c index 06e1a9889..f368074d4 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1408,7 +1408,7 @@ static void fio_load_file(int out, char const* path) int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, - char **errormsg) + char **errormsg, BackupPageHeader2 **headers) { struct { fio_header hdr; @@ -1511,7 +1511,18 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt else if (hdr.cop == FIO_SEND_FILE_EOF) { /* n_blocks_read reported by EOF */ - n_blocks_read = hdr.size; + n_blocks_read = hdr.arg; + + /* receive headers if any */ + if (hdr.size > 0) + { + *headers = pgut_malloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); + file->n_headers = hdr.size / sizeof(BackupPageHeader2); + +// elog(INFO, "Size: %u", hdr.size); + } + break; } else if (hdr.cop == FIO_PAGE) @@ -1554,7 +1565,9 @@ static void fio_send_pages_impl(int out, char* buf) FILE *in = NULL; BlockNumber blknum = 0; BlockNumber n_blocks_read = 0; - XLogRecPtr page_lsn = 0; + PageState page_st; +// XLogRecPtr page_lsn = 0; +// uint16 page_crc = 0; char read_buffer[BLCKSZ+1]; char in_buf[STDIO_BUFSIZE]; fio_header hdr; @@ -1567,6 +1580,10 @@ static void fio_send_pages_impl(int out, char* buf) /* parse buffer */ datapagemap_t *map = NULL; datapagemap_iterator_t *iter = NULL; + /* page headers */ + int32 hdr_num = -1; + int32 hdr_cur_pos = 0; + BackupPageHeader2 *headers = NULL; /* open source file */ in = fopen(from_fullpath, PG_BINARY_R); @@ -1644,7 +1661,9 @@ static void fio_send_pages_impl(int out, char* buf) } read_len = fread(read_buffer, 1, BLCKSZ, in); - page_lsn = InvalidXLogRecPtr; + + page_st.lsn = InvalidXLogRecPtr; + page_st.checksum = 0; current_pos += BLCKSZ; @@ -1669,7 +1688,8 @@ static void fio_send_pages_impl(int out, char* buf) if (read_len == BLCKSZ) { rc = validate_one_page(read_buffer, req->segmentno + blknum, - InvalidXLogRecPtr, &page_lsn, req->checksumVersion); + InvalidXLogRecPtr, &page_st, + req->checksumVersion); /* TODO: optimize copy of zeroed page */ if (rc == PAGE_IS_ZEROED) @@ -1719,32 +1739,45 @@ static void fio_send_pages_impl(int out, char* buf) * there is no sense to add more checks. */ if ((req->horizonLsn == InvalidXLogRecPtr) || - (page_lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_lsn >= req->horizonLsn)) /* delta */ + (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ + (req->horizonLsn > 0 && page_st.lsn >= req->horizonLsn)) /* delta */ { + int compressed_size = 0; char write_buffer[BLCKSZ*2]; - BackupPageHeader* bph = (BackupPageHeader*)write_buffer; /* compress page */ hdr.cop = FIO_PAGE; - hdr.arg = bph->block = blknum; - hdr.size = sizeof(BackupPageHeader); + hdr.arg = blknum; - bph->compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), - sizeof(write_buffer) - sizeof(BackupPageHeader), - read_buffer, BLCKSZ, req->calg, req->clevel, - NULL); + compressed_size = do_compress(write_buffer, sizeof(write_buffer), + read_buffer, BLCKSZ, req->calg, req->clevel, + NULL); - if (bph->compressed_size <= 0 || bph->compressed_size >= BLCKSZ) + if (compressed_size <= 0 || compressed_size >= BLCKSZ) { /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); - bph->compressed_size = BLCKSZ; + memcpy(write_buffer, read_buffer, BLCKSZ); + compressed_size = BLCKSZ; } - hdr.size += MAXALIGN(bph->compressed_size); + hdr.size = MAXALIGN(compressed_size); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); + + /* set page header for this file */ + hdr_num++; + if (!headers) + headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); + else + headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); + + headers[hdr_num].block = blknum; + headers[hdr_num].lsn = page_st.lsn; + headers[hdr_num].checksum = page_st.checksum; + headers[hdr_num].pos = hdr_cur_pos; + headers[hdr_num].compressed_size = hdr.size; + + hdr_cur_pos += hdr.size; } /* next block */ @@ -1761,14 +1794,23 @@ static void fio_send_pages_impl(int out, char* buf) eof: /* We are done, send eof */ hdr.cop = FIO_SEND_FILE_EOF; - hdr.arg = 0; - hdr.size = n_blocks_read; /* TODO: report number of backed up blocks */ + hdr.arg = n_blocks_read; + hdr.size = (hdr_num+1) * sizeof(BackupPageHeader2); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); + + /* send headers */ +// hdr.cop = FIO_SEND_FILE_HEADERS; +// hdr.arg = hdr_num +1; +// hdr.size = hdr.arg * sizeof(BackupPageHeader2); +// IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +// IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); cleanup: pg_free(map); pg_free(iter); pg_free(errormsg); + pg_free(headers); if (in) fclose(in); return; diff --git a/src/utils/file.h b/src/utils/file.h index 5a6415c81..0cc382119 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -45,6 +45,7 @@ typedef enum // FIO_CHUNK, FIO_SEND_FILE_EOF, FIO_SEND_FILE_CORRUPTION, + FIO_SEND_FILE_HEADERS, /* messages for closing connection */ FIO_DISCONNECT, /* message for compatibility check */ diff --git a/src/validate.c b/src/validate.c index 79c7d3cc2..0579df14e 100644 --- a/src/validate.c +++ b/src/validate.c @@ -363,7 +363,7 @@ pgBackupValidateFiles(void *arg) * check page headers, checksums (if enabled) * and compute checksum of the file */ - if (!check_file_pages(file, file_fullpath, arguments->stop_lsn, + if (!check_file_pages_new(file, file_fullpath, arguments->stop_lsn, arguments->checksum_version, arguments->backup_version)) arguments->corrupted = true; From 29adb5bfdf9158d10f0a3903f93f7c0bd2800f03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 16 Jun 2020 01:51:39 +0300 Subject: [PATCH 1390/2107] [Issue #228] fix remote mode --- src/data.c | 10 +++++++--- src/utils/file.c | 11 +++++++---- src/utils/file.h | 7 ++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/data.c b/src/data.c index 3615aed64..9e9f6b4ab 100644 --- a/src/data.c +++ b/src/data.c @@ -829,7 +829,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* handle hdr */ /* TODO: move in separate function */ - if (headers) + if (headers && file->n_headers > 0) { size_t hdr_size; char to_fullpath_hdr[MAXPGPATH]; @@ -862,10 +862,12 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, if (fclose(out)) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); -// elog(INFO, "n_headers: %u", file->n_headers); + /* TODO: fsync */ - pg_free(headers); +// elog(INFO, "n_headers: %u", file->n_headers); } + + pg_free(headers); } /* @@ -1177,6 +1179,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in = headers[n_hdr].pos; } +// elog(INFO, "BKLKUM: %u", blknum); + // elog(INFO, "Cur_pos: %u", ftell(in)); /* read a page from file */ diff --git a/src/utils/file.c b/src/utils/file.c index f368074d4..70191b6c6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1519,8 +1519,6 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt *headers = pgut_malloc(hdr.size); IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); file->n_headers = hdr.size / sizeof(BackupPageHeader2); - -// elog(INFO, "Size: %u", hdr.size); } break; @@ -1795,9 +1793,14 @@ static void fio_send_pages_impl(int out, char* buf) /* We are done, send eof */ hdr.cop = FIO_SEND_FILE_EOF; hdr.arg = n_blocks_read; - hdr.size = (hdr_num+1) * sizeof(BackupPageHeader2); + hdr.size = 0; + + if (headers) + hdr.size = (hdr_num+1) * sizeof(BackupPageHeader2); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); + + if (headers) + IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); /* send headers */ // hdr.cop = FIO_SEND_FILE_HEADERS; diff --git a/src/utils/file.h b/src/utils/file.h index 0cc382119..19fcf6d5d 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -71,9 +71,10 @@ typedef enum typedef struct { // fio_operations cop; - unsigned cop : 6; - unsigned handle : 6; - unsigned size : 20; +// 16 + unsigned cop : 32; + unsigned handle : 32; + unsigned size : 32; unsigned arg; } fio_header; From 9b36081659e96939b9cb09302e724bb7518b052a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Jun 2020 13:39:42 +0300 Subject: [PATCH 1391/2107] [Issue #228] various fixes and improvements --- src/backup.c | 28 +- src/catalog.c | 5 + src/data.c | 720 +++++++++++++++++++++++++++++---------------- src/dir.c | 6 +- src/merge.c | 26 +- src/pg_probackup.h | 22 +- src/utils/file.c | 12 +- src/validate.c | 2 +- 8 files changed, 547 insertions(+), 274 deletions(-) diff --git a/src/backup.c b/src/backup.c index 00896637e..57bde384b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -664,8 +664,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool for (i = 0; i < parray_num(backup_files_list); i++) { - char to_fullpath[MAXPGPATH]; - pgFile *file = (pgFile *) parray_get(backup_files_list, i); + char to_fullpath[MAXPGPATH]; + pgFile *file = (pgFile *) parray_get(backup_files_list, i); /* TODO: sync directory ? */ if (S_ISDIR(file->mode)) @@ -687,7 +687,21 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); + elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno)); + + /* fsync header file */ + if (file->external_dir_num == 0 && + file->is_datafile && !file->is_cfs && + file->n_headers > 0) + { + char to_fullpath_hdr[MAXPGPATH]; + + snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); + + if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + + } } time(&end_time); @@ -2131,7 +2145,7 @@ backup_files(void *arg) /* backup file */ if (file->is_datafile && !file->is_cfs) { - backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, + backup_data_file_new(&(arguments->conn_arg), file, from_fullpath, to_fullpath, arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, @@ -2147,12 +2161,6 @@ backup_files(void *arg) current.backup_mode, current.parent_backup, true); } - /* No point in storing empty, missing or not changed files */ - if (file->write_size <= 0) - unlink(to_fullpath); -// elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath, -// strerror(errno)); - if (file->write_size == FILE_NOT_FOUND) continue; diff --git a/src/catalog.c b/src/catalog.c index 686b21e70..450f1100a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1906,7 +1906,12 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); if (file->n_headers > 0) + { len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); + len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); + +// elog(INFO, "CRC INT: %li, CRC UINT: %u", file->crc_hdr, file->crc_hdr); + } sprintf(line+len, "}\n"); diff --git a/src/data.c b/src/data.c index 9e9f6b4ab..c4ddcc989 100644 --- a/src/data.c +++ b/src/data.c @@ -32,6 +32,7 @@ typedef union DataPage } DataPage; static BackupPageHeader2* get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version); +static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_fullpath); static bool get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c); @@ -472,6 +473,7 @@ prepare_page(ConnectionArgs *conn_arg, return PageIsOk; } +/* split this function in two: compress() and backup() */ static int compress_and_backup_page(pgFile *file, BlockNumber blknum, FILE *in, FILE *out, pg_crc32 *crc, @@ -526,7 +528,174 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->write_size += write_buffer_size; file->uncompressed_size += BLCKSZ; - return compressed_size; + return write_buffer_size; +} + +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +void +backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, + const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, bool missing_ok) +{ + int rc; + bool use_pagemap; + char *errmsg = NULL; + BlockNumber err_blknum = 0; + /* page headers */ + BackupPageHeader2 *headers = NULL; + + /* sanity */ + if (file->size % BLCKSZ != 0) + elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + file->n_blocks = file->size/BLCKSZ; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if ((backup_mode == BACKUP_MODE_DIFF_PAGE || + backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want to make + * incremental backup, so we should exit. + */ + file->write_size = BYTES_INVALID; + return; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + file->uncompressed_size = 0; + INIT_FILE_CRC32(true, file->crc); + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * In PTRACK 1.x there was a problem + * of data files with missing _ptrack map. + * Such files should be fully copied. + */ + + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev || + !file->pagemap.bitmap) + use_pagemap = false; + else + use_pagemap = true; + + /* Remote mode */ + if (fio_is_remote(FIO_DB_HOST)) + { + + rc = fio_send_pages(to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + calg, clevel, checksum_version, + /* send pagemap if any */ + use_pagemap ? &file->pagemap : NULL, + /* variables for error reporting */ + &err_blknum, &errmsg, &headers); + } + else + { + /* TODO: stop handling errors internally */ + rc = send_pages(conn_arg, to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + backup_mode == BACKUP_MODE_DIFF_DELTA && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + calg, clevel, checksum_version, + /* send pagemap if any */ + use_pagemap ? &file->pagemap : NULL, + &headers, backup_mode, ptrack_version_num, ptrack_schema); + } + + /* check for errors */ + if (rc == FILE_MISSING) + { + elog(LOG, "File \"%s\" is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + err_blknum, to_fullpath, strerror(errno)); + + else if (rc == PAGE_CORRUPTION) + { + if (errmsg) + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, err_blknum, errmsg); + else + elog(ERROR, "Corruption detected in file \"%s\", block %u", + from_fullpath, err_blknum); + } + /* OPEN_FAILED and READ_FAILED */ + else if (rc == OPEN_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot open file \"%s\"", from_fullpath); + } + else if (rc == READ_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot read file \"%s\"", from_fullpath); + } + + file->read_size = rc * BLCKSZ; + + /* refresh n_blocks for FULL and DELTA */ + if (backup_mode == BACKUP_MODE_FULL || + backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = file->read_size / BLCKSZ; + + /* Determine that file didn`t changed in case of incremental backup */ + if (backup_mode != BACKUP_MODE_FULL && + file->exists_in_prev && + file->write_size == 0 && + file->n_blocks > 0) + { + file->write_size = BYTES_INVALID; + } + +cleanup: + + /* finish CRC calculation */ + FIN_FILE_CRC32(true, file->crc); + + /* dump page headers */ + write_page_headers(headers, file, to_fullpath); + + pg_free(errmsg); + pg_free(file->pagemap.bitmap); + pg_free(headers); } /* @@ -600,17 +769,6 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); - /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - /* * Read each page, verify checksum and write it to backup. * If page map is empty or file is not present in previous backup @@ -628,17 +786,13 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, else use_pagemap = true; - /* enable stdio buffering for output file */ - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); - /* Remote mode */ if (fio_is_remote(FIO_DB_HOST)) { char *errmsg = NULL; BlockNumber err_blknum = 0; - int rc = fio_send_pages(out, from_fullpath, file, + int rc = fio_send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, @@ -692,6 +846,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* Local mode */ else { + uint cur_pos_out = 0; /* open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) @@ -753,6 +908,11 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, else if (rc == PageIsOk) { + + /* open local backup file for write */ + if (!out) + out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); + hdr_num++; if (!headers) @@ -763,16 +923,15 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, headers[hdr_num].block = blknum; headers[hdr_num].lsn = page_st.lsn; headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = ftell(out); /* optimize */ - -// elog(INFO, "CRC: %u", headers[hdr_num].checksum); -// elog(INFO, "POS: %u", headers[hdr_num].pos); + headers[hdr_num].pos = cur_pos_out; /* optimize */ headers[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), rc, curr_page, calg, clevel, from_fullpath, to_fullpath); file->n_headers = hdr_num +1; + + cur_pos_out += headers[hdr_num].compressed_size; } /* TODO: handle PageIsCorrupted, currently it is done in prepare_page */ else @@ -846,8 +1005,6 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); - /* TODO: calculate checksums */ - /* TODO: save file to backup_content metainformation */ hdr_size = file->n_headers * sizeof(BackupPageHeader2); // elog(INFO, "Size: %lu, aligh: %lu", hdr_size, MAXALIGN(hdr_size)); @@ -996,8 +1153,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* set stdio buffering for input data file */ setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + /* get headers for this file */ headers = get_data_file_headers(from_fullpath, tmp_file, parse_program_version(backup->program_version)); + if (!headers && tmp_file->n_headers > 0) + elog(ERROR, "Failed to get headers for file \"%s\"", from_fullpath); + /* * Restore the file. * Datafiles are backed up block by block and every block @@ -1136,7 +1297,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers break; if (compressed_size > BLCKSZ) - elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum); + elog(ERROR, "Size of a blknum %i exceed BLCKSZ: %i", blknum, compressed_size); /* incremental restore in LSN mode */ if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) @@ -1171,7 +1332,6 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (headers && cur_pos_in != headers[n_hdr].pos) { - elog(INFO, "Seek to %u", headers[n_hdr].pos); if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) elog(ERROR, "Cannot seek to offset %u of '%s': %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); @@ -1179,10 +1339,6 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in = headers[n_hdr].pos; } -// elog(INFO, "BKLKUM: %u", blknum); - -// elog(INFO, "Cur_pos: %u", ftell(in)); - /* read a page from file */ read_len = fread(page.data, 1, MAXALIGN(compressed_size), in); @@ -1712,7 +1868,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool -check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, +validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version) { size_t read_len = 0; @@ -1726,21 +1882,21 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); + /* nothing to validate */ + if (backup_version >= 20400 && + file->n_headers <= 0) + return true; + in = fopen(fullpath, PG_BINARY_R); if (in == NULL) - { - if (errno == ENOENT) - { - elog(WARNING, "File \"%s\" is not found", fullpath); - return false; - } - elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); - } headers = get_data_file_headers(fullpath, file, backup_version); + if (!headers && file->n_headers > 0) + elog(ERROR, "Failed to get headers for file \"%s\"", fullpath); + /* calc CRC of backup file */ INIT_FILE_CRC32(use_crc32c, crc); @@ -1767,6 +1923,8 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, blknum = headers[n_hdr].block; compressed_size = headers[n_hdr].compressed_size; +// elog(INFO, "POS: %u", headers[n_hdr].pos); + if (cur_pos != headers[n_hdr].pos && fio_fseek(in, headers[n_hdr].pos) < 0) { @@ -1774,14 +1932,25 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, blknum, fullpath, strerror(errno)); } } - else if (!get_compressed_page_meta(in, fullpath, &compressed_size, - &blknum, &crc, use_crc32c)) + else { + if (!get_compressed_page_meta(in, fullpath, &compressed_size, + &blknum, &crc, use_crc32c)) break; } - read_len = fread(compressed_page.data, 1, - MAXALIGN(compressed_size), in); + /* backward compatibility kludge TODO: remove in 3.0 */ + if (compressed_size == PageIsTruncated) + { + elog(LOG, "Block %u of \"%s\" is truncated", + blknum, fullpath); + continue; + } + + Assert(compressed_size <= BLCKSZ); + Assert(compressed_size > 0); + + read_len = fread(compressed_page.data, 1, MAXALIGN(compressed_size), in); if (read_len != MAXALIGN(compressed_size)) { elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", @@ -1789,7 +1958,12 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return false; } - cur_pos += MAXALIGN(compressed_size); +// elog(INFO, "POS1: %lu", cur_pos); + + cur_pos += read_len; + +// elog(INFO, "POS2: %lu", cur_pos); +// elog(INFO, "Compressed size: %i", compressed_size); COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); @@ -1806,8 +1980,11 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, file->compress_alg, &errormsg); if (uncompressed_size < 0 && errormsg != NULL) + { elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", blknum, fullpath, errormsg); + return false; + } if (uncompressed_size != BLCKSZ) { @@ -1816,8 +1993,8 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, is_valid = false; continue; } - elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - fullpath, uncompressed_size); + elog(WARNING, "Page %u of file \"%s\" uncompressed to %d bytes. != BLCKSZ", + blknum, fullpath, uncompressed_size); return false; } @@ -1872,185 +2049,9 @@ check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return is_valid; } -/* Valiate pages of datafile in backup one by one */ -bool -check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version) -{ - size_t read_len = 0; - bool is_valid = true; - FILE *in; - pg_crc32 crc; - bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - - elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); - - in = fopen(fullpath, PG_BINARY_R); - if (in == NULL) - { - if (errno == ENOENT) - { - elog(WARNING, "File \"%s\" is not found", fullpath); - return false; - } - - elog(ERROR, "Cannot open file \"%s\": %s", - fullpath, strerror(errno)); - } - - /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); - - /* read and validate pages one by one */ - while (true) - { - int rc = 0; - DataPage compressed_page; /* used as read buffer */ - DataPage page; - BackupPageHeader header; - BlockNumber blknum = 0; - XLogRecPtr page_lsn = 0; - - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during data file validation"); - - /* read BackupPageHeader */ - read_len = fread(&header, 1, sizeof(header), in); - - if (ferror(in)) - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - - if (read_len != sizeof(header)) - { - if (read_len == 0 && feof(in)) - break; /* EOF found */ - else if (read_len != 0 && feof(in)) - elog(WARNING, - "Odd size page found at block %u of \"%s\"", - blknum, fullpath); - else - elog(WARNING, "Cannot read header of block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - return false; - } - - COMP_FILE_CRC32(use_crc32c, crc, &header, read_len); - - if (header.block == 0 && header.compressed_size == 0) - { - elog(VERBOSE, "Skip empty block of \"%s\"", fullpath); - continue; - } - - if (header.block < blknum) - { - elog(WARNING, "Backup is broken at block %u of \"%s\"", - blknum, fullpath); - return false; - } - - blknum = header.block; - - if (header.compressed_size == PageIsTruncated) - { - elog(LOG, "Block %u of \"%s\" is truncated", - blknum, fullpath); - continue; - } - - Assert(header.compressed_size <= BLCKSZ); - - read_len = fread(compressed_page.data, 1, - MAXALIGN(header.compressed_size), in); - if (read_len != MAXALIGN(header.compressed_size)) - { - elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, fullpath, read_len, header.compressed_size); - return false; - } - - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); - - if (header.compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, - backup_version)) - { - int32 uncompressed_size = 0; - const char *errormsg = NULL; - - uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - header.compressed_size, - file->compress_alg, - &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, fullpath, errormsg); - - if (uncompressed_size != BLCKSZ) - { - if (header.compressed_size == BLCKSZ) - { - is_valid = false; - continue; - } - elog(WARNING, "Page of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - fullpath, uncompressed_size); - return false; - } - -// rc = validate_one_page(page.data, -// file->segno * RELSEG_SIZE + blknum, -// stop_lsn, &page_lsn, NULL, checksum_version); - } - else -// rc = validate_one_page(compressed_page.data, -// file->segno * RELSEG_SIZE + blknum, -// stop_lsn, &page_lsn, NULL, checksum_version); - - switch (rc) - { - case PAGE_IS_NOT_FOUND: - elog(LOG, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); - break; - case PAGE_IS_ZEROED: - elog(LOG, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); - break; - case PAGE_HEADER_IS_INVALID: - elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); - is_valid = false; - break; - case PAGE_CHECKSUM_MISMATCH: - elog(WARNING, "File: %s blknum %u have wrong checksum", file->rel_path, blknum); - is_valid = false; - break; - case PAGE_LSN_FROM_FUTURE: - elog(WARNING, "File: %s, block %u, checksum is %s. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->rel_path, blknum, - checksum_version ? "correct" : "not enabled", - (uint32) (page_lsn >> 32), (uint32) page_lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - break; - } - } - - FIN_FILE_CRC32(use_crc32c, crc); - fclose(in); - - if (crc != file->crc) - { - elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", - fullpath, crc, file->crc); - is_valid = false; - } - - return is_valid; -} - /* read local data file and construct map with block checksums */ -PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, +PageState* +get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { PageState *checksum_map = NULL; @@ -2186,6 +2187,197 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } +/* */ +bool +get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, + BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c) +{ + + /* read BackupPageHeader */ + BackupPageHeader header; + size_t read_len = fread(&header, 1, sizeof(header), in); + + if (ferror(in)) + elog(ERROR, "Cannot read file \"%s\": %s", + fullpath, strerror(errno)); + + if (read_len != sizeof(header)) + { + if (read_len == 0 && feof(in)) + return false; /* EOF found */ + else if (read_len != 0 && feof(in)) + elog(ERROR, + "Odd size page found at block %u of \"%s\"", + *blknum, fullpath); + else + elog(ERROR, "Cannot read header of block %u of \"%s\": %s", + *blknum, fullpath, strerror(errno)); + } + + if (crc) + COMP_FILE_CRC32(use_crc32c, *crc, &header, read_len); + + if (header.block == 0 && header.compressed_size == 0) + elog(ERROR, "Empty block in file \"%s\"", fullpath); + + + *blknum = header.block; + *compressed_size = header.compressed_size; + + elog(INFO, "blknum: %i", header.block); + elog(INFO, "size: %i", header.compressed_size); + elog(INFO, "size2: %i", *compressed_size); + + elog(INFO, "BLKNUM: %i", *blknum); + elog(INFO, "File: %s", fullpath); + + Assert(*compressed_size != 0); + return true; + +} + +/* Open local backup file for writing, set permissions and buffering */ +FILE* +open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) +{ + FILE *out = NULL; + /* open backup file for write */ + out = fopen(to_fullpath, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open backup file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + /* enable stdio buffering for output file */ + *out_buf = pgut_malloc(buf_size); + setvbuf(out, *out_buf, _IOFBF, buf_size); + + return out; +} + +int +send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, + uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, + BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) +{ + FILE *in = NULL; + FILE *out = NULL; + int hdr_num = -1; + uint cur_pos_out = 0; + char curr_page[BLCKSZ]; + int n_blocks_read = 0; + BlockNumber blknum = 0; + datapagemap_iterator_t *iter = NULL; + + /* stdio buffers */ + char *in_buf = NULL; + char *out_buf = NULL; + + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + return FILE_MISSING; + + elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); + } + + /* + * Enable stdio buffering for local input file, + * unless the pagemap is involved, which + * imply a lot of random access. + */ + + if (use_pagemap) + { + iter = datapagemap_iterate(&file->pagemap); + datapagemap_next(iter, &blknum); /* set first block */ + + setvbuf(in, NULL, _IONBF, BUFSIZ); + } + else + { + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + } + + while (blknum < file->n_blocks) + { + PageState page_st; + int rc = prepare_page(conn_arg, file, prev_backup_start_lsn, + blknum, in, backup_mode, curr_page, + true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath, &page_st); + if (rc == PageIsTruncated) + break; + + else if (rc == PageIsOk) + { + /* lazily open backup file (useful for s3) */ + if (!out) + out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); + + hdr_num++; + + if (!*headers) + *headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); + else + *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); + + (*headers)[hdr_num].block = blknum; + (*headers)[hdr_num].pos = cur_pos_out; + (*headers)[hdr_num].lsn = page_st.lsn; + (*headers)[hdr_num].checksum = page_st.checksum; + + (*headers)[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), + rc, curr_page, calg, clevel, + from_fullpath, to_fullpath); + cur_pos_out += MAXALIGN((*headers)[hdr_num].compressed_size); + } + + n_blocks_read++; + + /* next block */ + if (use_pagemap) + { + /* exit if pagemap is exhausted */ + if (!datapagemap_next(iter, &blknum)) + break; + } + else + blknum++; + } + + file->n_headers = hdr_num +1; + + /* cleanup */ + if (in && fclose(in)) + elog(ERROR, "Cannot close the source file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* close local output file */ + if (out && fclose(out)) + elog(ERROR, "Cannot close the backup file \"%s\": %s", + to_fullpath, strerror(errno)); + + pg_free(iter); + pg_free(in_buf); + pg_free(out_buf); + + return n_blocks_read; +} + /* attempt to open header file, read content and return as * array of headers. */ @@ -2194,11 +2386,23 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) { int len; FILE *in = NULL; + pg_crc32 hdr_crc; char fullpath_hdr[MAXPGPATH]; BackupPageHeader2 *headers = NULL; +// elog(INFO, "Backup Version: %u", backup_version); + if (backup_version < 20400) + { + elog(INFO, "HELLO1"); return NULL; + } + + if (file->n_headers <= 0) + { + elog(INFO, "HELLO2"); + return NULL; + } snprintf(fullpath_hdr, MAXPGPATH, "%s_hdr", fullpath); @@ -2213,51 +2417,55 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) if (fread(headers, 1, len, in) != len) elog(ERROR, "Cannot read header file \"%s\": %s", fullpath_hdr, strerror(errno)); + /* validate checksum */ + INIT_FILE_CRC32(true, hdr_crc); + COMP_FILE_CRC32(true, hdr_crc, headers, len); + FIN_FILE_CRC32(true, hdr_crc); + + if (hdr_crc != file->hdr_crc) + { + elog(ERROR, "Header file crc mismatch \"%s\", current: %u, expected: %u", + fullpath_hdr, hdr_crc, file->hdr_crc); + } + if (fclose(in)) elog(ERROR, "Cannot close header file \"%s\": %s", fullpath_hdr, strerror(errno)); return headers; } -/* */ -bool -get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, - BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c) +void +write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_fullpath) { + FILE *out = NULL; + size_t hdr_size = 0; + char to_fullpath_hdr[MAXPGPATH]; - /* read BackupPageHeader */ - BackupPageHeader header; - size_t read_len = fread(&header, 1, sizeof(header), in); - - if (ferror(in)) - elog(ERROR, "Cannot read file \"%s\": %s", - fullpath, strerror(errno)); + if (file->n_headers <= 0) + return; - if (read_len != sizeof(header)) - { - if (read_len == 0 && feof(in)) - return false; /* EOF found */ - else if (read_len != 0 && feof(in)) - elog(ERROR, - "Odd size page found at block %u of \"%s\"", - *blknum, fullpath); - else - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - *blknum, fullpath, strerror(errno)); - } + snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); - if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, &header, read_len); - - if (header.block == 0 && header.compressed_size == 0) - elog(ERROR, "Empty block in file \"%s\"", fullpath); + out = fopen(to_fullpath_hdr, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open header file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + hdr_size = file->n_headers * sizeof(BackupPageHeader2); - *blknum = header.block; - *compressed_size = header.compressed_size; + /* calculate checksums */ + INIT_FILE_CRC32(true, file->hdr_crc); + COMP_FILE_CRC32(true, file->hdr_crc, headers, hdr_size); + FIN_FILE_CRC32(true, file->hdr_crc); - Assert(*compressed_size <= BLCKSZ); - return true; + if (fwrite(headers, 1, hdr_size, out) != hdr_size) + elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + if (fclose(out)) + elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); } - diff --git a/src/dir.c b/src/dir.c index 7b367a34e..a7e93fa1a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1550,7 +1550,8 @@ dir_read_file_list(const char *root, const char *external_prefix, segno, n_blocks, n_headers, - dbOid; /* used for partial restore */ + dbOid, /* used for partial restore */ + hdr_crc; pgFile *file; COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); @@ -1594,6 +1595,9 @@ dir_read_file_list(const char *root, const char *external_prefix, if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) file->n_headers = (int) n_headers; + if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) + file->hdr_crc = (pg_crc32) hdr_crc; + parray_append(files, file); } diff --git a/src/merge.c b/src/merge.c index fb2f39181..c0709e3e9 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1020,6 +1020,9 @@ merge_files(void *arg) tmp_file->n_blocks = file->n_blocks; tmp_file->compress_alg = file->compress_alg; tmp_file->uncompressed_size = file->n_blocks * BLCKSZ; + + tmp_file->n_headers = file->n_headers; + tmp_file->hdr_crc = file->hdr_crc; } else tmp_file->uncompressed_size = tmp_file->write_size; @@ -1140,8 +1143,11 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, FILE *out = NULL; char *buffer = pgut_malloc(STDIO_BUFSIZE); char to_fullpath[MAXPGPATH]; + char to_fullpath_hdr[MAXPGPATH]; char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ + char to_fullpath_tmp2_hdr[MAXPGPATH]; + /* The next possible optimization is copying "as is" the file * from intermediate incremental backup, that didn`t changed in @@ -1152,6 +1158,9 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, join_path_components(to_fullpath, full_database_dir, tmp_file->rel_path); snprintf(to_fullpath_tmp1, MAXPGPATH, "%s_tmp1", to_fullpath); snprintf(to_fullpath_tmp2, MAXPGPATH, "%s_tmp2", to_fullpath); + /* header files */ + snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); + snprintf(to_fullpath_tmp2_hdr, MAXPGPATH, "%s_hdr", to_fullpath_tmp2); /* open temp file */ out = fopen(to_fullpath_tmp1, PG_BINARY_W); @@ -1177,7 +1186,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, were n_blocks is missing. */ - backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file_new(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, false); @@ -1207,11 +1216,26 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); + /* sync header file */ + if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot sync temp header file \"%s\": %s", + to_fullpath_tmp2_hdr, strerror(errno)); + +//<- CRITICAL SECTION + /* Do atomic rename from second temp file to destination file */ if (rename(to_fullpath_tmp2, to_fullpath) == -1) elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", to_fullpath_tmp2, to_fullpath, strerror(errno)); +//<- If we crash here, merge cannot be continued. + + /* Do atomic rename from header file */ + if (rename(to_fullpath_tmp2_hdr, to_fullpath_hdr) == -1) + elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", + to_fullpath_tmp2, to_fullpath, strerror(errno)); +//<- + /* drop temp file */ unlink(to_fullpath_tmp1); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fcaf39577..3f3e90a32 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -192,6 +192,7 @@ typedef struct pgFile */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ + pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ char *rel_path; /* relative path of the file */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ @@ -585,7 +586,7 @@ typedef struct BackupPageHeader typedef struct BackupPageHeader2 { int32 block; /* block number */ - int32 pos; + int32 pos; /* position in backup file */ int32 compressed_size; XLogRecPtr lsn; uint16 checksum; @@ -963,6 +964,11 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok); +extern void backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, + const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, @@ -993,8 +999,8 @@ extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); extern pid_t check_postmaster(const char *pgdata); -extern bool check_file_pages_new(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version); +extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, + uint32 checksum_version, uint32 backup_version); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, XLogRecPtr startpoint, TimeLineID start_tli, @@ -1076,8 +1082,16 @@ extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); +/* open local file to writing */ +extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size); + +extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, + uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, + BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); + /* FIO */ -extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, +extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers); /* return codes for fio_send_pages */ diff --git a/src/utils/file.c b/src/utils/file.c index 70191b6c6..75fbca21f 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1405,11 +1405,13 @@ static void fio_load_file(int out, char const* path) * In case of DELTA mode horizonLsn must be a valid lsn, * otherwise it should be set to InvalidXLogRecPtr. */ -int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, +int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers) { + FILE *out = NULL; + char *out_buf = NULL; struct { fio_header hdr; fio_send_request arg; @@ -1532,6 +1534,10 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + /* lazily open backup file */ + if (!out) + out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); + if (fio_fwrite(out, buf, hdr.size) != hdr.size) { fio_fclose(out); @@ -1545,6 +1551,10 @@ int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPt elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); } + if (out) + fclose(out); + pg_free(out_buf); + return n_blocks_read; } diff --git a/src/validate.c b/src/validate.c index 0579df14e..d7c6ff6a7 100644 --- a/src/validate.c +++ b/src/validate.c @@ -363,7 +363,7 @@ pgBackupValidateFiles(void *arg) * check page headers, checksums (if enabled) * and compute checksum of the file */ - if (!check_file_pages_new(file, file_fullpath, arguments->stop_lsn, + if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn, arguments->checksum_version, arguments->backup_version)) arguments->corrupted = true; From 8cbbce0b152e99885545b6895d4ed971c7fa1df6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Jun 2020 13:47:25 +0300 Subject: [PATCH 1392/2107] [Issue #228] drop old backup_data_file --- src/backup.c | 2 +- src/data.c | 331 +-------------------------------------------- src/merge.c | 2 +- src/pg_probackup.h | 5 - 4 files changed, 3 insertions(+), 337 deletions(-) diff --git a/src/backup.c b/src/backup.c index 57bde384b..d87df4b54 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2145,7 +2145,7 @@ backup_files(void *arg) /* backup file */ if (file->is_datafile && !file->is_cfs) { - backup_data_file_new(&(arguments->conn_arg), file, from_fullpath, to_fullpath, + backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, arguments->prev_start_lsn, current.backup_mode, instance_config.compress_alg, diff --git a/src/data.c b/src/data.c index c4ddcc989..553a32d7c 100644 --- a/src/data.c +++ b/src/data.c @@ -540,7 +540,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, * backup with special header. */ void -backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, +backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, @@ -698,335 +698,6 @@ backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, pg_free(headers); } -/* - * Backup data file in the from_root directory to the to_root directory with - * same relative path. If prev_backup_start_lsn is not NULL, only pages with - * higher lsn will be copied. - * Not just copy file, but read it block by block (use bitmap in case of - * incremental backup), validate checksum, optionally compress and write to - * backup with special header. - */ -void -backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok) -{ - FILE *in = NULL; - FILE *out = NULL; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; /* number of blocks in source file */ - BlockNumber n_blocks_skipped = 0; - int rc; - char curr_page[BLCKSZ]; - bool use_pagemap; - datapagemap_iterator_t *iter = NULL; - - /* headers */ - int hdr_num = -1; - BackupPageHeader2 *headers = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; - - /* sanity */ - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; - - /* set n_blocks for a file */ - file->n_blocks = nblocks; - - /* - * Skip unchanged file only if it exists in previous backup. - * This way we can correctly handle null-sized files which are - * not tracked by pagemap and thus always marked as unchanged. - */ - if ((backup_mode == BACKUP_MODE_DIFF_PAGE || - backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev && !file->pagemap_isabsent) - { - /* - * There are no changed blocks since last backup. We want to make - * incremental backup, so we should exit. - */ - file->write_size = BYTES_INVALID; - return; - } - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); - - /* - * Read each page, verify checksum and write it to backup. - * If page map is empty or file is not present in previous backup - * backup all pages of the relation. - * - * In PTRACK 1.x there was a problem - * of data files with missing _ptrack map. - * Such files should be fully copied. - */ - - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; - - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - char *errmsg = NULL; - BlockNumber err_blknum = 0; - - int rc = fio_send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, - /* variables for error reporting */ - &err_blknum, &errmsg, &headers); - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Failed to open for reading remote file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Failed to read from remote file \"%s\"", from_fullpath); - } - - file->read_size = rc * BLCKSZ; - pg_free(errmsg); - - } - /* Local mode */ - else - { - uint cur_pos_out = 0; - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - - /* In all other cases throw an error */ - elog(ERROR, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - /* Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - - while (blknum < nblocks) - { - PageState page_st; - rc = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath, &page_st); - - if (rc == PageIsTruncated) - break; - - /* TODO: remove */ - else if (rc == SkipCurrentPage) - n_blocks_skipped++; - - else if (rc == PageIsOk) - { - - /* open local backup file for write */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - - hdr_num++; - - if (!headers) - headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); - - headers[hdr_num].block = blknum; - headers[hdr_num].lsn = page_st.lsn; - headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = cur_pos_out; /* optimize */ - - headers[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), - rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); - - file->n_headers = hdr_num +1; - - cur_pos_out += headers[hdr_num].compressed_size; - } - /* TODO: handle PageIsCorrupted, currently it is done in prepare_page */ - else - Assert(false); - - - file->read_size += BLCKSZ; - - /* next block */ - if (use_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } - } - - pg_free(file->pagemap.bitmap); - pg_free(iter); - - /* refresh n_blocks for FULL and DELTA */ - if (backup_mode == BACKUP_MODE_FULL || - backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; - - /* Determine that file didn`t changed in case of incremental backup */ - if (backup_mode != BACKUP_MODE_FULL && - file->exists_in_prev && - file->write_size == 0 && - file->n_blocks > 0) - { - file->write_size = BYTES_INVALID; - } - -cleanup: - /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); - - /* close local input file */ - if (in && fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* close local output file */ - if (out && fclose(out)) - elog(ERROR, "Cannot close the backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - pg_free(in_buf); - pg_free(out_buf); - - /* handle hdr */ - /* TODO: move in separate function */ - if (headers && file->n_headers > 0) - { - size_t hdr_size; - char to_fullpath_hdr[MAXPGPATH]; - - snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); - - out = fopen(to_fullpath_hdr, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - hdr_size = file->n_headers * sizeof(BackupPageHeader2); - -// elog(INFO, "Size: %lu, aligh: %lu", hdr_size, MAXALIGN(hdr_size)); -// elog(INFO, "checksum: %u, lsn: %lu", headers[file->n_headers-1].checksum, headers[file->n_headers-1].lsn); -// elog(INFO, "POS: %u", headers[file->n_headers-1].pos); -// elog(INFO, "blknum: %u", headers[file->n_headers-1].block); -// elog(INFO, "size: %u", headers[file->n_headers-1].compressed_size); - - if (fwrite(headers, 1, hdr_size, out) != hdr_size) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno)); - - if (fclose(out)) - elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); - - /* TODO: fsync */ - -// elog(INFO, "n_headers: %u", file->n_headers); - } - - pg_free(headers); -} - /* * Backup non data file * We do not apply compression to this file. diff --git a/src/merge.c b/src/merge.c index c0709e3e9..fbc271b7c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1186,7 +1186,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, were n_blocks is missing. */ - backup_data_file_new(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3f3e90a32..5e2504258 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -964,11 +964,6 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, bool missing_ok); -extern void backup_data_file_new(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, From 76f9267efe0b5dc9d0039b9fc5bff9432b7a0d1f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Jun 2020 17:56:17 +0300 Subject: [PATCH 1393/2107] [Issue #228] fix decompression errors --- src/data.c | 2 +- src/dir.c | 15 +++++++++++++++ src/utils/file.c | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 553a32d7c..19e55592d 100644 --- a/src/data.c +++ b/src/data.c @@ -528,7 +528,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->write_size += write_buffer_size; file->uncompressed_size += BLCKSZ; - return write_buffer_size; + return compressed_size; } /* diff --git a/src/dir.c b/src/dir.c index a7e93fa1a..db3df2f2b 100644 --- a/src/dir.c +++ b/src/dir.c @@ -250,6 +250,21 @@ pgFileDelete(pgFile *file, const char *full_path) elog(ERROR, "Cannot remove file \"%s\": %s", full_path, strerror(errno)); } + + if (file->n_headers > 0) + { + char full_path_hdr[MAXPGPATH]; + + snprintf(full_path_hdr, MAXPGPATH, "%s_hdr", full_path); + + if (remove(full_path_hdr) == -1) + { + if (errno == ENOENT) + return; + elog(ERROR, "Cannot remove file \"%s\": %s", full_path_hdr, + strerror(errno)); + } + } } /* diff --git a/src/utils/file.c b/src/utils/file.c index 75fbca21f..8639fff07 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1783,7 +1783,7 @@ static void fio_send_pages_impl(int out, char* buf) headers[hdr_num].lsn = page_st.lsn; headers[hdr_num].checksum = page_st.checksum; headers[hdr_num].pos = hdr_cur_pos; - headers[hdr_num].compressed_size = hdr.size; + headers[hdr_num].compressed_size = compressed_size; hdr_cur_pos += hdr.size; } From ddbc6fd5e68e1677e6363828072cd7277baf3b69 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Jun 2020 22:56:46 +0300 Subject: [PATCH 1394/2107] [Issue #228] remove compressed_size attribute from BackupPageHeader2 --- src/data.c | 128 ++++++++++++++++++++++++++------------------- src/pg_probackup.h | 3 +- src/utils/file.c | 24 +++++---- 3 files changed, 88 insertions(+), 67 deletions(-) diff --git a/src/data.c b/src/data.c index 19e55592d..7ca293c87 100644 --- a/src/data.c +++ b/src/data.c @@ -506,7 +506,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, { // memcpy(write_buffer, &header, sizeof(header)); memcpy(write_buffer, compressed_page, compressed_size); - write_buffer_size = MAXALIGN(compressed_size); + write_buffer_size = compressed_size; } /* Non-positive value means that compression failed. Write it as is. */ else @@ -875,6 +875,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers off_t cur_pos_out = 0; off_t cur_pos_in = 0; + /* should not be possible */ + Assert(!(backup_version >= 20400 && file->n_headers <= 0)); + /* * We rely on stdio buffering of input and output. * For buffering to be efficient, we try to minimize the @@ -915,7 +918,13 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers blknum = headers[n_hdr].block; page_lsn = headers[n_hdr].lsn; page_crc = headers[n_hdr].checksum; - compressed_size = headers[n_hdr].compressed_size; + /* calculate payload size by comparing current and next page positions */ + compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos; + + Assert(compressed_size > 0); + Assert(compressed_size <= BLCKSZ); + + read_len = compressed_size; } else { @@ -923,6 +932,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers &blknum, NULL, false)) { cur_pos_in += sizeof(BackupPageHeader); + + /* backward compatibility kludge TODO: remove in 3.0 */ + read_len = MAXALIGN(compressed_size); } else break; @@ -963,6 +975,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers break; } + Assert(compressed_size > 0); + Assert(compressed_size <= BLCKSZ); + /* no point in writing redundant data */ if (nblocks > 0 && blknum >= nblocks) break; @@ -992,8 +1007,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers /* if this page is marked as already restored, then skip it */ if (map && datapagemap_is_set(map, blknum)) { -// elog(INFO, "HAHAHA"); - /* skip to the next page for backup withot header file */ + /* Backward compatibility kludge TODO: remove in 3.0 + * skip to the next page for backup withot header file + */ if (!headers && fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) elog(ERROR, "Cannot seek block %u of '%s': %s", blknum, from_fullpath, strerror(errno)); @@ -1011,13 +1027,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } /* read a page from file */ - read_len = fread(page.data, 1, MAXALIGN(compressed_size), in); - -// elog(INFO, "Blocknum %u, zread: %u ", blknum, compressed_size); - - if (read_len != MAXALIGN(compressed_size)) - elog(ERROR, "Cannot read block %u of \"%s\", read %zu of %d", - blknum, from_fullpath, read_len, compressed_size); + if (fread(page.data, 1, read_len, in) != read_len) + elog(ERROR, "Cannot read block %u file \"%s\": %s", + blknum, from_fullpath, strerror(errno)); cur_pos_in += read_len; @@ -1072,6 +1084,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers write_len += BLCKSZ; cur_pos_out += BLCKSZ; /* update current write position */ + /* Mark page as restored, to avoid reading this page when restoring parent backups */ if (map) datapagemap_add(map, blknum); } @@ -1553,10 +1566,8 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); - /* nothing to validate */ - if (backup_version >= 20400 && - file->n_headers <= 0) - return true; + /* should not be possible */ + Assert(!(backup_version >= 20400 && file->n_headers <= 0)); in = fopen(fullpath, PG_BINARY_R); if (in == NULL) @@ -1592,22 +1603,34 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, break; blknum = headers[n_hdr].block; - compressed_size = headers[n_hdr].compressed_size; + /* calculate payload size by comparing current and next page positions */ + compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos; -// elog(INFO, "POS: %u", headers[n_hdr].pos); + Assert(compressed_size > 0); + Assert(compressed_size <= BLCKSZ); - if (cur_pos != headers[n_hdr].pos && - fio_fseek(in, headers[n_hdr].pos) < 0) + if (cur_pos != headers[n_hdr].pos) { - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); + if (fio_fseek(in, headers[n_hdr].pos) < 0) + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, fullpath, strerror(errno)); + + cur_pos = headers[n_hdr].pos; } + + read_len = compressed_size; } + /* old backups rely on header located directly in data file */ else { if (!get_compressed_page_meta(in, fullpath, &compressed_size, &blknum, &crc, use_crc32c)) break; + + /* Backward compatibility kludge, TODO: remove in 3.0 + * for some reason we padded compressed pages in old versions + */ + read_len = MAXALIGN(compressed_size); } /* backward compatibility kludge TODO: remove in 3.0 */ @@ -1621,21 +1644,16 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, Assert(compressed_size <= BLCKSZ); Assert(compressed_size > 0); - read_len = fread(compressed_page.data, 1, MAXALIGN(compressed_size), in); - if (read_len != MAXALIGN(compressed_size)) + if (fread(compressed_page.data, 1, read_len, in) != read_len) { - elog(WARNING, "Cannot read block %u of \"%s\" read %zu of %d", - blknum, fullpath, read_len, compressed_size); + elog(WARNING, "Cannot read block %u file \"%s\": %s", + blknum, fullpath, strerror(errno)); return false; } -// elog(INFO, "POS1: %lu", cur_pos); - + /* update current position */ cur_pos += read_len; -// elog(INFO, "POS2: %lu", cur_pos); -// elog(INFO, "Compressed size: %i", compressed_size); - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); if (compressed_size != BLCKSZ @@ -1895,12 +1913,12 @@ get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, *blknum = header.block; *compressed_size = header.compressed_size; - elog(INFO, "blknum: %i", header.block); - elog(INFO, "size: %i", header.compressed_size); - elog(INFO, "size2: %i", *compressed_size); - - elog(INFO, "BLKNUM: %i", *blknum); - elog(INFO, "File: %s", fullpath); +// elog(INFO, "blknum: %i", header.block); +// elog(INFO, "size: %i", header.compressed_size); +// elog(INFO, "size2: %i", *compressed_size); +// +// elog(INFO, "BLKNUM: %i", *blknum); +// elog(INFO, "File: %s", fullpath); Assert(*compressed_size != 0); return true; @@ -1930,6 +1948,7 @@ open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) return out; } +/* backup local file */ int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, @@ -1944,6 +1963,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f int n_blocks_read = 0; BlockNumber blknum = 0; datapagemap_iterator_t *iter = NULL; + int compressed_size = 0; /* stdio buffers */ char *in_buf = NULL; @@ -2004,17 +2024,17 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f if (!*headers) *headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); else - *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); + *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+1) * sizeof(BackupPageHeader2)); (*headers)[hdr_num].block = blknum; (*headers)[hdr_num].pos = cur_pos_out; (*headers)[hdr_num].lsn = page_st.lsn; (*headers)[hdr_num].checksum = page_st.checksum; - (*headers)[hdr_num].compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), - rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); - cur_pos_out += MAXALIGN((*headers)[hdr_num].compressed_size); + compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), + rc, curr_page, calg, clevel, + from_fullpath, to_fullpath); + cur_pos_out += compressed_size; } n_blocks_read++; @@ -2030,7 +2050,13 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f blknum++; } - file->n_headers = hdr_num +1; + /* add one additional header */ + if (*headers) + { + file->n_headers = hdr_num +1; + *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+2) * sizeof(BackupPageHeader2)); + (*headers)[hdr_num+1].pos = cur_pos_out; + } /* cleanup */ if (in && fclose(in)) @@ -2061,19 +2087,11 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) char fullpath_hdr[MAXPGPATH]; BackupPageHeader2 *headers = NULL; -// elog(INFO, "Backup Version: %u", backup_version); - if (backup_version < 20400) - { - elog(INFO, "HELLO1"); return NULL; - } if (file->n_headers <= 0) - { - elog(INFO, "HELLO2"); return NULL; - } snprintf(fullpath_hdr, MAXPGPATH, "%s_hdr", fullpath); @@ -2082,7 +2100,11 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) if (!in) elog(ERROR, "Cannot open header file \"%s\": %s", fullpath_hdr, strerror(errno)); - len = file->n_headers * sizeof(BackupPageHeader2); + /* + * the actual number of headers in header file is n+1, last one is a dummy header, + * used for calculation of compressed_size for actual last header. + */ + len = (file->n_headers+1) * sizeof(BackupPageHeader2); headers = pgut_malloc(len); if (fread(headers, 1, len, in) != len) @@ -2094,10 +2116,8 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) FIN_FILE_CRC32(true, hdr_crc); if (hdr_crc != file->hdr_crc) - { elog(ERROR, "Header file crc mismatch \"%s\", current: %u, expected: %u", fullpath_hdr, hdr_crc, file->hdr_crc); - } if (fclose(in)) elog(ERROR, "Cannot close header file \"%s\": %s", fullpath_hdr, strerror(errno)); @@ -2127,7 +2147,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_full elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); - hdr_size = file->n_headers * sizeof(BackupPageHeader2); + hdr_size = (file->n_headers+1) * sizeof(BackupPageHeader2); /* calculate checksums */ INIT_FILE_CRC32(true, file->hdr_crc); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5e2504258..d1eb4c039 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -585,10 +585,9 @@ typedef struct BackupPageHeader /* 4MB for 1GB file */ typedef struct BackupPageHeader2 { + XLogRecPtr lsn; int32 block; /* block number */ int32 pos; /* position in backup file */ - int32 compressed_size; - XLogRecPtr lsn; uint16 checksum; } BackupPageHeader2; diff --git a/src/utils/file.c b/src/utils/file.c index 8639fff07..c2a17fbea 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1427,9 +1427,6 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f -------------------------------------------------------------- */ -// elog(WARNING, "Size: %lu", sizeof(fio_header)); -// elog(ERROR, "Size: %lu", MAXALIGN(sizeof(fio_header))); - req.hdr.cop = FIO_SEND_PAGES; if (pagemap) @@ -1520,7 +1517,7 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f { *headers = pgut_malloc(hdr.size); IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); - file->n_headers = hdr.size / sizeof(BackupPageHeader2); + file->n_headers = (hdr.size / sizeof(BackupPageHeader2)) -1; } break; @@ -1590,7 +1587,7 @@ static void fio_send_pages_impl(int out, char* buf) datapagemap_iterator_t *iter = NULL; /* page headers */ int32 hdr_num = -1; - int32 hdr_cur_pos = 0; + int32 cur_pos_out = 0; BackupPageHeader2 *headers = NULL; /* open source file */ @@ -1767,7 +1764,7 @@ static void fio_send_pages_impl(int out, char* buf) memcpy(write_buffer, read_buffer, BLCKSZ); compressed_size = BLCKSZ; } - hdr.size = MAXALIGN(compressed_size); + hdr.size = compressed_size; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); @@ -1777,15 +1774,14 @@ static void fio_send_pages_impl(int out, char* buf) if (!headers) headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); else - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1 ) * sizeof(BackupPageHeader2)); + headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1) * sizeof(BackupPageHeader2)); headers[hdr_num].block = blknum; headers[hdr_num].lsn = page_st.lsn; headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = hdr_cur_pos; - headers[hdr_num].compressed_size = compressed_size; + headers[hdr_num].pos = cur_pos_out; - hdr_cur_pos += hdr.size; + cur_pos_out += compressed_size; } /* next block */ @@ -1806,7 +1802,13 @@ static void fio_send_pages_impl(int out, char* buf) hdr.size = 0; if (headers) - hdr.size = (hdr_num+1) * sizeof(BackupPageHeader2); + { + hdr.size = (hdr_num+2) * sizeof(BackupPageHeader2); + + /* add one additional header */ + headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+2) * sizeof(BackupPageHeader2)); + headers[hdr_num+1].pos = cur_pos_out; + } IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (headers) From 07831038675fc5c8172f4731d916eefdf4e0f945 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jun 2020 03:33:44 +0300 Subject: [PATCH 1395/2107] [Issue #228] return BackupPageHeader into data file to fix merge retry, remote 8 bytes alignment in datafiles, do not skip header when running data file validation --- src/data.c | 181 +++++++++++++++++++++++++++------------------ src/merge.c | 20 ++++- src/pg_probackup.h | 2 +- src/restore.c | 2 +- src/utils/file.c | 21 +++--- 5 files changed, 143 insertions(+), 83 deletions(-) diff --git a/src/data.c b/src/data.c index 7ca293c87..25ec01e81 100644 --- a/src/data.c +++ b/src/data.c @@ -25,16 +25,16 @@ #include "utils/thread.h" /* Union to ease operations on relation pages */ -typedef union DataPage +typedef struct DataPage { - PageHeaderData page_data; + BackupPageHeader bph; char data[BLCKSZ]; } DataPage; static BackupPageHeader2* get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version); static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_fullpath); -static bool get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, - BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c); +static bool get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, + pg_crc32 *crc, bool use_crc32c); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ @@ -481,17 +481,15 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, CompressAlg calg, int clevel, const char *from_fullpath, const char *to_fullpath) { -// BackupPageHeader header; - int compressed_size; + int compressed_size = 0; size_t write_buffer_size = 0; - char write_buffer[BLCKSZ]; - char compressed_page[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ + char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ + BackupPageHeader* bph = (BackupPageHeader*)write_buffer; const char *errormsg = NULL; -// header.block = blknum; - /* Compress the page */ - compressed_size = do_compress(compressed_page, sizeof(compressed_page), + compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), + sizeof(write_buffer) - sizeof(BackupPageHeader), page, BLCKSZ, calg, clevel, &errormsg); /* Something went wrong and errormsg was assigned, throw a warning */ @@ -501,21 +499,16 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, file->compress_alg = calg; /* TODO: wtf? why here? */ - /* The page was successfully compressed. */ - if (compressed_size > 0 && compressed_size < BLCKSZ) - { -// memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer, compressed_page, compressed_size); - write_buffer_size = compressed_size; - } - /* Non-positive value means that compression failed. Write it as is. */ - else + /* compression didn`t worked */ + if (compressed_size <= 0 || compressed_size >= BLCKSZ) { + /* Do not compress page */ + memcpy(write_buffer + sizeof(BackupPageHeader), page, BLCKSZ); compressed_size = BLCKSZ; -// memcpy(write_buffer, &header, sizeof(header)); - memcpy(write_buffer, page, BLCKSZ); - write_buffer_size = compressed_size; } + bph->block = blknum; + bph->compressed_size = compressed_size; + write_buffer_size = compressed_size + sizeof(BackupPageHeader); /* Update CRC */ COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); @@ -747,7 +740,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map) + XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool is_merge) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); @@ -825,9 +818,10 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* get headers for this file */ - headers = get_data_file_headers(from_fullpath, tmp_file, parse_program_version(backup->program_version)); + if (!is_merge) + headers = get_data_file_headers(from_fullpath, tmp_file, parse_program_version(backup->program_version)); - if (!headers && tmp_file->n_headers > 0) + if (!is_merge && !headers && tmp_file->n_headers > 0) elog(ERROR, "Failed to get headers for file \"%s\"", from_fullpath); /* @@ -895,6 +889,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers for (;;) { off_t write_pos; + size_t len; size_t read_len; DataPage page; int32 compressed_size = 0; @@ -918,23 +913,44 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers blknum = headers[n_hdr].block; page_lsn = headers[n_hdr].lsn; page_crc = headers[n_hdr].checksum; - /* calculate payload size by comparing current and next page positions */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos; + /* calculate payload size by comparing current and next page positions, + * page header is not included */ + compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); Assert(compressed_size > 0); Assert(compressed_size <= BLCKSZ); - read_len = compressed_size; + read_len = compressed_size + sizeof(BackupPageHeader); } else { - if (get_compressed_page_meta(in, from_fullpath, &compressed_size, - &blknum, NULL, false)) + /* We get into this function either when restoring old backup + * or when merging something. Aligh read_len only in restoring + * or merging old backup. + */ + if (get_compressed_page_meta(in, from_fullpath, &(page).bph, NULL, false)) { cur_pos_in += sizeof(BackupPageHeader); /* backward compatibility kludge TODO: remove in 3.0 */ - read_len = MAXALIGN(compressed_size); + blknum = page.bph.block; + compressed_size = page.bph.compressed_size; + + /* this will backfire when retrying merge of old backups, + * just pray that this will never happen. + */ + if (backup_version >= 20400) + read_len = compressed_size; + else + read_len = MAXALIGN(compressed_size); + +// elog(INFO, "FILE: %s", from_fullpath); +// elog(INFO, "blknum: %i", blknum); +// +// elog(INFO, "POS: %u", cur_pos_in); +// elog(INFO, "SIZE: %i", compressed_size); +// elog(INFO, "ASIZE: %i", read_len); + } else break; @@ -1008,9 +1024,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (map && datapagemap_is_set(map, blknum)) { /* Backward compatibility kludge TODO: remove in 3.0 - * skip to the next page for backup withot header file + * go to the next page. */ - if (!headers && fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0) + if (!headers && fseek(in, read_len, SEEK_CUR) != 0) elog(ERROR, "Cannot seek block %u of '%s': %s", blknum, from_fullpath, strerror(errno)); continue; @@ -1027,9 +1043,14 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } /* read a page from file */ - if (fread(page.data, 1, read_len, in) != read_len) + if (headers) + len = fread(&page, 1, read_len, in); + else + len = fread(page.data, 1, read_len, in); + + if (len != read_len) elog(ERROR, "Cannot read block %u file \"%s\": %s", - blknum, from_fullpath, strerror(errno)); + blknum, from_fullpath, strerror(errno)); cur_pos_in += read_len; @@ -1562,7 +1583,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; BackupPageHeader2 *headers = NULL; int n_hdr = -1; - off_t cur_pos = 0; + off_t cur_pos_in = 0; elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); @@ -1586,6 +1607,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, while (true) { int rc = 0; + size_t len = 0; DataPage compressed_page; /* used as read buffer */ int compressed_size = 0; DataPage page; @@ -1603,40 +1625,47 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, break; blknum = headers[n_hdr].block; - /* calculate payload size by comparing current and next page positions */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos; + /* calculate payload size by comparing current and next page positions, + * page header is not included. + */ + compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); Assert(compressed_size > 0); Assert(compressed_size <= BLCKSZ); - if (cur_pos != headers[n_hdr].pos) + read_len = sizeof(BackupPageHeader) + compressed_size; + + if (cur_pos_in != headers[n_hdr].pos) { if (fio_fseek(in, headers[n_hdr].pos) < 0) elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); + else + elog(INFO, "Seek to %u", headers[n_hdr].pos); - cur_pos = headers[n_hdr].pos; + cur_pos_in = headers[n_hdr].pos; } - - read_len = compressed_size; } /* old backups rely on header located directly in data file */ else { - if (!get_compressed_page_meta(in, fullpath, &compressed_size, - &blknum, &crc, use_crc32c)) - break; - - /* Backward compatibility kludge, TODO: remove in 3.0 - * for some reason we padded compressed pages in old versions - */ - read_len = MAXALIGN(compressed_size); + if (get_compressed_page_meta(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) + { + /* Backward compatibility kludge, TODO: remove in 3.0 + * for some reason we padded compressed pages in old versions + */ + blknum = compressed_page.bph.block; + compressed_size = compressed_page.bph.compressed_size; + read_len = MAXALIGN(compressed_size); + } + else + break; } /* backward compatibility kludge TODO: remove in 3.0 */ if (compressed_size == PageIsTruncated) { - elog(LOG, "Block %u of \"%s\" is truncated", + elog(INFO, "Block %u of \"%s\" is truncated", blknum, fullpath); continue; } @@ -1644,7 +1673,17 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, Assert(compressed_size <= BLCKSZ); Assert(compressed_size > 0); - if (fread(compressed_page.data, 1, read_len, in) != read_len) + if (headers) + len = fread(&compressed_page, 1, read_len, in); + else + len = fread(compressed_page.data, 1, read_len, in); + +// elog(INFO, "POS: %u", cur_pos_in); +// +// elog(INFO, "LEN: %i", len); +// elog(INFO, "READ_LEN: %i", read_len); + + if (len != read_len) { elog(WARNING, "Cannot read block %u file \"%s\": %s", blknum, fullpath, strerror(errno)); @@ -1652,9 +1691,12 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } /* update current position */ - cur_pos += read_len; + cur_pos_in += read_len; - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + if (headers) + COMP_FILE_CRC32(use_crc32c, crc, &compressed_page, read_len); + else + COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); if (compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, @@ -1878,40 +1920,39 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* */ bool -get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, - BlockNumber *blknum, pg_crc32 *crc, bool use_crc32c) +get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, + pg_crc32 *crc, bool use_crc32c) { /* read BackupPageHeader */ - BackupPageHeader header; - size_t read_len = fread(&header, 1, sizeof(header), in); + size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); if (ferror(in)) elog(ERROR, "Cannot read file \"%s\": %s", fullpath, strerror(errno)); - if (read_len != sizeof(header)) + if (read_len != sizeof(BackupPageHeader)) { if (read_len == 0 && feof(in)) return false; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "Odd size page found at block %u of \"%s\"", - *blknum, fullpath); + "Odd size page found at offset %lu of \"%s\"", + ftell(in), fullpath); else - elog(ERROR, "Cannot read header of block %u of \"%s\": %s", - *blknum, fullpath, strerror(errno)); + elog(ERROR, "Cannot read header at offset %lu of \"%s\": %s", + ftell(in), fullpath, strerror(errno)); } if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, &header, read_len); + COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); - if (header.block == 0 && header.compressed_size == 0) + if (bph->block == 0 && bph->compressed_size == 0) elog(ERROR, "Empty block in file \"%s\"", fullpath); - *blknum = header.block; - *compressed_size = header.compressed_size; +// *blknum = header.block; +// *compressed_size = header.compressed_size; // elog(INFO, "blknum: %i", header.block); // elog(INFO, "size: %i", header.compressed_size); @@ -1920,7 +1961,7 @@ get_compressed_page_meta(FILE *in, const char *fullpath, int32 *compressed_size, // elog(INFO, "BLKNUM: %i", *blknum); // elog(INFO, "File: %s", fullpath); - Assert(*compressed_size != 0); + Assert(bph->compressed_size != 0); return true; } @@ -2034,7 +2075,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), rc, curr_page, calg, clevel, from_fullpath, to_fullpath); - cur_pos_out += compressed_size; + cur_pos_out += compressed_size + sizeof(BackupPageHeader); } n_blocks_read++; diff --git a/src/merge.c b/src/merge.c index fbc271b7c..1599f3b41 100644 --- a/src/merge.c +++ b/src/merge.c @@ -464,7 +464,10 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (!dest_backup) elog(ERROR, "Destination backup is missing, cannot continue merge"); - elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); + if (dest_backup->status == BACKUP_STATUS_MERGING) + elog(INFO, "Retry failed merge of backup %s with parent chain", base36enc(dest_backup->start_time)); + else + elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); /* sanity */ if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && @@ -539,6 +542,19 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) elog(INFO, "Validate parent chain for backup %s", base36enc(dest_backup->start_time)); + /* forbid merge retry for failed merges between 2.4.0 and any + * older version. Several format changes makes it impossible + * to determine the exact format any speific file is got. + */ + if (full_backup->status == BACKUP_STATUS_MERGING && + parse_program_version(dest_backup->program_version) >= 20400 && + parse_program_version(full_backup->program_version) < 20400) + { + elog(ERROR, "Retry of failed merge for backups with different between minor " + "versions is forbidden to avoid data corruption because of storage format " + "changes introduced in 2.4.0 version, please take a new full backup"); + } + /* * Validate or revalidate all members of parent chain * with sole exception of FULL backup. If it has MERGING status @@ -1171,7 +1187,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, - use_bitmap, NULL, InvalidXLogRecPtr, NULL); + use_bitmap, NULL, InvalidXLogRecPtr, NULL, true); if (fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_tmp1, strerror(errno)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d1eb4c039..0ff51480a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -974,7 +974,7 @@ extern void backup_non_data_file_internal(const char *from_fullpath, extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map); + XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool is_merge); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, diff --git a/src/restore.c b/src/restore.c index 5995a1e21..d99ce6711 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1144,7 +1144,7 @@ restore_files(void *arg) arguments->restored_bytes += restore_data_file(arguments->parent_chain, dest_file, out, to_fullpath, arguments->use_bitmap, checksum_map, - arguments->shift_lsn, lsn_map); + arguments->shift_lsn, lsn_map, true); } else { diff --git a/src/utils/file.c b/src/utils/file.c index c2a17fbea..f3d451bea 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1569,16 +1569,14 @@ static void fio_send_pages_impl(int out, char* buf) { FILE *in = NULL; BlockNumber blknum = 0; + int current_pos = 0; BlockNumber n_blocks_read = 0; PageState page_st; -// XLogRecPtr page_lsn = 0; -// uint16 page_crc = 0; char read_buffer[BLCKSZ+1]; char in_buf[STDIO_BUFSIZE]; fio_header hdr; fio_send_request *req = (fio_send_request*) buf; char *from_fullpath = (char*) buf + sizeof(fio_send_request); - int current_pos = 0; bool with_pagemap = req->bitmapsize > 0 ? true : false; /* error reporting */ char *errormsg = NULL; @@ -1749,22 +1747,27 @@ static void fio_send_pages_impl(int out, char* buf) { int compressed_size = 0; char write_buffer[BLCKSZ*2]; + BackupPageHeader* bph = (BackupPageHeader*)write_buffer; /* compress page */ hdr.cop = FIO_PAGE; hdr.arg = blknum; - compressed_size = do_compress(write_buffer, sizeof(write_buffer), - read_buffer, BLCKSZ, req->calg, req->clevel, - NULL); + compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), + sizeof(write_buffer) - sizeof(BackupPageHeader), + read_buffer, BLCKSZ, req->calg, req->clevel, + NULL); if (compressed_size <= 0 || compressed_size >= BLCKSZ) { /* Do not compress page */ - memcpy(write_buffer, read_buffer, BLCKSZ); + memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); compressed_size = BLCKSZ; } - hdr.size = compressed_size; + bph->block = blknum; + bph->compressed_size = compressed_size; + + hdr.size = compressed_size + sizeof(BackupPageHeader); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); @@ -1781,7 +1784,7 @@ static void fio_send_pages_impl(int out, char* buf) headers[hdr_num].checksum = page_st.checksum; headers[hdr_num].pos = cur_pos_out; - cur_pos_out += compressed_size; + cur_pos_out += hdr.size; } /* next block */ From 367c91b2e764b89b736118018d4fcb8624a72bec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jun 2020 03:36:25 +0300 Subject: [PATCH 1396/2107] tests: new compatibility tests --- tests/compatibility.py | 195 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index f2e208e44..ff1359820 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -309,6 +309,11 @@ def test_backward_compatibility_ptrack(self): self.set_archiving(backup_dir, 'node', node, old_binary=True) node.slow_start() + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.pgbench_init(scale=10) # FULL backup with old binary @@ -802,6 +807,196 @@ def test_backward_compatibility_merge_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_3(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=50) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, old_binary=True, options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + page1 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE from pgbench_accounts where ctid > '(10,1)'") + + # PAGE2 backup with OLD binary + page2 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata2 = self.pgdata_content(node.data_dir) + + # PAGE3 backup with OLD binary + page3 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE4 backup with NEW binary + page4 = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + pgdata4 = self.pgdata_content(node.data_dir) + + # merge backups one by one and check data correctness + # merge PAGE1 + self.merge_backup( + backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + + # check data correctness for PAGE1 + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, backup_id=page1, + options=['--log-level-file=VERBOSE']) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + # merge PAGE2 + self.merge_backup(backup_dir, "node", page2) + + # check data correctness for PAGE2 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + # merge PAGE3 + self.show_pb(backup_dir, 'node', page3) + self.merge_backup(backup_dir, "node", page3) + self.show_pb(backup_dir, 'node', page3) + + # check data correctness for PAGE3 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # merge PAGE4 + self.merge_backup(backup_dir, "node", page4) + + # check data correctness for PAGE4 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata4, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_4(self): + """ + Start merge between minor version, crash and retry it. + old binary version =< 2.2.7 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, old_binary=True, options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "20", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE backup with NEW binary + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + pgdata = self.pgdata_content(node.data_dir) + + # merge PAGE4 + gdb = self.merge_backup(backup_dir, "node", page_id, gdb=True) + + gdb.set_breakpoint('rename') + gdb.run_until_break() + gdb.continue_execution_until_break(1000) + gdb._execute('signal SIGKILL') + + self.merge_backup(backup_dir, "node", page_id) + + # check data correctness for PAGE + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page_id) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_page_vacuum_truncate(self): """ From cb7050320b3374086c6d90cea2096af6fe54ad8a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 18 Jun 2020 22:16:30 +0300 Subject: [PATCH 1397/2107] [Issue #228] use one map per backup for all files --- src/backup.c | 10 ++++- src/catalog.c | 29 +++++++++---- src/data.c | 104 +++++++++++++++++++++++++-------------------- src/dir.c | 6 ++- src/merge.c | 48 +++++++++------------ src/pg_probackup.h | 29 ++++++++++--- src/validate.c | 5 ++- 7 files changed, 143 insertions(+), 88 deletions(-) diff --git a/src/backup.c b/src/backup.c index d87df4b54..c8d6cb08d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -523,6 +523,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool arg->prev_start_lsn = prev_backup_start_lsn; arg->conn_arg.conn = NULL; arg->conn_arg.cancel_conn = NULL; + arg->hdr_map = &(current).hdr_map; arg->thread_num = i+1; /* By default there are some error */ arg->ret = 1; @@ -594,6 +595,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool set_min_recovery_point(pg_control, database_path, current.stop_lsn); } + /* close block header map */ + if (current.hdr_map.fp && fclose(current.hdr_map.fp)) + elog(ERROR, "Cannot close file \"%s\"", current.hdr_map.path); + /* close ssh session in main thread */ fio_disconnect(); @@ -704,6 +709,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } } + if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -2153,7 +2161,7 @@ backup_files(void *arg) arguments->nodeInfo->checksum_version, arguments->nodeInfo->ptrack_version_num, arguments->nodeInfo->ptrack_schema, - true); + arguments->hdr_map, true); } else { diff --git a/src/catalog.c b/src/catalog.c index 450f1100a..3a98b7649 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -474,6 +474,11 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); + /* block header map */ + backup->hdr_map.path = pgut_malloc(MAXPGPATH); + join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + backup->hdr_map.fp = NULL; + /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID @@ -847,6 +852,11 @@ pgBackupCreateDir(pgBackup *backup) backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); + /* block header map */ + backup->hdr_map.path = pgut_malloc(MAXPGPATH); + join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + backup->hdr_map.fp = NULL; + /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { @@ -1909,8 +1919,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, { len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); - -// elog(INFO, "CRC INT: %li, CRC UINT: %u", file->crc_hdr, file->crc_hdr); + len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off); } sprintf(line+len, "}\n"); @@ -2272,6 +2281,9 @@ pgBackupInit(pgBackup *backup) backup->note = NULL; backup->content_crc = 0; + backup->hdr_map.path = NULL; + backup->hdr_map.fp = NULL; + backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; } /* free pgBackup object */ @@ -2280,12 +2292,13 @@ pgBackupFree(void *backup) { pgBackup *b = (pgBackup *) backup; - pfree(b->primary_conninfo); - pfree(b->external_dir_str); - pfree(b->root_dir); - pfree(b->database_dir); - pfree(b->note); - pfree(backup); + pg_free(b->primary_conninfo); + pg_free(b->external_dir_str); + pg_free(b->root_dir); + pg_free(b->database_dir); + pg_free(b->note); + pg_free(b->hdr_map.path); + pg_free(backup); } /* Compare two pgBackup with their IDs (start time) in ascending order */ diff --git a/src/data.c b/src/data.c index 25ec01e81..28230b7ec 100644 --- a/src/data.c +++ b/src/data.c @@ -31,8 +31,8 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; -static BackupPageHeader2* get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version); -static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_fullpath); +static BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version); +static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map); static bool get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, pg_crc32 *crc, bool use_crc32c); @@ -537,7 +537,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok) + int ptrack_version_num, const char *ptrack_schema, + HeaderMap *hdr_map, bool missing_ok) { int rc; bool use_pagemap; @@ -684,7 +685,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, FIN_FILE_CRC32(true, file->crc); /* dump page headers */ - write_page_headers(headers, file, to_fullpath); + write_page_headers(headers, file, hdr_map); pg_free(errmsg); pg_free(file->pagemap.bitmap); @@ -740,7 +741,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool is_merge) + XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers) { size_t total_write_len = 0; char *in_buf = pgut_malloc(STDIO_BUFSIZE); @@ -818,10 +819,11 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* get headers for this file */ - if (!is_merge) - headers = get_data_file_headers(from_fullpath, tmp_file, parse_program_version(backup->program_version)); + if (use_headers) + headers = get_data_file_headers(&(backup->hdr_map), tmp_file, + parse_program_version(backup->program_version)); - if (!is_merge && !headers && tmp_file->n_headers > 0) + if (use_headers && !headers && tmp_file->n_headers > 0) elog(ERROR, "Failed to get headers for file \"%s\"", from_fullpath); /* @@ -1574,7 +1576,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version) + uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map) { size_t read_len = 0; bool is_valid = true; @@ -1595,7 +1597,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); - headers = get_data_file_headers(fullpath, file, backup_version); + headers = get_data_file_headers(hdr_map, file, backup_version); if (!headers && file->n_headers > 0) elog(ERROR, "Failed to get headers for file \"%s\"", fullpath); @@ -2120,12 +2122,11 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f * array of headers. */ BackupPageHeader2* -get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) +get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) { - int len; - FILE *in = NULL; + size_t read_len = 0; + FILE *in = NULL; pg_crc32 hdr_crc; - char fullpath_hdr[MAXPGPATH]; BackupPageHeader2 *headers = NULL; if (backup_version < 20400) @@ -2134,70 +2135,83 @@ get_data_file_headers(const char *fullpath, pgFile *file, uint32 backup_version) if (file->n_headers <= 0) return NULL; - snprintf(fullpath_hdr, MAXPGPATH, "%s_hdr", fullpath); - - in = fopen(fullpath_hdr, PG_BINARY_R); + in = fopen(hdr_map->path, PG_BINARY_R); if (!in) - elog(ERROR, "Cannot open header file \"%s\": %s", fullpath_hdr, strerror(errno)); + elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + + /* disable buffering */ + setvbuf(in, NULL, _IONBF, BUFSIZ); + + if (fseek(in, file->hdr_off, SEEK_SET)) + elog(ERROR, "Cannot seek to position %lu in header map \"%s\": %s", + file->hdr_off, hdr_map->path, strerror(errno)); /* * the actual number of headers in header file is n+1, last one is a dummy header, * used for calculation of compressed_size for actual last header. */ - len = (file->n_headers+1) * sizeof(BackupPageHeader2); - headers = pgut_malloc(len); + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + headers = pgut_malloc(read_len); - if (fread(headers, 1, len, in) != len) - elog(ERROR, "Cannot read header file \"%s\": %s", fullpath_hdr, strerror(errno)); + if (fread(headers, 1, read_len, in) != read_len) + elog(ERROR, "Cannot read header file \"%s\": %s", hdr_map->path, strerror(errno)); /* validate checksum */ INIT_FILE_CRC32(true, hdr_crc); - COMP_FILE_CRC32(true, hdr_crc, headers, len); + COMP_FILE_CRC32(true, hdr_crc, headers, read_len); FIN_FILE_CRC32(true, hdr_crc); if (hdr_crc != file->hdr_crc) elog(ERROR, "Header file crc mismatch \"%s\", current: %u, expected: %u", - fullpath_hdr, hdr_crc, file->hdr_crc); + hdr_map->path, hdr_crc, file->hdr_crc); if (fclose(in)) - elog(ERROR, "Cannot close header file \"%s\": %s", fullpath_hdr, strerror(errno)); + elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno)); return headers; } void -write_page_headers(BackupPageHeader2 *headers, pgFile *file, const char* to_fullpath) +write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map) { - FILE *out = NULL; - size_t hdr_size = 0; - char to_fullpath_hdr[MAXPGPATH]; + size_t read_len = 0; if (file->n_headers <= 0) return; - snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); + /* writing to header map must be serialized */ + pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ - out = fopen(to_fullpath_hdr, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath_hdr, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + if (!hdr_map->fp) + { + hdr_map->fp = fopen(hdr_map->path, PG_BINARY_W); + if (hdr_map->fp == NULL) + elog(ERROR, "Cannot open header file \"%s\": %s", + hdr_map->path, strerror(errno)); + + /* disable buffering for header file */ + setvbuf(hdr_map->fp, NULL, _IONBF, BUFSIZ); + + /* update file permission */ + if (chmod(hdr_map->path, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", hdr_map->path, + strerror(errno)); + + file->hdr_off = 0; + } + else + file->hdr_off = ftell(hdr_map->fp); /* TODO: replace by counter */ - hdr_size = (file->n_headers+1) * sizeof(BackupPageHeader2); + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); /* calculate checksums */ INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, hdr_size); + COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); FIN_FILE_CRC32(true, file->hdr_crc); - if (fwrite(headers, 1, hdr_size, out) != hdr_size) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + if (fwrite(headers, 1, read_len, hdr_map->fp) != read_len) + elog(ERROR, "Cannot write to file \"%s\": %s", hdr_map->path, strerror(errno)); - if (fclose(out)) - elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_hdr, strerror(errno)); + pthread_mutex_unlock(&(hdr_map->mutex)); } diff --git a/src/dir.c b/src/dir.c index db3df2f2b..a8ca12a33 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1566,7 +1566,8 @@ dir_read_file_list(const char *root, const char *external_prefix, n_blocks, n_headers, dbOid, /* used for partial restore */ - hdr_crc; + hdr_crc, + hdr_off; pgFile *file; COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); @@ -1613,6 +1614,9 @@ dir_read_file_list(const char *root, const char *external_prefix, if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) file->hdr_crc = (pg_crc32) hdr_crc; + if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) + file->hdr_off = hdr_off; + parray_append(files, file); } diff --git a/src/merge.c b/src/merge.c index 1599f3b41..9548c308e 100644 --- a/src/merge.c +++ b/src/merge.c @@ -29,6 +29,7 @@ typedef struct bool compression_match; bool program_version_match; bool use_bitmap; + bool is_retry; /* * Return value from the thread. @@ -48,7 +49,8 @@ get_external_index(const char *key, const parray *list); static void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, - pgFile *tmp_file, const char *to_root, bool use_bitmap); + pgFile *tmp_file, const char *to_root, bool use_bitmap, + bool is_retry); static void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, @@ -441,6 +443,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) parray *result_filelist = NULL; bool use_bitmap = true; + bool is_retry = false; // size_t total_in_place_merge_bytes = 0; pthread_t *threads = NULL; @@ -464,8 +467,13 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (!dest_backup) elog(ERROR, "Destination backup is missing, cannot continue merge"); - if (dest_backup->status == BACKUP_STATUS_MERGING) + if (dest_backup->status == BACKUP_STATUS_MERGING || + full_backup->status == BACKUP_STATUS_MERGING || + full_backup->status == BACKUP_STATUS_MERGED) + { + is_retry = true; elog(INFO, "Retry failed merge of backup %s with parent chain", base36enc(dest_backup->start_time)); + } else elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); @@ -661,6 +669,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) arg->compression_match = compression_match; arg->program_version_match = program_version_match; arg->use_bitmap = use_bitmap; + arg->is_retry = is_retry; /* By default there are some error */ arg->ret = 1; @@ -1054,7 +1063,8 @@ merge_files(void *arg) arguments->dest_backup, dest_file, tmp_file, arguments->full_database_dir, - arguments->use_bitmap); + arguments->use_bitmap, + arguments->is_retry); else merge_non_data_file(arguments->parent_chain, arguments->full_backup, @@ -1154,16 +1164,13 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, bool use_bitmap) + const char *full_database_dir, bool use_bitmap, bool is_retry) { FILE *out = NULL; char *buffer = pgut_malloc(STDIO_BUFSIZE); char to_fullpath[MAXPGPATH]; - char to_fullpath_hdr[MAXPGPATH]; char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ - char to_fullpath_tmp2_hdr[MAXPGPATH]; - /* The next possible optimization is copying "as is" the file * from intermediate incremental backup, that didn`t changed in @@ -1174,9 +1181,6 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, join_path_components(to_fullpath, full_database_dir, tmp_file->rel_path); snprintf(to_fullpath_tmp1, MAXPGPATH, "%s_tmp1", to_fullpath); snprintf(to_fullpath_tmp2, MAXPGPATH, "%s_tmp2", to_fullpath); - /* header files */ - snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); - snprintf(to_fullpath_tmp2_hdr, MAXPGPATH, "%s_hdr", to_fullpath_tmp2); /* open temp file */ out = fopen(to_fullpath_tmp1, PG_BINARY_W); @@ -1187,7 +1191,9 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, /* restore file into temp file */ tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, - use_bitmap, NULL, InvalidXLogRecPtr, NULL, true); + use_bitmap, NULL, InvalidXLogRecPtr, NULL, + /* when retrying merge header map cannot be trusted */ + is_retry ? false : true); if (fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath_tmp1, strerror(errno)); @@ -1205,7 +1211,10 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, - dest_backup->checksum_version, 0, NULL, false); + dest_backup->checksum_version, 0, NULL, + + /* TODO: add header map */ + NULL, false); /* drop restored temp file */ if (unlink(to_fullpath_tmp1) == -1) @@ -1232,26 +1241,11 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); - /* sync header file */ - if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync temp header file \"%s\": %s", - to_fullpath_tmp2_hdr, strerror(errno)); - -//<- CRITICAL SECTION - /* Do atomic rename from second temp file to destination file */ if (rename(to_fullpath_tmp2, to_fullpath) == -1) elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", to_fullpath_tmp2, to_fullpath, strerror(errno)); -//<- If we crash here, merge cannot be continued. - - /* Do atomic rename from header file */ - if (rename(to_fullpath_tmp2_hdr, to_fullpath_hdr) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - to_fullpath_tmp2, to_fullpath, strerror(errno)); -//<- - /* drop temp file */ unlink(to_fullpath_tmp1); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 0ff51480a..73266e51a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -37,6 +37,7 @@ #include "utils/file.h" #include "datapagemap.h" +#include "utils/thread.h" /* pgut client variables and full path */ extern const char *PROGRAM_NAME; @@ -65,6 +66,7 @@ extern const char *PROGRAM_EMAIL; #define PG_TABLESPACE_MAP_FILE "tablespace_map" #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" +#define HEADER_MAP "block_header_map" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 @@ -192,7 +194,6 @@ typedef struct pgFile */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ - pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ char *rel_path; /* relative path of the file */ char *linked; /* path of the linked file */ bool is_datafile; /* true if the file is PostgreSQL data file */ @@ -202,7 +203,6 @@ typedef struct pgFile ForkName forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ int n_blocks; /* number of blocks in the data file in data directory */ - int n_headers; /* number of blocks in the data file in backup */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ bool is_database; /* Flag used strictly by ptrack 1.x backup */ int external_dir_num; /* Number of external directory. 0 if not external */ @@ -213,6 +213,10 @@ typedef struct pgFile may take up to 16kB per file */ bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, * i.e. datafiles without _ptrack */ + /* coordinates in header map */ + int n_headers; /* number of blocks in the data file in backup */ + pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ + off_t hdr_off; /* offset in header map */ } pgFile; typedef struct page_map_entry @@ -355,6 +359,16 @@ typedef struct PGNodeInfo } PGNodeInfo; +/* structure used for access to block header map */ +typedef struct HeaderMap +{ + char *path; + FILE *fp; + off_t offset; + pthread_mutex_t mutex; + +} HeaderMap; + typedef struct pgBackup pgBackup; /* Information about single backup stored in backup.conf */ @@ -432,6 +446,9 @@ struct pgBackup char *note; pg_crc32 content_crc; + + /* mutex used for write access to block header map */ + HeaderMap hdr_map; }; /* Recovery target for restore and validate subcommands */ @@ -504,6 +521,7 @@ typedef struct ConnectionArgs conn_arg; int thread_num; + HeaderMap *hdr_map; /* * Return value from the thread. @@ -962,7 +980,8 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, bool missing_ok); + int ptrack_version_num, const char *ptrack_schema, + HeaderMap *hdr_map, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, @@ -974,7 +993,7 @@ extern void backup_non_data_file_internal(const char *from_fullpath, extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool is_merge); + XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, const char *from_fullpath, const char *to_fullpath, int nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, @@ -994,7 +1013,7 @@ extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, extern pid_t check_postmaster(const char *pgdata); extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version); + uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, XLogRecPtr startpoint, TimeLineID start_tli, diff --git a/src/validate.c b/src/validate.c index d7c6ff6a7..cd35047b5 100644 --- a/src/validate.c +++ b/src/validate.c @@ -32,6 +32,7 @@ typedef struct BackupMode backup_mode; parray *dbOid_exclude_list; const char *external_prefix; + HeaderMap *hdr_map; /* * Return value from the thread. @@ -156,6 +157,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) arg->checksum_version = backup->checksum_version; arg->backup_version = parse_program_version(backup->program_version); arg->external_prefix = external_prefix; + arg->hdr_map = &(backup->hdr_map); // arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -365,7 +367,8 @@ pgBackupValidateFiles(void *arg) */ if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn, arguments->checksum_version, - arguments->backup_version)) + arguments->backup_version, + arguments->hdr_map)) arguments->corrupted = true; } } From d9fbced324466ff4af6056f82331e60b9564c709 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 02:32:43 +0300 Subject: [PATCH 1398/2107] [Issue #228] fix merge --- src/backup.c | 2 +- src/catalog.c | 8 +++++++- src/data.c | 41 ++++++++++++++++++++++++++--------------- src/merge.c | 46 ++++++++++++++++++++++++++++++++++++++++------ src/pg_probackup.h | 5 +++++ 5 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/backup.c b/src/backup.c index c8d6cb08d..1b94a2384 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2161,7 +2161,7 @@ backup_files(void *arg) arguments->nodeInfo->checksum_version, arguments->nodeInfo->ptrack_version_num, arguments->nodeInfo->ptrack_schema, - arguments->hdr_map, true); + arguments->hdr_map, false); } else { diff --git a/src/catalog.c b/src/catalog.c index 3a98b7649..185ba28b1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -474,9 +474,11 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); - /* block header map */ + /* block header map, TODO: move to separate function */ backup->hdr_map.path = pgut_malloc(MAXPGPATH); join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); + join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP); backup->hdr_map.fp = NULL; /* TODO: save encoded backup id */ @@ -855,6 +857,8 @@ pgBackupCreateDir(pgBackup *backup) /* block header map */ backup->hdr_map.path = pgut_malloc(MAXPGPATH); join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); + join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP); backup->hdr_map.fp = NULL; /* create directories for actual backup files */ @@ -2282,6 +2286,7 @@ pgBackupInit(pgBackup *backup) backup->content_crc = 0; backup->hdr_map.path = NULL; + backup->hdr_map.path_tmp = NULL; backup->hdr_map.fp = NULL; backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; } @@ -2298,6 +2303,7 @@ pgBackupFree(void *backup) pg_free(b->database_dir); pg_free(b->note); pg_free(b->hdr_map.path); + pg_free(b->hdr_map.path_tmp); pg_free(backup); } diff --git a/src/data.c b/src/data.c index 28230b7ec..132255afd 100644 --- a/src/data.c +++ b/src/data.c @@ -31,8 +31,6 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; -static BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version); -static void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map); static bool get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, pg_crc32 *crc, bool use_crc32c); @@ -538,7 +536,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, - HeaderMap *hdr_map, bool missing_ok) + HeaderMap *hdr_map, bool is_merge) { int rc; bool use_pagemap; @@ -629,7 +627,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* check for errors */ if (rc == FILE_MISSING) { - elog(LOG, "File \"%s\" is not found", from_fullpath); + elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } @@ -685,7 +683,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, FIN_FILE_CRC32(true, file->crc); /* dump page headers */ - write_page_headers(headers, file, hdr_map); + write_page_headers(headers, file, hdr_map, is_merge); pg_free(errmsg); pg_free(file->pagemap.bitmap); @@ -818,8 +816,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* set stdio buffering for input data file */ setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); +// elog(INFO, "N_HEADERS: %i", tmp_file->n_headers); +// elog(INFO, "File: %s", tmp_file->rel_path); +// elog(INFO, "Backup: %s", base36enc(backup->start_time)); + /* get headers for this file */ - if (use_headers) + if (use_headers && tmp_file->n_headers > 0) headers = get_data_file_headers(&(backup->hdr_map), tmp_file, parse_program_version(backup->program_version)); @@ -2163,8 +2165,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) FIN_FILE_CRC32(true, hdr_crc); if (hdr_crc != file->hdr_crc) - elog(ERROR, "Header file crc mismatch \"%s\", current: %u, expected: %u", - hdr_map->path, hdr_crc, file->hdr_crc); + elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u", + file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); if (fclose(in)) elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno)); @@ -2173,29 +2175,35 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) } void -write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map) +write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge) { - size_t read_len = 0; + size_t read_len = 0; + char *map_path = NULL; if (file->n_headers <= 0) return; + /* when running merge we must save headers into the temp map */ + map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; + /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ if (!hdr_map->fp) { - hdr_map->fp = fopen(hdr_map->path, PG_BINARY_W); + elog(LOG, "Creating page header map \"%s\"", map_path); + + hdr_map->fp = fopen(map_path, PG_BINARY_W); if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", - hdr_map->path, strerror(errno)); + map_path, strerror(errno)); /* disable buffering for header file */ setvbuf(hdr_map->fp, NULL, _IONBF, BUFSIZ); /* update file permission */ - if (chmod(hdr_map->path, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", hdr_map->path, + if (chmod(map_path, FILE_PERMISSION) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", map_path, strerror(errno)); file->hdr_off = 0; @@ -2211,7 +2219,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map) FIN_FILE_CRC32(true, file->hdr_crc); if (fwrite(headers, 1, read_len, hdr_map->fp) != read_len) - elog(ERROR, "Cannot write to file \"%s\": %s", hdr_map->path, strerror(errno)); + elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); + + elog(VERBOSE, "Writing header map for file \"%s\" offset: %lu, len: %lu, crc: %u", + file->rel_path, file->hdr_off, read_len, file->hdr_crc); pthread_mutex_unlock(&(hdr_map->mutex)); } diff --git a/src/merge.c b/src/merge.c index 9548c308e..a51756df0 100644 --- a/src/merge.c +++ b/src/merge.c @@ -550,11 +550,11 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) elog(INFO, "Validate parent chain for backup %s", base36enc(dest_backup->start_time)); - /* forbid merge retry for failed merges between 2.4.0 and any + /* Forbid merge retry for failed merges between 2.4.0 and any * older version. Several format changes makes it impossible * to determine the exact format any speific file is got. */ - if (full_backup->status == BACKUP_STATUS_MERGING && + if (is_retry && parse_program_version(dest_backup->program_version) >= 20400 && parse_program_version(full_backup->program_version) < 20400) { @@ -705,6 +705,23 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) elog(ERROR, "Backup files merging failed, time elapsed: %s", pretty_time); + /* If temp header map descriptor is open, then close it and make rename */ + if (full_backup->hdr_map.fp) + { + if (fclose(full_backup->hdr_map.fp)) + elog(ERROR, "Cannot close file \"%s\"", full_backup->hdr_map.path); + + /* sync new header map to dist */ + if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot sync temp header map \"%s\": %s", + full_backup->hdr_map.path_tmp, strerror(errno)); + + /* Replace old header map with new one */ + if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path) == -1) + elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", + full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno)); + } + /* * Update FULL backup metadata. * We cannot set backup status to OK just yet, @@ -847,6 +864,12 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->root_dir = pgut_strdup(destination_path); } + /* Reinit some path variables */ + join_path_components(full_backup->database_dir, full_backup->root_dir, DATABASE_DIR); + join_path_components(full_backup->hdr_map.path, full_backup->database_dir, HEADER_MAP); + join_path_components(full_backup->hdr_map.path_tmp, full_backup->database_dir, HEADER_MAP_TMP); + full_backup->hdr_map.fp = NULL; + /* If we crash here, it will produce full backup in MERGED * status, located in directory with wrong backup id. * It should not be a problem. @@ -1033,6 +1056,7 @@ merge_files(void *arg) if (file && file->n_blocks == dest_file->n_blocks) { + BackupPageHeader2 *headers = NULL; elog(VERBOSE, "The file didn`t changed since FULL backup, skip merge: \"%s\"", file->rel_path); @@ -1052,6 +1076,18 @@ merge_files(void *arg) else tmp_file->uncompressed_size = tmp_file->write_size; + /* Copy header metadata from old map into a new one */ + tmp_file->n_headers = file->n_headers; + headers = get_data_file_headers(&(arguments->full_backup->hdr_map), file, + parse_program_version(arguments->full_backup->program_version)); + + /* sanity */ + if (!headers && file->n_headers > 0) + elog(ERROR, "Failed to get headers for file \"%s\"", file->rel_path); + + write_page_headers(headers, tmp_file, &(arguments->full_backup->hdr_map), true); + pg_free(headers); + //TODO: report in_place merge bytes. goto done; } @@ -1205,16 +1241,14 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 16KB. * TODO: maybe we should just trust dest_file->n_blocks? * No, we can`t, because current binary can be used to merge - * 2 backups of old versions, were n_blocks is missing. + * 2 backups of old versions, where n_blocks is missing. */ backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, - - /* TODO: add header map */ - NULL, false); + &(full_backup->hdr_map), true); /* drop restored temp file */ if (unlink(to_fullpath_tmp1) == -1) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 73266e51a..a3a06d68f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -67,6 +67,7 @@ extern const char *PROGRAM_EMAIL; #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" #define HEADER_MAP "block_header_map" +#define HEADER_MAP_TMP "block_header_map_tmp" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 @@ -363,6 +364,7 @@ typedef struct PGNodeInfo typedef struct HeaderMap { char *path; + char *path_tmp; /* used only in merge */ FILE *fp; off_t offset; pthread_mutex_t mutex; @@ -1014,6 +1016,9 @@ extern pid_t check_postmaster(const char *pgdata); extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); + +extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version); +extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, XLogRecPtr startpoint, TimeLineID start_tli, From d383697b8842544ded304df00eba33429a6d7d3c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 02:36:38 +0300 Subject: [PATCH 1399/2107] tests: minor fixes in merge module --- tests/merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index c56489aec..aa9a3bd1b 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -345,7 +345,7 @@ def test_merge_compressed_and_uncompressed_backups_1(self): self.assertEqual(show_backup["backup-mode"], "FULL") # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) pgbench.wait() # Do uncompressed DELTA backup @@ -354,7 +354,7 @@ def test_merge_compressed_and_uncompressed_backups_1(self): options=['--stream']) # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed PAGE backup From a0ba9167f2f169eed37922c281e083b130b8ca8e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 10:09:04 +0300 Subject: [PATCH 1400/2107] tests: some fixes --- tests/backup.py | 4 ++-- tests/compatibility.py | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 8f97577b4..964661c0c 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1387,7 +1387,7 @@ def test_drop_rel_during_backup_delta(self): with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() self.assertTrue( - 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() @@ -1523,7 +1523,7 @@ def test_drop_rel_during_backup_ptrack(self): with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: log_content = f.read() self.assertTrue( - 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() diff --git a/tests/compatibility.py b/tests/compatibility.py index ff1359820..49ea50d30 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -936,7 +936,7 @@ def test_backward_compatibility_merge_3(self): def test_backward_compatibility_merge_4(self): """ Start merge between minor version, crash and retry it. - old binary version =< 2.2.7 + old binary version =< 2.4.0 """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -982,17 +982,25 @@ def test_backward_compatibility_merge_4(self): gdb.set_breakpoint('rename') gdb.run_until_break() - gdb.continue_execution_until_break(1000) + gdb.continue_execution_until_break(500) gdb._execute('signal SIGKILL') - self.merge_backup(backup_dir, "node", page_id) - - # check data correctness for PAGE - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page_id) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + try: + self.merge_backup(backup_dir, "node", page_id) + self.assertEqual( + 1, 0, + "Expecting Error because of format changes.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Retry of failed merge for backups with different " + "between minor versions is forbidden to avoid data corruption " + "because of storage format changes introduced in 2.4.0 version, " + "please take a new full backup", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) # Clean after yourself self.del_test_dir(module_name, fname) From 6f89a53ce011f0f7e9e7ac38ddfea110972616ca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 10:39:29 +0300 Subject: [PATCH 1401/2107] tests: for basic smoke tests avoid crashish of PostgreSQL cluster when cleaning up test directories --- tests/archive.py | 2 +- tests/backup.py | 8 ++++---- tests/checkdb.py | 2 +- tests/compression.py | 3 +-- tests/delta.py | 2 +- tests/helpers/ptrack_helpers.py | 5 ++++- tests/merge.py | 3 +-- tests/page.py | 2 +- tests/replica.py | 2 +- tests/retention.py | 6 +++--- tests/validate.py | 2 +- 11 files changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 5508f54b8..444b3dd09 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -988,7 +988,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.backup_node(backup_dir, 'master', replica) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[master, replica]) # @unittest.expectedFailure # @unittest.skip("skip") diff --git a/tests/backup.py b/tests/backup.py index 694c9a448..f4712ad73 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1068,7 +1068,7 @@ def test_basic_tablespace_handling(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[node]) # @unittest.skip("skip") def test_tablespace_handling_1(self): @@ -1646,7 +1646,7 @@ def test_backup_concurrent_drop_table(self): self.assertEqual(show_backup['status'], "OK") # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, nodes=[node]) # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): @@ -1930,7 +1930,7 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 700) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): @@ -1973,7 +1973,7 @@ def test_basic_missing_dir_permissions(self): os.chmod(full_path, 700) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): diff --git a/tests/checkdb.py b/tests/checkdb.py index 033a6c253..6c25293ab 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -348,7 +348,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): diff --git a/tests/compression.py b/tests/compression.py index b8788a46c..321461d6e 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -115,10 +115,9 @@ def test_basic_compression_stream_zlib(self): delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) - node.cleanup() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_compression_archive_zlib(self): """ diff --git a/tests/delta.py b/tests/delta.py index fdbaf1271..6b2ebf922 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -87,7 +87,7 @@ def test_basic_delta_vacuum_truncate(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index eec57a3ba..bba6cf2a0 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1448,13 +1448,16 @@ def get_version(self, node): def get_bin_path(self, binary): return testgres.get_bin_path(binary) - def del_test_dir(self, module_name, fname): + def del_test_dir(self, module_name, fname, nodes=[]): """ Del testdir and optimistically try to del module dir""" try: testgres.clean_all() except: pass + for node in nodes: + node.stop() + shutil.rmtree( os.path.join( self.tmp_path, diff --git a/tests/merge.py b/tests/merge.py index 54b7e2884..93d41c5d4 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -100,8 +100,7 @@ def test_basic_merge_full_page(self): self.assertEqual(count1, count2) # Clean after yourself - node.cleanup() - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_merge_compressed_backups(self): """ diff --git a/tests/page.py b/tests/page.py index a73328e1c..201f825e8 100644 --- a/tests/page.py +++ b/tests/page.py @@ -100,7 +100,7 @@ def test_basic_page_vacuum_truncate(self): self.assertEqual(result1, result2) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node, node_restored]) # @unittest.skip("skip") def test_page_vacuum_truncate_1(self): diff --git a/tests/replica.py b/tests/replica.py index ab6eaf592..aa3de9aef 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -344,7 +344,7 @@ def test_basic_make_replica_via_restore(self): options=['--archive-timeout=30s', '--stream']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [master, replica]) # @unittest.skip("skip") def test_take_backup_from_delayed_replica(self): diff --git a/tests/retention.py b/tests/retention.py index e797d3c60..876bcbca4 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1009,7 +1009,7 @@ def test_basic_window_merge_multiple_descendants(self): 'FULL') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): @@ -1276,7 +1276,7 @@ def test_basic_window_merge_multiple_descendants_1(self): '--delete-expired', '--log-level-console=log']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_window_chains(self): @@ -2534,4 +2534,4 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) diff --git a/tests/validate.py b/tests/validate.py index 8e27c50cc..c2d1405bc 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -298,7 +298,7 @@ def test_basic_validate_corrupted_intermediate_backup(self): 'Backup STATUS should be "ORPHAN"') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): From 615a54edb56d88f119cb81c764c91f8e636102a8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 10:49:32 +0300 Subject: [PATCH 1402/2107] tests: minor fixes --- tests/backup.py | 2 +- tests/delta.py | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index f4712ad73..fadc2aa53 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1601,7 +1601,7 @@ def test_basic_temp_slot_for_stream_backup(self): options=['--stream', '--slot=slot_1', '--temp-slot']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): diff --git a/tests/delta.py b/tests/delta.py index 6b2ebf922..0abdd1c2c 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -45,39 +45,32 @@ def test_basic_delta_vacuum_truncate(self): "create table t_heap as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) + "from generate_series(0,1024) i;") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) + "delete from t_heap where ctid >= '(11,0)'") node.safe_psql( "postgres", - "vacuum t_heap" - ) + "vacuum t_heap") self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) + backup_dir, 'node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) self.restore_node( - backup_dir, 'node', node_restored - ) + backup_dir, 'node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -87,7 +80,7 @@ def test_basic_delta_vacuum_truncate(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname, [node, node_restored]) # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): From a7eb948a75ae8450f4595787c9fa0cca6c0e5f87 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 19 Jun 2020 14:02:00 +0300 Subject: [PATCH 1403/2107] [Issue #228]: remove artefacts of previous implementation --- src/backup.c | 27 ++++++++------------------- src/dir.c | 15 --------------- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/src/backup.c b/src/backup.c index 1b94a2384..eb507143a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -596,8 +596,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } /* close block header map */ - if (current.hdr_map.fp && fclose(current.hdr_map.fp)) - elog(ERROR, "Cannot close file \"%s\"", current.hdr_map.path); + if (current.hdr_map.fp) + { + if (fclose(current.hdr_map.fp)) + elog(ERROR, "Cannot close file \"%s\"", current.hdr_map.path); + + if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); + } /* close ssh session in main thread */ fio_disconnect(); @@ -693,25 +699,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno)); - - /* fsync header file */ - if (file->external_dir_num == 0 && - file->is_datafile && !file->is_cfs && - file->n_headers > 0) - { - char to_fullpath_hdr[MAXPGPATH]; - - snprintf(to_fullpath_hdr, MAXPGPATH, "%s_hdr", to_fullpath); - - if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath_hdr, strerror(errno)); - - } } - if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); - time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); diff --git a/src/dir.c b/src/dir.c index a8ca12a33..cf7cc7afa 100644 --- a/src/dir.c +++ b/src/dir.c @@ -250,21 +250,6 @@ pgFileDelete(pgFile *file, const char *full_path) elog(ERROR, "Cannot remove file \"%s\": %s", full_path, strerror(errno)); } - - if (file->n_headers > 0) - { - char full_path_hdr[MAXPGPATH]; - - snprintf(full_path_hdr, MAXPGPATH, "%s_hdr", full_path); - - if (remove(full_path_hdr) == -1) - { - if (errno == ENOENT) - return; - elog(ERROR, "Cannot remove file \"%s\": %s", full_path_hdr, - strerror(errno)); - } - } } /* From 0322c97a5ec57be31ddaea05b1bf1a3a1c0f1bec Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Jun 2020 19:08:14 +0300 Subject: [PATCH 1404/2107] [Issue #228]: bufferization of writes to page header map --- src/catalog.c | 14 ++++++++------ src/data.c | 5 +++-- src/merge.c | 8 ++++++-- src/pg_probackup.h | 3 ++- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 185ba28b1..d5be578c1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -475,11 +475,12 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); /* block header map, TODO: move to separate function */ + backup->hdr_map.fp = NULL; + backup->hdr_map.buf = NULL; backup->hdr_map.path = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP); - backup->hdr_map.fp = NULL; + join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; @@ -855,11 +856,12 @@ pgBackupCreateDir(pgBackup *backup) join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); /* block header map */ + backup->hdr_map.fp = NULL; + backup->hdr_map.buf = NULL; backup->hdr_map.path = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path, backup->database_dir, HEADER_MAP); + join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path_tmp, backup->database_dir, HEADER_MAP_TMP); - backup->hdr_map.fp = NULL; + join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) diff --git a/src/data.c b/src/data.c index 132255afd..80f0ff02f 100644 --- a/src/data.c +++ b/src/data.c @@ -2198,8 +2198,9 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); - /* disable buffering for header file */ - setvbuf(hdr_map->fp, NULL, _IONBF, BUFSIZ); + /* enable buffering for header file */ + hdr_map->buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, STDIO_BUFSIZE); /* update file permission */ if (chmod(map_path, FILE_PERMISSION) == -1) diff --git a/src/merge.c b/src/merge.c index a51756df0..93ee83009 100644 --- a/src/merge.c +++ b/src/merge.c @@ -720,6 +720,10 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path) == -1) elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno)); + + full_backup->hdr_map.fp = NULL; + pg_free(full_backup->hdr_map.buf); + full_backup->hdr_map.buf = NULL; } /* @@ -866,8 +870,8 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* Reinit some path variables */ join_path_components(full_backup->database_dir, full_backup->root_dir, DATABASE_DIR); - join_path_components(full_backup->hdr_map.path, full_backup->database_dir, HEADER_MAP); - join_path_components(full_backup->hdr_map.path_tmp, full_backup->database_dir, HEADER_MAP_TMP); + join_path_components(full_backup->hdr_map.path, full_backup->root_dir, HEADER_MAP); + join_path_components(full_backup->hdr_map.path_tmp, full_backup->root_dir, HEADER_MAP_TMP); full_backup->hdr_map.fp = NULL; /* If we crash here, it will produce full backup in MERGED diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e5d981889..5941eac6e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -365,6 +365,7 @@ typedef struct HeaderMap { char *path; char *path_tmp; /* used only in merge */ + char *buf; /* buffer */ FILE *fp; off_t offset; pthread_mutex_t mutex; @@ -449,7 +450,7 @@ struct pgBackup pg_crc32 content_crc; - /* mutex used for write access to block header map */ + /* map used for access to page headers */ HeaderMap hdr_map; }; From a0d31d530b60b47371f136fb9a31adcd264e7852 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Jun 2020 19:13:31 +0300 Subject: [PATCH 1405/2107] [Issue #228]: bufferization quick fix --- src/backup.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backup.c b/src/backup.c index eb507143a..cfc977527 100644 --- a/src/backup.c +++ b/src/backup.c @@ -603,6 +603,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); + + current.hdr_map.fp = NULL; + pg_free(current.hdr_map.buf); + current.hdr_map.buf = NULL; } /* close ssh session in main thread */ From 09e87a54f7da2d070b4991c3c292ee9dd07e235d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Jun 2020 19:14:38 +0300 Subject: [PATCH 1406/2107] [Issue #228]: added page header map compression --- src/catalog.c | 1 + src/data.c | 65 +++++++++++++++++++++++++++++++++++++++++----- src/dir.c | 6 ++++- src/pg_probackup.h | 3 ++- 4 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index d5be578c1..afcc8976f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1926,6 +1926,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off); + len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size); } sprintf(line+len, "}\n"); diff --git a/src/data.c b/src/data.c index 80f0ff02f..b10aa22f9 100644 --- a/src/data.c +++ b/src/data.c @@ -2130,6 +2130,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) FILE *in = NULL; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL; + /* header decompression */ + char *zheaders = NULL; + const char *errormsg = NULL; if (backup_version < 20400) return NULL; @@ -2150,14 +2153,33 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) file->hdr_off, hdr_map->path, strerror(errno)); /* - * the actual number of headers in header file is n+1, last one is a dummy header, - * used for calculation of compressed_size for actual last header. + * The actual number of headers in header file is n+1, last one is a dummy header, + * used for calculation of read_len for actual last header. */ read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + + /* allocate memory for compressed and uncompressed headers */ headers = pgut_malloc(read_len); + memset(headers, 0, read_len); + zheaders = pgut_malloc(file->hdr_size); + memset(zheaders, 0, file->hdr_size); + + if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) + elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s", + file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); - if (fread(headers, 1, read_len, in) != read_len) - elog(ERROR, "Cannot read header file \"%s\": %s", hdr_map->path, strerror(errno)); +// elog(INFO, "zsize: %i, size: %i", file->hdr_size, read_len); + + if (do_decompress(headers, read_len, zheaders, file->hdr_size, + ZLIB_COMPRESS, &errormsg) != read_len) + { + if (errormsg) + elog(ERROR, "An error occured during metadata decompression for file \"%s\": %s", + file->rel_path, errormsg); + else + elog(ERROR, "An error occured during metadata decompression for file \"%s\"", + file->rel_path); + } /* validate checksum */ INIT_FILE_CRC32(true, hdr_crc); @@ -2171,6 +2193,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) if (fclose(in)) elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno)); + pg_free(zheaders); + return headers; } @@ -2179,6 +2203,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, { size_t read_len = 0; char *map_path = NULL; + /* header compression */ + int z_len = 0; + char *zheaders = NULL; + const char *errormsg = NULL; if (file->n_headers <= 0) return; @@ -2219,11 +2247,34 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); FIN_FILE_CRC32(true, file->hdr_crc); - if (fwrite(headers, 1, read_len, hdr_map->fp) != read_len) + zheaders = pgut_malloc(read_len*2); + memset(zheaders, 0, read_len*2); + + z_len = do_compress(zheaders, read_len*2, headers, + read_len, ZLIB_COMPRESS, 1, &errormsg); + + if (z_len < 0) + { + if (errormsg) + elog(ERROR, "An error occured during compressing metadata for file \"%s\": %s", + file->rel_path, errormsg); + else + elog(ERROR, "An error occured during compressing metadata for file \"%s\": %i", + file->rel_path, z_len); + } + + if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); - elog(VERBOSE, "Writing header map for file \"%s\" offset: %lu, len: %lu, crc: %u", - file->rel_path, file->hdr_off, read_len, file->hdr_crc); + elog(VERBOSE, "Writing header map for file \"%s\" offset: %li, len: %i, crc: %u", + file->rel_path, file->hdr_off, z_len, file->hdr_crc); + +// elog(INFO, "File: %s, Unzip: %i, zip: %i", file->rel_path, read_len, z_len); + file->hdr_size = z_len; + + /* End critical section */ pthread_mutex_unlock(&(hdr_map->mutex)); + + pg_free(zheaders); } diff --git a/src/dir.c b/src/dir.c index cf7cc7afa..219126196 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1552,7 +1552,8 @@ dir_read_file_list(const char *root, const char *external_prefix, n_headers, dbOid, /* used for partial restore */ hdr_crc, - hdr_off; + hdr_off, + hdr_size; pgFile *file; COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); @@ -1602,6 +1603,9 @@ dir_read_file_list(const char *root, const char *external_prefix, if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) file->hdr_off = hdr_off; + if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false)) + file->hdr_size = (int) hdr_size; + parray_append(files, file); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5941eac6e..262912a86 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -214,10 +214,11 @@ typedef struct pgFile may take up to 16kB per file */ bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, * i.e. datafiles without _ptrack */ - /* coordinates in header map */ + /* Coordinates in header map */ int n_headers; /* number of blocks in the data file in backup */ pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ off_t hdr_off; /* offset in header map */ + int hdr_size; /* offset in header map */ } pgFile; typedef struct page_map_entry From e49a4909434541f5133419684b037164baad26a3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 01:55:00 +0300 Subject: [PATCH 1407/2107] [Issue #228]: improve page header map bufferization and share descriptors between threads --- src/backup.c | 14 +++--- src/catalog.c | 23 ++-------- src/data.c | 103 ++++++++++++++++++++++++++++++++++----------- src/merge.c | 21 +++++---- src/pg_probackup.h | 16 ++++--- src/restore.c | 7 +++ src/validate.c | 1 + 7 files changed, 116 insertions(+), 69 deletions(-) diff --git a/src/backup.c b/src/backup.c index cfc977527..17b7044ea 100644 --- a/src/backup.c +++ b/src/backup.c @@ -505,6 +505,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool instance_config.pgdata, external_dirs, true); write_backup(¤t, true); + /* Init backup page header map */ + init_header_map(¤t); + /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); @@ -595,18 +598,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool set_min_recovery_point(pg_control, database_path, current.stop_lsn); } - /* close block header map */ - if (current.hdr_map.fp) + /* close and sync page header map */ + if (current.hdr_map.w_fp) { - if (fclose(current.hdr_map.fp)) - elog(ERROR, "Cannot close file \"%s\"", current.hdr_map.path); + cleanup_header_map(&(current.hdr_map)); if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); - - current.hdr_map.fp = NULL; - pg_free(current.hdr_map.buf); - current.hdr_map.buf = NULL; } /* close ssh session in main thread */ diff --git a/src/catalog.c b/src/catalog.c index afcc8976f..388f31e43 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -474,13 +474,8 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); - /* block header map, TODO: move to separate function */ - backup->hdr_map.fp = NULL; - backup->hdr_map.buf = NULL; - backup->hdr_map.path = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); - backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); + /* Initialize page header map */ + init_header_map(backup); /* TODO: save encoded backup id */ backup->backup_id = backup->start_time; @@ -856,12 +851,7 @@ pgBackupCreateDir(pgBackup *backup) join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); /* block header map */ - backup->hdr_map.fp = NULL; - backup->hdr_map.buf = NULL; - backup->hdr_map.path = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); - backup->hdr_map.path_tmp = pgut_malloc(MAXPGPATH); - join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); + init_header_map(backup); /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) @@ -2287,11 +2277,6 @@ pgBackupInit(pgBackup *backup) backup->files = NULL; backup->note = NULL; backup->content_crc = 0; - - backup->hdr_map.path = NULL; - backup->hdr_map.path_tmp = NULL; - backup->hdr_map.fp = NULL; - backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; } /* free pgBackup object */ @@ -2305,8 +2290,6 @@ pgBackupFree(void *backup) pg_free(b->root_dir); pg_free(b->database_dir); pg_free(b->note); - pg_free(b->hdr_map.path); - pg_free(b->hdr_map.path_tmp); pg_free(backup); } diff --git a/src/data.c b/src/data.c index b10aa22f9..2d4ba57e5 100644 --- a/src/data.c +++ b/src/data.c @@ -2127,10 +2127,10 @@ BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) { size_t read_len = 0; - FILE *in = NULL; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL; /* header decompression */ + int z_len = 0; char *zheaders = NULL; const char *errormsg = NULL; @@ -2140,16 +2140,36 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) if (file->n_headers <= 0) return NULL; - in = fopen(hdr_map->path, PG_BINARY_R); +// in = fopen(hdr_map->path, PG_BINARY_R); +// +// if (!in) +// elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); - if (!in) - elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + if (!hdr_map->r_fp) + { + pthread_lock(&(hdr_map->mutex)); - /* disable buffering */ - setvbuf(in, NULL, _IONBF, BUFSIZ); + /* it is possible for another contender got here first, so double check */ + if (!hdr_map->r_fp) /* this file will be closed in restore.c and merge.c */ + { + elog(LOG, "Opening page header map \"%s\"", hdr_map->path); + + hdr_map->r_fp = fopen(hdr_map->path, PG_BINARY_R); + if (hdr_map->r_fp == NULL) + elog(ERROR, "Cannot open header file \"%s\": %s", + hdr_map->path, strerror(errno)); - if (fseek(in, file->hdr_off, SEEK_SET)) - elog(ERROR, "Cannot seek to position %lu in header map \"%s\": %s", + /* enable buffering for header file */ + hdr_map->r_buf = pgut_malloc(LARGE_CHUNK_SIZE); + setvbuf(hdr_map->r_fp, hdr_map->r_buf, _IOFBF, LARGE_CHUNK_SIZE); + } + + /* End critical section */ + pthread_mutex_unlock(&(hdr_map->mutex)); + } + + if (fseek(hdr_map->r_fp, file->hdr_off, SEEK_SET)) + elog(ERROR, "Cannot seek to position %lu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); /* @@ -2164,21 +2184,22 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) zheaders = pgut_malloc(file->hdr_size); memset(zheaders, 0, file->hdr_size); - if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) + if (fread(zheaders, 1, file->hdr_size, hdr_map->r_fp) != file->hdr_size) elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); // elog(INFO, "zsize: %i, size: %i", file->hdr_size, read_len); - if (do_decompress(headers, read_len, zheaders, file->hdr_size, - ZLIB_COMPRESS, &errormsg) != read_len) + z_len = do_decompress(headers, read_len, zheaders, file->hdr_size, + ZLIB_COMPRESS, &errormsg); + if (z_len <= 0) { if (errormsg) elog(ERROR, "An error occured during metadata decompression for file \"%s\": %s", file->rel_path, errormsg); else - elog(ERROR, "An error occured during metadata decompression for file \"%s\"", - file->rel_path); + elog(ERROR, "An error occured during metadata decompression for file \"%s\": %i", + file->rel_path, z_len); } /* validate checksum */ @@ -2190,9 +2211,6 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); - if (fclose(in)) - elog(ERROR, "Cannot close header file \"%s\": %s", hdr_map->path, strerror(errno)); - pg_free(zheaders); return headers; @@ -2217,18 +2235,18 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ - if (!hdr_map->fp) + if (!hdr_map->w_fp) { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->fp = fopen(map_path, PG_BINARY_W); - if (hdr_map->fp == NULL) + hdr_map->w_fp = fopen(map_path, PG_BINARY_W); + if (hdr_map->w_fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); /* enable buffering for header file */ - hdr_map->buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, STDIO_BUFSIZE); + hdr_map->w_buf = pgut_malloc(LARGE_CHUNK_SIZE); + setvbuf(hdr_map->w_fp, hdr_map->w_buf, _IOFBF, LARGE_CHUNK_SIZE); /* update file permission */ if (chmod(map_path, FILE_PERMISSION) == -1) @@ -2238,7 +2256,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->hdr_off = 0; } else - file->hdr_off = ftell(hdr_map->fp); /* TODO: replace by counter */ + file->hdr_off = ftell(hdr_map->w_fp); /* TODO: replace by counter */ read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); @@ -2253,7 +2271,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, z_len = do_compress(zheaders, read_len*2, headers, read_len, ZLIB_COMPRESS, 1, &errormsg); - if (z_len < 0) + if (z_len <= 0) { if (errormsg) elog(ERROR, "An error occured during compressing metadata for file \"%s\": %s", @@ -2263,13 +2281,13 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->rel_path, z_len); } - if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) + if (fwrite(zheaders, 1, z_len, hdr_map->w_fp) != z_len) elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); elog(VERBOSE, "Writing header map for file \"%s\" offset: %li, len: %i, crc: %u", file->rel_path, file->hdr_off, z_len, file->hdr_crc); -// elog(INFO, "File: %s, Unzip: %i, zip: %i", file->rel_path, read_len, z_len); + elog(INFO, "File: %s, Unzip: %li, zip: %i", file->rel_path, read_len, z_len); file->hdr_size = z_len; @@ -2278,3 +2296,38 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, pg_free(zheaders); } + +void +init_header_map(pgBackup *backup) +{ + backup->hdr_map.r_fp = NULL; + backup->hdr_map.w_fp = NULL; + backup->hdr_map.r_buf = NULL; + backup->hdr_map.w_buf = NULL; + join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); + join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); + backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; +} + +void +cleanup_header_map(HeaderMap *hdr_map) +{ + + /* cleanup read descriptor */ + if (hdr_map->r_fp && fclose(hdr_map->r_fp)) + elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + + hdr_map->r_fp = NULL; + pg_free(hdr_map->r_buf); + hdr_map->r_buf = NULL; + hdr_map->r_offset = 0; + + /* cleanup write descriptor */ + if (hdr_map->w_fp && fclose(hdr_map->w_fp)) + elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + + hdr_map->w_fp = NULL; + pg_free(hdr_map->w_buf); + hdr_map->w_buf = NULL; + hdr_map->w_offset = 0; +} diff --git a/src/merge.c b/src/merge.c index 93ee83009..5267df49c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -706,10 +706,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) pretty_time); /* If temp header map descriptor is open, then close it and make rename */ - if (full_backup->hdr_map.fp) + if (full_backup->hdr_map.w_fp) { - if (fclose(full_backup->hdr_map.fp)) - elog(ERROR, "Cannot close file \"%s\"", full_backup->hdr_map.path); + cleanup_header_map(&(full_backup->hdr_map)); /* sync new header map to dist */ if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) @@ -717,13 +716,16 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->hdr_map.path_tmp, strerror(errno)); /* Replace old header map with new one */ - if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path) == -1) + if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path)) elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno)); + } - full_backup->hdr_map.fp = NULL; - pg_free(full_backup->hdr_map.buf); - full_backup->hdr_map.buf = NULL; + /* Close page header maps */ + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + cleanup_header_map(&(backup->hdr_map)); } /* @@ -868,11 +870,8 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->root_dir = pgut_strdup(destination_path); } - /* Reinit some path variables */ + /* Reinit path to database_dir */ join_path_components(full_backup->database_dir, full_backup->root_dir, DATABASE_DIR); - join_path_components(full_backup->hdr_map.path, full_backup->root_dir, HEADER_MAP); - join_path_components(full_backup->hdr_map.path_tmp, full_backup->root_dir, HEADER_MAP_TMP); - full_backup->hdr_map.fp = NULL; /* If we crash here, it will produce full backup in MERGED * status, located in directory with wrong backup id. diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 262912a86..5dd86acf0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -92,6 +92,7 @@ extern const char *PROGRAM_EMAIL; #define ERRMSG_MAX_LEN 2048 #define CHUNK_SIZE (128 * 1024) +#define LARGE_CHUNK_SIZE (4 * 1024 * 1024) #define OUT_BUF_SIZE (512 * 1024) /* retry attempts */ @@ -364,11 +365,14 @@ typedef struct PGNodeInfo /* structure used for access to block header map */ typedef struct HeaderMap { - char *path; - char *path_tmp; /* used only in merge */ - char *buf; /* buffer */ - FILE *fp; - off_t offset; + char path[MAXPGPATH]; + char path_tmp[MAXPGPATH]; /* used only in merge */ + char *r_buf; /* buffer */ + char *w_buf; /* buffer */ + FILE *r_fp; /* descriptor used for reading */ + FILE *w_fp; /* descriptor used for writing */ + off_t r_offset; /* current position in r_fp */ + off_t w_offset; /* current position in w_fp */ pthread_mutex_t mutex; } HeaderMap; @@ -1021,6 +1025,8 @@ extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr s extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version); extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); +extern void init_header_map(pgBackup *backup); +extern void cleanup_header_map(HeaderMap *hdr_map); /* parsexlog.c */ extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, XLogRecPtr startpoint, TimeLineID start_tli, diff --git a/src/restore.c b/src/restore.c index d99ce6711..21dd73a70 100644 --- a/src/restore.c +++ b/src/restore.c @@ -925,6 +925,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s", pretty_total_bytes, pretty_time); + /* Close page header maps */ + for (i = parray_num(parent_chain) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + cleanup_header_map(&(backup->hdr_map)); + } + if (no_sync) elog(WARNING, "Restored files are not synced to disk"); else diff --git a/src/validate.c b/src/validate.c index cd35047b5..d16d27677 100644 --- a/src/validate.c +++ b/src/validate.c @@ -185,6 +185,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); + cleanup_header_map(&(backup->hdr_map)); /* Update backup status */ if (corrupted) From ac1ff37adadaae6f0fd92f3cfde29228a04e0ea8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 03:17:28 +0300 Subject: [PATCH 1408/2107] [Issue #228]: fixes --- src/backup.c | 2 +- src/data.c | 118 +++++++++++++++++---------------------------- src/merge.c | 6 +-- src/pg_probackup.h | 15 +++--- 4 files changed, 54 insertions(+), 87 deletions(-) diff --git a/src/backup.c b/src/backup.c index 17b7044ea..92967218d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -599,7 +599,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } /* close and sync page header map */ - if (current.hdr_map.w_fp) + if (current.hdr_map.fp) { cleanup_header_map(&(current.hdr_map)); diff --git a/src/data.c b/src/data.c index 2d4ba57e5..5189c419e 100644 --- a/src/data.c +++ b/src/data.c @@ -2126,6 +2126,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) { + FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; BackupPageHeader2 *headers = NULL; @@ -2140,35 +2141,15 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) if (file->n_headers <= 0) return NULL; -// in = fopen(hdr_map->path, PG_BINARY_R); -// -// if (!in) -// elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); - - if (!hdr_map->r_fp) - { - pthread_lock(&(hdr_map->mutex)); - - /* it is possible for another contender got here first, so double check */ - if (!hdr_map->r_fp) /* this file will be closed in restore.c and merge.c */ - { - elog(LOG, "Opening page header map \"%s\"", hdr_map->path); - - hdr_map->r_fp = fopen(hdr_map->path, PG_BINARY_R); - if (hdr_map->r_fp == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - hdr_map->path, strerror(errno)); + /* TODO: consider to make this descriptor thread-specific */ + in = fopen(hdr_map->path, PG_BINARY_R); - /* enable buffering for header file */ - hdr_map->r_buf = pgut_malloc(LARGE_CHUNK_SIZE); - setvbuf(hdr_map->r_fp, hdr_map->r_buf, _IOFBF, LARGE_CHUNK_SIZE); - } - - /* End critical section */ - pthread_mutex_unlock(&(hdr_map->mutex)); - } + if (!in) + elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + /* disable buffering for header file */ + setvbuf(in, NULL, _IONBF, BUFSIZ); - if (fseek(hdr_map->r_fp, file->hdr_off, SEEK_SET)) + if (fseek(in, file->hdr_off, SEEK_SET)) elog(ERROR, "Cannot seek to position %lu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); @@ -2184,7 +2165,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) zheaders = pgut_malloc(file->hdr_size); memset(zheaders, 0, file->hdr_size); - if (fread(zheaders, 1, file->hdr_size, hdr_map->r_fp) != file->hdr_size) + if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); @@ -2211,6 +2192,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); + if (fclose(in)) + elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + pg_free(zheaders); return headers; @@ -2231,22 +2215,35 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, /* when running merge we must save headers into the temp map */ map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + + /* calculate checksums */ + INIT_FILE_CRC32(true, file->hdr_crc); + COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); + FIN_FILE_CRC32(true, file->hdr_crc); + + zheaders = pgut_malloc(read_len*2); + memset(zheaders, 0, read_len*2); + + /* compress headers */ + z_len = do_compress(zheaders, read_len*2, headers, + read_len, ZLIB_COMPRESS, 1, &errormsg); /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ - if (!hdr_map->w_fp) + if (!hdr_map->fp) { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->w_fp = fopen(map_path, PG_BINARY_W); - if (hdr_map->w_fp == NULL) + hdr_map->fp = fopen(map_path, PG_BINARY_W); + if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); /* enable buffering for header file */ - hdr_map->w_buf = pgut_malloc(LARGE_CHUNK_SIZE); - setvbuf(hdr_map->w_fp, hdr_map->w_buf, _IOFBF, LARGE_CHUNK_SIZE); + hdr_map->buf = pgut_malloc(LARGE_CHUNK_SIZE); + setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, LARGE_CHUNK_SIZE); /* update file permission */ if (chmod(map_path, FILE_PERMISSION) == -1) @@ -2256,20 +2253,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->hdr_off = 0; } else - file->hdr_off = ftell(hdr_map->w_fp); /* TODO: replace by counter */ - - read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); - - /* calculate checksums */ - INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, file->hdr_crc); - - zheaders = pgut_malloc(read_len*2); - memset(zheaders, 0, read_len*2); - - z_len = do_compress(zheaders, read_len*2, headers, - read_len, ZLIB_COMPRESS, 1, &errormsg); + file->hdr_off = hdr_map->offset; if (z_len <= 0) { @@ -2281,15 +2265,14 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->rel_path, z_len); } - if (fwrite(zheaders, 1, z_len, hdr_map->w_fp) != z_len) - elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); - - elog(VERBOSE, "Writing header map for file \"%s\" offset: %li, len: %i, crc: %u", + elog(VERBOSE, "Writing headers for file \"%s\" offset: %li, len: %i, crc: %u", file->rel_path, file->hdr_off, z_len, file->hdr_crc); - elog(INFO, "File: %s, Unzip: %li, zip: %i", file->rel_path, read_len, z_len); + if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) + elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); - file->hdr_size = z_len; + file->hdr_size = z_len; /* save the length of compressed headers */ + hdr_map->offset += z_len; /* update current offset in map */ /* End critical section */ pthread_mutex_unlock(&(hdr_map->mutex)); @@ -2300,10 +2283,8 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, void init_header_map(pgBackup *backup) { - backup->hdr_map.r_fp = NULL; - backup->hdr_map.w_fp = NULL; - backup->hdr_map.r_buf = NULL; - backup->hdr_map.w_buf = NULL; + backup->hdr_map.fp = NULL; + backup->hdr_map.buf = NULL; join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; @@ -2312,22 +2293,11 @@ init_header_map(pgBackup *backup) void cleanup_header_map(HeaderMap *hdr_map) { - - /* cleanup read descriptor */ - if (hdr_map->r_fp && fclose(hdr_map->r_fp)) + /* cleanup descriptor */ + if (hdr_map->fp && fclose(hdr_map->fp)) elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - - hdr_map->r_fp = NULL; - pg_free(hdr_map->r_buf); - hdr_map->r_buf = NULL; - hdr_map->r_offset = 0; - - /* cleanup write descriptor */ - if (hdr_map->w_fp && fclose(hdr_map->w_fp)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - - hdr_map->w_fp = NULL; - pg_free(hdr_map->w_buf); - hdr_map->w_buf = NULL; - hdr_map->w_offset = 0; + hdr_map->fp = NULL; + hdr_map->offset = 0; + pg_free(hdr_map->buf); + hdr_map->buf = NULL; } diff --git a/src/merge.c b/src/merge.c index 5267df49c..dd1d37131 100644 --- a/src/merge.c +++ b/src/merge.c @@ -705,12 +705,12 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) elog(ERROR, "Backup files merging failed, time elapsed: %s", pretty_time); - /* If temp header map descriptor is open, then close it and make rename */ - if (full_backup->hdr_map.w_fp) + /* If temp header map is open, then close it and make rename */ + if (full_backup->hdr_map.fp) { cleanup_header_map(&(full_backup->hdr_map)); - /* sync new header map to dist */ + /* sync new header map to disk */ if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync temp header map \"%s\": %s", full_backup->hdr_map.path_tmp, strerror(errno)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5dd86acf0..ee7c8c7f9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -66,8 +66,8 @@ extern const char *PROGRAM_EMAIL; #define PG_TABLESPACE_MAP_FILE "tablespace_map" #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" -#define HEADER_MAP "block_header_map" -#define HEADER_MAP_TMP "block_header_map_tmp" +#define HEADER_MAP "page_header_map" +#define HEADER_MAP_TMP "page_header_map_tmp" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 @@ -366,13 +366,10 @@ typedef struct PGNodeInfo typedef struct HeaderMap { char path[MAXPGPATH]; - char path_tmp[MAXPGPATH]; /* used only in merge */ - char *r_buf; /* buffer */ - char *w_buf; /* buffer */ - FILE *r_fp; /* descriptor used for reading */ - FILE *w_fp; /* descriptor used for writing */ - off_t r_offset; /* current position in r_fp */ - off_t w_offset; /* current position in w_fp */ + char path_tmp[MAXPGPATH]; /* used only in merge */ + FILE *fp; /* used only for writing */ + char *buf; /* buffer */ + off_t offset; /* current position in fp */ pthread_mutex_t mutex; } HeaderMap; From 73a69151616abacaece428c26a0849385061d89c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 13:54:29 +0300 Subject: [PATCH 1409/2107] [Issue #228]: add strict mode to get_data_file_headers() --- src/data.c | 63 ++++++++++++++++++++++++++++++++++------------ src/merge.c | 3 ++- src/pg_probackup.h | 2 +- src/restore.c | 4 +-- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/src/data.c b/src/data.c index 5189c419e..051c0cdd6 100644 --- a/src/data.c +++ b/src/data.c @@ -823,10 +823,11 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* get headers for this file */ if (use_headers && tmp_file->n_headers > 0) headers = get_data_file_headers(&(backup->hdr_map), tmp_file, - parse_program_version(backup->program_version)); + parse_program_version(backup->program_version), + true); if (use_headers && !headers && tmp_file->n_headers > 0) - elog(ERROR, "Failed to get headers for file \"%s\"", from_fullpath); + elog(ERROR, "Failed to get page headers for file \"%s\"", from_fullpath); /* * Restore the file. @@ -1599,10 +1600,13 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); - headers = get_data_file_headers(hdr_map, file, backup_version); + headers = get_data_file_headers(hdr_map, file, backup_version, false); if (!headers && file->n_headers > 0) - elog(ERROR, "Failed to get headers for file \"%s\"", fullpath); + { + elog(WARNING, "Cannot get page headers for file \"%s\"", fullpath); + return false; + } /* calc CRC of backup file */ INIT_FILE_CRC32(use_crc32c, crc); @@ -2124,8 +2128,9 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f * array of headers. */ BackupPageHeader2* -get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) +get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) { + bool success = false; FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; @@ -2145,13 +2150,19 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) in = fopen(hdr_map->path, PG_BINARY_R); if (!in) - elog(ERROR, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + { + elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + return NULL; + } /* disable buffering for header file */ setvbuf(in, NULL, _IONBF, BUFSIZ); if (fseek(in, file->hdr_off, SEEK_SET)) - elog(ERROR, "Cannot seek to position %lu in page header map \"%s\": %s", + { + elog(strict ? ERROR : WARNING, "Cannot seek to position %lu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); + goto cleanup; + } /* * The actual number of headers in header file is n+1, last one is a dummy header, @@ -2159,28 +2170,35 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) */ read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); - /* allocate memory for compressed and uncompressed headers */ - headers = pgut_malloc(read_len); - memset(headers, 0, read_len); + /* allocate memory for compressed headers */ zheaders = pgut_malloc(file->hdr_size); memset(zheaders, 0, file->hdr_size); if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) - elog(ERROR, "Cannot read header file at offset: %li len: %i \"%s\": %s", + { + elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %li len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); + goto cleanup; + } // elog(INFO, "zsize: %i, size: %i", file->hdr_size, read_len); + /* allocate memory for uncompressed headers */ + headers = pgut_malloc(read_len); + memset(headers, 0, read_len); + z_len = do_decompress(headers, read_len, zheaders, file->hdr_size, ZLIB_COMPRESS, &errormsg); if (z_len <= 0) { if (errormsg) - elog(ERROR, "An error occured during metadata decompression for file \"%s\": %s", + elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %s", file->rel_path, errormsg); else - elog(ERROR, "An error occured during metadata decompression for file \"%s\": %i", + elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %i", file->rel_path, z_len); + + goto cleanup; } /* validate checksum */ @@ -2189,13 +2207,26 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version) FIN_FILE_CRC32(true, hdr_crc); if (hdr_crc != file->hdr_crc) - elog(ERROR, "Header map for file \"%s\" crc mismatch \"%s\" offset: %lu, len: %lu, current: %u, expected: %u", + { + elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " + "offset: %lu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); + goto cleanup; + } - if (fclose(in)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + success = true; + +cleanup: pg_free(zheaders); + if (in && fclose(in)) + elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + + if (!success) + { + pg_free(headers); + headers = NULL; + } return headers; } diff --git a/src/merge.c b/src/merge.c index dd1d37131..97f6bc7c5 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1082,7 +1082,8 @@ merge_files(void *arg) /* Copy header metadata from old map into a new one */ tmp_file->n_headers = file->n_headers; headers = get_data_file_headers(&(arguments->full_backup->hdr_map), file, - parse_program_version(arguments->full_backup->program_version)); + parse_program_version(arguments->full_backup->program_version), + true); /* sanity */ if (!headers && file->n_headers > 0) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ee7c8c7f9..2b5b36f64 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1020,7 +1020,7 @@ extern pid_t check_postmaster(const char *pgdata); extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); -extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version); +extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); extern void init_header_map(pgBackup *backup); extern void cleanup_header_map(HeaderMap *hdr_map); diff --git a/src/restore.c b/src/restore.c index 21dd73a70..5cae45132 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1144,7 +1144,7 @@ restore_files(void *arg) /* Restore destination file */ if (dest_file->is_datafile && !dest_file->is_cfs) { - /* enable stdio buffering for local destination non-data file */ + /* enable stdio buffering for local destination data file */ if (!fio_is_remote_file(out)) setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); /* Destination file is data file */ @@ -1155,7 +1155,7 @@ restore_files(void *arg) } else { - /* disable stdio buffering for local destination data file */ + /* disable stdio buffering for local destination nonedata file */ if (!fio_is_remote_file(out)) setvbuf(out, NULL, _IONBF, BUFSIZ); /* Destination file is non-data file */ From 3df12374bee24f951789b56e5f9370babb5012db Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 13:55:15 +0300 Subject: [PATCH 1410/2107] [Issue #228]: added tests for corruption, truncation and missing conditions --- tests/validate.py | 196 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index c2d1405bc..c84ea5294 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3768,6 +3768,202 @@ def test_not_validate_diffenent_pg_version(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_corrupt_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # Corrupt tablespace_map file in FULL backup + with open(page_header_map, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: An error occured during metadata decompression' in e.message and + 'data error' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: An error occured during metadata decompression' in e.message and + 'data error' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + + self.assertIn("WARNING: Some backups are not valid", e.message) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node]) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_truncated_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # truncate page_header_map file + with open(page_header_map, "rb+", 0) as f: + f.truncate(121) + f.flush() + f.close + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + self.assertIn("WARNING: Some backups are not valid", e.message) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node]) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_missing_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # unlink page_header_map file + os.remove(page_header_map) + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + self.assertIn("WARNING: Some backups are not valid", e.message) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node]) # validate empty backup list # page from future during validate From c8acde3f54c576b45082a250fa61b7a340005c65 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 22:11:43 +0300 Subject: [PATCH 1411/2107] tests: fix tests.archive.ArchiveTest.test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 47bf3d60f..2e09807ef 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1919,7 +1919,8 @@ def test_archive_pg_receivexlog_partial_handling(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '10s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1942,9 +1943,6 @@ def test_archive_pg_receivexlog_partial_handling(self): replica.slow_start(replica=True) - # FULL - self.backup_node(backup_dir, 'replica', replica, options=['--stream']) - if self.get_version(replica) < 100000: pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: @@ -1965,9 +1963,8 @@ def test_archive_pg_receivexlog_partial_handling(self): 'Failed to start pg_receivexlog: {0}'.format( pg_receivexlog.communicate()[1])) - replica.safe_psql( - 'postgres', - 'CHECKPOINT') + # FULL + self.backup_node(backup_dir, 'replica', replica, options=['--stream']) node.safe_psql( "postgres", @@ -1976,7 +1973,8 @@ def test_archive_pg_receivexlog_partial_handling(self): "from generate_series(0,1000000) i") # PAGE - self.backup_node(backup_dir, 'replica', replica, backup_type='page') + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', options=['--stream']) node.safe_psql( "postgres", @@ -2013,7 +2011,8 @@ def test_archive_pg_receivexlog_partial_handling(self): # Clean after yourself pg_receivexlog.kill() - self.del_test_dir(module_name, fname) + self.del_test_dir( + module_name, fname, [node, replica, node_restored]) @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): From 2267d5be01b91ce2e578091e8c8359931ee7f9e3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 22:45:03 +0300 Subject: [PATCH 1412/2107] [Issue #228] fix possible metadata corruption in merge retry --- src/merge.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 97f6bc7c5..042931558 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1046,8 +1046,10 @@ merge_files(void *arg) /* * In-place merge means that file in FULL backup stays as it is, * no additional actions are required. + * page header map cannot be trusted when retrying, so no + * in place merge for retry. */ - if (in_place) + if (in_place && !arguments->is_retry) { pgFile **res_file = NULL; pgFile *file = NULL; From 75e5496da1973bf5b321ac5cbdd3fc3e26516d1d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Jun 2020 23:02:53 +0300 Subject: [PATCH 1413/2107] tests: new test tests.merge.MergeTest.test_merge_page_header_map_retry --- tests/merge.py | 61 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 988eb0558..3444056d2 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2244,7 +2244,7 @@ def test_merge_multiple_descendants(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_smart_merge(self): @@ -2304,7 +2304,7 @@ def test_smart_merge(self): logfile_content = f.read() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_idempotent_merge(self): """ @@ -2379,8 +2379,7 @@ def test_idempotent_merge(self): self.assertEqual( page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) - - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_merge_correct_inheritance(self): """ @@ -2435,7 +2434,7 @@ def test_merge_correct_inheritance(self): page_meta['expire-time'], self.show_pb(backup_dir, 'node', page_id)['expire-time']) - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) def test_merge_correct_inheritance_1(self): """ @@ -2485,7 +2484,7 @@ def test_merge_correct_inheritance_1(self): 'expire-time', self.show_pb(backup_dir, 'node', page_id)) - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") # @unittest.expectedFailure @@ -2603,6 +2602,56 @@ def test_multi_timeline_merge(self): '--amcheck', '-d', 'postgres', '-p', str(node_restored.port)]) + # Clean after yourself + self.del_test_dir(module_name, fname, [node, node_restored]) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_merge_page_header_map_retry(self): + """ + page header map cannot be trusted when + running retry + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=20) + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + gdb = self.merge_backup(backup_dir, 'node', delta_id, gdb=True) + + # our goal here is to get full backup with merged data files, + # but with old page header map + gdb.set_breakpoint('cleanup_header_map') + gdb.run_until_break() + gdb._execute('signal SIGKILL') + + self.merge_backup(backup_dir, 'node', delta_id) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + # Clean after yourself self.del_test_dir(module_name, fname) From 345c26be796f2d19cd3fe730bf6bd4c23dc1ba67 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 13:42:06 +0300 Subject: [PATCH 1414/2107] [Issue #228] nitpicking --- src/backup.c | 3 +- src/data.c | 91 ++++++++++++++++++++++-------------------------- src/restore.c | 6 ++-- src/utils/file.c | 29 ++++++--------- src/utils/file.h | 1 + 5 files changed, 57 insertions(+), 73 deletions(-) diff --git a/src/backup.c b/src/backup.c index 92967218d..18cad5bf3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -212,6 +212,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool "trying to look up on previous timelines", current.tli); + /* TODO: use read_timeline_history */ tli_list = catalog_get_timelines(&instance_config); if (parray_num(tli_list) == 0) @@ -526,7 +527,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool arg->prev_start_lsn = prev_backup_start_lsn; arg->conn_arg.conn = NULL; arg->conn_arg.cancel_conn = NULL; - arg->hdr_map = &(current).hdr_map; + arg->hdr_map = &(current.hdr_map); arg->thread_num = i+1; /* By default there are some error */ arg->ret = 1; diff --git a/src/data.c b/src/data.c index 051c0cdd6..b2176d577 100644 --- a/src/data.c +++ b/src/data.c @@ -31,8 +31,8 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; -static bool get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c); +static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, + pg_crc32 *crc, bool use_crc32c); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ @@ -861,6 +861,10 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, * If "nblocks" is greater than zero, then skip restoring blocks, * whose position if greater than "nblocks". * If map is NULL, then page bitmap cannot be used for restore optimization + * Page bitmap optimize restore of incremental chains, consisting of more than one + * backup. We restoring from newest to oldest and page, once restored, marked in map. + * When the same page, but in older backup, encountered, we check the map, if it is + * marked as already restored, then page is skipped. */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, @@ -930,10 +934,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers else { /* We get into this function either when restoring old backup - * or when merging something. Aligh read_len only in restoring - * or merging old backup. + * or when merging something. Align read_len only in restoring + * or merging old backups. */ - if (get_compressed_page_meta(in, from_fullpath, &(page).bph, NULL, false)) + if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) { cur_pos_in += sizeof(BackupPageHeader); @@ -941,28 +945,26 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers blknum = page.bph.block; compressed_size = page.bph.compressed_size; - /* this will backfire when retrying merge of old backups, - * just pray that this will never happen. + /* this has a potential to backfire when retrying merge of old backups, + * so we just forbid the retrying of failed merges between versions >= 2.4.0 and + * version < 2.4.0 */ if (backup_version >= 20400) read_len = compressed_size; else + /* For some unknown and possibly dump reason I/O operations + * in versions < 2.4.0 were always aligned to 8 bytes. + * Now we have to deal with backward compatibility. + */ read_len = MAXALIGN(compressed_size); -// elog(INFO, "FILE: %s", from_fullpath); -// elog(INFO, "blknum: %i", blknum); -// -// elog(INFO, "POS: %u", cur_pos_in); -// elog(INFO, "SIZE: %i", compressed_size); -// elog(INFO, "ASIZE: %i", read_len); - } else break; } /* - * Backupward compatibility kludge: in the good old days + * Backward compatibility kludge: in the good old days * n_blocks attribute was available only in DELTA backups. * File truncate in PAGE and PTRACK happened on the fly when * special value PageIsTruncated is encountered. @@ -1006,13 +1008,13 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (compressed_size > BLCKSZ) elog(ERROR, "Size of a blknum %i exceed BLCKSZ: %i", blknum, compressed_size); - /* incremental restore in LSN mode */ + /* Incremental restore in LSN mode */ if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) datapagemap_add(map, blknum); if (map && checksum_map && checksum_map[blknum].checksum != 0) { -// elog(INFO, "HDR CRC: %u, MAP CRC: %u", page_crc, checksum_map[blknum].checksum); + //elog(INFO, "HDR CRC: %u, MAP CRC: %u", page_crc, checksum_map[blknum].checksum); /* * The heart of incremental restore in CHECKSUM mode * If page in backup has the same checksum and lsn as @@ -1110,7 +1112,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers write_len += BLCKSZ; cur_pos_out += BLCKSZ; /* update current write position */ - /* Mark page as restored, to avoid reading this page when restoring parent backups */ + /* Mark page as restored to avoid reading this page when restoring parent backups */ if (map) datapagemap_add(map, blknum); } @@ -1238,7 +1240,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* incremental restore */ if (already_exists) { - /* compare checksumms of remote and local files */ + /* compare checksums of already existing file and backup file */ pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); if (file_crc == tmp_file->crc) @@ -1625,7 +1627,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file validation"); - /* newer backups have headers in separate storage */ + /* newer backups have page headers in separate storage */ if (headers) { n_hdr++; @@ -1657,7 +1659,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* old backups rely on header located directly in data file */ else { - if (get_compressed_page_meta(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) + if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) { /* Backward compatibility kludge, TODO: remove in 3.0 * for some reason we padded compressed pages in old versions @@ -1686,11 +1688,6 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, else len = fread(compressed_page.data, 1, read_len, in); -// elog(INFO, "POS: %u", cur_pos_in); -// -// elog(INFO, "LEN: %i", len); -// elog(INFO, "READ_LEN: %i", read_len); - if (len != read_len) { elog(WARNING, "Cannot read block %u file \"%s\": %s", @@ -1886,11 +1883,8 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, for (blknum = 0; blknum < n_blocks; blknum++) { size_t read_len = fread(read_buffer, 1, BLCKSZ, in); -// page_lsn = InvalidXLogRecPtr; PageState page_st; -// page_st.lsn = InvalidXLogRecPtr - /* report error */ if (ferror(in)) elog(ERROR, "Cannot read block %u of \"%s\": %s", @@ -1905,7 +1899,8 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, datapagemap_add(lsn_map, blknum); } else - elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); + elog(ERROR, "Cannot read block %u from file \"%s\": %s", + blknum, fullpath, strerror(errno)); if (feof(in)) break; @@ -1926,10 +1921,10 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } -/* */ +/* Every page in data file contains BackupPageHeader, extract it */ bool -get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c) +get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, + pg_crc32 *crc, bool use_crc32c) { /* read BackupPageHeader */ @@ -1952,26 +1947,18 @@ get_compressed_page_meta(FILE *in, const char *fullpath, BackupPageHeader* bph, ftell(in), fullpath, strerror(errno)); } + /* In older versions < 2.4.0, when crc for file was calculated, header was + * not included in crc calculations. Now it is. And now we have + * the problem of backward compatibility for backups of old versions + */ if (crc) COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); if (bph->block == 0 && bph->compressed_size == 0) elog(ERROR, "Empty block in file \"%s\"", fullpath); - -// *blknum = header.block; -// *compressed_size = header.compressed_size; - -// elog(INFO, "blknum: %i", header.block); -// elog(INFO, "size: %i", header.compressed_size); -// elog(INFO, "size2: %i", *compressed_size); -// -// elog(INFO, "BLKNUM: %i", *blknum); -// elog(INFO, "File: %s", fullpath); - Assert(bph->compressed_size != 0); return true; - } /* Open local backup file for writing, set permissions and buffering */ @@ -2099,7 +2086,10 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f blknum++; } - /* add one additional header */ + /* + * Add dummy header, so we can later extract the length of last header + * as difference between their offsets. + */ if (*headers) { file->n_headers = hdr_num +1; @@ -2124,8 +2114,11 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f return n_blocks_read; } -/* attempt to open header file, read content and return as +/* + * Attempt to open header file, read content and return as * array of headers. + * TODO: some access optimizations would be great here: + * less fseeks, buffering, descriptor sharing, etc. */ BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) @@ -2181,8 +2174,6 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b goto cleanup; } -// elog(INFO, "zsize: %i, size: %i", file->hdr_size, read_len); - /* allocate memory for uncompressed headers */ headers = pgut_malloc(read_len); memset(headers, 0, read_len); @@ -2244,7 +2235,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, if (file->n_headers <= 0) return; - /* when running merge we must save headers into the temp map */ + /* when running merge we must write headers into temp map */ map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); diff --git a/src/restore.c b/src/restore.c index 5cae45132..ff554d04f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -345,7 +345,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty) { elog(INFO, "Destination directory and tablespace directories are empty, " - "disabled incremental restore"); + "disable incremental restore"); params->incremental_mode = INCR_NONE; } @@ -699,7 +699,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_qsort(backup->files, pgFileCompareRelPathWithExternal); } - /* If dest backup version is older than 2.3.0, then bitmap optimization + /* If dest backup version is older than 2.4.0, then bitmap optimization * is impossible to use, because bitmap restore rely on pgFile.n_blocks, * which is not always available in old backups. */ @@ -2009,7 +2009,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, { elog(WARNING, "Destination directory contains \"backup_control\" file. " "This does NOT mean that you should delete this file and retry, only that " - "incremental restore in 'lsn' mode can produce incorrect result, when applied " + "incremental restore in 'lsn' mode may produce incorrect result, when applied " "to cluster with pg_control not synchronized with cluster state." "Consider to use incremental restore in 'checksum' mode"); success = false; diff --git a/src/utils/file.c b/src/utils/file.c index f3d451bea..19dc5f01c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -455,7 +455,7 @@ fio_disconnect(void) hdr.size = 0; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND_FILE_EOF); + Assert(hdr.cop == FIO_DISCONNECTED); SYS_CHECK(close(fio_stdin)); SYS_CHECK(close(fio_stdout)); fio_stdin = 0; @@ -1652,7 +1652,8 @@ static void fio_send_pages_impl(int out, char* buf) /* read page, check header and validate checksumms */ for (;;) { - /* Optimize stdio buffer usage, fseek only when current position + /* + * Optimize stdio buffer usage, fseek only when current position * does not match the position of requested block. */ if (current_pos != blknum*BLCKSZ) @@ -1665,10 +1666,7 @@ static void fio_send_pages_impl(int out, char* buf) read_len = fread(read_buffer, 1, BLCKSZ, in); - page_st.lsn = InvalidXLogRecPtr; - page_st.checksum = 0; - - current_pos += BLCKSZ; + current_pos += read_len; /* report error */ if (ferror(in)) @@ -1741,9 +1739,9 @@ static void fio_send_pages_impl(int out, char* buf) * As far as unsigned number are always greater or equal than zero, * there is no sense to add more checks. */ - if ((req->horizonLsn == InvalidXLogRecPtr) || + if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page, ptrack */ (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_st.lsn >= req->horizonLsn)) /* delta */ + (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta */ { int compressed_size = 0; char write_buffer[BLCKSZ*2]; @@ -1808,7 +1806,7 @@ static void fio_send_pages_impl(int out, char* buf) { hdr.size = (hdr_num+2) * sizeof(BackupPageHeader2); - /* add one additional header */ + /* add dummy header */ headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+2) * sizeof(BackupPageHeader2)); headers[hdr_num+1].pos = cur_pos_out; } @@ -1817,13 +1815,6 @@ static void fio_send_pages_impl(int out, char* buf) if (headers) IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); - /* send headers */ -// hdr.cop = FIO_SEND_FILE_HEADERS; -// hdr.arg = hdr_num +1; -// hdr.size = hdr.arg * sizeof(BackupPageHeader2); -// IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); -// IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); - cleanup: pg_free(map); pg_free(iter); @@ -2388,7 +2379,7 @@ static void fio_get_checksum_map_impl(int out, char *buf) req->n_blocks, req->stop_lsn, req->segmentno); hdr.size = req->n_blocks; - /* send arrays of checksums to main process */ + /* send array of PageState`s to main process */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > 0) IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); @@ -2458,7 +2449,7 @@ static void fio_get_lsn_map_impl(int out, char *buf) else hdr.size = 0; - /* send arrays of checksums to main process */ + /* send bitmap to main process */ IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > 0) IO_CHECK(fio_write_all(out, lsn_map->bitmap, hdr.size), hdr.size); @@ -2691,7 +2682,7 @@ void fio_communicate(int in, int out) fio_check_postmaster_impl(out, buf); break; case FIO_DISCONNECT: - hdr.cop = FIO_SEND_FILE_EOF; + hdr.cop = FIO_DISCONNECTED; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; default: diff --git a/src/utils/file.h b/src/utils/file.h index 19fcf6d5d..8302662f3 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -48,6 +48,7 @@ typedef enum FIO_SEND_FILE_HEADERS, /* messages for closing connection */ FIO_DISCONNECT, + FIO_DISCONNECTED, /* message for compatibility check */ FIO_AGENT_VERSION, FIO_LIST_DIR, From 0410ff84084010ca0210298fdf156ede2d46eef7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 13:49:47 +0300 Subject: [PATCH 1415/2107] [Issue #228] documentation update --- doc/pgprobackup.xml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index efbbf980e..f62c68d97 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1732,7 +1732,7 @@ pg_probackup restore -B backup_dir --instance @@ -1740,7 +1740,7 @@ pg_probackup restore -B backup_dir --instance pg_control in the data directory to obtain redo LSN and redo TLI, which allows to determine a point in history(shiftpoint), where data directory - state shifted from backup chain history. If shiftpoint is not within + state shifted from target backup chain history. If shiftpoint is not within reach of backup chain history, then restore is aborted. If shiftpoint is within reach of backup chain history, then read all data files in the data directory, validate header and checksum in @@ -1755,7 +1755,7 @@ pg_probackup restore -B backup_dir --instance pg_control file must be synched with state of data directory. This condition cannot checked at the start of restore, so it is a user responsibility to ensure - that pg_control contain valid information. Because of that is not + that pg_control contain valid information. Therefore it is not recommended to use LSN mode in any situation, where pg_control has been tampered with: after pg_resetxlog execution, @@ -1769,7 +1769,13 @@ pg_probackup restore -B backup_dir --instance - + + Regardless of chosen incremental mode, pg_probackup will check, that postmaster + in given destination directory is not running and system-identifier is + the same as in the backup. + + + Suppose you want to return an old master as replica after switchover using incremental restore in LSN mode: @@ -1794,12 +1800,12 @@ INFO: Redundant files are removed, time elapsed: 1s INFO: Start restoring backup files. PGDATA size: 15GB INFO: Backup files are restored. Transfered bytes: 1693MB, time elapsed: 43s INFO: Restore incremental ratio (less is better): 11% (1693MB/15GB) -INFO: Restore of backup QBRHT8 completed. +INFO: Restore of backup QBRNBP completed. Incremental restore is possible only for backups with - program_version equal or greater than 2.3.0. + program_version equal or greater than 2.4.0. From c670de211a5a24b1ccf17112d73d5c7c588563c5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 13:52:56 +0300 Subject: [PATCH 1416/2107] tests: update compatibility module --- tests/compatibility.py | 151 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 7 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 49ea50d30..8cbd61782 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -1041,7 +1041,8 @@ def test_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node, old_binary=True) + id1 = self.backup_node(backup_dir, 'node', node, old_binary=True) + pgdata1 = self.pgdata_content(node.data_dir) node.safe_psql( "postgres", @@ -1051,8 +1052,9 @@ def test_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( + id2 = self.backup_node( backup_dir, 'node', node, backup_type='page', old_binary=True) + pgdata2 = self.pgdata_content(node.data_dir) node.safe_psql( "postgres", @@ -1061,23 +1063,49 @@ def test_page_vacuum_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1) i") - self.backup_node( + id3 = self.backup_node( backup_dir, 'node', node, backup_type='page', old_binary=True) - - pgdata = self.pgdata_content(node.data_dir) + pgdata3 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id1) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + self.compare_pgdata(pgdata1, pgdata_restored) self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id2) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id3) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() # Clean after yourself self.del_test_dir(module_name, fname) @@ -1162,6 +1190,115 @@ def test_page_vacuum_truncate_compression(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_page_vacuum_truncate_compressed_1(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id1 = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--compress']) + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id2 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + pgdata2 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + id3 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + pgdata3 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id1) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id2) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id3) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_hidden_files(self): """ From 1ea48b19de96ee2c078472963a720005a6780c38 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 14:26:24 +0300 Subject: [PATCH 1417/2107] [Issue #228] a bit of cleaning --- src/data.c | 9 ++------- src/dir.c | 2 +- src/pg_probackup.h | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/data.c b/src/data.c index b2176d577..b4df5217f 100644 --- a/src/data.c +++ b/src/data.c @@ -816,10 +816,6 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* set stdio buffering for input data file */ setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); -// elog(INFO, "N_HEADERS: %i", tmp_file->n_headers); -// elog(INFO, "File: %s", tmp_file->rel_path); -// elog(INFO, "Backup: %s", base36enc(backup->start_time)); - /* get headers for this file */ if (use_headers && tmp_file->n_headers > 0) headers = get_data_file_headers(&(backup->hdr_map), tmp_file, @@ -934,7 +930,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers else { /* We get into this function either when restoring old backup - * or when merging something. Align read_len only in restoring + * or when merging something. Align read_len only when restoring * or merging old backups. */ if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) @@ -1926,10 +1922,9 @@ bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, pg_crc32 *crc, bool use_crc32c) { - /* read BackupPageHeader */ size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); - + if (ferror(in)) elog(ERROR, "Cannot read file \"%s\": %s", fullpath, strerror(errno)); diff --git a/src/dir.c b/src/dir.c index 219126196..2c6dd65da 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1213,7 +1213,7 @@ read_tablespace_map(parray *files, const char *backup_dir) * * * TODO: maybe when running incremental restore with tablespace remapping, then - * new tablespace directory MUST be empty? because there is no + * new tablespace directory MUST be empty? because there is no way * we can be sure, that files laying there belong to our instance. */ void diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2b5b36f64..e24a6d29a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1110,8 +1110,8 @@ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, - uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); + uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, + BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); /* FIO */ extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, From b5782d080467d1fc937bda7730c80b5ed8880302 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 16:30:31 +0300 Subject: [PATCH 1418/2107] [Issue #213] Fix pgbadger parsing --- src/archive.c | 315 +++++++++++++++++++++------------------------ src/pg_probackup.c | 5 + src/pg_probackup.h | 3 + src/utils/logger.c | 22 +++- 4 files changed, 174 insertions(+), 171 deletions(-) diff --git a/src/archive.c b/src/archive.c index 487f6c0e1..4780db063 100644 --- a/src/archive.c +++ b/src/archive.c @@ -15,18 +15,18 @@ static int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, - int thread_num, uint32 archive_timeout); + uint32 archive_timeout); #ifdef HAVE_LIBZ static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, int thread_num, uint32 archive_timeout); + int compress_level, uint32 archive_timeout); #endif static void *push_files(void *arg); static void *get_files(void *arg); static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, - bool prefetch_mode, int thread_num); + bool prefetch_mode); static int get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress, int thread_num); + bool is_decompress); #ifdef HAVE_LIBZ static const char *get_gz_error(gzFile gzf, int errnum); #endif @@ -41,7 +41,7 @@ static uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir uint32 wal_seg_size); static bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, const char *prefetch_dir, const char *absolute_wal_file_path, - uint32 wal_seg_size, bool parse_wal, int thread_num); + uint32 wal_seg_size, bool parse_wal); static uint32 maintain_prefetch(const char *prefetch_dir, XLogSegNo first_segno, uint32 wal_seg_size); @@ -97,7 +97,7 @@ static int push_file(WALSegno *xlogfile, const char *archive_status_dir, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout, bool no_ready_rename, bool is_compress, - int compress_level, int thread_num); + int compress_level); static parray *setup_push_filelist(const char *archive_status_dir, const char *first_file, int batch_size); @@ -133,7 +133,6 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, uint32 n_total_pushed = 0; uint32 n_total_skipped = 0; uint32 n_total_failed = 0; - pid_t my_pid; instr_time start_time, end_time; double push_time; char pretty_time_str[20]; @@ -142,8 +141,6 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, parray *batch_files = NULL; int n_threads; - my_pid = getpid(); - if (wal_file_name == NULL) elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); @@ -182,9 +179,9 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, if (num_threads > parray_num(batch_files)) n_threads = parray_num(batch_files); - elog(INFO, "PID [%d]: pg_probackup archive-push WAL file: %s, " + elog(INFO, "pg_probackup archive-push WAL file: %s, " "threads: %i/%i, batch: %lu/%i, compression: %s", - my_pid, wal_file_name, n_threads, num_threads, + wal_file_name, n_threads, num_threads, parray_num(batch_files), batch_size, is_compress ? "zlib" : "none"); @@ -214,7 +211,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, instance->archive_timeout, no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false, is_compress && IsXLogFileName(xlogfile->name) ? true : false, - instance->compress_level, 1); + instance->compress_level); if (rc == 0) n_total_pushed++; else @@ -292,13 +289,13 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, if (push_isok) /* report number of files pushed into archive */ - elog(INFO, "PID [%d]: pg_probackup archive-push completed successfully, " + elog(INFO, "pg_probackup archive-push completed successfully, " "pushed: %u, skipped: %u, time elapsed: %s", - my_pid, n_total_pushed, n_total_skipped, pretty_time_str); + n_total_pushed, n_total_skipped, pretty_time_str); else - elog(ERROR, "PID [%d]: pg_probackup archive-push failed, " + elog(ERROR, "pg_probackup archive-push failed, " "pushed: %i, skipped: %u, failed: %u, time elapsed: %s", - my_pid, n_total_pushed, n_total_skipped, n_total_failed, + n_total_pushed, n_total_skipped, n_total_failed, pretty_time_str); } @@ -313,6 +310,8 @@ push_files(void *arg) int rc; archive_push_arg *args = (archive_push_arg *) arg; + my_thread_num = args->thread_num; + for (i = 0; i < parray_num(args->files); i++) { bool no_ready_rename = args->no_ready_rename; @@ -334,7 +333,7 @@ push_files(void *arg) args->archive_timeout, no_ready_rename, /* do not compress .backup, .partial and .history files */ args->compress && IsXLogFileName(xlogfile->name) ? true : false, - args->compress_level, args->thread_num); + args->compress_level); if (rc == 0) args->n_pushed++; @@ -354,25 +353,25 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout, bool no_ready_rename, bool is_compress, - int compress_level, int thread_num) + int compress_level) { int rc; char wal_file_dummy[MAXPGPATH]; join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); - elog(LOG, "Thread [%d]: pushing file \"%s\"", thread_num, xlogfile->name); + elog(LOG, "pushing file \"%s\"", xlogfile->name); /* If compression is not required, then just copy it as is */ if (!is_compress) rc = push_file_internal_uncompressed(xlogfile->name, pg_xlog_dir, archive_dir, overwrite, no_sync, - thread_num, archive_timeout); + archive_timeout); #ifdef HAVE_LIBZ else rc = push_file_internal_gz(xlogfile->name, pg_xlog_dir, archive_dir, overwrite, no_sync, compress_level, - thread_num, archive_timeout); + archive_timeout); #endif /* take '--no-ready-rename' flag into account */ @@ -387,13 +386,12 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, canonicalize_path(wal_file_ready); canonicalize_path(wal_file_done); /* It is ok to rename status file in archive_status directory */ - elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", thread_num, - wal_file_ready, wal_file_done); + elog(VERBOSE, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); /* do not error out, if rename failed */ if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) - elog(WARNING, "Thread [%d]: Cannot rename ready file \"%s\" to \"%s\": %s", - thread_num, wal_file_ready, wal_file_done, strerror(errno)); + elog(WARNING, "Cannot rename ready file \"%s\" to \"%s\": %s", + wal_file_ready, wal_file_done, strerror(errno)); } return rc; @@ -410,7 +408,7 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, - int thread_num, uint32 archive_timeout) + uint32 archive_timeout) { FILE *in = NULL; int out = -1; @@ -434,8 +432,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) - elog(ERROR, "Thread [%d]: Cannot open source file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); + elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -448,8 +445,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) - elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Failed to open temp WAL file \"%s\": %s", + to_fullpath_part, strerror(errno)); /* Already existing destination temp file is not an error condition */ } else @@ -479,24 +476,22 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) - elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Failed to open temp WAL file \"%s\": %s", + to_fullpath_part, strerror(errno)); } else /* Successfully created partial file */ break; } else - elog(ERROR, "Thread [%d]: Cannot stat temp WAL file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); } /* first round */ if (!partial_try_count) { - elog(LOG, "Thread [%d]: Temp WAL file already exists, " - "waiting on it %u seconds: \"%s\"", - thread_num, archive_timeout, to_fullpath_part); + elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", + archive_timeout, to_fullpath_part); partial_file_size = st.st_size; } @@ -520,23 +515,20 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (!partial_is_stale) - elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\" in %i seconds", - thread_num, to_fullpath_part, archive_timeout); + elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", + to_fullpath_part, archive_timeout); /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Thread [%d]: Reusing stale temp WAL file \"%s\"", - thread_num, to_fullpath_part); + elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) - elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); } part_opened: - elog(VERBOSE, "Thread [%d]: Temp WAL file successfully created: \"%s\"", - thread_num, to_fullpath_part); + elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); /* Check if possible to skip copying */ if (fileExists(to_fullpath, FIO_BACKUP_HOST)) { @@ -548,8 +540,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (crc32_src == crc32_dst) { - elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); + elog(LOG, "WAL file already exists in archive with the same " + "checksum, skip pushing: \"%s\"", from_fullpath); /* cleanup */ fclose(in); fio_close(out); @@ -559,16 +551,16 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d else { if (overwrite) - elog(LOG, "Thread [%d]: WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", thread_num, to_fullpath); + elog(LOG, "WAL file already exists in archive with " + "different checksum, overwriting: \"%s\"", to_fullpath); else { /* Overwriting is forbidden, * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: WAL file already exists in archive with " - "different checksum: \"%s\"", thread_num, to_fullpath); + elog(ERROR, "WAL file already exists in archive with " + "different checksum: \"%s\"", to_fullpath); } } } @@ -583,15 +575,15 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (ferror(in)) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot read source file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); + elog(ERROR, "Cannot read source file \"%s\": %s", + from_fullpath, strerror(errno)); } if (read_len > 0 && fio_write(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot write to destination temp file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Cannot write to destination temp file \"%s\": %s", + to_fullpath_part, strerror(errno)); } if (feof(in)) @@ -605,20 +597,19 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (fio_close(out) != 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot close temp WAL file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Cannot close temp WAL file \"%s\": %s", + to_fullpath_part, strerror(errno)); } /* sync temp file to disk */ if (!no_sync) { if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Thread [%d]: Failed to sync file \"%s\": %s", - thread_num, to_fullpath_part, strerror(errno)); + elog(ERROR, "Failed to sync file \"%s\": %s", + to_fullpath_part, strerror(errno)); } - elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", - thread_num, to_fullpath_part, to_fullpath); + elog(VERBOSE, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -626,8 +617,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot rename file \"%s\" to \"%s\": %s", - thread_num, to_fullpath_part, to_fullpath, strerror(errno)); + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + to_fullpath_part, to_fullpath, strerror(errno)); } pg_free(buf); @@ -645,7 +636,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, int thread_num, uint32 archive_timeout) + int compress_level, uint32 archive_timeout) { FILE *in = NULL; gzFile out = NULL; @@ -677,8 +668,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) - elog(ERROR, "Thread [%d]: Cannot open source WAL file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); + elog(ERROR, "Cannot open source WAL file \"%s\": %s", + from_fullpath, strerror(errno)); /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -688,8 +679,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) - elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Cannot open temp WAL file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); /* Already existing destination temp file is not an error condition */ } else @@ -719,24 +710,23 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) - elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Failed to open temp WAL file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); } else /* Successfully created partial file */ break; } else - elog(ERROR, "Thread [%d]: Cannot stat temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); } /* first round */ if (!partial_try_count) { - elog(LOG, "Thread [%d]: Temp WAL file already exists, " - "waiting on it %u seconds: \"%s\"", - thread_num, archive_timeout, to_fullpath_gz_part); + elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", + archive_timeout, to_fullpath_gz_part); partial_file_size = st.st_size; } @@ -760,23 +750,21 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (!partial_is_stale) - elog(ERROR, "Thread [%d]: Failed to open temp WAL file \"%s\" in %i seconds", - thread_num, to_fullpath_gz_part, archive_timeout); + elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", + to_fullpath_gz_part, archive_timeout); /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Thread [%d]: Reusing stale temp WAL file \"%s\"", - thread_num, to_fullpath_gz_part); + elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (out == NULL) - elog(ERROR, "Thread [%d]: Cannot open temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Cannot open temp WAL file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); } part_opened: - elog(VERBOSE, "Thread [%d]: Temp WAL file successfully created: \"%s\"", - thread_num, to_fullpath_gz_part); + elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); /* Check if possible to skip copying, */ if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) @@ -790,8 +778,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (crc32_src == crc32_dst) { - elog(LOG, "Thread [%d]: WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", thread_num, from_fullpath); + elog(LOG, "WAL file already exists in archive with the same " + "checksum, skip pushing: \"%s\"", from_fullpath); /* cleanup */ fclose(in); fio_gzclose(out); @@ -801,16 +789,16 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, else { if (overwrite) - elog(LOG, "Thread [%d]: WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", thread_num, to_fullpath_gz); + elog(LOG, "WAL file already exists in archive with " + "different checksum, overwriting: \"%s\"", to_fullpath_gz); else { /* Overwriting is forbidden, * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: WAL file already exists in archive with " - "different checksum: \"%s\"", thread_num, to_fullpath_gz); + elog(ERROR, "WAL file already exists in archive with " + "different checksum: \"%s\"", to_fullpath_gz); } } } @@ -825,15 +813,15 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (ferror(in)) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot read from source file \"%s\": %s", - thread_num, from_fullpath, strerror(errno)); + elog(ERROR, "Cannot read from source file \"%s\": %s", + from_fullpath, strerror(errno)); } if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot write to compressed temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, get_gz_error(out, errno)); + elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", + to_fullpath_gz_part, get_gz_error(out, errno)); } if (feof(in)) @@ -847,20 +835,20 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (fio_gzclose(out) != 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot close compressed temp WAL file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); } /* sync temp file to disk */ if (!no_sync) { if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Thread [%d]: Failed to sync file \"%s\": %s", - thread_num, to_fullpath_gz_part, strerror(errno)); + elog(ERROR, "Failed to sync file \"%s\": %s", + to_fullpath_gz_part, strerror(errno)); } - elog(VERBOSE, "Thread [%d]: Rename \"%s\" to \"%s\"", - thread_num, to_fullpath_gz_part, to_fullpath_gz); + elog(VERBOSE, "Rename \"%s\" to \"%s\"", + to_fullpath_gz_part, to_fullpath_gz); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -868,8 +856,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - elog(ERROR, "Thread [%d]: Cannot rename file \"%s\" to \"%s\": %s", - thread_num, to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); } pg_free(buf); @@ -1012,7 +1000,6 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, char prefetched_file[MAXPGPATH]; /* reporting */ - pid_t my_pid = getpid(); uint32 n_fetched = 0; int n_actual_threads = num_threads; uint32 n_files_in_prefetch = 0; @@ -1023,13 +1010,13 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, char pretty_time_str[20]; if (wal_file_name == NULL) - elog(ERROR, "PID [%d]: Required parameter not specified: --wal-file-name %%f", my_pid); + elog(ERROR, "Required parameter not specified: --wal-file-name %%f"); if (wal_file_path == NULL) - elog(ERROR, "PID [%d]: Required parameter not specified: --wal_file_path %%p", my_pid); + elog(ERROR, "Required parameter not specified: --wal_file_path %%p"); if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "PID [%d]: getcwd() error", my_pid); + elog(ERROR, "getcwd() error"); /* path to PGDATA/pg_wal directory */ join_path_components(pg_xlog_dir, current_dir, XLOGDIR); @@ -1044,8 +1031,8 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, INSTR_TIME_SET_CURRENT(start_time); if (num_threads > batch_size) n_actual_threads = batch_size; - elog(INFO, "PID [%d]: pg_probackup archive-get WAL file: %s, remote: %s, threads: %i/%i, batch: %i", - my_pid, wal_file_name, IsSshProtocol() ? "ssh" : "none", n_actual_threads, num_threads, batch_size); + elog(INFO, "pg_probackup archive-get WAL file: %s, remote: %s, threads: %i/%i, batch: %i", + wal_file_name, IsSshProtocol() ? "ssh" : "none", n_actual_threads, num_threads, batch_size); num_threads = n_actual_threads; @@ -1098,11 +1085,11 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, absolute_wal_file_path, instance->xlog_seg_size, - validate_wal, 0)) + validate_wal)) { n_files_in_prefetch--; - elog(INFO, "PID [%d]: pg_probackup archive-get used prefetched WAL segment %s, prefetch state: %u/%u", - my_pid, wal_file_name, n_files_in_prefetch, batch_size); + elog(INFO, "pg_probackup archive-get used prefetched WAL segment %s, prefetch state: %u/%u", + wal_file_name, n_files_in_prefetch, batch_size); goto get_done; } else @@ -1136,11 +1123,11 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, n_files_in_prefetch = maintain_prefetch(prefetch_dir, segno, instance->xlog_seg_size); if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, absolute_wal_file_path, - instance->xlog_seg_size, validate_wal, 0)) + instance->xlog_seg_size, validate_wal)) { n_files_in_prefetch--; - elog(INFO, "PID [%d]: pg_probackup archive-get copied WAL file %s, prefetch state: %u/%u", - my_pid, wal_file_name, n_files_in_prefetch, batch_size); + elog(INFO, "pg_probackup archive-get copied WAL file %s, prefetch state: %u/%u", + wal_file_name, n_files_in_prefetch, batch_size); goto get_done; } // else @@ -1168,19 +1155,17 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, while (fail_count < 3) { - if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false, 0)) + if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false)) { fail_count = 0; - elog(INFO, "PID [%d]: pg_probackup archive-get copied WAL file %s", - my_pid, wal_file_name); + elog(INFO, "pg_probackup archive-get copied WAL file %s", wal_file_name); n_fetched++; break; } else fail_count++; - elog(LOG, "PID [%d]: Failed to get WAL file %s, retry %i/3", - 0, wal_file_name, fail_count); + elog(LOG, "Failed to get WAL file %s, retry %i/3", wal_file_name, fail_count); } /* TODO/TOTHINK: @@ -1201,11 +1186,11 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, pretty_time_interval(get_time, pretty_time_str, 20); if (fail_count == 0) - elog(INFO, "PID [%d]: pg_probackup archive-get completed successfully, fetched: %i/%i, time elapsed: %s", - my_pid, n_fetched, batch_size, pretty_time_str); + elog(INFO, "pg_probackup archive-get completed successfully, fetched: %i/%i, time elapsed: %s", + n_fetched, batch_size, pretty_time_str); else - elog(ERROR, "PID [%d]: pg_probackup archive-get failed to deliver WAL file: %s, time elapsed: %s", - my_pid, wal_file_name, pretty_time_str); + elog(ERROR, "pg_probackup archive-get failed to deliver WAL file: %s, time elapsed: %s", + wal_file_name, pretty_time_str); } /* @@ -1253,7 +1238,7 @@ uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, /* It is ok, maybe requested batch is greater than the number of available * files in the archive */ - if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true, 0)) + if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true)) { elog(LOG, "Thread [%d]: Failed to prefetch WAL segment %s", 0, xlogfile->name); break; @@ -1312,6 +1297,8 @@ get_files(void *arg) char from_fullpath[MAXPGPATH]; archive_get_arg *args = (archive_get_arg *) arg; + my_thread_num = args->thread_num; + for (i = 0; i < parray_num(args->files); i++) { WALSegno *xlogfile = (WALSegno *) parray_get(args->files, i); @@ -1325,13 +1312,12 @@ get_files(void *arg) join_path_components(from_fullpath, args->archive_dir, xlogfile->name); join_path_components(to_fullpath, args->prefetch_dir, xlogfile->name); - if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true, args->thread_num)) + if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true)) { /* It is ok, maybe requested batch is greater than the number of available * files in the archive */ - elog(LOG, "Thread [%d]: Failed to prefetch WAL segment %s", - args->thread_num, xlogfile->name); + elog(LOG, "Failed to prefetch WAL segment %s", xlogfile->name); prefetch_stop = true; break; } @@ -1351,7 +1337,7 @@ get_files(void *arg) */ bool get_wal_file(const char *filename, const char *from_fullpath, - const char *to_fullpath, bool prefetch_mode, int thread_num) + const char *to_fullpath, bool prefetch_mode) { int rc = FILE_MISSING; FILE *out; @@ -1364,15 +1350,15 @@ get_wal_file(const char *filename, const char *from_fullpath, out = fopen(to_fullpath, PG_BINARY_W); if (!out) { - elog(WARNING, "Thread [%d]: Failed to open file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Failed to open file '%s': %s", + to_fullpath, strerror(errno)); return false; } if (chmod(to_fullpath, FILE_PERMISSION) == -1) { - elog(WARNING, "Thread [%d]: Cannot change mode of file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Cannot change mode of file '%s': %s", + to_fullpath, strerror(errno)); fclose(out); unlink(to_fullpath); return false; @@ -1420,11 +1406,11 @@ get_wal_file(const char *filename, const char *from_fullpath, } if (rc == WRITE_FAILED) - elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Cannot write to file '%s': %s", + to_fullpath, strerror(errno)); if (errmsg) - elog(WARNING, "Thread [%d]: %s", thread_num, errmsg); + elog(WARNING, "%s", errmsg); pg_free(errmsg); } @@ -1434,11 +1420,11 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ if (IsXLogFileName(filename)) - rc = get_wal_file_internal(from_fullpath_gz, to_fullpath, out, true, thread_num); + rc = get_wal_file_internal(from_fullpath_gz, to_fullpath, out, true); if (rc == FILE_MISSING) #endif /* ... failing that, use uncompressed */ - rc = get_wal_file_internal(from_fullpath, to_fullpath, out, false, thread_num); + rc = get_wal_file_internal(from_fullpath, to_fullpath, out, false); /* When not in prefetch mode, try to use partial file */ if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) @@ -1448,15 +1434,13 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* '.gz.partial' goes first ... */ snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, - out, true, thread_num); + rc = get_wal_file_internal(from_partial, to_fullpath, out, true); if (rc == FILE_MISSING) #endif { /* ... failing that, use '.partial' */ snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, out, - false, thread_num); + rc = get_wal_file_internal(from_partial, to_fullpath, out, false); } if (rc == SEND_OK) @@ -1465,8 +1449,7 @@ get_wal_file(const char *filename, const char *from_fullpath, } if (!prefetch_mode && (rc == FILE_MISSING)) - elog(LOG, "Thread [%d]: Target WAL file is missing: %s", - thread_num, filename); + elog(LOG, "Target WAL file is missing: %s", filename); if (rc < 0) { @@ -1484,8 +1467,7 @@ get_wal_file(const char *filename, const char *from_fullpath, if (fflush(out) != 0) { - elog(WARNING, "Thread [%d]: Cannot flush file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); fclose(out); unlink(to_fullpath); return false; @@ -1493,8 +1475,7 @@ get_wal_file(const char *filename, const char *from_fullpath, if (ftruncate(fileno(out), xlog_seg_size) != 0) { - elog(WARNING, "Thread [%d]: Cannot extend file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Cannot extend file \"%s\": %s", to_fullpath, strerror(errno)); fclose(out); unlink(to_fullpath); return false; @@ -1503,14 +1484,13 @@ get_wal_file(const char *filename, const char *from_fullpath, if (fclose(out) != 0) { - elog(WARNING, "Thread [%d]: Cannot close file '%s': %s", - thread_num, to_fullpath, strerror(errno)); + elog(WARNING, "Cannot close file '%s': %s", to_fullpath, strerror(errno)); unlink(to_fullpath); return false; } - elog(LOG, "Thread [%d]: WAL file successfully %s: %s", - thread_num, prefetch_mode ? "prefetched" : "copied", filename); + elog(LOG, "WAL file successfully %s: %s", + prefetch_mode ? "prefetched" : "copied", filename); return true; } @@ -1525,7 +1505,7 @@ get_wal_file(const char *filename, const char *from_fullpath, */ int get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress, int thread_num) + bool is_decompress) { #ifdef HAVE_LIBZ gzFile gz_in = NULL; @@ -1534,8 +1514,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ int exit_code = 0; - elog(VERBOSE, "Thread [%d]: Attempting to %s WAL file '%s'", - thread_num, is_decompress ? "open compressed" : "open", from_path); + elog(VERBOSE, "Attempting to %s WAL file '%s'", + is_decompress ? "open compressed" : "open", from_path); /* open source file for read */ if (!is_decompress) @@ -1547,8 +1527,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, exit_code = FILE_MISSING; else { - elog(WARNING, "Thread [%d]: Cannot open source WAL file \"%s\": %s", - thread_num, from_path, strerror(errno)); + elog(WARNING, "Cannot open source WAL file \"%s\": %s", + from_path, strerror(errno)); exit_code = OPEN_FAILED; } goto cleanup; @@ -1567,8 +1547,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, exit_code = FILE_MISSING; else { - elog(WARNING, "Thread [%d]: Cannot open compressed WAL file \"%s\": %s", - thread_num, from_path, strerror(errno)); + elog(WARNING, "Cannot open compressed WAL file \"%s\": %s", + from_path, strerror(errno)); exit_code = OPEN_FAILED; } @@ -1593,8 +1573,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, break; else { - elog(WARNING, "Thread [%d]: Cannot read compressed WAL file \"%s\": %s", - thread_num, from_path, get_gz_error(gz_in, errno)); + elog(WARNING, "Cannot read compressed WAL file \"%s\": %s", + from_path, get_gz_error(gz_in, errno)); exit_code = READ_FAILED; break; } @@ -1607,8 +1587,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, if (ferror(in)) { - elog(WARNING, "Thread [%d]: Cannot read source WAL file \"%s\": %s", - thread_num, from_path, strerror(errno)); + elog(WARNING, "Cannot read source WAL file \"%s\": %s", + from_path, strerror(errno)); exit_code = READ_FAILED; break; } @@ -1621,8 +1601,8 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, { if (fwrite(buf, 1, read_len, out) != read_len) { - elog(WARNING, "Thread [%d]: Cannot write to WAL file '%s': %s", - thread_num, to_path, strerror(errno)); + elog(WARNING, "Cannot write to WAL file '%s': %s", + to_path, strerror(errno)); exit_code = WRITE_FAILED; break; } @@ -1663,7 +1643,7 @@ bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefet */ bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, const char *prefetch_dir, const char *absolute_wal_file_path, - uint32 wal_seg_size, bool parse_wal, int thread_num) + uint32 wal_seg_size, bool parse_wal) { char prefetched_file[MAXPGPATH]; @@ -1683,8 +1663,7 @@ bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_ if (parse_wal && !validate_wal_segment(tli, segno, prefetch_dir, wal_seg_size)) { /* prefetched WAL segment is not looking good */ - elog(LOG, "Thread [%d]: Prefetched WAL segment %s is invalid, cannot use it", - thread_num, wal_file_name); + elog(LOG, "Prefetched WAL segment %s is invalid, cannot use it", wal_file_name); unlink(prefetched_file); return false; } @@ -1694,8 +1673,8 @@ bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_ return true; else { - elog(WARNING, "Thread [%d]: Cannot rename file '%s' to '%s': %s", - thread_num, prefetched_file, absolute_wal_file_path, strerror(errno)); + elog(WARNING, "Cannot rename file '%s' to '%s': %s", + prefetched_file, absolute_wal_file_path, strerror(errno)); unlink(prefetched_file); } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9bebc5926..f2aca75fd 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -67,6 +67,9 @@ char *externaldir = NULL; static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; +bool is_archive_cmd = false; +pid_t my_pid = 0; +__thread int my_thread_num = 1; bool progress = false; bool no_sync = false; #if PG_VERSION_NUM >= 100000 @@ -294,6 +297,7 @@ main(int argc, char *argv[]) /* Get current time */ current_time = time(NULL); + my_pid = getpid(); //set_pglocale_pgservice(argv[0], "pgscripts"); #if PG_VERSION_NUM >= 110000 @@ -578,6 +582,7 @@ main(int argc, char *argv[]) backup_subcmd == ARCHIVE_PUSH_CMD)) { instance_config.logger.log_level_file = LOG_OFF; + is_archive_cmd = true; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e24a6d29a..4159553d7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -705,9 +705,12 @@ extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; /* common options */ +extern pid_t my_pid; +extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; extern bool progress; +extern bool is_archive_cmd; /* true for archive-{get,push} */ #if PG_VERSION_NUM >= 100000 /* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; diff --git a/src/utils/logger.c b/src/utils/logger.c index c485af864..5aee41b46 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -192,6 +192,7 @@ elog_internal(int elevel, bool file_only, const char *message) write_to_stderr; time_t log_time = (time_t) time(NULL); char strfbuf[128]; + char str_pid[128]; write_to_file = elevel >= logger_config.log_level_file && logger_config.log_directory @@ -208,10 +209,12 @@ elog_internal(int elevel, bool file_only, const char *message) pthread_lock(&log_file_mutex); loggin_in_progress = true; - if (write_to_file || write_to_error_log) + if (write_to_file || write_to_error_log || is_archive_cmd) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); + snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); + /* * Write message to log file. * Do not write to file if this error was raised during write previous @@ -227,7 +230,8 @@ elog_internal(int elevel, bool file_only, const char *message) open_logfile(&log_file, logger_config.log_filename); } - fprintf(log_file, "%s: ", strfbuf); + fprintf(log_file, "%s ", strfbuf); + fprintf(log_file, "%s ", str_pid); write_elevel(log_file, elevel); fprintf(log_file, "%s\n", message); @@ -244,7 +248,8 @@ elog_internal(int elevel, bool file_only, const char *message) if (error_log_file == NULL) open_logfile(&error_log_file, logger_config.error_log_filename); - fprintf(error_log_file, "%s: ", strfbuf); + fprintf(error_log_file, "%s ", strfbuf); + fprintf(error_log_file, "%s ", str_pid); write_elevel(error_log_file, elevel); fprintf(error_log_file, "%s\n", message); @@ -257,6 +262,17 @@ elog_internal(int elevel, bool file_only, const char *message) */ if (write_to_stderr) { + if (is_archive_cmd) + { + char str_thread[64]; + /* [Issue #213] fix pgbadger parsing */ + snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); + + fprintf(stderr, "%s ", strfbuf); + fprintf(stderr, "%s ", str_pid); + fprintf(stderr, "%s ", str_thread); + } + write_elevel(stderr, elevel); fprintf(stderr, "%s\n", message); From f3248993c501ce0ac9979358bd5deee826eb27d3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 16:58:24 +0300 Subject: [PATCH 1419/2107] [Issue #228] minor fixes in tests --- tests/archive.py | 2 +- tests/helpers/ptrack_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 2e09807ef..01ff5c062 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2242,7 +2242,7 @@ def test_archive_get_prefetch_corruption(self): # at this point replica is consistent restore_command = self.get_restore_command(backup_dir, 'node', replica) - restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' + restore_command += ' -j5 --batch-size=10 --log-level-console=VERBOSE' #restore_command += ' --batch-size=2 --log-level-console=VERBOSE' if node.major_version >= 12: diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bba6cf2a0..19d399d4b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1178,7 +1178,7 @@ def set_archiving( if overwrite: options['archive_command'] += '--overwrite ' - options['archive_command'] += '--log-level-console=verbose ' + options['archive_command'] += '--log-level-console=VERBOSE ' options['archive_command'] += '-j 5 ' options['archive_command'] += '--batch-size 10 ' options['archive_command'] += '--no-sync ' From aed8260bf8df1a2489f64bca453e30c14a19824e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 18:39:01 +0300 Subject: [PATCH 1420/2107] [Issue #213] fix compiling on Windows --- src/pg_probackup.h | 6 ++++++ src/utils/file.c | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4159553d7..8e1420bf0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -39,6 +39,12 @@ #include "datapagemap.h" #include "utils/thread.h" +#ifdef WIN32 +#define __thread __declspec(thread) +#else +#include +#endif + /* pgut client variables and full path */ extern const char *PROGRAM_NAME; extern const char *PROGRAM_NAME_FULL; diff --git a/src/utils/file.c b/src/utils/file.c index 19dc5f01c..3a1afd028 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2,12 +2,6 @@ #include #include -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - #include "pg_probackup.h" #include "file.h" #include "storage/checksum.h" From cf99f6dc77d67c61a5b7b23f0155515bdcd3ea03 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 18:44:18 +0300 Subject: [PATCH 1421/2107] another Window fix --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index b4df5217f..54df90cb6 100644 --- a/src/data.c +++ b/src/data.c @@ -1989,7 +1989,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f FILE *in = NULL; FILE *out = NULL; int hdr_num = -1; - uint cur_pos_out = 0; + off_t cur_pos_out = 0; char curr_page[BLCKSZ]; int n_blocks_read = 0; BlockNumber blknum = 0; From c3cc6a044727599157958a4de3ccd23c911eac76 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 18:57:10 +0300 Subject: [PATCH 1422/2107] replcate truncate with ftruncate --- src/data.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/data.c b/src/data.c index 54df90cb6..a99cf0ff3 100644 --- a/src/data.c +++ b/src/data.c @@ -1792,15 +1792,16 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, char read_buffer[BLCKSZ]; char in_buf[STDIO_BUFSIZE]; - /* truncate up to blocks */ - if (truncate(fullpath, n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", - n_blocks, fullpath, strerror(errno)); - /* open file */ in = fopen(fullpath, PG_BINARY_R); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + + /* truncate up to blocks */ + if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + n_blocks, fullpath, strerror(errno)); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* initialize array of checksums */ @@ -1860,17 +1861,18 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, char in_buf[STDIO_BUFSIZE]; datapagemap_t *lsn_map = NULL; - /* truncate up to blocks */ - if (truncate(fullpath, n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", - n_blocks, fullpath, strerror(errno)); - Assert(shift_lsn > 0); /* open file */ in = fopen(fullpath, PG_BINARY_R); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + + /* truncate up to blocks */ + if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) + elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + n_blocks, fullpath, strerror(errno)); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); lsn_map = pgut_malloc(sizeof(datapagemap_t)); From 9f227cc78f55f89a39fcc00a961a2dc680f31838 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Jun 2020 19:20:49 +0300 Subject: [PATCH 1423/2107] [Issue #229] In 'remote operations' section add requirement about programm version --- doc/pgprobackup.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index f62c68d97..9d3417c2c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1081,6 +1081,13 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; . + + + Operating in remote mode requires pg_probackup + binary to be installed on both local and remote systems. + The versions of local and remote binary must be the same. + + When started in the remote mode, the main pg_probackup process From f5e7e335c4f2a5b1f54899adae69c1d5d8083d31 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 00:15:37 +0300 Subject: [PATCH 1424/2107] [Issue #66] Correctly handle zero-sized nonedata files during incremental restore --- src/data.c | 28 ++++++++++++++++++---------- src/merge.c | 6 +++--- src/restore.c | 23 ++++++++++++++++------- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/data.c b/src/data.c index a99cf0ff3..27e15ab00 100644 --- a/src/data.c +++ b/src/data.c @@ -711,7 +711,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, } /* - * If non-data file exists in previous backup + * If nonedata file exists in previous backup * and its mtime is less than parent backup start time ... */ if (prev_file && file->exists_in_prev && file->mtime <= parent_backup_time) @@ -1136,7 +1136,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during non-data file restore"); + elog(ERROR, "Interrupted during nonedata file restore"); read_len = fread(buf, 1, STDIO_BUFSIZE, in); @@ -1165,7 +1165,6 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pgFile *dest_file, FILE *out, const char *to_fullpath, bool already_exists) { -// int i; char from_root[MAXPGPATH]; char from_fullpath[MAXPGPATH]; FILE *in = NULL; @@ -1203,14 +1202,19 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, */ if (!tmp_file) { - elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", dest_file->rel_path, base36enc(tmp_backup->start_time)); continue; } /* Full copy is found and it is null sized, nothing to do here */ if (tmp_file->write_size == 0) + { + /* In case of incremental restore truncate file just to be safe */ + if (already_exists && fio_ftruncate(out, 0)) + elog(ERROR, "Cannot truncate file \"%s\": %s", strerror(errno)); return 0; + } /* Full copy is found */ if (tmp_file->write_size > 0) @@ -1222,14 +1226,14 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* sanity */ if (!tmp_backup) - elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", + elog(ERROR, "Failed to locate a backup containing full copy of nonedata file \"%s\"", to_fullpath); if (!tmp_file) - elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); + elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of non-data file has invalid size. " + elog(ERROR, "Full copy of nonedata file has invalid size. " "Metadata corruption in backup %s in file: \"%s\"", base36enc(tmp_backup->start_time), to_fullpath); @@ -1245,6 +1249,10 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, to_fullpath); return 0; } + + /* Checksum mismatch, truncate file and overwrite it */ + if (fio_ftruncate(out, 0)) + elog(ERROR, "Cannot truncate file \"%s\": %s", strerror(errno)); } if (tmp_file->external_dir_num == 0) @@ -1264,7 +1272,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - /* disable stdio buffering for non-data files */ + /* disable stdio buffering for nonedata files */ setvbuf(in, NULL, _IONBF, BUFSIZ); /* do actual work */ @@ -1793,7 +1801,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, char in_buf[STDIO_BUFSIZE]; /* open file */ - in = fopen(fullpath, PG_BINARY_R); + in = fopen(fullpath, "r+"); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); @@ -1864,7 +1872,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, Assert(shift_lsn > 0); /* open file */ - in = fopen(fullpath, PG_BINARY_R); + in = fopen(fullpath, "r+"); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); diff --git a/src/merge.c b/src/merge.c index 042931558..7d367461c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1342,7 +1342,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, */ if (!from_file) { - elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", dest_file->rel_path, base36enc(from_backup->start_time)); continue; } @@ -1353,11 +1353,11 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, /* sanity */ if (!from_backup) - elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", + elog(ERROR, "Failed to found a backup containing full copy of nonedata file \"%s\"", dest_file->rel_path); if (!from_file) - elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", dest_file->rel_path); + elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", dest_file->rel_path); /* set path to source file */ if (from_file->external_dir_num) diff --git a/src/restore.c b/src/restore.c index ff554d04f..22e095503 100644 --- a/src/restore.c +++ b/src/restore.c @@ -470,8 +470,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); else - elog(INFO, "Destination directory redo point %X/%X on tli %i is within reach of " - "backup %s with Stop LSN %X/%X on tli %i, incremental restore in 'lsn' mode is possible", + elog(INFO, "Destination directory redo point %X/%X on tli %i is " + "within reach of backup %s with Stop LSN %X/%X on tli %i", (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, base36enc(tmp_backup->start_time), (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, @@ -1118,11 +1118,20 @@ restore_files(void *arg) } } - /* open destination file */ - if (already_exists) - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); - else + /* + * Open dest file and truncate it to zero, if destination + * file already exists and dest file size is zero, or + * if file do not exist + */ + if ((already_exists && dest_file->write_size == 0) || !already_exists) out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); + /* + * If file already exists and dest size is not zero, + * then open it for reading and writing. + */ + else + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + if (out == NULL) elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); @@ -1158,7 +1167,7 @@ restore_files(void *arg) /* disable stdio buffering for local destination nonedata file */ if (!fio_is_remote_file(out)) setvbuf(out, NULL, _IONBF, BUFSIZ); - /* Destination file is non-data file */ + /* Destination file is nonedata file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, arguments->dest_backup, dest_file, out, to_fullpath, already_exists); From 0445bc6e09eead80d38f59204d0584d573f44616 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 00:16:08 +0300 Subject: [PATCH 1425/2107] documentation minor fix --- doc/pgprobackup.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 9d3417c2c..26211b30e 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1753,7 +1753,7 @@ pg_probackup restore -B backup_dir --instance data checksums parameter must be enabled in data directory (to avoid corruption @@ -1762,9 +1762,9 @@ pg_probackup restore -B backup_dir --instance pg_control file must be synched with state of data directory. This condition cannot checked at the start of restore, so it is a user responsibility to ensure - that pg_control contain valid information. Therefore it is not - recommended to use LSN mode in any situation, where pg_control has - been tampered with: + that pg_control contain valid information. + Therefore it is not recommended to use LSN mode in any situation, + where pg_control cannot be trusted or has been tampered with: after pg_resetxlog execution, after restore from backup without recovery been run, etc. @@ -1797,9 +1797,9 @@ pg_probackup restore -B backup_dir --instance Date: Tue, 23 Jun 2020 00:17:22 +0300 Subject: [PATCH 1426/2107] new tests for module 'incr_restore' --- tests/incr_restore.py | 187 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 170 insertions(+), 17 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 859885b1a..ce4e0d37f 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -12,7 +12,7 @@ from testgres import QueryException -module_name = 'restore' +module_name = 'incr_restore' class IncrRestoreTest(ProbackupTest, unittest.TestCase): @@ -967,7 +967,7 @@ def test_incr_lsn_restore_backward(self): print(self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=[ - "-j", "4", '--incremental-mode=lsn', + "-j", "4", '--incremental-mode=lsn', '--log-level-file=VERBOSE', '--recovery-target=immediate', '--recovery-target-action=pause'])) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1310,9 +1310,9 @@ def test_incr_checksum_long_xact(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=checksum'])) + options=["-j", "4", '--incremental-mode=checksum']) node.slow_start() @@ -1452,9 +1452,9 @@ def test_incr_lsn_long_xact_2(self): backup_dir, 'node', node, backup_type="delta", options=["-j", "4", "--stream"]) - print(node.safe_psql( - 'postgres', - "select * from page_header(get_raw_page('t1', 0))")) +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) con.commit() @@ -1463,26 +1463,24 @@ def test_incr_lsn_long_xact_2(self): 'postgres', 'select * from t1') - print(node.safe_psql( - 'postgres', - "select * from page_header(get_raw_page('t1', 0))")) - - print("HELLO") +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) con2.commit() node.safe_psql( 'postgres', 'select * from t1') - print(node.safe_psql( - 'postgres', - "select * from page_header(get_raw_page('t1', 0))")) +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn'])) + options=["-j", "4", '--incremental-mode=lsn']) node.slow_start() @@ -1494,5 +1492,160 @@ def test_incr_lsn_long_xact_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_zero_size_file_checksum(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + fullpath = os.path.join(node.data_dir, 'simple_file') + with open(fullpath, "w", 0) as f: + f.flush() + f.close + + # FULL backup + id1 = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + pgdata1 = self.pgdata_content(node.data_dir) + + with open(fullpath, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + id2 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata2 = self.pgdata_content(node.data_dir) + + with open(fullpath, "w") as f: + f.close() + + id3 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata3 = self.pgdata_content(node.data_dir) + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=id1, + options=["-j", "4", '-I', 'checksum'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + self.restore_node( + backup_dir, 'node', node, backup_id=id2, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.restore_node( + backup_dir, 'node', node, backup_id=id3, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_zero_size_file_lsn(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + fullpath = os.path.join(node.data_dir, 'simple_file') + with open(fullpath, "w", 0) as f: + f.flush() + f.close + + # FULL backup + id1 = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + pgdata1 = self.pgdata_content(node.data_dir) + + with open(fullpath, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + id2 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata2 = self.pgdata_content(node.data_dir) + + with open(fullpath, "w") as f: + f.close() + + id3 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata3 = self.pgdata_content(node.data_dir) + + node.stop() + + print(self.restore_node( + backup_dir, 'node', node, backup_id=id1, + options=["-j", "4", '-I', 'checksum'])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + node.slow_start() + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id2, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + node.slow_start() + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id3, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From 425b0a437a880e4109169b93e4643593efae136c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 00:37:00 +0300 Subject: [PATCH 1427/2107] fix some elog messages --- src/data.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/data.c b/src/data.c index 27e15ab00..4cf99bb39 100644 --- a/src/data.c +++ b/src/data.c @@ -1212,7 +1212,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, { /* In case of incremental restore truncate file just to be safe */ if (already_exists && fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", strerror(errno)); + elog(ERROR, "Cannot truncate file \"%s\": %s", + to_fullpath, strerror(errno)); return 0; } @@ -1233,9 +1234,10 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of nonedata file has invalid size. " + elog(ERROR, "Full copy of nonedata file has invalid size: %li. " "Metadata corruption in backup %s in file: \"%s\"", - base36enc(tmp_backup->start_time), to_fullpath); + tmp_file->write_size, base36enc(tmp_backup->start_time), + to_fullpath); /* incremental restore */ if (already_exists) @@ -1245,14 +1247,15 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (file_crc == tmp_file->crc) { - elog(VERBOSE, "Remote nonedata file \"%s\" is unchanged, skip restore", + elog(VERBOSE, "Already existing nonedata file \"%s\" has the same checksum, skip restore", to_fullpath); return 0; } /* Checksum mismatch, truncate file and overwrite it */ if (fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", strerror(errno)); + elog(ERROR, "Cannot truncate file \"%s\": %s", + to_fullpath, strerror(errno)); } if (tmp_file->external_dir_num == 0) From 5fa64b960985b862d91b317f0e902c21d92280ad Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 12:16:21 +0300 Subject: [PATCH 1428/2107] tests: minor fixes --- tests/backup.py | 7 +------ tests/expected/option_help.out | 7 ++++--- tests/expected/option_version.out | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 361c72807..73eb21022 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -324,7 +324,7 @@ def test_page_detect_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - ptrack_enable=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -333,11 +333,6 @@ def test_page_detect_corruption(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index aaf663536..2170e2773 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -78,14 +78,15 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--recovery-target=immediate|latest] [--recovery-target-name=target-name] [--recovery-target-action=pause|promote|shutdown] - [--restore-as-replica] [--force] + [--restore-command=cmdline] + [-R | --restore-as-replica] [--force] [--primary-conninfo=primary_conninfo] [-S | --primary-slot-name=slotname] [--no-validate] [--skip-block-validation] [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] - [--skip-external-dirs] [--restore-command=cmdline] - [--no-sync] + [--skip-external-dirs] [--no-sync] + [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index abc5c2424..45fae8a96 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.3.5 \ No newline at end of file +pg_probackup 2.4.0 \ No newline at end of file From fe5dd3591eaaf22efca93de485ab4d232158a9a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 12:32:08 +0300 Subject: [PATCH 1429/2107] fix in remote file removing for Windows --- src/restore.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/restore.c b/src/restore.c index 22e095503..fcf6fc7d8 100644 --- a/src/restore.c +++ b/src/restore.c @@ -831,12 +831,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, /* if file does not exists in destination list, then we can safely unlink it */ if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal) == NULL) { - char full_file_path[MAXPGPATH]; + char fullpath[MAXPGPATH]; - join_path_components(full_file_path, pgdata_path, file->rel_path); + join_path_components(fullpath, pgdata_path, file->rel_path); - fio_pgFileDelete(file, full_file_path); - elog(VERBOSE, "Deleted file \"%s\"", full_file_path); +// fio_pgFileDelete(file, full_file_path); + pgFileDelete(file, fullpath); + elog(VERBOSE, "Deleted file \"%s\"", fullpath); /* shrink pgdata list */ parray_remove(pgdata_files, i); From de794064b503fc1ca11df595dd86e7dce1449b37 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 15:31:23 +0300 Subject: [PATCH 1430/2107] fix stdio buffering issue for Windows --- src/data.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/data.c b/src/data.c index 4cf99bb39..3c5ae566a 100644 --- a/src/data.c +++ b/src/data.c @@ -1030,7 +1030,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * go to the next page. */ if (!headers && fseek(in, read_len, SEEK_CUR) != 0) - elog(ERROR, "Cannot seek block %u of '%s': %s", + elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, from_fullpath, strerror(errno)); continue; } @@ -1039,7 +1039,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of '%s': %s", + elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; @@ -1802,6 +1802,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, BlockNumber blknum = 0; char read_buffer[BLCKSZ]; char in_buf[STDIO_BUFSIZE]; + off_t cur_pos = 0; /* open file */ in = fopen(fullpath, "r+"); @@ -1819,16 +1820,32 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); memset(checksum_map, 0, n_blocks * sizeof(PageState)); - for (blknum = 0; blknum < n_blocks; blknum++) + for (;;) { - size_t read_len = fread(read_buffer, 1, BLCKSZ, in); PageState page_st; + size_t read_len = 0; + + if (blknum >= n_blocks) + break; + + if (cur_pos != blknum * BLCKSZ && + fseek(in, blknum * BLCKSZ, SEEK_SET)) + { + elog(ERROR, "Cannot seek to offset %u in file \"%s\": %s", + blknum * BLCKSZ, fullpath, strerror(errno)); + } + + read_len = fread(read_buffer, 1, BLCKSZ, in); + cur_pos += read_len; /* report error */ if (ferror(in)) elog(ERROR, "Cannot read block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); + if (read_len == 0 && feof(in)) + break; + if (read_len == BLCKSZ) { int rc = validate_one_page(read_buffer, segmentno + blknum, @@ -1844,12 +1861,11 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, checksum_map[blknum].lsn = page_st.lsn; } + + blknum++; } else - elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); - - if (feof(in)) - break; + elog(WARNING, "Odd size read len %lu for blknum %u in file \"%s\"", read_len, blknum, fullpath); if (interrupted) elog(ERROR, "Interrupted during page reading"); From 97ef6db6af7731cc7664ff56b963cd0f2016821d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 17:27:22 +0300 Subject: [PATCH 1431/2107] Additional fix for Windows --- src/data.c | 68 +++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/data.c b/src/data.c index 3c5ae566a..d00ae8193 100644 --- a/src/data.c +++ b/src/data.c @@ -31,6 +31,11 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; +typedef struct DataBlock +{ + char data[BLCKSZ]; +} DataBlock; + static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, pg_crc32 *crc, bool use_crc32c); @@ -1792,29 +1797,33 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return is_valid; } -/* read local data file and construct map with block checksums */ +/* read local data file and construct map with block checksums + * bufsize must be divisible by BLCKSZ + */ PageState* get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) + int n_blocks, XLogRecPtr dest_stop_lsn, + BlockNumber segmentno) { PageState *checksum_map = NULL; FILE *in = NULL; BlockNumber blknum = 0; - char read_buffer[BLCKSZ]; - char in_buf[STDIO_BUFSIZE]; - off_t cur_pos = 0; + DataBlock *read_buffer; + int bufsize = LARGE_CHUNK_SIZE; /* open file */ in = fopen(fullpath, "r+"); if (!in) - elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); + + setvbuf(in, NULL, _IONBF, BUFSIZ); /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", n_blocks, fullpath, strerror(errno)); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + read_buffer = pgut_malloc(bufsize); /* initialize array of checksums */ checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); @@ -1822,21 +1831,15 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, for (;;) { + int rc; + int block; PageState page_st; - size_t read_len = 0; - - if (blknum >= n_blocks) - break; + size_t read_len = 0; - if (cur_pos != blknum * BLCKSZ && - fseek(in, blknum * BLCKSZ, SEEK_SET)) - { - elog(ERROR, "Cannot seek to offset %u in file \"%s\": %s", - blknum * BLCKSZ, fullpath, strerror(errno)); - } + if (interrupted) + elog(ERROR, "Interrupted during page reading"); - read_len = fread(read_buffer, 1, BLCKSZ, in); - cur_pos += read_len; + read_len = fread(read_buffer, 1, bufsize, in); /* report error */ if (ferror(in)) @@ -1846,34 +1849,37 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, if (read_len == 0 && feof(in)) break; - if (read_len == BLCKSZ) + for (block = 0; block < read_len / BLCKSZ; block++) { - int rc = validate_one_page(read_buffer, segmentno + blknum, + + if (blknum >= n_blocks) + elog(ERROR, "Concurrent writing to restored cluster detected"); + + rc = validate_one_page(read_buffer[block].data, segmentno + blknum, dest_stop_lsn, &page_st, checksum_version); + /* we care only about valid pages */ if (rc == PAGE_IS_VALID) { - if (checksum_version) - checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; - else - checksum_map[blknum].checksum = page_st.checksum; +// if (checksum_version) +// checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; +// else +// checksum_map[blknum].checksum = page_st.checksum; + checksum_map[blknum].checksum = page_st.checksum; checksum_map[blknum].lsn = page_st.lsn; } blknum++; } - else - elog(WARNING, "Odd size read len %lu for blknum %u in file \"%s\"", read_len, blknum, fullpath); - - if (interrupted) - elog(ERROR, "Interrupted during page reading"); } if (in) fclose(in); + pg_free(read_buffer); + return checksum_map; } @@ -1893,7 +1899,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* open file */ in = fopen(fullpath, "r+"); if (!in) - elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); + elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) From 0aad77e9591fe2f50eeb0e13f8717fb02faebed1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 17:31:52 +0300 Subject: [PATCH 1432/2107] tests: minor fix in "incr_restore" module --- tests/incr_restore.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index ce4e0d37f..14cf45183 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -19,11 +19,12 @@ class IncrRestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_basic_incr_restore(self): - """recovery to target timeline""" + """incremental restore in CHECKSUM mode""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -31,7 +32,7 @@ def test_basic_incr_restore(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=10) + node.pgbench_init(scale=50) self.backup_node(backup_dir, 'node', node) @@ -69,9 +70,9 @@ def test_basic_incr_restore(self): node.stop() - self.restore_node( + print(self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) + options=["-j", "4", "--incremental-mode=checksum"])) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) From c6c43f65b40a37a84f4c97489e01144d17804a37 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 18:06:12 +0300 Subject: [PATCH 1433/2107] Revert "Additional fix for Windows" This reverts commit 97ef6db6af7731cc7664ff56b963cd0f2016821d. --- src/data.c | 68 +++++++++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/src/data.c b/src/data.c index d00ae8193..3c5ae566a 100644 --- a/src/data.c +++ b/src/data.c @@ -31,11 +31,6 @@ typedef struct DataPage char data[BLCKSZ]; } DataPage; -typedef struct DataBlock -{ - char data[BLCKSZ]; -} DataBlock; - static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, pg_crc32 *crc, bool use_crc32c); @@ -1797,33 +1792,29 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, return is_valid; } -/* read local data file and construct map with block checksums - * bufsize must be divisible by BLCKSZ - */ +/* read local data file and construct map with block checksums */ PageState* get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, - BlockNumber segmentno) + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { PageState *checksum_map = NULL; FILE *in = NULL; BlockNumber blknum = 0; - DataBlock *read_buffer; - int bufsize = LARGE_CHUNK_SIZE; + char read_buffer[BLCKSZ]; + char in_buf[STDIO_BUFSIZE]; + off_t cur_pos = 0; /* open file */ in = fopen(fullpath, "r+"); if (!in) - elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); - - setvbuf(in, NULL, _IONBF, BUFSIZ); + elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", n_blocks, fullpath, strerror(errno)); - read_buffer = pgut_malloc(bufsize); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); /* initialize array of checksums */ checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); @@ -1831,15 +1822,21 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, for (;;) { - int rc; - int block; PageState page_st; - size_t read_len = 0; + size_t read_len = 0; - if (interrupted) - elog(ERROR, "Interrupted during page reading"); + if (blknum >= n_blocks) + break; - read_len = fread(read_buffer, 1, bufsize, in); + if (cur_pos != blknum * BLCKSZ && + fseek(in, blknum * BLCKSZ, SEEK_SET)) + { + elog(ERROR, "Cannot seek to offset %u in file \"%s\": %s", + blknum * BLCKSZ, fullpath, strerror(errno)); + } + + read_len = fread(read_buffer, 1, BLCKSZ, in); + cur_pos += read_len; /* report error */ if (ferror(in)) @@ -1849,37 +1846,34 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, if (read_len == 0 && feof(in)) break; - for (block = 0; block < read_len / BLCKSZ; block++) + if (read_len == BLCKSZ) { - - if (blknum >= n_blocks) - elog(ERROR, "Concurrent writing to restored cluster detected"); - - rc = validate_one_page(read_buffer[block].data, segmentno + blknum, + int rc = validate_one_page(read_buffer, segmentno + blknum, dest_stop_lsn, &page_st, checksum_version); - /* we care only about valid pages */ if (rc == PAGE_IS_VALID) { -// if (checksum_version) -// checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; -// else -// checksum_map[blknum].checksum = page_st.checksum; + if (checksum_version) + checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; + else + checksum_map[blknum].checksum = page_st.checksum; - checksum_map[blknum].checksum = page_st.checksum; checksum_map[blknum].lsn = page_st.lsn; } blknum++; } + else + elog(WARNING, "Odd size read len %lu for blknum %u in file \"%s\"", read_len, blknum, fullpath); + + if (interrupted) + elog(ERROR, "Interrupted during page reading"); } if (in) fclose(in); - pg_free(read_buffer); - return checksum_map; } @@ -1899,7 +1893,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* open file */ in = fopen(fullpath, "r+"); if (!in) - elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); + elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) From 339bd6d37cdaa901990e68ec9a415fea4f7f11f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 18:06:21 +0300 Subject: [PATCH 1434/2107] Revert "fix stdio buffering issue for Windows" This reverts commit de794064b503fc1ca11df595dd86e7dce1449b37. --- src/data.c | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/data.c b/src/data.c index 3c5ae566a..4cf99bb39 100644 --- a/src/data.c +++ b/src/data.c @@ -1030,7 +1030,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * go to the next page. */ if (!headers && fseek(in, read_len, SEEK_CUR) != 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", + elog(ERROR, "Cannot seek block %u of '%s': %s", blknum, from_fullpath, strerror(errno)); continue; } @@ -1039,7 +1039,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", + elog(ERROR, "Cannot seek to offset %u of '%s': %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; @@ -1802,7 +1802,6 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, BlockNumber blknum = 0; char read_buffer[BLCKSZ]; char in_buf[STDIO_BUFSIZE]; - off_t cur_pos = 0; /* open file */ in = fopen(fullpath, "r+"); @@ -1820,32 +1819,16 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); memset(checksum_map, 0, n_blocks * sizeof(PageState)); - for (;;) + for (blknum = 0; blknum < n_blocks; blknum++) { + size_t read_len = fread(read_buffer, 1, BLCKSZ, in); PageState page_st; - size_t read_len = 0; - - if (blknum >= n_blocks) - break; - - if (cur_pos != blknum * BLCKSZ && - fseek(in, blknum * BLCKSZ, SEEK_SET)) - { - elog(ERROR, "Cannot seek to offset %u in file \"%s\": %s", - blknum * BLCKSZ, fullpath, strerror(errno)); - } - - read_len = fread(read_buffer, 1, BLCKSZ, in); - cur_pos += read_len; /* report error */ if (ferror(in)) elog(ERROR, "Cannot read block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); - if (read_len == 0 && feof(in)) - break; - if (read_len == BLCKSZ) { int rc = validate_one_page(read_buffer, segmentno + blknum, @@ -1861,11 +1844,12 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, checksum_map[blknum].lsn = page_st.lsn; } - - blknum++; } else - elog(WARNING, "Odd size read len %lu for blknum %u in file \"%s\"", read_len, blknum, fullpath); + elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); + + if (feof(in)) + break; if (interrupted) elog(ERROR, "Interrupted during page reading"); From 173486cf810bbb91bdbe02c56dfa23700773b8f0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 18:11:57 +0300 Subject: [PATCH 1435/2107] Fix fopen flags for Windows --- src/data.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/data.c b/src/data.c index 4cf99bb39..389ff6b6a 100644 --- a/src/data.c +++ b/src/data.c @@ -1030,7 +1030,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * go to the next page. */ if (!headers && fseek(in, read_len, SEEK_CUR) != 0) - elog(ERROR, "Cannot seek block %u of '%s': %s", + elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, from_fullpath, strerror(errno)); continue; } @@ -1039,7 +1039,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of '%s': %s", + elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; @@ -1804,7 +1804,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, char in_buf[STDIO_BUFSIZE]; /* open file */ - in = fopen(fullpath, "r+"); + in = fopen(fullpath, "r+b"); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); @@ -1837,11 +1837,11 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, if (rc == PAGE_IS_VALID) { - if (checksum_version) - checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; - else - checksum_map[blknum].checksum = page_st.checksum; - +// if (checksum_version) +// checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; +// else +// checksum_map[blknum].checksum = page_st.checksum; + checksum_map[blknum].lsn = page_st.checksum; checksum_map[blknum].lsn = page_st.lsn; } } @@ -1875,7 +1875,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, Assert(shift_lsn > 0); /* open file */ - in = fopen(fullpath, "r+"); + in = fopen(fullpath, "r+b"); if (!in) elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); From b6900aa3ff6204258ed0a9ce5c2f89b31efca764 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 18:14:27 +0300 Subject: [PATCH 1436/2107] quick fix --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 389ff6b6a..6fc904ca0 100644 --- a/src/data.c +++ b/src/data.c @@ -1841,7 +1841,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, // checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; // else // checksum_map[blknum].checksum = page_st.checksum; - checksum_map[blknum].lsn = page_st.checksum; + checksum_map[blknum].checksum = page_st.checksum; checksum_map[blknum].lsn = page_st.lsn; } } From 852c99323178b9eca506c0d96dc67e34c6ea4d56 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 18:44:41 +0300 Subject: [PATCH 1437/2107] tests: improve node cleanup in "incr_restore" module --- tests/incr_restore.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 14cf45183..64d9c939e 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -80,7 +80,6 @@ def test_basic_incr_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_checksum_corruption_detection(self): """recovery to target timeline""" @@ -105,7 +104,7 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.backup_node(backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -113,7 +112,7 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.backup_node(backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -121,16 +120,14 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.stop() - # corrupt block - self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=checksum"]) + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -349,7 +346,7 @@ def test_basic_incr_restore_sanity(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") def test_incr_checksum_restore(self): @@ -438,7 +435,7 @@ def test_incr_checksum_restore(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node, node_1]) # @unittest.skip("skip") @@ -527,7 +524,7 @@ def test_incr_lsn_restore(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node, node_1]) # @unittest.skip("skip") def test_incr_lsn_sanity(self): @@ -596,7 +593,7 @@ def test_incr_lsn_sanity(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node_1]) # @unittest.skip("skip") def test_incr_checksum_sanity(self): @@ -655,7 +652,7 @@ def test_incr_checksum_sanity(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node_1]) # @unittest.skip("skip") @@ -1178,7 +1175,7 @@ def test_make_replica_via_incr_checksum_restore(self): pgbench.wait() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [new_master, old_master]) # @unittest.skip("skip") def test_make_replica_via_incr_lsn_restore(self): @@ -1251,7 +1248,7 @@ def test_make_replica_via_incr_lsn_restore(self): pgbench.wait() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [new_master, old_master]) # @unittest.skip("skip") # @unittest.expectedFailure @@ -1492,7 +1489,7 @@ def test_incr_lsn_long_xact_2(self): '1') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [node]) # @unittest.skip("skip") # @unittest.expectedFailure From 6ec34f83441a8b59f92aa135b91907b4e16704b7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 20:06:38 +0300 Subject: [PATCH 1438/2107] fix file delete on Windows --- src/delete.c | 2 +- src/dir.c | 36 ++---------------------------------- src/merge.c | 4 ++-- src/pg_probackup.h | 3 ++- src/restore.c | 2 +- src/utils/file.c | 35 +++++++++++++++++++++++++++++++++++ src/utils/file.h | 1 + 7 files changed, 44 insertions(+), 39 deletions(-) diff --git a/src/delete.c b/src/delete.c index e5424cc1b..b3c50a4b9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -768,7 +768,7 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", i + 1, num_files, full_path); - pgFileDelete(file, full_path); + pgFileDelete(file->mode, full_path); } parray_walk(files, pgFileFree); diff --git a/src/dir.c b/src/dir.c index 2c6dd65da..dfbb6e8c4 100644 --- a/src/dir.c +++ b/src/dir.c @@ -225,9 +225,9 @@ pgFileInit(const char *rel_path) * If the pgFile points directory, the directory must be empty. */ void -pgFileDelete(pgFile *file, const char *full_path) +pgFileDelete(mode_t mode, const char *full_path) { - if (S_ISDIR(file->mode)) + if (S_ISDIR(mode)) { if (rmdir(full_path) == -1) { @@ -252,38 +252,6 @@ pgFileDelete(pgFile *file, const char *full_path) } } -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -fio_pgFileDelete(pgFile *file, const char *full_path) -{ - if (S_ISDIR(file->mode)) - { - if (fio_unlink(full_path, FIO_DB_HOST) == -1) - { - if (errno == ENOENT) - return; - else if (errno == ENOTDIR) /* could be symbolic link */ - goto delete_file; - - elog(ERROR, "Cannot remove directory \"%s\": %s", - full_path, strerror(errno)); - } - return; - } - -delete_file: - if (fio_unlink(full_path, FIO_DB_HOST) == -1) - { - if (errno == ENOENT) - return; - elog(ERROR, "Cannot remove file \"%s\": %s", full_path, - strerror(errno)); - } -} - /* * Read the local file to compute its CRC. * We cannot make decision about file decompression because diff --git a/src/merge.c b/src/merge.c index 7d367461c..296bdf055 100644 --- a/src/merge.c +++ b/src/merge.c @@ -806,7 +806,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* We need full path, file object has relative path */ join_path_components(full_file_path, full_database_dir, full_file->rel_path); - pgFileDelete(full_file, full_file_path); + pgFileDelete(full_file->mode, full_file_path); elog(VERBOSE, "Deleted \"%s\"", full_file_path); } } @@ -1141,7 +1141,7 @@ remove_dir_with_files(const char *path) join_path_components(full_path, path, file->rel_path); - pgFileDelete(file, full_path); + pgFileDelete(file->mode, full_path); elog(VERBOSE, "Deleted \"%s\"", full_path); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8e1420bf0..305dcdd00 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -970,7 +970,7 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location); extern pgFile *pgFileInit(const char *rel_path); -extern void pgFileDelete(pgFile *file, const char *full_path); +extern void pgFileDelete(mode_t mode, const char *full_path); extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); @@ -1123,6 +1123,7 @@ extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const c BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); /* FIO */ +extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers); diff --git a/src/restore.c b/src/restore.c index fcf6fc7d8..2ade54fa8 100644 --- a/src/restore.c +++ b/src/restore.c @@ -836,7 +836,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(fullpath, pgdata_path, file->rel_path); // fio_pgFileDelete(file, full_file_path); - pgFileDelete(file, fullpath); + fio_delete(file->mode, fullpath, FIO_DB_HOST); elog(VERBOSE, "Deleted file \"%s\"", fullpath); /* shrink pgdata list */ diff --git a/src/utils/file.c b/src/utils/file.c index 3a1afd028..db5924a7c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2492,6 +2492,37 @@ static void fio_check_postmaster_impl(int out, char *buf) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } +/* + * Delete file pointed by the pgFile. + * If the pgFile points directory, the directory must be empty. + */ +void +fio_delete(mode_t mode, const char *fullpath, fio_location location) +{ + if (fio_is_remote(location)) + { + fio_header hdr; + + hdr.cop = FIO_DELETE; + hdr.size = strlen(fullpath) + 1; + hdr.arg = mode; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, fullpath, hdr.size), hdr.size); + + } + else + pgFileDelete(mode, fullpath); +} + +static void +fio_delete_impl(mode_t mode, char *buf) +{ + char *fullpath = (char*) buf; + + pgFileDelete(mode, fullpath); +} + /* Execute commands at remote host */ void fio_communicate(int in, int out) { @@ -2675,6 +2706,10 @@ void fio_communicate(int in, int out) /* calculate crc32 for a file */ fio_check_postmaster_impl(out, buf); break; + case FIO_DELETE: + /* delete file */ + fio_delete_impl(hdr.arg, buf); + break; case FIO_DISCONNECT: hdr.cop = FIO_DISCONNECTED; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); diff --git a/src/utils/file.h b/src/utils/file.h index 8302662f3..612bda18f 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -23,6 +23,7 @@ typedef enum FIO_CHMOD, FIO_SEEK, FIO_TRUNCATE, + FIO_DELETE, FIO_PREAD, FIO_READ, FIO_LOAD, From dfcf2d4bf9fa05d4a1a2350594299bf8d2acce5c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 23 Jun 2020 23:07:53 +0300 Subject: [PATCH 1439/2107] tests: fix external tests for Windows --- tests/incr_restore.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 64d9c939e..2a31223e4 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -808,7 +808,8 @@ def test_incr_restore_multiple_external(self): self.set_config( backup_dir, 'node', - options=['-E{0}:{1}'.format(external_dir1, external_dir2)]) + options=['-E{0}{1}{2}'.format( + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) # cmdline option MUST override options in config self.backup_node( @@ -879,7 +880,8 @@ def test_incr_lsn_restore_multiple_external(self): self.set_config( backup_dir, 'node', - options=['-E{0}:{1}'.format(external_dir1, external_dir2)]) + options=['-E{0}{1}{2}'.format( + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) # cmdline option MUST override options in config self.backup_node( From 678237080f0dfaf7d7bdaed782e363463fe2d499 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Jun 2020 22:41:23 +0300 Subject: [PATCH 1440/2107] disable in-place merge if backup versions are different --- src/merge.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/merge.c b/src/merge.c index 296bdf055..a453a073c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -532,23 +532,19 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * If current program version differs from destination backup version, * then in-place merge is not possible. */ - if (parse_program_version(dest_backup->program_version) == - parse_program_version(PROGRAM_VERSION)) + if ((parse_program_version(full_backup->program_version) == + parse_program_version(dest_backup->program_version)) && + (parse_program_version(dest_backup->program_version) == + parse_program_version(PROGRAM_VERSION))) program_version_match = true; else elog(WARNING, "In-place merge is disabled because of program " - "versions mismatch: backup %s was produced by version %s, " - "but current program version is %s", - base36enc(dest_backup->start_time), - dest_backup->program_version, PROGRAM_VERSION); - - /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ - join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); - /* Construct path to external dir: /backup_dir/instance_name/FULL/external */ - join_path_components(full_external_prefix, full_backup->root_dir, EXTERNAL_DIR); - - elog(INFO, "Validate parent chain for backup %s", - base36enc(dest_backup->start_time)); + "versions mismatch. Full backup version: %s, " + "destination backup version: %s, " + "current program version: %s", + full_backup->program_version, + dest_backup->program_version, + PROGRAM_VERSION); /* Forbid merge retry for failed merges between 2.4.0 and any * older version. Several format changes makes it impossible @@ -568,6 +564,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * with sole exception of FULL backup. If it has MERGING status * then it isn't valid backup until merging is finished. */ + elog(INFO, "Validate parent chain for backup %s", + base36enc(dest_backup->start_time)); + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -611,6 +610,11 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name, true); } + /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ + join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); + /* Construct path to external dir: /backup_dir/instance_name/FULL/external */ + join_path_components(full_external_prefix, full_backup->root_dir, EXTERNAL_DIR); + /* Create directories */ create_data_directories(dest_backup->files, full_database_dir, dest_backup->root_dir, false, false, FIO_BACKUP_HOST); @@ -627,6 +631,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) if (full_externals && dest_externals) reorder_external_dirs(full_backup, full_externals, dest_externals); + /* bitmap optimization rely on n_blocks, which is generally available since 2.3.0 */ if (parse_program_version(dest_backup->program_version) < 20300) use_bitmap = false; @@ -981,7 +986,8 @@ merge_files(void *arg) * In-place merge is also impossible, if program version of destination * backup differs from PROGRAM_VERSION */ - if (arguments->program_version_match && arguments->compression_match) + if (arguments->program_version_match && arguments->compression_match && + !arguments->is_retry) { /* * Case 1: @@ -1049,7 +1055,7 @@ merge_files(void *arg) * page header map cannot be trusted when retrying, so no * in place merge for retry. */ - if (in_place && !arguments->is_retry) + if (in_place) { pgFile **res_file = NULL; pgFile *file = NULL; From 5a41524251b688318feb87a7f7a129f3f3cbcc7d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Jun 2020 22:42:35 +0300 Subject: [PATCH 1441/2107] tests: improve test coverage of partial incremental restore case --- tests/compatibility.py | 12 +- tests/external.py | 5 +- tests/incr_restore.py | 339 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 352 insertions(+), 4 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 8cbd61782..d2db2be28 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -359,7 +359,10 @@ def test_backward_compatibility_ptrack(self): node_restored.cleanup() self.restore_node( backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + options=[ + "-j", "4", + "--recovery-target=latest", + "--recovery-target-action=promote"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -384,7 +387,10 @@ def test_backward_compatibility_ptrack(self): self.restore_node( backup_dir, 'node', node_restored, - options=["-j", "4", "--recovery-target-action=promote"]) + options=[ + "-j", "4", + "--recovery-target=latest", + "--recovery-target-action=promote"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -593,7 +599,7 @@ def test_backward_compatibility_merge(self): self.merge_backup(backup_dir, "node", backup_id) - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + self.show_pb(backup_dir, as_text=True, as_json=False) # restore OLD FULL with new binary node_restored = self.make_simple_node( diff --git a/tests/external.py b/tests/external.py index e80935eb7..9d14d7558 100644 --- a/tests/external.py +++ b/tests/external.py @@ -743,8 +743,11 @@ def test_external_merge(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) + print(self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) + # Merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + print(self.merge_backup(backup_dir, 'node', backup_id=backup_id, + options=['--log-level-file=VERBOSE'])) # RESTORE node.cleanup() diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 2a31223e4..9caa479c0 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1647,5 +1647,344 @@ def test_incr_restore_zero_size_file_lsn(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_incremental_partial_restore_exclude_checksum(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + node.pgbench_init(scale=20) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # restore FULL backup into second node2 + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1')) + node1.cleanup() + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + + # restore some data into node2 + self.restore_node(backup_dir, 'node', node2) + + # partial restore backup into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore backup into node2 + print(self.restore_node( + backup_dir, 'node', + node2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-I", "checksum"])) + + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node, node2]) + + def test_incremental_partial_restore_exclude_lsn(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + node.pgbench_init(scale=20) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.stop() + + # restore FULL backup into second node2 + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1')) + node1.cleanup() + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + + # restore some data into node2 + self.restore_node(backup_dir, 'node', node2) + + # partial restore backup into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore backup into node2 + node2.port = node.port + node2.slow_start() + node2.stop() + print(self.restore_node( + backup_dir, 'node', + node2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-I", "lsn"])) + + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node2]) + + def test_incremental_partial_restore_exclude_tablespace_checksum(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # cat_version = node.get_control_data()["Catalog version number"] + # version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version + + # PG_10_201707211 + # pg_tblspc/33172/PG_9.5_201510051/16386/ + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # node1 + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1')) + node1.cleanup() + node1_tablespace = self.get_tblspace_path(node1, 'somedata') + + # node2 + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + node2_tablespace = self.get_tblspace_path(node2, 'somedata') + + # in node2 restore full backup + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + + # partial restore into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node1_tablespace)]) + +# with open(os.path.join(node1_tablespace, "hello"), "w") as f: +# f.close() + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore into node2 + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + + self.set_auto_conf(node2, {'port': node2.port}) + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + # Clean after yourself + self.del_test_dir(module_name, fname, [node2]) + # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From ced96b23874b92d08eb8c193712b05b837d7b6b1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 24 Jun 2020 22:46:11 +0300 Subject: [PATCH 1442/2107] Version 2.4.1 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 305dcdd00..b5bdd5c21 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -276,8 +276,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.0" -#define AGENT_PROTOCOL_VERSION 20400 +#define PROGRAM_VERSION "2.4.1" +#define AGENT_PROTOCOL_VERSION 20401 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 45fae8a96..3a8236d68 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.0 \ No newline at end of file +pg_probackup 2.4.1 \ No newline at end of file From 9622b239c43cf60f2e75bac54c42056610ac2584 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Jun 2020 16:00:17 +0300 Subject: [PATCH 1443/2107] Readme: update --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 813abbcc8..64b58baa6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ The utility is compatible with: As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. +* Incremental restore: page-level incremental restore allows you dramatically speed up restore by reusing valid unchanged pages in destination directory. * Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to do periodical full backups. * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery * Verification: on-demand verification of PostgreSQL instance with the `checkdb` command. @@ -54,7 +55,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## Current release -[2.3.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.5) +[2.4.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.4.1) ## Documentation @@ -62,7 +63,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.5). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.1). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 5ed94447b392a1c5466df5e48399832ce98362f6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Jun 2020 22:58:27 +0300 Subject: [PATCH 1444/2107] bugfix: correctly calculate wal-bytes for STREAM backups --- src/backup.c | 7 ++++++- src/catalog.c | 2 +- src/data.c | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index 18cad5bf3..bbcc04ef7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -642,10 +642,15 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool */ pg_free(file->rel_path); + /* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */ file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path)); + file->name = last_dir_separator(file->rel_path); - /* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */ + if (file->name == NULL) + file->name = file->rel_path; + else + file->name++; } /* Add xlog files into the list of backed up files */ diff --git a/src/catalog.c b/src/catalog.c index 388f31e43..e47f0367b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1880,7 +1880,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, * Size of WAL files in 'pg_wal' is counted separately * TODO: in 3.0 add attribute is_walfile */ - if (IsXLogFileName(file->name) && (file->external_dir_num == 0)) + if (IsXLogFileName(file->name) && file->external_dir_num == 0) wal_size_on_disk += file->write_size; else { diff --git a/src/data.c b/src/data.c index 6fc904ca0..ea61ab3eb 100644 --- a/src/data.c +++ b/src/data.c @@ -2230,6 +2230,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b return headers; } +/* write headers of all blocks belonging to file to header map and + * save its offset and size */ void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge) { From 3963313e31485254de568733c9261322c08b481b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Jun 2020 16:36:14 +0300 Subject: [PATCH 1445/2107] bugfix: message for getting agent version must have the smallest serial number --- src/utils/file.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.h b/src/utils/file.h index 612bda18f..255512f1f 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -12,6 +12,8 @@ typedef enum { + /* message for compatibility check */ + FIO_AGENT_VERSION, /* never move this */ FIO_OPEN, FIO_CLOSE, FIO_WRITE, @@ -50,8 +52,6 @@ typedef enum /* messages for closing connection */ FIO_DISCONNECT, FIO_DISCONNECTED, - /* message for compatibility check */ - FIO_AGENT_VERSION, FIO_LIST_DIR, FIO_CHECK_POSTMASTER } fio_operations; From fb3417868b08c1b98ccf0e4f56b04aae4a96bb9c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Jun 2020 18:36:48 +0300 Subject: [PATCH 1446/2107] Readme: remove current release section --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 64b58baa6..7827f8b64 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,6 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * Remote backup via ssh on Windows currently is not supported. * When running remote operations via ssh, remote and local pg_probackup versions must be the same. -## Current release - -[2.4.1](https://github.com/postgrespro/pg_probackup/releases/tag/2.4.1) - ## Documentation Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup) From c89f87b8db353fa43e8a082934662ae949f57d76 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Jun 2020 22:45:06 +0300 Subject: [PATCH 1447/2107] fix args for fio_send_pages --- src/data.c | 4 +--- src/pg_probackup.h | 7 ++++--- src/utils/file.c | 18 +++++++++--------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/data.c b/src/data.c index ea61ab3eb..14995b462 100644 --- a/src/data.c +++ b/src/data.c @@ -618,9 +618,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* send prev backup START_LSN */ backup_mode == BACKUP_MODE_DIFF_DELTA && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, + calg, clevel, checksum_version, use_pagemap, &headers, backup_mode, ptrack_version_num, ptrack_schema); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b5bdd5c21..8750e79cf 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1124,9 +1124,10 @@ extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const c /* FIO */ extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); -extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, - int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, - BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers); +extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber* err_blknum, char **errormsg, + BackupPageHeader2 **headers); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, diff --git a/src/utils/file.c b/src/utils/file.c index db5924a7c..b29a67070 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1399,10 +1399,10 @@ static void fio_load_file(int out, char const* path) * In case of DELTA mode horizonLsn must be a valid lsn, * otherwise it should be set to InvalidXLogRecPtr. */ -int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, - int calg, int clevel, uint32 checksum_version, - datapagemap_t *pagemap, BlockNumber* err_blknum, - char **errormsg, BackupPageHeader2 **headers) +int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber* err_blknum, char **errormsg, + BackupPageHeader2 **headers) { FILE *out = NULL; char *out_buf = NULL; @@ -1423,10 +1423,10 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f req.hdr.cop = FIO_SEND_PAGES; - if (pagemap) + if (use_pagemap) { - req.hdr.size = sizeof(fio_send_request) + pagemap->bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = pagemap->bitmapsize; + req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; + req.arg.bitmapsize = (*file).pagemap.bitmapsize; /* TODO: add optimization for the case of pagemap * containing small number of blocks with big serial numbers: @@ -1465,8 +1465,8 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); /* send pagemap if any */ - if (pagemap) - IO_CHECK(fio_write_all(fio_stdout, pagemap->bitmap, pagemap->bitmapsize), pagemap->bitmapsize); + if (use_pagemap) + IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); while (true) { From f941a7a68ea8b9ed0d0b56020b38b7ad42c70351 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 Jul 2020 00:26:02 +0300 Subject: [PATCH 1448/2107] add comment about pending improvement of pgFileInit --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index bbcc04ef7..e2293fe4c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -647,7 +647,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool file->name = last_dir_separator(file->rel_path); - if (file->name == NULL) + if (file->name == NULL) // TODO: do it in pgFileInit file->name = file->rel_path; else file->name++; From 7fc70c85f265a9fc888baee81f6588c7bab9d06e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 Jul 2020 00:27:58 +0300 Subject: [PATCH 1449/2107] Version 2.4.2 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8750e79cf..46770e8b7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -276,8 +276,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.1" -#define AGENT_PROTOCOL_VERSION 20401 +#define PROGRAM_VERSION "2.4.2" +#define AGENT_PROTOCOL_VERSION 20402 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 3a8236d68..47c19fef5 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.1 \ No newline at end of file +pg_probackup 2.4.2 \ No newline at end of file From 8db55b42aeece064b01d1400749129866ebc2db9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 1 Jul 2020 03:07:46 +0300 Subject: [PATCH 1450/2107] make compiler happy --- src/data.c | 2 +- src/pg_probackup.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 14995b462..fc3107513 100644 --- a/src/data.c +++ b/src/data.c @@ -607,7 +607,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, calg, clevel, checksum_version, /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, + use_pagemap, /* variables for error reporting */ &err_blknum, &errmsg, &headers); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 46770e8b7..b995be062 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1126,7 +1126,7 @@ extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const c extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg, + bool use_pagemap, BlockNumber *err_blknum, char **errormsg, BackupPageHeader2 **headers); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); From 718e9febf81abba718c0b67c3d8291ddfb63b208 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 3 Jul 2020 14:04:56 +0300 Subject: [PATCH 1451/2107] Readme: add SUSE installation instructions --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7827f8b64..1f9f480b3 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,16 @@ rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclel yum install pg_probackup-{12,11,10,9.6,9.5} yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo -#SRPM Packages +#SRPM Centos|RHEL|OracleLinux Packages yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} +#RPM SUSE|SLES Packages +zypper install --allow-unsigned-rpm -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm +zypper --gpg-auto-import-keys install -y pg_probackup-{12,11,10,9.6,9.5} + +#SRPM SUSE|SLES Packages +zypper si pg_probackup-{12,11,10,9.6,9.5} + #RPM ALT Linux 7 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update From 25309e4d94a18f083e54d8d620ee510d9dc8d2b4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 3 Jul 2020 14:09:54 +0300 Subject: [PATCH 1452/2107] Readme: use https instead http for SUSE repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f9f480b3..dd5df41fb 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} #RPM SUSE|SLES Packages -zypper install --allow-unsigned-rpm -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm +zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm zypper --gpg-auto-import-keys install -y pg_probackup-{12,11,10,9.6,9.5} #SRPM SUSE|SLES Packages From 075346b038a6c66d90834721d4ac070256bbc4ff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 3 Jul 2020 15:11:21 +0300 Subject: [PATCH 1453/2107] Readme: added debuginfo packages for SUSE --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dd5df41fb..e26169f2d 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm zypper --gpg-auto-import-keys install -y pg_probackup-{12,11,10,9.6,9.5} +zypper install pg_probackup-{12,11,10,9.6,9.5}-debuginfo #SRPM SUSE|SLES Packages zypper si pg_probackup-{12,11,10,9.6,9.5} From 814700126dcde27cab33a5c4dcdab606752a5cfc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 6 Jul 2020 14:05:13 +0300 Subject: [PATCH 1454/2107] tests: fix streaming in tests.incr_restore.IncrRestoreTest.test_basic_incr_restore_sanity --- tests/incr_restore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 9caa479c0..3e6829532 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -288,6 +288,7 @@ def test_basic_incr_restore_sanity(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], + set_replication=True, pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 5cd672d4826e765362874dd938f14e843fd5ef32 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 Jul 2020 16:33:07 +0300 Subject: [PATCH 1455/2107] [Issue #224] New flag '--color' for coloring the show plain backup info and console ERROR and WARNING messages --- doc/pgprobackup.xml | 14 +++++- src/backup.c | 2 +- src/catalog.c | 19 +++++++-- src/delete.c | 2 +- src/help.c | 9 +++- src/pg_probackup.c | 6 +++ src/pg_probackup.h | 21 ++++++++- src/show.c | 15 ++++--- src/util.c | 23 ++++++++++ src/utils/logger.c | 101 +++++++++++++++++++++++++++++++++++++++++++- src/utils/logger.h | 1 + tests/show.py | 36 ++++++++++++++++ 12 files changed, 233 insertions(+), 16 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 26211b30e..b1469946e 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3566,7 +3566,7 @@ pg_probackup show-config -B backup_dir --instance show pg_probackup show -B backup_dir -[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] +[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] [--color] Shows the contents of the backup catalog. If @@ -3581,6 +3581,8 @@ pg_probackup show -B backup_dir plain text. You can specify the --format=json option to get the result in the JSON format. + If --color flag is used with plain text format, + then output is colored. For details on usage, see the sections @@ -4603,6 +4605,16 @@ pg_probackup archive-get -B backup_dir --instance + + + + + + Color the console log messages of warning and error levels. + + + + diff --git a/src/backup.c b/src/backup.c index e2293fe4c..2d8fbb467 100644 --- a/src/backup.c +++ b/src/backup.c @@ -843,7 +843,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), + PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t, false), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); diff --git a/src/catalog.c b/src/catalog.c index e47f0367b..939d46749 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -331,9 +331,22 @@ lock_backup(pgBackup *backup, bool strict) * Get backup_mode in string representation. */ const char * -pgBackupGetBackupMode(pgBackup *backup) +pgBackupGetBackupMode(pgBackup *backup, bool show_color) { - return backupModes[backup->backup_mode]; + if (show_color) + { + /* color the Backup mode */ + char *mode = pgut_malloc(24); /* leaking memory here */ + + if (backup->backup_mode == BACKUP_MODE_FULL) + snprintf(mode, 24, "%s%s%s", TC_GREEN_BOLD, backupModes[backup->backup_mode], TC_RESET); + else + snprintf(mode, 24, "%s%s%s", TC_BLUE_BOLD, backupModes[backup->backup_mode], TC_RESET); + + return mode; + } + else + return backupModes[backup->backup_mode]; } static bool @@ -1684,7 +1697,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) char timestamp[100]; fio_fprintf(out, "#Configuration\n"); - fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup, false)); fio_fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); fio_fprintf(out, "compress-alg = %s\n", deparse_compress_alg(backup->compress_alg)); diff --git a/src/delete.c b/src/delete.c index b3c50a4b9..542352aac 100644 --- a/src/delete.c +++ b/src/delete.c @@ -402,7 +402,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* TODO: add ancestor(chain full backup) ID */ elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", base36enc(backup->start_time), - pgBackupGetBackupMode(backup), + pgBackupGetBackupMode(backup, false), status2str(backup->status), cur_full_backup_num, instance_config.retention_redundancy, diff --git a/src/help.c b/src/help.c index 2b5bcd06e..f4f7dce58 100644 --- a/src/help.c +++ b/src/help.c @@ -310,6 +310,7 @@ help_backup(void) printf(_(" (example: --note='backup before app update to v13.1')\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -470,6 +471,7 @@ help_restore(void) printf(_(" -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -536,6 +538,7 @@ help_validate(void) printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -580,6 +583,7 @@ help_checkdb(void) printf(_(" can be used only with '--amcheck' option\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -620,7 +624,8 @@ help_show(void) printf(_(" --instance=instance_name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); printf(_(" --archive show WAL archive information\n")); - printf(_(" --format=format show format=PLAIN|JSON\n\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --color color the info for plain format\n\n")); } static void @@ -655,6 +660,7 @@ help_delete(void) printf(_(" --status=backup_status delete all backups with specified status\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -697,6 +703,7 @@ help_merge(void) printf(_(" --progress show progress\n")); printf(_("\n Logging options:\n")); + printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f2aca75fd..aa2b09f19 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -67,6 +67,7 @@ char *externaldir = NULL; static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; +bool show_color = false; bool is_archive_cmd = false; pid_t my_pid = 0; __thread int my_thread_num = 1; @@ -178,6 +179,7 @@ static ConfigOption cmd_options[] = { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, + { 'b', 134, "color", &show_color, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -440,6 +442,10 @@ main(int argc, char *argv[]) pgut_init(); + /* Check terminal presense and initialize ANSI escape codes for Windows */ + if (show_color) + init_console(); + if (help_opt) help_command(command_name); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b995be062..6cad098fe 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -111,6 +111,22 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +/* Text Coloring macro */ +#define TC_LEN 11 +#define TC_RED "\033[0;31m" +#define TC_RED_BOLD "\033[1;31m" +#define TC_BLUE "\033[0;34m" +#define TC_BLUE_BOLD "\033[1;34m" +#define TC_GREEN "\033[0;32m" +#define TC_GREEN_BOLD "\033[1;32m" +#define TC_YELLOW "\033[0;33m" +#define TC_YELLOW_BOLD "\033[1;33m" +#define TC_MAGENTA "\033[0;35m" +#define TC_MAGENTA_BOLD "\033[1;35m" +#define TC_CYAN "\033[0;36m" +#define TC_CYAN_BOLD "\033[1;36m" +#define TC_RESET "\033[0m" + typedef struct RedoParams { TimeLineID tli; @@ -715,6 +731,7 @@ extern pid_t my_pid; extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; +extern bool show_color; extern bool progress; extern bool is_archive_cmd; /* true for archive-{get,push} */ #if PG_VERSION_NUM >= 100000 @@ -866,7 +883,8 @@ extern void write_backup_status(pgBackup *backup, BackupStatus status, extern void write_backup_data_bytes(pgBackup *backup); extern bool lock_backup(pgBackup *backup, bool strict); -extern const char *pgBackupGetBackupMode(pgBackup *backup); +extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color); +extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode); extern parray *catalog_get_instance_list(void); extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); @@ -1076,6 +1094,7 @@ extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_loc extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); +const char *status2str_color(BackupStatus status); extern BackupStatus str2status(const char *status); extern const char *base36enc(long unsigned int value); extern char *base36enc_dup(long unsigned int value); diff --git a/src/show.c b/src/show.c index 81a16ad64..d8c0cba2e 100644 --- a/src/show.c +++ b/src/show.c @@ -321,7 +321,7 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add_value(buf, "parent-backup-id", base36enc(backup->parent_backup), json_level, true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup, false), json_level, true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", @@ -554,8 +554,8 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na cur++; /* Mode */ - row->mode = pgBackupGetBackupMode(backup); - widths[cur] = Max(widths[cur], strlen(row->mode)); + row->mode = pgBackupGetBackupMode(backup, show_color); + widths[cur] = Max(widths[cur], strlen(row->mode) - (show_color ? TC_LEN : 0)); cur++; /* WAL mode*/ @@ -628,8 +628,9 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na cur++; /* Status */ - row->status = status2str(backup->status); - widths[cur] = Max(widths[cur], strlen(row->status)); + row->status = show_color ? status2str_color(backup->status) : status2str(backup->status); + widths[cur] = Max(widths[cur], strlen(row->status) - (show_color ? TC_LEN : 0)); + } for (i = 0; i < SHOW_FIELDS_COUNT; i++) @@ -679,7 +680,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->recovery_time); cur++; - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), row->mode); cur++; @@ -715,7 +716,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->stop_lsn); cur++; - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), row->status); cur++; diff --git a/src/util.c b/src/util.c index 5ad751df2..0d6974bfb 100644 --- a/src/util.c +++ b/src/util.c @@ -516,6 +516,29 @@ status2str(BackupStatus status) return statusName[status]; } +const char * +status2str_color(BackupStatus status) +{ + char *status_str = pgut_malloc(20); + + /* UNKNOWN */ + if (status == BACKUP_STATUS_INVALID) + snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, "UNKNOWN", TC_RESET); + /* CORRUPT, ERROR and ORPHAN */ + else if (status == BACKUP_STATUS_CORRUPT || status == BACKUP_STATUS_ERROR || + status == BACKUP_STATUS_ORPHAN) + snprintf(status_str, 20, "%s%s%s", TC_RED_BOLD, statusName[status], TC_RESET); + /* MERGING, MERGED, DELETING and DELETED */ + else if (status == BACKUP_STATUS_MERGING || status == BACKUP_STATUS_MERGED || + status == BACKUP_STATUS_DELETING || status == BACKUP_STATUS_DELETED) + snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, statusName[status], TC_RESET); + /* OK and DONE */ + else + snprintf(status_str, 20, "%s%s%s", TC_GREEN_BOLD, statusName[status], TC_RESET); + + return status_str; +} + BackupStatus str2status(const char *status) { diff --git a/src/utils/logger.c b/src/utils/logger.c index 5aee41b46..aad8303dc 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -39,6 +39,10 @@ typedef enum PG_FATAL } eLogType; +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif + void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); static void elog_internal(int elevel, bool file_only, const char *message); @@ -115,6 +119,85 @@ init_logger(const char *root_path, LoggerConfig *config) #endif } +/* + * Check that we are connected to terminal and + * enable ANSI escape codes for Windows if possible + */ +void +init_console(void) +{ + + /* no point in tex coloring if we do not connected to terminal */ + if (!isatty(fileno(stderr)) || + !isatty(fileno(stdout))) + { + show_color = false; + elog(WARNING, "No terminal detected, ignoring '--color' flag"); + return; + } + +#ifdef WIN32 + HANDLE hOut = INVALID_HANDLE_VALUE; + HANDLE hErr = INVALID_HANDLE_VALUE; + DWORD dwMode_out = 0; + DWORD dwMode_err = 0; + + hOut = GetStdHandle(STD_OUTPUT_HANDLE); + if (hOut == INVALID_HANDLE_VALUE || !hOut) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get terminal stdout handle: %s", strerror(errno)); + return; + } + + hErr = GetStdHandle(STD_ERROR_HANDLE); + if (hErr == INVALID_HANDLE_VALUE || !hErr) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get terminal stderror handle: %s", strerror(errno)); + return; + } + + if (!GetConsoleMode(hOut, &dwMode_out)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get console mode for stdout: %s", strerror(errno)); + return; + } + + if (!GetConsoleMode(hErr, &dwMode_err)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get console mode for stderr: %s", strerror(errno)); + return; + } + + /* Add ANSI codes support */ + dwMode_out |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + dwMode_err |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + + if (!SetConsoleMode(hOut, dwMode_out)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Cannot set console mode for stdout: %s", strerror(errno)); + return; + } + + if (!SetConsoleMode(hErr, dwMode_err)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Cannot set console mode for stderr: %s", strerror(errno)); + return; + } +#endif +} + static void write_elevel(FILE *stream, int elevel) { @@ -272,10 +355,26 @@ elog_internal(int elevel, bool file_only, const char *message) fprintf(stderr, "%s ", str_pid); fprintf(stderr, "%s ", str_thread); } + else if (show_color) + { + /* color WARNING and ERROR messages */ + if (elevel == WARNING) + fprintf(stderr, "%s", TC_YELLOW_BOLD); + else if (elevel == ERROR) + fprintf(stderr, "%s", TC_RED_BOLD); + } write_elevel(stderr, elevel); - fprintf(stderr, "%s\n", message); + /* main payload */ + fprintf(stderr, "%s", message); + + /* reset color to default */ + if (show_color && (elevel == WARNING || elevel == ERROR)) + fprintf(stderr, "%s", TC_RESET); + + fprintf(stderr, "\n"); + fflush(stderr); } diff --git a/src/utils/logger.h b/src/utils/logger.h index 37b6ff095..6a7407e41 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -51,6 +51,7 @@ extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void init_logger(const char *root_path, LoggerConfig *config); +extern void init_console(void); extern int parse_log_level(const char *level); extern const char *deparse_log_level(int level); diff --git a/tests/show.py b/tests/show.py index 0da95dcbb..1018ed63d 100644 --- a/tests/show.py +++ b/tests/show.py @@ -538,3 +538,39 @@ def test_corrupt_correctness_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_color_with_no_terminal(self): + """backup.control contains invalid option""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + try: + self.backup_node( + backup_dir, 'node', node, options=['--color', '--archive-timeout=1s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because archiving is disabled\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + '[0m', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 26578edda1391396e63609928329b98f024ea396 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 Jul 2020 16:36:09 +0300 Subject: [PATCH 1456/2107] bump version to 2.5.0 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6cad098fe..f25fdb66d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -292,8 +292,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.2" -#define AGENT_PROTOCOL_VERSION 20402 +#define PROGRAM_VERSION "2.5.0" +#define AGENT_PROTOCOL_VERSION 20500 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 47c19fef5..342842b29 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.2 \ No newline at end of file +pg_probackup 2.5.0 \ No newline at end of file From 383650988f4068b486c3afbaeb4ccca478f194a0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 7 Jul 2020 23:58:28 +0300 Subject: [PATCH 1457/2107] [Issue #224] enable the stdout and stderr coloring by default --- doc/pgprobackup.xml | 10 +++++----- src/help.c | 26 +++++++++++++------------- src/pg_probackup.c | 13 ++++++++----- src/utils/logger.c | 1 - tests/archive.py | 5 +++++ tests/show.py | 2 +- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index b1469946e..844cf5b44 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3566,7 +3566,7 @@ pg_probackup show-config -B backup_dir --instance show pg_probackup show -B backup_dir -[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] [--color] +[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] [--no-color] Shows the contents of the backup catalog. If @@ -3581,8 +3581,8 @@ pg_probackup show -B backup_dir plain text. You can specify the --format=json option to get the result in the JSON format. - If --color flag is used with plain text format, - then output is colored. + If --no-color flag is used, + then the output is not colored. For details on usage, see the sections @@ -4607,10 +4607,10 @@ pg_probackup archive-get -B backup_dir --instance - + - Color the console log messages of warning and error levels. + Disable the coloring for console log messages of warning and error levels. diff --git a/src/help.c b/src/help.c index f4f7dce58..c82c8cf63 100644 --- a/src/help.c +++ b/src/help.c @@ -127,7 +127,7 @@ help_pg_probackup(void) printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); @@ -188,7 +188,7 @@ help_pg_probackup(void) printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); printf(_(" [--instance=instance_name [-i backup-id]]\n")); printf(_(" [--format=format] [--archive]\n")); - printf(_(" [--help]\n")); + printf(_(" [--no-color] [--help]\n")); printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-j num-threads] [--progress]\n")); @@ -273,7 +273,7 @@ help_backup(void) printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); @@ -310,7 +310,6 @@ help_backup(void) printf(_(" (example: --note='backup before app update to v13.1')\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -330,6 +329,7 @@ help_backup(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Retention options:\n")); printf(_(" --delete-expired delete backups expired according to current\n")); @@ -471,7 +471,6 @@ help_restore(void) printf(_(" -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -491,6 +490,7 @@ help_restore(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); @@ -538,7 +538,6 @@ help_validate(void) printf(_(" --skip-block-validation set to validate only file-level checksum\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -557,7 +556,8 @@ help_validate(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void @@ -583,7 +583,6 @@ help_checkdb(void) printf(_(" can be used only with '--amcheck' option\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -603,6 +602,7 @@ help_checkdb(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Connection options:\n")); printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); @@ -625,7 +625,7 @@ help_show(void) printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); printf(_(" --archive show WAL archive information\n")); printf(_(" --format=format show format=PLAIN|JSON\n")); - printf(_(" --color color the info for plain format\n\n")); + printf(_(" --no-color disable the coloring for plain format\n\n")); } static void @@ -660,7 +660,6 @@ help_delete(void) printf(_(" --status=backup_status delete all backups with specified status\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -679,7 +678,8 @@ help_delete(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void @@ -703,7 +703,6 @@ help_merge(void) printf(_(" --progress show progress\n")); printf(_("\n Logging options:\n")); - printf(_(" --color color the error and warning console messages\n")); printf(_(" --log-level-console=log-level-console\n")); printf(_(" level for console logging (default: info)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); @@ -722,7 +721,8 @@ help_merge(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index aa2b09f19..bbc2a5bf0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -67,7 +67,8 @@ char *externaldir = NULL; static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; -bool show_color = false; +bool no_color = false; +bool show_color = true; bool is_archive_cmd = false; pid_t my_pid = 0; __thread int my_thread_num = 1; @@ -179,7 +180,7 @@ static ConfigOption cmd_options[] = { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, - { 'b', 134, "color", &show_color, SOURCE_CMD_STRICT }, + { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -287,6 +288,9 @@ main(int argc, char *argv[]) PROGRAM_NAME_FULL = argv[0]; + /* Check terminal presense and initialize ANSI escape codes for Windows */ + init_console(); + /* Initialize current backup */ pgBackupInit(¤t); @@ -442,9 +446,8 @@ main(int argc, char *argv[]) pgut_init(); - /* Check terminal presense and initialize ANSI escape codes for Windows */ - if (show_color) - init_console(); + if (no_color) + show_color = false; if (help_opt) help_command(command_name); diff --git a/src/utils/logger.c b/src/utils/logger.c index aad8303dc..149f4c62e 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -132,7 +132,6 @@ init_console(void) !isatty(fileno(stdout))) { show_color = false; - elog(WARNING, "No terminal detected, ignoring '--color' flag"); return; } diff --git a/tests/archive.py b/tests/archive.py index 01ff5c062..576a08ab4 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -432,6 +432,11 @@ def test_archive_push_file_exists(self): 'pg_probackup archive-push completed successfully', log_content) + # btw check that console coloring codes are not slipped into log file + self.assertNotIn('[0m', log_content) + + print(log_content) + # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/show.py b/tests/show.py index 1018ed63d..92ef392da 100644 --- a/tests/show.py +++ b/tests/show.py @@ -559,7 +559,7 @@ def test_color_with_no_terminal(self): # FULL try: self.backup_node( - backup_dir, 'node', node, options=['--color', '--archive-timeout=1s']) + backup_dir, 'node', node, options=['--archive-timeout=1s']) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, From 4849767e31715f6c1ee702c8e6c20a800e2cfd55 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 10 Jul 2020 15:51:11 +0300 Subject: [PATCH 1458/2107] DOC: fix typo --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 26211b30e..dbead635b 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -399,7 +399,7 @@ doc/src/sgml/pgprobackup.sgml - On Unix systems, for PostgreSQL 10 or higher, + On Unix systems, for PostgreSQL 10 or lower, a backup can be made only by the same OS user that has started the PostgreSQL server. For example, if PostgreSQL server is started by user postgres, the backup command must also be run From f5879a4864298125ccc0d3c21469ad0a9e660cb7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 17 Jul 2020 12:00:13 +0300 Subject: [PATCH 1459/2107] [Issue #241] for ptrack 1.x compare ptrack_control LSN with previous backup StopLSN --- src/backup.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/backup.c b/src/backup.c index e2293fe4c..f07f88964 100644 --- a/src/backup.c +++ b/src/backup.c @@ -261,13 +261,29 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); - if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) + if (nodeInfo->ptrack_version_num < 20) { - elog(ERROR, "LSN from ptrack_control %X/%X differs from Start LSN of previous backup %X/%X.\n" - "Create new full backup before an incremental one.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (prev_backup->start_lsn >> 32), - (uint32) (prev_backup->start_lsn)); + // backward compatibility kludge: use Stop LSN for ptrack 1.x, + if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control %X/%X differs from Stop LSN of previous backup %X/%X.\n" + "Create new full backup before an incremental one.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (prev_backup->stop_lsn >> 32), + (uint32) (prev_backup->stop_lsn)); + } + } + else + { + // new ptrack is more robust and checks Start LSN + if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) + { + elog(ERROR, "LSN from ptrack_control %X/%X is greater than Start LSN of previous backup %X/%X.\n" + "Create new full backup before an incremental one.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (prev_backup->start_lsn >> 32), + (uint32) (prev_backup->start_lsn)); + } } } From 970bd02b116b12ab4215d3d5672b45916de7a3d7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 17 Jul 2020 12:08:23 +0300 Subject: [PATCH 1460/2107] tests: added tests.ptrack.PtrackTest.test_ptrack_eat_my_data --- tests/ptrack.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index a709afb74..893e84b4c 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -15,6 +15,87 @@ class PtrackTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + def test_ptrack_eat_my_data(self): + """ + PGPRO-4051 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 12: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=50) + + self.backup_node(backup_dir, 'node', node) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) + + for i in range(10): + print("Iteration: {0}".format(i)) + + sleep(2) + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored.cleanup() +# +# self.restore_node(backup_dir, 'node', node_restored) +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# +# self.compare_pgdata(pgdata, pgdata_restored) + + pgbench.terminate() + pgbench.wait() + + self.switch_wal_segment(node) + + result = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + balance = node_restored.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + + self.assertEqual('0', balance) + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql( + 'postgres', + 'SELECT * FROM pgbench_accounts'), + 'Data loss') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_ptrack_simple(self): """make node, make full and ptrack stream backups," From 93f0f5e6cdafa5822383fccd35dced45004d7e93 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 22 Jul 2020 16:35:11 +0300 Subject: [PATCH 1461/2107] Update error message inside test_ptrack_pg_resetxlog --- tests/ptrack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 893e84b4c..cca955ec0 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4029,7 +4029,7 @@ def test_ptrack_pg_resetxlog(self): except ProbackupException as e: self.assertTrue( 'ERROR: LSN from ptrack_control ' in e.message and - 'differs from Start LSN of previous backup' in e.message, + 'is greater than Start LSN of previous backup' in e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) From 801de81b69b936f6ca27c0e085c5cc1b7eb89d0d Mon Sep 17 00:00:00 2001 From: Dmitry Maslyuk Date: Wed, 29 Jul 2020 15:28:25 +0300 Subject: [PATCH 1462/2107] PGPRO-4075: Fixed build rules to improve stability --- Makefile | 100 ++++++++++++++++++++++++------------------------------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 41502286b..4d1e46f64 100644 --- a/Makefile +++ b/Makefile @@ -17,92 +17,78 @@ EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c src/instr_time.h -INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h src/instr_time.h +ifdef top_srcdir +srchome := $(abspath $(top_srcdir)) +else +top_srcdir=../.. +ifneq (,$(wildcard ../../../contrib/pg_probackup)) +# separate build directory support +srchome := $(abspath $(top_srcdir)/..) +else +srchome := $(abspath $(top_srcdir)) +endif +endif + +# OBJS variable must be finally defined before invoking the include directive +ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +endif +ifneq (,$(wildcard $(srchome)/src/bin/pg_rewind/logging.h)) +EXTRA_CLEAN += src/logging.h +endif ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# !USE_PGXS else subdir=contrib/pg_probackup top_builddir=../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk -endif # USE_PGXS - -ifeq ($(top_srcdir),../..) - ifeq ($(LN_S),ln -s) - srchome=$(top_srcdir)/.. - endif -else -srchome=$(top_srcdir) -endif - -#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) -ifneq (12,$(MAJORVERSION)) -EXTRA_CLEAN += src/logging.h -INCLUDES += src/logging.h endif -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -OBJS += src/walmethods.o -EXTRA_CLEAN += src/walmethods.c src/walmethods.h -INCLUDES += src/walmethods.h -endif - - -PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(srchome)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} -all: checksrcdir $(INCLUDES); - -$(PROGRAM): $(OBJS) +src/utils/configuration.o: src/datapagemap.h +src/archive.o: src/instr_time.h +src/backup.o: src/receivelog.h src/streamutil.h +ifneq (,$(wildcard $(srchome)/src/bin/pg_rewind/logging.h)) +src/datapagemap.o: src/logging.h +endif -src/instr_time.h: $(top_srcdir)/src/include/portability/instr_time.h +src/instr_time.h: $(srchome)/src/include/portability/instr_time.h rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@ -src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c +src/datapagemap.c: $(srchome)/src/bin/pg_rewind/datapagemap.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ -src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h +src/datapagemap.h: $(srchome)/src/bin/pg_rewind/datapagemap.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ -src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c +src/pg_crc.c: $(srchome)/src/backend/utils/hash/pg_crc.c rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@ -src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c +src/receivelog.c: $(srchome)/src/bin/pg_basebackup/receivelog.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ -src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h +ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) +src/receivelog.h: src/walmethods.h $(srchome)/src/bin/pg_basebackup/receivelog.h +else +src/receivelog.h: $(srchome)/src/bin/pg_basebackup/receivelog.h +endif rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ -src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c +src/streamutil.c: $(srchome)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ -src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h +src/streamutil.h: $(srchome)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c +src/xlogreader.c: $(srchome)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ - -#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) -ifneq (12,$(MAJORVERSION)) -src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h +src/logging.h: $(srchome)/src/bin/pg_rewind/logging.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ -endif - -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c +src/walmethods.c: $(srchome)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ -src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h +src/walmethods.h: $(srchome)/src/bin/pg_basebackup/walmethods.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ -endif ifeq ($(PORTNAME), aix) CC=xlc_r endif - -# This rule's only purpose is to give the user instructions on how to pass -# the path to PostgreSQL source tree to the makefile. -.PHONY: checksrcdir -checksrcdir: -ifndef top_srcdir - @echo "You must have PostgreSQL source tree available to compile." - @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" - @echo "variable: \"make top_srcdir=\"" - @exit 1 -endif From 29f28e55881c84da01f16264a086cc37794a2223 Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 6 Aug 2020 18:04:52 +0300 Subject: [PATCH 1463/2107] Disable incremental backups with ptrack 1.X, as it is unsafe. Instead of throwing an error, throw a warning and fall back to performing DELTA backup, as we don't want to break existing scripts. --- src/ptrack.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ptrack.c b/src/ptrack.c index 3f2591137..1119a76e5 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -202,9 +202,21 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) else if (strcmp(ptrack_version_str, "2.1") == 0) nodeInfo->ptrack_version_num = 21; else - elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", + elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s", ptrack_version_str); + /* ptrack 1.X is buggy, so fall back to DELTA backup strategy for safety */ + if (nodeInfo->ptrack_version_num >= 15 && nodeInfo->ptrack_version_num < 20) + { + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s. " + "Fall back to DELTA backup.", + ptrack_version_str); + current.backup_mode = BACKUP_MODE_DIFF_DELTA; + } + } + PQclear(res_db); } From b7a44394338d12b12170465aae1b2b64b4dcad40 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 14 Aug 2020 14:47:02 +0300 Subject: [PATCH 1464/2107] [Issue #246] added tests.exclude.ExcludeTest.test_exclude_temp_files --- tests/exclude.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/exclude.py b/tests/exclude.py index c9efe22af..a4fec58a5 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -8,6 +8,45 @@ class ExcludeTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + def test_exclude_temp_files(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + oid = node.safe_psql( + 'postgres', + "select oid from pg_database where datname = 'postgres'").rstrip() + + file = os.path.join(node.data_dir, 'base', oid, 'pgsql_tmp7351.16') + with open(file, 'w') as f: + f.write("HELLO") + f.flush() + f.close + + full_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + file = os.path.join( + backup_dir, 'backups', 'node', full_id, + 'database', 'base', oid, 'pgsql_tmp7351.16') + self.assertFalse(os.path.exists(file)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure def test_exclude_temp_tables(self): From 066bb5027418d5e4df138b2149c54e37deaa481d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 19 Aug 2020 14:05:55 +0300 Subject: [PATCH 1465/2107] tests: added tests.ptrack.PtrackTest.test_ptrack_stop_pg --- tests/helpers/ptrack_helpers.py | 2 +- tests/ptrack.py | 101 ++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 19d399d4b..d87c8212b 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -365,7 +365,7 @@ def make_simple_node( options['max_wal_senders'] = 10 if ptrack_enable: - if node.major_version > 11: + if node.major_version >= 11: options['ptrack.map_size'] = '128' options['shared_preload_libraries'] = 'ptrack' else: diff --git a/tests/ptrack.py b/tests/ptrack.py index cca955ec0..698732788 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -15,6 +15,45 @@ class PtrackTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + def test_ptrack_stop_pg(self): + """ + create node, take full backup, + restart node, check that ptrack backup + can be taken + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 11: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.stop() + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_ptrack_eat_my_data(self): """ @@ -33,7 +72,7 @@ def test_ptrack_eat_my_data(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -113,7 +152,7 @@ def test_ptrack_simple(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -357,7 +396,7 @@ def test_ptrack_enable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -408,7 +447,7 @@ def test_ptrack_disable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -417,7 +456,7 @@ def test_ptrack_disable(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) # DISABLE PTRACK - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql('postgres', "alter system set ptrack.map_size to 0") else: node.safe_psql('postgres', "alter system set ptrack_enable to off") @@ -425,7 +464,7 @@ def test_ptrack_disable(self): node.slow_start() # ENABLE PTRACK - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") else: @@ -478,7 +517,7 @@ def test_ptrack_uncommitted_xact(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -539,7 +578,7 @@ def test_ptrack_vacuum_full(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -629,7 +668,7 @@ def test_ptrack_vacuum_truncate(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -710,7 +749,7 @@ def test_ptrack_get_block(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") node.safe_psql( "postgres", @@ -786,7 +825,7 @@ def test_ptrack_stream(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -879,7 +918,7 @@ def test_ptrack_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1215,7 +1254,7 @@ def test_create_db(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1335,7 +1374,7 @@ def test_create_db_on_replica(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1435,7 +1474,7 @@ def test_alter_table_set_tablespace_ptrack(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1529,7 +1568,7 @@ def test_alter_database_set_tablespace_ptrack(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1601,7 +1640,7 @@ def test_drop_tablespace(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1698,7 +1737,7 @@ def test_ptrack_alter_tablespace(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -1816,7 +1855,7 @@ def test_ptrack_multiple_segments(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2233,7 +2272,7 @@ def test_ptrack_cluster_on_btree(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2298,7 +2337,7 @@ def test_ptrack_cluster_on_gist(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2546,7 +2585,7 @@ def test_ptrack_empty(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2722,7 +2761,7 @@ def test_ptrack_truncate(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2916,7 +2955,7 @@ def test_ptrack_vacuum(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3112,7 +3151,7 @@ def test_ptrack_vacuum_bits_frozen(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3291,7 +3330,7 @@ def test_ptrack_vacuum_bits_visibility(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3370,7 +3409,7 @@ def test_ptrack_vacuum_full(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3553,7 +3592,7 @@ def test_ptrack_vacuum_truncate(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3891,7 +3930,7 @@ def test_ptrack_zero_changes(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3941,7 +3980,7 @@ def test_ptrack_pg_resetxlog(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -4067,7 +4106,7 @@ def test_corrupt_ptrack_map(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: + if node.major_version >= 11: node.safe_psql( "postgres", "CREATE EXTENSION ptrack") From 8b43cae3a834f6f590864719f3d8f13e48a5622f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 24 Aug 2020 19:01:35 +0300 Subject: [PATCH 1466/2107] tests: ptrack2 support for PG11 --- tests/ptrack.py | 275 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 229 insertions(+), 46 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 698732788..1636737a8 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -54,6 +54,161 @@ def test_ptrack_stop_pg(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup(self): + """ + t2 /------P2 + t1 ------F---*-----P1 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 11: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + sleep(15) + + xid = node.safe_psql( + 'postgres', + 'SELECT txid_current()').rstrip() + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + node.cleanup() + + # Restore from full backup to create Timeline 2 + print(self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(xid), + '--recovery-target-action=promote'])) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + + self.assertEqual('0', balance) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup_1(self): + """ + t2 /------ + t1 ---F---P1---* + + # delete P1 + t2 /------P2 + t1 ---F--------* + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 11: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # delete old PTRACK backup + self.delete_pb(backup_dir, 'node', backup_id=ptrack_id) + + # take new PTRACK backup + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + + self.assertEqual('0', balance) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_ptrack_eat_my_data(self): """ @@ -330,7 +485,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - if node.major_version < 12: + if node.major_version < 11: fnames = [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', @@ -1868,7 +2023,7 @@ def test_ptrack_multiple_segments(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) # PTRACK STUFF - if node.major_version < 12: + if node.major_version < 11: idx_ptrack = {'type': 'heap'} idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') @@ -1881,7 +2036,7 @@ def test_ptrack_multiple_segments(self): node.safe_psql("postgres", "checkpoint") - if node.major_version < 12: + if node.major_version < 11: idx_ptrack['new_size'] = self.get_fork_size( node, 'pgbench_accounts') @@ -2299,7 +2454,7 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -2317,7 +2472,7 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) # Clean after yourself @@ -2378,7 +2533,20 @@ def test_ptrack_cluster_on_gist(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - self.check_ptrack_map_sanity(node, idx_ptrack) + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2397,7 +2565,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2459,7 +2627,23 @@ def test_ptrack_cluster_on_btree_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - self.check_ptrack_map_sanity(replica, idx_ptrack) + if master.major_version < 11: + self.check_ptrack_map_sanity(replica, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname) @@ -2477,7 +2661,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2537,13 +2721,13 @@ def test_ptrack_cluster_on_gist_replica(self): master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1') master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist') - if master.major_version < 12: + if master.major_version < 11: master.safe_psql('postgres', 'CHECKPOINT') # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) - if master.major_version < 12: + if master.major_version < 11: replica.safe_psql('postgres', 'CHECKPOINT') self.check_ptrack_map_sanity(replica, idx_ptrack) @@ -2660,15 +2844,14 @@ def test_ptrack_empty_replica(self): set_replication=True, initdb_params=['--data-checksums'], ptrack_enable=True, - pg_options={ - 'autovacuum': 'off'}) + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2777,7 +2960,7 @@ def test_ptrack_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': node.safe_psql( @@ -2793,7 +2976,7 @@ def test_ptrack_truncate(self): node.safe_psql('postgres', 'truncate t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -2810,7 +2993,7 @@ def test_ptrack_truncate(self): pgdata = self.pgdata_content(node.data_dir) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -2840,14 +3023,15 @@ def test_ptrack_truncate_replica(self): pg_options={ 'max_wal_size': '32MB', 'archive_timeout': '10s', - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2885,7 +3069,7 @@ def test_ptrack_truncate_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -2905,7 +3089,7 @@ def test_ptrack_truncate_replica(self): '--master-db=postgres', '--master-port={0}'.format(master.port)]) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -2915,7 +3099,6 @@ def test_ptrack_truncate_replica(self): # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', @@ -2987,7 +3170,7 @@ def test_ptrack_vacuum(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3006,7 +3189,7 @@ def test_ptrack_vacuum(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3044,7 +3227,7 @@ def test_ptrack_vacuum_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3092,7 +3275,7 @@ def test_ptrack_vacuum_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3115,7 +3298,7 @@ def test_ptrack_vacuum_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3184,7 +3367,7 @@ def test_ptrack_vacuum_bits_frozen(self): node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3195,7 +3378,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3230,7 +3413,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3279,7 +3462,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3297,7 +3480,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3360,7 +3543,7 @@ def test_ptrack_vacuum_bits_visibility(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3374,7 +3557,7 @@ def test_ptrack_vacuum_bits_visibility(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3438,7 +3621,7 @@ def test_ptrack_vacuum_full(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3452,7 +3635,7 @@ def test_ptrack_vacuum_full(self): node.safe_psql('postgres', 'vacuum full t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3490,7 +3673,7 @@ def test_ptrack_vacuum_full_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3514,7 +3697,7 @@ def test_ptrack_vacuum_full_replica(self): "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " "tsvector from generate_series(0,256000) i") - if master.major_version < 12: + if master.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': master.safe_psql( @@ -3541,7 +3724,7 @@ def test_ptrack_vacuum_full_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3559,7 +3742,7 @@ def test_ptrack_vacuum_full_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3606,7 +3789,7 @@ def test_ptrack_vacuum_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': node.safe_psql( @@ -3619,7 +3802,7 @@ def test_ptrack_vacuum_truncate(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3634,7 +3817,7 @@ def test_ptrack_vacuum_truncate(self): node.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3670,7 +3853,7 @@ def test_ptrack_vacuum_truncate_replica(self): self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3718,7 +3901,7 @@ def test_ptrack_vacuum_truncate_replica(self): ] ) - if master.major_version < 12: + if master.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3737,7 +3920,7 @@ def test_ptrack_vacuum_truncate_replica(self): replica.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - if master.major_version < 12: + if master.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( From a3de01fcf434a118ea51dc3d475aae69f0acfc47 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 25 Aug 2020 18:27:19 +0300 Subject: [PATCH 1467/2107] DOC: ptrack1 is deprecated --- doc/pgprobackup.xml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index dbead635b..c03688bd7 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1161,6 +1161,14 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; PostgreSQL. Links to PTRACK patches can be found here. + + + PTRACK versions lower than 2.0 are deprecated. Postgres Pro Standard and Postgres Pro Enterprise + versions starting with 11.9.1 contain PTRACK 2.0. Upgrade your server to avoid issues in backups + that you will take in future and be sure to take fresh backups of your clusters with the upgraded + PTRACK since the backups taken with PTRACK 1.x might be corrupt. + + If you are going to use PTRACK backups, complete the following additional steps. The role that will perform PTRACK backups @@ -1168,7 +1176,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; access to all the databases of the cluster. - For PostgreSQL 12 or higher: + For PostgreSQL 11 or higher: @@ -1226,7 +1234,7 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; For older PostgreSQL versions, PTRACK required taking backups in the exclusive mode to provide exclusive access to bitmaps with changed blocks. - To set up PTRACK backups for PostgreSQL 11 + To set up PTRACK backups for PostgreSQL 10 or lower, do the following: From e4e72db8573c4e5fcbb6ea90b082bd51bdd0096c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 26 Aug 2020 12:01:18 +0300 Subject: [PATCH 1468/2107] include ptrack.map into backup --- src/dir.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index dfbb6e8c4..ec94426d5 100644 --- a/src/dir.c +++ b/src/dir.c @@ -703,9 +703,9 @@ dir_check_file(pgFile *file, bool backup_logs) { if (strcmp(file->name, "pg_internal.init") == 0) return CHECK_FALSE; - /* Do not backup ptrack2.x map files */ - else if (strcmp(file->name, "ptrack.map") == 0) - return CHECK_FALSE; + /* Do not backup ptrack2.x temp map files */ +// else if (strcmp(file->name, "ptrack.map") == 0) +// return CHECK_FALSE; else if (strcmp(file->name, "ptrack.map.mmap") == 0) return CHECK_FALSE; else if (strcmp(file->name, "ptrack.map.tmp") == 0) From 6f52d883752799f8e4b852b8936f15f4f42e0b21 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 17:29:49 +0300 Subject: [PATCH 1469/2107] [Issue #251] use correct timeline ID when running instance validation with multitimeline incremental chains --- src/validate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/validate.c b/src/validate.c index d16d27677..678673d0b 100644 --- a/src/validate.c +++ b/src/validate.c @@ -583,7 +583,7 @@ do_validate_instance(void) /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli, + 0, 0, current_backup->tli, instance_config.xlog_seg_size); /* @@ -680,7 +680,7 @@ do_validate_instance(void) /* Revalidation successful, validate corresponding WAL files */ validate_wal(backup, arclog_path, 0, - 0, 0, current_backup->tli, + 0, 0, backup->tli, instance_config.xlog_seg_size); } } From 24c38743976b72ba6dc3347e8ac62ac5b8f19450 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 17:34:35 +0300 Subject: [PATCH 1470/2107] tests: added tests.replica.ReplicaTest.test_replica_switchover --- tests/replica.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/replica.py b/tests/replica.py index de00c195c..9a75a7aa0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -13,6 +13,84 @@ class ReplicaTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_switchover(self): + """ + check that archiving on replica works correctly + over the course of several switchovers + https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru + """ + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + node1.slow_start() + + # take full backup and restore it + self.backup_node(backup_dir, 'node1', node1, options=['--stream']) + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + + # create replica + self.restore_node(backup_dir, 'node1', node2) + + # setup replica + self.add_instance(backup_dir, 'node2', node2) + self.set_archiving(backup_dir, 'node2', node2, replica=True) + self.set_replica(node1, node2, synchronous=False) + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start(replica=True) + + # generate some data + node1.pgbench_init(scale=5) + + # take full backup on replica + self.backup_node(backup_dir, 'node2', node2, options=['--stream']) + + # first switchover + node1.stop() + node2.promote() + + self.set_replica(node2, node1, synchronous=False) + node2.reload() + node1.slow_start(replica=True) + + # take incremental backup from new master + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # second switchover + node2.stop() + node1.promote() + self.set_replica(node1, node2, synchronous=False) + node1.reload() + node2.slow_start(replica=True) + + # generate some more data + node1.pgbench_init(scale=5) + + # take incremental backup from replica + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # https://github.com/postgrespro/pg_probackup/issues/251 + self.validate_pb(backup_dir) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure def test_replica_stream_ptrack_backup(self): From c90b45ae3b0562a674ef9114bb4b002fea278bf8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 20:40:16 +0300 Subject: [PATCH 1471/2107] tests: added tests.merge.MergeTest.test_missing_data_file --- tests/merge.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 3444056d2..26e5d1ea9 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2655,5 +2655,69 @@ def test_merge_page_header_map_retry(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_missing_data_file(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Change data + pgbench = node.pgbench(options=['-T', '5', '-c', '1']) + pgbench.wait() + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").rstrip() + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', path) + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), + logfile_content) + + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From 7b6b782bf64e7c5f7569ab5975cf93677148e183 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 21:02:44 +0300 Subject: [PATCH 1472/2107] tests: added tests.merge.MergeTest.test_missing_non_data_file --- tests/merge.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index 26e5d1ea9..2881a3fdd 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2689,11 +2689,6 @@ def test_missing_data_file(self): 'postgres', "select pg_relation_filepath('pgbench_accounts')").rstrip() - pgdata = self.pgdata_content(node.data_dir) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - gdb = self.merge_backup( backup_dir, "node", delta_id, options=['--log-level-file=VERBOSE'], gdb=True) @@ -2719,5 +2714,63 @@ def test_missing_data_file(self): self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_missing_non_data_file(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', 'backup_label') + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: File "{0}" is not found'.format(file_to_remove), + logfile_content) + + self.assertIn( + 'ERROR: Backup files merging failed', + logfile_content) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From c1ed6ffaba6c63b21737aa98472c73fddde6382c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 22:01:10 +0300 Subject: [PATCH 1473/2107] [Issue #250] tests: added tests.merge.MergeTest.test_merge_remote_mode --- tests/merge.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/merge.py b/tests/merge.py index 2881a3fdd..214a8eef0 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2772,5 +2772,62 @@ def test_missing_non_data_file(self): self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_merge_remote_mode(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + self.set_config(backup_dir, 'node', options=['--retention-window=1']) + + backups = os.path.join(backup_dir, 'backups', 'node') + with open( + os.path.join( + backups, full_id, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=5))) + + gdb = self.backup_node( + backup_dir, "node", node, + options=['--log-level-file=VERBOSE', '--merge-expired'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + + with open(logfile, "rw+") as f: + f.truncate() + + gdb.continue_execution_until_exit() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertNotIn( + 'SSH', logfile_content) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From f2ddb9f352f0abb3b5129acc965136fbc74006f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 Aug 2020 22:03:46 +0300 Subject: [PATCH 1474/2107] [Issue #250] disable remote mode when running retention purge after backup --- src/data.c | 2 +- src/delete.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index fc3107513..8f08f9c9e 100644 --- a/src/data.c +++ b/src/data.c @@ -726,7 +726,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, } backup_non_data_file_internal(from_fullpath, FIO_DB_HOST, - to_fullpath, file, true); + to_fullpath, file, missing_ok); } /* diff --git a/src/delete.c b/src/delete.c index b3c50a4b9..b2960e7a7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -135,6 +135,9 @@ void do_retention(void) backup_deleted = false; backup_merged = false; + /* For now retention is possible only locally */ + MyLocation = FIO_LOCAL_HOST; + /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); From ff2053be211a604b8e2d1c78c31cee3667eeafe4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 31 Aug 2020 14:02:11 +0300 Subject: [PATCH 1475/2107] [Issue #249] added tests.restore.RestoreTest.test_issue_249 --- tests/restore.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index 9c105175e..70de7e357 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3409,3 +3409,73 @@ def test_restore_primary_slot_info(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_issue_249(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE database db1') + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'INSERT INTO pgbench_accounts SELECT * FROM t1') + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored_1')) + node_restored_1.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=["--db-include=db1"]) + + self.set_auto_conf( + node_restored_1, + {'port': node_restored_1.port, 'hot_standby': 'on'}) + + node_restored_1.slow_start() + + node_restored_1.safe_psql( + 'db1', + 'select 1') + + try: + node_restored_1.safe_psql( + 'postgres', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 28dfccd9bbc812ad5dc8022e6539ca3f2271d766 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 31 Aug 2020 14:33:05 +0300 Subject: [PATCH 1476/2107] [Issue #246] fix tests.exclude.ExcludeTest.test_exclude_temp_files --- tests/exclude.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/exclude.py b/tests/exclude.py index a4fec58a5..7148c8b09 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -30,7 +30,11 @@ def test_exclude_temp_files(self): 'postgres', "select oid from pg_database where datname = 'postgres'").rstrip() - file = os.path.join(node.data_dir, 'base', oid, 'pgsql_tmp7351.16') + pgsql_tmp_dir = os.path.join(node.data_dir, 'base', 'pgsql_tmp') + + os.mkdir(pgsql_tmp_dir) + + file = os.path.join(pgsql_tmp_dir, 'pgsql_tmp7351.16') with open(file, 'w') as f: f.write("HELLO") f.flush() @@ -41,8 +45,13 @@ def test_exclude_temp_files(self): file = os.path.join( backup_dir, 'backups', 'node', full_id, - 'database', 'base', oid, 'pgsql_tmp7351.16') - self.assertFalse(os.path.exists(file)) + 'database', 'base', 'pgsql_tmp', 'pgsql_tmp7351.16') + + self.assertFalse( + os.path.exists(file), + "File must be excluded: {0}".format(file)) + + # TODO check temporary tablespaces # Clean after yourself self.del_test_dir(module_name, fname) From 46c7dba3bb2b9e1aac41b38d26510ed75019f058 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 31 Aug 2020 14:58:19 +0300 Subject: [PATCH 1477/2107] [Issue #246] correctly exclude nested directories such as 'pgsql_tmp' --- src/dir.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dir.c b/src/dir.c index ec94426d5..9d7e3701d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -613,8 +613,8 @@ dir_check_file(pgFile *file, bool backup_logs) */ for (i = 0; pgdata_exclude_dir[i]; i++) { - /* relative path exclude */ - if (strcmp(file->rel_path, pgdata_exclude_dir[i]) == 0) + /* exclude by dirname */ + if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) { elog(VERBOSE, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; From 9d8b4d865520ac6ed7420b155d3fe3e76fb4e577 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 2 Sep 2020 19:28:44 +0300 Subject: [PATCH 1478/2107] Extens page bitmap by doubling its size --- .gitignore | 3 -- Makefile | 18 ++------ src/datapagemap.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++ src/datapagemap.h | 29 ++++++++++++ src/util.c | 28 +----------- 5 files changed, 146 insertions(+), 45 deletions(-) create mode 100644 src/datapagemap.c create mode 100644 src/datapagemap.h diff --git a/.gitignore b/.gitignore index 0258e4e2e..6241c238d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,6 @@ # Extra files /src/pg_crc.c -/src/datapagemap.c -/src/datapagemap.h -/src/logging.h /src/receivelog.c /src/receivelog.h /src/streamutil.c diff --git a/Makefile b/Makefile index 4d1e46f64..bc37860be 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ - src/validate.o + src/validate.o src/datapagemap.o # borrowed files -OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ +OBJS += src/pg_crc.o src/receivelog.o src/streamutil.o \ src/xlogreader.o -EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \ +EXTRA_CLEAN = src/pg_crc.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c src/instr_time.h @@ -34,9 +34,6 @@ ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) OBJS += src/walmethods.o EXTRA_CLEAN += src/walmethods.c src/walmethods.h endif -ifneq (,$(wildcard $(srchome)/src/bin/pg_rewind/logging.h)) -EXTRA_CLEAN += src/logging.h -endif ifdef USE_PGXS PG_CONFIG = pg_config @@ -56,16 +53,9 @@ PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} src/utils/configuration.o: src/datapagemap.h src/archive.o: src/instr_time.h src/backup.o: src/receivelog.h src/streamutil.h -ifneq (,$(wildcard $(srchome)/src/bin/pg_rewind/logging.h)) -src/datapagemap.o: src/logging.h -endif src/instr_time.h: $(srchome)/src/include/portability/instr_time.h rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@ -src/datapagemap.c: $(srchome)/src/bin/pg_rewind/datapagemap.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ -src/datapagemap.h: $(srchome)/src/bin/pg_rewind/datapagemap.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ src/pg_crc.c: $(srchome)/src/backend/utils/hash/pg_crc.c rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@ src/receivelog.c: $(srchome)/src/bin/pg_basebackup/receivelog.c @@ -82,8 +72,6 @@ src/streamutil.h: $(srchome)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ src/xlogreader.c: $(srchome)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ -src/logging.h: $(srchome)/src/bin/pg_rewind/logging.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ src/walmethods.c: $(srchome)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ src/walmethods.h: $(srchome)/src/bin/pg_basebackup/walmethods.h diff --git a/src/datapagemap.c b/src/datapagemap.c new file mode 100644 index 000000000..7e4202a72 --- /dev/null +++ b/src/datapagemap.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * datapagemap.c + * A data structure for keeping track of data pages that have changed. + * + * This is a fairly simple bitmap. + * + * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "datapagemap.h" + +struct datapagemap_iterator +{ + datapagemap_t *map; + BlockNumber nextblkno; +}; + +/***** + * Public functions + */ + +/* + * Add a block to the bitmap. + */ +void +datapagemap_add(datapagemap_t *map, BlockNumber blkno) +{ + int offset; + int bitno; + int oldsize = map->bitmapsize; + + offset = blkno / 8; + bitno = blkno % 8; + + /* enlarge or create bitmap if needed */ + if (oldsize <= offset) + { + int newsize; + + /* + * The minimum to hold the new bit is offset + 1. But add some + * headroom, so that we don't need to repeatedly enlarge the bitmap in + * the common case that blocks are modified in order, from beginning + * of a relation to the end. + */ + newsize = (oldsize == 0) ? 16 : oldsize; + while (newsize <= offset) { + newsize <<= 1; + } + + map->bitmap = pg_realloc(map->bitmap, newsize); + + /* zero out the newly allocated region */ + memset(&map->bitmap[oldsize], 0, newsize - oldsize); + + map->bitmapsize = newsize; + } + + /* Set the bit */ + map->bitmap[offset] |= (1 << bitno); +} + +/* + * Start iterating through all entries in the page map. + * + * After datapagemap_iterate, call datapagemap_next to return the entries, + * until it returns false. After you're done, use pg_free() to destroy the + * iterator. + */ +datapagemap_iterator_t * +datapagemap_iterate(datapagemap_t *map) +{ + datapagemap_iterator_t *iter; + + iter = pg_malloc(sizeof(datapagemap_iterator_t)); + iter->map = map; + iter->nextblkno = 0; + + return iter; +} + +bool +datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno) +{ + datapagemap_t *map = iter->map; + + for (;;) + { + BlockNumber blk = iter->nextblkno; + int nextoff = blk / 8; + int bitno = blk % 8; + + if (nextoff >= map->bitmapsize) + break; + + iter->nextblkno++; + + if (map->bitmap[nextoff] & (1 << bitno)) + { + *blkno = blk; + return true; + } + } + + /* no more set bits in this bitmap. */ + return false; +} + diff --git a/src/datapagemap.h b/src/datapagemap.h new file mode 100644 index 000000000..6af54713b --- /dev/null +++ b/src/datapagemap.h @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------- + * + * datapagemap.h + * + * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#ifndef DATAPAGEMAP_H +#define DATAPAGEMAP_H + +#include "storage/relfilenode.h" +#include "storage/block.h" + + +struct datapagemap +{ + char *bitmap; + int bitmapsize; +}; + +typedef struct datapagemap datapagemap_t; +typedef struct datapagemap_iterator datapagemap_iterator_t; + +extern void datapagemap_add(datapagemap_t *map, BlockNumber blkno); +extern datapagemap_iterator_t *datapagemap_iterate(datapagemap_t *map); +extern bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno); + +#endif /* DATAPAGEMAP_H */ diff --git a/src/util.c b/src/util.c index 5ad751df2..946957819 100644 --- a/src/util.c +++ b/src/util.c @@ -538,33 +538,7 @@ datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) offset = blkno / 8; bitno = blkno % 8; - /* enlarge or create bitmap if needed */ - if (map->bitmapsize <= offset) - { - int oldsize = map->bitmapsize; - int newsize; - - /* - * The minimum to hold the new bit is offset + 1. But add some - * headroom, so that we don't need to repeatedly enlarge the bitmap in - * the common case that blocks are modified in order, from beginning - * of a relation to the end. - */ - newsize = offset + 1; - newsize += 10; - - map->bitmap = pg_realloc(map->bitmap, newsize); - - /* zero out the newly allocated region */ - memset(&map->bitmap[oldsize], 0, newsize - oldsize); - - map->bitmapsize = newsize; - } - - //datapagemap_print(map); - - /* check the bit */ - return map->bitmap[offset] & (1 << bitno); + return (map->bitmapsize <= offset) ? false : (map->bitmap[offset] & (1 << bitno)) != 0; } /* From b4c03bb4ad93a1ed710e748ed4d5d42e3e4c1d91 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 6 Sep 2020 11:43:53 +0300 Subject: [PATCH 1479/2107] bump version to 2.4.3 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b995be062..35d3f0669 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -276,8 +276,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.2" -#define AGENT_PROTOCOL_VERSION 20402 +#define PROGRAM_VERSION "2.4.3" +#define AGENT_PROTOCOL_VERSION 20403 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 47c19fef5..f30235a44 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.2 \ No newline at end of file +pg_probackup 2.4.3 \ No newline at end of file From 2002573a3568778cae0806164a9aee108d6d2589 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Sep 2020 21:48:38 +0300 Subject: [PATCH 1480/2107] [Issue #252] additional debug messages --- src/datapagemap.c | 4 ++++ src/parsexlog.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/datapagemap.c b/src/datapagemap.c index 7e4202a72..4d09cb096 100644 --- a/src/datapagemap.c +++ b/src/datapagemap.c @@ -13,6 +13,7 @@ #include "postgres_fe.h" #include "datapagemap.h" +#include "utils/logger.h" struct datapagemap_iterator { @@ -61,6 +62,9 @@ datapagemap_add(datapagemap_t *map, BlockNumber blkno) map->bitmapsize = newsize; } + if (map->bitmapsize > 16384) + elog(WARNING, "Bitmapsize: %u", map->bitmapsize); + /* Set the bit */ map->bitmap[offset] |= (1 << bitno); } diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a33d3045..acded10ff 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1771,6 +1771,10 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, if (forknum != MAIN_FORKNUM) continue; + if (blkno > 131072) + elog(WARNING, "Detecting big blkno %u in record %X/%X", + blkno, (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr)); + process_block_change(forknum, rnode, blkno); } } From d67fe9c2ec12538a617dd2699a8db4400bd8e8d3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Sep 2020 22:43:48 +0300 Subject: [PATCH 1481/2107] [Issue #252] fix for additional debug messages --- src/backup.c | 3 +++ src/parsexlog.c | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index f07f88964..d30d8f7b9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2376,6 +2376,9 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) */ if (file_item) { + if (blkno_inseg > 131072) + elog(WARNING, "Detecting big blkno %u", blkno_inseg); + /* We need critical section only we use more than one threads */ if (num_threads > 1) pthread_lock(&backup_pagemap_mutex); diff --git a/src/parsexlog.c b/src/parsexlog.c index acded10ff..5a33d3045 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1771,10 +1771,6 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, if (forknum != MAIN_FORKNUM) continue; - if (blkno > 131072) - elog(WARNING, "Detecting big blkno %u in record %X/%X", - blkno, (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr)); - process_block_change(forknum, rnode, blkno); } } From 56e093368f09f0c424f7dafafec5d7d308615f39 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Sep 2020 02:15:22 +0300 Subject: [PATCH 1482/2107] [Issue #252] fix memory leak in process_block_change() --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index d30d8f7b9..2448e2dff 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2389,7 +2389,10 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pthread_mutex_unlock(&backup_pagemap_mutex); } + if (segno > 0) + pg_free(f.rel_path); pg_free(rel_path); + } /* From 8cdc7cacb5bb0b0aaae32436e8d60f375a6041b2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Sep 2020 02:15:38 +0300 Subject: [PATCH 1483/2107] Revert "[Issue #252] additional debug messages" This reverts commit 2002573a3568778cae0806164a9aee108d6d2589. --- src/datapagemap.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/datapagemap.c b/src/datapagemap.c index 4d09cb096..7e4202a72 100644 --- a/src/datapagemap.c +++ b/src/datapagemap.c @@ -13,7 +13,6 @@ #include "postgres_fe.h" #include "datapagemap.h" -#include "utils/logger.h" struct datapagemap_iterator { @@ -62,9 +61,6 @@ datapagemap_add(datapagemap_t *map, BlockNumber blkno) map->bitmapsize = newsize; } - if (map->bitmapsize > 16384) - elog(WARNING, "Bitmapsize: %u", map->bitmapsize); - /* Set the bit */ map->bitmap[offset] |= (1 << bitno); } From 1a04646682a082934c39259d857f9da6f640d582 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Sep 2020 02:16:53 +0300 Subject: [PATCH 1484/2107] Revert "[Issue #252] fix for additional debug messages" This reverts commit d67fe9c2ec12538a617dd2699a8db4400bd8e8d3. --- src/backup.c | 3 --- src/parsexlog.c | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 2448e2dff..8a8cb795c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2376,9 +2376,6 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) */ if (file_item) { - if (blkno_inseg > 131072) - elog(WARNING, "Detecting big blkno %u", blkno_inseg); - /* We need critical section only we use more than one threads */ if (num_threads > 1) pthread_lock(&backup_pagemap_mutex); diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a33d3045..acded10ff 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1771,6 +1771,10 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, if (forknum != MAIN_FORKNUM) continue; + if (blkno > 131072) + elog(WARNING, "Detecting big blkno %u in record %X/%X", + blkno, (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr)); + process_block_change(forknum, rnode, blkno); } } From a693fb54afcaebad211dbf452b71cf81471ab807 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 10 Sep 2020 02:18:09 +0300 Subject: [PATCH 1485/2107] [Issue #252] cleanup debug messages --- src/parsexlog.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index acded10ff..5a33d3045 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1771,10 +1771,6 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, if (forknum != MAIN_FORKNUM) continue; - if (blkno > 131072) - elog(WARNING, "Detecting big blkno %u in record %X/%X", - blkno, (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr)); - process_block_change(forknum, rnode, blkno); } } From eca14e02619f1642ad6957eab72c69ab28b382ad Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Sep 2020 11:09:42 +0300 Subject: [PATCH 1486/2107] [Issue #261] PG13 support --- src/backup.c | 13 +++---- src/parsexlog.c | 100 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/src/backup.c b/src/backup.c index 8a8cb795c..6fea8b628 100644 --- a/src/backup.c +++ b/src/backup.c @@ -422,14 +422,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* Extract information about files in backup_list parsing their names:*/ parse_filelist_filenames(backup_files_list, instance_config.pgdata); + elog(LOG, "Current Start LSN: %X/%X, TLI: %X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + current.tli); if (current.backup_mode != BACKUP_MODE_FULL) - { - elog(LOG, "Current tli: %X", current.tli); - elog(LOG, "Parent start_lsn: %X/%X", - (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); - elog(LOG, "start_lsn: %X/%X", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); - } + elog(LOG, "Parent Start LSN: %X/%X, TLI: %X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), + prev_backup->tli); /* * Build page mapping in incremental mode. diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a33d3045..32432e5be 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -148,10 +148,12 @@ typedef struct int ret; } xlog_thread_arg; +static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg); +static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data); + static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI); + int reqLen, XLogRecPtr targetRecPtr, char *readBuf); static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, TimeLineID tli, uint32 segment_size, @@ -551,7 +553,13 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, TimestampTz last_time = 0; char *errormsg; - record = XLogReadRecord(xlogreader, startpoint, &errormsg); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; @@ -615,7 +623,13 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, xlogreader->system_identifier = instance_config.system_identifier; - res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(target_lsn)) + target_lsn = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, target_lsn); +#endif + + res = WalReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ if (errormsg) @@ -656,6 +670,12 @@ get_first_record_lsn(const char *archivedir, XLogSegNo segno, /* Set startpoint to 0 in segno */ GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + while (attempts <= timeout) { record = XLogFindNextRecord(xlogreader, startpoint); @@ -710,6 +730,12 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, /* Set startpoint to 0 in segno */ GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -733,7 +759,7 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, if (interrupted) elog(ERROR, "Interrupted during WAL reading"); - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { @@ -822,6 +848,13 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, XLogRecPtr found; GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); + +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -846,7 +879,7 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, if (interrupted) elog(ERROR, "Interrupted during WAL reading"); - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; @@ -905,8 +938,7 @@ get_gz_error(gzFile gzf) /* XLogreader callback function, to read a WAL page */ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI) + int reqLen, XLogRecPtr targetRecPtr, char *readBuf) { XLogReaderData *reader_data; uint32 targetPageOff; @@ -1040,7 +1072,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, reader_data->prev_page_off == targetPageOff) { memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); - *pageTLI = reader_data->tli; +// *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } @@ -1084,7 +1116,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); reader_data->prev_page_off = targetPageOff; - *pageTLI = reader_data->tli; +// *pageTLI = reader_data->tli; return XLOG_BLCKSZ; } @@ -1109,12 +1141,7 @@ InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, if (allocate_reader) { -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, - reader_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); -#endif + xlogreader = WalReaderAllocate(wal_seg_size, reader_data); if (xlogreader == NULL) elog(ERROR, "Out of memory"); xlogreader->system_identifier = instance_config.system_identifier; @@ -1314,16 +1341,18 @@ XLogThreadWorker(void *arg) uint32 prev_page_off = 0; bool need_read = true; -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, - reader_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); -#endif + xlogreader = WalReaderAllocate(wal_seg_size, reader_data); + if (xlogreader == NULL) elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); xlogreader->system_identifier = instance_config.system_identifier; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(thread_arg->startpoint)) + thread_arg->startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, thread_arg->startpoint); +#endif + found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); /* @@ -1376,7 +1405,7 @@ XLogThreadWorker(void *arg) !SwitchThreadToNextWal(xlogreader, thread_arg)) break; - record = XLogReadRecord(xlogreader, thread_arg->startpoint, &errormsg); + record = WalReadRecord(xlogreader, thread_arg->startpoint, &errormsg); if (record == NULL) { @@ -1857,3 +1886,28 @@ bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_ return rc; } +static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg) +{ + +#if PG_VERSION_NUM >= 130000 + return XLogReadRecord(xlogreader, errormsg); +#else + return XLogReadRecord(xlogreader, startpoint, errormsg); +#endif + +} + +static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data) +{ + +#if PG_VERSION_NUM >= 130000 + return XLogReaderAllocate(wal_seg_size, NULL, + XL_ROUTINE(.page_read = &SimpleXLogPageRead), + reader_data); +#elif PG_VERSION_NUM >= 110000 + return XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); +#else + return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); +#endif +} \ No newline at end of file From c1e52db6519fe9ea035da31b31e07c4d3111e6c5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Sep 2020 11:47:44 +0300 Subject: [PATCH 1487/2107] [Issue #261] fix compatibility with older(<13) PG versions --- src/parsexlog.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 32432e5be..535975239 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -153,7 +153,11 @@ static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *r static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf); + int reqLen, XLogRecPtr targetRecPtr, char *readBuf +#if PG_VERSION_NUM < 130000 + ,TimeLineID *pageTLI +#endif + ); static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, TimeLineID tli, uint32 segment_size, @@ -938,7 +942,11 @@ get_gz_error(gzFile gzf) /* XLogreader callback function, to read a WAL page */ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf) + int reqLen, XLogRecPtr targetRecPtr, char *readBuf +#if PG_VERSION_NUM < 130000 + ,TimeLineID *pageTLI +#endif + ) { XLogReaderData *reader_data; uint32 targetPageOff; @@ -1072,7 +1080,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, reader_data->prev_page_off == targetPageOff) { memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); -// *pageTLI = reader_data->tli; +#if PG_VERSION_NUM < 130000 + *pageTLI = reader_data->tli; +#endif return XLOG_BLCKSZ; } @@ -1116,7 +1126,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); reader_data->prev_page_off = targetPageOff; -// *pageTLI = reader_data->tli; +#if PG_VERSION_NUM < 130000 + *pageTLI = reader_data->tli; +#endif return XLOG_BLCKSZ; } From ee6d0327de537a276d2d6b1b2c31d5f563c57d7e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Sep 2020 11:49:05 +0300 Subject: [PATCH 1488/2107] [Issue #261] fix tests --- tests/archive.py | 2 +- tests/helpers/ptrack_helpers.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index 01ff5c062..0ac0ba191 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2212,7 +2212,7 @@ def test_archive_get_prefetch_corruption(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_keep_segments': '200'}) + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d87c8212b..01a0be83a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -371,6 +371,9 @@ def make_simple_node( else: options['ptrack_enable'] = 'on' + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + # set default values self.set_auto_conf(node, options) From 4db82314621e66c74bca9d10406a25b0a8a12dab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 25 Sep 2020 13:22:15 +0300 Subject: [PATCH 1489/2107] [Issue #260] fix integer overflow --- src/backup.c | 4 ++-- src/checkdb.c | 2 +- src/ptrack.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6fea8b628..1a4437281 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1269,7 +1269,7 @@ get_database_map(PGconn *conn) db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); /* get Oid */ - db_entry->dbOid = atoi(PQgetvalue(res, i, 0)); + db_entry->dbOid = atoll(PQgetvalue(res, i, 0)); /* get datname */ datname = PQgetvalue(res, i, 1); @@ -2644,7 +2644,7 @@ IdentifySystem(StreamThreadArg *stream_thread_arg) } stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); - stream_conn_tli = atoi(PQgetvalue(res, 0, 1)); + stream_conn_tli = atoll(PQgetvalue(res, 0, 1)); /* Additional sanity, primary for PG 9.5, * where system id can be obtained only via "IDENTIFY SYSTEM" diff --git a/src/checkdb.c b/src/checkdb.c index 3a97fc2c7..e8911f7d5 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -439,7 +439,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, char *namespace = NULL; /* index oid */ - ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + ind->indexrelid = atoll(PQgetvalue(res, i, 0)); /* index relname */ name = PQgetvalue(res, i, 1); diff --git a/src/ptrack.c b/src/ptrack.c index 1119a76e5..06ba44eeb 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -284,8 +284,8 @@ pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) if (strcmp(dbname, "template0") == 0) continue; - dbOid = atoi(PQgetvalue(res_db, i, 1)); - tblspcOid = atoi(PQgetvalue(res_db, i, 2)); + dbOid = atoll(PQgetvalue(res_db, i, 1)); + tblspcOid = atoll(PQgetvalue(res_db, i, 2)); tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, dbname, From 6b01210216cec4a42bb6e89bb0e22a7af7aefb2a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 26 Sep 2020 09:27:46 +0300 Subject: [PATCH 1490/2107] tests: use wal_keep_segments in make_simple_node --- tests/helpers/ptrack_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 01a0be83a..8bd56a499 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -373,6 +373,8 @@ def make_simple_node( if node.major_version >= 13: options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '100' # set default values self.set_auto_conf(node, options) From 95a3fbaa1128f285de71bca5a04b8ccef0def3a1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 26 Sep 2020 09:28:48 +0300 Subject: [PATCH 1491/2107] Version 2.4.4 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 35d3f0669..9e9fd01a8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -276,8 +276,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.3" -#define AGENT_PROTOCOL_VERSION 20403 +#define PROGRAM_VERSION "2.4.4" +#define AGENT_PROTOCOL_VERSION 20404 typedef struct ConnectionOptions diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index f30235a44..2d00a86ce 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.3 \ No newline at end of file +pg_probackup 2.4.4 \ No newline at end of file From 1e7ea20e7556551a22d71cb01afa44771c2069cd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 29 Sep 2020 02:26:45 +0300 Subject: [PATCH 1492/2107] Readme: update --- README.md | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e26169f2d..706f92201 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.5, 9.6, 10, 11, 12; +* PostgreSQL 9.5, 9.6, 10, 11, 12, 13; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,8 +41,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* vanilla PostgreSQL 12 with [ptrack extension](https://github.com/postgrespro/ptrack) +* vanilla PostgreSQL 12,13 with [ptrack extension](https://github.com/postgrespro/ptrack) * Postgres Pro Standard 9.6, 10, 11, 12 * Postgres Pro Enterprise 9.6, 10, 11, 12 @@ -59,7 +58,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.1). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.4). ### Linux Installation #### pg_probackup for vanilla PostgreSQL @@ -67,57 +66,57 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg-probackup-{12,11,10,9.6,9.5}-dbg +sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5} +sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}-dbg #DEB-SRC Packages sudo echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update -sudo apt-get source pg-probackup-{12,11,10,9.6,9.5} +sudo apt-get source pg-probackup-{13,12,11,10,9.6,9.5} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{13,12,11,10,9.6,9.5} +yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{13,12,11,10,9.6,9.5} +yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{13,12,11,10,9.6,9.5} +yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} +yumdownloader --source pg_probackup-{13,12,11,10,9.6,9.5} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{12,11,10,9.6,9.5} -zypper install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{13,12,11,10,9.6,9.5} +zypper install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{12,11,10,9.6,9.5} +zypper si pg_probackup-{13,12,11,10,9.6,9.5} #RPM ALT Linux 7 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux 8 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux 9 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} +sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise From 5fb52348a749f27dc8f737f144c62eaa6f50fbe3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 29 Sep 2020 13:29:36 +0300 Subject: [PATCH 1493/2107] fix checkdb integer overflow --- src/checkdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/checkdb.c b/src/checkdb.c index e8911f7d5..c994a6692 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -479,7 +479,7 @@ amcheck_one_index(check_indexes_arg *arguments, params[0] = palloc(64); /* first argument is index oid */ - sprintf(params[0], "%i", ind->indexrelid); + sprintf(params[0], "%u", ind->indexrelid); /* second argument is heapallindexed */ params[1] = heapallindexed ? "true" : "false"; From a1f576752d6b898014dd12de38bcc07e29d3723c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 19 Oct 2020 18:57:48 +0300 Subject: [PATCH 1494/2107] tests: kludge for testgres, which currently does not support PG13 --- tests/helpers/ptrack_helpers.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 8bd56a499..b0b7ceb5d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -382,6 +382,13 @@ def make_simple_node( # Apply given parameters self.set_auto_conf(node, pg_options) + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + self.set_auto_conf( + node, {}, 'postgresql.conf', ['wal_keep_segments']) + return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): @@ -1231,7 +1238,9 @@ def get_restore_command(self, backup_dir, instance, node): return restore_command - def set_auto_conf(self, node, options, config='postgresql.auto.conf'): + # rm_options - list of parameter name that should be deleted from current config, + # example: ['wal_keep_segments', 'max_wal_size'] + def set_auto_conf(self, node, options, config='postgresql.auto.conf', rm_options={}): # parse postgresql.auto.conf path = os.path.join(node.data_dir, config) @@ -1259,6 +1268,11 @@ def set_auto_conf(self, node, options, config='postgresql.auto.conf'): var = var.strip() var = var.strip('"') var = var.strip("'") + + # remove options specified in rm_options list + if name in rm_options: + continue + current_options[name] = var for option in options: From 6a28319875744dbf4ad05c71ee2f3a553e7e2761 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Oct 2020 21:37:34 +0300 Subject: [PATCH 1495/2107] tests: rename test_ptrack_truncate_replica to test_basic_ptrack_truncate_replica --- tests/ptrack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 1636737a8..9004c95be 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3013,7 +3013,7 @@ def test_ptrack_truncate(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_ptrack_truncate_replica(self): + def test_basic_ptrack_truncate_replica(self): fname = self.id().split('.')[3] master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), @@ -3121,7 +3121,7 @@ def test_ptrack_truncate_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, fname, [master, replica]) # @unittest.skip("skip") # @unittest.expectedFailure From 6ecfa8e15f5c59a0d1cb870b9042c82a506edf01 Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Fri, 23 Oct 2020 17:19:28 +0700 Subject: [PATCH 1496/2107] Disable autovacuum for correct checksum comparing --- tests/archive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/archive.py b/tests/archive.py index 0ac0ba191..358541b34 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -899,6 +899,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', + 'autovacuum': 'off', 'archive_timeout': '10s'}) replica = self.make_simple_node( From 98372aa8c493abb4bab7bba2b330bda980c3a306 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 27 Oct 2020 21:10:04 +0300 Subject: [PATCH 1497/2107] tests: honor the paranoia flag in tests.ptrack.PtrackTest.test_basic_ptrack_truncate_replica --- tests/ptrack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 9004c95be..b54006201 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3118,7 +3118,9 @@ def test_basic_ptrack_truncate_replica(self): self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself self.del_test_dir(module_name, fname, [master, replica]) From 585f1ab5d337f92130ab8e06c134540736f8bb6c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Oct 2020 23:07:52 +0300 Subject: [PATCH 1498/2107] [Issue #271] when updating backup status overwrite control file only if status has changed --- src/catalog.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/catalog.c b/src/catalog.c index e47f0367b..f88d90b56 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -120,6 +120,13 @@ write_backup_status(pgBackup *backup, BackupStatus status, return; } + /* overwrite control file only if status has changed */ + if (backup->status == status) + { + pgBackupFree(tmp); + return; + } + backup->status = status; tmp->status = backup->status; tmp->root_dir = pgut_strdup(backup->root_dir); From e96d84caa6d9dacf0c9e510ca953ddccbb42ec8e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Oct 2020 23:10:45 +0300 Subject: [PATCH 1499/2107] add .vscode directory to .gitignore --- .gitignore | 1 + src/validate.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6241c238d..474df1c73 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ # Misc .python-version +.vscode diff --git a/src/validate.c b/src/validate.c index 678673d0b..8159881a5 100644 --- a/src/validate.c +++ b/src/validate.c @@ -190,6 +190,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) /* Update backup status */ if (corrupted) backup->status = BACKUP_STATUS_CORRUPT; + write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : BACKUP_STATUS_OK, instance_name, true); @@ -218,7 +219,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); } - } } From 3df428871df98858db7694af402aa866f1dba1a8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 29 Oct 2020 11:05:07 +0300 Subject: [PATCH 1500/2107] [Issue #271] minor fix --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index f88d90b56..a05626ce1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -121,7 +121,7 @@ write_backup_status(pgBackup *backup, BackupStatus status, } /* overwrite control file only if status has changed */ - if (backup->status == status) + if (tmp->status == status) { pgBackupFree(tmp); return; From e8def1d344ecf85c27229f2993e32710eed3528e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 02:33:21 +0300 Subject: [PATCH 1501/2107] check for interrupt when running show --- src/show.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/show.c b/src/show.c index 81a16ad64..ce6604ac3 100644 --- a/src/show.c +++ b/src/show.c @@ -101,6 +101,9 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive InstanceConfig *instance = parray_get(instances, i); char backup_instance_path[MAXPGPATH]; + if (interrupted) + elog(ERROR, "Interrupted during show"); + sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance->name); if (show_archive) From b4933f6c36cf19e121317c5ebc199ddb80be06c6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 02:33:45 +0300 Subject: [PATCH 1502/2107] reset errno after pg_logging_init --- src/utils/logger.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/logger.c b/src/utils/logger.c index 5aee41b46..719ede436 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -92,6 +92,7 @@ init_logger(const char *root_path, LoggerConfig *config) #if PG_VERSION_NUM >= 120000 /* Setup logging for functions from other modules called by pg_probackup */ pg_logging_init(PROGRAM_NAME); + errno = 0; /* sometimes pg_logging_init sets errno */ switch (logger_config.log_level_console) { From 842e6ce0690e116f43cedf1a2354d6389fda9b74 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 02:38:39 +0300 Subject: [PATCH 1503/2107] fix race condition in atexit callbacks for connection canceling --- src/checkdb.c | 1 + src/utils/pgut.c | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/checkdb.c b/src/checkdb.c index c994a6692..4ea1d0800 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -292,6 +292,7 @@ check_indexes(void *arg) int i; check_indexes_arg *arguments = (check_indexes_arg *) arg; int n_indexes = 0; + my_thread_num = arguments->thread_num; if (arguments->index_list) n_indexes = parray_num(arguments->index_list); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6d996f47f..27cfe6d1f 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -35,6 +35,9 @@ bool interrupted = false; bool in_cleanup = false; bool in_password = false; +/* critical section when adding disconnect callbackups */ +static pthread_mutex_t atexit_callback_disconnect_mutex = PTHREAD_MUTEX_INITIALIZER; + /* Connection routines */ static void init_cancel_handler(void); static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); @@ -48,6 +51,7 @@ static void pgut_pgfnames_cleanup(char **filenames); void discard_response(PGconn *conn); +/* Note that atexit handlers always called on the main thread */ void pgut_init(void) { @@ -237,7 +241,9 @@ pgut_connect(const char *host, const char *port, if (PQstatus(conn) == CONNECTION_OK) { + pthread_lock(&atexit_callback_disconnect_mutex); pgut_atexit_push(pgut_disconnect_callback, conn); + pthread_mutex_unlock(&atexit_callback_disconnect_mutex); return conn; } @@ -365,7 +371,10 @@ pgut_disconnect(PGconn *conn) { if (conn) PQfinish(conn); + + pthread_lock(&atexit_callback_disconnect_mutex); pgut_atexit_pop(pgut_disconnect_callback, conn); + pthread_mutex_unlock(&atexit_callback_disconnect_mutex); } @@ -840,7 +849,9 @@ call_atexit_callbacks(bool fatal) { pgut_atexit_item *item; pgut_atexit_item *next; - for (item = pgut_atexit_stack; item; item = next){ + + for (item = pgut_atexit_stack; item; item = next) + { next = item->next; item->callback(fatal, item->userdata); } From 3c00972bd412f7c6a759dcc9978d575e7d39addb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 02:47:06 +0300 Subject: [PATCH 1504/2107] tests: move to python3 --- tests/Readme.md | 2 +- tests/archive.py | 26 ++++++++++++++++++------- tests/backup.py | 27 ++++++++++++++------------ tests/checkdb.py | 11 ++++++----- tests/delta.py | 6 +++--- tests/exclude.py | 4 ++-- tests/external.py | 19 +++++++++--------- tests/helpers/ptrack_helpers.py | 30 ++++++++++++++++++----------- tests/incr_restore.py | 12 ++++++------ tests/logging.py | 2 +- tests/merge.py | 14 +++++++------- tests/pgpro2068.py | 4 ++-- tests/restore.py | 9 ++++----- tests/retention.py | 16 +++++++++------- tests/set_backup.py | 2 +- tests/validate.py | 34 +++++++++++++++++---------------- 16 files changed, 123 insertions(+), 95 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index c1dd9a63d..3adf0c019 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -34,7 +34,7 @@ Run ptrack tests: Usage: - pip install testgres==1.8.2 + pip install testgres export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` diff --git a/tests/archive.py b/tests/archive.py index 358541b34..78aa6d045 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -316,7 +316,7 @@ def test_pgpro434_4(self): "postgres", "SELECT pid " "FROM pg_stat_activity " - "WHERE application_name = 'pg_probackup'").rstrip() + "WHERE application_name = 'pg_probackup'").decode('utf-8').rstrip() os.environ["PGAPPNAME"] = "pg_probackup" @@ -371,7 +371,7 @@ def test_archive_push_file_exists(self): filename = '000000010000000000000001' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") f.flush() f.close() @@ -458,7 +458,7 @@ def test_archive_push_file_exists_overwrite(self): filename = '000000010000000000000001' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") f.flush() f.close() @@ -538,7 +538,7 @@ def test_archive_push_partial_file_exists(self): xid = node.safe_psql( "postgres", - "INSERT INTO t1 VALUES (1) RETURNING (xmin)").rstrip() + "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() if self.get_version(node) < 100000: filename_orig = node.safe_psql( @@ -551,6 +551,8 @@ def test_archive_push_partial_file_exists(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = filename_orig.decode('utf-8') + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: @@ -561,7 +563,7 @@ def test_archive_push_partial_file_exists(self): file = os.path.join(wals_dir, filename) # emulate stale .part file - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblah") f.flush() f.close() @@ -628,6 +630,8 @@ def test_archive_push_part_file_exists_not_stale(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = filename_orig.decode('utf-8') + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: @@ -637,7 +641,7 @@ def test_archive_push_part_file_exists_not_stale(self): filename = filename_orig + '.part' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblah") f.flush() f.close() @@ -645,7 +649,7 @@ def test_archive_push_part_file_exists_not_stale(self): self.switch_wal_segment(node) sleep(30) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblahblahblah") f.flush() f.close() @@ -2378,6 +2382,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2400,6 +2406,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2422,6 +2430,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2444,6 +2454,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( diff --git a/tests/backup.py b/tests/backup.py index 73eb21022..957231cb7 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -349,7 +349,7 @@ def test_page_detect_corruption(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() path = os.path.join(node.data_dir, heap_path) with open(path, "rb+", 0) as f: @@ -416,7 +416,7 @@ def test_backup_detect_corruption(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -558,7 +558,7 @@ def test_backup_detect_invalid_block_header(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -699,7 +699,7 @@ def test_backup_detect_missing_permissions(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -840,7 +840,7 @@ def test_backup_truncate_misaligned(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() heap_size = node.safe_psql( "postgres", @@ -922,7 +922,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): relfilenode = node.safe_psql( "postgres", "select 't_heap1'::regclass::oid" - ).rstrip() + ).decode('utf-8').rstrip() list = [] for root, dirs, files in os.walk(os.path.join( @@ -1205,11 +1205,11 @@ def test_drop_rel_during_full_backup(self): relative_path_1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() relative_path_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() absolute_path_1 = os.path.join(node.data_dir, relative_path_1) absolute_path_2 = os.path.join(node.data_dir, relative_path_2) @@ -1350,7 +1350,7 @@ def test_drop_rel_during_backup_delta(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() absolute_path = os.path.join(node.data_dir, relative_path) @@ -1418,7 +1418,7 @@ def test_drop_rel_during_backup_page(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() absolute_path = os.path.join(node.data_dir, relative_path) @@ -1443,6 +1443,7 @@ def test_drop_rel_during_backup_page(self): # File removed, we can proceed with backup gdb.continue_execution_until_exit() + gdb.kill() pgdata = self.pgdata_content(node.data_dir) @@ -1492,7 +1493,7 @@ def test_drop_rel_during_backup_ptrack(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() absolute_path = os.path.join(node.data_dir, relative_path) @@ -1637,6 +1638,7 @@ def test_backup_concurrent_drop_table(self): gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() + gdb.kill() show_backup = self.show_pb(backup_dir, 'node')[0] @@ -1761,6 +1763,7 @@ def test_sigint_handling(self): gdb._execute('signal SIGINT') gdb.continue_execution_until_error() + gdb.kill() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] @@ -1901,7 +1904,7 @@ def test_basic_missing_file_permissions(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pg_class')").rstrip() + "select pg_relation_filepath('pg_class')").decode('utf-8').rstrip() full_path = os.path.join(node.data_dir, relative_path) diff --git a/tests/checkdb.py b/tests/checkdb.py index 6c25293ab..a7527d579 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -259,13 +259,13 @@ def test_basic_checkdb_amcheck_only_sanity(self): node.data_dir, node.safe_psql( "db1", - "select pg_relation_filepath('pgbench_accounts_pkey')").rstrip()) + "select pg_relation_filepath('pgbench_accounts_pkey')").decode('utf-8').rstrip()) index_path_2 = os.path.join( node.data_dir, node.safe_psql( "db2", - "select pg_relation_filepath('some_index')").rstrip()) + "select pg_relation_filepath('some_index')").decode('utf-8').rstrip()) try: self.checkdb_node( @@ -376,7 +376,7 @@ def test_checkdb_block_validation_sanity(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # sanity try: @@ -473,14 +473,15 @@ def test_checkdb_sigint_handling(self): gdb = self.checkdb_node( backup_dir, 'node', gdb=True, options=[ - '-d', 'postgres', '-j', '4', + '-d', 'postgres', '-j', '2', '--skip-block-validation', + '--progress', '--amcheck', '-p', str(node.port)]) gdb.set_breakpoint('amcheck_one_index') gdb.run_until_break() - gdb.continue_execution_until_break(10) + gdb.continue_execution_until_break(20) gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') diff --git a/tests/delta.py b/tests/delta.py index 0abdd1c2c..47e6278cd 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -206,7 +206,7 @@ def test_delta_vacuum_truncate_2(self): filepath = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')" - ).rstrip() + ).decode('utf-8').rstrip() self.backup_node(backup_dir, 'node', node) @@ -708,7 +708,7 @@ def test_exists_in_previous_backup(self): node.safe_psql("postgres", "SELECT * FROM t_heap") filepath = node.safe_psql( "postgres", - "SELECT pg_relation_filepath('t_heap')").rstrip() + "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', @@ -1063,7 +1063,7 @@ def test_delta_nullified_heap_page_backup(self): file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( "postgres", diff --git a/tests/exclude.py b/tests/exclude.py index 7148c8b09..7ee315fa5 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -227,7 +227,7 @@ def test_exclude_log_dir(self): log_dir = node.safe_psql( 'postgres', - 'show log_directory').rstrip() + 'show log_directory').decode('utf-8').rstrip() node.cleanup() @@ -264,7 +264,7 @@ def test_exclude_log_dir_1(self): log_dir = node.safe_psql( 'postgres', - 'show log_directory').rstrip() + 'show log_directory').decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, diff --git a/tests/external.py b/tests/external.py index 9d14d7558..5658de2bf 100644 --- a/tests/external.py +++ b/tests/external.py @@ -38,7 +38,8 @@ def test_basic_external(self): # take FULL backup with external directory pointing to a file file_path = os.path.join(core_dir, 'file') - open(file_path, "w+") + with open(file_path, "w+") as f: + pass try: self.backup_node( @@ -1745,7 +1746,7 @@ def test_external_dir_contain_symlink_on_file(self): # create symlink to directory in external directory src_file = os.path.join(symlinked_dir, 'postgresql.conf') os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) os.symlink(src_file, file_in_external_dir) # FULL backup with external directories @@ -2123,7 +2124,7 @@ def test_restore_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2189,7 +2190,7 @@ def test_merge_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2258,12 +2259,12 @@ def test_restore_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0700) + os.chmod(external_dir_1, 0o0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0700) + os.chmod(external_dir_2, 0o0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() @@ -2340,12 +2341,12 @@ def test_merge_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0700) + os.chmod(external_dir_1, 0o0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0700) + os.chmod(external_dir_2, 0o0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() @@ -2458,7 +2459,7 @@ def test_smart_restore_externals(self): logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = f.read() # get delta between FULL and PAGE filelists filelist_full = self.get_backup_filelist( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b0b7ceb5d..f4d81def6 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -130,13 +130,13 @@ def slow_start(self, replica=False): self.start() while True: try: - output = self.safe_psql('template1', query).rstrip() + output = self.safe_psql('template1', query).decode("utf-8").rstrip() if output == 't': break except testgres.QueryException as e: - if 'database system is starting up' in e[0]: + if 'database system is starting up' in e.message: continue else: raise e @@ -439,7 +439,8 @@ def get_fork_path(self, node, fork_name): def get_md5_per_page_for_fork(self, file, size_in_pages): pages_per_segment = {} md5_per_page = {} - nsegments = size_in_pages/131072 + size_in_pages = int(size_in_pages) + nsegments = int(size_in_pages/131072) if size_in_pages % 131072 != 0: nsegments = nsegments + 1 @@ -1424,7 +1425,7 @@ def switch_wal_segment(self, node): """ if isinstance(node, testgres.PostgresNode): if self.version_to_num( - node.safe_psql('postgres', 'show server_version') + node.safe_psql('postgres', 'show server_version').decode('utf-8') ) >= self.version_to_num('10.0'): node.safe_psql('postgres', 'select pg_switch_wal()') else: @@ -1441,10 +1442,11 @@ def switch_wal_segment(self, node): def wait_until_replica_catch_with_master(self, master, replica): - if self.version_to_num( - master.safe_psql( - 'postgres', - 'show server_version')) >= self.version_to_num('10.0'): + version = master.safe_psql( + 'postgres', + 'show server_version').decode('utf-8').rstrip() + + if self.version_to_num(version) >= self.version_to_num('10.0'): master_function = 'pg_catalog.pg_current_wal_lsn()' replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' else: @@ -1453,7 +1455,7 @@ def wait_until_replica_catch_with_master(self, master, replica): lsn = master.safe_psql( 'postgres', - 'SELECT {0}'.format(master_function)).rstrip() + 'SELECT {0}'.format(master_function)).decode('utf-8').rstrip() # Wait until replica catch up with master replica.poll_query_until( @@ -1527,8 +1529,10 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_fullpath = os.path.join(root, file) file_relpath = os.path.relpath(file_fullpath, pgdata) directory_dict['files'][file_relpath] = {'is_datafile': False} - directory_dict['files'][file_relpath]['md5'] = hashlib.md5( - open(file_fullpath, 'rb').read()).hexdigest() + with open(file_fullpath, 'rb') as f: + directory_dict['files'][file_relpath]['md5'] = hashlib.md5(f.read()).hexdigest() +# directory_dict['files'][file_relpath]['md5'] = hashlib.md5( +# f = open(file_fullpath, 'rb').read()).hexdigest() # crappy algorithm if file.isdigit(): @@ -1764,6 +1768,10 @@ def __init__(self, cmd, verbose, attach=False): else: break + def kill(self): + self.proc.kill() + self.proc.wait() + def set_breakpoint(self, location): result = self._execute('break ' + location) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 3e6829532..6fb0edbca 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -382,7 +382,7 @@ def test_incr_checksum_restore(self): xid = node.safe_psql( 'postgres', - 'select txid_current()').rstrip() + 'select txid_current()').decode('utf-8').rstrip() # --A-----B--------X pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) @@ -472,7 +472,7 @@ def test_incr_lsn_restore(self): xid = node.safe_psql( 'postgres', - 'select txid_current()').rstrip() + 'select txid_current()').decode('utf-8').rstrip() # --A-----B--------X pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) @@ -684,7 +684,7 @@ def test_incr_checksum_corruption_detection(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() @@ -742,7 +742,7 @@ def test_incr_lsn_corruption_detection(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() @@ -1512,7 +1512,7 @@ def test_incr_restore_zero_size_file_checksum(self): node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w", 0) as f: + with open(fullpath, "w+b", 0) as f: f.flush() f.close @@ -1586,7 +1586,7 @@ def test_incr_restore_zero_size_file_lsn(self): node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w", 0) as f: + with open(fullpath, "w+b", 0) as f: f.flush() f.close diff --git a/tests/logging.py b/tests/logging.py index efde1d0b9..757600004 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -255,7 +255,7 @@ def test_garbage_in_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) # mangle .rotation file - with open(rotation_file_path, "wt", 0) as f: + with open(rotation_file_path, "w+b", 0) as f: f.write(b"blah") f.flush() f.close diff --git a/tests/merge.py b/tests/merge.py index 214a8eef0..d7dbae206 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1093,7 +1093,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): old_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP self.backup_node( @@ -1109,7 +1109,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): new_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP backup_id_2 = self.backup_node( @@ -1508,7 +1508,7 @@ def test_crash_after_opening_backup_control_2(self): path = node.safe_psql( 'postgres', - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() fsm_path = path + '_fsm' @@ -1601,7 +1601,7 @@ def test_losing_file_after_failed_merge(self): path = node.safe_psql( 'postgres', - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( 'postgres', @@ -1682,7 +1682,7 @@ def test_failed_merge_after_delete(self): dboid = node.safe_psql( "postgres", - "select oid from pg_database where datname = 'testdb'").rstrip() + "select oid from pg_database where datname = 'testdb'").decode('utf-8').rstrip() # take FULL backup full_id = self.backup_node( @@ -2687,7 +2687,7 @@ def test_missing_data_file(self): path = node.safe_psql( 'postgres', - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() gdb = self.merge_backup( backup_dir, "node", delta_id, @@ -2812,7 +2812,7 @@ def test_merge_remote_mode(self): logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, "rw+") as f: + with open(logfile, "w+") as f: f.truncate() gdb.continue_execution_until_exit() diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 253be3441..e373a84b8 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -58,7 +58,7 @@ def test_minrecpoint_on_replica(self): # we need those later node.safe_psql( "postgres", - "CREATE EXTENSION plpythonu") + "CREATE EXTENSION plpython3u") node.safe_psql( "postgres", @@ -174,7 +174,7 @@ def test_minrecpoint_on_replica(self): found_corruption = True if found_corruption: plpy.error('Found Corruption') -$$ LANGUAGE plpythonu; +$$ LANGUAGE plpython3u; ''' # Find blocks from future diff --git a/tests/restore.py b/tests/restore.py index 70de7e357..2a5fac6a6 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2,7 +2,6 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess -from datetime import datetime import sys from time import sleep from datetime import datetime, timedelta @@ -1986,7 +1985,7 @@ def test_restore_target_new_options(self): target_name = 'savepoint' - target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + target_time = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z") with node.connect("postgres") as con: res = con.execute( "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") @@ -2470,7 +2469,7 @@ def test_partial_restore_exclude_tablespace(self): 'postgres', "SELECT oid " "FROM pg_tablespace " - "WHERE spcname = 'somedata'").rstrip() + "WHERE spcname = 'somedata'").decode('utf-8').rstrip() for i in range(1, 10, 1): node.safe_psql( @@ -2480,7 +2479,7 @@ def test_partial_restore_exclude_tablespace(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -3308,7 +3307,7 @@ def test_stream_restore_command_option(self): timeline_id = node.safe_psql( 'postgres', - 'select timeline_id from pg_control_checkpoint()').rstrip() + 'select timeline_id from pg_control_checkpoint()').decode('utf-8').rstrip() self.assertEqual('2', timeline_id) diff --git a/tests/retention.py b/tests/retention.py index 0d1c72b41..6ab796b48 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1881,7 +1881,7 @@ def test_wal_depth_1(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=1) @@ -1917,7 +1917,7 @@ def test_wal_depth_1(self): target_xid = node_restored.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_restored.pgbench_init(scale=2) @@ -2003,7 +2003,7 @@ def test_wal_purge(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=5) # B2 FULL on TLI1 @@ -2012,6 +2012,8 @@ def test_wal_purge(self): B3 = self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) + self.delete_pb(backup_dir, 'node', options=['--delete-wal']) + # TLI 2 node_tli2 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_tli2')) @@ -2034,7 +2036,7 @@ def test_wal_purge(self): target_xid = node_tli2.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) B4 = self.backup_node( @@ -2224,7 +2226,7 @@ def test_wal_depth_2(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=5) # B2 FULL on TLI1 @@ -2255,7 +2257,7 @@ def test_wal_depth_2(self): target_xid = node_tli2.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) B4 = self.backup_node( @@ -2490,7 +2492,7 @@ def test_basic_wal_depth(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() self.switch_wal_segment(node) diff --git a/tests/set_backup.py b/tests/set_backup.py index db039c92d..daba9a216 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -463,7 +463,7 @@ def test_add_big_note(self): note = node.safe_psql( "postgres", - "SELECT repeat('hello', 200)").rstrip() + "SELECT repeat('hello', 200)").decode('utf-8').rstrip() backup_id = self.backup_node( backup_dir, 'node', node, diff --git a/tests/validate.py b/tests/validate.py index c84ea5294..6777b92a1 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -34,7 +34,7 @@ def test_basic_validate_nullified_heap_page_backup(self): file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( "postgres", @@ -244,7 +244,7 @@ def test_basic_validate_corrupted_intermediate_backup(self): "from generate_series(0,10000) i") file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -326,7 +326,7 @@ def test_validate_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -337,7 +337,7 @@ def test_validate_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_path_t_heap_1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -632,7 +632,7 @@ def test_validate_corrupted_intermediate_backups_1(self): "from generate_series(0,10000) i") file_page_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() backup_id_3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -662,7 +662,7 @@ def test_validate_corrupted_intermediate_backups_1(self): "from generate_series(0,10000) i") file_page_5 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() backup_id_6 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -829,7 +829,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_page_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() backup_id_3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -853,7 +853,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30001, 30001) i RETURNING (xmin)").rstrip() + "from generate_series(30001, 30001) i RETURNING (xmin)").decode('utf-8').rstrip() backup_id_5 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -866,7 +866,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_page_5 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() backup_id_6 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1015,7 +1015,7 @@ def test_validate_instance_with_corrupted_page(self): "from generate_series(0,10000) i") file_path_t_heap1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1152,7 +1152,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1248,7 +1248,7 @@ def test_validate_instance_with_corrupted_full(self): file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1559,11 +1559,11 @@ def test_validate_corrupt_wal_between_backups(self): if self.get_version(node) < self.version_to_num('10.0'): walfile = node.safe_psql( 'postgres', - 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() else: walfile = node.safe_psql( 'postgres', - 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -2640,7 +2640,7 @@ def test_file_size_corruption_no_validate(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() heap_size = node.safe_psql( "postgres", "select pg_relation_size('t_heap')") @@ -3746,7 +3746,9 @@ def test_not_validate_diffenent_pg_version(self): with open(control_file, 'r') as f: data = f.read(); - data = data.replace(str(pg_version), str(fake_new_pg_version)) + data = data.replace( + "server-version = {0}".format(str(pg_version)), + "server-version = {0}".format(str(fake_new_pg_version))) with open(control_file, 'w') as f: f.write(data); From 288c1223e4aa89b9cd02dacdd03a940b7fe97102 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 03:01:11 +0300 Subject: [PATCH 1505/2107] [Issue #270] issue VERBOSE message instead of WARNING if backup directory name does not match backup ID --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index a05626ce1..a3b7e85ef 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -472,7 +472,7 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) } else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) { - elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", + elog(VERBOSE, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", base36enc(backup->start_time), backup_conf_path); } From 59251c3d041cdc470b08ed7a7e163031bbf70fed Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 30 Oct 2020 13:47:48 +0300 Subject: [PATCH 1506/2107] do not fsync WAL segments during streaming --- src/backup.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 1a4437281..b1bc75d26 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2394,6 +2394,8 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) /* * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is * set by pg_stop_backup(). + * + * TODO: Add streamed file to file list when segment is finished */ static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) @@ -2499,7 +2501,11 @@ StreamLog(void *arg) ctl.sysidentifier = NULL; #if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, +// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, + 0, + true); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; #if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 @@ -2509,6 +2515,8 @@ StreamLog(void *arg) ctl.basedir = (char *) stream_arg->basedir; #endif + ctl.do_sync = false; /* We sync all files at the end of backup */ +// ctl.mark_done /* for future use in s3 */ ctl.stream_stop = stop_streaming; ctl.standby_message_timeout = standby_message_timeout; ctl.partial_suffix = NULL; From a5feea243554f8765d394666f9c76c8e8981a551 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 31 Oct 2020 14:07:48 +0300 Subject: [PATCH 1507/2107] fix 59251c3d for 9.6 and 9.5 --- src/backup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index b1bc75d26..931a83676 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2508,6 +2508,8 @@ StreamLog(void *arg) true); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; + ctl.do_sync = false; /* We sync all files at the end of backup */ +// ctl.mark_done /* for future use in s3 */ #if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 ctl.temp_slot = temp_slot; #endif @@ -2515,8 +2517,6 @@ StreamLog(void *arg) ctl.basedir = (char *) stream_arg->basedir; #endif - ctl.do_sync = false; /* We sync all files at the end of backup */ -// ctl.mark_done /* for future use in s3 */ ctl.stream_stop = stop_streaming; ctl.standby_message_timeout = standby_message_timeout; ctl.partial_suffix = NULL; From c796ab2c4294ee26a22f300a739bd51f24a62101 Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Mon, 2 Nov 2020 14:07:12 +0700 Subject: [PATCH 1508/2107] tests: fix errors for python3 in ptrack_helpers --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f4d81def6..5ba43e929 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -492,10 +492,10 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): if os.path.getsize(file) == 0: return ptrack_bits_for_fork byte_size = os.path.getsize(file + '_ptrack') - npages = byte_size/8192 + npages = int(byte_size/8192) if byte_size % 8192 != 0: print('Ptrack page is not 8k aligned') - sys.exit(1) + exit(1) file = os.open(file + '_ptrack', os.O_RDONLY) From e806cf957e1a59f117438a0b3b55aef0ae7ad84a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 4 Nov 2020 16:31:22 +0000 Subject: [PATCH 1509/2107] tests: move ptrack module to python3 --- tests/ptrack.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index b54006201..f4e556518 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -89,7 +89,7 @@ def test_ptrack_multi_timeline_backup(self): xid = node.safe_psql( 'postgres', - 'SELECT txid_current()').rstrip() + 'SELECT txid_current()').decode('utf-8').rstrip() pgbench.wait() self.backup_node(backup_dir, 'node', node, backup_type='ptrack') @@ -126,7 +126,7 @@ def test_ptrack_multi_timeline_backup(self): 'select (select sum(tbalance) from pgbench_tellers) - ' '( select sum(bbalance) from pgbench_branches) + ' '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() self.assertEqual('0', balance) @@ -275,7 +275,7 @@ def test_ptrack_eat_my_data(self): 'select (select sum(tbalance) from pgbench_tellers) - ' '( select sum(bbalance) from pgbench_branches) + ' '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() self.assertEqual('0', balance) @@ -1108,6 +1108,13 @@ def test_ptrack_archive(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(200, 300) i") + # Drop Node node.cleanup() From 7fa6dd2d4d8ffcf106003ac53ffe095c43c5b0f7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 8 Nov 2020 14:59:07 +0300 Subject: [PATCH 1510/2107] [Issue #274] added tests.logging.LogTest.test_issue_274 --- tests/logging.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/logging.py b/tests/logging.py index 757600004..ab9425d16 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -300,3 +300,41 @@ def test_garbage_in_rotation_file(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_issue_274(self): + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + log_dir = os.path.join(backup_dir, "somedir") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--log-level-console=verbose', '--log-level-file=verbose', + '--log-directory={0}'.format(log_dir), '-j1', + '--log-filename=somelog.txt', '--archive-timeout=5s']) + except: + pass + + log_file_path = os.path.join( + log_dir, 'somelog.txt') + + self.assertTrue(os.path.isfile(log_file_path)) + + with open(log_file_path, "r+") as f: + log_content = f.read() + + self.assertIn('INFO: command:', log_content) + print(log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 895b57f67cc8b3469879e519dee6a3f0ffd8c810 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 8 Nov 2020 17:05:10 +0300 Subject: [PATCH 1511/2107] [Issue #274] tests: minor fix --- tests/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logging.py b/tests/logging.py index ab9425d16..5d0bf8f3d 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -334,7 +334,7 @@ def test_issue_274(self): log_content = f.read() self.assertIn('INFO: command:', log_content) - print(log_content) + # print(log_content) # Clean after yourself self.del_test_dir(module_name, fname) From 104600f5c233cce98a0d1984446c3b8ff60c5bbb Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Mon, 9 Nov 2020 16:18:32 +0700 Subject: [PATCH 1512/2107] tests: another fix ptrack module to python3 --- tests/ptrack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index f4e556518..18425237f 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -202,7 +202,8 @@ def test_ptrack_multi_timeline_backup_1(self): 'select (select sum(tbalance) from pgbench_tellers) - ' '( select sum(bbalance) from pgbench_branches) + ' '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').rstrip() + '(select sum(delta) from pgbench_history) as must_be_zero').\ + decode('utf-8').rstrip() self.assertEqual('0', balance) From caa98327b726dfeadd11dc2b26e9c0fc0fcee890 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 9 Nov 2020 13:50:56 +0300 Subject: [PATCH 1513/2107] [Issue #274] execute atexit callbackups in correct order to avoid truncating log file --- src/utils/logger.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/utils/logger.c b/src/utils/logger.c index 719ede436..f039d4a5d 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -48,7 +48,7 @@ static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf( /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); -static void release_logfile(void); +static void release_logfile(bool fatal, void *userdata); static char *logfile_getname(const char *format, time_t timestamp); static FILE *logfile_open(const char *filename, const char *mode); @@ -224,12 +224,7 @@ elog_internal(int elevel, bool file_only, const char *message) if (write_to_file) { if (log_file == NULL) - { - if (logger_config.log_filename == NULL) - open_logfile(&log_file, LOG_FILENAME_DEFAULT); - else - open_logfile(&log_file, logger_config.log_filename); - } + open_logfile(&log_file, logger_config.log_filename ? logger_config.log_filename : LOG_FILENAME_DEFAULT); fprintf(log_file, "%s ", strfbuf); fprintf(log_file, "%s ", str_pid); @@ -699,7 +694,7 @@ open_logfile(FILE **file, const char *filename_format) */ if (!exit_hook_registered) { - atexit(release_logfile); + pgut_atexit_push(release_logfile, NULL); exit_hook_registered = true; } } @@ -708,7 +703,7 @@ open_logfile(FILE **file, const char *filename_format) * Closes opened file. */ static void -release_logfile(void) +release_logfile(bool fatal, void *userdata) { if (log_file) { From 038f4cf8693f5c5defaebff280f7e79f90baf340 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 9 Nov 2020 13:53:38 +0300 Subject: [PATCH 1514/2107] [Issue #274] improve test coverage --- tests/logging.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/logging.py b/tests/logging.py index 5d0bf8f3d..47143cfb7 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -311,19 +311,51 @@ def test_issue_274(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) node.slow_start() + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,45600) i") + log_dir = os.path.join(backup_dir, "somedir") try: self.backup_node( - backup_dir, 'node', node, backup_type='page', + backup_dir, 'node', replica, backup_type='page', options=[ '--log-level-console=verbose', '--log-level-file=verbose', '--log-directory={0}'.format(log_dir), '-j1', - '--log-filename=somelog.txt', '--archive-timeout=5s']) - except: - pass + '--log-filename=somelog.txt', '--archive-timeout=5s', + '--no-validate', '--log-rotation-size=100KB']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of archiving timeout" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: WAL segment', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) log_file_path = os.path.join( log_dir, 'somelog.txt') @@ -334,7 +366,6 @@ def test_issue_274(self): log_content = f.read() self.assertIn('INFO: command:', log_content) - # print(log_content) # Clean after yourself self.del_test_dir(module_name, fname) From dad2747ddc593382058358cff50969022ef54832 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 11 Nov 2020 22:19:09 +0300 Subject: [PATCH 1515/2107] Refactor code related to WAL streaming. Move it to stream.c --- Makefile | 4 +- src/backup.c | 314 +------------------------------------- src/pg_probackup.h | 7 + src/stream.c | 368 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 380 insertions(+), 313 deletions(-) create mode 100644 src/stream.c diff --git a/Makefile b/Makefile index bc37860be..1431be4ef 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ - src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ - src/validate.o src/datapagemap.o + src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ + src/util.o src/validate.o src/datapagemap.o # borrowed files OBJS += src/pg_crc.o src/receivelog.o src/streamutil.o \ diff --git a/src/backup.c b/src/backup.c index 931a83676..c65a21710 100644 --- a/src/backup.c +++ b/src/backup.c @@ -15,7 +15,6 @@ #endif #include "catalog/pg_tablespace.h" #include "pgtar.h" -#include "receivelog.h" #include "streamutil.h" #include @@ -25,18 +24,6 @@ #include "utils/thread.h" #include "utils/file.h" -static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ -static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; -static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; - -/* - * How long we should wait for streaming end in seconds. - * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 - */ -static uint32 stream_stop_timeout = 0; -/* Time in which we started to wait for streaming end */ -static time_t stream_stop_begin = 0; - const char *progname = "pg_probackup"; /* list of files contained in backup */ @@ -45,26 +32,6 @@ static parray *backup_files_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; -/* - * We need to wait end of WAL streaming before execute pg_stop_backup(). - */ -typedef struct -{ - const char *basedir; - PGconn *conn; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; - - XLogRecPtr startpos; - TimeLineID starttli; -} StreamThreadArg; - -static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1}; bool exclusive_backup = false; @@ -86,15 +53,11 @@ static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); -static int checkpoint_timeout(PGconn *backup_conn); static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir); -static void *StreamLog(void *arg); -static void IdentifySystem(StreamThreadArg *stream_thread_arg); - static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); static parray *get_database_map(PGconn *pg_startbackup_conn); @@ -310,33 +273,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* start stream replication */ if (stream_wal) { - /* How long we should wait for streaming end after pg_stop_backup */ - stream_stop_timeout = checkpoint_timeout(backup_conn); - stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; - join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); - stream_thread_arg.basedir = dst_backup_path; - - /* - * Connect in replication mode to the server. - */ - stream_thread_arg.conn = pgut_connect_replication(instance_config.conn_opt.pghost, - instance_config.conn_opt.pgport, - instance_config.conn_opt.pgdatabase, - instance_config.conn_opt.pguser); - /* sanity */ - IdentifySystem(&stream_thread_arg); - - /* By default there are some error */ - stream_thread_arg.ret = 1; - /* we must use startpos as start_lsn from start_backup */ - stream_thread_arg.startpos = current.start_lsn; - stream_thread_arg.starttli = current.tli; - - thread_interrupted = false; - pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + start_WAL_streaming(backup_conn, dst_backup_path, &instance_config.conn_opt, + current.start_lsn, current.tli); } /* initialize backup list */ @@ -1979,10 +1920,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (stream_wal) { - /* Wait for the completion of stream */ - pthread_join(stream_thread, NULL); - if (stream_thread_arg.ret == 1) - elog(ERROR, "WAL streaming failed"); + wait_WAL_streaming_end(); pgBackupGetPath2(backup, stream_xlog_path, lengthof(stream_xlog_path), @@ -2009,35 +1947,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, } } -/* - * Retrieve checkpoint_timeout GUC value in seconds. - */ -static int -checkpoint_timeout(PGconn *backup_conn) -{ - PGresult *res; - const char *val; - const char *hintmsg; - int val_int; - - res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); - val = PQgetvalue(res, 0, 0); - - if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) - { - PQclear(res); - if (hintmsg) - elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, - hintmsg); - else - elog(ERROR, "Invalid value of checkout_timeout %s", val); - } - - PQclear(res); - - return val_int; -} - /* * Notify end of backup to server when "backup_label" is in the root directory * of the DB cluster. @@ -2391,164 +2300,6 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) } -/* - * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is - * set by pg_stop_backup(). - * - * TODO: Add streamed file to file list when segment is finished - */ -static bool -stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) -{ - static uint32 prevtimeline = 0; - static XLogRecPtr prevpos = InvalidXLogRecPtr; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during WAL streaming"); - - /* we assume that we get called once at the end of each segment */ - if (segment_finished) - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), - (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - - /* - * Note that we report the previous, not current, position here. After a - * timeline switch, xlogpos points to the beginning of the segment because - * that's where we always begin streaming. Reporting the end of previous - * timeline isn't totally accurate, because the next timeline can begin - * slightly before the end of the WAL that we received on the previous - * timeline, but it's close enough for reporting purposes. - */ - if (prevtimeline != 0 && prevtimeline != timeline) - elog(LOG, _("switched to timeline %u at %X/%X\n"), - timeline, (uint32) (prevpos >> 32), (uint32) prevpos); - - if (!XLogRecPtrIsInvalid(stop_backup_lsn)) - { - if (xlogpos >= stop_backup_lsn) - { - stop_stream_lsn = xlogpos; - return true; - } - - /* pg_stop_backup() was executed, wait for the completion of stream */ - if (stream_stop_begin == 0) - { - elog(INFO, "Wait for LSN %X/%X to be streamed", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); - - stream_stop_begin = time(NULL); - } - - if (time(NULL) - stream_stop_begin > stream_stop_timeout) - elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, - stream_stop_timeout); - } - - prevtimeline = timeline; - prevpos = xlogpos; - - return false; -} - -/* - * Start the log streaming - */ -static void * -StreamLog(void *arg) -{ - StreamThreadArg *stream_arg = (StreamThreadArg *) arg; - - /* - * Always start streaming at the beginning of a segment - */ - stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; - - /* Initialize timeout */ - stream_stop_begin = 0; - -#if PG_VERSION_NUM >= 100000 - /* if slot name was not provided for temp slot, use default slot name */ - if (!replication_slot && temp_slot) - replication_slot = "pg_probackup_slot"; -#endif - - -#if PG_VERSION_NUM >= 110000 - /* Create temp repslot */ - if (temp_slot) - CreateReplicationSlot(stream_arg->conn, replication_slot, - NULL, temp_slot, true, true, false); -#endif - - /* - * Start the replication - */ - elog(LOG, "started streaming WAL at %X/%X (timeline %u)", - (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, - stream_arg->starttli); - -#if PG_VERSION_NUM >= 90600 - { - StreamCtl ctl; - - MemSet(&ctl, 0, sizeof(ctl)); - - ctl.startpos = stream_arg->startpos; - ctl.timeline = stream_arg->starttli; - ctl.sysidentifier = NULL; - -#if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod( - stream_arg->basedir, -// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, - 0, - true); - ctl.replication_slot = replication_slot; - ctl.stop_socket = PGINVALID_SOCKET; - ctl.do_sync = false; /* We sync all files at the end of backup */ -// ctl.mark_done /* for future use in s3 */ -#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 - ctl.temp_slot = temp_slot; -#endif -#else - ctl.basedir = (char *) stream_arg->basedir; -#endif - - ctl.stream_stop = stop_streaming; - ctl.standby_message_timeout = standby_message_timeout; - ctl.partial_suffix = NULL; - ctl.synchronous = false; - ctl.mark_done = false; - - if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) - elog(ERROR, "Problem in receivexlog"); - -#if PG_VERSION_NUM >= 100000 - if (!ctl.walmethod->finish()) - elog(ERROR, "Could not finish writing WAL files: %s", - strerror(errno)); -#endif - } -#else - if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, - NULL, (char *) stream_arg->basedir, stop_streaming, - standby_message_timeout, NULL, false, false) == false) - elog(ERROR, "Problem in receivexlog"); -#endif - - elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", - (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); - stream_arg->ret = 0; - - PQfinish(stream_arg->conn); - stream_arg->conn = NULL; - - return NULL; -} - static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) { @@ -2613,62 +2364,3 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) } } } - -/* - * Run IDENTIFY_SYSTEM through a given connection and - * check system identifier and timeline are matching - */ -void -IdentifySystem(StreamThreadArg *stream_thread_arg) -{ - PGresult *res; - - uint64 stream_conn_sysidentifier = 0; - char *stream_conn_sysidentifier_str; - TimeLineID stream_conn_tli = 0; - - if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) - { - PQfinish(stream_thread_arg->conn); - /* - * Error message already written in CheckServerVersionForStreaming(). - * There's no hope of recovering from a version mismatch, so don't - * retry. - */ - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - /* - * Identify server, obtain server system identifier and timeline - */ - res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING,"Could not send replication command \"%s\": %s", - "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); - PQfinish(stream_thread_arg->conn); - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); - stream_conn_tli = atoll(PQgetvalue(res, 0, 1)); - - /* Additional sanity, primary for PG 9.5, - * where system id can be obtained only via "IDENTIFY SYSTEM" - */ - if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) - elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); - - if (stream_conn_sysidentifier != instance_config.system_identifier) - elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " - "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", - stream_conn_sysidentifier, instance_config.system_identifier); - - if (stream_conn_tli != current.tli) - elog(ERROR, "Timeline identifier mismatch. " - "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", - stream_conn_tli, current.tli); - - PQclear(res); -} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 9e9fd01a8..1827780ad 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1172,4 +1172,11 @@ datapagemap_is_set(datapagemap_t *map, BlockNumber blkno); extern void datapagemap_print_debug(datapagemap_t *map); +/* in stream.c */ +extern XLogRecPtr stop_backup_lsn; +extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, + ConnectionOptions *conn_opt, + XLogRecPtr startpos, TimeLineID starttli); +extern void wait_WAL_streaming_end(void); + #endif /* PG_PROBACKUP_H */ diff --git a/src/stream.c b/src/stream.c new file mode 100644 index 000000000..e7d6ce68e --- /dev/null +++ b/src/stream.c @@ -0,0 +1,368 @@ +/*------------------------------------------------------------------------- + * + * stream.c: pg_probackup specific code for WAL streaming + * + * Portions Copyright (c) 2015-2020, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "receivelog.h" +#include "streamutil.h" + +#include +#include + +/* + * global variable needed by ReceiveXlogStream() + * + * standby_message_timeout controls how often we send a message + * back to the primary letting it know our progress, in milliseconds. + * + * in pg_probackup we use a default setting = 10 sec + */ +static int standby_message_timeout = 10 * 1000; + +/* stop_backup_lsn is set by pg_stop_backup() to stop streaming */ +XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + const char *basedir; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; + + XLogRecPtr startpos; + TimeLineID starttli; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static void IdentifySystem(StreamThreadArg *stream_thread_arg); +static int checkpoint_timeout(PGconn *backup_conn); +static void *StreamLog(void *arg); +static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, + bool segment_finished); + +/* + * Run IDENTIFY_SYSTEM through a given connection and + * check system identifier and timeline are matching + */ +static void +IdentifySystem(StreamThreadArg *stream_thread_arg) +{ + PGresult *res; + + uint64 stream_conn_sysidentifier = 0; + char *stream_conn_sysidentifier_str; + TimeLineID stream_conn_tli = 0; + + if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) + { + PQfinish(stream_thread_arg->conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtain server system identifier and timeline + */ + res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING,"Could not send replication command \"%s\": %s", + "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); + PQfinish(stream_thread_arg->conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); + stream_conn_tli = atoll(PQgetvalue(res, 0, 1)); + + /* Additional sanity, primary for PG 9.5, + * where system id can be obtained only via "IDENTIFY SYSTEM" + */ + if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) + elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); + + if (stream_conn_sysidentifier != instance_config.system_identifier) + elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " + "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", + stream_conn_sysidentifier, instance_config.system_identifier); + + if (stream_conn_tli != current.tli) + elog(ERROR, "Timeline identifier mismatch. " + "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", + stream_conn_tli, current.tli); + + PQclear(res); +} + +/* + * Retrieve checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(PGconn *backup_conn) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * Always start streaming at the beginning of a segment + */ + stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; + + /* Initialize timeout */ + stream_stop_begin = 0; + +#if PG_VERSION_NUM >= 100000 + /* if slot name was not provided for temp slot, use default slot name */ + if (!replication_slot && temp_slot) + replication_slot = "pg_probackup_slot"; +#endif + + +#if PG_VERSION_NUM >= 110000 + /* Create temp repslot */ + if (temp_slot) + CreateReplicationSlot(stream_arg->conn, replication_slot, + NULL, temp_slot, true, true, false); +#endif + + /* + * Start the replication + */ + elog(LOG, "started streaming WAL at %X/%X (timeline %u)", + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = stream_arg->startpos; + ctl.timeline = stream_arg->starttli; + ctl.sysidentifier = NULL; + +#if PG_VERSION_NUM >= 100000 + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, +// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, + 0, + true); + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; + ctl.do_sync = false; /* We sync all files at the end of backup */ +// ctl.mark_done /* for future use in s3 */ +#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 + ctl.temp_slot = temp_slot; +#endif +#else + ctl.basedir = (char *) stream_arg->basedir; +#endif + + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + + if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + elog(ERROR, "Problem in receivexlog"); + +#if PG_VERSION_NUM >= 100000 + if (!ctl.walmethod->finish()) + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); +#endif + } +#else + if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, + NULL, (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + elog(ERROR, "Problem in receivexlog"); +#endif + + elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * for ReceiveXlogStream + * + * The stream_stop callback will be called every time data + * is received, and whenever a segment is completed. If it returns + * true, the streaming will stop and the function + * return. As long as it returns false, streaming will continue + * indefinitely. + * + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + * + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during WAL streaming"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + { + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + /* TODO Add streamed file to file list */ + } + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos >= stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_begin == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + + +/* --- External API --- */ + +/* + * Maybe add a StreamOptions struct ? + * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. + */ +void +start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, + XLogRecPtr startpos, TimeLineID starttli) +{ + /* How long we should wait for streaming end after pg_stop_backup */ + stream_stop_timeout = checkpoint_timeout(backup_conn); + //TODO Add a comment about this calculation + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + stream_thread_arg.basedir = stream_dst_path; + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser); + /* sanity check*/ + IdentifySystem(&stream_thread_arg); + + /* Set error exit code as default */ + stream_thread_arg.ret = 1; + /* we must use startpos as start_lsn from start_backup */ + stream_thread_arg.startpos = current.start_lsn; + stream_thread_arg.starttli = current.tli; + + thread_interrupted = false; + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); +} + +/* Wait for the completion of stream */ +void +wait_WAL_streaming_end(void) +{ + pthread_join(stream_thread, NULL); + if (stream_thread_arg.ret == 1) + elog(ERROR, "WAL streaming failed"); +} From 1d3c333641038c335b0e843146bde5e3a15f3ef9 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 11 Nov 2020 23:10:17 +0300 Subject: [PATCH 1516/2107] Refactor code related to WAL streaming. Optimize filelist gathering for streamed xlog files. Now streaming thread calculates CRC and adds file info to the filelist after each finished segment. --- src/backup.c | 60 +++++++++------------------------------------- src/pg_probackup.h | 3 +-- src/stream.c | 42 ++++++++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 56 deletions(-) diff --git a/src/backup.c b/src/backup.c index c65a21710..5afb9675d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -270,6 +270,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool pgBackupGetPath(¤t, external_prefix, lengthof(external_prefix), EXTERNAL_DIR); + /* initialize backup's file list */ + backup_files_list = parray_new(); + /* start stream replication */ if (stream_wal) { @@ -280,9 +283,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool current.start_lsn, current.tli); } - /* initialize backup list */ - backup_files_list = parray_new(); - /* list files with the logical path. omit $PGDATA */ if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(backup_files_list, instance_config.pgdata, @@ -567,52 +567,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* close ssh session in main thread */ fio_disconnect(); - /* Add archived xlog files into the list of files of this backup */ - if (stream_wal) - { - parray *xlog_files_list; - char pg_xlog_path[MAXPGPATH]; - char wal_full_path[MAXPGPATH]; - - /* Scan backup PG_XLOG_DIR */ - xlog_files_list = parray_new(); - join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, true, 0, - FIO_BACKUP_HOST); - - /* TODO: Drop streamed WAL segments greater than stop_lsn */ - for (i = 0; i < parray_num(xlog_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - - join_path_components(wal_full_path, pg_xlog_path, file->rel_path); - - if (!S_ISREG(file->mode)) - continue; - - file->crc = pgFileGetCRC(wal_full_path, true, false); - file->write_size = file->size; - - /* overwrite rel_path, because now it is relative to - * /backup_dir/backups/instance_name/backup_id/database/pg_xlog/ - */ - pg_free(file->rel_path); - - /* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */ - file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path)); - - file->name = last_dir_separator(file->rel_path); - - if (file->name == NULL) // TODO: do it in pgFileInit - file->name = file->rel_path; - else - file->name++; - } + /* + * Add archived xlog files into the list of files of this backup + * NOTHING TO DO HERE + */ - /* Add xlog files into the list of backed up files */ - parray_concat(backup_files_list, xlog_files_list); - parray_free(xlog_files_list); - } /* write database map to file and add it to control file */ if (database_map) @@ -1920,7 +1879,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (stream_wal) { - wait_WAL_streaming_end(); + /* This function will also add list of xlog files + * to the passed filelist */ + if(wait_WAL_streaming_end(backup_files_list)) + elog(ERROR, "WAL streaming failed"); pgBackupGetPath2(backup, stream_xlog_path, lengthof(stream_xlog_path), diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1827780ad..c38cae8cb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1177,6 +1177,5 @@ extern XLogRecPtr stop_backup_lsn; extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, XLogRecPtr startpos, TimeLineID starttli); -extern void wait_WAL_streaming_end(void); - +extern int wait_WAL_streaming_end(parray *backup_files_list); #endif /* PG_PROBACKUP_H */ diff --git a/src/stream.c b/src/stream.c index e7d6ce68e..2161250f8 100644 --- a/src/stream.c +++ b/src/stream.c @@ -57,6 +57,8 @@ typedef struct static pthread_t stream_thread; static StreamThreadArg stream_thread_arg = {"", NULL, 1}; +static parray *xlog_files_list = NULL; + static void IdentifySystem(StreamThreadArg *stream_thread_arg); static int checkpoint_timeout(PGconn *backup_conn); static void *StreamLog(void *arg); @@ -273,10 +275,38 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* we assume that we get called once at the end of each segment */ if (segment_finished) { + XLogSegNo xlog_segno; + char wal_segment_name[MAXPGPATH]; + char wal_segment_fullpath[MAXPGPATH]; + pgFile *file; + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - /* TODO Add streamed file to file list */ + /* Add streamed xlog file into the backup's list of files */ + if (!xlog_files_list) + xlog_files_list = parray_new(); + + GetXLogSegNo(xlogpos, xlog_segno, instance_config.xlog_seg_size); + GetXLogFileName(wal_segment_name, timeline, xlog_segno, + instance_config.xlog_seg_size); + + join_path_components(wal_segment_fullpath, + stream_thread_arg.basedir, wal_segment_name); + + /* + * NOTE We pass wal_segment_name as a relpath, since now we don't have + * any subdirs in wal directory structure + */ + file = pgFileNew(wal_segment_fullpath, wal_segment_name, false, 0, + FIO_BACKUP_HOST); + file->name = file->rel_path; + file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + + /* Should we recheck it using stat? */ + file->write_size = instance_config.xlog_seg_size; + file->uncompressed_size = instance_config.xlog_seg_size; + parray_append(xlog_files_list, file); } /* @@ -359,10 +389,12 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption } /* Wait for the completion of stream */ -void -wait_WAL_streaming_end(void) +int +wait_WAL_streaming_end(parray *backup_files_list) { + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + pthread_join(stream_thread, NULL); - if (stream_thread_arg.ret == 1) - elog(ERROR, "WAL streaming failed"); + return stream_thread_arg.ret; } From 3e1d0010350e041a6348f381b5e31dc4b68a5471 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 12 Nov 2020 01:03:59 +0300 Subject: [PATCH 1517/2107] [Issue #267] fix segfault and rel_path for streamed WAL segments --- src/stream.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/stream.c b/src/stream.c index 2161250f8..09c1ef205 100644 --- a/src/stream.c +++ b/src/stream.c @@ -251,7 +251,7 @@ StreamLog(void *arg) /* * for ReceiveXlogStream - * + * * The stream_stop callback will be called every time data * is received, and whenever a segment is completed. If it returns * true, the streaming will stop and the function @@ -276,30 +276,33 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) if (segment_finished) { XLogSegNo xlog_segno; - char wal_segment_name[MAXPGPATH]; + char wal_segment_name[MAXNAMLEN]; + char wal_segment_relpath[MAXPGPATH]; char wal_segment_fullpath[MAXPGPATH]; pgFile *file; - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + elog(INFO, _("finished segment at %X/%X (timeline %u)"), (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); /* Add streamed xlog file into the backup's list of files */ if (!xlog_files_list) xlog_files_list = parray_new(); - + GetXLogSegNo(xlogpos, xlog_segno, instance_config.xlog_seg_size); + + /* xlogpos points to the current segment, and we need the finished - previous one */ + xlog_segno--; GetXLogFileName(wal_segment_name, timeline, xlog_segno, instance_config.xlog_seg_size); join_path_components(wal_segment_fullpath, stream_thread_arg.basedir, wal_segment_name); - /* - * NOTE We pass wal_segment_name as a relpath, since now we don't have - * any subdirs in wal directory structure - */ - file = pgFileNew(wal_segment_fullpath, wal_segment_name, false, 0, - FIO_BACKUP_HOST); + join_path_components(wal_segment_relpath, + PG_XLOG_DIR, wal_segment_name); + + /* append file to filelist */ + file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); file->name = file->rel_path; file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); From 1fed5633e6af3df866858501e203b841c0ab4f95 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 12 Nov 2020 12:00:48 +0300 Subject: [PATCH 1518/2107] [Issue #267] use spaces for indentation --- src/stream.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/stream.c b/src/stream.c index 09c1ef205..db9ffe136 100644 --- a/src/stream.c +++ b/src/stream.c @@ -277,12 +277,12 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) { XLogSegNo xlog_segno; char wal_segment_name[MAXNAMLEN]; - char wal_segment_relpath[MAXPGPATH]; + char wal_segment_relpath[MAXPGPATH]; char wal_segment_fullpath[MAXPGPATH]; - pgFile *file; + pgFile *file = NULL; - elog(INFO, _("finished segment at %X/%X (timeline %u)"), - (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); /* Add streamed xlog file into the backup's list of files */ if (!xlog_files_list) @@ -290,18 +290,18 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) GetXLogSegNo(xlogpos, xlog_segno, instance_config.xlog_seg_size); - /* xlogpos points to the current segment, and we need the finished - previous one */ - xlog_segno--; + /* xlogpos points to the current segment, and we need the finished - previous one */ + xlog_segno--; GetXLogFileName(wal_segment_name, timeline, xlog_segno, instance_config.xlog_seg_size); join_path_components(wal_segment_fullpath, stream_thread_arg.basedir, wal_segment_name); - join_path_components(wal_segment_relpath, + join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); - /* append file to filelist */ + /* append file to filelist */ file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); file->name = file->rel_path; file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); From 6123e9467c1456c55524e41a3dee74c62d1e6544 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 12 Nov 2020 13:29:23 +0300 Subject: [PATCH 1519/2107] [PBCKP-107] fix build for Ubuntu 20.10 --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 931a83676..eb84966fe 100644 --- a/src/backup.c +++ b/src/backup.c @@ -37,7 +37,7 @@ static uint32 stream_stop_timeout = 0; /* Time in which we started to wait for streaming end */ static time_t stream_stop_begin = 0; -const char *progname = "pg_probackup"; +//const char *progname = "pg_probackup"; /* list of files contained in backup */ static parray *backup_files_list = NULL; From 6611e5e8c0c4d7fbdd0d2e58bed5892de78b63c8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 12 Nov 2020 20:40:44 +0300 Subject: [PATCH 1520/2107] [PBCKP-110] fix build error on windows --- src/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index db9ffe136..dbb536603 100644 --- a/src/stream.c +++ b/src/stream.c @@ -276,7 +276,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) if (segment_finished) { XLogSegNo xlog_segno; - char wal_segment_name[MAXNAMLEN]; + char wal_segment_name[MAXFNAMELEN]; char wal_segment_relpath[MAXPGPATH]; char wal_segment_fullpath[MAXPGPATH]; pgFile *file = NULL; From 2808cc673c9819a01c09d7e1f69df3349417a25b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 14 Nov 2020 04:11:33 +0300 Subject: [PATCH 1521/2107] tests: fix test_checkdb_with_least_privileges --- tests/checkdb.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index a7527d579..db7af1b45 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -618,14 +618,10 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' ) -# if ProbackupTest.enterprise: -# node.safe_psql( -# "backupdb", -# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") -# -# node.safe_psql( -# "backupdb", -# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") # checkdb try: From cbb721427df48068e178f1b3d92c7b0d91e2fb58 Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Mon, 16 Nov 2020 15:15:05 +0700 Subject: [PATCH 1522/2107] tests: ignore ptrack.map.tmp in pgdata_content --- tests/helpers/ptrack_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 5ba43e929..626f15750 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1505,7 +1505,8 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): 'backup_label', 'tablespace_map', 'recovery.conf', 'ptrack_control', 'ptrack_init', 'pg_control', 'probackup_recovery.conf', 'recovery.signal', - 'standby.signal', 'ptrack.map', 'ptrack.map.mmap' + 'standby.signal', 'ptrack.map', 'ptrack.map.mmap', + 'ptrack.map.tmp' ] if exclude_dirs: From 210c303694cae8450e8d91df32d56a8bcbad3cb0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 18 Nov 2020 15:46:03 +0300 Subject: [PATCH 1523/2107] tests: start restored node in test_basic_ptrack_truncate_replica --- tests/ptrack.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 18425237f..410f264ad 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3031,7 +3031,7 @@ def test_basic_ptrack_truncate_replica(self): pg_options={ 'max_wal_size': '32MB', 'archive_timeout': '10s', - 'checkpoint_timeout': '30s', + 'checkpoint_timeout': '5min', 'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3130,6 +3130,14 @@ def test_basic_ptrack_truncate_replica(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + # Clean after yourself self.del_test_dir(module_name, fname, [master, replica]) From 9453cf0f296cadc110f807eb535470b931c66e9c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 19 Nov 2020 14:38:04 +0300 Subject: [PATCH 1524/2107] bugfix: correctly append streamed segments to backup filelist --- src/pg_probackup.h | 5 +++++ src/stream.c | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c38cae8cb..67147a0f3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -649,6 +649,9 @@ typedef struct BackupPageHeader2 strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) #if PG_VERSION_NUM >= 110000 + +#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ + XLogSegmentOffset(xlogptr, wal_segsz_bytes) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) #define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ @@ -668,6 +671,8 @@ typedef struct BackupPageHeader2 #define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) #else +#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ + XLogSegmentOffset(xlogptr) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo) #define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ diff --git a/src/stream.c b/src/stream.c index dbb536603..bf4885ab2 100644 --- a/src/stream.c +++ b/src/stream.c @@ -290,8 +290,13 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) GetXLogSegNo(xlogpos, xlog_segno, instance_config.xlog_seg_size); - /* xlogpos points to the current segment, and we need the finished - previous one */ - xlog_segno--; + /* + * xlogpos points to the current segment, and we need the finished - previous one + * inless xlogpos points to not 0 offset in segment + */ + if (WalSegmentOffset(xlogpos, instance_config.xlog_seg_size) == 0) + xlog_segno--; + GetXLogFileName(wal_segment_name, timeline, xlog_segno, instance_config.xlog_seg_size); From 90b6d043bbbf597b62cb7abfb98269dfb937fb94 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 19 Nov 2020 19:02:55 +0300 Subject: [PATCH 1525/2107] bugfix: fix assert error for PG13, when start_lsn and stop_lsn are in the same WAL segment, also some segfaults in wait_WAL_streaming_end --- src/backup.c | 4 +- src/parsexlog.c | 7 +++ src/stream.c | 118 +++++++++++++++++++++++++++++++----------------- 3 files changed, 85 insertions(+), 44 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9a1cd9001..0578f2102 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1382,8 +1382,8 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, { XLogRecPtr res; - res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, - in_prev_segment, instance_config.xlog_seg_size); + res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, + in_prev_segment, instance_config.xlog_seg_size); if (!XLogRecPtrIsInvalid(res)) { diff --git a/src/parsexlog.c b/src/parsexlog.c index 535975239..38c62c6a7 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -846,7 +846,14 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, */ GetXLogSegNo(start_lsn, start_segno, wal_seg_size); if (start_segno == segno) + { startpoint = start_lsn; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + } else { XLogRecPtr found; diff --git a/src/stream.c b/src/stream.c index bf4885ab2..f47ea9e6b 100644 --- a/src/stream.c +++ b/src/stream.c @@ -64,6 +64,9 @@ static int checkpoint_timeout(PGconn *backup_conn); static void *StreamLog(void *arg); static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished); +static void append_wal_segment(parray *filelist, uint32 timeline, + XLogRecPtr xlogpos, char *basedir, + uint32 xlog_seg_size); /* * Run IDENTIFY_SYSTEM through a given connection and @@ -166,6 +169,8 @@ StreamLog(void *arg) */ stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; + xlog_files_list = parray_new(); + /* Initialize timeout */ stream_stop_begin = 0; @@ -239,6 +244,18 @@ StreamLog(void *arg) elog(ERROR, "Problem in receivexlog"); #endif + /* sort xlog_files_list */ + parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); + + append_wal_segment(xlog_files_list, stream_arg->starttli, + stop_stream_lsn, (char *) stream_arg->basedir, + instance_config.xlog_seg_size); + + /* + * TODO: remove redundant WAL segments + * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 + */ + elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); stream_arg->ret = 0; @@ -275,46 +292,12 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* we assume that we get called once at the end of each segment */ if (segment_finished) { - XLogSegNo xlog_segno; - char wal_segment_name[MAXFNAMELEN]; - char wal_segment_relpath[MAXPGPATH]; - char wal_segment_fullpath[MAXPGPATH]; - pgFile *file = NULL; - - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + elog(INFO, _("finished segment at %X/%X (timeline %u)"), (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - /* Add streamed xlog file into the backup's list of files */ - if (!xlog_files_list) - xlog_files_list = parray_new(); - - GetXLogSegNo(xlogpos, xlog_segno, instance_config.xlog_seg_size); - - /* - * xlogpos points to the current segment, and we need the finished - previous one - * inless xlogpos points to not 0 offset in segment - */ - if (WalSegmentOffset(xlogpos, instance_config.xlog_seg_size) == 0) - xlog_segno--; - - GetXLogFileName(wal_segment_name, timeline, xlog_segno, - instance_config.xlog_seg_size); - - join_path_components(wal_segment_fullpath, - stream_thread_arg.basedir, wal_segment_name); - - join_path_components(wal_segment_relpath, - PG_XLOG_DIR, wal_segment_name); - - /* append file to filelist */ - file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); - file->name = file->rel_path; - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); - - /* Should we recheck it using stat? */ - file->write_size = instance_config.xlog_seg_size; - file->uncompressed_size = instance_config.xlog_seg_size; - parray_append(xlog_files_list, file); + append_wal_segment(xlog_files_list, timeline, xlogpos, + (char*) stream_thread_arg.basedir, + instance_config.xlog_seg_size); } /* @@ -400,9 +383,60 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption int wait_WAL_streaming_end(parray *backup_files_list) { - parray_concat(backup_files_list, xlog_files_list); - parray_free(xlog_files_list); + pthread_join(stream_thread, NULL); - pthread_join(stream_thread, NULL); - return stream_thread_arg.ret; + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + return stream_thread_arg.ret; } + +/* Append streamed WAL segment to filelist */ +void +append_wal_segment(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) +{ + XLogSegNo xlog_segno; + char wal_segment_name[MAXFNAMELEN]; + char wal_segment_relpath[MAXPGPATH]; + char wal_segment_fullpath[MAXPGPATH]; + pgFile *file = NULL; + + GetXLogSegNo(xlogpos, xlog_segno, xlog_seg_size); + + /* + * xlogpos points to the current segment, and we need the finished - previous one + * inless xlogpos points to not 0 offset in segment + */ + if (WalSegmentOffset(xlogpos, xlog_seg_size) == 0) + xlog_segno--; + + GetXLogFileName(wal_segment_name, timeline, xlog_segno, xlog_seg_size); + + join_path_components(wal_segment_fullpath, basedir, wal_segment_name); + join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); + + file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); + file->name = file->rel_path; + + /* + * Check if file is already in the list + * stop_lsn segment can be added to this list twice, so + * try not to add duplicates + */ + if (parray_bsearch(filelist, file, pgFileCompareRelPathWithExternal)) + { + if (!parray_rm(filelist, file, pgFileCompareRelPathWithExternal)) + elog(ERROR, "Failed to remove duplicate from array of streamed segments: %s", + file->rel_path); + } + + /* calculate crc */ + file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + + /* Should we recheck it using stat? */ + file->write_size = xlog_seg_size; + file->uncompressed_size = xlog_seg_size; + + /* append file to filelist */ + elog(VERBOSE, "Append WAL segment: \"%s\"", wal_segment_relpath); + parray_append(filelist, file); +} \ No newline at end of file From f4900236946be91ffdbb3170fa8310e64e2707e4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 19 Nov 2020 19:06:05 +0300 Subject: [PATCH 1526/2107] minor fixes --- src/stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stream.c b/src/stream.c index f47ea9e6b..145c32b4b 100644 --- a/src/stream.c +++ b/src/stream.c @@ -292,7 +292,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* we assume that we get called once at the end of each segment */ if (segment_finished) { - elog(INFO, _("finished segment at %X/%X (timeline %u)"), + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); append_wal_segment(xlog_files_list, timeline, xlogpos, @@ -439,4 +439,4 @@ append_wal_segment(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char * /* append file to filelist */ elog(VERBOSE, "Append WAL segment: \"%s\"", wal_segment_relpath); parray_append(filelist, file); -} \ No newline at end of file +} From 698ea115bb16db30d80e0adf8a6c76b209d6e148 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Nov 2020 00:05:00 +0300 Subject: [PATCH 1527/2107] 90b6d043bb review --- src/backup.c | 4 ++-- src/pg_probackup.h | 2 +- src/stream.c | 47 ++++++++++++++++++++++++++++------------------ 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/backup.c b/src/backup.c index 0578f2102..5d8348a80 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1382,8 +1382,8 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, { XLogRecPtr res; - res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, - in_prev_segment, instance_config.xlog_seg_size); + res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, + in_prev_segment, instance_config.xlog_seg_size); if (!XLogRecPtrIsInvalid(res)) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 67147a0f3..ce70bf656 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -672,7 +672,7 @@ typedef struct BackupPageHeader2 XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) #else #define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ - XLogSegmentOffset(xlogptr) + ((xlogptr) & ((XLogSegSize) - 1)) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo) #define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ diff --git a/src/stream.c b/src/stream.c index 145c32b4b..dbb149148 100644 --- a/src/stream.c +++ b/src/stream.c @@ -64,9 +64,9 @@ static int checkpoint_timeout(PGconn *backup_conn); static void *StreamLog(void *arg); static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished); -static void append_wal_segment(parray *filelist, uint32 timeline, - XLogRecPtr xlogpos, char *basedir, - uint32 xlog_seg_size); +static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, + XLogRecPtr xlogpos, char *basedir, + uint32 xlog_seg_size); /* * Run IDENTIFY_SYSTEM through a given connection and @@ -244,12 +244,16 @@ StreamLog(void *arg) elog(ERROR, "Problem in receivexlog"); #endif - /* sort xlog_files_list */ + /* be paranoid and sort xlog_files_list, + * so if stop_lsn segno is already in the list, + * then list must be sorted to detect duplicates. + */ parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); - append_wal_segment(xlog_files_list, stream_arg->starttli, - stop_stream_lsn, (char *) stream_arg->basedir, - instance_config.xlog_seg_size); + /* Add the last segment to the list */ + add_walsegment_to_filelist(xlog_files_list, stream_arg->starttli, + stop_stream_lsn, (char *) stream_arg->basedir, + instance_config.xlog_seg_size); /* * TODO: remove redundant WAL segments @@ -295,9 +299,9 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - append_wal_segment(xlog_files_list, timeline, xlogpos, - (char*) stream_thread_arg.basedir, - instance_config.xlog_seg_size); + add_walsegment_to_filelist(xlog_files_list, timeline, xlogpos, + (char*) stream_thread_arg.basedir, + instance_config.xlog_seg_size); } /* @@ -392,19 +396,22 @@ wait_WAL_streaming_end(parray *backup_files_list) /* Append streamed WAL segment to filelist */ void -append_wal_segment(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) +add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) { XLogSegNo xlog_segno; char wal_segment_name[MAXFNAMELEN]; char wal_segment_relpath[MAXPGPATH]; char wal_segment_fullpath[MAXPGPATH]; pgFile *file = NULL; + pgFile **existing_file = NULL; GetXLogSegNo(xlogpos, xlog_segno, xlog_seg_size); /* - * xlogpos points to the current segment, and we need the finished - previous one - * inless xlogpos points to not 0 offset in segment + * When xlogpos points to the zero offset (0/3000000), + * it means that previous segment was just successfully streamed. + * When xlogpos points to the positive offset, + * then current segment is successfully streamed. */ if (WalSegmentOffset(xlogpos, xlog_seg_size) == 0) xlog_segno--; @@ -422,11 +429,16 @@ append_wal_segment(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char * * stop_lsn segment can be added to this list twice, so * try not to add duplicates */ - if (parray_bsearch(filelist, file, pgFileCompareRelPathWithExternal)) + + existing_file = (pgFile **) parray_bsearch(filelist, file, pgFileCompareRelPathWithExternal); + + if (existing_file) { - if (!parray_rm(filelist, file, pgFileCompareRelPathWithExternal)) - elog(ERROR, "Failed to remove duplicate from array of streamed segments: %s", - file->rel_path); + (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + (*existing_file)->write_size = xlog_seg_size; + (*existing_file)->uncompressed_size = xlog_seg_size; + + return; } /* calculate crc */ @@ -437,6 +449,5 @@ append_wal_segment(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char * file->uncompressed_size = xlog_seg_size; /* append file to filelist */ - elog(VERBOSE, "Append WAL segment: \"%s\"", wal_segment_relpath); parray_append(filelist, file); } From cde244e1bab9f25e03380f17c8f7d921f7f2e9c5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 20 Nov 2020 00:05:35 +0300 Subject: [PATCH 1528/2107] tests: added tests.replica.ReplicaTest.test_start_stop_lsn_in_the_same_segno --- tests/replica.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/replica.py b/tests/replica.py index 9a75a7aa0..8baaaa36c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1073,6 +1073,87 @@ def test_replica_toast(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_start_stop_lsn_in_the_same_segno(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'autovacuum': 'off', + 'checkpoint_timeout': '1h', + 'wal_level': 'replica', + 'shared_buffers': '128MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + master.safe_psql( + 'postgres', + 'CHECKPOINT') + + self.wait_until_replica_catch_with_master(master, replica) + + sleep(60) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + # Clean after yourself + self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_replica_promote_1(self): """ From 10bd31097f0df7fc1fdbca0929f0112cbf90e57f Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Fri, 20 Nov 2020 12:24:39 +0700 Subject: [PATCH 1529/2107] tests: stop "node" before deletion test dir --- tests/ptrack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 410f264ad..3aa18c7dd 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3139,7 +3139,7 @@ def test_basic_ptrack_truncate_replica(self): 'select 1') # Clean after yourself - self.del_test_dir(module_name, fname, [master, replica]) + self.del_test_dir(module_name, fname, [master, replica, node]) # @unittest.skip("skip") # @unittest.expectedFailure From bd29752456b352df1f8e53125e4042155a45b7c5 Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 20 Nov 2020 13:42:55 +0300 Subject: [PATCH 1530/2107] Fix test test_basic_ptrack_truncate_replica. Pause replication before taking backup to make datadir comparison more stable. --- tests/ptrack.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 3aa18c7dd..c45ecd6ec 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3108,6 +3108,15 @@ def test_basic_ptrack_truncate_replica(self): # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) + if replica.major_version < 10: + replica.safe_psql( + "postgres", + "select pg_xlog_replay_pause()") + else: + replica.safe_psql( + "postgres", + "select pg_wal_replay_pause()") + self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', options=[ From 1cf233d3a0cb90bcd06e74ce162217a39b5c5a93 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 1 Dec 2020 12:36:36 +0300 Subject: [PATCH 1531/2107] [Issue #281] Non-exclusive locking mode for read-only tasks, such as restore and validate --- src/backup.c | 2 +- src/catalog.c | 464 +++++++++++++++++++++++++++----- src/delete.c | 10 +- src/merge.c | 2 +- src/pg_probackup.h | 8 +- src/restore.c | 17 +- src/validate.c | 6 +- tests/backup.py | 4 +- tests/checkdb.py | 7 +- tests/delta.py | 15 +- tests/helpers/ptrack_helpers.py | 4 +- tests/incr_restore.py | 7 +- tests/locking.py | 44 +-- tests/page.py | 4 +- tests/restore.py | 1 + tests/retention.py | 109 ++++---- 16 files changed, 521 insertions(+), 183 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5d8348a80..557727ef3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -765,7 +765,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, /* Create backup directory and BACKUP_CONTROL_FILE */ if (pgBackupCreateDir(¤t)) elog(ERROR, "Cannot create backup directory"); - if (!lock_backup(¤t, true)) + if (!lock_backup(¤t, true, true)) elog(ERROR, "Cannot lock backup %s directory", base36enc(current.start_time)); write_backup(¤t, true); diff --git a/src/catalog.c b/src/catalog.c index a3b7e85ef..d154253f3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -24,9 +24,14 @@ static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); -static bool exit_hook_registered = false; +static bool backup_lock_exit_hook_registered = false; static parray *lock_files = NULL; +static int lock_backup_exclusive(pgBackup *backup, bool strict); +static bool lock_backup_internal(pgBackup *backup, bool exclusive); +static bool lock_backup_read_only(pgBackup *backup); +static bool wait_read_only_owners(pgBackup *backup); + static timelineInfo * timelineInfoNew(TimeLineID tli) { @@ -131,29 +136,140 @@ write_backup_status(pgBackup *backup, BackupStatus status, tmp->status = backup->status; tmp->root_dir = pgut_strdup(backup->root_dir); + /* lock backup in exclusive mode */ + if (!lock_backup(tmp, strict, true)) + elog(ERROR, "Cannot lock backup %s directory", base36enc(backup->start_time)); + write_backup(tmp, strict); pgBackupFree(tmp); } /* - * Create exclusive lockfile in the backup's directory. + * Lock backup in either exclusive or non-exclusive (read-only) mode. + * "strict" flag allows to ignore "out of space" errors and should be + * used only by DELETE command to free disk space on filled up + * filesystem. + * + * Only read only tasks (validate, restore) are allowed to take non-exclusive locks. + * Changing backup metadata must be done with exclusive lock. + * + * Only one process can hold exclusive lock at any time. + * Exlusive lock - PID of process, holding the lock - is placed in + * lock file: BACKUP_LOCK_FILE. + * + * Multiple proccess are allowed to take non-exclusive locks simultaneously. + * Non-exclusive locks - PIDs of proccesses, holding the lock - are placed in + * separate lock file: BACKUP_RO_LOCK_FILE. + * When taking RO lock, a brief exclusive lock is taken. + * + * TODO: lock-timeout as parameter + * TODO: we must think about more fine grain unlock mechanism - separate unlock_backup() function. */ bool -lock_backup(pgBackup *backup, bool strict) +lock_backup(pgBackup *backup, bool strict, bool exclusive) +{ + int rc; + char lock_file[MAXPGPATH]; + bool enospc_detected = false; + + join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); + + rc = lock_backup_exclusive(backup, strict); + + if (rc == 1) + return false; + else if (rc == 2) + { + enospc_detected = true; + if (strict) + return false; + } + + /* + * We have exclusive lock, now there are following scenarios: + * + * 1. If we are for exlusive lock, then we must open the RO lock file + * and check if any of the processes listed there are still alive. + * If some processes are alive and are not going away in lock_timeout, + * then return false. + * + * 2. If we are here for non-exlusive lock, then write the pid + * into RO lock list and release the exclusive lock. + */ + + if (lock_backup_internal(backup, exclusive)) + { + if (!exclusive) + { + /* release exclusive lock */ + if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) + elog(ERROR, "Could not remove old lock file \"%s\": %s", + lock_file, strerror(errno)); + + /* we are done */ + return true; + } + + /* When locking backup in lax exclusive mode, + * we should wait until all RO locks owners are gone. + */ + if (!strict && enospc_detected) + { + /* We are in lax mode and EONSPC was encountered: once again try to grab exclusive lock, + * because there is a chance that lock_backup_read_only may have freed some space on filesystem, + * thanks to unlinking of BACKUP_RO_LOCK_FILE. + * If somebody concurrently acquired exclusive lock first, then we should give up. + */ + if (lock_backup_exclusive(backup, strict) == 1) + return false; + + return true; + } + } + else + return false; + + /* + * Arrange to unlink the lock file(s) at proc_exit. + */ + if (!backup_lock_exit_hook_registered) + { + atexit(unlink_lock_atexit); + backup_lock_exit_hook_registered = true; + } + + /* Use parray so that the lock files are unlinked in a loop */ + if (lock_files == NULL) + lock_files = parray_new(); + parray_append(lock_files, pgut_strdup(lock_file)); + + return true; +} + +/* Lock backup in exclusive mode + * Result codes: + * 0 Success + * 1 Failed to acquire lock in lock_timeout time + * 2 Failed to acquire lock due to ENOSPC + */ +int +lock_backup_exclusive(pgBackup *backup, bool strict) { char lock_file[MAXPGPATH]; - int fd; + int fd = 0; char buffer[MAXPGPATH * 2 + 256]; - int ntries; + int ntries = LOCK_TIMEOUT; + int log_freq = ntries / 5; int len; int encoded_pid; - pid_t my_pid, - my_p_pid; + pid_t my_p_pid; - join_path_components(lock_file, backup->root_dir, BACKUP_CATALOG_PID); + join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); /* + * TODO: is this stuff with ppid below is relevant for us ? + * * If the PID in the lockfile is our own PID or our parent's or * grandparent's PID, then the file must be stale (probably left over from * a previous system boot cycle). We need to check this because of the @@ -171,7 +287,6 @@ lock_backup(pgBackup *backup, bool strict) * would surely never launch a competing postmaster or pg_ctl process * directly. */ - my_pid = getpid(); #ifndef WIN32 my_p_pid = getppid(); #else @@ -188,8 +303,14 @@ lock_backup(pgBackup *backup, bool strict) * (for example, a non-writable $backup_instance_path directory might cause a failure * that won't go away). 100 tries seems like plenty. */ - for (ntries = 0;; ntries++) + do { + FILE *fp_out = NULL; + + if (interrupted) + elog(ERROR, "Interrupted while locking backup %s", + base36enc(backup->start_time)); + /* * Try to create the lock file --- O_EXCL makes this atomic. * @@ -202,8 +323,11 @@ lock_backup(pgBackup *backup, bool strict) /* * Couldn't create the pid file. Probably it already exists. + * If file already exists or we have some permission problem (???), + * then retry; */ - if ((errno != EEXIST && errno != EACCES) || ntries > 100) +// if ((errno != EEXIST && errno != EACCES)) + if (errno != EEXIST) elog(ERROR, "Could not create lock file \"%s\": %s", lock_file, strerror(errno)); @@ -211,28 +335,38 @@ lock_backup(pgBackup *backup, bool strict) * Read the file to get the old owner's PID. Note race condition * here: file might have been deleted since we tried to create it. */ - fd = fio_open(lock_file, O_RDONLY, FIO_BACKUP_HOST); - if (fd < 0) + + fp_out = fopen(lock_file, "r"); + if (fp_out == NULL) { if (errno == ENOENT) - continue; /* race condition; try again */ - elog(ERROR, "Could not open lock file \"%s\": %s", - lock_file, strerror(errno)); + continue; /* race condition; try again */ + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); } - if ((len = fio_read(fd, buffer, sizeof(buffer) - 1)) < 0) - elog(ERROR, "Could not read lock file \"%s\": %s", - lock_file, strerror(errno)); - fio_close(fd); + len = fread(buffer, 1, sizeof(buffer) - 1, fp_out); + if (ferror(fp_out)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_out); + + /* + * It should be possible only as a result of system crash, + * so its hypothetical owner should be dead by now + */ if (len == 0) - elog(ERROR, "Lock file \"%s\" is empty", lock_file); + { + elog(WARNING, "Lock file \"%s\" is empty", lock_file); + goto grab_lock; + } - buffer[len] = '\0'; encoded_pid = atoi(buffer); if (encoded_pid <= 0) - elog(ERROR, "Bogus data in lock file \"%s\": \"%s\"", + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); + goto grab_lock; + } /* * Check to see if the other process still exists @@ -247,9 +381,19 @@ lock_backup(pgBackup *backup, bool strict) { if (kill(encoded_pid, 0) == 0) { - elog(WARNING, "Process %d is using backup %s and still is running", - encoded_pid, base36enc(backup->start_time)); - return false; + /* complain every fifth interval */ + if ((ntries % log_freq) == 0) + { + elog(WARNING, "Process %d is using backup %s, and is still running", + encoded_pid, base36enc(backup->start_time)); + + elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, base36enc(backup->start_time)); + } + + sleep(1); + + /* try again */ + continue; } else { @@ -262,15 +406,25 @@ lock_backup(pgBackup *backup, bool strict) } } +grab_lock: /* * Looks like nobody's home. Unlink the file and try again to create * it. Need a loop because of possible race condition against other * would-be creators. */ if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) + { + if (errno == ENOENT) + continue; /* race condition, again */ elog(ERROR, "Could not remove old lock file \"%s\": %s", lock_file, strerror(errno)); - } + } + + } while (ntries--); + + /* Failed to acquire exclusive lock in time */ + if (fd <= 0) + return 1; /* * Successfully created the file, now fill it. @@ -284,52 +438,215 @@ lock_backup(pgBackup *backup, bool strict) fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); - /* if write didn't set errno, assume problem is no disk space */ - errno = save_errno ? save_errno : ENOSPC; /* In lax mode if we failed to grab lock because of 'out of space error', * then treat backup as locked. * Only delete command should be run in lax mode. */ - if (!strict && errno == ENOSPC) - return true; - - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + if (!strict && save_errno == ENOSPC) + return 2; + else + elog(ERROR, "Could not write lock file \"%s\": %s", + lock_file, strerror(save_errno)); } + if (fio_flush(fd) != 0) { - int save_errno = errno; + int save_errno = errno; fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); - errno = save_errno; - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + + /* In lax mode if we failed to grab lock because of 'out of space error', + * then treat backup as locked. + * Only delete command should be run in lax mode. + */ + if (!strict && save_errno == ENOSPC) + return 2; + else + elog(ERROR, "Could not flush lock file \"%s\": %s", + lock_file, strerror(save_errno)); } + if (fio_close(fd) != 0) { int save_errno = errno; fio_unlink(lock_file, FIO_BACKUP_HOST); - errno = save_errno; - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + + if (!strict && errno == ENOSPC) + return 2; + else + elog(ERROR, "Could not close lock file \"%s\": %s", + lock_file, strerror(save_errno)); } - /* - * Arrange to unlink the lock file(s) at proc_exit. - */ - if (!exit_hook_registered) + return 0; +} + +/* Wait until all read-only lock owners are gone */ +bool +wait_read_only_owners(pgBackup *backup) +{ + FILE *fp = NULL; + char buffer[256]; + pid_t encoded_pid; + int ntries = LOCK_TIMEOUT; + int log_freq = ntries / 5; + char lock_file[MAXPGPATH]; + + join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); + + fp = fopen(lock_file, "r"); + if (fp == NULL && errno != ENOENT) + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + + /* iterate over pids in lock file */ + while (fp && fgets(buffer, sizeof(buffer), fp)) + { + encoded_pid = atoi(buffer); + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); + continue; + } + + /* wait until RO lock owners go away */ + do + { + if (interrupted) + elog(ERROR, "Interrupted while locking backup %s", + base36enc(backup->start_time)); + + if (encoded_pid != my_pid) + { + if (kill(encoded_pid, 0) == 0) + { + if ((ntries % log_freq) == 0) + { + elog(WARNING, "Process %d is using backup %s in read only mode, and is still running", + encoded_pid, base36enc(backup->start_time)); + + elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, + base36enc(backup->start_time)); + } + + sleep(1); + + /* try again */ + continue; + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } + + /* locker is dead */ + break; + + } while (ntries--); + + if (ntries <= 0) + { + elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns read-only lock", + base36enc(backup->start_time), encoded_pid); + return false; + } + } + + if (fp && ferror(fp)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + + if (fp) + fclose(fp); + + /* unlink RO lock list */ + fio_unlink(lock_file, FIO_BACKUP_HOST); + return true; +} + +bool +lock_backup_internal(pgBackup *backup, bool exclusive) +{ + if (exclusive) + return wait_read_only_owners(backup); + else + return lock_backup_read_only(backup); +} + +bool +lock_backup_read_only(pgBackup *backup) +{ + FILE *fp_in = NULL; + FILE *fp_out = NULL; + char buf_in[256]; + pid_t encoded_pid; + char lock_file[MAXPGPATH]; + + char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ + char lock_file_tmp[MAXPGPATH]; + int buffer_len = 0; + + join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); + snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + + /* open already existing lock files */ + fp_in = fopen(lock_file, "r"); + if (fp_in == NULL && errno != ENOENT) + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + + /* read PIDs of owners */ + while (fp_in && fgets(buf_in, sizeof(buf_in), fp_in)) { - atexit(unlink_lock_atexit); - exit_hook_registered = true; + encoded_pid = atoi(buf_in); + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); + continue; + } + + if (encoded_pid != my_pid) + { + if (kill(encoded_pid, 0) == 0) + { + /* + * Somebody is still using this backup in RO mode, + * copy this pid into a new file. + */ + buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } + } + + if (fp_in) + { + if (ferror(fp_in)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_in); } - /* Use parray so that the lock files are unlinked in a loop */ - if (lock_files == NULL) - lock_files = parray_new(); - parray_append(lock_files, pgut_strdup(lock_file)); + fp_out = fopen(lock_file_tmp, "w"); + if (fp_out == NULL) + elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + /* add my own pid */ + buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); + + /* write out the collected PIDs to temp lock file */ + fwrite(buffer, 1, buffer_len, fp_out); + + if (ferror(fp_out)) + elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); + + if (fclose(fp_out) != 0) + elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + if (rename(lock_file_tmp, lock_file) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + lock_file_tmp, lock_file, strerror(errno)); return true; } @@ -570,7 +887,7 @@ get_backup_filelist(pgBackup *backup, bool strict) * Lock list of backups. Function goes in backward direction. */ void -catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict) +catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict, bool exclusive) { int start_idx, end_idx; @@ -585,7 +902,7 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool str for (i = start_idx; i >= end_idx; i--) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (!lock_backup(backup, strict)) + if (!lock_backup(backup, strict, exclusive)) elog(ERROR, "Cannot lock backup %s directory", base36enc(backup->start_time)); } @@ -1785,21 +2102,23 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* * Save the backup content into BACKUP_CONTROL_FILE. - * TODO: honor the strict flag + * Flag strict allows to ignore "out of space" error + * when attempting to lock backup. Only delete is allowed + * to use this functionality. */ void write_backup(pgBackup *backup, bool strict) { - FILE *fp = NULL; + FILE *fp_out = NULL; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; - char buf[4096]; + char buf[8192]; join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - fp = fopen(path_temp, PG_BINARY_W); - if (fp == NULL) + fp_out = fopen(path_temp, PG_BINARY_W); + if (fp_out == NULL) elog(ERROR, "Cannot open control file \"%s\": %s", path_temp, strerror(errno)); @@ -1807,19 +2126,34 @@ write_backup(pgBackup *backup, bool strict) elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, strerror(errno)); - setvbuf(fp, buf, _IOFBF, sizeof(buf)); + setvbuf(fp_out, buf, _IOFBF, sizeof(buf)); - pgBackupWriteControl(fp, backup); + pgBackupWriteControl(fp_out, backup); - if (fflush(fp) != 0) - elog(ERROR, "Cannot flush control file \"%s\": %s", - path_temp, strerror(errno)); + /* Ignore 'out of space' error in lax mode */ + if (fflush(fp_out) != 0) + { + int elevel = ERROR; + int save_errno = errno; + + if (!strict && (errno == ENOSPC)) + elevel = WARNING; + + elog(elevel, "Cannot flush control file \"%s\": %s", + path_temp, strerror(save_errno)); + + if (!strict && (save_errno == ENOSPC)) + { + fclose(fp_out); + return; + } + } - if (fsync(fileno(fp)) < 0) + if (fsync(fileno(fp_out)) < 0) elog(ERROR, "Cannot sync control file \"%s\": %s", path_temp, strerror(errno)); - if (fclose(fp) != 0) + if (fclose(fp_out) != 0) elog(ERROR, "Cannot close control file \"%s\": %s", path_temp, strerror(errno)); diff --git a/src/delete.c b/src/delete.c index b2960e7a7..b2010c288 100644 --- a/src/delete.c +++ b/src/delete.c @@ -89,7 +89,7 @@ do_delete(time_t backup_id) if (!dry_run) { /* Lock marked for delete backups */ - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false); + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false, true); /* Delete backups from the end of list */ for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) @@ -513,7 +513,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l parray_rm(to_purge_list, full_backup, pgBackupCompareId); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* Consider this extreme case */ // PAGEa1 PAGEb1 both valid @@ -630,7 +630,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) continue; /* Actual purge */ - if (!lock_backup(delete_backup, false)) + if (!lock_backup(delete_backup, false, true)) { /* If the backup still is used, do not interrupt and go to the next */ elog(WARNING, "Cannot lock backup %s directory, skip purging", @@ -975,7 +975,7 @@ do_delete_instance(void) /* Delete all backups. */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); - catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true); + catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true); for (i = 0; i < parray_num(backup_list); i++) { @@ -1081,7 +1081,7 @@ do_delete_status(InstanceConfig *instance_config, const char *status) if (backup->stream) size_to_delete += backup->wal_bytes; - if (!dry_run && lock_backup(backup, false)) + if (!dry_run && lock_backup(backup, false, true)) delete_backup_files(backup); n_deleted++; diff --git a/src/merge.c b/src/merge.c index a453a073c..0c50dd73f 100644 --- a/src/merge.c +++ b/src/merge.c @@ -400,7 +400,7 @@ do_merge(time_t backup_id) parray_append(merge_list, full_backup); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* do actual merge */ merge_chain(merge_list, full_backup, dest_backup); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ce70bf656..d0b4891de 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -66,7 +66,8 @@ extern const char *PROGRAM_EMAIL; #define PG_GLOBAL_DIR "global" #define BACKUP_CONTROL_FILE "backup.control" #define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" -#define BACKUP_CATALOG_PID "backup.pid" +#define BACKUP_LOCK_FILE "backup.pid" +#define BACKUP_RO_LOCK_FILE "backup_ro.pid" #define DATABASE_FILE_LIST "backup_content.control" #define PG_BACKUP_LABEL_FILE "backup_label" #define PG_TABLESPACE_MAP_FILE "tablespace_map" @@ -78,6 +79,7 @@ extern const char *PROGRAM_EMAIL; /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 +#define LOCK_TIMEOUT 30 /* Directory/File permission */ #define DIR_PERMISSION (0700) @@ -869,14 +871,14 @@ extern void write_backup(pgBackup *backup, bool strict); extern void write_backup_status(pgBackup *backup, BackupStatus status, const char *instance_name, bool strict); extern void write_backup_data_bytes(pgBackup *backup); -extern bool lock_backup(pgBackup *backup, bool strict); +extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); extern const char *pgBackupGetBackupMode(pgBackup *backup); extern parray *catalog_get_instance_list(void); extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, - int to_idx, bool strict); + int to_idx, bool strict, bool exclusive); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); diff --git a/src/restore.c b/src/restore.c index 2ade54fa8..7bfc1b7f0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -496,18 +496,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { tmp_backup = (pgBackup *) parray_get(parent_chain, i); - /* Do not interrupt, validate the next backup */ - if (!lock_backup(tmp_backup, true)) + /* lock every backup in chain in read-only mode */ + if (!lock_backup(tmp_backup, true, false)) { - if (params->is_restore) - elog(ERROR, "Cannot lock backup %s directory", - base36enc(tmp_backup->start_time)); - else - { - elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(tmp_backup->start_time)); - continue; - } + elog(ERROR, "Cannot lock backup %s directory", + base36enc(tmp_backup->start_time)); } /* validate datafiles only */ @@ -660,7 +653,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - if (!lock_backup(backup, true)) + if (!lock_backup(backup, true, false)) elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); if (backup->status != BACKUP_STATUS_OK && diff --git a/src/validate.c b/src/validate.c index 8159881a5..30bfbf3e4 100644 --- a/src/validate.c +++ b/src/validate.c @@ -192,7 +192,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK, instance_name, true); + BACKUP_STATUS_OK, instance_name, true); if (corrupted) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); @@ -570,7 +570,7 @@ do_validate_instance(void) base_full_backup = current_backup; /* Do not interrupt, validate the next backup */ - if (!lock_backup(current_backup, true)) + if (!lock_backup(current_backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(current_backup->start_time)); @@ -665,7 +665,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_ORPHAN) { /* Do not interrupt, validate the next backup */ - if (!lock_backup(backup, true)) + if (!lock_backup(backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", base36enc(backup->start_time)); diff --git a/tests/backup.py b/tests/backup.py index 957231cb7..bf0cbca81 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -212,9 +212,7 @@ def test_incremental_backup_corrupt_full(self): except ProbackupException as e: self.assertTrue( "INFO: Validate backups of the instance 'node'" in e.message and - "WARNING: Backup file".format( - file) in e.message and - "is not found".format(file) in e.message and + "WARNING: Backup file" in e.message and "is not found" in e.message and "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and "WARNING: Some backups are not valid" in e.message, diff --git a/tests/checkdb.py b/tests/checkdb.py index db7af1b45..3349ad2ef 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -210,6 +210,7 @@ def test_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -495,6 +496,7 @@ def test_checkdb_sigint_handling(self): self.assertNotIn('connection to client lost', output) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -588,7 +590,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' +# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' ) # >= 10 @@ -655,3 +657,6 @@ def test_checkdb_with_least_privileges(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index 47e6278cd..daa423d49 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -433,11 +433,11 @@ def test_delta_multiple_segments(self): node.safe_psql("postgres", "checkpoint") # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") # delta BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream']) + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) @@ -450,8 +450,10 @@ def test_delta_multiple_segments(self): restored_node, 'somedata_restored') self.restore_node( - backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format( + tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM NODE_RESTORED pgdata_restored = self.pgdata_content(restored_node.data_dir) @@ -461,7 +463,8 @@ def test_delta_multiple_segments(self): restored_node.slow_start() result_new = restored_node.safe_psql( - "postgres", "select * from pgbench_accounts") + "postgres", + "select count(*) from pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 626f15750..79e7086c6 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1102,7 +1102,7 @@ def validate_pb( def delete_pb( self, backup_dir, instance, - backup_id=None, options=[], old_binary=False): + backup_id=None, options=[], old_binary=False, gdb=False): cmd_list = [ 'delete', '-B', backup_dir @@ -1112,7 +1112,7 @@ def delete_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb) def delete_expired( self, backup_dir, instance, options=[], old_binary=False): diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 6fb0edbca..c71b952c8 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -142,6 +142,7 @@ def test_incr_restore_with_tablespace(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -184,6 +185,7 @@ def test_incr_restore_with_tablespace_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], + set_replication=True, pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -245,6 +247,7 @@ def test_incr_restore_with_tablespace_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], + set_replication=True, pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1320,7 +1323,7 @@ def test_incr_checksum_long_xact(self): self.assertEqual( node.safe_psql( 'postgres', - 'select count(*) from t1').rstrip(), + 'select count(*) from t1').decode('utf-8').rstrip(), '1') # Clean after yourself @@ -1488,7 +1491,7 @@ def test_incr_lsn_long_xact_2(self): self.assertEqual( node.safe_psql( 'postgres', - 'select count(*) from t1').rstrip(), + 'select count(*) from t1').decode('utf-8').rstrip(), '1') # Clean after yourself diff --git a/tests/locking.py b/tests/locking.py index 2da2415ea..92c779c8a 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -50,7 +50,7 @@ def test_locking_running_validate_1(self): backup_id = self.show_pb(backup_dir, 'node')[1]['id'] self.assertIn( - "is using backup {0} and still is running".format(backup_id), + "is using backup {0}, and is still running".format(backup_id), validate_output, '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) @@ -61,7 +61,8 @@ def test_locking_running_validate_1(self): 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - # self.del_test_dir(module_name, fname) + gdb.kill() + self.del_test_dir(module_name, fname) def test_locking_running_validate_2(self): """ @@ -129,6 +130,7 @@ def test_locking_running_validate_2(self): 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_locking_running_validate_2_specific_id(self): @@ -227,6 +229,7 @@ def test_locking_running_validate_2_specific_id(self): repr(e.message), self.cmd)) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_locking_running_3(self): @@ -296,6 +299,7 @@ def test_locking_running_3(self): 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_locking_restore_locked(self): @@ -303,8 +307,8 @@ def test_locking_restore_locked(self): make node, take full backup, take two page backups, launch validate on PAGE1 and stop it in the middle, launch restore of PAGE2. - Expect restore to fail because validation of - intermediate backup is impossible + Expect restore to sucseed because read-only locks + do not conflict """ fname = self.id().split('.')[3] node = self.make_simple_node( @@ -334,24 +338,13 @@ def test_locking_restore_locked(self): node.cleanup() - try: - self.restore_node(backup_dir, 'node', node) - self.assertEqual( - 1, 0, - "Expecting Error because restore without whole chain validation " - "is prohibited unless --no-validate provided.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Cannot lock backup {0} directory\n".format(full_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.restore_node(backup_dir, 'node', node) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) - def test_locking_restore_locked_without_validation(self): + def test_concurrent_delete_and_restore(self): """ make node, take full backup, take page backup, launch validate on FULL and stop it in the middle, @@ -376,10 +369,11 @@ def test_locking_restore_locked_without_validation(self): # PAGE1 restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - gdb = self.validate_pb( + gdb = self.delete_pb( backup_dir, 'node', backup_id=backup_id, gdb=True) - gdb.set_breakpoint('pgBackupValidate') + # gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() node.cleanup() @@ -397,13 +391,14 @@ def test_locking_restore_locked_without_validation(self): self.assertTrue( "Backup {0} is used without validation".format( restore_id) in e.message and - 'is using backup {0} and still is running'.format( + 'is using backup {0}, and is still running'.format( backup_id) in e.message and 'ERROR: Cannot lock backup' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_locking_concurrent_validate_and_backup(self): @@ -439,6 +434,7 @@ def test_locking_concurrent_validate_and_backup(self): self.backup_node(backup_dir, 'node', node, backup_type='page') # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_locking_concurren_restore_and_delete(self): @@ -467,7 +463,6 @@ def test_locking_concurren_restore_and_delete(self): gdb.set_breakpoint('create_data_directories') gdb.run_until_break() - # This PAGE backup is expected to be successfull try: self.delete_pb(backup_dir, 'node', full_id) self.assertEqual( @@ -483,6 +478,7 @@ def test_locking_concurren_restore_and_delete(self): repr(e.message), self.cmd)) # Clean after yourself + gdb.kill() self.del_test_dir(module_name, fname) def test_backup_directory_name(self): @@ -538,3 +534,7 @@ def test_backup_directory_name(self): # Clean after yourself self.del_test_dir(module_name, fname) + +# TODO: +# test that concurrent validation and restore are not locking each other +# check that quick exclusive lock, when taking RO-lock, is really quick diff --git a/tests/page.py b/tests/page.py index 201f825e8..323c0a6de 100644 --- a/tests/page.py +++ b/tests/page.py @@ -393,7 +393,7 @@ def test_page_multiple_segments(self): pgbench.wait() # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") # PAGE BACKUP self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -422,7 +422,7 @@ def test_page_multiple_segments(self): restored_node.slow_start() result_new = restored_node.safe_psql( - "postgres", "select * from pgbench_accounts") + "postgres", "select count(*) from pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') diff --git a/tests/restore.py b/tests/restore.py index 2a5fac6a6..14afb993f 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3093,6 +3093,7 @@ def test_missing_database_map(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " diff --git a/tests/retention.py b/tests/retention.py index 6ab796b48..6dc8536ca 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -41,7 +41,7 @@ def test_retention_redundancy_1(self): output_before = self.show_archive(backup_dir, 'node', tli=1) # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) @@ -142,13 +142,13 @@ def test_retention_window_3(self): node.slow_start() # take FULL BACKUP - backup_id_1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take second FULL BACKUP - backup_id_2 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take third FULL BACKUP - backup_id_3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): @@ -189,7 +189,7 @@ def test_retention_window_4(self): node.slow_start() # take FULL BACKUPs - backup_id_1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) backup_id_2 = self.backup_node(backup_dir, 'node', node) @@ -444,8 +444,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -632,7 +631,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): node.pgbench_init(scale=5) # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() @@ -663,13 +662,13 @@ def test_window_merge_interleaved_incremental_chains_1(self): # PAGEa1 ERROR # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) @@ -711,7 +710,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=3))) - output = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--retention-window=1', '--expired', '--merge-expired']) @@ -1305,26 +1304,26 @@ def test_window_chains(self): node.pgbench_init(scale=3) # Chain A - backup_id_a = self.backup_node(backup_dir, 'node', node) - page_id_a1 = self.backup_node( + self.backup_node(backup_dir, 'node', node) + self.backup_node( backup_dir, 'node', node, backup_type='page') - page_id_a2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Chain B - backup_id_b = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) @@ -1347,7 +1346,7 @@ def test_window_chains(self): conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=3))) - output = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=[ '--retention-window=1', '--expired', @@ -1391,26 +1390,26 @@ def test_window_chains_1(self): node.pgbench_init(scale=3) # Chain A - backup_id_a = self.backup_node(backup_dir, 'node', node) - page_id_a1 = self.backup_node( + self.backup_node(backup_dir, 'node', node) + self.backup_node( backup_dir, 'node', node, backup_type='page') - page_id_a2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Chain B - backup_id_b = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta') - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='delta') - pgdata = self.pgdata_content(node.data_dir) + self.pgdata_content(node.data_dir) # Purge backups backups = os.path.join(backup_dir, 'backups', 'node') @@ -1483,15 +1482,15 @@ def test_window_error_backups(self): node.slow_start() # Take FULL BACKUPs - backup_id_a1 = self.backup_node(backup_dir, 'node', node) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) + self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, backup_type='page') - page_id_a3 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # Clean after yourself self.del_test_dir(module_name, fname) @@ -1516,7 +1515,7 @@ def test_window_error_backups_1(self): node.slow_start() # Take FULL BACKUP - full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take PAGE BACKUP gdb = self.backup_node( @@ -1528,15 +1527,15 @@ def test_window_error_backups_1(self): gdb._execute('signal SIGINT') gdb.continue_execution_until_error() - page_id = self.show_pb(backup_dir, 'node')[1]['id'] + self.show_pb(backup_dir, 'node')[1]['id'] # Take DELTA backup - delta_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) # Take FULL BACKUP - full2_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) @@ -1563,7 +1562,7 @@ def test_window_error_backups_2(self): node.slow_start() # Take FULL BACKUP - full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take PAGE BACKUP gdb = self.backup_node( @@ -1574,7 +1573,7 @@ def test_window_error_backups_2(self): gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() - page_id = self.show_pb(backup_dir, 'node')[1]['id'] + self.show_pb(backup_dir, 'node')[1]['id'] if self.get_version(node) < 90600: node.safe_psql( @@ -1582,7 +1581,7 @@ def test_window_error_backups_2(self): 'SELECT pg_catalog.pg_stop_backup()') # Take DELTA backup - delta_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) @@ -1630,7 +1629,7 @@ def test_retention_redundancy_overlapping_chains(self): self.backup_node(backup_dir, 'node', node, backup_type="page") # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) @@ -1639,7 +1638,7 @@ def test_retention_redundancy_overlapping_chains(self): # Clean after yourself self.del_test_dir(module_name, fname) - def test_retention_redundancy_overlapping_chains(self): + def test_retention_redundancy_overlapping_chains_1(self): """""" fname = self.id().split('.')[3] node = self.make_simple_node( @@ -1678,7 +1677,7 @@ def test_retention_redundancy_overlapping_chains(self): self.backup_node(backup_dir, 'node', node, backup_type="page") # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) @@ -1869,7 +1868,7 @@ def test_wal_depth_1(self): # FULL node.pgbench_init(scale=1) - B1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # PAGE node.pgbench_init(scale=1) @@ -1885,12 +1884,12 @@ def test_wal_depth_1(self): node.pgbench_init(scale=1) - B3 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') node.pgbench_init(scale=1) - B4 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Timeline 2 @@ -1940,11 +1939,11 @@ def test_wal_depth_1(self): node_restored.slow_start() node_restored.pgbench_init(scale=1) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) node.pgbench_init(scale=1) - B6 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] @@ -2007,9 +2006,9 @@ def test_wal_purge(self): node.pgbench_init(scale=5) # B2 FULL on TLI1 - B2 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) - B3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) self.delete_pb(backup_dir, 'node', options=['--delete-wal']) @@ -2023,7 +2022,7 @@ def test_wal_purge(self): backup_dir, 'node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-timeline=1', '--recovery-target-action=promote']) self.assertIn( @@ -2039,11 +2038,11 @@ def test_wal_purge(self): "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) - B4 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() @@ -2086,7 +2085,7 @@ def test_wal_purge(self): node_tli4.pgbench_init(scale=5) - B6 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() @@ -2232,7 +2231,7 @@ def test_wal_depth_2(self): # B2 FULL on TLI1 B2 = self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) - B3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) # TLI 2 @@ -2244,7 +2243,7 @@ def test_wal_depth_2(self): backup_dir, 'node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-timeline=1', '--recovery-target-action=promote']) self.assertIn( @@ -2264,7 +2263,7 @@ def test_wal_depth_2(self): backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() @@ -2307,7 +2306,7 @@ def test_wal_depth_2(self): node_tli4.pgbench_init(scale=5) - B6 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() From 6c21833a177a569efd608c87148e2df1c99f803e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 11:15:21 +0300 Subject: [PATCH 1532/2107] [Issue #272] fix DST time skew, store backup metadata timestamps in UTC --- src/backup.c | 2 +- src/catalog.c | 39 ++++++++++++++-------------- src/delete.c | 4 +-- src/parsexlog.c | 6 ++--- src/pg_probackup.h | 11 ++++---- src/restore.c | 4 +-- src/show.c | 12 ++++----- src/utils/configuration.c | 53 ++++++++++++++++++++++++++++----------- src/utils/configuration.h | 2 +- 9 files changed, 78 insertions(+), 55 deletions(-) diff --git a/src/backup.c b/src/backup.c index 557727ef3..393e08f53 100644 --- a/src/backup.c +++ b/src/backup.c @@ -133,7 +133,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time); + time2iso(label, lengthof(label), current.start_time, false); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); diff --git a/src/catalog.c b/src/catalog.c index d154253f3..4da6ff053 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -789,7 +789,7 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) } else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) { - elog(VERBOSE, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", + elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", base36enc(backup->start_time), backup_conf_path); } @@ -1952,7 +1952,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) { char expire_timestamp[100]; - time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time); + time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time, false); elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), expire_timestamp); } @@ -2003,7 +2003,7 @@ add_note(pgBackup *target_backup, char *note) * Write information about backup.in to stream "out". */ void -pgBackupWriteControl(FILE *out, pgBackup *backup) +pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) { char timestamp[100]; @@ -2035,27 +2035,27 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, utc); fio_fprintf(out, "start-time = '%s'\n", timestamp); if (backup->merge_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->merge_time); + time2iso(timestamp, lengthof(timestamp), backup->merge_time, utc); fio_fprintf(out, "merge-time = '%s'\n", timestamp); } if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, utc); fio_fprintf(out, "end-time = '%s'\n", timestamp); } fio_fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, utc); fio_fprintf(out, "recovery-time = '%s'\n", timestamp); } if (backup->expire_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->expire_time); + time2iso(timestamp, lengthof(timestamp), backup->expire_time, utc); fio_fprintf(out, "expire-time = '%s'\n", timestamp); } @@ -2109,7 +2109,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) void write_backup(pgBackup *backup, bool strict) { - FILE *fp_out = NULL; + FILE *fp = NULL; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; char buf[8192]; @@ -2117,8 +2117,8 @@ write_backup(pgBackup *backup, bool strict) join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - fp_out = fopen(path_temp, PG_BINARY_W); - if (fp_out == NULL) + fp = fopen(path_temp, PG_BINARY_W); + if (fp == NULL) elog(ERROR, "Cannot open control file \"%s\": %s", path_temp, strerror(errno)); @@ -2126,12 +2126,12 @@ write_backup(pgBackup *backup, bool strict) elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, strerror(errno)); - setvbuf(fp_out, buf, _IOFBF, sizeof(buf)); + setvbuf(fp, buf, _IOFBF, sizeof(buf)); - pgBackupWriteControl(fp_out, backup); + pgBackupWriteControl(fp, backup, true); /* Ignore 'out of space' error in lax mode */ - if (fflush(fp_out) != 0) + if (fflush(fp) != 0) { int elevel = ERROR; int save_errno = errno; @@ -2144,17 +2144,18 @@ write_backup(pgBackup *backup, bool strict) if (!strict && (save_errno == ENOSPC)) { - fclose(fp_out); + fclose(fp); + fio_unlink(path_temp, FIO_BACKUP_HOST); return; } } - if (fsync(fileno(fp_out)) < 0) - elog(ERROR, "Cannot sync control file \"%s\": %s", + if (fclose(fp) != 0) + elog(ERROR, "Cannot close control file \"%s\": %s", path_temp, strerror(errno)); - if (fclose(fp_out) != 0) - elog(ERROR, "Cannot close control file \"%s\": %s", + if (fio_sync(path_temp, FIO_BACKUP_HOST) < 0) + elog(ERROR, "Cannot sync control file \"%s\": %s", path_temp, strerror(errno)); if (rename(path_temp, path) < 0) diff --git a/src/delete.c b/src/delete.c index b2010c288..a6d6b51ba 100644 --- a/src/delete.c +++ b/src/delete.c @@ -316,7 +316,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg (backup->expire_time > current_time)) { char expire_timestamp[100]; - time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time); + time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time, false); elog(LOG, "Backup %s is pinned until '%s', retain", base36enc(backup->start_time), expire_timestamp); @@ -740,7 +740,7 @@ delete_backup_files(pgBackup *backup) return; } - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); elog(INFO, "Delete: %s %s", base36enc(backup->start_time), timestamp); diff --git a/src/parsexlog.c b/src/parsexlog.c index 38c62c6a7..41a410d30 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -480,7 +480,7 @@ validate_wal(pgBackup *backup, const char *archivedir, last_rec.rec_xid = backup->recovery_xid; last_rec.rec_lsn = backup->stop_lsn; - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, false); if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) || (target_time != 0 && backup->recovery_time >= target_time) @@ -493,7 +493,7 @@ validate_wal(pgBackup *backup, const char *archivedir, InvalidXLogRecPtr, true, validateXLogRecord, &last_rec, true); if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_rec.rec_time)); + timestamptz_to_time_t(last_rec.rec_time), false); /* There are all needed WAL records */ if (all_wal) @@ -508,7 +508,7 @@ validate_wal(pgBackup *backup, const char *archivedir, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time, false); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "Not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d0b4891de..edd03304f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -394,10 +394,9 @@ struct pgBackup TimeLineID tli; /* timeline of start and stop backup lsns */ XLogRecPtr start_lsn; /* backup's starting transaction log location */ XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ - time_t start_time; /* since this moment backup has status - * BACKUP_STATUS_RUNNING */ - time_t merge_dest_backup; /* start_time of incremental backup, - * this backup is merging with. + time_t start_time; /* UTC time of backup creation */ + time_t merge_dest_backup; /* start_time of incremental backup with + * which this backup is merging with. * Only available for FULL backups * with MERGING or MERGED statuses */ time_t merge_time; /* the moment when merge was started or 0 */ @@ -892,7 +891,7 @@ extern void do_set_backup(const char *instance_name, time_t backup_id, extern void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params); extern void add_note(pgBackup *target_backup, char *note); -extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); @@ -1081,7 +1080,7 @@ extern void set_min_recovery_point(pgFile *file, const char *backup_path, extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, const char *to_fullpath, fio_location to_location, pgFile *file); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); extern BackupStatus str2status(const char *status); extern const char *base36enc(long unsigned int value); diff --git a/src/restore.c b/src/restore.c index 7bfc1b7f0..81b50cb30 100644 --- a/src/restore.c +++ b/src/restore.c @@ -643,7 +643,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time_t start_time, end_time; /* Preparations for actual restoring */ - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); + time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); elog(INFO, "Restoring the database from backup at %s", timestamp); dest_files = get_backup_filelist(dest_backup, true); @@ -1465,7 +1465,7 @@ pg12_recovery_config(pgBackup *backup, bool add_include) { char current_time_str[100]; - time2iso(current_time_str, lengthof(current_time_str), current_time); + time2iso(current_time_str, lengthof(current_time_str), current_time, false); snprintf(postgres_auto_path, lengthof(postgres_auto_path), "%s/postgresql.auto.conf", instance_config.pgdata); diff --git a/src/show.c b/src/show.c index ce6604ac3..c1482772e 100644 --- a/src/show.c +++ b/src/show.c @@ -374,12 +374,12 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, false); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -388,13 +388,13 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); json_add_value(buf, "recovery-time", timestamp, json_level, true); } if (backup->expire_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->expire_time); + time2iso(timestamp, lengthof(timestamp), backup->expire_time, false); json_add_value(buf, "expire-time", timestamp, json_level, true); } @@ -482,7 +482,7 @@ show_backup(const char *instance_name, time_t requested_backup_id) } if (show_format == SHOW_PLAIN) - pgBackupWriteControl(stdout, backup); + pgBackupWriteControl(stdout, backup, false); else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -550,7 +550,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na /* Recovery Time */ if (backup->recovery_time != (time_t) 0) time2iso(row->recovery_time, lengthof(row->recovery_time), - backup->recovery_time); + backup->recovery_time, false); else StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time)); widths[cur] = Max(widths[cur], strlen(row->recovery_time)); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 1ef332ed5..a7dbfab20 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -679,7 +679,7 @@ option_get_value(ConfigOption *opt) if (t > 0) { timestamp = palloc(100); - time2iso(timestamp, 100, t); + time2iso(timestamp, 100, t, false); } else timestamp = palloc0(1 /* just null termination */); @@ -1111,6 +1111,8 @@ parse_time(const char *value, time_t *result, bool utc_default) struct tm tm; char junk[2]; + char *local_tz = getenv("TZ"); + /* tmp = replace( value, !isalnum, ' ' ) */ tmp = pgut_malloc(strlen(value) + + 1); len = 0; @@ -1221,26 +1223,27 @@ parse_time(const char *value, time_t *result, bool utc_default) /* determine whether Daylight Saving Time is in effect */ tm.tm_isdst = -1; + /* set timezone to UTC */ + setenv("TZ", "UTC", 1); + + /* convert time to utc unix time */ *result = mktime(&tm); + /* return old timezone back if any */ + if (local_tz) + setenv("TZ", local_tz, 1); + else + unsetenv("TZ"); + /* adjust time zone */ if (tz_set || utc_default) { - time_t ltime = time(NULL); - struct tm *ptm = gmtime(<ime); - time_t gmt = mktime(ptm); - time_t offset; - /* UTC time */ *result -= tz; - - /* Get local time */ - ptm = localtime(<ime); - offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); - - *result += offset; } + pg_free(local_tz); + return true; } @@ -1462,14 +1465,32 @@ convert_from_base_unit_u(uint64 base_value, int base_unit, * Convert time_t value to ISO-8601 format string. Always set timezone offset. */ void -time2iso(char *buf, size_t len, time_t time) +time2iso(char *buf, size_t len, time_t time, bool utc) { - struct tm *ptm = gmtime(&time); - time_t gmt = mktime(ptm); + struct tm *ptm = NULL; + time_t gmt; time_t offset; char *ptr = buf; + char *local_tz = getenv("TZ"); + /* set timezone to UTC if requested */ + if (utc) + setenv("TZ", "UTC", 1); + + ptm = gmtime(&time); + gmt = mktime(ptm); ptm = localtime(&time); + + if (utc) + { + /* return old timezone back if any */ + if (local_tz) + setenv("TZ", local_tz, 1); + else + unsetenv("TZ"); + } + + /* adjust timezone offset */ offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); @@ -1485,4 +1506,6 @@ time2iso(char *buf, size_t len, time_t time) snprintf(ptr, len - (ptr - buf), ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } + + pg_free(local_tz); } diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 46b5d6c1b..eea8c7746 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -96,7 +96,7 @@ extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg); extern bool parse_lsn(const char *value, XLogRecPtr *result); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern void convert_from_base_unit(int64 base_value, int base_unit, int64 *value, const char **unit); From d3425bdc882fff4e584edff489ffe119e2c0279c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 18:17:31 +0300 Subject: [PATCH 1533/2107] [Issue #272] if timezone offset is not specified, treat it as local timezone --- src/utils/configuration.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index a7dbfab20..76920a9fd 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1223,8 +1223,10 @@ parse_time(const char *value, time_t *result, bool utc_default) /* determine whether Daylight Saving Time is in effect */ tm.tm_isdst = -1; - /* set timezone to UTC */ - setenv("TZ", "UTC", 1); + /* if tz is not set, treat it as local timezone */ + if (tz_set) + /* set timezone to UTC */ + setenv("TZ", "UTC", 1); /* convert time to utc unix time */ *result = mktime(&tm); From 91bcb9bdd96becde1d78e7c71df6df4063917944 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 18:17:53 +0300 Subject: [PATCH 1534/2107] [Issue #272] documentation update --- doc/pgprobackup.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index c03688bd7..5a82f6ba9 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4409,6 +4409,10 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which recovery will proceed. + If timezone offset is not specified, local timezone is used. + + + Example: --recovery-target-time='2020-01-01 00:00:00+03' @@ -4595,6 +4599,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which the backup will stay pinned. Must be an ISO-8601 complaint timestamp. + If timezone offset is not specified, local timezone is used. Example: --expire-time='2020-01-01 00:00:00+03' From d11e3398f0d7f05f83ecdb5147105563b1f19078 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 18:32:26 +0300 Subject: [PATCH 1535/2107] [Issue #238] in PG>=12 use postgresql.auto.conf for recovery parameters --- src/pg_probackup.c | 7 +- src/pg_probackup.h | 12 + src/restore.c | 463 ++++++++++++++++++-------------- tests/helpers/ptrack_helpers.py | 43 ++- tests/restore.py | 222 ++++++++++++++- 5 files changed, 520 insertions(+), 227 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f2aca75fd..e34195ae6 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -99,7 +99,7 @@ static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; time_t current_time = 0; -bool restore_as_replica = false; +static bool restore_as_replica = false; bool no_validate = false; IncrRestoreMode incremental_mode = INCR_NONE; @@ -705,15 +705,14 @@ main(int argc, char *argv[]) if (force) no_validate = true; - if (replication_slot != NULL) - restore_as_replica = true; - /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); restore_params->force = force; restore_params->no_validate = no_validate; restore_params->restore_as_replica = restore_as_replica; + restore_params->recovery_settings_mode = DEFAULT; + restore_params->primary_slot_name = replication_slot; restore_params->skip_block_validation = skip_block_validation; restore_params->skip_external_dirs = skip_external_dirs; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index edd03304f..4c670b770 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -146,6 +146,16 @@ typedef enum PartialRestoreType EXCLUDE, } PartialRestoreType; +typedef enum RecoverySettingsMode +{ + DEFAULT, /* not set */ + DONTWRITE, /* explicitly forbid to update recovery settings */ + //TODO Should we always clean/preserve old recovery settings, + // or make it configurable? + PITR_REQUESTED, /* can be set based on other parameters + * if not explicitly forbidden */ +} RecoverySettingsMode; + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, @@ -490,6 +500,8 @@ typedef struct pgRestoreParams bool is_restore; bool no_validate; bool restore_as_replica; + //TODO maybe somehow add restore_as_replica as one of RecoverySettingsModes + RecoverySettingsMode recovery_settings_mode; bool skip_external_dirs; bool skip_block_validation; //Start using it const char *restore_command; diff --git a/src/restore.c b/src/restore.c index 81b50cb30..7220eb469 100644 --- a/src/restore.c +++ b/src/restore.c @@ -39,13 +39,29 @@ typedef struct int ret; } restore_files_arg; + +static void +print_recovery_settings(FILE *fp, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +static void +print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params); + +#if PG_VERSION_NUM >= 120000 +static void +update_recovery_options(pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +#else +static void +update_recovery_options_before_v12(pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +#endif + static void create_recovery_conf(time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); -static void pg12_recovery_config(pgBackup *backup, bool add_include); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, @@ -597,6 +613,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, instance_config.pgdata, no_sync); + //TODO rename and update comment /* Create recovery.conf with given recovery target parameters */ create_recovery_conf(target_backup_id, rt, dest_backup, params); } @@ -1204,13 +1221,9 @@ create_recovery_conf(time_t backup_id, pgBackup *backup, pgRestoreParams *params) { - char path[MAXPGPATH]; - FILE *fp; - bool pitr_requested; bool target_latest; bool target_immediate; bool restore_command_provided = false; - char restore_command_guc[16384]; if (instance_config.restore_command && (pg_strcasecmp(instance_config.restore_command, "none") != 0)) @@ -1242,265 +1255,323 @@ create_recovery_conf(time_t backup_id, * We will get a replica that is "in the future" to the master. * We accept this risk because its probability is low. */ - pitr_requested = !backup->stream || rt->time_string || + if (!backup->stream || rt->time_string || rt->xid_string || rt->lsn_string || rt->target_name || - target_immediate || target_latest || restore_command_provided; - - /* No need to generate recovery.conf at all. */ - if (!(pitr_requested || params->restore_as_replica)) - { - /* - * Restoring STREAM backup without PITR and not as replica, - * recovery.signal and standby.signal for PG12 are not needed - * - * We do not add "include" option in this case because - * here we are creating empty "probackup_recovery.conf" - * to handle possible already existing "include" - * directive pointing to "probackup_recovery.conf". - * If don`t do that, recovery will fail. - */ - pg12_recovery_config(backup, false); - return; - } + target_immediate || target_latest || restore_command_provided) + params->recovery_settings_mode = PITR_REQUESTED; elog(LOG, "----------------------------------------"); + #if PG_VERSION_NUM >= 120000 - elog(LOG, "creating probackup_recovery.conf"); - pg12_recovery_config(backup, true); - snprintf(path, lengthof(path), "%s/probackup_recovery.conf", instance_config.pgdata); + update_recovery_options(backup, params, rt); #else - elog(LOG, "creating recovery.conf"); - snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); + update_recovery_options_before_v12(backup, params, rt); #endif +} - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); -#if PG_VERSION_NUM >= 120000 - fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); -#else - fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); -#endif +/* TODO get rid of using global variables: instance_config, backup_path, instance_name */ +static void +print_recovery_settings(FILE *fp, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) +{ + char restore_command_guc[16384]; + fio_fprintf(fp, "## recovery settings\n"); - /* construct restore_command */ - if (pitr_requested) + /* If restore_command is provided, use it. Otherwise construct it from scratch. */ + if (instance_config.restore_command && + (pg_strcasecmp(instance_config.restore_command, "none") != 0)) + sprintf(restore_command_guc, "%s", instance_config.restore_command); + else { - fio_fprintf(fp, "\n## recovery settings\n"); - /* If restore_command is provided, use it. Otherwise construct it from scratch. */ - if (restore_command_provided) - sprintf(restore_command_guc, "%s", instance_config.restore_command); - else + /* default cmdline, ok for local restore */ + sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " + "--wal-file-path=%%p --wal-file-name=%%f", + PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, + backup_path, instance_name); + + /* append --remote-* parameters provided via --archive-* settings */ + if (instance_config.archive.host) { - /* default cmdline, ok for local restore */ - sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " - "--wal-file-path=%%p --wal-file-name=%%f", - PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, - backup_path, instance_name); - - /* append --remote-* parameters provided via --archive-* settings */ - if (instance_config.archive.host) - { - strcat(restore_command_guc, " --remote-host="); - strcat(restore_command_guc, instance_config.archive.host); - } + strcat(restore_command_guc, " --remote-host="); + strcat(restore_command_guc, instance_config.archive.host); + } - if (instance_config.archive.port) - { - strcat(restore_command_guc, " --remote-port="); - strcat(restore_command_guc, instance_config.archive.port); - } + if (instance_config.archive.port) + { + strcat(restore_command_guc, " --remote-port="); + strcat(restore_command_guc, instance_config.archive.port); + } - if (instance_config.archive.user) - { - strcat(restore_command_guc, " --remote-user="); - strcat(restore_command_guc, instance_config.archive.user); - } + if (instance_config.archive.user) + { + strcat(restore_command_guc, " --remote-user="); + strcat(restore_command_guc, instance_config.archive.user); } + } - /* - * We've already checked that only one of the four following mutually - * exclusive options is specified, so the order of calls is insignificant. - */ - if (rt->target_name) - fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->target_name) + fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); - if (rt->time_string) - fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); + if (rt->time_string) + fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); - if (rt->xid_string) - fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); + if (rt->xid_string) + fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); - if (rt->lsn_string) - fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); + if (rt->lsn_string) + fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - if (rt->target_stop && target_immediate) - fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); + if (rt->target_stop && (strcmp(rt->target_stop, "immediate") == 0)) + fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); - if (rt->inclusive_specified) - fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", - rt->target_inclusive ? "true" : "false"); + if (rt->inclusive_specified) + fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->target_inclusive ? "true" : "false"); + + if (rt->target_tli) + fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + else + { +#if PG_VERSION_NUM >= 120000 - if (rt->target_tli) - fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); - else - { /* * In PG12 default recovery target timeline was changed to 'latest', which * is extremely risky. Explicitly preserve old behavior of recovering to current * timneline for PG12. */ -#if PG_VERSION_NUM >= 120000 fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); #endif - } - - if (rt->target_action) - fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); - else - /* default recovery_target_action is 'pause' */ - fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); } - if (pitr_requested) - { - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); - } + if (rt->target_action) + fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); + else + /* default recovery_target_action is 'pause' */ + fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); + + elog(LOG, "Setting restore_command to '%s'", restore_command_guc); + fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); +} + +static void +print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params) +{ + fio_fprintf(fp, "\n## standby settings\n"); + if (params->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); + else if (backup->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + + if (params->primary_slot_name != NULL) + fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); +} - if (params->restore_as_replica) - { - fio_fprintf(fp, "\n## standby settings\n"); - /* standby_mode was removed in PG12 */ #if PG_VERSION_NUM < 120000 - fio_fprintf(fp, "standby_mode = 'on'\n"); -#endif +static void +update_recovery_options_before_v12(pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) +{ + FILE *fp; + char path[MAXPGPATH]; + + elog(LOG, "update recovery settings in recovery.conf"); + snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); - if (params->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); - else if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + fp = fio_fopen(path, "w", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); + + fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (params->recovery_settings_mode == PITR_REQUESTED) + print_recovery_settings(fp, backup, params, rt); - if (params->primary_slot_name != NULL) - fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); + if (params->restore_as_replica) + { + print_standby_settings_common(fp, backup, params); + fio_fprintf(fp, "standby_mode = 'on'\n"); } if (fio_fflush(fp) != 0 || fio_fclose(fp)) elog(ERROR, "cannot write file \"%s\": %s", path, strerror(errno)); +} +#endif +/* + * Read postgresql.auto.conf, clean old recovery options, + * to avoid unexpected intersections. + * Write recovery options for this backup. + */ #if PG_VERSION_NUM >= 120000 - /* - * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. - * In older versions presense of recovery.conf alone was enough. - * To keep behaviour consistent with older versions, - * we are forced to create "recovery.signal" - * even when only restore_command is provided. - * Presense of "recovery.signal" by itself determine only - * one thing: do PostgreSQL must switch to a new timeline - * after successfull recovery or not? - */ - if (pitr_requested) - { - elog(LOG, "creating recovery.signal file"); - snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); +static void +update_recovery_options(pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); +{ + char postgres_auto_path[MAXPGPATH]; + char postgres_auto_path_tmp[MAXPGPATH]; + char path[MAXPGPATH]; + FILE *fp; + FILE *fp_tmp; + struct stat st; + char current_time_str[100]; + /* postgresql.auto.conf parsing */ + char line[16384] = "\0"; + char *buf = NULL; + int buf_len = 0; + int buf_len_max = 16384; - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(LOG, "update recovery settings in postgresql.auto.conf"); + + time2iso(current_time_str, lengthof(current_time_str), current_time, false); + + snprintf(postgres_auto_path, lengthof(postgres_auto_path), + "%s/postgresql.auto.conf", instance_config.pgdata); + + if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) + { + /* file not found is not an error case */ + if (errno != ENOENT) + elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path, strerror(errno)); } - if (params->restore_as_replica) + fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); + if (fp == NULL && errno != ENOENT) + elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); + + sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); + fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); + if (fp_tmp == NULL) + elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); + + while (fp && fgets(line, lengthof(line), fp)) { - elog(LOG, "creating standby.signal file"); - snprintf(path, lengthof(path), "%s/standby.signal", instance_config.pgdata); + /* ignore "include 'probackup_recovery.conf'" directive */ + if (strstr(line, "include") && + strstr(line, "probackup_recovery.conf")) + { + continue; + } - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); + /* ignore already existing recovery options */ + if (strstr(line, "restore_command") || + strstr(line, "recovery_target")) + { + continue; + } - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, - strerror(errno)); - } -#endif -} + if (!buf) + buf = pgut_malloc(buf_len_max); -/* - * Create empty probackup_recovery.conf in PGDATA and - * add "include" directive to postgresql.auto.conf + /* avoid buffer overflow */ + if ((buf_len + strlen(line)) >= buf_len_max) + { + buf_len_max += (buf_len + strlen(line)) *2; + buf = pgut_realloc(buf, buf_len_max); + } - * When restoring PG12 we always(!) must do this, even - * when restoring STREAM backup without PITR or replica options - * because restored instance may have been previously backed up - * and restored again and user didn`t cleaned up postgresql.auto.conf. + buf_len += snprintf(buf+buf_len, sizeof(line), "%s", line); + } - * So for recovery to work regardless of all this factors - * we must always create empty probackup_recovery.conf file. - */ -static void -pg12_recovery_config(pgBackup *backup, bool add_include) -{ -#if PG_VERSION_NUM >= 120000 - char probackup_recovery_path[MAXPGPATH]; - char postgres_auto_path[MAXPGPATH]; - FILE *fp; + /* close input postgresql.auto.conf */ + if (fp) + fio_close_stream(fp); - if (add_include) - { - char current_time_str[100]; + /* TODO: detect remote error */ + if (buf_len > 0) + fio_fwrite(fp_tmp, buf, buf_len); - time2iso(current_time_str, lengthof(current_time_str), current_time, false); + if (fio_fflush(fp_tmp) != 0 || + fio_fclose(fp_tmp)) + elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path_tmp, + strerror(errno)); + pg_free(buf); + + if (fio_rename(postgres_auto_path_tmp, postgres_auto_path, FIO_DB_HOST) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + postgres_auto_path_tmp, postgres_auto_path, strerror(errno)); - snprintf(postgres_auto_path, lengthof(postgres_auto_path), - "%s/postgresql.auto.conf", instance_config.pgdata); + if (fio_chmod(postgres_auto_path, FILE_PERMISSION, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", postgres_auto_path, strerror(errno)); + if (params) + { fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + elog(ERROR, "cannot open file \"%s\": %s", postgres_auto_path, strerror(errno)); - // TODO: check if include 'probackup_recovery.conf' already exists - fio_fprintf(fp, "\n# created by pg_probackup restore of backup %s at '%s'\n", - base36enc(backup->start_time), current_time_str); - fio_fprintf(fp, "include '%s'\n", "probackup_recovery.conf"); + fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", + base36enc(backup->start_time), current_time_str); + + if (params->recovery_settings_mode == PITR_REQUESTED) + print_recovery_settings(fp, backup, params, rt); + + if (params->restore_as_replica) + print_standby_settings_common(fp, backup, params); if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + elog(ERROR, "cannot write file \"%s\": %s", postgres_auto_path, strerror(errno)); - } - /* Create empty probackup_recovery.conf */ - snprintf(probackup_recovery_path, lengthof(probackup_recovery_path), - "%s/probackup_recovery.conf", instance_config.pgdata); - fp = fio_fopen(probackup_recovery_path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", probackup_recovery_path, - strerror(errno)); + /* + * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. + * In older versions presense of recovery.conf alone was enough. + * To keep behaviour consistent with older versions, + * we are forced to create "recovery.signal" + * even when only restore_command is provided. + * Presense of "recovery.signal" by itself determine only + * one thing: do PostgreSQL must switch to a new timeline + * after successfull recovery or not? + */ + if (params->recovery_settings_mode == PITR_REQUESTED) + { + elog(LOG, "creating recovery.signal file"); + snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write to file \"%s\": %s", probackup_recovery_path, - strerror(errno)); -#endif - return; + fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write file \"%s\": %s", path, + strerror(errno)); + } + + if (params->restore_as_replica) + { + elog(LOG, "creating standby.signal file"); + snprintf(path, lengthof(path), "%s/standby.signal", instance_config.pgdata); + + fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "cannot write file \"%s\": %s", path, + strerror(errno)); + } + } } +#endif /* * Try to read a timeline's history file. diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 79e7086c6..f2bcbba8c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -246,18 +246,6 @@ def __init__(self, *args, **kwargs): print('pg_probackup binary is not found') exit(1) - self.probackup_version = None - - try: - self.probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8')) - - self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) - if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( @@ -280,6 +268,32 @@ def __init__(self, *args, **kwargs): if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') + self.probackup_version = None + self.old_probackup_version = None + + try: + self.probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8')) + + if self.probackup_old_path: + old_probackup_version_output = subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.old_probackup_version = re.search( + r"\d+\.\d+\.\d+", + subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + ).group(0) + + self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) + self.remote = False self.remote_host = None self.remote_port = None @@ -1142,8 +1156,9 @@ def get_recovery_conf(self, node): out_dict = {} if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf_path = os.path.join( - node.data_dir, 'probackup_recovery.conf') + recovery_conf_path = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf_path, 'r') as f: + print(f.read()) else: recovery_conf_path = os.path.join(node.data_dir, 'recovery.conf') diff --git a/tests/restore.py b/tests/restore.py index 14afb993f..b82da1a45 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -51,8 +51,11 @@ def test_restore_full_to_latest(self): repr(self.output), self.cmd)) # 2 - Test that recovery.conf was created + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') self.assertEqual(os.path.isfile(recovery_conf), True) @@ -1807,8 +1810,11 @@ def test_restore_target_immediate_stream(self): pgdata = self.pgdata_content(node.data_dir) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1862,8 +1868,11 @@ def test_restore_target_immediate_archive(self): pgdata = self.pgdata_content(node.data_dir) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1912,7 +1921,7 @@ def test_restore_target_latest_archive(self): self.backup_node(backup_dir, 'node', node) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1924,22 +1933,34 @@ def test_restore_target_latest_archive(self): # open(recovery_conf, 'rb').read()).hexdigest() with open(recovery_conf, 'r') as f: - content_1 = f.read() + content_1 = '' + while True: + line = f.readline() - # restore - node.cleanup() + if not line: + break + if line.startswith("#"): + continue + content_1 += line + node.cleanup() self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) # hash_2 = hashlib.md5( # open(recovery_conf, 'rb').read()).hexdigest() with open(recovery_conf, 'r') as f: - content_2 = f.read() + content_2 = '' + while True: + line = f.readline() - self.assertEqual(content_1, content_2) + if not line: + break + if line.startswith("#"): + continue + content_2 += line - # self.assertEqual(hash_1, hash_2) + self.assertEqual(content_1, content_2) # Clean after yourself self.del_test_dir(module_name, fname) @@ -1965,8 +1986,11 @@ def test_restore_target_new_options(self): # Take FULL self.backup_node(backup_dir, 'node', node) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -3264,8 +3288,11 @@ def test_stream_restore_command_option(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -3352,8 +3379,11 @@ def test_restore_primary_conninfo(self): os.path.isfile(standby_signal), "File '{0}' do not exists".format(standby_signal)) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(replica.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(replica.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(replica.data_dir, 'recovery.conf') @@ -3479,3 +3509,169 @@ def test_issue_249(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_pg_12_probackup_recovery_conf_compatibility(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.old_probackup_version: + if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): + return unittest.skip('You need pg_probackup < 2.4.5 for this test') + + if self.pg_config_version < self.version_to_num('12.0'): + return unittest.skip('You need PostgreSQL >= 12 for this test') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + time = node.safe_psql( + 'SELECT current_timestamp(0)::timestamptz;').decode('utf-8').rstrip() + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-time={0}".format(time), + "--recovery-target-action=promote"], + old_binary=True) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + xid = node.safe_psql( + 'SELECT txid_current()').decode('utf-8').rstrip() + node.pgbench_init(scale=1) + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-xid={0}".format(xid), + "--recovery-target-action=promote"]) + + node.slow_start() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_drop_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + return unittest.skip('You need PostgreSQL >= 12 for this test') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # drop postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + os.remove(auto_path) + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + def test_truncate_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + return unittest.skip('You need PostgreSQL >= 12 for this test') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # truncate postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + with open(auto_path, "w+") as f: + f.truncate() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From e645874a641c24623feb0e564619522ec269bf18 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 22:21:54 +0300 Subject: [PATCH 1536/2107] [Issue #266] Disable in-place merge only if merge chain contains backup with version less than STORAGE_FORMAT_VERSION, which bumped each time storage format is changed or some bug in backup/restore procedures is fixed --- src/merge.c | 71 +++++++++++++++++++++++++++++++++++++--------- src/pg_probackup.h | 2 ++ 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/merge.c b/src/merge.c index 0c50dd73f..05220a5d6 100644 --- a/src/merge.c +++ b/src/merge.c @@ -58,6 +58,8 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, pgFile *tmp_file, const char *full_database_dir, const char *full_external_prefix); +static bool is_forward_compatible(parray *parent_chain); + /* * Implementation of MERGE command. * @@ -532,19 +534,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * If current program version differs from destination backup version, * then in-place merge is not possible. */ - if ((parse_program_version(full_backup->program_version) == - parse_program_version(dest_backup->program_version)) && - (parse_program_version(dest_backup->program_version) == - parse_program_version(PROGRAM_VERSION))) - program_version_match = true; - else - elog(WARNING, "In-place merge is disabled because of program " - "versions mismatch. Full backup version: %s, " - "destination backup version: %s, " - "current program version: %s", - full_backup->program_version, - dest_backup->program_version, - PROGRAM_VERSION); + program_version_match = is_forward_compatible(parent_chain); /* Forbid merge retry for failed merges between 2.4.0 and any * older version. Several format changes makes it impossible @@ -1398,3 +1388,58 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, to_fullpath_tmp, to_fullpath, strerror(errno)); } + +/* + * If file format in incremental chain is compatible + * with current storage format. + * If not, then in-place merge is not possible. + * + * Consider the following examples: + * STORAGE_FORMAT_VERSION = 2.4.4 + * 2.3.3 \ + * 2.3.4 \ disable in-place merge, because + * 2.4.1 / current STORAGE_FORMAT_VERSION > 2.3.3 + * 2.4.3 / + * + * 2.4.4 \ enable in_place merge, because + * 2.4.5 / current STORAGE_FORMAT_VERSION == 2.4.4 + * + * 2.4.5 \ enable in_place merge, because + * 2.4.6 / current STORAGE_FORMAT_VERSION < 2.4.5 + * + */ +bool +is_forward_compatible(parray *parent_chain) +{ + int i; + pgBackup *oldest_ver_backup = NULL; + uint32 oldest_ver_in_chain = parse_program_version(PROGRAM_VERSION); + + for (i = 0; i < parray_num(parent_chain); i++) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + uint32 current_version = parse_program_version(backup->program_version); + + if (!oldest_ver_backup) + oldest_ver_backup = backup; + + if (current_version < oldest_ver_in_chain) + { + oldest_ver_in_chain = current_version; + oldest_ver_backup = backup; + } + } + + if (oldest_ver_in_chain < parse_program_version(STORAGE_FORMAT_VERSION)) + { + elog(WARNING, "In-place merge is disabled because of storage format incompatibility. " + "Backup %s storage format version: %s, " + "current storage format version: %s", + base36enc(oldest_ver_backup->start_time), + oldest_ver_backup->program_version, + STORAGE_FORMAT_VERSION); + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4c670b770..a7fa00f8f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -291,6 +291,8 @@ typedef enum ShowFormat #define PROGRAM_VERSION "2.4.4" #define AGENT_PROTOCOL_VERSION 20404 +/* update only when changing storage format */ +#define STORAGE_FORMAT_VERSION "2.4.4" typedef struct ConnectionOptions { From e645a6c2add293e30ae4f953fac65a89d0f7ffff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 23:34:28 +0300 Subject: [PATCH 1537/2107] [Issue #266] improve test coverage --- tests/compatibility.py | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index d2db2be28..066df9626 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -1011,6 +1011,87 @@ def test_backward_compatibility_merge_4(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_5(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version >= STORAGE_FORMAT_VERSION (2.4.4) + """ + + if self.version_to_num(self.old_probackup_version) < self.version_to_num('2.4.4'): + return unittest.skip('OLD pg_probackup binary must be == 2.4.4 for this test') + + self.assertNotEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num(self.probackup_version)) + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + # PAGE2 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + # merge chain created by old binary with new binary + output = self.merge_backup(backup_dir, "node", backup_id) + + # check that in-place is disabled + self.assertNotIn( + "WARNING: In-place merge is disabled " + "because of program versions mismatch", output) + + # restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_page_vacuum_truncate(self): """ From dfe28b1ef874bb805bbf744348b53636a88e4fb7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 23:35:11 +0300 Subject: [PATCH 1538/2107] tests: minor tweaks in merge module --- tests/merge.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/merge.py b/tests/merge.py index d7dbae206..44652066a 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1475,7 +1475,7 @@ def test_crash_after_opening_backup_control_1(self): self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_crash_after_opening_backup_control_2(self): """ check that crashing after opening backup_content.control @@ -1529,8 +1529,8 @@ def test_crash_after_opening_backup_control_2(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('sprintf') - gdb.continue_execution_until_break(1) +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(1) gdb._execute('signal SIGKILL') @@ -1567,7 +1567,7 @@ def test_crash_after_opening_backup_control_2(self): self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_losing_file_after_failed_merge(self): """ check that crashing after opening backup_content.control @@ -1622,8 +1622,8 @@ def test_losing_file_after_failed_merge(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('sprintf') - gdb.continue_execution_until_break(20) +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') From 17b3e2b5863126d2f0f1babe24871569328dea21 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 23:39:26 +0300 Subject: [PATCH 1539/2107] [Issue #272] improve test coverage --- tests/show.py | 2 +- tests/time_stamp.py | 153 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/tests/show.py b/tests/show.py index 0da95dcbb..3fdd85d04 100644 --- a/tests/show.py +++ b/tests/show.py @@ -6,7 +6,7 @@ module_name = 'show' -class OptionTest(ProbackupTest, unittest.TestCase): +class ShowTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 8abd55a2b..0d0c6e2f8 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -1,6 +1,8 @@ import os import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from time import sleep module_name = 'time_stamp' @@ -72,3 +74,154 @@ def test_server_date_style(self): # Clean after yourself self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_dst_timezone_handling(self): + """for manual testing""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + print(subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + output = self.backup_node(backup_dir, 'node', node, return_id=False) + self.assertNotIn("backup ID in control file", output) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', return_id=False) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # Clean after yourself + self.del_test_dir(module_name, fname) + + @unittest.skip("skip") + def test_dst_timezone_handling_backward_compatibilty(self): + """for manual testing""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + self.backup_node(backup_dir, 'node', node, old_binary=True, return_id=False) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', old_binary=True, return_id=False) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # Clean after yourself + self.del_test_dir(module_name, fname) From 50d2d9c3afa1a7bff8324f648614c1e6a9dc7c38 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 2 Dec 2020 23:47:17 +0300 Subject: [PATCH 1540/2107] Version 2.4.5 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a7fa00f8f..33990a18e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -288,8 +288,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.4" -#define AGENT_PROTOCOL_VERSION 20404 +#define PROGRAM_VERSION "2.4.5" +#define AGENT_PROTOCOL_VERSION 20405 /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 2d00a86ce..e49101319 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.4 \ No newline at end of file +pg_probackup 2.4.5 \ No newline at end of file From b6ce030d36ae5f138bba5ec2e3a40e78db871fea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 12:21:53 +0300 Subject: [PATCH 1541/2107] Timestamps with missing timezone offset should be treated as UTC only when parsing config(due to some old bug), otherwise as local timezone --- src/utils/configuration.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 76920a9fd..fb420ef43 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1223,8 +1223,11 @@ parse_time(const char *value, time_t *result, bool utc_default) /* determine whether Daylight Saving Time is in effect */ tm.tm_isdst = -1; - /* if tz is not set, treat it as local timezone */ - if (tz_set) + /* + * If tz is not set, + * treat it as UTC if requested, otherwise as local timezone + */ + if (tz_set || utc_default) /* set timezone to UTC */ setenv("TZ", "UTC", 1); From 7ddf8ee18ad6e4885d78f5940f5a3f9128934d64 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 12:23:33 +0300 Subject: [PATCH 1542/2107] tests: fixes for time_stamp module --- tests/time_stamp.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 0d0c6e2f8..d159ea0a0 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -49,8 +49,14 @@ def test_start_time_format(self): show_backup = show_backup + self.show_pb(backup_dir, 'node') i += 1 + print(show_backup[1]['id']) + print(show_backup[2]['id']) + self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + node.stop() # Clean after yourself self.del_test_dir(module_name, fname) @@ -92,7 +98,7 @@ def test_dst_timezone_handling(self): node.slow_start() print(subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'US/Detroit'], + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()) @@ -168,7 +174,7 @@ def test_dst_timezone_handling_backward_compatibilty(self): node.slow_start() subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'US/Detroit'], + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() From 186fb72e1c4d16568061bb2cebef5ce1e64d11a7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 13:15:08 +0300 Subject: [PATCH 1543/2107] tests: minor fixes --- src/restore.c | 2 +- tests/archive.py | 6 +++--- tests/compatibility.py | 4 ++-- tests/helpers/ptrack_helpers.py | 5 +---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/restore.c b/src/restore.c index 7220eb469..f2e55cdef 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1212,7 +1212,7 @@ restore_files(void *arg) } /* - * Create recovery.conf (probackup_recovery.conf in case of PG12) + * Create recovery.conf (postgresql.auto.conf in case of PG12) * with given recovery target parameters */ static void diff --git a/tests/archive.py b/tests/archive.py index 78aa6d045..482a2502b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1607,7 +1607,7 @@ def test_archive_options(self): ]) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1682,7 +1682,7 @@ def test_archive_options_1(self): self.restore_node(backup_dir, 'node', node) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -2096,7 +2096,7 @@ def test_multi_timeline_recovery_prefetching(self): if node.major_version >= 12: node.append_conf( - 'probackup_recovery.conf', "restore_command = '{0}'".format(restore_command)) + 'postgresql.auto.conf', "restore_command = '{0}'".format(restore_command)) else: node.append_conf( 'recovery.conf', "restore_command = '{0}'".format(restore_command)) diff --git a/tests/compatibility.py b/tests/compatibility.py index 066df9626..547933344 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -675,7 +675,7 @@ def test_backward_compatibility_merge_1(self): # check that in-place is disabled self.assertIn( "WARNING: In-place merge is disabled " - "because of program versions mismatch", output) + "because of storage format incompatibility", output) # restore merged backup node_restored = self.make_simple_node( @@ -1077,7 +1077,7 @@ def test_backward_compatibility_merge_5(self): # check that in-place is disabled self.assertNotIn( "WARNING: In-place merge is disabled " - "because of program versions mismatch", output) + "because of storage format incompatibility", output) # restore merged backup node_restored = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f2bcbba8c..8932a4bc8 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1326,10 +1326,6 @@ def set_replica( f.close() config = 'postgresql.auto.conf' - probackup_recovery_path = os.path.join(replica.data_dir, 'probackup_recovery.conf') - if os.path.exists(probackup_recovery_path): - if os.stat(probackup_recovery_path).st_size > 0: - config = 'probackup_recovery.conf' if not log_shipping: self.set_auto_conf( @@ -1547,6 +1543,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): directory_dict['files'][file_relpath] = {'is_datafile': False} with open(file_fullpath, 'rb') as f: directory_dict['files'][file_relpath]['md5'] = hashlib.md5(f.read()).hexdigest() + f.close() # directory_dict['files'][file_relpath]['md5'] = hashlib.md5( # f = open(file_fullpath, 'rb').read()).hexdigest() From 196d4fae383216a02038da597e5409731c1a8356 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 21:30:42 +0300 Subject: [PATCH 1544/2107] bugfix: do not create empty recovery.conf --- src/restore.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/restore.c b/src/restore.c index f2e55cdef..bf278f8b2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1379,6 +1379,16 @@ update_recovery_options_before_v12(pgBackup *backup, FILE *fp; char path[MAXPGPATH]; + /* + * If PITR is not requested and instance is not restored as replica, + * then recovery.conf should not be created. + */ + if (params->recovery_settings_mode != PITR_REQUESTED && + !params->restore_as_replica) + { + return; + } + elog(LOG, "update recovery settings in recovery.conf"); snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); From 84e63979cdd98e4cee2fce1d1554416bcb8eae74 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 21:31:01 +0300 Subject: [PATCH 1545/2107] tests: minor fixes --- tests/restore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/restore.py b/tests/restore.py index b82da1a45..d7899e0eb 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3492,7 +3492,7 @@ def test_issue_249(self): self.set_auto_conf( node_restored_1, - {'port': node_restored_1.port, 'hot_standby': 'on'}) + {'port': node_restored_1.port, 'hot_standby': 'off'}) node_restored_1.slow_start() From 9619608f9249a1314c3d0df61cd3b0c6af0b6b1a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 21:54:44 +0300 Subject: [PATCH 1546/2107] bugfix: cross-platform setenv and getenv --- src/pg_probackup.h | 3 ++ src/utils/configuration.c | 12 ++++---- src/utils/pgut.c | 58 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 33990a18e..27d8b89b6 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1158,6 +1158,9 @@ extern void fio_list_dir(parray *files, const char *root, bool exclude, bool fol extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); +extern void pgut_setenv(const char *key, const char *val); +extern void pgut_unsetenv(const char *key); + extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index fb420ef43..56d873dca 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1229,16 +1229,16 @@ parse_time(const char *value, time_t *result, bool utc_default) */ if (tz_set || utc_default) /* set timezone to UTC */ - setenv("TZ", "UTC", 1); + pgut_setenv("TZ", "UTC"); /* convert time to utc unix time */ *result = mktime(&tm); /* return old timezone back if any */ if (local_tz) - setenv("TZ", local_tz, 1); + pgut_setenv("TZ", local_tz); else - unsetenv("TZ"); + pgut_unsetenv("TZ"); /* adjust time zone */ if (tz_set || utc_default) @@ -1480,7 +1480,7 @@ time2iso(char *buf, size_t len, time_t time, bool utc) /* set timezone to UTC if requested */ if (utc) - setenv("TZ", "UTC", 1); + pgut_setenv("TZ", "UTC"); ptm = gmtime(&time); gmt = mktime(ptm); @@ -1490,9 +1490,9 @@ time2iso(char *buf, size_t len, time_t time, bool utc) { /* return old timezone back if any */ if (local_tz) - setenv("TZ", local_tz, 1); + pgut_setenv("TZ", local_tz); else - unsetenv("TZ"); + pgut_unsetenv("TZ"); } /* adjust timezone offset */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 27cfe6d1f..316581900 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1213,3 +1213,61 @@ pgut_rmtree(const char *path, bool rmtopdir, bool strict) return result; } + +/* cross-platform setenv */ +void +pgut_setenv(const char *key, const char *val) +{ +#ifdef WIN32 + char *envstr = NULL; + envstr = psprintf("%s=%s", key, val); + putenv(envstr); +#else + setenv(key, val, 1); +#endif +} + +/* stolen from unsetenv.c */ +void +pgut_unsetenv(const char *key) +{ +#ifdef WIN32 + char *envstr = NULL; + + if (getenv(key) == NULL) + return; /* no work */ + + /* + * The technique embodied here works if libc follows the Single Unix Spec + * and actually uses the storage passed to putenv() to hold the environ + * entry. When we clobber the entry in the second step we are ensuring + * that we zap the actual environ member. However, there are some libc + * implementations (notably recent BSDs) that do not obey SUS but copy the + * presented string. This method fails on such platforms. Hopefully all + * such platforms have unsetenv() and thus won't be using this hack. See: + * http://www.greenend.org.uk/rjk/2008/putenv.html + * + * Note that repeatedly setting and unsetting a var using this code will + * leak memory. + */ + + envstr = (char *) malloc(strlen(key) + 2); + if (!envstr) /* not much we can do if no memory */ + return; + + /* Override the existing setting by forcibly defining the var */ + sprintf(envstr, "%s=", key); + putenv(envstr); + + /* Now we can clobber the variable definition this way: */ + strcpy(envstr, "="); + + /* + * This last putenv cleans up if we have multiple zero-length names as a + * result of unsetting multiple things. + */ + putenv(envstr); +#else + unsetenv(key); +#endif +} From 3543dae5ead7ae3111f5c35c27377da0f8a7fcc1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 22:13:19 +0300 Subject: [PATCH 1547/2107] fix build on Windows --- gen_probackup_project.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index b78f4699e..abc779a40 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -167,6 +167,7 @@ sub build_pgprobackup 'pg_probackup.c', 'restore.c', 'show.c', + 'stream.c', 'util.c', 'validate.c', 'checkdb.c', From 667f2dc7c0e31f24b0202588094f3a04b3986c2c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 3 Dec 2020 14:58:52 -0500 Subject: [PATCH 1548/2107] use tzset on Windows --- src/utils/configuration.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 56d873dca..f61868290 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1228,8 +1228,13 @@ parse_time(const char *value, time_t *result, bool utc_default) * treat it as UTC if requested, otherwise as local timezone */ if (tz_set || utc_default) + { /* set timezone to UTC */ pgut_setenv("TZ", "UTC"); +#ifdef WIN32 + tzset(); +#endif + } /* convert time to utc unix time */ *result = mktime(&tm); @@ -1240,6 +1245,10 @@ parse_time(const char *value, time_t *result, bool utc_default) else pgut_unsetenv("TZ"); +#ifdef WIN32 + tzset(); +#endif + /* adjust time zone */ if (tz_set || utc_default) { @@ -1480,7 +1489,12 @@ time2iso(char *buf, size_t len, time_t time, bool utc) /* set timezone to UTC if requested */ if (utc) + { pgut_setenv("TZ", "UTC"); +#ifdef WIN32 + tzset(); +#endif + } ptm = gmtime(&time); gmt = mktime(ptm); @@ -1493,6 +1507,9 @@ time2iso(char *buf, size_t len, time_t time, bool utc) pgut_setenv("TZ", local_tz); else pgut_unsetenv("TZ"); +#ifdef WIN32 + tzset(); +#endif } /* adjust timezone offset */ From c63a2b81c797a2c07a6876659327b6eaa6de3480 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 4 Dec 2020 00:27:50 +0300 Subject: [PATCH 1549/2107] Version 2.4.6 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 27d8b89b6..6d234a5f8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -288,8 +288,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.5" -#define AGENT_PROTOCOL_VERSION 20405 +#define PROGRAM_VERSION "2.4.6" +#define AGENT_PROTOCOL_VERSION 20406 /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e49101319..94ea2d6ed 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.5 \ No newline at end of file +pg_probackup 2.4.6 \ No newline at end of file From a261a83492fc6f8adccfa7618693bb77202a354d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 5 Dec 2020 12:02:58 +0300 Subject: [PATCH 1550/2107] tests: added tests.restore.RestoreTest.test_concurrent_restore --- tests/restore.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index d7899e0eb..35fc6dceb 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3675,3 +3675,62 @@ def test_truncate_postgresql_auto_conf(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_concurrent_restore(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress']) + + pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress', '--no-validate']) + + pgdata1 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + + node.cleanup() + node_restored.cleanup() + + gdb = self.restore_node( + backup_dir, 'node', node, options=['--no-validate'], gdb=True) + + gdb.set_breakpoint('restore_data_file') + gdb.run_until_break() + + self.restore_node( + backup_dir, 'node', node_restored, options=['--no-validate']) + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + pgdata2 = self.pgdata_content(node.data_dir) + pgdata3 = self.pgdata_content(node_restored.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + self.compare_pgdata(pgdata2, pgdata3) + + # Clean after yourself + self.del_test_dir(module_name, fname) From da50370dbbe38f6255bc77d70f1a84d260d283ab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 00:47:00 +0300 Subject: [PATCH 1551/2107] [Issue #283] added tests.replica.ReplicaTest.test_replica_via_basebackup --- tests/replica.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/replica.py b/tests/replica.py index 8baaaa36c..61ba21b60 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1728,6 +1728,81 @@ def test_instance_from_the_past(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_replica_via_basebackup(self): + """ + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off', 'hot_standby': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=10) + + #FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=['--recovery-target=latest', '--recovery-target-action=promote']) + node.slow_start() + + # Timeline 2 + # Take stream page backup from instance in timeline2 + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + node.cleanup() + + # restore stream backup + self.restore_node(backup_dir, 'node', node) + + xlog_dir = 'pg_wal' + if self.get_version(node) < 100000: + xlog_dir = 'pg_xlog' + + filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") + self.assertTrue( + os.path.exists(filepath), + "History file do not exists: {0}".format(filepath)) + + node.slow_start() + + # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " + # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + pg_basebackup_path = self.get_bin_path('pg_basebackup') + + self.run_binary( + [ + pg_basebackup_path, '-p', str(node.port), '-h', 'localhost', + '-R', '-X', 'stream', '-D', node_restored.data_dir + ]) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start(replica=True) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) # archiving from promoted delayed replica From 8350e174303db41a9d303862559accfef0eef3f4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 00:48:16 +0300 Subject: [PATCH 1552/2107] [Issue #283] add history file to STREAM backup filelist --- src/pg_probackup.h | 4 ++-- src/stream.c | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6d234a5f8..467685851 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -995,8 +995,8 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool missing_ok, bool use_crc32c); +extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); diff --git a/src/stream.c b/src/stream.c index dbb149148..ad73f296b 100644 --- a/src/stream.c +++ b/src/stream.c @@ -67,6 +67,8 @@ static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size); +static void add_history_file_to_filelist(parray *filelist, uint32 timeline, + char *basedir); /* * Run IDENTIFY_SYSTEM through a given connection and @@ -255,6 +257,9 @@ StreamLog(void *arg) stop_stream_lsn, (char *) stream_arg->basedir, instance_config.xlog_seg_size); + /* append history file to walsegment filelist */ + add_history_file_to_filelist(xlog_files_list, stream_arg->starttli, (char *) stream_arg->basedir); + /* * TODO: remove redundant WAL segments * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 @@ -398,7 +403,7 @@ wait_WAL_streaming_end(parray *backup_files_list) void add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) { - XLogSegNo xlog_segno; + XLogSegNo xlog_segno; char wal_segment_name[MAXFNAMELEN]; char wal_segment_relpath[MAXPGPATH]; char wal_segment_fullpath[MAXPGPATH]; @@ -451,3 +456,32 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos /* append file to filelist */ parray_append(filelist, file); } + +/* Append streamed WAL segment to filelist */ +void +add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) +{ + char filename[MAXFNAMELEN]; + char fullpath[MAXPGPATH]; + char relpath[MAXPGPATH]; + pgFile *file = NULL; + + /* Timeline 1 does not have a history file */ + if (timeline == 1) + return; + + snprintf(filename, lengthof(filename), "%08X.history", timeline); + join_path_components(fullpath, basedir, filename); + join_path_components(relpath, PG_XLOG_DIR, filename); + + file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); + file->name = file->rel_path; + + /* calculate crc */ + file->crc = pgFileGetCRC(fullpath, true, false); + file->write_size = file->size; + file->uncompressed_size = file->size; + + /* append file to filelist */ + parray_append(filelist, file); +} From cd2655b4e28c26fa2e7d5ee9206adfb4b0e1dbca Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 02:34:49 +0300 Subject: [PATCH 1553/2107] bugfix: correctly calculate backup WAL size for STREAM backup --- src/stream.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/stream.c b/src/stream.c index ad73f296b..c5c739ae8 100644 --- a/src/stream.c +++ b/src/stream.c @@ -427,7 +427,6 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); - file->name = file->rel_path; /* * Check if file is already in the list @@ -457,7 +456,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos parray_append(filelist, file); } -/* Append streamed WAL segment to filelist */ +/* Append history file to filelist */ void add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) { @@ -475,7 +474,6 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) join_path_components(relpath, PG_XLOG_DIR, filename); file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); - file->name = file->rel_path; /* calculate crc */ file->crc = pgFileGetCRC(fullpath, true, false); From 623c0d3a46a0308c7e1eb41ad71c297cc222241a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 09:23:50 +0300 Subject: [PATCH 1554/2107] do not fsync streamed WAL segments --- src/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index c5c739ae8..825aa0e7d 100644 --- a/src/stream.c +++ b/src/stream.c @@ -212,7 +212,7 @@ StreamLog(void *arg) stream_arg->basedir, // (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, 0, - true); + false); ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; ctl.do_sync = false; /* We sync all files at the end of backup */ From f586c96e3d1dc6e2ba0b6ec0769f1454daaf9382 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 09:31:31 +0300 Subject: [PATCH 1555/2107] Version 2.4.7 --- src/backup.c | 1 - src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 393e08f53..f6d3a958b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -572,7 +572,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool * NOTHING TO DO HERE */ - /* write database map to file and add it to control file */ if (database_map) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 467685851..9e6a5c74e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -288,8 +288,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.6" -#define AGENT_PROTOCOL_VERSION 20406 +#define PROGRAM_VERSION "2.4.7" +#define AGENT_PROTOCOL_VERSION 20407 /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 94ea2d6ed..059e0f990 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.6 \ No newline at end of file +pg_probackup 2.4.7 \ No newline at end of file From 33d288380e069e75e9d198f95b9887193893f2fe Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 7 Dec 2020 17:13:51 +0300 Subject: [PATCH 1556/2107] tests: minor fixes --- tests/replica.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 61ba21b60..f664ca886 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1764,8 +1764,9 @@ def test_replica_via_basebackup(self): # Timeline 2 # Take stream page backup from instance in timeline2 - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--log-level-file=verbose']) node.cleanup() @@ -1783,8 +1784,6 @@ def test_replica_via_basebackup(self): node.slow_start() - # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " - # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() From c525dad4c7337d007e72121cd8fcad1da4962eff Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 00:16:00 +0300 Subject: [PATCH 1557/2107] [Issue #249] hot_standby breaks partial restore for PG < 12 --- doc/pgprobackup.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 5a82f6ba9..7a9d79f14 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1880,6 +1880,14 @@ pg_probackup restore -B backup_dir --instance template1 databases are always restored. + + + Due to how recovery works in PostgreSQL < 12 it is advisable to + disable option, when running partial + restore of PostgreSQL cluster of version less than . + Otherwise recovery may fail. + + From 775b260aa4a0f5668fe48911c857cd04260b14fc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 10:03:37 +0300 Subject: [PATCH 1558/2107] [Issue #263] added tests.set_backup.SetBackupTest.test_add_big_note_1 --- tests/set_backup.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/set_backup.py b/tests/set_backup.py index daba9a216..02ce007bf 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -473,4 +473,37 @@ def test_add_big_note(self): self.assertEqual(backup_meta['note'], note) # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_add_big_note_1(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + note = node.safe_psql( + "postgres", + "SELECT repeat('q', 1024)").decode('utf-8').rstrip() + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + self.set_backup( + backup_dir, 'node', backup_id, + options=['--note={0}'.format(note)]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + + print(backup_meta) + self.assertEqual(backup_meta['note'], note) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 6f9e1df95dd5444d7007d0f5adc8095221c08a49 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 10:04:32 +0300 Subject: [PATCH 1559/2107] [Issue #263] fix parsing of big size values --- src/utils/configuration.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index f61868290..2aeab039b 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -490,9 +490,9 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, bool strict, bool missing_ok) { FILE *fp; - char buf[1024]; + char buf[4096]; char key[1024]; - char value[1024]; + char value[2048]; int parsed_options = 0; if (!options) From 27310b3684033d08415b10a185cd1027e25cc092 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 10:50:49 +0300 Subject: [PATCH 1560/2107] [Issue #284] added tests.time_stamp.TimeStamp.test_handling_of_TZ_env_variable --- tests/helpers/ptrack_helpers.py | 21 +++++++++++++-------- tests/time_stamp.py | 26 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 8932a4bc8..d340213c2 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -707,7 +707,7 @@ def check_ptrack_clean(self, idx_dict, size): ) ) - def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True): + def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True, env=None): if not self.probackup_old_path and old_binary: print('PGPROBACKUPBIN_OLD is not set') exit(1) @@ -717,6 +717,9 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur else: binary_path = self.probackup_path + if not env: + env=self.test_env + try: self.cmd = [' '.join(map(str, [binary_path] + command))] if self.verbose: @@ -728,13 +731,13 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.test_env + env=env ) else: self.output = subprocess.check_output( [binary_path] + command, stderr=subprocess.STDOUT, - env=self.test_env + env=env ).decode('utf-8') if command[0] == 'backup' and return_id: # return backup ID @@ -845,7 +848,8 @@ def backup_node( self, backup_dir, instance, node, data_dir=False, backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, - old_binary=False, return_id=True, no_remote=False + old_binary=False, return_id=True, no_remote=False, + env=None ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -878,7 +882,7 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( self, backup_dir=False, instance=False, data_dir=False, @@ -942,7 +946,8 @@ def restore_node( def show_pb( self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False, as_json=True, old_binary=False + options=[], as_text=False, as_json=True, old_binary=False, + env=None ): backup_list = [] @@ -963,7 +968,7 @@ def show_pb( if as_text: # You should print it when calling as_text=true - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, old_binary=old_binary, env=env) # get show result as list of lines if as_json: @@ -988,7 +993,7 @@ def show_pb( return backup_list else: show_splitted = self.run_pb( - cmd_list + options, old_binary=old_binary).splitlines() + cmd_list + options, old_binary=old_binary, env=env).splitlines() if instance is not None and backup_id is None: # cut header(ID, Mode, etc) from show as single string header = show_splitted[1:2][0] diff --git a/tests/time_stamp.py b/tests/time_stamp.py index d159ea0a0..9fcb7a445 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -7,7 +7,7 @@ module_name = 'time_stamp' -class CheckTimeStamp(ProbackupTest, unittest.TestCase): +class TimeStamp(ProbackupTest, unittest.TestCase): def test_start_time_format(self): """Test backup ID changing after start-time editing in backup.control. @@ -81,6 +81,30 @@ def test_server_date_style(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_handling_of_TZ_env_variable(self): + """Issue #112""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(module_name, fname), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + my_env = os.environ.copy() + my_env["TZ"] = "America/Detroit" + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) + + self.show_pb(backup_dir, 'node', env=my_env) + + # Clean after yourself + self.del_test_dir(module_name, fname) + @unittest.skip("skip") # @unittest.expectedFailure def test_dst_timezone_handling(self): From e9ddcafdf9b02bc2581f47c110a668f44a30ec8e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 03:07:45 -0500 Subject: [PATCH 1561/2107] [Issue #284] fix test --- tests/time_stamp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 9fcb7a445..bf6e91e0a 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -100,7 +100,9 @@ def test_handling_of_TZ_env_variable(self): self.backup_node( backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) - self.show_pb(backup_dir, 'node', env=my_env) + output = self.show_pb(backup_dir, 'node', as_json=False, as_text=True, env=my_env) + + self.assertNotIn("backup ID in control file", output) # Clean after yourself self.del_test_dir(module_name, fname) From 0d65e055cfbbc22cc05f5cf78561c546b2888e5c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 03:10:26 -0500 Subject: [PATCH 1562/2107] [Issue #284] fix segfault in local timezone handling --- src/utils/configuration.c | 4 ---- src/utils/pgut.c | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 2aeab039b..d6a7d069e 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1256,8 +1256,6 @@ parse_time(const char *value, time_t *result, bool utc_default) *result -= tz; } - pg_free(local_tz); - return true; } @@ -1528,6 +1526,4 @@ time2iso(char *buf, size_t len, time_t time, bool utc) snprintf(ptr, len - (ptr - buf), ":%02d", abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); } - - pg_free(local_tz); } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 316581900..bc35a24ff 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1251,7 +1251,7 @@ pgut_unsetenv(const char *key) * leak memory. */ - envstr = (char *) malloc(strlen(key) + 2); + envstr = (char *) pgut_malloc(strlen(key) + 2); if (!envstr) /* not much we can do if no memory */ return; From d9f95db7261c25c9708773ac3f6a8896273a6f6a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 9 Dec 2020 03:12:29 -0500 Subject: [PATCH 1563/2107] Version 2.4.8 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 9e6a5c74e..2b8253cb9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -288,8 +288,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.7" -#define AGENT_PROTOCOL_VERSION 20407 +#define PROGRAM_VERSION "2.4.8" +#define AGENT_PROTOCOL_VERSION 20408 /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 059e0f990..9a1e5663b 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.7 \ No newline at end of file +pg_probackup 2.4.8 \ No newline at end of file From ba2a35bb34deaf68d8ead5d62b90105acd6be735 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Dec 2020 14:53:42 +0300 Subject: [PATCH 1564/2107] [Issue #231] added tests.backup.BackupTest.test_issue_231 --- tests/backup.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index bf0cbca81..fb4bd1885 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2899,3 +2899,31 @@ def test_issue_203(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_issue_231(self): + """ + https://github.com/postgrespro/pg_probackup/issues/231 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, options=['--streamblablah', '-j2']) + except: + pass + + self.backup_node(backup_dir, 'node', node, options=['--stream', '-j2']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From ac2da1e817d001f40e2c797e56e344d04872bea0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Dec 2020 15:05:29 +0300 Subject: [PATCH 1565/2107] [Issue #231] fix tests.backup.BackupTest.test_issue_231 --- tests/backup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index fb4bd1885..590d86dbf 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2917,13 +2917,16 @@ def test_issue_231(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + datadir = os.path.join(node.data_dir, '123') + try: self.backup_node( - backup_dir, 'node', node, options=['--streamblablah', '-j2']) + backup_dir, 'node', node, + data_dir='{0}'.format(datadir), return_id=False) except: pass - self.backup_node(backup_dir, 'node', node, options=['--stream', '-j2']) + self.backup_node(backup_dir, 'node', node, options=['--stream']) # Clean after yourself self.del_test_dir(module_name, fname) From 232f94b2700a466edd695c58ecf705fdbcdbe5d3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 11 Dec 2020 16:00:15 +0300 Subject: [PATCH 1566/2107] DOC: note about the usage of partial restore for multi-database cluster decoupling --- doc/pgprobackup.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7a9d79f14..0f844ec39 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1874,6 +1874,10 @@ pg_probackup restore -B backup_dir --instance DROP DATABASE command. + + If want to decouple a single cluster with multiple databases into separate clusters with minimal downtime, then + you can partially restore cluster as standby via options. + The template0 and From 3f0984591192e909121609ea0d5d5a6ff000624b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Tue, 29 Dec 2020 08:47:20 +0300 Subject: [PATCH 1567/2107] remove backup_subcmd global variable and useless command_name variable --- src/help.c | 99 ++++++++++-------- src/pg_probackup.c | 206 +++++++++++++------------------------- src/pg_probackup.h | 4 +- src/utils/configuration.c | 56 +++++++++++ src/utils/configuration.h | 27 +++++ src/utils/file.c | 18 ++++ 6 files changed, 231 insertions(+), 179 deletions(-) diff --git a/src/help.c b/src/help.c index c82c8cf63..2af045cde 100644 --- a/src/help.c +++ b/src/help.c @@ -7,8 +7,10 @@ *------------------------------------------------------------------------- */ +#include #include "pg_probackup.h" +static void help_nocmd(void); static void help_init(void); static void help_backup(void); static void help_restore(void); @@ -24,50 +26,52 @@ static void help_del_instance(void); static void help_archive_push(void); static void help_archive_get(void); static void help_checkdb(void); +static void help_help(void); void -help_command(char *command) +help_print_version(void) { - if (strcmp(command, "init") == 0) - help_init(); - else if (strcmp(command, "backup") == 0) - help_backup(); - else if (strcmp(command, "restore") == 0) - help_restore(); - else if (strcmp(command, "validate") == 0) - help_validate(); - else if (strcmp(command, "show") == 0) - help_show(); - else if (strcmp(command, "delete") == 0) - help_delete(); - else if (strcmp(command, "merge") == 0) - help_merge(); - else if (strcmp(command, "set-backup") == 0) - help_set_backup(); - else if (strcmp(command, "set-config") == 0) - help_set_config(); - else if (strcmp(command, "show-config") == 0) - help_show_config(); - else if (strcmp(command, "add-instance") == 0) - help_add_instance(); - else if (strcmp(command, "del-instance") == 0) - help_del_instance(); - else if (strcmp(command, "archive-push") == 0) - help_archive_push(); - else if (strcmp(command, "archive-get") == 0) - help_archive_get(); - else if (strcmp(command, "checkdb") == 0) - help_checkdb(); - else if (strcmp(command, "--help") == 0 - || strcmp(command, "help") == 0 - || strcmp(command, "-?") == 0 - || strcmp(command, "--version") == 0 - || strcmp(command, "version") == 0 - || strcmp(command, "-V") == 0) - printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); - else - printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); - exit(0); +#ifdef PGPRO_VERSION + fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stdout, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif +} + +void +help_command(ProbackupSubcmd const subcmd) +{ + typedef void (* help_function_ptr)(void); + /* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ + static help_function_ptr const help_functions[] = + { + &help_nocmd, + &help_init, + &help_add_instance, + &help_del_instance, + &help_archive_push, + &help_archive_get, + &help_backup, + &help_restore, + &help_validate, + &help_delete, + &help_merge, + &help_show, + &help_set_config, + &help_set_backup, + &help_show_config, + &help_checkdb, + &help_nocmd, // SSH_CMD + &help_nocmd, // AGENT_CMD + &help_help, + &help_help, // VERSION_CMD + }; + + Assert((int)subcmd < sizeof(help_functions) / sizeof(help_functions[0])); + help_functions[(int)subcmd](); } void @@ -247,7 +251,12 @@ help_pg_probackup(void) if (PROGRAM_EMAIL) printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); } - exit(0); +} + +static void +help_nocmd(void) +{ + printf(_("Unknown command. Try pg_probackup help\n")); } static void @@ -971,3 +980,9 @@ help_archive_get(void) printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } + +static void +help_help(void) +{ + printf(_("No help page required for \"help\" and \"version\" commands. Just try it!\n")); +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6fdf8bafb..855d24d92 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -27,27 +27,6 @@ const char *PROGRAM_FULL_PATH = NULL; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; -typedef enum ProbackupSubcmd -{ - NO_CMD = 0, - INIT_CMD, - ADD_INSTANCE_CMD, - DELETE_INSTANCE_CMD, - ARCHIVE_PUSH_CMD, - ARCHIVE_GET_CMD, - BACKUP_CMD, - RESTORE_CMD, - VALIDATE_CMD, - DELETE_CMD, - MERGE_CMD, - SHOW_CMD, - SET_CONFIG_CMD, - SET_BACKUP_CMD, - SHOW_CONFIG_CMD, - CHECKDB_CMD -} ProbackupSubcmd; - - /* directory options */ char *backup_path = NULL; /* @@ -152,7 +131,6 @@ static pgSetBackupParams *set_backup_params = NULL; /* current settings */ pgBackup current; -static ProbackupSubcmd backup_subcmd = NO_CMD; static bool help_opt = false; @@ -160,7 +138,7 @@ static void opt_incr_restore_mode(ConfigOption *opt, const char *arg); static void opt_backup_mode(ConfigOption *opt, const char *arg); static void opt_show_format(ConfigOption *opt, const char *arg); -static void compress_init(void); +static void compress_init(ProbackupSubcmd const subcmd); static void opt_datname_exclude_list(ConfigOption *opt, const char *arg); static void opt_datname_include_list(ConfigOption *opt, const char *arg); @@ -259,32 +237,14 @@ static ConfigOption cmd_options[] = { 0 } }; -static void -setMyLocation(void) -{ - -#ifdef WIN32 - if (IsSshProtocol()) - elog(ERROR, "Currently remote operations on Windows are not supported"); -#endif - - MyLocation = IsSshProtocol() - ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST - : (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == ADD_INSTANCE_CMD) - ? FIO_BACKUP_HOST - : FIO_LOCAL_HOST - : FIO_LOCAL_HOST; -} - /* * Entry point of pg_probackup command. */ int main(int argc, char *argv[]) { - char *command = NULL, - *command_name; + char *command = NULL; + ProbackupSubcmd backup_subcmd = NO_CMD; PROGRAM_NAME_FULL = argv[0]; @@ -322,91 +282,58 @@ main(int argc, char *argv[]) /* Parse subcommands and non-subcommand options */ if (argc > 1) { - if (strcmp(argv[1], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[1], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[1], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[1], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[1], "merge") == 0) - backup_subcmd = MERGE_CMD; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[1], "set-backup") == 0) - backup_subcmd = SET_BACKUP_CMD; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[1], "checkdb") == 0) - backup_subcmd = CHECKDB_CMD; -#ifdef WIN32 - else if (strcmp(argv[1], "ssh") == 0) - launch_ssh(argv); -#endif - else if (strcmp(argv[1], "agent") == 0) - { - /* 'No forward compatibility' sanity: - * /old/binary -> ssh execute -> /newer/binary agent version_num - * If we are executed as an agent for older binary, then exit with error - */ - if (argc > 2) - { - elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " - "is launched as an agent for pg_probackup binary with version '%s'", - PROGRAM_VERSION, argv[2]); - } - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - } - else if (strcmp(argv[1], "--help") == 0 || - strcmp(argv[1], "-?") == 0 || - strcmp(argv[1], "help") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) + backup_subcmd = parse_subcmd(argv[1]); + switch(backup_subcmd) { -#ifdef PGPRO_VERSION - fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); + case SSH_CMD: +#ifdef WIN32 + launch_ssh(argv); + break; #else - fprintf(stdout, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); + elog(ERROR, "\"ssh\" command implemented only for Windows"); #endif - exit(0); + case AGENT_CMD: + /* 'No forward compatibility' sanity: + * /old/binary -> ssh execute -> /newer/binary agent version_num + * If we are executed as an agent for older binary, then exit with error + */ + if (argc > 2) + elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " + "is launched as an agent for pg_probackup binary with version '%s'", + PROGRAM_VERSION, argv[2]); + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; + case HELP_CMD: + if (argc > 2) + { + /* 'pg_probackup help command' style */ + help_command(parse_subcmd(argv[2])); + exit(0); + } + else + { + help_pg_probackup(); + exit(0); + } + break; + case VERSION_CMD: + help_print_version(); + exit(0); + case NO_CMD: + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + default: + /* Silence compiler warnings */ + break; } - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); } - - if (backup_subcmd == NO_CMD) - elog(ERROR, "No subcommand specified"); + else + elog(ERROR, "No subcommand specified. Please run with \"help\" argument to see possible subcommands."); /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ /* TODO why do we do that only for some commands? */ - command_name = pstrdup(argv[1]); if (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == VALIDATE_CMD || @@ -450,9 +377,14 @@ main(int argc, char *argv[]) show_color = false; if (help_opt) - help_command(command_name); + { + /* 'pg_probackup command --help' style */ + help_command(backup_subcmd); + exit(0); + } + + setMyLocation(backup_subcmd); - /* backup_path is required for all pg_probackup commands except help and checkdb */ if (backup_path == NULL) { /* @@ -460,12 +392,8 @@ main(int argc, char *argv[]) * from environment variable */ backup_path = getenv("BACKUP_PATH"); - if (backup_path == NULL && backup_subcmd != CHECKDB_CMD) - elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } - setMyLocation(); - if (backup_path != NULL) { canonicalize_path(backup_path); @@ -474,11 +402,9 @@ main(int argc, char *argv[]) if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); } - - /* Ensure that backup_path is an absolute path */ - if (backup_path && !is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); - + /* backup_path is required for all pg_probackup commands except help, version and checkdb */ + if (backup_path == NULL && backup_subcmd != CHECKDB_CMD && backup_subcmd != HELP_CMD && backup_subcmd != VERSION_CMD) + elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); /* * Option --instance is required for all commands except @@ -574,7 +500,8 @@ main(int argc, char *argv[]) else config_read_opt(path, instance_options, ERROR, true, false); } - setMyLocation(); + /* Зачем второй раз устанавливать? */ + setMyLocation(backup_subcmd); } /* @@ -676,7 +603,7 @@ main(int argc, char *argv[]) backup_subcmd != SET_BACKUP_CMD && backup_subcmd != SHOW_CMD) elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); current.backup_id = base36dec(backup_id_string); if (current.backup_id == 0) @@ -709,7 +636,7 @@ main(int argc, char *argv[]) if (force && backup_subcmd != RESTORE_CMD) elog(ERROR, "You cannot specify \"--force\" flag with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); if (force) no_validate = true; @@ -779,7 +706,7 @@ main(int argc, char *argv[]) /* sanity */ if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); if (num_threads < 1) num_threads = 1; @@ -787,7 +714,7 @@ main(int argc, char *argv[]) if (batch_size < 1) batch_size = 1; - compress_init(); + compress_init(backup_subcmd); /* do actual operation */ switch (backup_subcmd) @@ -881,6 +808,13 @@ main(int argc, char *argv[]) case NO_CMD: /* Should not happen */ elog(ERROR, "Unknown subcommand"); + case SSH_CMD: + case AGENT_CMD: + /* Может перейти на использование какого-нибудь do_agent() для однобразия? */ + case HELP_CMD: + case VERSION_CMD: + /* Silence compiler warnings, these already handled earlier */ + break; } return 0; @@ -943,13 +877,13 @@ opt_show_format(ConfigOption *opt, const char *arg) * Initialize compress and sanity checks for compress. */ static void -compress_init(void) +compress_init(ProbackupSubcmd const subcmd) { /* Default algorithm is zlib */ if (compress_shortcut) instance_config.compress_alg = ZLIB_COMPRESS; - if (backup_subcmd != SET_CONFIG_CMD) + if (subcmd != SET_CONFIG_CMD) { if (instance_config.compress_level != COMPRESS_LEVEL_DEFAULT && instance_config.compress_alg == NOT_DEFINED_COMPRESS) @@ -963,7 +897,7 @@ compress_init(void) if (instance_config.compress_alg == ZLIB_COMPRESS && instance_config.compress_level == 0) elog(WARNING, "Compression level 0 will lead to data bloat!"); - if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + if (subcmd == BACKUP_CMD || subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ if (instance_config.compress_alg == ZLIB_COMPRESS) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c53d31e95..a2c3309f8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -876,8 +876,9 @@ extern char *slurpFile(const char *datadir, extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ +extern void help_print_version(void); extern void help_pg_probackup(void); -extern void help_command(char *command); +extern void help_command(ProbackupSubcmd const subcmd); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); @@ -1162,6 +1163,7 @@ extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const c BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); /* FIO */ +extern void setMyLocation(ProbackupSubcmd const subcmd); extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, diff --git a/src/utils/configuration.c b/src/utils/configuration.c index d6a7d069e..05baaae53 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -87,6 +87,62 @@ static const unit_conversion time_unit_conversion_table[] = {""} /* end of table marker */ }; +/* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ +static char const * const subcmd_names[] = +{ + "NO_CMD", + "init", + "add-instance", + "del-instance", + "archive-push", + "archive-get", + "backup", + "restore", + "validate", + "delete", + "merge", + "show", + "set-config", + "set-backup", + "show-config", + "checkdb", + "ssh", + "agent", + "help", + "version", +}; + +ProbackupSubcmd +parse_subcmd(char const * const subcmd_str) +{ + struct { + ProbackupSubcmd id; + char *name; + } + static const subcmd_additional_names[] = { + { HELP_CMD, "--help" }, + { HELP_CMD, "-?" }, + { VERSION_CMD, "--version" }, + { VERSION_CMD, "-V" }, + }; + + int i; + for(i = (int)NO_CMD + 1; i < sizeof(subcmd_names) / sizeof(subcmd_names[0]); ++i) + if(strcmp(subcmd_str, subcmd_names[i]) == 0) + return (ProbackupSubcmd)i; + for(i = 0; i < sizeof(subcmd_additional_names) / sizeof(subcmd_additional_names[0]); ++i) + if(strcmp(subcmd_str, subcmd_additional_names[i].name) == 0) + return subcmd_additional_names[i].id; + return NO_CMD; +} + +char const * +get_subcmd_name(ProbackupSubcmd const subcmd) +{ + Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); + return subcmd_names[(int)subcmd]; +} + /* * Reading functions. */ diff --git a/src/utils/configuration.h b/src/utils/configuration.h index eea8c7746..4ed4e0e61 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -16,6 +16,31 @@ #define INFINITE_STR "INFINITE" +/* Order is important, keep it in sync with configuration.c:subcmd_names[] and help.c:help_command() */ +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SET_BACKUP_CMD, + SHOW_CONFIG_CMD, + CHECKDB_CMD, + SSH_CMD, + AGENT_CMD, + HELP_CMD, + VERSION_CMD +} ProbackupSubcmd; + typedef enum OptionSource { SOURCE_DEFAULT, @@ -75,6 +100,8 @@ struct ConfigOption #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) +extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); +extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], ConfigOption options[]); extern int config_read_opt(const char *path, ConfigOption options[], int elevel, diff --git a/src/utils/file.c b/src/utils/file.c index b29a67070..4adfa3fee 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -83,6 +83,24 @@ typedef struct #undef fopen(a, b) #endif +void +setMyLocation(ProbackupSubcmd const subcmd) +{ + +#ifdef WIN32 + if (IsSshProtocol()) + elog(ERROR, "Currently remote operations on Windows are not supported"); +#endif + + MyLocation = IsSshProtocol() + ? (subcmd == ARCHIVE_PUSH_CMD || subcmd == ARCHIVE_GET_CMD) + ? FIO_DB_HOST + : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD) + ? FIO_BACKUP_HOST + : FIO_LOCAL_HOST + : FIO_LOCAL_HOST; +} + /* Use specified file descriptors as stdin/stdout for FIO functions */ void fio_redirect(int in, int out, int err) { From 964d8dbbce996336b8347df48655102c4aad4ebb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 29 Dec 2020 18:26:43 +0300 Subject: [PATCH 1568/2107] add some comments about starting ssh client on Windows --- src/utils/remote.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/remote.c b/src/utils/remote.c index f590a82b4..88b1ce7a6 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -82,6 +82,11 @@ void wait_ssh(void) #endif } +/* + * On windows we launch a new pbk process via 'pg_probackup ssh ...' + * so this process would new that it should exec ssh, because + * there is no fork on Windows. + */ #ifdef WIN32 void launch_ssh(char* argv[]) { From 89931317e9bd935f6cfa165ec074fbd588dd9fb9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 29 Dec 2020 19:07:17 +0300 Subject: [PATCH 1569/2107] tests: fix backup.BackupTest.test_basic_missing_dir_permissions --- tests/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index 590d86dbf..de53c207b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1968,7 +1968,7 @@ def test_basic_missing_dir_permissions(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - os.chmod(full_path, 700) + os.rmdir(full_path) # Clean after yourself self.del_test_dir(module_name, fname, [node]) From 8aea21d70380c5f89d301afce4cc195b026c1aa9 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 30 Dec 2020 06:59:32 +0300 Subject: [PATCH 1570/2107] add version and help help pages --- src/help.c | 30 +++++++++++++++++++++++------- src/utils/configuration.c | 2 +- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/help.c b/src/help.c index 2af045cde..cab143ad8 100644 --- a/src/help.c +++ b/src/help.c @@ -11,6 +11,7 @@ #include "pg_probackup.h" static void help_nocmd(void); +static void help_internal(void); static void help_init(void); static void help_backup(void); static void help_restore(void); @@ -27,6 +28,7 @@ static void help_archive_push(void); static void help_archive_get(void); static void help_checkdb(void); static void help_help(void); +static void help_version(void); void help_print_version(void) @@ -64,10 +66,10 @@ help_command(ProbackupSubcmd const subcmd) &help_set_backup, &help_show_config, &help_checkdb, - &help_nocmd, // SSH_CMD - &help_nocmd, // AGENT_CMD + &help_internal, // SSH_CMD + &help_internal, // AGENT_CMD &help_help, - &help_help, // VERSION_CMD + &help_version, }; Assert((int)subcmd < sizeof(help_functions) / sizeof(help_functions[0])); @@ -77,9 +79,9 @@ help_command(ProbackupSubcmd const subcmd) void help_pg_probackup(void) { - printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n"), PROGRAM_NAME); - printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + printf(_("\n %s help [COMMAND]\n"), PROGRAM_NAME); printf(_("\n %s version\n"), PROGRAM_NAME); @@ -256,7 +258,13 @@ help_pg_probackup(void) static void help_nocmd(void) { - printf(_("Unknown command. Try pg_probackup help\n")); + printf(_("\nUnknown command. Try pg_probackup help\n\n")); +} + +static void +help_internal(void) +{ + printf(_("\nThis command is intended for internal use\n\n")); } static void @@ -984,5 +992,13 @@ help_archive_get(void) static void help_help(void) { - printf(_("No help page required for \"help\" and \"version\" commands. Just try it!\n")); + printf(_("\n%s help [command]\n"), PROGRAM_NAME); + printf(_("%s command --help\n\n"), PROGRAM_NAME); +} + +static void +help_version(void) +{ + printf(_("\n%s version\n"), PROGRAM_NAME); + printf(_("%s --version\n\n"), PROGRAM_NAME); } diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 05baaae53..afc1bc056 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -140,7 +140,7 @@ char const * get_subcmd_name(ProbackupSubcmd const subcmd) { Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); - return subcmd_names[(int)subcmd]; + return subcmd_names[(int)subcmd]; } /* From 7383ddd69c9f4b5f66ea5601f3764fdfda408fe3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 6 Jan 2021 19:15:59 +0300 Subject: [PATCH 1571/2107] add some comments --- src/pg_probackup.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 855d24d92..4b03f6a8c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -383,6 +383,7 @@ main(int argc, char *argv[]) exit(0); } + /* set location based on cmdline options only */ setMyLocation(backup_subcmd); if (backup_path == NULL) @@ -499,9 +500,14 @@ main(int argc, char *argv[]) config_read_opt(path, instance_options, ERROR, true, true); else config_read_opt(path, instance_options, ERROR, true, false); + + /* + * We can determine our location only after reading the configuration file, + * unless we are running arcive-push/archive-get - they are allowed to trust + * cmdline only. + */ + setMyLocation(backup_subcmd); } - /* Зачем второй раз устанавливать? */ - setMyLocation(backup_subcmd); } /* From 286d30dbff59c354e32680b6b1a990cc5612e9ee Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 20 Jan 2021 15:11:54 +0300 Subject: [PATCH 1572/2107] [Issue #231] make several attempts when creating backup directory, so two consecutive backups didn`t get the same backup ID --- src/backup.c | 34 ++++++++++++-------------- src/catalog.c | 60 ++++++++++++++++++++++++++++++++++++++-------- src/dir.c | 12 ++++++---- src/init.c | 10 ++++---- src/merge.c | 2 +- src/pg_probackup.c | 4 +--- src/pg_probackup.h | 6 ++--- src/utils/file.c | 4 ++-- tests/backup.py | 8 ++++--- 9 files changed, 90 insertions(+), 50 deletions(-) diff --git a/src/backup.c b/src/backup.c index f6d3a958b..364e86448 100644 --- a/src/backup.c +++ b/src/backup.c @@ -95,7 +95,6 @@ static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) { int i; - char database_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ char dst_backup_path[MAXPGPATH]; char label[1024]; @@ -265,10 +264,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* Update running backup meta with START LSN */ write_backup(¤t, true); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - pgBackupGetPath(¤t, external_prefix, lengthof(external_prefix), - EXTERNAL_DIR); + join_path_components(external_prefix, current.database_dir, EXTERNAL_DIR); /* initialize backup's file list */ backup_files_list = parray_new(); @@ -276,7 +272,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* start stream replication */ if (stream_wal) { - join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); + join_path_components(dst_backup_path, current.database_dir, PG_XLOG_DIR); fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); start_WAL_streaming(backup_conn, dst_backup_path, &instance_config.conn_opt, @@ -441,7 +437,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool join_path_components(dirpath, temp, file->rel_path); } else - join_path_components(dirpath, database_path, file->rel_path); + join_path_components(dirpath, current.database_dir, file->rel_path); elog(VERBOSE, "Create directory '%s'", dirpath); fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); @@ -475,7 +471,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool arg->nodeInfo = nodeInfo; arg->from_root = instance_config.pgdata; - arg->to_root = database_path; + arg->to_root = current.database_dir; arg->external_prefix = external_prefix; arg->external_dirs = external_dirs; arg->files_list = backup_files_list; @@ -552,7 +548,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool elog(ERROR, "Failed to find file \"%s\" in backup filelist.", XLOG_CONTROL_FILE); - set_min_recovery_point(pg_control, database_path, current.stop_lsn); + set_min_recovery_point(pg_control, current.database_dir, current.stop_lsn); } /* close and sync page header map */ @@ -609,7 +605,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* construct fullpath */ if (file->external_dir_num == 0) - join_path_components(to_fullpath, database_path, file->rel_path); + join_path_components(to_fullpath, current.database_dir, file->rel_path); else { char external_dst[MAXPGPATH]; @@ -726,8 +722,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) * Entry point of pg_probackup BACKUP subcommand. */ int -do_backup(time_t start_time, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs) +do_backup(pgSetBackupParams *set_backup_params, + bool no_validate, bool no_sync, bool backup_logs) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -736,13 +732,16 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); + /* Create backup directory and BACKUP_CONTROL_FILE */ + pgBackupCreateDir(¤t, backup_instance_path); + if (!instance_config.pgdata) elog(ERROR, "required parameter not specified: PGDATA " "(-D, --pgdata)"); /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; - current.start_time = start_time; + current.start_time = current.backup_id; StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); @@ -757,16 +756,13 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), + PROGRAM_VERSION, instance_name, base36enc(current.backup_id), pgBackupGetBackupMode(¤t), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); - /* Create backup directory and BACKUP_CONTROL_FILE */ - if (pgBackupCreateDir(¤t)) - elog(ERROR, "Cannot create backup directory"); if (!lock_backup(¤t, true, true)) elog(ERROR, "Cannot lock backup %s directory", - base36enc(current.start_time)); + base36enc(current.backup_id)); write_backup(¤t, true); /* set the error processing function for the backup process */ @@ -781,7 +777,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); if (current.from_replica) - elog(INFO, "Backup %s is going to be taken from standby", base36enc(start_time)); + elog(INFO, "Backup %s is going to be taken from standby", base36enc(current.backup_id)); /* TODO, print PostgreSQL full version */ //elog(INFO, "PostgreSQL version: %s", nodeInfo.server_version_str); diff --git a/src/catalog.c b/src/catalog.c index 4da6ff053..302154178 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -23,6 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); +static time_t create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; static parray *lock_files = NULL; @@ -1136,12 +1137,18 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; } -/* create backup directory in $BACKUP_PATH */ -int -pgBackupCreateDir(pgBackup *backup) +/* Create backup directory in $BACKUP_PATH + * Note, that backup_id attribute is updated, + * so it is possible to get diffrent values in + * pgBackup.start_time and pgBackup.backup_id. + * It may be ok or maybe not, so it's up to the caller + * to fix it or let it be. + */ + +void +pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) { int i; - char path[MAXPGPATH]; parray *subdirs = parray_new(); parray_append(subdirs, pg_strdup(DATABASE_DIR)); @@ -1163,13 +1170,10 @@ pgBackupCreateDir(pgBackup *backup) free_dir_list(external_list); } - pgBackupGetPath(backup, path, lengthof(path), NULL); - - if (!dir_is_empty(path, FIO_BACKUP_HOST)) - elog(ERROR, "backup destination is not empty \"%s\"", path); + backup->backup_id = create_backup_dir(backup, backup_instance_path); - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); - backup->root_dir = pgut_strdup(path); + if (backup->backup_id == 0) + elog(ERROR, "Cannot create backup directory: %s", strerror(errno)); backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); @@ -1180,11 +1184,47 @@ pgBackupCreateDir(pgBackup *backup) /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { + char path[MAXPGPATH]; + join_path_components(path, backup->root_dir, parray_get(subdirs, i)); fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); } free_dir_list(subdirs); +} + +/* + * Create root directory for backup, + * update pgBackup.root_dir if directory creation was a success + */ +time_t +create_backup_dir(pgBackup *backup, const char *backup_instance_path) +{ + int attempts = 10; + + while (attempts--) + { + int rc; + char path[MAXPGPATH]; + time_t backup_id = time(NULL); + + join_path_components(path, backup_instance_path, base36enc(backup_id)); + + /* TODO: add wrapper for remote mode */ + rc = dir_create_dir(path, DIR_PERMISSION, true); + + if (rc == 0) + { + backup->root_dir = pgut_strdup(path); + return backup_id; + } + else + { + elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); + sleep(1); + } + } + return 0; } diff --git a/src/dir.c b/src/dir.c index 9d7e3701d..7beede12e 100644 --- a/src/dir.c +++ b/src/dir.c @@ -138,9 +138,13 @@ static TablespaceList external_remap_list = {NULL, NULL}; /* * Create directory, also create parent directories if necessary. + * In strict mode treat already existing directory as error. + * Return values: + * 0 - ok + * -1 - error (check errno) */ int -dir_create_dir(const char *dir, mode_t mode) +dir_create_dir(const char *dir, mode_t mode, bool strict) { char parent[MAXPGPATH]; @@ -149,14 +153,14 @@ dir_create_dir(const char *dir, mode_t mode) /* Create parent first */ if (access(parent, F_OK) == -1) - dir_create_dir(parent, mode); + dir_create_dir(parent, mode, false); /* Create directory */ if (mkdir(dir, mode) == -1) { - if (errno == EEXIST) /* already exist */ + if (errno == EEXIST && !strict) /* already exist */ return 0; - elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + return -1; } return 0; diff --git a/src/init.c b/src/init.c index 431ea3b70..1ab6dc0f9 100644 --- a/src/init.c +++ b/src/init.c @@ -34,15 +34,15 @@ do_init(void) } /* create backup catalog root directory */ - dir_create_dir(backup_path, DIR_PERMISSION); + dir_create_dir(backup_path, DIR_PERMISSION, false); /* create backup catalog data directory */ join_path_components(path, backup_path, BACKUPS_DIR); - dir_create_dir(path, DIR_PERMISSION); + dir_create_dir(path, DIR_PERMISSION, false); /* create backup catalog wal directory */ join_path_components(arclog_path_dir, backup_path, "wal"); - dir_create_dir(arclog_path_dir, DIR_PERMISSION); + dir_create_dir(arclog_path_dir, DIR_PERMISSION, false); elog(INFO, "Backup catalog '%s' successfully inited", backup_path); return 0; @@ -91,8 +91,8 @@ do_add_instance(InstanceConfig *instance) instance->name, instance->arclog_path); /* Create directory for data files of this specific instance */ - dir_create_dir(instance->backup_instance_path, DIR_PERMISSION); - dir_create_dir(instance->arclog_path, DIR_PERMISSION); + dir_create_dir(instance->backup_instance_path, DIR_PERMISSION, false); + dir_create_dir(instance->arclog_path, DIR_PERMISSION, false); /* * Write initial configuration file. diff --git a/src/merge.c b/src/merge.c index 05220a5d6..3c51a1fae 100644 --- a/src/merge.c +++ b/src/merge.c @@ -639,7 +639,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) makeExternalDirPathByNum(new_container, full_external_prefix, file->external_dir_num); join_path_components(dirpath, new_container, file->rel_path); - dir_create_dir(dirpath, DIR_PERMISSION); + dir_create_dir(dirpath, DIR_PERMISSION, false); } pg_atomic_init_flag(&file->lock); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e34195ae6..90202fa84 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -799,8 +799,6 @@ main(int argc, char *argv[]) return do_init(); case BACKUP_CMD: { - time_t start_time = time(NULL); - current.stream = stream_wal; /* sanity */ @@ -808,7 +806,7 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs); + return do_backup(set_backup_params, no_validate, no_sync, backup_logs); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2b8253cb9..46775ebfc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -781,7 +781,7 @@ extern char** commands_args; extern const char *pgdata_exclude_dir[]; /* in backup.c */ -extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, +extern int do_backup(pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); @@ -916,7 +916,7 @@ extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, extern void pgBackupGetPathInInstance(const char *instance_name, const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); -extern int pgBackupCreateDir(pgBackup *backup); +extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); @@ -980,7 +980,7 @@ extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_external(const char *dir, parray *dirs_list); -extern int dir_create_dir(const char *path, mode_t mode); +extern int dir_create_dir(const char *path, mode_t mode, bool strict); extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); diff --git a/src/utils/file.c b/src/utils/file.c index b29a67070..c5c962edb 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1048,7 +1048,7 @@ int fio_mkdir(char const* path, int mode, fio_location location) } else { - return dir_create_dir(path, mode); + return dir_create_dir(path, mode, false); } } @@ -2648,7 +2648,7 @@ void fio_communicate(int in, int out) break; case FIO_MKDIR: /* Create directory */ hdr.size = 0; - hdr.arg = dir_create_dir(buf, hdr.arg); + hdr.arg = dir_create_dir(buf, hdr.arg, false); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CHMOD: /* Change file mode */ diff --git a/tests/backup.py b/tests/backup.py index de53c207b..66bc5d323 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2921,12 +2921,14 @@ def test_issue_231(self): try: self.backup_node( - backup_dir, 'node', node, - data_dir='{0}'.format(datadir), return_id=False) + backup_dir, 'node', node, data_dir='{0}'.format(datadir)) except: pass - self.backup_node(backup_dir, 'node', node, options=['--stream']) + out = self.backup_node(backup_dir, 'node', node, options=['--stream'], return_id=False) + + # it is a bit racy + self.assertIn("WARNING: Cannot create directory", out) # Clean after yourself self.del_test_dir(module_name, fname) From e7ffc15417a3a072aca2f8c8db7eacc6d042ac30 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 20 Jan 2021 15:39:38 +0300 Subject: [PATCH 1573/2107] [Issue #289] Wait for Start LSN after finding the suitable parent backup, not before --- src/backup.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index 364e86448..6e9cc224b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -146,15 +146,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool current.tli = get_current_timeline_from_control(false); #endif - /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal) - /* - * Do not wait start_lsn for stream backup. - * Because WAL streaming will start after pg_start_backup() in stream - * mode. - */ - wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); - /* * In incremental backup mode ensure that already-validated * backup on current timeline exists and get its filelist. @@ -264,10 +255,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* Update running backup meta with START LSN */ write_backup(¤t, true); - join_path_components(external_prefix, current.database_dir, EXTERNAL_DIR); - - /* initialize backup's file list */ - backup_files_list = parray_new(); + /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !stream_wal) + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); /* start stream replication */ if (stream_wal) @@ -279,6 +274,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool current.start_lsn, current.tli); } + /* initialize backup's file list */ + backup_files_list = parray_new(); + join_path_components(external_prefix, current.database_dir, EXTERNAL_DIR); + /* list files with the logical path. omit $PGDATA */ if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(backup_files_list, instance_config.pgdata, From 4edc9d94ad76421fb5afabd4aeeb3dddd09c9997 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 20 Jan 2021 15:40:18 +0300 Subject: [PATCH 1574/2107] [Issue #289] tests: added tests.backup.BackupTest.test_issue_289 --- tests/backup.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 66bc5d323..77787bd30 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2859,6 +2859,57 @@ def test_parent_backup_made_by_newer_version(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_issue_289(self): + """ + https://github.com/postgrespro/pg_probackup/issues/289 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) +# self.set_archiving(backup_dir, 'node', node) + +# os.rmdir( +# os.path.join(backup_dir, "wal", "node")) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "ERROR: Create new full backup before an incremental one", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + + exit(1) + + # Clean after yourself + self.del_test_dir(module_name, fname) + @unittest.skip("skip") def test_issue_203(self): """ From d624863d14289e4b8428c4b7b2a576126edd795e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 20 Jan 2021 15:55:25 +0300 Subject: [PATCH 1575/2107] [Issue #290] check WAL archive directory presence at the start of backup --- src/backup.c | 7 +++++++ tests/backup.py | 53 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6e9cc224b..00c1cc28b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -257,12 +257,19 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !stream_wal) + { + /* Check that archive_dir can be reached */ + if (fio_access(arclog_path, F_OK, FIO_BACKUP_HOST) != 0) + elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", + arclog_path, strerror(errno)); + /* * Do not wait start_lsn for stream backup. * Because WAL streaming will start after pg_start_backup() in stream * mode. */ wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); + } /* start stream replication */ if (stream_wal) diff --git a/tests/backup.py b/tests/backup.py index 77787bd30..3c0686f9b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2872,10 +2872,6 @@ def test_issue_289(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) -# self.set_archiving(backup_dir, 'node', node) - -# os.rmdir( -# os.path.join(backup_dir, "wal", "node")) node.slow_start() @@ -2905,7 +2901,54 @@ def test_issue_289(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - exit(1) + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_issue_290(self): + """ + https://github.com/postgrespro/pg_probackup/issues/290 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + os.rmdir( + os.path.join(backup_dir, "wal", "node")) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WAL archive directory is not accessible", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") # Clean after yourself self.del_test_dir(module_name, fname) From b16555acd6448276578398388555fa8641f65ebd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 21 Jan 2021 13:14:51 +0300 Subject: [PATCH 1576/2107] fix external dir path, introduced by 286d30dbff59 --- src/backup.c | 12 ++++++------ src/utils/file.c | 4 +++- src/validate.c | 4 +--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/backup.c b/src/backup.c index 00c1cc28b..2e0e08da6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -283,7 +283,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* initialize backup's file list */ backup_files_list = parray_new(); - join_path_components(external_prefix, current.database_dir, EXTERNAL_DIR); + join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR); /* list files with the logical path. omit $PGDATA */ if (fio_is_remote(FIO_DB_HOST)) @@ -738,6 +738,11 @@ do_backup(pgSetBackupParams *set_backup_params, /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); + /* Save list of external directories */ + if (instance_config.external_dir_str && + (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) + current.external_dir_str = instance_config.external_dir_str; + /* Create backup directory and BACKUP_CONTROL_FILE */ pgBackupCreateDir(¤t, backup_instance_path); @@ -755,11 +760,6 @@ do_backup(pgSetBackupParams *set_backup_params, current.compress_alg = instance_config.compress_alg; current.compress_level = instance_config.compress_level; - /* Save list of external directories */ - if (instance_config.external_dir_str && - (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) - current.external_dir_str = instance_config.external_dir_str; - elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", PROGRAM_VERSION, instance_name, base36enc(current.backup_id), pgBackupGetBackupMode(¤t), diff --git a/src/utils/file.c b/src/utils/file.c index c5c962edb..270d83fac 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1026,7 +1026,9 @@ int fio_unlink(char const* path, fio_location location) } } -/* Create directory */ +/* Create directory + * TODO: add strict flag + */ int fio_mkdir(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) diff --git a/src/validate.c b/src/validate.c index 30bfbf3e4..379c19c56 100644 --- a/src/validate.c +++ b/src/validate.c @@ -48,7 +48,6 @@ typedef struct void pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { - char base_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; parray *files = NULL; bool corrupted = false; @@ -115,7 +114,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->backup_mode != BACKUP_MODE_DIFF_DELTA) elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); - join_path_components(base_path, backup->root_dir, DATABASE_DIR); join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); files = get_backup_filelist(backup, false); @@ -149,7 +147,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { validate_files_arg *arg = &(threads_args[i]); - arg->base_path = base_path; + arg->base_path = backup->database_dir; arg->files = files; arg->corrupted = false; arg->backup_mode = backup->backup_mode; From 4e445024f2522706fb3fd60d50291bede5fcec62 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Jan 2021 15:56:14 +0300 Subject: [PATCH 1577/2107] [Issue #280] Expand "--force" flag for incremental restore, now in case of system ID mismatch the destination PGDATA will be deleted; the content of the directory, used as destination for tablespace remapping, is now also deleted. Tablespace map is now validated before reading. --- doc/pgprobackup.xml | 4 + src/backup.c | 8 +- src/delete.c | 1 + src/dir.c | 181 +++++++++++--- src/pg_probackup.h | 20 +- src/restore.c | 209 +++++++++++----- src/utils/file.c | 19 +- src/validate.c | 52 ++++ tests/helpers/ptrack_helpers.py | 29 +++ tests/incr_restore.py | 429 ++++++++++++++++++++++++++++++-- tests/restore.py | 136 ++++++++++ tests/time_stamp.py | 2 +- 12 files changed, 964 insertions(+), 126 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 0f844ec39..d6d69da4f 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3976,6 +3976,10 @@ pg_probackup restore -B backup_dir --instance PostgreSQL cluster from a corrupt or an invalid backup. Use with caution. + When used with incremental restore this flag + allows to replace already existing PGDATA with different system ID. In case of tablespaces, + remapped via --tablespace-mapping option into not empty directories, + the old content of such directories will be deleted. diff --git a/src/backup.c b/src/backup.c index 2e0e08da6..ca8baa777 100644 --- a/src/backup.c +++ b/src/backup.c @@ -286,12 +286,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR); /* list files with the logical path. omit $PGDATA */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0); - else - dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + fio_list_dir(backup_files_list, instance_config.pgdata, + true, true, false, backup_logs, true, 0); /* * Get database_map (name to oid) for use in partial restore feature. diff --git a/src/delete.c b/src/delete.c index a6d6b51ba..c2b935d67 100644 --- a/src/delete.c +++ b/src/delete.c @@ -720,6 +720,7 @@ do_retention_wal(bool dry_run) /* * Delete backup files of the backup and update the status of the backup to * BACKUP_STATUS_DELETED. + * TODO: delete files on multiple threads */ void delete_backup_files(pgBackup *backup) diff --git a/src/dir.c b/src/dir.c index 7beede12e..d07a4d2f5 100644 --- a/src/dir.c +++ b/src/dir.c @@ -130,6 +130,7 @@ static void dir_list_file_internal(parray *files, pgFile *parent, const char *pa bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); +static void cleanup_tablespace(const char *path); /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; @@ -518,6 +519,8 @@ db_map_entry_free(void *entry) * * When follow_symlink is true, symbolic link is ignored and only file or * directory linked to will be listed. + * + * TODO: make it strictly local */ void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, @@ -1088,7 +1091,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba const char *linked_path = get_tablespace_mapping((*link)->linked); if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory is not an absolute path: %s\n", + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", linked_path); join_path_components(to_path, data_dir, dir->rel_path); @@ -1128,7 +1131,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba * tablespace_map or tablespace_map.txt. */ void -read_tablespace_map(parray *files, const char *backup_dir) +read_tablespace_map(parray *links, const char *backup_dir) { FILE *fp; char db_path[MAXPGPATH], @@ -1138,16 +1141,9 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(db_path, backup_dir, DATABASE_DIR); join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); - /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path, FIO_BACKUP_HOST)) - { - elog(LOG, "there is no file tablespace_map"); - return; - } - fp = fio_open_stream(map_path, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno)); while (fgets(buf, lengthof(buf), fp)) { @@ -1166,7 +1162,7 @@ read_tablespace_map(parray *files, const char *backup_dir) file->linked = pgut_strdup(path); canonicalize_path(file->linked); - parray_append(files, file); + parray_append(links, file); } if (ferror(fp)) @@ -1183,30 +1179,49 @@ read_tablespace_map(parray *files, const char *backup_dir) * If tablespace-mapping option is supplied, all OLDDIR entries must have * entries in tablespace_map file. * - * - * TODO: maybe when running incremental restore with tablespace remapping, then - * new tablespace directory MUST be empty? because there is no way + * When running incremental restore with tablespace remapping, then + * new tablespace directory MUST be empty, because there is no way * we can be sure, that files laying there belong to our instance. + * But "force" flag allows to ignore this condition, by wiping out + * the current content on the directory. + * + * Exit codes: + * 1. backup has no tablespaces + * 2. backup has tablespaces and they are empty + * 3. backup has tablespaces and some of them are not empty */ -void -check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty) +int +check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty) { -// char this_backup_path[MAXPGPATH]; - parray *links; + parray *links = parray_new(); size_t i; TablespaceListCell *cell; pgFile *tmp_file = pgut_new(pgFile); + bool tblspaces_are_empty = true; - links = parray_new(); + elog(LOG, "Checking tablespace directories of backup %s", + base36enc(backup->start_time)); + + /* validate tablespace map, + * if there are no tablespaces, then there is nothing left to do + */ + if (!validate_tablespace_map(backup)) + { + /* + * Sanity check + * If there is no tablespaces in backup, + * then using the '--tablespace-mapping' option is a mistake. + */ + if (tablespace_dirs.head != NULL) + elog(ERROR, "Backup %s has no tablespaceses, nothing to remap " + "via \"--tablespace-mapping\" option", base36enc(backup->backup_id)); + return NoTblspc; + } -// pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); read_tablespace_map(links, backup->root_dir); /* Sort links by the path of a linked file*/ parray_qsort(links, pgFileCompareLinked); - elog(LOG, "check tablespace directories of backup %s", - base36enc(backup->start_time)); - /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ for (cell = tablespace_dirs.head; cell; cell = cell->next) { @@ -1216,52 +1231,109 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are elog(ERROR, "--tablespace-mapping option's old directory " "doesn't have an entry in tablespace_map file: \"%s\"", cell->old_dir); - - /* For incremental restore, check that new directory is empty */ -// if (incremental) -// { -// if (!is_absolute_path(cell->new_dir)) -// elog(ERROR, "tablespace directory is not an absolute path: %s\n", -// cell->new_dir); -// -// if (!dir_is_empty(cell->new_dir, FIO_DB_HOST)) -// elog(ERROR, "restore tablespace destination is not empty: \"%s\"", -// cell->new_dir); -// } } + /* + * There is difference between incremental restore of already existing + * tablespaceses and remapped tablespaceses. + * Former are allowed to be not empty, because we treat them like an + * extension of PGDATA. + * The latter are not, unless "--force" flag is used. + * in which case the remapped directory is nuked - just to be safe, + * because it is hard to be sure that there are no some tricky corner + * cases of pages from different systems having the same crc. + * This is a strict approach. + * + * Why can`t we not nuke it and just let it roll ? + * What if user just wants to rerun failed restore with the same + * parameters? Nuking is bad for this case. + * + * Consider the example of existing PGDATA: + * .... + * pg_tablespace + * 100500-> /somedirectory + * .... + * + * We want to remap it during restore like that: + * .... + * pg_tablespace + * 100500-> /somedirectory1 + * .... + * + * Usually it is required for "/somedirectory1" to be empty, but + * in case of incremental restore with 'force' flag, which required + * of us to drop already existing content of "/somedirectory1". + * + * TODO: Ideally in case of incremental restore we must also + * drop the "/somedirectory" directory first, but currently + * we don`t do that. + */ + /* 2 - all linked directories must be empty */ for (i = 0; i < parray_num(links); i++) { pgFile *link = (pgFile *) parray_get(links, i); const char *linked_path = link->linked; TablespaceListCell *cell; + bool remapped = false; for (cell = tablespace_dirs.head; cell; cell = cell->next) if (strcmp(link->linked, cell->old_dir) == 0) { linked_path = cell->new_dir; + remapped = true; break; } if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", linked_path); if (!dir_is_empty(linked_path, FIO_DB_HOST)) { + if (!incremental) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - *tblspaces_are_empty = false; + elog(ERROR, "Restore tablespace destination is not empty: \"%s\"", linked_path); + + else if (remapped && !force) + elog(ERROR, "Remapped tablespace destination is not empty: \"%s\". " + "Use \"--force\" flag if you want to automatically clean up the " + "content of new tablespace destination", + linked_path); + + else if (pgdata_is_empty && !force) + elog(ERROR, "PGDATA is empty, but tablespace destination is not: \"%s\". " + "Use \"--force\" flag is you want to automatically clean up the " + "content of tablespace destination", + linked_path); + + /* + * TODO: compile the list of tblspc Oids to delete later, + * similar to what we do with database_map. + */ + else if (force && (pgdata_is_empty || remapped)) + { + elog(WARNING, "Cleaning up the content of %s directory: \"%s\"", + remapped ? "remapped tablespace" : "tablespace", linked_path); + cleanup_tablespace(linked_path); + continue; + } + + tblspaces_are_empty = false; } } free(tmp_file); parray_walk(links, pgFileFree); parray_free(links); + + if (tblspaces_are_empty) + return EmptyTblspc; + + return NotEmptyTblspc; } +/* TODO: Make it consistent with check_tablespace_mapping */ void check_external_dir_mapping(pgBackup *backup, bool incremental) { @@ -1854,3 +1926,34 @@ read_database_map(pgBackup *backup) return database_map; } + +/* + * Use it to cleanup tablespaces + * TODO: Current algorihtm is not very efficient in remote mode, + * due to round-trip to delete every file. + */ +void +cleanup_tablespace(const char *path) +{ + int i; + char fullpath[MAXPGPATH]; + parray *files = parray_new(); + + fio_list_dir(files, path, false, false, false, false, false, 0); + + /* delete leaf node first */ + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + join_path_components(fullpath, path, file->rel_path); + + fio_delete(file->mode, fullpath, FIO_DB_HOST); + elog(VERBOSE, "Deleted file \"%s\"", fullpath); + } + + parray_walk(files, pgFileFree); + parray_free(files); +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 46775ebfc..1aab841c2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -132,6 +132,16 @@ typedef struct db_map_entry char *datname; } db_map_entry; +/* State of pgdata in the context of its compatibility for incremental restore */ +typedef enum DestDirIncrCompatibility +{ + POSTMASTER_IS_RUNNING, + SYSTEM_ID_MISMATCH, + BACKUP_LABEL_EXISTS, + DEST_IS_NOT_OK, + DEST_OK +} DestDirIncrCompatibility; + typedef enum IncrRestoreMode { INCR_NONE, @@ -250,6 +260,11 @@ typedef struct page_map_entry /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ +/* Return codes for check_tablespace_mapping */ +#define NoTblspc 0 +#define EmptyTblspc 1 +#define NotEmptyTblspc 2 + /* Current state of backup */ typedef enum BackupStatus { @@ -868,6 +883,7 @@ extern int do_validate_all(void); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); +extern bool validate_tablespace_map(pgBackup *backup); /* return codes for validate_one_page */ /* TODO: use enum */ @@ -957,10 +973,10 @@ extern void create_data_directories(parray *dest_files, bool incremental, fio_location location); -extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void read_tablespace_map(parray *links, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); -extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty); +extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty); extern void check_external_dir_mapping(pgBackup *backup, bool incremental); extern char *get_external_remap(char *current_dir); diff --git a/src/restore.c b/src/restore.c index bf278f8b2..ede049ecc 100644 --- a/src/restore.c +++ b/src/restore.c @@ -65,9 +65,10 @@ static void set_orphan_status(parray *backups, pgBackup *parent_backup); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync); -static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode); + const char *pgdata_path, bool no_sync, bool cleanup_pgdata, + bool backup_has_tblspc); +static DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, + IncrRestoreMode incremental_mode); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -131,39 +132,87 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray *parent_chain = NULL; parray *dbOid_exclude_list = NULL; bool pgdata_is_empty = true; - bool tblspaces_are_empty = true; + bool cleanup_pgdata = false; + bool backup_has_tblspc = true; /* backup contain tablespace */ XLogRecPtr shift_lsn = InvalidXLogRecPtr; + if (instance_name == NULL) + elog(ERROR, "required parameter not specified: --instance"); + if (params->is_restore) { if (instance_config.pgdata == NULL) elog(ERROR, "required parameter not specified: PGDATA (-D, --pgdata)"); + /* Check if restore destination empty */ if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) { + /* if destination directory is empty, then incremental restore may be disabled */ + pgdata_is_empty = false; + /* Check that remote system is NOT running and systemd id is the same as ours */ if (params->incremental_mode != INCR_NONE) { + DestDirIncrCompatibility rc; + bool ok_to_go = true; + elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", instance_config.pgdata); - check_incremental_compatibility(instance_config.pgdata, - instance_config.system_identifier, - params->incremental_mode); + rc = check_incremental_compatibility(instance_config.pgdata, + instance_config.system_identifier, + params->incremental_mode); + if (rc == POSTMASTER_IS_RUNNING) + { + /* Even with force flag it is unwise to run + * incremental restore over running instance + */ + ok_to_go = false; + } + else if (rc == SYSTEM_ID_MISMATCH) + { + /* + * In force mode it is possible to ignore system id mismatch + * by just wiping clean the destination directory. + */ + if (params->incremental_mode != INCR_NONE && params->force) + cleanup_pgdata = true; + else + ok_to_go = false; + } + else if (rc == BACKUP_LABEL_EXISTS) + { + /* + * A big no-no for lsn-based incremental restore + * If there is backup label in PGDATA, then this cluster was probably + * restored from backup, but not started yet. Which means that values + * in pg_control are not synchronized with PGDATA and so we cannot use + * incremental restore in LSN mode, because it is relying on pg_control + * to calculate switchpoint. + */ + if (params->incremental_mode == INCR_LSN) + ok_to_go = false; + } + else if (rc == DEST_IS_NOT_OK) + { + /* + * Something else is wrong. For example, postmaster.pid is mangled, + * so we cannot be sure that postmaster is running or not. + * It is better to just error out. + */ + ok_to_go = false; + } + + if (!ok_to_go) + elog(ERROR, "Incremental restore is not allowed"); } else elog(ERROR, "Restore destination is not empty: \"%s\"", instance_config.pgdata); - - /* if destination directory is empty, then incremental restore may be disabled */ - pgdata_is_empty = false; } } - if (instance_name == NULL) - elog(ERROR, "required parameter not specified: --instance"); - elog(LOG, "%s begin.", action); /* Get list of all backups sorted in order of descending start time */ @@ -356,9 +405,15 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty); + int rc = check_tablespace_mapping(dest_backup, + params->incremental_mode != INCR_NONE, params->force, + pgdata_is_empty); - if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty) + /* backup contain no tablespaces */ + if (rc == NoTblspc) + backup_has_tblspc = false; + + if (params->incremental_mode != INCR_NONE && !cleanup_pgdata && pgdata_is_empty && (rc != NotEmptyTblspc)) { elog(INFO, "Destination directory and tablespace directories are empty, " "disable incremental restore"); @@ -366,6 +421,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* no point in checking external directories if their restore is not requested */ + //TODO: + // - make check_external_dir_mapping more like check_tablespace_mapping + // - honor force flag in case of incremental restore just like check_tablespace_mapping if (!params->skip_external_dirs) check_external_dir_mapping(dest_backup, params->incremental_mode != INCR_NONE); } @@ -610,8 +668,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, base36enc(dest_backup->start_time), dest_backup->server_version); - restore_chain(dest_backup, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata, no_sync); + restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, + instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); //TODO rename and update comment /* Create recovery.conf with given recovery target parameters */ @@ -634,11 +692,13 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * Restore backup chain. + * Flag 'cleanup_pgdata' demands the removing of already existing content in PGDATA. */ void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync) + const char *pgdata_path, bool no_sync, bool cleanup_pgdata, + bool backup_has_tblspc) { int i; char timestamp[100]; @@ -736,7 +796,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * Restore dest_backup internal directories. */ create_data_directories(dest_files, instance_config.pgdata, - dest_backup->root_dir, true, + dest_backup->root_dir, backup_has_tblspc, params->incremental_mode != INCR_NONE, FIO_DB_HOST); @@ -789,18 +849,24 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } /* Get list of files in destination directory and remove redundant files */ - if (params->incremental_mode != INCR_NONE) + if (params->incremental_mode != INCR_NONE || cleanup_pgdata) { pgdata_files = parray_new(); elog(INFO, "Extracting the content of destination directory for incremental restore"); time(&start_time); - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); - else - dir_list_file(pgdata_files, pgdata_path, - false, true, false, false, true, 0, FIO_LOCAL_HOST); + fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); + + /* + * TODO: + * 1. Currently we are cleaning the tablespaces in check_tablespace_mapping and PGDATA here. + * It would be great to do all this work in one place. + * + * 2. In case of tablespace remapping we do not cleanup the old tablespace path, + * it is just left as it is. + * Lookup tests.incr_restore.IncrRestoreTest.test_incr_restore_with_tablespace_5 + */ /* get external dirs content */ if (external_dirs) @@ -810,13 +876,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, char *external_path = parray_get(external_dirs, i); parray *external_files = parray_new(); - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(external_files, external_path, - false, true, false, false, true, i+1); - else - dir_list_file(external_files, external_path, - false, true, false, false, true, i+1, - FIO_LOCAL_HOST); + fio_list_dir(external_files, external_path, + false, true, false, false, true, i+1); parray_concat(pgdata_files, external_files); parray_free(external_files); @@ -836,25 +897,41 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time(&start_time); for (i = 0; i < parray_num(pgdata_files); i++) { - pgFile *file = (pgFile *) parray_get(pgdata_files, i); + bool redundant = true; + pgFile *file = (pgFile *) parray_get(pgdata_files, i); + + if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal)) + redundant = false; + + /* do not delete the useful internal directories */ + if (S_ISDIR(file->mode) && !redundant) + continue; /* if file does not exists in destination list, then we can safely unlink it */ - if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal) == NULL) + if (cleanup_pgdata || redundant) { char fullpath[MAXPGPATH]; join_path_components(fullpath, pgdata_path, file->rel_path); -// fio_pgFileDelete(file, full_file_path); fio_delete(file->mode, fullpath, FIO_DB_HOST); elog(VERBOSE, "Deleted file \"%s\"", fullpath); /* shrink pgdata list */ + pgFileFree(file); parray_remove(pgdata_files, i); i--; } } + if (cleanup_pgdata) + { + /* Destination PGDATA and tablespaces were cleaned up, so it's the regular restore from this point */ + params->incremental_mode = INCR_NONE; + parray_free(pgdata_files); + pgdata_files = NULL; + } + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -2033,36 +2110,21 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* Check that instance is suitable for incremental restore * Depending on type of incremental restore requirements are differs. + * + * TODO: add PG_CONTROL_IS_MISSING */ -void +DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, IncrRestoreMode incremental_mode) { uint64 system_id_pgdata; + bool system_id_match = false; bool success = true; + bool postmaster_is_up = false; + bool backup_label_exists = false; pid_t pid; char backup_label[MAXPGPATH]; - /* slurp pg_control and check that system ID is the same */ - /* check that instance is not running */ - /* if lsn_based, check that there is no backup_label files is around AND - * get redo point lsn from destination pg_control. - - * It is really important to be sure that pg_control is in cohesion with - * data files content, because based on pg_control information we will - * choose a backup suitable for lsn based incremental restore. - */ - - system_id_pgdata = get_system_identifier(pgdata); - - if (system_id_pgdata != instance_config.system_identifier) - { - elog(WARNING, "Backup catalog was initialized for system id %lu, " - "but destination directory system id is %lu", - system_identifier, system_id_pgdata); - success = false; - } - /* check postmaster pid */ pid = fio_check_postmaster(pgdata, FIO_DB_HOST); @@ -2080,8 +2142,28 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", pid, pgdata); success = false; + postmaster_is_up = true; } + /* slurp pg_control and check that system ID is the same + * check that instance is not running + * if lsn_based, check that there is no backup_label files is around AND + * get redo point lsn from destination pg_control. + + * It is really important to be sure that pg_control is in cohesion with + * data files content, because based on pg_control information we will + * choose a backup suitable for lsn based incremental restore. + */ + + system_id_pgdata = get_system_identifier(pgdata); + + if (system_id_pgdata == instance_config.system_identifier) + system_id_match = true; + else + elog(WARNING, "Backup catalog was initialized for system id %lu, " + "but destination directory system id is %lu", + system_identifier, system_id_pgdata); + /* * TODO: maybe there should be some other signs, pointing to pg_control * desynchronization with cluster state. @@ -2097,9 +2179,22 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, "to cluster with pg_control not synchronized with cluster state." "Consider to use incremental restore in 'checksum' mode"); success = false; + backup_label_exists = true; } } + if (postmaster_is_up) + return POSTMASTER_IS_RUNNING; + + if (!system_id_match) + return SYSTEM_ID_MISMATCH; + + if (backup_label_exists) + return BACKUP_LABEL_EXISTS; + + /* some other error condition */ if (!success) - elog(ERROR, "Incremental restore is impossible"); + return DEST_IS_NOT_OK; + + return DEST_OK; } diff --git a/src/utils/file.c b/src/utils/file.c index 270d83fac..e23674548 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2163,9 +2163,9 @@ static void fio_send_file_impl(int out, char const* path) } /* Compile the array of files located on remote machine in directory root */ -void fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) +static void fio_list_dir_internal(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, bool backup_logs, + bool skip_hidden, int external_dir_num) { fio_header hdr; fio_list_dir_request req; @@ -2321,6 +2321,19 @@ static void fio_list_dir_impl(int out, char* buf) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } +/* Wrapper for directory listing */ +void fio_list_dir(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, bool backup_logs, + bool skip_hidden, int external_dir_num) +{ + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir_internal(files, root, exclude, follow_symlink, add_root, + backup_logs, skip_hidden, external_dir_num); + else + dir_list_file(files, root, exclude, follow_symlink, add_root, + backup_logs, skip_hidden, external_dir_num, FIO_LOCAL_HOST); +} + PageState * fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) diff --git a/src/validate.c b/src/validate.c index 379c19c56..21900c8e4 100644 --- a/src/validate.c +++ b/src/validate.c @@ -698,3 +698,55 @@ do_validate_instance(void) parray_walk(backups, pgBackupFree); parray_free(backups); } + +/* + * Validate tablespace_map checksum. + * Error out in case of checksum mismatch. + * Return 'false' if there are no tablespaces in backup. + * + * TODO: it is a bad, that we read the whole filelist just for + * the sake of tablespace_map. Probably pgBackup should come with + * already filled pgBackup.files + */ +bool +validate_tablespace_map(pgBackup *backup) +{ + char map_path[MAXPGPATH]; + pgFile *dummy = NULL; + pgFile **tablespace_map = NULL; + pg_crc32 crc; + parray *files = get_backup_filelist(backup, true); + + parray_qsort(files, pgFileCompareRelPathWithExternal); + join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); + + dummy = pgFileInit(PG_TABLESPACE_MAP_FILE); + tablespace_map = (pgFile **) parray_bsearch(files, dummy, pgFileCompareRelPathWithExternal); + + if (!tablespace_map) + { + elog(LOG, "there is no file tablespace_map"); + parray_walk(files, pgFileFree); + parray_free(files); + return false; + } + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path, FIO_BACKUP_HOST)) + elog(ERROR, "Tablespace map is missing: \"%s\", " + "probably backup %s is corrupt, validate it", + map_path, base36enc(backup->backup_id)); + + /* check tablespace map checksumms */ + crc = pgFileGetCRC(map_path, true, false); + + if ((*tablespace_map)->crc != crc) + elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " + "probably backup %s is corrupt, validate it", + map_path, crc, (*tablespace_map)->crc, base36enc(backup->backup_id)); + + pgFileFree(dummy); + parray_walk(files, pgFileFree); + parray_free(files); + return true; +} diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d340213c2..3c75ca2e7 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -434,6 +434,35 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False # res[0], 0, # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + def drop_tblspace(self, node, tblspc_name): + res = node.execute( + 'postgres', + 'select exists' + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertTrue( + res[0][0], + 'Tablespace "{0}" do not exists'.format(tblspc_name) + ) + + rels = node.execute( + "postgres", + "SELECT relname FROM pg_class c " + "LEFT JOIN pg_tablespace t ON c.reltablespace = t.oid " + "where c.relkind = 'r' and t.spcname = '{0}'".format(tblspc_name)) + + for rel in rels: + node.safe_psql( + 'postgres', + "DROP TABLE {0}".format(rel[0])) + + node.safe_psql( + 'postgres', + 'DROP TABLESPACE {0}'.format(tblspc_name)) + + def get_tblspace_path(self, node, tblspc_name): return os.path.join(node.base_dir, tblspc_name) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index c71b952c8..54c9a9ebf 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -80,9 +80,59 @@ def test_basic_incr_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_basic_incr_restore_into_missing_directory(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + print(self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"])) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_checksum_corruption_detection(self): - """recovery to target timeline""" + """ + """ fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -127,7 +177,8 @@ def test_checksum_corruption_detection(self): node.stop() self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -169,7 +220,7 @@ def test_incr_restore_with_tablespace(self): self.restore_node( backup_dir, 'node', node, options=[ - "-j", "4", "--incremental-mode=checksum", + "-j", "4", "--incremental-mode=checksum", "--force", "-T{0}={1}".format(tblspace, some_directory)]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -255,30 +306,352 @@ def test_incr_restore_with_tablespace_2(self): self.add_instance(backup_dir, 'node', node) node.slow_start() + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node_1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_1')) + + # fill node1 with data + out = self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '--force']) + + self.assertIn("WARNING: Backup catalog was initialized for system id", out) + tblspace = self.get_tblspace_path(node, 'tblspace') self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=10, tablespace='tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') - self.backup_node(backup_dir, 'node', node, options=['--stream']) + node.safe_psql( + 'postgres', + 'vacuum') - pgdata = self.pgdata_content(node.data_dir) + self.backup_node(backup_dir, 'node', node, backup_type='delta', options=['--stream']) - node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + pgdata = self.pgdata_content(node.data_dir) - node_1.cleanup() + try: + self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped directory is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.restore_node( + out = self.restore_node( backup_dir, 'node', node, data_dir=node_1.data_dir, - options=['--incremental-mode=checksum']) + options=[ + '--force', '--incremental-mode=checksum', + '-T{0}={1}'.format(tblspace, tblspace)]) + + pgdata_restored = self.pgdata_content(node_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_3(self): + """ + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + + self.create_tblspace_in_node(node, 'tblspace2') + node.pgbench_init(scale=10, tablespace='tblspace2') + + node.stop() self.restore_node( backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) + options=[ + "-j", "4", + "--incremental-mode=checksum"]) - pgdata_restored = self.pgdata_content(node_1.data_dir) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_4(self): + """ + Check that system ID mismatch is detected, + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup of node1 with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + node.cleanup() + + # recreate node + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because destination directory has wrong system id.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + print(e.message) + self.assertIn( + 'WARNING: Backup catalog was initialized for system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is impossible', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_incr_restore_with_tablespace_5(self): + """ + More complicated case, we restore backup + with tablespace, which we remap into directory + with some old content, that belongs to an instance + with different system id. + """ + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + node1.slow_start() + + self.create_tblspace_in_node(node1, 'tblspace') + node1.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node1 with tblspace + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + pgdata = self.pgdata_content(node1.data_dir) + + node1.stop() + + # recreate node + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2'), + set_replication=True, + initdb_params=['--data-checksums']) + node2.slow_start() + + self.create_tblspace_in_node(node2, 'tblspace') + node2.pgbench_init(scale=10, tablespace='tblspace') + node2.stop() + + tblspc1_path = self.get_tblspace_path(node1, 'tblspace') + tblspc2_path = self.get_tblspace_path(node2, 'tblspace') + + out = self.restore_node( + backup_dir, 'node', node1, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum", + "-T{0}={1}".format(tblspc1_path, tblspc2_path)]) + + # check that tblspc1_path is empty + self.assertFalse( + os.listdir(tblspc1_path), + "Dir is not empty: '{0}'".format(tblspc1_path)) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_6(self): + """ + Empty pgdata, not empty tablespace + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because there is running postmaster " + "process in destination directory.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: PGDATA is empty, but tablespace destination is not', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + self.assertIn( + "INFO: Destination directory and tablespace directories are empty, " + "disable incremental restore", out) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_7(self): + """ + Restore backup without tablespace into + PGDATA with tablespace. + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') + node.stop() + +# try: +# self.restore_node( +# backup_dir, 'node', node, +# options=[ +# "-j", "4", +# "--incremental-mode=checksum"]) +# # we should die here because exception is what we expect to happen +# self.assertEqual( +# 1, 0, +# "Expecting Error because there is running postmaster " +# "process in destination directory.\n " +# "Output: {0} \n CMD: {1}".format( +# repr(self.output), self.cmd)) +# except ProbackupException as e: +# self.assertIn( +# 'ERROR: PGDATA is empty, but tablespace destination is not', +# e.message, +# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( +# repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--incremental-mode=checksum"]) + + print(out) + + pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself @@ -1943,24 +2316,44 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): "-T", "{0}={1}".format( node_tablespace, node1_tablespace)]) -# with open(os.path.join(node1_tablespace, "hello"), "w") as f: -# f.close() pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore into node2 + try: + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped tablespace contain old data .\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.restore_node( backup_dir, 'node', node2, options=[ - "-I", "checksum", + "-I", "checksum", "--force", "--db-exclude=db1", "--db-exclude=db5", "-T", "{0}={1}".format( node_tablespace, node2_tablespace)]) + pgdata2 = self.pgdata_content(node2.data_dir) self.compare_pgdata(pgdata1, pgdata2) - self.set_auto_conf(node2, {'port': node2.port}) node2.slow_start() diff --git a/tests/restore.py b/tests/restore.py index 35fc6dceb..561e0e46a 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -8,6 +8,7 @@ import hashlib import shutil import json +from shutil import copyfile from testgres import QueryException @@ -1021,6 +1022,141 @@ def test_restore_with_tablespace_mapping_2(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_restore_with_missing_or_corrupted_tablespace_map(self): + """restore backup with missing or corrupted tablespace_map""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=1, tablespace='tblspace') + + # Full backup + self.backup_node(backup_dir, 'node', node) + + # Change some data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Page backup + page_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2')) + node2.cleanup() + + olddir = self.get_tblspace_path(node, 'tblspace') + newdir = self.get_tblspace_path(node2, 'tblspace') + + # drop tablespace_map + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map') + + tablespace_map_tmp = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map_tmp') + + os.rename(tablespace_map, tablespace_map_tmp) + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + copyfile(tablespace_map_tmp, tablespace_map) + + with open(tablespace_map, "a") as f: + f.write("HELLO\n") + + print(tablespace_map) + + exit(1) + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # rename it back + os.rename(tablespace_map_tmp, tablespace_map) + + print(self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)])) + + pgdata_restored = self.pgdata_content(node2.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_archive_node_backup_stream_restore_to_recovery_time(self): """ diff --git a/tests/time_stamp.py b/tests/time_stamp.py index bf6e91e0a..4a4198c27 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -82,7 +82,7 @@ def test_server_date_style(self): self.del_test_dir(module_name, fname) def test_handling_of_TZ_env_variable(self): - """Issue #112""" + """Issue #284""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, fname), From 1d5015705342be21b363d609f01178a73ec7a1e0 Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 22 Jan 2021 21:48:21 +0300 Subject: [PATCH 1578/2107] calm down compiler warning --- src/pg_probackup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 30af998a4..dd2ac97ee 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -291,6 +291,7 @@ main(int argc, char *argv[]) break; #else elog(ERROR, "\"ssh\" command implemented only for Windows"); + break; #endif case AGENT_CMD: /* 'No forward compatibility' sanity: From 40aeb8be175e41aa9edc206faa4f13b0c11f12a8 Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 22 Jan 2021 22:43:06 +0300 Subject: [PATCH 1579/2107] fix merge conflict --- tests/expected/option_version.out | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 55d7c7f04..560b6b592 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1,5 +1 @@ -<<<<<<< HEAD pg_probackup 2.5.0 -======= -pg_probackup 2.4.8 ->>>>>>> master From 8ec995b4b49a2f0bf1f1d6feb70369bb92b4e50d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Jan 2021 02:43:31 +0300 Subject: [PATCH 1580/2107] tests: fixes for #280 --- tests/backup.py | 17 +++++++++-------- tests/incr_restore.py | 6 +++--- tests/restore.py | 8 ++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 3c0686f9b..8ca96609f 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -973,7 +973,7 @@ def test_basic_tablespace_handling(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -1028,9 +1028,10 @@ def test_basic_tablespace_handling(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1144,7 +1145,7 @@ def test_tablespace_handling_2(self): tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -1166,9 +1167,9 @@ def test_tablespace_handling_2(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 54c9a9ebf..add485b3c 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -460,7 +460,7 @@ def test_incr_restore_with_tablespace_4(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Incremental restore is impossible', + 'ERROR: Incremental restore is not allowed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -692,7 +692,7 @@ def test_basic_incr_restore_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Incremental restore is impossible', + 'ERROR: Incremental restore is not allowed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -717,7 +717,7 @@ def test_basic_incr_restore_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Incremental restore is impossible', + 'ERROR: Incremental restore is not allowed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/restore.py b/tests/restore.py index 561e0e46a..1da0b12b8 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -868,7 +868,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore tablespace destination is not empty:', + 'ERROR: Restore tablespace destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -891,7 +891,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore tablespace destination is not empty:', + 'ERROR: Restore tablespace destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1109,10 +1109,6 @@ def test_restore_with_missing_or_corrupted_tablespace_map(self): with open(tablespace_map, "a") as f: f.write("HELLO\n") - - print(tablespace_map) - - exit(1) try: self.restore_node( From c7f4865fe304b3999f4c71d6754f43f8e27e36e2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Jan 2021 02:45:12 +0300 Subject: [PATCH 1581/2107] DOC: improve wording for "--force" flag --- doc/pgprobackup.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index d6d69da4f..4f5e885d6 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3978,8 +3978,8 @@ pg_probackup restore -B backup_dir --instance incremental restore this flag allows to replace already existing PGDATA with different system ID. In case of tablespaces, - remapped via --tablespace-mapping option into not empty directories, - the old content of such directories will be deleted. + remapped via --tablespace-mapping option into non-empty directories, + the content of such directories will be deleted. From 88cb60fc4e4391d9b5444f89c20728a2827000a6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Jan 2021 03:15:50 +0300 Subject: [PATCH 1582/2107] DOC: minor fix --- doc/Readme.md | 2 +- doc/pgprobackup.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/Readme.md b/doc/Readme.md index b9c74769e..756c6aaa0 100644 --- a/doc/Readme.md +++ b/doc/Readme.md @@ -2,4 +2,4 @@ ``` xmllint --noout --valid probackup.xml xsltproc stylesheet.xsl probackup.xml >pg-probackup.html -``` \ No newline at end of file +``` diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 4f5e885d6..88ce237d5 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1886,7 +1886,7 @@ pg_probackup restore -B backup_dir --instance - Due to how recovery works in PostgreSQL < 12 it is advisable to + Due to how recovery works in PostgreSQL versions lower than 12 it is advisable to disable option, when running partial restore of PostgreSQL cluster of version less than . Otherwise recovery may fail. From 2e9c55f2976f6403cd73f9de786ba8c094f1b6aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Jan 2021 15:02:34 +0300 Subject: [PATCH 1583/2107] [Issue #240] Handle SIGPIPE. Also when remote agent encounters an error condition during exectuion of asynchronous tasks, the error message is saved in global variable and reported when agent status is checked --- src/archive.c | 25 +++- src/data.c | 9 +- src/pg_probackup.h | 2 +- src/restore.c | 5 + src/utils/file.c | 301 +++++++++++++++++++++++++++++++++++++---- src/utils/file.h | 13 +- src/utils/pgut.c | 1 + tests/compatibility.py | 8 +- tests/restore.py | 7 +- 9 files changed, 330 insertions(+), 41 deletions(-) diff --git a/src/archive.c b/src/archive.c index 4780db063..2d858a64c 100644 --- a/src/archive.c +++ b/src/archive.c @@ -421,6 +421,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d int partial_try_count = 0; int partial_file_size = 0; bool partial_is_stale = true; + /* remote agent error message */ + char *errmsg = NULL; /* from path */ join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); @@ -579,7 +581,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d from_fullpath, strerror(errno)); } - if (read_len > 0 && fio_write(out, buf, read_len) != read_len) + if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); elog(ERROR, "Cannot write to destination temp file \"%s\": %s", @@ -593,6 +595,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* close source file */ fclose(in); + /* Writing is asynchronous in case of push in remote mode, so check agent status */ + if (fio_check_error_fd(out, &errmsg)) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write to the remote file \"%s\": %s", + to_fullpath_part, errmsg); + } + /* close temp file */ if (fio_close(out) != 0) { @@ -652,6 +662,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, int partial_try_count = 0; int partial_file_size = 0; bool partial_is_stale = true; + /* remote agent errormsg */ + char *errmsg = NULL; /* from path */ join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); @@ -804,6 +816,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, } /* copy content */ + /* TODO: move to separate function */ for (;;) { size_t read_len = 0; @@ -831,7 +844,15 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* close source file */ fclose(in); - /* close temp file */ + /* Writing is asynchronous in case of push in remote mode, so check agent status */ + if (fio_check_error_fd_gz(out, &errmsg)) + { + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", + to_fullpath_gz_part, errmsg); + } + + /* close temp file, TODO: make it synchronous */ if (fio_gzclose(out) != 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); diff --git a/src/data.c b/src/data.c index 8f08f9c9e..a3baa3d3b 100644 --- a/src/data.c +++ b/src/data.c @@ -1084,13 +1084,14 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_out = write_pos; } - /* If page is compressed and restore is in remote mode, send compressed - * page to the remote side. + /* + * If page is compressed and restore is in remote mode, + * send compressed page to the remote side. */ if (is_compressed) { ssize_t rc; - rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg); + rc = fio_fwrite_async_compressed(out, page.data, compressed_size, file->compress_alg); if (!fio_is_remote_file(out) && rc != BLCKSZ) elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", @@ -1098,7 +1099,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } else { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) + if (fio_fwrite_async(out, page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "Cannot write block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1aab841c2..aec5c3e84 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1185,7 +1185,7 @@ extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_vers fio_location location); extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); -extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg); +extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg); /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) diff --git a/src/restore.c b/src/restore.c index ede049ecc..aa9bff9e2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1109,6 +1109,7 @@ restore_files(void *arg) bool already_exists = false; PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ + char *errmsg = NULL; /* remote agent error message */ pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ @@ -1262,6 +1263,10 @@ restore_files(void *arg) } done: + /* Writing is asynchronous in case of restore in remote mode, so check the agent status */ + if (fio_check_error_file(out, &errmsg)) + elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath, errmsg); + /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, diff --git a/src/utils/file.c b/src/utils/file.c index e23674548..15a7085ec 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -14,6 +14,7 @@ static __thread void* fio_stdin_buffer; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; static __thread int fio_stderr = 0; +static char *async_errormsg = NULL; fio_location MyLocation; @@ -420,6 +421,7 @@ int fio_open(char const* path, int mode, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + /* check results */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.arg != 0) @@ -583,7 +585,9 @@ int fio_ftruncate(FILE* f, off_t size) : ftruncate(fileno(f), size); } -/* Truncate file */ +/* Truncate file + * TODO: make it synchronous + */ int fio_truncate(int fd, off_t size) { if (fio_is_remote_fd(fd)) @@ -653,6 +657,7 @@ int fio_fseek(FILE* f, off_t offs) } /* Set position in file */ +/* TODO: make it synchronous or check async error */ int fio_seek(int fd, off_t offs) { if (fio_is_remote_fd(fd)) @@ -674,22 +679,105 @@ int fio_seek(int fd, off_t offs) } } +/* seek is asynchronous */ +static void +fio_seek_impl(int fd, off_t offs) +{ + int rc; + + /* Quick exit for tainted agent */ + if (async_errormsg) + return; + + rc = lseek(fd, offs, SEEK_SET); + + if (rc < 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } +} + /* Write data to stdio file */ size_t fio_fwrite(FILE* f, void const* buf, size_t size) +{ + if (fio_is_remote_file(f)) + return fio_write(fio_fileno(f), buf, size); + else + return fwrite(buf, 1, size, f); +} + +/* Write data to the file synchronously */ +ssize_t fio_write(int fd, void const* buf, size_t size) +{ + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_WRITE; + hdr.handle = fd & ~FIO_PIPE_MARKER; + hdr.size = size; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* set errno */ + if (hdr.arg > 0) + { + errno = hdr.arg; + return -1; + } + + return size; + } + else + { + return write(fd, buf, size); + } +} + +static void +fio_write_impl(int fd, void const* buf, size_t size, int out) +{ + int rc; + fio_header hdr; + + rc = write(fd, buf, size); + + hdr.arg = 0; + hdr.size = 0; + + if (rc < 0) + hdr.arg = errno; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + + return; +} + +size_t fio_fwrite_async(FILE* f, void const* buf, size_t size) { return fio_is_remote_file(f) - ? fio_write(fio_fileno(f), buf, size) + ? fio_write_async(fio_fileno(f), buf, size) : fwrite(buf, 1, size, f); } /* Write data to the file */ -ssize_t fio_write(int fd, void const* buf, size_t size) +/* TODO: support async report error */ +ssize_t fio_write_async(int fd, void const* buf, size_t size) { + if (size == 0) + return 0; + if (fio_is_remote_fd(fd)) { fio_header hdr; - hdr.cop = FIO_WRITE; + hdr.cop = FIO_WRITE_ASYNC; hdr.handle = fd & ~FIO_PIPE_MARKER; hdr.size = size; @@ -704,37 +792,57 @@ ssize_t fio_write(int fd, void const* buf, size_t size) } } +static void +fio_write_async_impl(int fd, void const* buf, size_t size, int out) +{ + int rc; + + /* Quick exit if agent is tainted */ + if (async_errormsg) + return; + + rc = write(fd, buf, size); + + if (rc <= 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } +} + int32 -fio_decompress(void* dst, void const* src, size_t size, int compress_alg) +fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg) { - const char *errormsg = NULL; + const char *internal_errormsg = NULL; int32 uncompressed_size = do_decompress(dst, BLCKSZ, src, size, - compress_alg, &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) + compress_alg, &internal_errormsg); + + if (uncompressed_size < 0 && internal_errormsg != NULL) { - elog(WARNING, "An error occured during decompressing block: %s", errormsg); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, "An error occured during decompressing block: %s", internal_errormsg); return -1; } if (uncompressed_size != BLCKSZ) { - elog(ERROR, "Page uncompressed to %d bytes != BLCKSZ", - uncompressed_size); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, "Page uncompressed to %d bytes != BLCKSZ", uncompressed_size); return -1; } return uncompressed_size; } /* Write data to the file */ -ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg) +ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) { if (fio_is_remote_file(f)) { fio_header hdr; - hdr.cop = FIO_WRITE_COMPRESSED; + hdr.cop = FIO_WRITE_COMPRESSED_ASYNC; hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; hdr.size = size; hdr.arg = compress_alg; @@ -747,20 +855,124 @@ ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compres else { char uncompressed_buf[BLCKSZ]; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); + char *errormsg = NULL; + int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &errormsg); + + if (uncompressed_size < 0) + elog(ERROR, "%s", errormsg); - return (uncompressed_size < 0) - ? uncompressed_size - : fwrite(uncompressed_buf, 1, uncompressed_size, f); + return fwrite(uncompressed_buf, 1, uncompressed_size, f); } } -static ssize_t +static void fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) { + int rc; + int32 uncompressed_size; char uncompressed_buf[BLCKSZ]; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); - return fio_write_all(fd, uncompressed_buf, uncompressed_size); + + /* If the previous command already have failed, + * then there is no point in bashing a head against the wall + */ + if (async_errormsg) + return; + + /* decompress chunk */ + uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &async_errormsg); + + if (uncompressed_size < 0) + return; + + rc = write(fd, uncompressed_buf, uncompressed_size); + + if (rc <= 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } +} + +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_file(FILE* f, char **errmsg) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + + return 0; +} + +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_fd(int fd, char **errmsg) +{ + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + return 0; +} + +static void +fio_get_async_error_impl(int out) +{ + fio_header hdr; + hdr.cop = FIO_GET_ASYNC_ERROR; + + /* send error message */ + if (async_errormsg) + { + hdr.size = strlen(async_errormsg) + 1; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* send message itself */ + IO_CHECK(fio_write_all(out, async_errormsg, hdr.size), hdr.size); + + //TODO: should we reset the tainted state ? +// pg_free(async_errormsg); +// async_errormsg = NULL; + } + else + { + hdr.size = 0; + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } } /* Read data from stdio file */ @@ -1082,6 +1294,11 @@ int fio_chmod(char const* path, int mode, fio_location location) #define ZLIB_BUFFER_SIZE (64*1024) #define MAX_WBITS 15 /* 32K LZ77 window */ #define DEF_MEM_LEVEL 8 +/* last bit used to differenciate remote gzFile from local gzFile + * TODO: this is insane, we should create our own scructure for this, + * not flip some bits in someone's else and hope that it will not break + * between zlib versions. + */ #define FIO_GZ_REMOTE_MARKER 1 typedef struct fioGZFile @@ -1094,6 +1311,32 @@ typedef struct fioGZFile Bytef buf[ZLIB_BUFFER_SIZE]; } fioGZFile; +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_fd_gz(gzFile f, char **errmsg) +{ + if (f && ((size_t)f & FIO_GZ_REMOTE_MARKER)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + return 0; +} + /* On error returns NULL and errno should be checked */ gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location) @@ -1266,7 +1509,7 @@ fio_gzwrite(gzFile f, void const* buf, unsigned size) break; } } - rc = fio_write(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + rc = fio_write_async(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); if (rc >= 0) { gz->strm.next_out += rc; @@ -2572,6 +2815,7 @@ void fio_communicate(int in, int out) } IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); } + errno = 0; /* reset errno */ switch (hdr.cop) { case FIO_LOAD: /* Send file content */ fio_load_file(out, buf); @@ -2610,10 +2854,14 @@ void fio_communicate(int in, int out) SYS_CHECK(close(fd[hdr.handle])); break; case FIO_WRITE: /* Write to the current position in file */ - IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); +// IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); + fio_write_impl(fd[hdr.handle], buf, hdr.size, out); break; - case FIO_WRITE_COMPRESSED: /* Write to the current position in file */ - IO_CHECK(fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg), BLCKSZ); + case FIO_WRITE_ASYNC: /* Write to the current position in file */ + fio_write_async_impl(fd[hdr.handle], buf, hdr.size, out); + break; + case FIO_WRITE_COMPRESSED_ASYNC: /* Write to the current position in file */ + fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg); break; case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { @@ -2670,7 +2918,7 @@ void fio_communicate(int in, int out) SYS_CHECK(chmod(buf, hdr.arg)); break; case FIO_SEEK: /* Set current position in file */ - SYS_CHECK(lseek(fd[hdr.handle], hdr.arg, SEEK_SET)); + fio_seek_impl(fd[hdr.handle], hdr.arg); break; case FIO_TRUNCATE: /* Truncate file */ SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); @@ -2729,6 +2977,9 @@ void fio_communicate(int in, int out) hdr.cop = FIO_DISCONNECTED; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; + case FIO_GET_ASYNC_ERROR: + fio_get_async_error_impl(out); + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index 255512f1f..1eafe543d 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -36,7 +36,7 @@ typedef enum FIO_READDIR, FIO_CLOSEDIR, FIO_PAGE, - FIO_WRITE_COMPRESSED, + FIO_WRITE_COMPRESSED_ASYNC, FIO_GET_CRC32, /* used for incremental restore */ FIO_GET_CHECKSUM_MAP, @@ -53,7 +53,9 @@ typedef enum FIO_DISCONNECT, FIO_DISCONNECTED, FIO_LIST_DIR, - FIO_CHECK_POSTMASTER + FIO_CHECK_POSTMASTER, + FIO_GET_ASYNC_ERROR, + FIO_WRITE_ASYNC } fio_operations; typedef enum @@ -91,7 +93,11 @@ extern void fio_communicate(int in, int out); extern int fio_get_agent_version(void); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg); +extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); +extern size_t fio_fwrite_async(FILE* f, void const* buf, size_t size); +extern int fio_check_error_file(FILE* f, char **errmsg); +extern int fio_check_error_fd(int fd, char **errmsg); +extern int fio_check_error_fd_gz(gzFile f, char **errmsg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); @@ -104,6 +110,7 @@ extern void fio_error(int rc, int size, char const* file, int line); extern int fio_open(char const* name, int mode, fio_location location); extern ssize_t fio_write(int fd, void const* buf, size_t size); +extern ssize_t fio_write_async(int fd, void const* buf, size_t size); extern ssize_t fio_read(int fd, void* buf, size_t size); extern int fio_flush(int fd); extern int fio_seek(int fd, off_t offs); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index bc35a24ff..ef3472e26 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -967,6 +967,7 @@ init_cancel_handler(void) oldhandler = pqsignal(SIGINT, handle_interrupt); pqsignal(SIGQUIT, handle_interrupt); pqsignal(SIGTERM, handle_interrupt); + pqsignal(SIGPIPE, handle_interrupt); } #else /* WIN32 */ diff --git a/tests/compatibility.py b/tests/compatibility.py index 547933344..18f601506 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -944,6 +944,10 @@ def test_backward_compatibility_merge_4(self): Start merge between minor version, crash and retry it. old binary version =< 2.4.0 """ + if self.version_to_num(self.old_probackup_version) > self.version_to_num('2.4.0'): + self.assertTrue( + False, 'You need pg_probackup old_binary =< 2.4.0 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1019,9 +1023,9 @@ def test_backward_compatibility_merge_5(self): merge them with new binary. old binary version >= STORAGE_FORMAT_VERSION (2.4.4) """ - if self.version_to_num(self.old_probackup_version) < self.version_to_num('2.4.4'): - return unittest.skip('OLD pg_probackup binary must be == 2.4.4 for this test') + self.assertTrue( + False, 'OLD pg_probackup binary must be >= 2.4.4 for this test') self.assertNotEqual( self.version_to_num(self.old_probackup_version), diff --git a/tests/restore.py b/tests/restore.py index 1da0b12b8..9db885f09 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3649,13 +3649,12 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): pg_probackup version must be 12 or greater """ - if self.old_probackup_version: - if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): - return unittest.skip('You need pg_probackup < 2.4.5 for this test') - if self.pg_config_version < self.version_to_num('12.0'): return unittest.skip('You need PostgreSQL >= 12 for this test') + if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): + self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( From 302455a4c60ef02b09a5961d1e509a22caeb01a6 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 26 Jan 2021 15:16:41 +0300 Subject: [PATCH 1584/2107] Remove outdated Documentation.md file. Documentation now lives in doc/ subdirectory. --- Documentation.md | 2096 ---------------------------------------------- 1 file changed, 2096 deletions(-) delete mode 100644 Documentation.md diff --git a/Documentation.md b/Documentation.md deleted file mode 100644 index 943ea0cfd..000000000 --- a/Documentation.md +++ /dev/null @@ -1,2096 +0,0 @@ -# pg_probackup - -pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. - -Current version - 2.2.5 - -1. [Synopsis](#synopsis) -2. [Versioning](#versioning) -3. [Overview](#overview) - * [Limitations](#limitations) - -4. [Installation and Setup](#installation-and-setup) - * [Initializing the Backup Catalog](#initializing-the-backup-catalog) - * [Adding a New Backup Instance](#adding-a-new-backup-instance) - * [Configuring the Database Cluster](#configuring-the-database-cluster) - * [Setting up STREAM Backups](#setting-up-stream-backups) - * [Setting up Continuous WAL Archiving](#setting-up-continuous-wal-archiving) - * [Setting up Backup from Standby](#setting-up-backup-from-standby) - * [Setting up Cluster Verification](#setting-up-cluster-verification) - * [Setting up Partial Restore](#setting-up-partial-restore) - * [Configuring the Remote Mode](#configuring-the-remote-mode) - * [Setting up PTRACK Backups](#setting-up-ptrack-backups) - -5. [Usage](#usage) - * [Creating a Backup](#creating-a-backup) - * [ARCHIVE WAL mode](#archive-mode) - * [STREAM WAL mode](#stream-mode) - * [Page validation](#page-validation) - * [External directories](#external-directories) - * [Verifying a Cluster](#verifying-a-cluster) - * [Validating a Backup](#validating-a-backup) - * [Restoring a Cluster](#restoring-a-cluster) - * [Partial Restore](#partial-restore) - * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) - * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - * [Configuring pg_probackup](#configuring-pg_probackup) - * [Managing the Backup Catalog](#managing-the-backup-catalog) - * [Viewing Backup Information](#viewing-backup-information) - * [Viewing WAL Archive Information](#viewing-wal-archive-information) - * [Configuring Retention Policy](#configuring-retention-policy) - * [Backup Retention Policy](#backup-retention-policy) - * [Backup Pinning](#backup-pinning) - * [WAL Archive Retention Policy](#wal-archive-retention-policy) - * [Merging Backups](#merging-backups) - * [Deleting Backups](#deleting-backups) - -6. [Command-Line Reference](#command-line-reference) - * [Commands](#commands) - * [version](#version) - * [help](#help) - * [init](#init) - * [add-instance](#add-instance) - * [del-instance](#del-instance) - * [set-config](#set-config) - * [set-backup](#set-backup) - * [show-config](#show-config) - * [show](#show) - * [backup](#backup) - * [restore](#restore) - * [checkdb](#checkdb) - * [validate](#validate) - * [merge](#merge) - * [delete](#delete) - * [archive-push](#archive-push) - * [archive-get](#archive-get) - * [Options](#options) - * [Common Options](#common-options) - * [Recovery Target Options](#recovery-target-options) - * [Retention Options](#retention-options) - * [Pinning Options](#pinning-options) - * [Logging Options](#logging-options) - * [Connection Options](#connection-options) - * [Compression Options](#compression-options) - * [Archiving Options](#archiving-options) - * [Remote Mode Options](#remote-mode-options) - * [Remote WAL Archive Options](#remote-wal-archive-options) - * [Partial Restore Options](#partial-restore-options) - * [Replica Options](#replica-options) - -7. [Howto](#howto) - * [Minimal setup](#minimal-setup) -8. [Authors](#authors) -9. [Credits](#credits) - - -## Synopsis - -`pg_probackup version` - -`pg_probackup help [command]` - -`pg_probackup init -B backup_dir` - -`pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name` - -`pg_probackup del-instance -B backup_dir --instance instance_name` - -`pg_probackup set-config -B backup_dir --instance instance_name [option...]` - -`pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id [option...]` - -`pg_probackup show-config -B backup_dir --instance instance_name [--format=format]` - -`pg_probackup show -B backup_dir [option...]` - -`pg_probackup backup -B backup_dir --instance instance_name -b backup_mode [option...]` - -`pg_probackup restore -B backup_dir --instance instance_name [option...]` - -`pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` - -`pg_probackup validate -B backup_dir [option...]` - -`pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` - -`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired } [option...]` - -`pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` - -`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` - - -## Versioning - -pg_probackup is following the [semantic](https://semver.org/) versioning. - -## Overview - -As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: - -- Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow -- Validation: automatic data consistency checks and on-demand backup validation without actual data recovery -- Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` -- Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting 'time to live' for backups -- Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads -- Compression: storing backup data in a compressed state to save disk space -- Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) -- Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it -- Backup from replica: avoid extra load on the master server by taking backups from a standby -- External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats -- Archive Catalog: get list of all WAL timelines and corresponding meta information in `plain` or `json` formats -- Partial Restore: restore only the specified databases or skip the specified databases. - -To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. - -Using pg_probackup, you can take full or incremental [backups](#creating-a-backup): - -- FULL backups contain all the data files required to restore the database cluster. -- Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: - - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. - -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen backup mode (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery modes`: - -- [ARCHIVE](#archive-mode). Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery mode. -- [STREAM](#stream-mode). Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. Because of that backups of this WAL mode are called `autonomous` or `standalone`. - -### Limitations - -pg_probackup currently has the following limitations: - -- Only PostgreSQL of versions 9.5 and newer are supported. -- Currently remode mode of operations is not supported on Windows systems. -- On Unix systems backup of PostgreSQL verions =< 10 is possible only by the same OS user PostgreSQL server is running by. For example, if PostgreSQL server is running by user *postgres*, then backup must be run by user *postgres*. If backup is running in [remote mode](#using-pg_probackup-in-the-remote-mode) using `ssh`, then this limitation apply differently: value for `--remote-user` option should be *postgres*. -- During backup of PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` will be executed only if backup role is superuser. Because of that backup of a cluster with low amount of WAL traffic with non-superuser role may take more time than backup of the same cluster with superuser role. -- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. Also depending on cluster configuration PostgreSQL itself may apply additional restrictions such as CPU architecture platform and libc/libicu versions. -- Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. - -## Installation and Setup - -Once you have pg_probackup installed, complete the following setup: - -- Initialize the backup catalog. -- Add a new backup instance to the backup catalog. -- Configure the database cluster to enable pg_probackup backups. -- Optionally, configure SSH for running pg_probackup operations in remote mode. - -### Initializing the Backup Catalog - -pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. - -To initialize the backup catalog, run the following command: - - pg_probackup init -B backup_dir - -Where *backup_dir* is the path to backup catalog. If the *backup_dir* already exists, it must be empty. Otherwise, pg_probackup returns an error. - -The user launching pg_probackup must have full access to *backup_dir* directory. - -pg_probackup creates the backup_dir backup catalog, with the following subdirectories: - -- wal/ — directory for WAL files. -- backups/ — directory for backup files. - -Once the backup catalog is initialized, you can add a new backup instance. - -### Adding a New Backup Instance - -pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. - -To add a new backup instance, run the following command: - - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] - -Where: - -- *data_dir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. -- *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. - -pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls pg_probackup settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to pg_probackup.conf. - -For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). - -The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. - ->NOTE: For PostgreSQL >= 11 it is recommended to use [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature, so backup can be done by any OS user in the same group as the cluster owner. In this case the user should have read permissions on the cluster directory. - -### Configuring the Database Cluster - -Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. - -To perform [backup](#backup), the following permissions for role *backup* are required only in database **used for connection** to PostgreSQL server: - -For PostgreSQL 9.5: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -COMMIT; -``` - -For PostgreSQL 9.6: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; -``` - -For PostgreSQL >= 10: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; -``` - -In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow connection to database cluster on behalf of the *backup* role. - -Since pg_probackup needs to read cluster files directly, pg_probackup must be started by (in case of remote backup - connected to) OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. - -Depending on whether you are plan to take [autonomous](#stream-mode) and/or [archive](#archive-mode) backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server, run pg_probackup in remote mode or create PTRACK backups, additional setup is required. - -For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#setting-up-backup-from-standby), [Configuring the Remote Mode](#configuring-the-remote-mode), [Setting up Partial Restore](#setting-up-partial-restore) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). - -### Setting up STREAM Backups - -To set up the cluster for [STREAM](#stream-mode) backups, complete the following steps: - -- Grant the REPLICATION privilege to the backup role: - - ALTER ROLE backup WITH REPLICATION; - -- In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow replication on behalf of the *backup* role. -- Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. -- Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. - -If you are planning to take PAGE backups in STREAM mode or perform PITR with STREAM backups, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). - -Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups with [STREAM](#stream-mode) WAL mode. - -### Setting up continuous WAL archiving - -Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time-pitr-recovery) and making backups with [ARCHIVE](#archive-mode) WAL delivery mode require [continuous WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continuous archiving in the cluster, complete the following steps: - -- Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. -- If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on` or `always`. To perform archiving on standby, set this parameter to `always`. -- Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' - -Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote host. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). - -Once these steps are complete, you can start making backups with [ARCHIVE](#archive-mode) WAL-mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). - -Current state of WAL Archive can be obtained via [show](#show) command. For details, see the sections [Viewing WAL Archive information](#viewing-wal-archive-information). - -If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to set the value of this setting slightly lower than pg_probackup parameter `--archive-timeout` (default 5 min), so there should be enough time for rotated segment to be streamed to replica and send to archive before backup is aborted because of `--archive-timeout`. - ->NOTE: using pg_probackup command [archive-push](#archive-push) for continuous archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be `gzip`, and '.gz' suffix in filename is mandatory. - ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. - -### Setting up Backup from Standby - -For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: - -- On the standby server, set the parameter [hot_standby](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-HOT-STANDBY) to `on`. -- On the master server, set the parameter [full_page_writes](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-FULL-PAGE-WRITES) to `on`. -- To perform autonomous backups on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) -- To perform archive backups on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) - -Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups with appropriate WAL delivery mode: ARCHIVE or STREAM, from the standby server. - -Backup from the standby server has the following limitations: - -- If the standby is promoted to the master during backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like pg_compresslog as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. - -### Setting up Cluster Verification - -Logical verification of database cluster requires the following additional setup. Role *backup* is used as an example: - -- Install extension [amcheck](https://www.postgresql.org/docs/current/amcheck.html) or [amcheck_next](https://github.com/petergeoghegan/amcheck) **in every database** of the cluster: - - CREATE EXTENSION amcheck; - -- To perform logical verification the following permissions are required **in every database** of the cluster: - -``` -GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; -``` - -### Setting up Partial Restore - -If you are plalling to use partial restore, complete the following additional step: - -- Grant the read-only acces to 'pg_catalog.pg_database' to the *backup* role only in database **used for connection** to PostgreSQL server: - - GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - -### Configuring the Remote Mode - -pg_probackup supports the remote mode that allows to perform backup, restore and WAL archiving operations remotely. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to backup and/or to restore is located on a remote system. Currently the only supported remote protocol is SSH. - -#### Setup SSH - -If you are going to use pg_probackup in remote mode via ssh, complete the following steps: - -- Install pg_probackup on both systems: `backup_host` and `db_host`. -- For communication between the hosts setup the passwordless SSH connection between *backup* user on `backup_host` and *postgres* user on `db_host`: - - [backup@backup_host] ssh-copy-id postgres@db_host - -- If you are planning to rely on [continuous WAL archiving](#setting-up-continuous-wal-archiving), then setup passwordless SSH connection between *postgres* user on `db_host` and *backup* user on `backup_host`: - - [postgres@db_host] ssh-copy-id backup@backup_host - -Where: - -- *backup_host* is the system with *backup catalog*. -- *db_host* is the system with PostgreSQL cluster. -- *backup* is the OS user on *backup_host* used to run pg_probackup. -- *postgres* is the OS user on *db_host* used to run PostgreSQL cluster. Note, that for PostgreSQL versions >= 11, a more secure approach can used thanks to [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature. - -pg_probackup in remote mode via `ssh` works as follows: - -- only the following commands can be launched in remote mode: [add-instance](#add-instance), [backup](#backup), [restore](#restore), [archive-push](#archive-push), [archive-get](#archive-get). -- when started in remote mode the main pg_probackup process on local system connects via ssh to remote system and launches there number of agent proccesses equal to specified value of option `-j/--threads`. -- the main pg_probackup process use remote agents to access remote files and transfer data between local and remote systems. -- remote agents are smart and capable of handling some logic on their own to minimize the network traffic and number of round-trips between hosts. -- usually the main proccess is started on *backup_host* and connects to *db_host*, but in case of `archive-push` and `archive-get` commands the main process is started on *db_host* and connects to *backup_host*. -- after completition of data transfer the remote agents are terminated and ssh connections are closed. -- if an error condition is encountered by a remote agent, then all agents are terminated and error details are reported by the main pg_probackup process, which exits with error. -- compression is always done on *db_host*. -- decompression is always done on *backup_host*. - ->NOTE: You can improse [additional restrictions](https://man.openbsd.org/OpenBSD-current/man8/sshd.8#AUTHORIZED_KEYS_FILE_FORMAT) on ssh settings to protect the system in the event of account compromise. - -### Setting up PTRACK Backups - -Backup mode PTACK can be used only on Postgrespro Standart and Postgrespro Enterprise installations or patched vanilla PostgreSQL. Links to ptrack patches can be found [here](https://github.com/postgrespro/pg_probackup#ptrack-support). - -If you are going to use PTRACK backups, complete the following additional steps: - -- Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute `ptrack` functions to the *backup* role **in every database** of the cluster: - - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; - -- The *backup* role must have access to all the databases of the cluster. - -## Usage - -### Creating a Backup - -To create a backup, run the following command: - - pg_probackup backup -B backup_dir --instance instance_name -b backup_mode - -Where *backup_mode* can take one of the following values: - -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have generated since the previous full or incremental backup was taken. Only changed blocks are readed from data files. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. - -When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `the backup chain`. You must create at least one full backup before taking incremental ones. - -#### ARCHIVE mode - -ARCHIVE is the default WAL delivery mode. - -For example, to make a FULL backup in ARCHIVE mode, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL - -Unlike backup in STREAM mode, ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - -During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. - -#### STREAM mode - -STREAM is the optional WAL delivery mode. - -For example, to make a FULL backup in STREAM mode, add the `--stream` flag to the command from the previous example: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot - -The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. - -Unlike backup in ARCHIVE mode, STREAM backup include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - -During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN to '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. - -Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: - -- STREAM backups can be restored on the server that has no file access to WAL archive. -- STREAM backups enable you to restore the cluster state at the point in time for which WAL files in archive are no longer available. -- Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. - -#### Page validation - -If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. -Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. - -Page is considered corrupted if checksumm comparison failed more than 100 times, in this case backup is aborted. - -Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". - -#### External directories - -To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons, on Windows system paths must be separated by semicolon instead. - -For example, to include `'/etc/dir1/'` and `'/etc/dir2/'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 - -For example, to include `'C:\dir1\'` and `'C:\dir2\'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory on Windows system, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 - -pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. - -To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. - -### Verifying a Cluster - -To verify that PostgreSQL database cluster is free of corruption, run the following command: - - pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] - -This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: - -- `checkdb` is read-only -- if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated -- `checkdb` do not strictly require *the backup catalog*, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). - -If *backup_dir* and *instance_name* are omitted, then [connection options](#connection-options) and *data_dir* must be provided via environment variables or command-line options. - -Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. -Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. - -If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` flag when running [checkdb](#checkdb) command: - - pg_probackup checkdb -D data_dir --amcheck - -Physical verification can be skipped if `--skip-block-validation` flag is used. For logical only verification *backup_dir* and *data_dir* are optional, only [connection options](#connection-options) are mandatory: - - pg_probackup checkdb --amcheck --skip-block-validation {connection_options} - -Logical verification can be done more thoroughly with flag `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. - -### Validating a Backup - -pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `the backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruption. - -If you would like to skip backup validation, you can specify the `--no-validate` flag when running [backup](#backup) and [restore](#restore) commands. - -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact [recovery target options](#recovery-target-options) you are going to use for recovery. - -For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: - - pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 - -If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. - -If you specify *backup_id* via `-i/--backup-id` option, then only backup copy with specified backup ID will be validated. If *backup_id* is specified with [recovery target options](#recovery-target-options) then validate will check whether it is possible to restore the specified backup to the specified `recovery target`. - -For example, to check that you can restore the database cluster from a backup copy with *backup_id* up to the specified timestamp, run this command: - - pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' - -If *backup_id* belong to incremental backup, then all its parents starting from FULL backup will be validated. - -If you omit all the parameters, all backups are validated. - -### Restoring a Cluster - -To restore the database cluster from a backup, run the restore command with at least the following options: - - pg_probackup restore -B backup_dir --instance instance_name -i backup_id - -Where: - -- *backup_dir* is the backup catalog that stores all backup files and meta information. -- *instance_name* is the backup instance for the cluster to be restored. -- *backup_id* specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. - -If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. - -When using the `--tablespace-mapping/-T` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: - - pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir - -Once the restore command is complete, start the database service. - -If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. - -To restore cluster on remote host see the section [Using pg_probackup in the Remote Mode](#using-pg-probackup-in-the-remote-mode). - ->NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` flag to skip validation and speed up the recovery. - -#### Partial Restore - -If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can restore or exclude from restore the arbitrary number of specific databases using [partial restore options](#partial-restore-options) with the [restore](#restore) commands. - -To restore only one or more databases, run the restore command with the following options: - - pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name - -The option `--db-include` can be specified multiple times. For example, to restore only databases `db1` and `db2`, run the following command: - - pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 - -To exclude one or more specific databases from restore, run the following options: - - pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name - -The option `--db-exclude` can be specified multiple times. For example, to exclude the databases `db1` and `db2` from restore, run the following command: - - pg_probackup restore -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 - -Partial restore rely on lax behaviour of PostgreSQL recovery process toward truncated files. Files of excluded databases restored as null sized files, allowing recovery to work properly. After successfull starting of PostgreSQL cluster, you must drop excluded databases using `DROP DATABASE` command. - ->NOTE: The databases `template0` and `template1` are always restored. - -### Performing Point-in-Time (PITR) Recovery - -If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. - -If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore *backup_id* to the specified recovery target. - -- To restore the cluster state at the exact time, specify the `--recovery-target-time` option, in the timestamp format. For example: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' - -- To restore the cluster state up to a specific transaction ID, use the `--recovery-target-xid` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 - -- To restore the cluster state up to a specific LSN, use `--recovery-target-lsn` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 - -- To restore the cluster state up to a specific named restore point, use `--recovery-target-name` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' - -- To restore the backup to the latest state available in archive, use `--recovery-target` option with `latest` value: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' - -- To restore the cluster to the earliest point of consistency, use `--recovery-target` option with `immediate` value: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' - -### Using pg_probackup in the Remote Mode - -pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. - -Do note that pg_probackup rely on passwordless SSH connection for communication between the hosts. - -The typical workflow is as follows: - - - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. - -- If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery mode, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. - -- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. - -For example, to create archive full backup using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - -For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - -Restoring of ARCHIVE backup or performing PITR in remote mode require additional information: destination address, port and username for establishing ssh connection **from** a host with database **to** a host with backup catalog. This information will be used by `restore_command` to copy via ssh WAL segments from archive to PostgreSQL 'pg_wal' directory. - -To solve this problem you can use [Remote Wal Archive Options](#remote-wal-archive-options). - -For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302` and user `backup` on backup catalog host with address `192.168.0.3` via port `2303`, run: - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup - -Provided arguments will be used to construct 'restore_command' in recovery.conf: -``` -# recovery.conf generated by pg_probackup 2.1.5 -restore_command = 'pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' -``` - -Alternatively you can use `--restore-command` option to provide an entire 'restore_command': - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' - ->NOTE: The remote backup mode is currently unavailable for Windows systems. - -### Running pg_probackup on Parallel Threads - -[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk and network bandwidth). - -Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 - ->NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. - -### Configuring pg_probackup - -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. - -For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. - ->NOTE: It is **not recommended** to edit pg_probackup.conf manually. - -Initially, pg_probackup.conf contains the following settings: - -- PGDATA — the path to the data directory of the cluster to back up. -- system-identifier — the unique identifier of the PostgreSQL instance. - -Additionally, you can define [remote](#remote-mode-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: - - pg_probackup set-config -B backup_dir --instance instance_name - [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] - -To view the current settings, run the following command: - - pg_probackup show-config -B backup_dir --instance instance_name - -You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. - -### Specifying Connection Settings - -If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. - -If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. - -### Managing the Backup Catalog - -With pg_probackup, you can manage backups from the command line: - -- [View backup information](#viewing-backup-information) -- [View WAL Archive Information](#viewing-wal-archive-information) -- [Validate backups](#validating-a-backup) -- [Merge backups](#merging-backups) -- [Delete backups](#deleting-backups) - -#### Viewing Backup Information - -To view the list of existing backups for every instance, run the command: - - pg_probackup show -B backup_dir - -pg_probackup displays the list of all the available backups. For example: - -``` -BACKUP INSTANCE 'node' -====================================================================================================================================== - Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -====================================================================================================================================== - node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK -``` - -For each backup, the following information is provided: - -- Instance — the instance name. -- Version — PostgreSQL major version. -- ID — the backup identifier. -- Recovery time — the earliest moment for which you can restore the state of the database cluster. -- Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. -- WAL Mode — the WAL delivery mode. Possible values: STREAM and ARCHIVE. -- TLI — timeline identifiers of current backup and its parent. -- Time — the time it took to perform the backup. -- Data — the size of the data files in this backup. This value does not include the size of WAL files. In case of STREAM backup the total size of backup can be calculated as 'Data' + 'WAL'. -- WAL — the uncompressed size of WAL files required to apply by PostgreSQL recovery process to reach consistency. -- Zratio — compression ratio calculated as 'uncompressed-bytes' / 'data-bytes'. -- Start LSN — WAL log sequence number corresponding to the start of the backup process. REDO point for PostgreSQL recovery process to start from. -- Stop LSN — WAL log sequence number corresponding to the end of the backup process. Consistency point for PostgreSQL recovery process. -- Status — backup status. Possible values: - - - OK — the backup is complete and valid. - - DONE — the backup is complete, but was not validated. - - RUNNING — the backup is in progress. - - MERGING — the backup is being merged. - - DELETING — the backup files are being deleted. - - CORRUPT — some of the backup files are corrupted. - - ERROR — the backup was aborted because of an unexpected error. - - ORPHAN — the backup is invalid because one of its parent backups is corrupt or missing. - -You can restore the cluster from the backup only if the backup status is OK or DONE. - -To get more detailed information about the backup, run the show with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name -i backup_id - -The sample output is as follows: - -``` -#Configuration -backup-mode = FULL -stream = false -compress-alg = zlib -compress-level = 1 -from-replica = false - -#Compatibility -block-size = 8192 -wal-block-size = 8192 -checksum-version = 1 -program-version = 2.1.3 -server-version = 10 - -#Result backup info -timelineid = 1 -start-lsn = 0/04000028 -stop-lsn = 0/040000f8 -start-time = '2017-05-16 12:57:29' -end-time = '2017-05-16 12:57:31' -recovery-xid = 597 -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-05-16 12:57:31' -data-bytes = 22288792 -wal-bytes = 16777216 -uncompressed-bytes = 39961833 -pgdata-bytes = 39859393 -status = OK -parent-backup-id = 'PT8XFX' -primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' -``` - -Detailed output has additional attributes: -- compress-alg — compression algorithm used during backup. Possible values: 'zlib', 'pglz', 'none'. -- compress-level — compression level used during backup. -- from-replica — the fact that backup was taken from standby server. Possible values: '1', '0'. -- block-size — (block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. -- wal-block-size — (wal_block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. -- checksum-version — the fact that PostgreSQL cluster, from which backup is taken, has enabled [data block checksumms](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS). Possible values: '1', '0'. -- program-version — full version of pg_probackup binary used to create backup. -- start-time — the backup starting time. -- end-time — the backup ending time. -- expire-time — if the backup was pinned, then until this point in time the backup cannot be removed by retention purge. -- uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used. -- pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'. -- recovery-xid — current transaction id at the moment of backup ending. -- parent-backup-id — backup ID of parent backup. Available only for incremental backups. -- primary_conninfo — libpq conninfo used for connection to PostgreSQL cluster during backup. The password is not included. - -To get more detailed information about the backup in json format, run the show with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id - -The sample output is as follows: - -``` -[ - { - "instance": "node", - "backups": [ - { - "id": "PT91HZ", - "parent-backup-id": "PT8XFX", - "backup-mode": "DELTA", - "wal": "ARCHIVE", - "compress-alg": "zlib", - "compress-level": 1, - "from-replica": false, - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.3", - "server-version": "10", - "current-tli": 16, - "parent-tli": 2, - "start-lsn": "0/8000028", - "stop-lsn": "0/8000160", - "start-time": "2019-06-17 18:25:11+03", - "end-time": "2019-06-17 18:25:16+03", - "recovery-xid": 0, - "recovery-time": "2019-06-17 18:25:15+03", - "data-bytes": 106733, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } -] -``` - -#### Viewing WAL Archive Information - -To view the information about WAL archive for every instance, run the command: - - pg_probackup show -B backup_dir [--instance instance_name] --archive - -pg_probackup displays the list of all the available WAL files grouped by timelines. For example: - -``` -ARCHIVE INSTANCE 'node' -=================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=================================================================================================================== - 5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK - 4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK - 3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK - 2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED - 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK - -``` - -For each backup, the following information is provided: - -- TLI — timeline identifier. -- Parent TLI — identifier of timeline TLI branched off. -- Switchpoint — LSN of the moment when the timeline branched off from "Parent TLI". -- Min Segno — number of the first existing WAL segment belonging to the timeline. -- Max Segno — number of the last existing WAL segment belonging to the timeline. -- N segments — number of WAL segments belonging to the timeline. -- Size — the size files take on disk. -- Zratio — compression ratio calculated as 'N segments' * wal_seg_size / 'Size'. -- N backups — number of backups belonging to the timeline. To get the details about backups, use json format. -- Status — archive status for this exact timeline. Possible values: - - OK — all WAL segments between Min and Max are present. - - DEGRADED — some WAL segments between Min and Max are lost. To get details about lost files, use json format. - -To get more detailed information about the WAL archive in json format, run the command: - - pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json - -The sample output is as follows: - -``` -[ - { - "instance": "replica", - "timelines": [ - { - "tli": 5, - "parent-tli": 1, - "switchpoint": "0/B000000", - "min-segno": "000000000000000B", - "max-segno": "000000000000000C", - "n-segments": 2, - "size": 685320, - "zratio": 48.00, - "closest-backup-id": "PXS92O", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 4, - "parent-tli": 3, - "switchpoint": "0/18000000", - "min-segno": "0000000000000018", - "max-segno": "000000000000001A", - "n-segments": 3, - "size": 648625, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 3, - "parent-tli": 2, - "switchpoint": "0/15000000", - "min-segno": "0000000000000015", - "max-segno": "0000000000000017", - "n-segments": 3, - "size": 648911, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 2, - "parent-tli": 1, - "switchpoint": "0/B000108", - "min-segno": "000000000000000B", - "max-segno": "0000000000000015", - "n-segments": 5, - "size": 892173, - "zratio": 94.00, - "closest-backup-id": "PXS92O", - "status": "DEGRADED", - "lost-segments": [ - { - "begin-segno": "000000000000000D", - "end-segno": "000000000000000E" - }, - { - "begin-segno": "0000000000000010", - "end-segno": "0000000000000012" - } - ], - "backups": [ - { - "id": "PXS9CE", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 2, - "parent-tli": 0, - "start-lsn": "0/C000028", - "stop-lsn": "0/C000160", - "start-time": "2019-09-13 21:43:26+03", - "end-time": "2019-09-13 21:43:30+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:43:29+03", - "data-bytes": 104674852, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - }, - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "0000000000000001", - "max-segno": "000000000000000A", - "n-segments": 10, - "size": 8774805, - "zratio": 19.00, - "closest-backup-id": "", - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92O", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "true", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/4000028", - "stop-lsn": "0/6000028", - "start-time": "2019-09-13 21:37:36+03", - "end-time": "2019-09-13 21:38:45+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 25987319, - "wal-bytes": 50331648, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - }, - { - "instance": "master", - "timelines": [ - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "0000000000000001", - "max-segno": "000000000000000B", - "n-segments": 11, - "size": 8860892, - "zratio": 20.00, - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92H", - "parent-backup-id": "PXS92C", - "backup-mode": "PAGE", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 1, - "start-lsn": "0/4000028", - "stop-lsn": "0/50000B8", - "start-time": "2019-09-13 21:37:29+03", - "end-time": "2019-09-13 21:37:31+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 1328461, - "wal-bytes": 33554432, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - }, - { - "id": "PXS92C", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/2000028", - "stop-lsn": "0/2000160", - "start-time": "2019-09-13 21:37:24+03", - "end-time": "2019-09-13 21:37:29+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:28+03", - "data-bytes": 24871902, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - } -] -``` - -Most fields are consistent with plain format, with some exceptions: - -- size is in bytes. -- 'closest-backup-id' attribute contain ID of valid backup closest to the timeline, located on some of the previous timelines. This backup is the closest starting point to reach the timeline from other timelines by PITR. Closest backup always has a valid status, either OK or DONE. If such backup do not exists, then string is empty. -- DEGRADED timelines contain 'lost-segments' array with information about intervals of missing segments. In OK timelines 'lost-segments' array is empty. -- 'N backups' attribute is replaced with 'backups' array containing backups belonging to the timeline. If timeline has no backups, then 'backups' array is empty. - -### Configuring Retention Policy - -With pg_probackup, you can set retention policies for backups and WAL archive. All policies can be combined together in any way. - -#### Backup Retention Policy - -By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. - -To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): - - --retention-redundancy=redundancy -Specifies **the number of full backup copies** to keep in the backup catalog. - - --retention-window=window -Defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in **the number of days** from the current moment. For example, if `retention-window=7`, pg_probackup must delete all backup copies that are older than seven days, with all the corresponding WAL files. - -If both `--retention-redundancy` and `--retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `--retention-redundancy=2` and `--retention-window=7`, pg_probackup purges the backup catalog to keep only two full backup copies and all backups that are newer than seven days: - - pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 - -To clean up the backup catalog in accordance with retention policy, run: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired - -pg_probackup deletes all backup copies that do not conform to the defined retention policy. - -If you would like to also remove the WAL files that are no longer required for any of the backups, add the `--delete-wal` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal - ->NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. - -You can set or override the current retention policy by specifying `--retention-redundancy` and `--retention-window` options directly when running `delete` or `backup` commands: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 - -Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` flag when running [backup](#backup) or [delete](#delete) commands. - -Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `--retention-window` option is set to *7*, and you have the following backups available on April 10, 2019: - -``` -BACKUP INSTANCE 'node' -=================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -=================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK - -------------------------------------------------------retention window-------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK -``` - -Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` flag, only the P7XDFT full backup will be removed. - -With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: - - pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired - pg_probackup show -B backup_dir - -``` -BACKUP INSTANCE 'node' -================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK -``` - ->NOTE: The Time field for the merged backup displays the time required for the merge. - -#### Backup Pinning - -If you have the necessity to exclude certain backups from established retention policy then it is possible to pin a backup for an arbitrary amount of time. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d - -This command will set `expire-time` of the specified backup to 30 days starting from backup `recovery-time` attribute. Basically `expire-time` = `recovery-time` + `ttl`. - -Also you can set `expire-time` explicitly using `--expire-time` option. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' - -Alternatively you can use the `--ttl` and `--expire-time` options with the [backup](#backup) command to pin newly created backup: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d - pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' - -You can determine the fact that backup is pinned and check due expire time by looking up `expire-time` attribute in backup metadata via [show](#show) command: - - pg_probackup show --instance instance_name -i backup_id - -Pinned backup has `expire-time` attribute: -``` -... -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-01-01 00:00:00+03' -data-bytes = 22288792 -... -``` - -You can unpin the backup by setting the `--ttl` option to zero using [set-backup](#set-backup) command. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 - -Only pinned backups have the `expire-time` attribute in the backup metadata. - ->NOTE: Pinned incremental backup will also implicitly pin all its parent backups. - -#### WAL Archive Retention Policy - -By default, pg_probackup treatment of WAL Archive is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any of the existing backups in the backup catalog. To save disk space, you can configure WAL Archive retention policy, that allows to keep WAL of limited depth measured in backups per timeline. - -Suppose you have backed up the *node* instance in the *backup_dir* directory with configured [WAL archiving](#setting-up-continuous-wal-archiving): - - pg_probackup show -B backup_dir --instance node - -``` -BACKUP INSTANCE 'node' -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK - node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK -``` - -The state of WAL archive can be determined by using [show](#command) command with `--archive` flag: - - pg_probackup show -B backup_dir --instance node --archive - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK -``` - -General WAL purge without `wal-depth` cannot achieve much, only one segment can be removed: - - pg_probackup delete -B backup_dir --instance node --delete-wal - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK -``` - -If you would like, for example, to keep only those WAL segments that can be applied to the last valid backup, use the `--wal-depth` option: - - pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 - -``` -ARCHIVE INSTANCE 'node' -================================================================================================================ - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -================================================================================================================ - 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK -``` - -Alternatively you can use the `--wal-depth` option with the [backup](#backup) command: - - pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK -``` - -### Merging Backups - -As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: - - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - -This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes, I/O and network traffic in case of [remote](#using-pg_probackup-in-the-remote-mode) backup. - -Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the [show](#show) command with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name -i backup_id - -If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. - -### Deleting Backups - -To delete a backup that is no longer required, run the following command: - - pg_probackup delete -B backup_dir --instance instance_name -i backup_id - -This command will delete the backup with the specified *backup_id*, together with all the incremental backups that descend from *backup_id* if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. - -To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-wal - -To delete backups that are expired according to the current retention policy, use the `--delete-expired` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired - -Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the `--merge-expired` flag when running this command: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired - -In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. - -Before merging or deleting backups, you can run the `delete` command with the `--dry-run` flag, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. - -## Command-Line Reference -### Commands - -This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). - -#### version - - pg_probackup version - -Prints pg_probackup version. - -#### help - - pg_probackup help [command] - -Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. - -#### init - - pg_probackup init -B backup_dir [--help] - -Initializes the backup catalog in *backup_dir* that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified *backup_dir* already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. - -For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). - -#### add-instance - - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name - [--help] - -Initializes a new backup instance inside the backup catalog *backup_dir* and generates the pg_probackup.conf configuration file that controls pg_probackup settings for the cluster with the specified *data_dir* data directory. - -For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). - -#### del-instance - - pg_probackup del-instance -B backup_dir --instance instance_name - [--help] - -Deletes all backups and WAL files associated with the specified instance. - -#### set-config - - pg_probackup set-config -B backup_dir --instance instance_name - [--help] [--pgdata=pgdata-path] - [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] - [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] - [-d dbname] [-h host] [-p port] [-U username] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [--restore-command=cmdline] - [remote_options] [remote_archive_options] [logging_options] - -Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. - -For all available settings, see the [Options](#options) section. - -It is **not recommended** to edit pg_probackup.conf manually. - -#### set-backup - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id - {--ttl=ttl | --expire-time=time} [--help] - -Sets the provided backup-specific settings into the backup.control configuration file, or modifies previously defined values. - -For all available settings, see the section [Pinning Options](#pinning-options). - -#### show-config - - pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] - -Displays the contents of the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. - -To edit pg_probackup.conf, use the [set-config](#set-config) command. - -#### show - - pg_probackup show -B backup_dir - [--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] - -Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. If `--archive` option is specified, shows the content of WAL archive of the backup catalog. - -By default, the contents of the backup catalog is shown as plain text. - -For details on usage, see the sections [Managing the Backup Catalog](#managing-the-backup-catalog) and [Viewing WAL Archive Information](#viewing-wal-archive-information). - - -#### backup - - pg_probackup backup -B backup_dir -b backup_mode --instance instance_name - [--help] [-j num_threads] [--progress] - [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] - [--no-validate] [--skip-block-validation] - [-w --no-password] [-W --password] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [connection_options] [compression_options] [remote_options] - [retention_options] [pinning_options] [logging_options] - -Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. - - -b mode - --backup-mode=mode - -Specifies the backup mode to use. Possible values are: - -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. - -``` --C ---smooth-checkpoint -``` -Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. - - --stream -Makes an [STREAM](#stream-mode) backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. - - --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This flag can only be used together with the `--stream` flag. Default slot name is `pg_probackup_slot`, which can be changed via option `--slot/-S`. - - -S slot_name - --slot=slot_name -Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` flag. - - --backup-pg-log -Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. - - -E external_directory_path - --external-dirs=external_directory_path -Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. - - --archive-timeout=wait_time -Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. - - --skip-block-validation -Disables block-level checksum verification to speed up backup. - - --no-validate -Skips automatic validation after successfull backup. You can use this flag if you validate backups regularly and would like to save time when running backup operations. - -Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Pinning Options](#pinning-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. - -For details on usage, see the section [Creating a Backup](#creating-a-backup). - -#### restore - - pg_probackup restore -B backup_dir --instance instance_name - [--help] [-D data_dir] [-i backup_id] - [-j num_threads] [--progress] - [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] - [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] - [--restore-command=cmdline] - [recovery_options] [logging_options] [remote_options] - [partial_restore_options] [remote_archive_options] - -Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a [recovery target option](#recovery-target-options), pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. - - -R | --restore-as-replica -Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. - - -T OLDDIR=NEWDIR - --tablespace-mapping=OLDDIR=NEWDIR - -Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. - - --external-mapping=OLDDIR=NEWDIR -Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. - - --skip-external-dirs -Skip external directories included into the backup with the `--external-dirs` option. The contents of these directories will not be restored. - - --skip-block-validation -Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. - - --no-validate -Skips backup validation. You can use this flag if you validate backups regularly and would like to save time when running restore operations. - - --restore-command=cmdline -Set the [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) parameter to specified command. Example: `--restore-command='cp /mnt/server/archivedir/%f "%p"'` - - --force -Allows to ignore the invalid status of the backup. You can use this flag if you for some reason have the necessity to restore PostgreSQL cluster from corrupted or invalid backup. Use with caution. - -Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Remote WAL Archive Options](#remote-wal-archive-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. - -For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). - -#### checkdb - - pg_probackup checkdb - [-B backup_dir] [--instance instance_name] [-D data_dir] - [--help] [-j num_threads] [--progress] - [--skip-block-validation] [--amcheck] [--heapallindexed] - [connection_options] [logging_options] - -Verifies the PostgreSQL database cluster correctness by detecting physical and logical corruption. - - --amcheck -Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. - - --skip-block-validation -Skip validation of data files. Can be used only with `--amcheck` flag, so only logical verification of indexes is performed. - - --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this flag only together with the `--amcheck` flag. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. - -Additionally [Connection Options](#connection-options) and [Logging Options](#logging-options) can be used. - -For details on usage, see the section [Verifying a Cluster](#verifying-a-cluster). - -#### validate - - pg_probackup validate -B backup_dir - [--help] [--instance instance_name] [-i backup_id] - [-j num_threads] [--progress] - [--skip-block-validation] - [recovery_target_options] [logging_options] - -Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target options](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. - -For details, see the section [Validating a Backup](#validating-a-backup). - -#### merge - - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - [--help] [-j num_threads] [--progress] - [logging_options] - -Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. - -For details, see the section [Merging Backups](#merging-backups). - -#### delete - - pg_probackup delete -B backup_dir --instance instance_name - [--help] [-j num_threads] [--progress] - [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] - [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} - [--dry-run] - [logging_options] - -Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. - -For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-options) and [Configuring Retention Policy](#configuring-retention-policy). - -#### archive-push - - pg_probackup archive-push -B backup_dir --instance instance_name - --wal-file-path=wal_file_path --wal-file-name=wal_file_name - [--help] [--compress] [--compress-algorithm=compression_algorithm] - [--compress-level=compression_level] [--overwrite] - [remote_options] [logging_options] - -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by *instance_name* and *system-identifier*. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. - -If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` flag. - -Copying is done to temporary file with `.part` suffix or, if [compression](#compression-options) is used, with `.gz.part` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. -Copied to archive WAL segments are synced to disk. - -You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). - -For details, see sections [Archiving Options](#archiving-options) and [Compression Options](#compression-options). - -#### archive-get - - pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name - [--help] [remote_options] [logging_options] - -Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. - -### Options - -This section describes command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. - -For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). - -If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. - -#### Common Options -The list of general options. - - -B directory - --backup-path=directory - BACKUP_PATH -Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. - - -D directory - --pgdata=directory - PGDATA -Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the [add-instance](#add-instance) command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. - - -i backup_id - -backup-id=backup_id -Specifies the unique identifier of the backup. - - -j num_threads - --threads=num_threads -Sets the number of parallel threads for backup, restore, merge, validation and verification processes. - - --progress -Shows the progress of operations. - - --help -Shows detailed information about the options that can be used with this command. - -#### Recovery Target Options - -If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. - - --recovery-target=immediate|latest -Defines when to stop the recovery: - -- `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. -- `latest` value continues the recovery until all WAL segments available in the archive are applied. - -Default value of `--recovery-target` depends on WAL delivery method of restored backup, `immediate` for STREAM backup and `latest` for ARCHIVE. - - --recovery-target-timeline=timeline -Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. - - --recovery-target-lsn=lsn -Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring database cluster of major version 10 or higher. - - --recovery-target-name=recovery_target_name -Specifies a named savepoint up to which to restore the cluster data. - - --recovery-target-time=time -Specifies the timestamp up to which recovery will proceed. - - --recovery-target-xid=xid -Specifies the transaction ID up to which recovery will proceed. - - --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default depends on [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. - - --recovery-target-action=pause|promote|shutdown - Default: pause -Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. - -#### Retention Options - -You can use these options together with [backup](#backup) and [delete](#delete) commands. - -For details on configuring retention policy, see the section [Configuring Retention Policy](#configuring-retention-policy). - - --retention-redundancy=redundancy - Default: 0 -Specifies the number of full backup copies to keep in the data directory. Must be a positive integer. The zero value disables this setting. - - --retention-window=window - Default: 0 -Number of days of recoverability. Must be a positive integer. The zero value disables this setting. - - --wal-depth=wal_depth - Default: 0 -Number of latest valid backups on every timeline that must retain the ability to perform PITR. Must be a positive integer. The zero value disables this setting. - - --delete-wal -Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. - - --delete-expired -Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. - - --merge-expired -Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. - - --dry-run -Displays the current status of all the available backups, without deleting or merging expired backups, if any. - -##### Pinning Options - -You can use these options together with [backup](#backup) and [set-delete](#set-backup) commands. - -For details on backup pinning, see the section [Backup Pinning](#backup-pinning). - - --ttl=ttl -Specifies the amount of time the backup should be pinned. Must be a positive integer. The zero value unpin already pinned backup. Supported units: ms, s, min, h, d (s by default). Example: `--ttl=30d`. - - --expire-time=time -Specifies the timestamp up to which the backup will stay pinned. Must be a ISO-8601 complaint timestamp. Example: `--expire-time='2020-01-01 00:00:00+03'` - -#### Logging Options - -You can use these options with any command. - - --log-level-console=log_level - Default: info -Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. - ->NOTE: all console log messages are going to stderr, so output from [show](#show) and [show-config](#show-config) commands do not mingle with log messages. - - --log-level-file=log_level - Default: off -Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. - - --log-filename=log_filename - Default: pg_probackup.log -Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. - -For example, if you specify the 'pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. - -This option takes effect if file logging is enabled by the `log-level-file` option. - - --error-log-filename=error_log_filename - Default: none -Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. - -For example, if you specify the 'error-pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. - -This option is useful for troubleshooting and monitoring. - - --log-directory=log_directory - Default: $BACKUP_PATH/log/ -Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. - - --log-rotation-size=log_rotation_size - Default: 0 -Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). - - --log-rotation-age=log_rotation_age - Default: 0 -Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). - -#### Connection Options - -You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. - -All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. - - -d dbname - --pgdatabase=dbname - PGDATABASE -Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. - - -h host - --pghost=host - PGHOST - Default: local socket -Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. - - -p port - --pgport=port - PGPORT - Default: 5432 -Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. - - -U username - --pguser=username - PGUSER -User name to connect as. - - -w - --no-password - Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file or PGPASSWORD environment variable, the connection attempt will fail. This flag can be useful in batch jobs and scripts where no user is present to enter a password. - - -W - --password -Forces a password prompt. - -#### Compression Options - -You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. - - --compress-algorithm=compression_algorithm - Default: none -Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. -For the [archive-push](#archive-push) command, the pglz compression algorithm is not supported. - - --compress-level=compression_level - Default: 1 -Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with `--compress-algorithm` option. - - --compress -Alias for `--compress-algorithm=zlib` and `--compress-level=1`. - -#### Archiving Options - -These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. - -Additionally [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. - - --wal-file-path=wal_file_path -Provides the path to the WAL file in `archive_command` and `restore_command`. The `%p` variable as value for this option is required for correct processing. - - --wal-file-name=wal_file_name -Provides the name of the WAL file in `archive_command` and `restore_command`. The `%f` variable as value is required for correct processing. - - --overwrite -Overwrites archived WAL file. Use this flag together with the [archive-push](#archive-push) command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` flag. - -#### Remote Mode Options - -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. - -For details on configuring and usage of remote operation mode, see the sections [Configuring the Remote Mode](#configuring-the-remote-mode) and [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). - - --remote-proto=proto -Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: - -- `ssh` enables the remote backup mode via SSH. This is the Default value. -- `none` explicitly disables the remote mode. - -You can omit this option if the `--remote-host` option is specified. - - --remote-host=destination -Specifies the remote host IP address or hostname to connect to. - - --remote-port=port - Default: 22 -Specifies the remote host port to connect to. - - --remote-user=username - Default: current user -Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. - - --remote-path=path -Specifies pg_probackup installation directory on the remote system. - - --ssh-options=ssh_options -Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found on [ssh_config manual page](https://man.openbsd.org/ssh_config.5). - -#### Remote WAL Archive Options - -This section describes the options used to provide the arguments for [Remote Mode Options](#remote-mode-options) in [archive-get](#archive-get) used in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) command when restoring ARCHIVE backup or performing PITR. - - --archive-host=destination -Provides the argument for `--remote-host` option in `archive-get` command. - - --archive-port=port - Default: 22 -Provides the argument for `--remote-port` option in `archive-get` command. - - --archive-user=username - Default: PostgreSQL user -Provides the argument for `--remote-user` option in `archive-get` command. If you omit this option, the the user running PostgreSQL cluster is used. - -#### Partial Restore Options - -This section describes the options related to partial restore of a cluster from backup. These options can be used with [restore](#restore) command. - - --db-exclude=dbname -Specifies database name to exclude from restore. All other databases in the cluster will be restored as usual, including `template0` and `template1`. This option can be specified multiple times for multiple databases. - - --db-include=dbname -Specifies database name to restore from backup. All other databases in the cluster will not be restored, with exception of `template0` and `template1`. This option can be specified multiple times for multiple databases. - -#### Replica Options - -This section describes the options related to taking a backup from standby. - ->NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. - - --master-db=dbname - Default: postgres, the default PostgreSQL database. -Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the [set-config](#set-config) command. - - --master-host=host -Deprecated. Specifies the host name of the system on which the master server is running. - - --master-port=port - Default: 5432, the PostgreSQL default port. -Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. - - --master-user=username - Default: postgres, the PostgreSQL default user name. -Deprecated. User name to connect as. - - --replica-timeout=timeout - Default: 300 sec -Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. - -## Howto - -All examples below assume the remote mode of operations via `ssh`. If you are planning to run backup and restore operation locally then step `Setup passwordless SSH connection` can be skipped and all `--remote-*` options can be ommited. - -Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. - -- *backup_host* - host with backup catalog. -- *backupman* - user on `backup_host` running all pg_probackup operations. -- */mnt/backups* - directory on `backup_host` where backup catalog is stored. -- *postgres_host* - host with PostgreSQL cluster. -- *postgres* - user on `postgres_host` which run PostgreSQL cluster. -- */var/lib/postgresql/11/main* - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. -- *backupdb* - database used for connection to PostgreSQL cluster. - -### Minimal Setup - -This setup is relying on autonomous FULL and DELTA backups. - -#### Setup passwordless SSH connection from `backup_host` to `postgres_host` -``` -[backupman@backup_host] ssh-copy-id postgres@postgres_host -``` - -#### Setup PostgreSQL cluster - -It is recommended from security purposes to use separate database for backup operations. -``` -postgres=# -CREATE DATABASE backupdb; -``` - -Connect to `backupdb` database, create role `probackup` and grant to it the following permissions: -``` -backupdb=# -BEGIN; -CREATE ROLE probackup WITH LOGIN REPLICATION; -GRANT USAGE ON SCHEMA pg_catalog TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO probackup; -COMMIT; -``` - -#### Init the backup catalog -``` -[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups -INFO: Backup catalog '/mnt/backups' successfully inited -``` - -#### Add instance 'pg-11' to backup catalog -``` -[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main -INFO: Instance 'node' successfully inited -``` - -#### Take FULL backup -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YK2 -INFO: Backup PZ7YK2 data files are valid -INFO: Backup PZ7YK2 resident size: 196MB -INFO: Backup PZ7YK2 completed -``` - -#### Lets take a look at the backup catalog -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' - -BACKUP INSTANCE 'pg-11' -================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK -``` - -#### Take incremental backup in DELTA mode -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YK2 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YMP -INFO: Backup PZ7YMP data files are valid -INFO: Backup PZ7YMP resident size: 32MB -INFO: Backup PZ7YMP completed -``` - -#### Lets hide some parameters into config, so cmdline can be less crowdy -``` -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -``` - -#### Take another incremental backup in DELTA mode, omitting some of the previous parameters: -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YMP -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YR5 -INFO: Backup PZ7YR5 data files are valid -INFO: Backup PZ7YR5 resident size: 32MB -INFO: Backup PZ7YR5 completed -``` - -#### Lets take a look at instance config -``` -[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' - -# Backup instance information -pgdata = /var/lib/postgresql/11/main -system-identifier = 6746586934060931492 -xlog-seg-size = 16777216 -# Connection parameters -pgdatabase = backupdb -pghost = postgres_host -pguser = probackup -# Replica parameters -replica-timeout = 5min -# Archive parameters -archive-timeout = 5min -# Logging parameters -log-level-console = INFO -log-level-file = OFF -log-filename = pg_probackup.log -log-rotation-size = 0 -log-rotation-age = 0 -# Retention parameters -retention-redundancy = 0 -retention-window = 0 -wal-depth = 0 -# Compression parameters -compress-algorithm = none -compress-level = 1 -# Remote access parameters -remote-proto = ssh -remote-host = postgres_host -``` - -Note, that we are getting default values for other options, that were not overwritten by set-config command. - - -#### Lets take a look at the backup catalog -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' - -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK -``` - -## Authors -Postgres Professional, Moscow, Russia. - -## Credits -pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. From 0f296ffbf5bc3ac9cac35851057e6d40959baab5 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 26 Jan 2021 15:24:01 +0300 Subject: [PATCH 1585/2107] [issue #301] Only run CI tests for master branch commits to limit our travis usage. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index fc7ecc059..457835d3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,3 +45,8 @@ env: jobs: allow_failures: - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) + +# Only run CI for master branch commits to limit our travis usage +branches: + only: + - master From 7b3d13f1645bedf6e85100a5539cdeadb64f0ab4 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 26 Jan 2021 15:38:54 +0300 Subject: [PATCH 1586/2107] Update README: Add Development section. Fix broken link to documentaiton --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 706f92201..1c61413ce 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,14 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup) +## Development + +* Stable version state can be found under the respective [release tag](https://github.com/postgrespro/pg_probackup/releases). +* `master` branch contains minor fixes that are planned to the nearest minor release. +* Upcoming major release is developed in a release branch i.e. `release_2_5`. + +For detailed release plans check [Milestones](https://github.com/postgrespro/pg_probackup/milestones) + ## Installation and Setup ### Windows Installation Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.4). @@ -160,7 +168,7 @@ sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo ``` -Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). ## Building from source ### Linux From 315ab4f062f6f7e99d7d004a9aa25d6707529d62 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Tue, 26 Jan 2021 17:06:31 +0300 Subject: [PATCH 1587/2107] [DOC] Edits to recent document changes --- doc/pgprobackup.xml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 88ce237d5..a394cbcd4 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1875,8 +1875,9 @@ pg_probackup restore -B backup_dir --instance DROP DATABASE command. - If want to decouple a single cluster with multiple databases into separate clusters with minimal downtime, then - you can partially restore cluster as standby via options. + To decouple a single cluster containing multiple databases into separate clusters with minimal downtime, + you can do partial restore of the cluster as a standby using the option + for specific databases. @@ -1886,10 +1887,12 @@ pg_probackup restore -B backup_dir --instance - Due to how recovery works in PostgreSQL versions lower than 12 it is advisable to - disable option, when running partial - restore of PostgreSQL cluster of version less than . - Otherwise recovery may fail. + Due to recovery specifics of PostgreSQL versions earlier than 12, + it is advisable that you set the + hot_standby + parameter to off when running partial + restore of a PostgreSQL cluster of version earlier than 12. + Otherwise the recovery may fail. @@ -3976,10 +3979,11 @@ pg_probackup restore -B backup_dir --instance PostgreSQL cluster from a corrupt or an invalid backup. Use with caution. - When used with incremental restore this flag - allows to replace already existing PGDATA with different system ID. In case of tablespaces, - remapped via --tablespace-mapping option into non-empty directories, - the content of such directories will be deleted. + If PGDATA contains a non-empty directory with system ID different from that + of the backup being restored, incremental restore + with this flag overwrites the directory contents (while an error occurs without the flag). If tablespaces + are remapped through the --tablespace-mapping option into non-empty directories, + the contents of such directories will be deleted. @@ -4425,7 +4429,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which recovery will proceed. - If timezone offset is not specified, local timezone is used. + If the time zone offset is not specified, the local time zone is used. Example: --recovery-target-time='2020-01-01 00:00:00+03' @@ -4615,7 +4619,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which the backup will stay pinned. Must be an ISO-8601 complaint timestamp. - If timezone offset is not specified, local timezone is used. + If the time zone offset is not specified, the local time zone is used. Example: --expire-time='2020-01-01 00:00:00+03' From c12f8467e0cd80f7a28a03c36409c92f26a26fc4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 26 Jan 2021 20:16:36 +0300 Subject: [PATCH 1588/2107] Version 2.4.9 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index aec5c3e84..2ec09babb 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -303,8 +303,8 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.8" -#define AGENT_PROTOCOL_VERSION 20408 +#define PROGRAM_VERSION "2.4.9" +#define AGENT_PROTOCOL_VERSION 20409 /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 9a1e5663b..1481e32db 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.8 \ No newline at end of file +pg_probackup 2.4.9 \ No newline at end of file From 807cc27ebe4f3cbb260aec85691b8710c844651f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Jan 2021 00:23:59 +0300 Subject: [PATCH 1589/2107] Fix overwriting of 0-sized postgresql.auto.conf in remote mode, restore nonedata files asynchronously --- src/data.c | 2 +- src/restore.c | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/data.c b/src/data.c index a3baa3d3b..d3f67f43c 100644 --- a/src/data.c +++ b/src/data.c @@ -1145,7 +1145,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, if (read_len > 0) { - if (fio_fwrite(out, buf, read_len) != read_len) + if (fio_fwrite_async(out, buf, read_len) != read_len) elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); } diff --git a/src/restore.c b/src/restore.c index aa9bff9e2..a1f009b04 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1515,8 +1515,8 @@ update_recovery_options(pgBackup *backup, char postgres_auto_path[MAXPGPATH]; char postgres_auto_path_tmp[MAXPGPATH]; char path[MAXPGPATH]; - FILE *fp; - FILE *fp_tmp; + FILE *fp = NULL; + FILE *fp_tmp = NULL; struct stat st; char current_time_str[100]; /* postgresql.auto.conf parsing */ @@ -1540,9 +1540,13 @@ update_recovery_options(pgBackup *backup, strerror(errno)); } - fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); - if (fp == NULL && errno != ENOENT) - elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); + /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ + if (st.st_size > 0) + { + fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); + if (fp == NULL && errno != ENOENT) + elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); + } sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); @@ -1582,9 +1586,11 @@ update_recovery_options(pgBackup *backup, if (fp) fio_close_stream(fp); - /* TODO: detect remote error */ - if (buf_len > 0) - fio_fwrite(fp_tmp, buf, buf_len); + /* Write data to postgresql.auto.conf.tmp */ + if (buf_len > 0 && + (fio_fwrite(fp_tmp, buf, buf_len) != buf_len)) + elog(ERROR, "Cannot write to \"%s\": %s", + postgres_auto_path_tmp, strerror(errno)); if (fio_fflush(fp_tmp) != 0 || fio_fclose(fp_tmp)) From 25fd6c22066588e30fd0702495085ed462ba9571 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Jan 2021 01:48:38 +0300 Subject: [PATCH 1590/2107] fix: do not ignore ENOENT, when overwriting postgresql.auto.conf --- src/restore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restore.c b/src/restore.c index a1f009b04..3f0adf7b7 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1544,7 +1544,7 @@ update_recovery_options(pgBackup *backup, if (st.st_size > 0) { fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); - if (fp == NULL && errno != ENOENT) + if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); } From 2620042019c702423febc58444e11495d746417e Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Wed, 27 Jan 2021 10:25:25 +0300 Subject: [PATCH 1591/2107] [DOC] Remove outdated content from the documentation --- doc/pgprobackup.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a394cbcd4..b0a0f6763 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1206,18 +1206,6 @@ CREATE EXTENSION ptrack; does not affect PTRACK operation. The maximum allowed value is 1024. - - - Grant the right to execute PTRACK - functions to the backup role - in the database used to connect to the cluster: - - -GRANT EXECUTE ON FUNCTION pg_ptrack_get_pagemapset(pg_lsn) TO backup; -GRANT EXECUTE ON FUNCTION pg_ptrack_control_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - - @@ -1254,7 +1242,6 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; From 0fbf1a2fabb97fd9004bf6a4a74739024eb9bda8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 27 Jan 2021 15:55:36 +0300 Subject: [PATCH 1592/2107] README: update link to latest Windows installers --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c61413ce..2ecaf9695 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ For detailed release plans check [Milestones](https://github.com/postgrespro/pg_ ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.4). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.9). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 7d64d58755bd5314d035ff66a65ee2ffb240ceea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Feb 2021 16:09:02 +0300 Subject: [PATCH 1593/2107] [Issue #308] Wait on empty exclusive lock file --- src/catalog.c | 52 ++++++++++++++++++++++++++++++++++------------ src/pg_probackup.h | 4 +++- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 302154178..94c01cff9 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -205,7 +205,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) { /* release exclusive lock */ if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) - elog(ERROR, "Could not remove old lock file \"%s\": %s", + elog(ERROR, "Could not remove exclusive lock file \"%s\": %s", lock_file, strerror(errno)); /* we are done */ @@ -261,7 +261,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) int fd = 0; char buffer[MAXPGPATH * 2 + 256]; int ntries = LOCK_TIMEOUT; - int log_freq = ntries / 5; + int empty_tries = LOCK_STALE_TIMEOUT; int len; int encoded_pid; pid_t my_p_pid; @@ -351,13 +351,39 @@ lock_backup_exclusive(pgBackup *backup, bool strict) fclose(fp_out); /* - * It should be possible only as a result of system crash, - * so its hypothetical owner should be dead by now + * There are several possible reasons for lock file + * to be empty: + * - system crash + * - process crash + * - race between writer and reader + * + * Consider empty file to stale after LOCK_STALE_TIMEOUT + * attempts. + * + * TODO: alternatively we can write into temp file (lock_file_%pid), + * rename it and then re-read lock file to make sure, + * that we are successfully acquired the lock. */ if (len == 0) { - elog(WARNING, "Lock file \"%s\" is empty", lock_file); - goto grab_lock; + if (empty_tries == 0) + { + elog(WARNING, "Lock file \"%s\" is empty", lock_file); + goto grab_lock; + } + + if ((empty_tries % LOG_FREQ) == 0) + elog(WARNING, "Waiting %u seconds on empty exclusive lock for backup %s", + empty_tries, base36enc(backup->start_time)); + + sleep(1); + /* + * waiting on empty lock file should not affect + * the timer for concurrent lockers (ntries). + */ + empty_tries--; + ntries++; + continue; } encoded_pid = atoi(buffer); @@ -383,12 +409,13 @@ lock_backup_exclusive(pgBackup *backup, bool strict) if (kill(encoded_pid, 0) == 0) { /* complain every fifth interval */ - if ((ntries % log_freq) == 0) + if ((ntries % LOG_FREQ) == 0) { elog(WARNING, "Process %d is using backup %s, and is still running", encoded_pid, base36enc(backup->start_time)); - elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, base36enc(backup->start_time)); + elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", + ntries, base36enc(backup->start_time)); } sleep(1); @@ -435,7 +462,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) errno = 0; if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) { - int save_errno = errno; + int save_errno = errno; fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); @@ -453,7 +480,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) if (fio_flush(fd) != 0) { - int save_errno = errno; + int save_errno = errno; fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); @@ -471,7 +498,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) if (fio_close(fd) != 0) { - int save_errno = errno; + int save_errno = errno; fio_unlink(lock_file, FIO_BACKUP_HOST); @@ -493,7 +520,6 @@ wait_read_only_owners(pgBackup *backup) char buffer[256]; pid_t encoded_pid; int ntries = LOCK_TIMEOUT; - int log_freq = ntries / 5; char lock_file[MAXPGPATH]; join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); @@ -523,7 +549,7 @@ wait_read_only_owners(pgBackup *backup) { if (kill(encoded_pid, 0) == 0) { - if ((ntries % log_freq) == 0) + if ((ntries % LOG_FREQ) == 0) { elog(WARNING, "Process %d is using backup %s in read only mode, and is still running", encoded_pid, base36enc(backup->start_time)); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2ec09babb..f4adc98cc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -79,7 +79,9 @@ extern const char *PROGRAM_EMAIL; /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 -#define LOCK_TIMEOUT 30 +#define LOCK_TIMEOUT 60 +#define LOCK_STALE_TIMEOUT 30 +#define LOG_FREQ 10 /* Directory/File permission */ #define DIR_PERMISSION (0700) From d9d6c34e2576e75e06ced80276d41e8e0db555ea Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 1 Feb 2021 16:42:44 +0300 Subject: [PATCH 1594/2107] [Issue #308] fix typo --- src/catalog.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 94c01cff9..63bb6862e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -357,8 +357,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) * - process crash * - race between writer and reader * - * Consider empty file to stale after LOCK_STALE_TIMEOUT - * attempts. + * Consider empty file to be stale after LOCK_STALE_TIMEOUT attempts. * * TODO: alternatively we can write into temp file (lock_file_%pid), * rename it and then re-read lock file to make sure, From 2ae2908ea7d327ae57b56389060cd040a56a4dcb Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 2 Feb 2021 18:19:47 +0300 Subject: [PATCH 1595/2107] Code cleanup. Remove unused functions --- src/catalog.c | 47 ---------------------------------------------- src/pg_probackup.h | 4 ---- 2 files changed, 51 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 57da50c8d..2936033e4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2741,33 +2741,6 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, base36enc(backup->start_time), subdir1, subdir2); } -/* - * independent from global variable backup_instance_path - * Still depends from backup_path - */ -void -pgBackupGetPathInInstance(const char *instance_name, - const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2) -{ - char backup_instance_path[MAXPGPATH]; - - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - - /* If "subdir1" is NULL do not check "subdir2" */ - if (!subdir1) - snprintf(path, len, "%s/%s", backup_instance_path, - base36enc(backup->start_time)); - else if (!subdir2) - snprintf(path, len, "%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1); - /* "subdir1" and "subdir2" is not NULL */ - else - snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1, subdir2); -} - /* * Check if multiple backups consider target backup to be their direct parent */ @@ -2917,26 +2890,6 @@ is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive) return false; } -/* - * Return backup index number. - * Note: this index number holds true until new sorting of backup list - */ -int -get_backup_index_number(parray *backup_list, pgBackup *backup) -{ - int i; - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); - - if (tmp_backup->start_time == backup->start_time) - return i; - } - elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time)); - return -1; -} - /* On backup_list lookup children of target_backup and append them to append_list */ void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 217c8a7f1..721840af2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -948,9 +948,6 @@ extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); -extern void pgBackupGetPathInInstance(const char *instance_name, - const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2); extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); @@ -968,7 +965,6 @@ extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern bool is_prolific(parray *backup_list, pgBackup *target_backup); -extern int get_backup_index_number(parray *backup_list, pgBackup *backup); extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list); extern bool launch_agent(void); extern void launch_ssh(char* argv[]); From 0fd2fdeec0ead56468af408e2566b3d2b04fd53c Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 2 Feb 2021 19:12:08 +0300 Subject: [PATCH 1596/2107] Code cleanup. Remove unused pgBackupGetPath() function --- src/backup.c | 2 +- src/catalog.c | 10 ---------- src/dir.c | 1 - src/pg_probackup.h | 2 -- src/validate.c | 1 - 5 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/backup.c b/src/backup.c index 7bdf9d6fb..0b61234f1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1774,7 +1774,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!exclusive_backup) { Assert(PQnfields(res) >= 4); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); + pgBackupGetPath2(backup, path, lengthof(path), DATABASE_DIR, NULL); /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); diff --git a/src/catalog.c b/src/catalog.c index 2936033e4..741a007ff 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2710,16 +2710,6 @@ pgBackupCompareIdDesc(const void *l, const void *r) return -pgBackupCompareId(l, r); } -/* - * Construct absolute path of the backup directory. - * If subdir is not NULL, it will be appended after the path. - */ -void -pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) -{ - pgBackupGetPath2(backup, path, len, subdir, NULL); -} - /* * Construct absolute path of the backup directory. * Append "subdir1" and "subdir2" to the backup directory. diff --git a/src/dir.c b/src/dir.c index d07a4d2f5..2bcd87b57 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1880,7 +1880,6 @@ read_database_map(pgBackup *backup) char path[MAXPGPATH]; char database_map_path[MAXPGPATH]; -// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 721840af2..06fab18dd 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -944,8 +944,6 @@ extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, - const char *subdir); extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, const char *subdir1, const char *subdir2); extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path); diff --git a/src/validate.c b/src/validate.c index 21900c8e4..b44f4b1b8 100644 --- a/src/validate.c +++ b/src/validate.c @@ -205,7 +205,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { char path[MAXPGPATH]; - //pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); if (pgFileSize(path) >= (BLCKSZ*500)) From f26c95964701a7666b585d3e8ce61eab3e3a0bf1 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 2 Feb 2021 19:36:39 +0300 Subject: [PATCH 1597/2107] remove unneeded funtion dir_read_file_list() --- src/catalog.c | 109 +++++++++++++++++++++++++++++++++++++++-- src/dir.c | 119 +-------------------------------------------- src/pg_probackup.h | 4 +- 3 files changed, 107 insertions(+), 125 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 741a007ff..11b9a27d1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -876,19 +876,118 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) } /* - * Create list of backup datafiles. - * If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error. - * If valid backup id is passed only matching backup will be added to the list. - * TODO this function only used once. Is it really needed? + * Get list of files in the backup from the DATABASE_FILE_LIST. */ parray * get_backup_filelist(pgBackup *backup, bool strict) { parray *files = NULL; char backup_filelist_path[MAXPGPATH]; + FILE *fp; + char buf[BLCKSZ]; + char stdio_buf[STDIO_BUFSIZE]; + pg_crc32 content_crc = 0; join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc); + + fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); + if (fp == NULL) + elog(ERROR, "cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); + + /* enable stdio buffering for local file */ + if (!fio_is_remote(FIO_BACKUP_HOST)) + setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); + + files = parray_new(); + + INIT_FILE_CRC32(true, content_crc); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + external_dir_num, + crc, + segno, + n_blocks, + n_headers, + dbOid, /* used for partial restore */ + hdr_crc, + hdr_off, + hdr_size; + pgFile *file; + + COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + + get_control_value(buf, "path", path, NULL, true); + get_control_value(buf, "size", NULL, &write_size, true); + get_control_value(buf, "mode", NULL, &mode, true); + get_control_value(buf, "is_datafile", NULL, &is_datafile, true); + get_control_value(buf, "is_cfs", NULL, &is_cfs, false); + get_control_value(buf, "crc", NULL, &crc, true); + get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); + get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); + get_control_value(buf, "dbOid", NULL, &dbOid, false); + + file = pgFileInit(path); + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + file->external_dir_num = external_dir_num; + file->dbOid = dbOid ? dbOid : 0; + + /* + * Optional fields + */ + if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + { + file->linked = pgut_strdup(linked); + canonicalize_path(file->linked); + } + + if (get_control_value(buf, "segno", NULL, &segno, false)) + file->segno = (int) segno; + + if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) + file->n_headers = (int) n_headers; + + if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) + file->hdr_crc = (pg_crc32) hdr_crc; + + if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) + file->hdr_off = hdr_off; + + if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false)) + file->hdr_size = (int) hdr_size; + + parray_append(files, file); + } + + FIN_FILE_CRC32(true, content_crc); + + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); + + fio_close_stream(fp); + + if (backup->content_crc != 0 && + backup->content_crc != content_crc) + { + elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", + backup_filelist_path, content_crc, backup->content_crc); + return NULL; + } /* redundant sanity? */ if (!files) diff --git a/src/dir.c b/src/dir.c index 2bcd87b57..1573f6880 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1430,7 +1430,7 @@ get_external_remap(char *current_dir) * * Returns true if the value was found in the line. */ -static bool +bool get_control_value(const char *str, const char *name, char *value_str, int64 *value_int64, bool is_mandatory) { @@ -1554,123 +1554,6 @@ get_control_value(const char *str, const char *name, return false; /* Make compiler happy */ } -/* - * Construct parray of pgFile from the backup content list. - * If root is not NULL, path will be absolute path. - */ -parray * -dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location, pg_crc32 expected_crc) -{ - FILE *fp; - parray *files; - char buf[BLCKSZ]; - char stdio_buf[STDIO_BUFSIZE]; - pg_crc32 content_crc = 0; - - fp = fio_open_stream(file_txt, location); - if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", file_txt, strerror(errno)); - - /* enable stdio buffering for local file */ - if (!fio_is_remote(location)) - setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); - - files = parray_new(); - - INIT_FILE_CRC32(true, content_crc); - - while (fgets(buf, lengthof(buf), fp)) - { - char path[MAXPGPATH]; - char linked[MAXPGPATH]; - char compress_alg_string[MAXPGPATH]; - int64 write_size, - mode, /* bit length of mode_t depends on platforms */ - is_datafile, - is_cfs, - external_dir_num, - crc, - segno, - n_blocks, - n_headers, - dbOid, /* used for partial restore */ - hdr_crc, - hdr_off, - hdr_size; - pgFile *file; - - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); - - get_control_value(buf, "path", path, NULL, true); - get_control_value(buf, "size", NULL, &write_size, true); - get_control_value(buf, "mode", NULL, &mode, true); - get_control_value(buf, "is_datafile", NULL, &is_datafile, true); - get_control_value(buf, "is_cfs", NULL, &is_cfs, false); - get_control_value(buf, "crc", NULL, &crc, true); - get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); - get_control_value(buf, "dbOid", NULL, &dbOid, false); - - file = pgFileInit(path); - file->write_size = (int64) write_size; - file->mode = (mode_t) mode; - file->is_datafile = is_datafile ? true : false; - file->is_cfs = is_cfs ? true : false; - file->crc = (pg_crc32) crc; - file->compress_alg = parse_compress_alg(compress_alg_string); - file->external_dir_num = external_dir_num; - file->dbOid = dbOid ? dbOid : 0; - - /* - * Optional fields - */ - - if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) - { - file->linked = pgut_strdup(linked); - canonicalize_path(file->linked); - } - - if (get_control_value(buf, "segno", NULL, &segno, false)) - file->segno = (int) segno; - - if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) - file->n_blocks = (int) n_blocks; - - if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) - file->n_headers = (int) n_headers; - - if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) - file->hdr_crc = (pg_crc32) hdr_crc; - - if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) - file->hdr_off = hdr_off; - - if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false)) - file->hdr_size = (int) hdr_size; - - parray_append(files, file); - } - - FIN_FILE_CRC32(true, content_crc); - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", file_txt); - - fio_close_stream(fp); - - if (expected_crc != 0 && - expected_crc != content_crc) - { - elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", - file_txt, content_crc, expected_crc); - return NULL; - } - - return files; -} - /* * Check if directory empty. */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 06fab18dd..2e72fe864 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -975,6 +975,8 @@ extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); /* in dir.c */ +extern bool get_control_value(const char *str, const char *name, + char *value_str, int64 *value_int64, bool is_mandatory); extern void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location); @@ -1000,8 +1002,6 @@ extern void db_map_entry_free(void *map); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *external_prefix, parray *external_list); -extern parray *dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location, pg_crc32 expected_crc); extern parray *make_external_directory_list(const char *colon_separated_dirs, bool remap); extern void free_dir_list(parray *list); From 53be6243f96aeb9e94f3d04ed57cfc1a34cfff40 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 2 Feb 2021 20:17:31 +0300 Subject: [PATCH 1598/2107] tests: remove debug messages in module "incr_restore" --- tests/incr_restore.py | 99 +++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index add485b3c..3aa84121f 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -70,9 +70,9 @@ def test_basic_incr_restore(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -119,9 +119,9 @@ def test_basic_incr_restore_into_missing_directory(self): node.cleanup() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -453,7 +453,6 @@ def test_incr_restore_with_tablespace_4(self): "Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - print(e.message) self.assertIn( 'WARNING: Backup catalog was initialized for system id', e.message, @@ -649,8 +648,6 @@ def test_incr_restore_with_tablespace_7(self): options=[ "-j", "4", "--incremental-mode=checksum"]) - print(out) - pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -800,9 +797,9 @@ def test_incr_checksum_restore(self): pgdata = self.pgdata_content(node_1.data_dir) - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -890,8 +887,8 @@ def test_incr_lsn_restore(self): pgdata = self.pgdata_content(node_1.data_dir) - print(self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"])) + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1079,9 +1076,9 @@ def test_incr_checksum_corruption_detection(self): f.flush() f.close - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1209,9 +1206,9 @@ def test_incr_restore_multiple_external(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=checksum', '--log-level-console=VERBOSE'])) + options=["-j", "4", '--incremental-mode=checksum']) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1281,9 +1278,9 @@ def test_incr_lsn_restore_multiple_external(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=lsn'])) + options=["-j", "4", '--incremental-mode=lsn']) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1341,11 +1338,13 @@ def test_incr_lsn_restore_backward(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=[ - "-j", "4", '--incremental-mode=lsn', '--log-level-file=VERBOSE', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(full_pgdata, pgdata_restored) @@ -1384,11 +1383,13 @@ def test_incr_lsn_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=delta_id, options=[ - "-j", "4", '--incremental-mode=lsn', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) @@ -1447,11 +1448,13 @@ def test_incr_checksum_restore_backward(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(full_pgdata, pgdata_restored) @@ -1459,11 +1462,13 @@ def test_incr_checksum_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=page_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(page_pgdata, pgdata_restored) @@ -1471,11 +1476,13 @@ def test_incr_checksum_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=delta_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) @@ -1542,9 +1549,9 @@ def test_make_replica_via_incr_checksum_restore(self): data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - print(self.restore_node( + self.restore_node( backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=checksum'])) + options=['-R', '--incremental-mode=checksum']) self.set_replica(new_master, old_master, synchronous=True) @@ -1615,9 +1622,9 @@ def test_make_replica_via_incr_lsn_restore(self): data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - print(self.restore_node( + self.restore_node( backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=lsn'])) + options=['-R', '--incremental-mode=lsn']) self.set_replica(new_master, old_master, synchronous=True) @@ -1762,9 +1769,9 @@ def test_incr_lsn_long_xact_1(self): node.stop() try: - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn'])) + options=["-j", "4", '--incremental-mode=lsn']) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -1920,9 +1927,9 @@ def test_incr_restore_zero_size_file_checksum(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum'])) + options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) @@ -1994,9 +2001,9 @@ def test_incr_restore_zero_size_file_lsn(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum'])) + options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) @@ -2088,12 +2095,12 @@ def test_incremental_partial_restore_exclude_checksum(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore backup into node2 - print(self.restore_node( + self.restore_node( backup_dir, 'node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "checksum"])) + "-I", "checksum"]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -2198,12 +2205,12 @@ def test_incremental_partial_restore_exclude_lsn(self): node2.port = node.port node2.slow_start() node2.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "lsn"])) + "-I", "lsn"]) pgdata2 = self.pgdata_content(node2.data_dir) From da2c49dfe1807238f3beb733ac0f1a951e09fe94 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 2 Feb 2021 22:41:45 +0300 Subject: [PATCH 1599/2107] code cleanup --- src/dir.c | 2 +- src/pg_probackup.c | 2 +- src/pg_probackup.h | 17 ----------------- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/dir.c b/src/dir.c index 1573f6880..0b724036a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -28,7 +28,7 @@ * start so they are not included in backups. The directories themselves are * kept and included as empty to preserve access permissions. */ -const char *pgdata_exclude_dir[] = +static const char *pgdata_exclude_dir[] = { PG_XLOG_DIR, /* diff --git a/src/pg_probackup.c b/src/pg_probackup.c index dd2ac97ee..37d872309 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -104,7 +104,7 @@ bool force = false; bool dry_run = false; static char *delete_status = NULL; /* compression options */ -bool compress_shortcut = false; +static bool compress_shortcut = false; /* other options */ char *instance_name; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2e72fe864..18bf87bbc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -672,16 +672,6 @@ typedef struct BackupPageHeader2 #define PageIsTruncated -2 #define PageIsCorrupted -3 /* used by checkdb */ - -/* - * return pointer that exceeds the length of prefix from character string. - * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". - * - * Deprecated. Do not use this in new code. - */ -#define GetRelativePath(str, prefix) \ - ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) - /* * Return timeline, xlog ID and record offset from an LSN of the type * 0/B000188, usual result from pg_stop_backup() and friends. @@ -789,9 +779,6 @@ extern bool delete_expired; extern bool merge_expired; extern bool dry_run; -/* compression options */ -extern bool compress_shortcut; - /* other options */ extern char *instance_name; @@ -808,10 +795,6 @@ extern pgBackup current; /* argv of the process */ extern char** commands_args; -/* in dir.c */ -/* exclude directory list for $PGDATA file listing */ -extern const char *pgdata_exclude_dir[]; - /* in backup.c */ extern int do_backup(pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs); From 94ada4c13745bc2e48a38a6d4b027bbbcbe9e644 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 3 Feb 2021 01:41:49 +0300 Subject: [PATCH 1600/2107] [Issue #308] test coverage and comments improvement --- src/catalog.c | 49 ++++++++------------------------- tests/helpers/ptrack_helpers.py | 14 +++++----- tests/locking.py | 46 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 63bb6862e..8ac3e5799 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -166,6 +166,9 @@ write_backup_status(pgBackup *backup, BackupStatus status, * * TODO: lock-timeout as parameter * TODO: we must think about more fine grain unlock mechanism - separate unlock_backup() function. + * TODO: more accurate naming + * -> exclusive lock -> acquire HW_LATCH and wait until all LW_LATCH`es are clear + * -> shared lock -> acquire HW_LATCH, acquire LW_LATCH, release HW_LATCH */ bool lock_backup(pgBackup *backup, bool strict, bool exclusive) @@ -264,45 +267,13 @@ lock_backup_exclusive(pgBackup *backup, bool strict) int empty_tries = LOCK_STALE_TIMEOUT; int len; int encoded_pid; - pid_t my_p_pid; join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); - /* - * TODO: is this stuff with ppid below is relevant for us ? - * - * If the PID in the lockfile is our own PID or our parent's or - * grandparent's PID, then the file must be stale (probably left over from - * a previous system boot cycle). We need to check this because of the - * likelihood that a reboot will assign exactly the same PID as we had in - * the previous reboot, or one that's only one or two counts larger and - * hence the lockfile's PID now refers to an ancestor shell process. We - * allow pg_ctl to pass down its parent shell PID (our grandparent PID) - * via the environment variable PG_GRANDPARENT_PID; this is so that - * launching the postmaster via pg_ctl can be just as reliable as - * launching it directly. There is no provision for detecting - * further-removed ancestor processes, but if the init script is written - * carefully then all but the immediate parent shell will be root-owned - * processes and so the kill test will fail with EPERM. Note that we - * cannot get a false negative this way, because an existing postmaster - * would surely never launch a competing postmaster or pg_ctl process - * directly. - */ -#ifndef WIN32 - my_p_pid = getppid(); -#else - - /* - * Windows hasn't got getppid(), but doesn't need it since it's not using - * real kill() either... - */ - my_p_pid = 0; -#endif - /* * We need a loop here because of race conditions. But don't loop forever * (for example, a non-writable $backup_instance_path directory might cause a failure - * that won't go away). 100 tries seems like plenty. + * that won't go away). */ do { @@ -396,14 +367,12 @@ lock_backup_exclusive(pgBackup *backup, bool strict) /* * Check to see if the other process still exists - * - * Per discussion above, my_pid, my_p_pid can be - * ignored as false matches. - * * Normally kill() will fail with ESRCH if the given PID doesn't * exist. */ - if (encoded_pid != my_pid && encoded_pid != my_p_pid) + if (encoded_pid == my_pid) + return 0; + else { if (kill(encoded_pid, 0) == 0) { @@ -508,6 +477,10 @@ lock_backup_exclusive(pgBackup *backup, bool strict) lock_file, strerror(save_errno)); } +// elog(LOG, "Acquired exclusive lock for backup %s after %ds", +// base36enc(backup->start_time), +// LOCK_TIMEOUT - ntries + LOCK_STALE_TIMEOUT - empty_tries); + return 0; } diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 3c75ca2e7..833e95a36 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -757,7 +757,7 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur return GDBobj([binary_path] + command, self.verbose) if asynchronous: return subprocess.Popen( - self.cmd, + [binary_path] + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env @@ -1133,8 +1133,8 @@ def show_archive( exit(1) def validate_pb( - self, backup_dir, instance=None, - backup_id=None, options=[], old_binary=False, gdb=False + self, backup_dir, instance=None, backup_id=None, + options=[], old_binary=False, gdb=False, asynchronous=False ): cmd_list = [ @@ -1146,11 +1146,11 @@ def validate_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) def delete_pb( - self, backup_dir, instance, - backup_id=None, options=[], old_binary=False, gdb=False): + self, backup_dir, instance, backup_id=None, + options=[], old_binary=False, gdb=False, asynchronous=False): cmd_list = [ 'delete', '-B', backup_dir @@ -1160,7 +1160,7 @@ def delete_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) def delete_expired( self, backup_dir, instance, options=[], old_binary=False): diff --git a/tests/locking.py b/tests/locking.py index 92c779c8a..540007838 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -535,6 +535,52 @@ def test_backup_directory_name(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_empty_lock_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/308 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=100) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + with open(lockfile, "w+") as f: + f.truncate() + + out = self.validate_pb(backup_dir, 'node', backup_id) + + self.assertIn( + "Waiting 30 seconds on empty exclusive lock for backup", out) + +# lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') +# with open(lockfile, "w+") as f: +# f.truncate() +# +# p1 = self.validate_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=validate.log']) +# sleep(3) +# p2 = self.delete_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=delete.log']) +# +# p1.wait() +# p2.wait() + + # Clean after yourself + self.del_test_dir(module_name, fname) + # TODO: # test that concurrent validation and restore are not locking each other # check that quick exclusive lock, when taking RO-lock, is really quick From c1e81edd49fdd93a0b6943b62fdf0b4c98ed828c Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 13:03:46 +0300 Subject: [PATCH 1601/2107] Code cleanup. Start removing global variables in pg_probackup.c --- src/init.c | 14 +++++++------- src/pg_probackup.c | 10 +++++++++- src/pg_probackup.h | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/init.c b/src/init.c index 1ab6dc0f9..51e628f0e 100644 --- a/src/init.c +++ b/src/init.c @@ -17,34 +17,34 @@ * Initialize backup catalog. */ int -do_init(void) +do_init(char *backup_catalog_path) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; int results; - results = pg_check_dir(backup_path); + results = pg_check_dir(backup_catalog_path); if (results == 4) /* exists and not empty*/ elog(ERROR, "backup catalog already exist and it's not empty"); else if (results == -1) /*trouble accessing directory*/ { int errno_tmp = errno; elog(ERROR, "cannot open backup catalog directory \"%s\": %s", - backup_path, strerror(errno_tmp)); + backup_catalog_path, strerror(errno_tmp)); } /* create backup catalog root directory */ - dir_create_dir(backup_path, DIR_PERMISSION, false); + dir_create_dir(backup_catalog_path, DIR_PERMISSION, false); /* create backup catalog data directory */ - join_path_components(path, backup_path, BACKUPS_DIR); + join_path_components(path, backup_catalog_path, BACKUPS_DIR); dir_create_dir(path, DIR_PERMISSION, false); /* create backup catalog wal directory */ - join_path_components(arclog_path_dir, backup_path, "wal"); + join_path_components(arclog_path_dir, backup_catalog_path, "wal"); dir_create_dir(arclog_path_dir, DIR_PERMISSION, false); - elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + elog(INFO, "Backup catalog '%s' successfully inited", backup_catalog_path); return 0; } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 37d872309..5479f78c7 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -2,6 +2,13 @@ * * pg_probackup.c: Backup/Recovery manager for PostgreSQL. * + * This is an entry point for the program. + * Parse command name and it's options, verify them and call a + * do_***() function that implements the command. + * + * Avoid using global variables in the code. + * Pass all needed information as funciton arguments. + * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION * Portions Copyright (c) 2015-2019, Postgres Professional * @@ -28,6 +35,7 @@ const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; /* directory options */ +/* TODO make it local variable, pass as an argument to all commands that need it. */ char *backup_path = NULL; /* * path or to the data files in the backup catalog @@ -739,7 +747,7 @@ main(int argc, char *argv[]) case DELETE_INSTANCE_CMD: return do_delete_instance(); case INIT_CMD: - return do_init(); + return do_init(backup_path); case BACKUP_CMD: { current.stream = stream_wal; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 18bf87bbc..b169b2237 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -839,7 +839,7 @@ extern void merge_chain(parray *parent_chain, extern parray *read_database_map(pgBackup *backup); /* in init.c */ -extern int do_init(void); +extern int do_init(char *backup_catalog_path); extern int do_add_instance(InstanceConfig *instance); /* in archive.c */ From 24a7a085c379e511910ff7b4407487e0e77f8a86 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 13:41:01 +0300 Subject: [PATCH 1602/2107] Code cleanup. Pass backup_catalog_path explicitly to functions that need it --- src/archive.c | 2 +- src/backup.c | 2 +- src/catalog.c | 4 ++-- src/init.c | 10 +++++----- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 9 +++++---- src/show.c | 7 ++++--- src/validate.c | 8 ++++---- 8 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/archive.c b/src/archive.c index 2d858a64c..28622cc57 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1046,7 +1046,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, join_path_components(absolute_wal_file_path, current_dir, wal_file_path); /* full filepath to WAL file in archive directory. - * backup_path/wal/instance_name/000000010000000000000001 */ + * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); INSTR_TIME_SET_CURRENT(start_time); diff --git a/src/backup.c b/src/backup.c index 0b61234f1..c84b1b7d5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -89,7 +89,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) /* * Take a backup of a single postgresql instance. - * Move files from 'pgdata' to a subdirectory in 'backup_path'. + * Move files from 'pgdata' to a subdirectory in backup catalog. */ static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) diff --git a/src/catalog.c b/src/catalog.c index 11b9a27d1..4d4df123d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -692,7 +692,7 @@ IsDir(const char *dirpath, const char *entry, fio_location location) * actual config of each instance. */ parray * -catalog_get_instance_list(void) +catalog_get_instance_list(char *backup_catalog_path) { char path[MAXPGPATH]; DIR *dir; @@ -702,7 +702,7 @@ catalog_get_instance_list(void) instances = parray_new(); /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); + join_path_components(path, backup_catalog_path, BACKUPS_DIR); dir = opendir(path); if (dir == NULL) elog(ERROR, "Cannot open directory \"%s\": %s", diff --git a/src/init.c b/src/init.c index 51e628f0e..255f5425d 100644 --- a/src/init.c +++ b/src/init.c @@ -49,7 +49,7 @@ do_init(char *backup_catalog_path) } int -do_add_instance(InstanceConfig *instance) +do_add_instance(char *backup_catalog_path, InstanceConfig *instance) { char path[MAXPGPATH]; char arclog_path_dir[MAXPGPATH]; @@ -66,14 +66,14 @@ do_add_instance(InstanceConfig *instance) instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); /* Ensure that all root directories already exist */ - if (access(backup_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", backup_path); + if (access(backup_catalog_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", backup_catalog_path); - join_path_components(path, backup_path, BACKUPS_DIR); + join_path_components(path, backup_catalog_path, BACKUPS_DIR); if (access(path, F_OK) != 0) elog(ERROR, "Directory does not exist: '%s'", path); - join_path_components(arclog_path_dir, backup_path, "wal"); + join_path_components(arclog_path_dir, backup_catalog_path, "wal"); if (access(arclog_path_dir, F_OK) != 0) elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5479f78c7..8a52ba515 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -743,7 +743,7 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - return do_add_instance(&instance_config); + return do_add_instance(backup_path, &instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(); case INIT_CMD: @@ -770,7 +770,7 @@ main(int argc, char *argv[]) if (datname_exclude_list || datname_include_list) elog(ERROR, "You must specify parameter (-i, --backup-id) for partial validation"); - return do_validate_all(); + return do_validate_all(backup_path); } else /* PITR validation and, optionally, partial validation */ @@ -779,7 +779,7 @@ main(int argc, char *argv[]) restore_params, no_sync); case SHOW_CMD: - return do_show(instance_name, current.backup_id, show_archive); + return do_show(backup_path, instance_name, current.backup_id, show_archive); case DELETE_CMD: if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b169b2237..a0120dc8a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -840,7 +840,7 @@ extern parray *read_database_map(pgBackup *backup); /* in init.c */ extern int do_init(char *backup_catalog_path); -extern int do_add_instance(InstanceConfig *instance); +extern int do_add_instance(char *backup_catalog_path, InstanceConfig *instance); /* in archive.c */ extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, @@ -856,7 +856,8 @@ extern void init_config(InstanceConfig *config, const char *instance_name); extern InstanceConfig *readInstanceConfigFile(const char *instance_name); /* in show.c */ -extern int do_show(const char *instance_name, time_t requested_backup_id, bool show_archive); +extern int do_show(char *backup_catalog_path, const char *instance_name, + time_t requested_backup_id, bool show_archive); /* in delete.c */ extern void do_delete(time_t backup_id); @@ -880,7 +881,7 @@ extern void help_command(ProbackupSubcmd const subcmd); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); -extern int do_validate_all(void); +extern int do_validate_all(char *backup_catalog_path); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); @@ -906,7 +907,7 @@ extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color); extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode); -extern parray *catalog_get_instance_list(void); +extern parray *catalog_get_instance_list(char *backup_catalog_path); extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict, bool exclusive); diff --git a/src/show.c b/src/show.c index 61bde9ef3..c88f4fae9 100644 --- a/src/show.c +++ b/src/show.c @@ -75,7 +75,8 @@ static int32 json_level = 0; * Entry point of pg_probackup SHOW subcommand. */ int -do_show(const char *instance_name, time_t requested_backup_id, bool show_archive) +do_show(char *backup_catalog_path, const char *instance_name, + time_t requested_backup_id, bool show_archive) { int i; @@ -93,7 +94,7 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive */ if (instance_name == NULL) { - parray *instances = catalog_get_instance_list(); + parray *instances = catalog_get_instance_list(backup_catalog_path); show_instance_start(); for (i = 0; i < parray_num(instances); i++) @@ -104,7 +105,7 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive if (interrupted) elog(ERROR, "Interrupted during show"); - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance->name); + sprintf(backup_instance_path, "%s/%s/%s", backup_catalog_path, BACKUPS_DIR, instance->name); if (show_archive) show_instance_archive(instance); diff --git a/src/validate.c b/src/validate.c index b44f4b1b8..40c94af67 100644 --- a/src/validate.c +++ b/src/validate.c @@ -382,7 +382,7 @@ pgBackupValidateFiles(void *arg) * If --instance option was provided, validate only backups of this instance. */ int -do_validate_all(void) +do_validate_all(char *backup_catalog_path) { corrupted_backup_found = false; skipped_due_to_lock = false; @@ -395,7 +395,7 @@ do_validate_all(void) struct dirent *dent; /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); + join_path_components(path, backup_catalog_path, BACKUPS_DIR); dir = opendir(path); if (dir == NULL) elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); @@ -425,8 +425,8 @@ do_validate_all(void) */ instance_name = dent->d_name; sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); + backup_catalog_path, BACKUPS_DIR, instance_name); + sprintf(arclog_path, "%s/%s/%s", backup_catalog_path, "wal", instance_name); join_path_components(conf_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); if (config_read_opt(conf_path, instance_options, ERROR, false, From 7ef802d02f7c14a0a3f54729ddcb0f3e0f3bd5cc Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 14:21:21 +0300 Subject: [PATCH 1603/2107] Add a comment declaring code refactoring plan --- src/pg_probackup.c | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 8a52ba515..8e09bccfc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -7,7 +7,29 @@ * do_***() function that implements the command. * * Avoid using global variables in the code. - * Pass all needed information as funciton arguments. + * Pass all needed information as funciton arguments: + * + + * + * TODO: + * + * Functions that work with a backup catalog accept catalogState, + * which currently only contains pathes to backup catalog subdirectories + * + function specific options. + * + * Functions that work with an instance accept instanceState argument, which + * includes catalogState, instance_name, + * info about pgdata associated with the instance, + * various instance config options, and list of backups belonging to the instance. + * + function specific options. + * + * Functions that work with multiple backups in the catalog + * accept instanceState and info needed to determine the range of backups to handle. + * + function specific options. + * + * Functions that work with a single backup accept backupState argument, + * which includes link to the instanceState, backup_id and backup-specific info. + * + function specific options. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION * Portions Copyright (c) 2015-2019, Postgres Professional @@ -34,6 +56,7 @@ const char *PROGRAM_FULL_PATH = NULL; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; +/* ================ catalogState =========== */ /* directory options */ /* TODO make it local variable, pass as an argument to all commands that need it. */ char *backup_path = NULL; @@ -48,10 +71,13 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; +/* ================ catalogState (END) =========== */ + + + /* colon separated external directories list ("/path1:/path2") */ char *externaldir = NULL; /* common options */ -static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; bool no_color = false; @@ -114,8 +140,9 @@ static char *delete_status = NULL; /* compression options */ static bool compress_shortcut = false; -/* other options */ +/* ================ instanceState =========== */ char *instance_name; +/* ================ instanceState (END) =========== */ /* archive push options */ int batch_size = 1; @@ -137,8 +164,10 @@ int64 ttl = -1; static char *expire_time_string = NULL; static pgSetBackupParams *set_backup_params = NULL; -/* current settings */ +/* ================ backupState =========== */ +static char *backup_id_string = NULL; pgBackup current; +/* ================ backupState (END) =========== */ static bool help_opt = false; From 2284c2b0069dcd6feadef524c1b8bd72a17ed5cb Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 15:14:00 +0300 Subject: [PATCH 1604/2107] Refactoring. Introduce catalogState structure. Update code in init.c --- src/init.c | 38 ++++++++++++++++---------------------- src/pg_probackup.c | 24 +++++++++++++++++++----- src/pg_probackup.h | 20 ++++++++++++++++++-- 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/init.c b/src/init.c index 255f5425d..07de8a98f 100644 --- a/src/init.c +++ b/src/init.c @@ -17,42 +17,37 @@ * Initialize backup catalog. */ int -do_init(char *backup_catalog_path) +do_init(CatalogState *catalogState) { - char path[MAXPGPATH]; - char arclog_path_dir[MAXPGPATH]; int results; - results = pg_check_dir(backup_catalog_path); + results = pg_check_dir(catalogState->catalog_path); + if (results == 4) /* exists and not empty*/ elog(ERROR, "backup catalog already exist and it's not empty"); else if (results == -1) /*trouble accessing directory*/ { int errno_tmp = errno; elog(ERROR, "cannot open backup catalog directory \"%s\": %s", - backup_catalog_path, strerror(errno_tmp)); + catalogState->catalog_path, strerror(errno_tmp)); } /* create backup catalog root directory */ - dir_create_dir(backup_catalog_path, DIR_PERMISSION, false); + dir_create_dir(catalogState->catalog_path, DIR_PERMISSION, false); /* create backup catalog data directory */ - join_path_components(path, backup_catalog_path, BACKUPS_DIR); - dir_create_dir(path, DIR_PERMISSION, false); + dir_create_dir(catalogState->backup_subdir_path, DIR_PERMISSION, false); /* create backup catalog wal directory */ - join_path_components(arclog_path_dir, backup_catalog_path, "wal"); - dir_create_dir(arclog_path_dir, DIR_PERMISSION, false); + dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); - elog(INFO, "Backup catalog '%s' successfully inited", backup_catalog_path); + elog(INFO, "Backup catalog '%s' successfully inited", catalogState->catalog_path); return 0; } int -do_add_instance(char *backup_catalog_path, InstanceConfig *instance) +do_add_instance(CatalogState *catalogState, InstanceConfig *instance) { - char path[MAXPGPATH]; - char arclog_path_dir[MAXPGPATH]; struct stat st; /* PGDATA is always required */ @@ -66,16 +61,15 @@ do_add_instance(char *backup_catalog_path, InstanceConfig *instance) instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); /* Ensure that all root directories already exist */ - if (access(backup_catalog_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", backup_catalog_path); + /* TODO maybe call do_init() here instead of error?*/ + if (access(catalogState->catalog_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->catalog_path); - join_path_components(path, backup_catalog_path, BACKUPS_DIR); - if (access(path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", path); + if (access(catalogState->backup_subdir_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->backup_subdir_path); - join_path_components(arclog_path_dir, backup_catalog_path, "wal"); - if (access(arclog_path_dir, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir); + if (access(catalogState->wal_subdir_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->wal_subdir_path); if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' backup directory already exists: '%s'", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 8e09bccfc..04693bdff 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -71,9 +71,9 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; -/* ================ catalogState (END) =========== */ - +static CatalogState *catalogState = NULL; +/* ================ catalogState (END) =========== */ /* colon separated external directories list ("/path1:/path2") */ char *externaldir = NULL; @@ -424,6 +424,7 @@ main(int argc, char *argv[]) /* set location based on cmdline options only */ setMyLocation(backup_subcmd); + /* ===== catalogState ======*/ if (backup_path == NULL) { /* @@ -440,11 +441,24 @@ main(int argc, char *argv[]) /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); + + catalogState = pgut_new(CatalogState); + strncpy(catalogState->catalog_path, backup_path, MAXPGPATH); + join_path_components(catalogState->backup_subdir_path, + catalogState->catalog_path, BACKUPS_DIR); + join_path_components(catalogState->wal_subdir_path, + catalogState->catalog_path, WAL_SUBDIR); } + /* backup_path is required for all pg_probackup commands except help, version and checkdb */ - if (backup_path == NULL && backup_subcmd != CHECKDB_CMD && backup_subcmd != HELP_CMD && backup_subcmd != VERSION_CMD) + if (backup_path == NULL && + backup_subcmd != CHECKDB_CMD && + backup_subcmd != HELP_CMD && + backup_subcmd != VERSION_CMD) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + /* ===== catalogState (END) ======*/ + /* * Option --instance is required for all commands except * init, show, checkdb and validate @@ -772,11 +786,11 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - return do_add_instance(backup_path, &instance_config); + return do_add_instance(catalogState, &instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(); case INIT_CMD: - return do_init(backup_path); + return do_init(catalogState); case BACKUP_CMD: { current.stream = stream_wal; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a0120dc8a..46575beae 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -55,6 +55,7 @@ extern const char *PROGRAM_EMAIL; /* Directory/File names */ #define DATABASE_DIR "database" #define BACKUPS_DIR "backups" +#define WAL_SUBDIR "wal" #if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" #define PG_LOG_DIR "log" @@ -129,6 +130,7 @@ extern const char *PROGRAM_EMAIL; #define TC_CYAN_BOLD "\033[1;36m" #define TC_RESET "\033[0m" + typedef struct RedoParams { TimeLineID tli; @@ -746,11 +748,25 @@ typedef struct BackupPageHeader2 #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) +/* ====== CatalogState ======= */ + /* directory options */ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; +typedef struct CatalogState +{ + /* $BACKUP_PATH */ + char catalog_path[MAXPGPATH]; //previously global var backup_path + /* $BACKUP_PATH/backups */ + char backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/wal */ + char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path +} CatalogState; + +/* ====== CatalogState (END) ======= */ + /* common options */ extern pid_t my_pid; extern __thread int my_thread_num; @@ -839,8 +855,8 @@ extern void merge_chain(parray *parent_chain, extern parray *read_database_map(pgBackup *backup); /* in init.c */ -extern int do_init(char *backup_catalog_path); -extern int do_add_instance(char *backup_catalog_path, InstanceConfig *instance); +extern int do_init(CatalogState *catalogState); +extern int do_add_instance(CatalogState *catalogState, InstanceConfig *instance); /* in archive.c */ extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, From 48b8f60fdbd89d83d8652f7ff29529cdeac73f57 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 17:35:14 +0300 Subject: [PATCH 1605/2107] Refactoring. Introduce instanceState structure. Update code in init.c --- src/init.c | 15 ++++++++------- src/pg_probackup.c | 20 +++++++++++++++++++- src/pg_probackup.h | 19 +++++++++++++++++-- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/init.c b/src/init.c index 07de8a98f..aa376389d 100644 --- a/src/init.c +++ b/src/init.c @@ -46,9 +46,10 @@ do_init(CatalogState *catalogState) } int -do_add_instance(CatalogState *catalogState, InstanceConfig *instance) +do_add_instance(InstanceState *instanceState, InstanceConfig *instance) { struct stat st; + CatalogState *catalogState = instanceState->catalog_state; /* PGDATA is always required */ if (instance->pgdata == NULL) @@ -71,22 +72,22 @@ do_add_instance(CatalogState *catalogState, InstanceConfig *instance) if (access(catalogState->wal_subdir_path, F_OK) != 0) elog(ERROR, "Directory does not exist: '%s'", catalogState->wal_subdir_path); - if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + if (stat(instanceState->instance_backup_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' backup directory already exists: '%s'", - instance->name, instance->backup_instance_path); + instanceState->instance_name, instanceState->instance_backup_subdir_path); /* * Create directory for wal files of this specific instance. * Existence check is extra paranoid because if we don't have such a * directory in data dir, we shouldn't have it in wal as well. */ - if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + if (stat(instanceState->instance_wal_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'", - instance->name, instance->arclog_path); + instanceState->instance_name, instanceState->instance_wal_subdir_path); /* Create directory for data files of this specific instance */ - dir_create_dir(instance->backup_instance_path, DIR_PERMISSION, false); - dir_create_dir(instance->arclog_path, DIR_PERMISSION, false); + dir_create_dir(instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); + dir_create_dir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); /* * Write initial configuration file. diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 04693bdff..ae5555beb 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -142,6 +142,9 @@ static bool compress_shortcut = false; /* ================ instanceState =========== */ char *instance_name; + +static InstanceState *instanceState = NULL; + /* ================ instanceState (END) =========== */ /* archive push options */ @@ -292,6 +295,7 @@ main(int argc, char *argv[]) pgBackupInit(¤t); /* Initialize current instance configuration */ + //TODO get git of this global variable craziness init_config(&instance_config, instance_name); PROGRAM_NAME = get_progname(argv[0]); @@ -459,6 +463,8 @@ main(int argc, char *argv[]) /* ===== catalogState (END) ======*/ + /* ===== instanceState ======*/ + /* * Option --instance is required for all commands except * init, show, checkdb and validate @@ -470,9 +476,21 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: --instance"); } else + { /* Set instance name */ instance_config.name = pgut_strdup(instance_name); + instanceState = pgut_new(InstanceState); + instanceState->catalog_state = catalogState; + + strncpy(instanceState->instance_name, instance_name, MAXPGPATH); + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instance_name); + } + /* ===== instanceState (END) ======*/ + /* * If --instance option was passed, construct paths for backup data and * xlog files of this backup instance. @@ -786,7 +804,7 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - return do_add_instance(catalogState, &instance_config); + return do_add_instance(instanceState, &instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(); case INIT_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 46575beae..635739a59 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -795,8 +795,23 @@ extern bool delete_expired; extern bool merge_expired; extern bool dry_run; -/* other options */ +/* ===== instanceState ===== */ extern char *instance_name; +typedef struct InstanceState +{ + /* catalog, this instance belongs to */ + CatalogState *catalog_state; + + char instance_name[MAXPGPATH]; //previously global var instance_name + /* $BACKUP_PATH/backups/instance_name */ + char instance_backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/backups/instance_name */ + char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path + + //TODO add config here +} InstanceState; + +/* ===== instanceState (END) ===== */ /* show options */ extern ShowFormat show_format; @@ -856,7 +871,7 @@ extern parray *read_database_map(pgBackup *backup); /* in init.c */ extern int do_init(CatalogState *catalogState); -extern int do_add_instance(CatalogState *catalogState, InstanceConfig *instance); +extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance); /* in archive.c */ extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, From 3d62e2a1d612c80d313b45c2c5f68480ddaf0bb8 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 17:48:59 +0300 Subject: [PATCH 1606/2107] Refactoting. Move new state definitions to a separate header --- src/pg_probackup.c | 3 ++- src/pg_probackup.h | 27 ++++-------------------- src/pg_probackup_state.h | 44 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 src/pg_probackup_state.h diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ae5555beb..3b6e8a499 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -11,7 +11,7 @@ * * - * TODO: + * TODO (see pg_probackup_state.h): * * Functions that work with a backup catalog accept catalogState, * which currently only contains pathes to backup catalog subdirectories @@ -38,6 +38,7 @@ */ #include "pg_probackup.h" +#include "pg_probackup_state.h" #include "pg_getopt.h" #include "streamutil.h" diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 635739a59..b5af7b6d4 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -10,6 +10,7 @@ #ifndef PG_PROBACKUP_H #define PG_PROBACKUP_H + #include "postgres_fe.h" #include "libpq-fe.h" #include "libpq-int.h" @@ -39,6 +40,9 @@ #include "datapagemap.h" #include "utils/thread.h" +#include "pg_probackup_state.h" + + #ifdef WIN32 #define __thread __declspec(thread) #else @@ -755,16 +759,6 @@ extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; -typedef struct CatalogState -{ - /* $BACKUP_PATH */ - char catalog_path[MAXPGPATH]; //previously global var backup_path - /* $BACKUP_PATH/backups */ - char backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/wal */ - char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path -} CatalogState; - /* ====== CatalogState (END) ======= */ /* common options */ @@ -797,19 +791,6 @@ extern bool dry_run; /* ===== instanceState ===== */ extern char *instance_name; -typedef struct InstanceState -{ - /* catalog, this instance belongs to */ - CatalogState *catalog_state; - - char instance_name[MAXPGPATH]; //previously global var instance_name - /* $BACKUP_PATH/backups/instance_name */ - char instance_backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/backups/instance_name */ - char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path - - //TODO add config here -} InstanceState; /* ===== instanceState (END) ===== */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h new file mode 100644 index 000000000..ab20a55a3 --- /dev/null +++ b/src/pg_probackup_state.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup_state.h: Definitions of internal pg_probackup states + * + * Portions Copyright (c) 2021-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_STATE_H +#define PG_PROBACKUP_STATE_H + +/* ====== CatalogState ======= */ + +typedef struct CatalogState +{ + /* $BACKUP_PATH */ + char catalog_path[MAXPGPATH]; //previously global var backup_path + /* $BACKUP_PATH/backups */ + char backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/wal */ + char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path +} CatalogState; + +/* ====== CatalogState (END) ======= */ + + +/* ===== instanceState ===== */ +typedef struct InstanceState +{ + /* catalog, this instance belongs to */ + CatalogState *catalog_state; + + char instance_name[MAXPGPATH]; //previously global var instance_name + /* $BACKUP_PATH/backups/instance_name */ + char instance_backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/backups/instance_name */ + char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path + + //TODO add config here +} InstanceState; + +/* ===== instanceState (END) ===== */ + +#endif /* PG_PROBACKUP_STATE_H */ From cc58553514ea4c54ddf3520953caacd2a8da848e Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 18:00:21 +0300 Subject: [PATCH 1607/2107] Refactoring. Cleanup comments --- src/pg_probackup.c | 6 ++++-- src/pg_probackup_state.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3b6e8a499..3d6990945 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -19,7 +19,7 @@ * * Functions that work with an instance accept instanceState argument, which * includes catalogState, instance_name, - * info about pgdata associated with the instance, + * info about pgdata associated with the instance (see pgState), * various instance config options, and list of backups belonging to the instance. * + function specific options. * @@ -31,6 +31,9 @@ * which includes link to the instanceState, backup_id and backup-specific info. * + function specific options. * + * Functions that work with a postgreSQL instance (i.e. checkdb) accept pgState, + * which includes info about pgdata directory and connection. + * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION * Portions Copyright (c) 2015-2019, Postgres Professional * @@ -72,7 +75,6 @@ char backup_instance_path[MAXPGPATH]; */ char arclog_path[MAXPGPATH] = ""; - static CatalogState *catalogState = NULL; /* ================ catalogState (END) =========== */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index ab20a55a3..19e096328 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -2,7 +2,7 @@ * * pg_probackup_state.h: Definitions of internal pg_probackup states * - * Portions Copyright (c) 2021-2018, Postgres Professional + * Portions Copyright (c) 2021, Postgres Professional * *------------------------------------------------------------------------- */ From dbf523308b07ae1d23e511110a0ab3c453559e87 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 18:39:51 +0300 Subject: [PATCH 1608/2107] Refactoring. Use instanceState instead of global variable instance_name --- src/backup.c | 14 +++++------ src/catalog.c | 2 +- src/delete.c | 16 ++++++------- src/init.c | 2 +- src/merge.c | 8 +++---- src/parsexlog.c | 2 +- src/pg_probackup.c | 17 +++++++------- src/pg_probackup.h | 20 +++++++++------- src/restore.c | 10 ++++---- src/validate.c | 58 ++++++++++++++++++++++++++++------------------ 10 files changed, 83 insertions(+), 66 deletions(-) diff --git a/src/backup.c b/src/backup.c index c84b1b7d5..bbd168db0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -47,7 +47,7 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); +static void do_backup_pg(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); @@ -92,7 +92,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) * Move files from 'pgdata' to a subdirectory in backup catalog. */ static void -do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) +do_backup_pg(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) { int i; char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ @@ -724,7 +724,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) * Entry point of pg_probackup BACKUP subcommand. */ int -do_backup(pgSetBackupParams *set_backup_params, +do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs) { PGconn *backup_conn = NULL; @@ -740,7 +740,7 @@ do_backup(pgSetBackupParams *set_backup_params, current.external_dir_str = instance_config.external_dir_str; /* Create backup directory and BACKUP_CONTROL_FILE */ - pgBackupCreateDir(¤t, backup_instance_path); + pgBackupCreateDir(¤t, instanceState->instance_backup_subdir_path); if (!instance_config.pgdata) elog(ERROR, "required parameter not specified: PGDATA " @@ -758,7 +758,7 @@ do_backup(pgSetBackupParams *set_backup_params, elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instance_name, base36enc(current.backup_id), pgBackupGetBackupMode(¤t, false), + PROGRAM_VERSION, instanceState->instance_name, base36enc(current.backup_id), pgBackupGetBackupMode(¤t, false), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); @@ -824,7 +824,7 @@ do_backup(pgSetBackupParams *set_backup_params, add_note(¤t, set_backup_params->note); /* backup data */ - do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs); + do_backup_pg(backup_conn, &nodeInfo, no_sync, backup_logs); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -879,7 +879,7 @@ do_backup(pgSetBackupParams *set_backup_params, * which are expired according to retention policies */ if (delete_expired || merge_expired || delete_wal) - do_retention(); + do_retention(instanceState); return 0; } diff --git a/src/catalog.c b/src/catalog.c index 4d4df123d..f215e90e2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -112,7 +112,7 @@ read_backup(const char *root_dir) */ void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name, bool strict) + bool strict) { pgBackup *tmp; diff --git a/src/delete.c b/src/delete.c index 2130f1a1d..b020716fa 100644 --- a/src/delete.c +++ b/src/delete.c @@ -29,7 +29,7 @@ static bool backup_merged = false; /* At least one merge was enacted */ static bool wal_deleted = false; /* At least one WAL segments was deleted */ void -do_delete(time_t backup_id) +do_delete(InstanceState *instanceState, time_t backup_id) { int i; parray *backup_list, @@ -39,7 +39,7 @@ do_delete(time_t backup_id) char size_to_delete_pretty[20]; /* Get complete list of backups */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); delete_list = parray_new(); @@ -123,7 +123,7 @@ do_delete(time_t backup_id) * which FULL backup should be keeped for redundancy obligation(only valid do), * but if invalid backup is not guarded by retention - it is removed */ -void do_retention(void) +void do_retention(InstanceState *instanceState) { parray *backup_list = NULL; parray *to_keep_list = parray_new(); @@ -139,7 +139,7 @@ void do_retention(void) MyLocation = FIO_LOCAL_HOST; /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup) * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which * the error occurs before deleting all backup files. */ - write_backup_status(backup, BACKUP_STATUS_DELETING, instance_name, false); + write_backup_status(backup, BACKUP_STATUS_DELETING, false); /* list files to be deleted */ files = parray_new(); @@ -966,7 +966,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, /* Delete all backup files and wal files of given instance. */ int -do_delete_instance(void) +do_delete_instance(InstanceState *instanceState) { parray *backup_list; int i; @@ -974,7 +974,7 @@ do_delete_instance(void) /* Delete all backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true); @@ -1008,7 +1008,7 @@ do_delete_instance(void) elog(ERROR, "Can't remove \"%s\": %s", arclog_path, strerror(errno)); - elog(INFO, "Instance '%s' successfully deleted", instance_name); + elog(INFO, "Instance '%s' successfully deleted", instanceState->instance_name); return 0; } diff --git a/src/init.c b/src/init.c index aa376389d..29506749b 100644 --- a/src/init.c +++ b/src/init.c @@ -121,6 +121,6 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) /* pgdata was set through command line */ do_set_config(true); - elog(INFO, "Instance '%s' successfully inited", instance_name); + elog(INFO, "Instance '%s' successfully inited", instanceState->instance_name); return 0; } diff --git a/src/merge.c b/src/merge.c index 3c51a1fae..6dc599fe4 100644 --- a/src/merge.c +++ b/src/merge.c @@ -68,7 +68,7 @@ static bool is_forward_compatible(parray *parent_chain); * - Remove unnecessary files, which doesn't exist in the target backup anymore */ void -do_merge(time_t backup_id) +do_merge(InstanceState *instanceState, time_t backup_id) { parray *backups; parray *merge_list = parray_new(); @@ -80,13 +80,13 @@ do_merge(time_t backup_id) if (backup_id == INVALID_BACKUP_ID) elog(ERROR, "required parameter is not specified: --backup-id"); - if (instance_name == NULL) + if (instanceState == NULL) elog(ERROR, "required parameter is not specified: --instance"); elog(INFO, "Merge started"); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); /* Find destination backup first */ for (i = 0; i < parray_num(backups); i++) @@ -597,7 +597,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) write_backup(backup, true); } else - write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_MERGING, true); } /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ diff --git a/src/parsexlog.c b/src/parsexlog.c index 41a410d30..e427041d0 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -385,7 +385,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3d6990945..9af364674 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -809,7 +809,7 @@ main(int argc, char *argv[]) case ADD_INSTANCE_CMD: return do_add_instance(instanceState, &instance_config); case DELETE_INSTANCE_CMD: - return do_delete_instance(); + return do_delete_instance(instanceState); case INIT_CMD: return do_init(catalogState); case BACKUP_CMD: @@ -821,10 +821,11 @@ main(int argc, char *argv[]) elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(set_backup_params, no_validate, no_sync, backup_logs); + return do_backup(instanceState, set_backup_params, + no_validate, no_sync, backup_logs); } case RESTORE_CMD: - return do_restore_or_validate(current.backup_id, + return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, restore_params, no_sync); case VALIDATE_CMD: @@ -834,11 +835,11 @@ main(int argc, char *argv[]) if (datname_exclude_list || datname_include_list) elog(ERROR, "You must specify parameter (-i, --backup-id) for partial validation"); - return do_validate_all(backup_path); + return do_validate_all(catalogState, instanceState); } else /* PITR validation and, optionally, partial validation */ - return do_restore_or_validate(current.backup_id, + return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, restore_params, no_sync); @@ -859,13 +860,13 @@ main(int argc, char *argv[]) if (delete_status) do_delete_status(&instance_config, delete_status); else - do_retention(); + do_retention(instanceState); } else - do_delete(current.backup_id); + do_delete(instanceState, current.backup_id); break; case MERGE_CMD: - do_merge(current.backup_id); + do_merge(instanceState, current.backup_id); break; case SHOW_CONFIG_CMD: do_show_config(); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b5af7b6d4..3125e81ad 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -49,6 +49,9 @@ #include #endif +/* Wrap the code that we're going to delete after refactoring in this define*/ +#define REFACTORE_ME + /* pgut client variables and full path */ extern const char *PROGRAM_NAME; extern const char *PROGRAM_NAME_FULL; @@ -808,7 +811,7 @@ extern pgBackup current; extern char** commands_args; /* in backup.c */ -extern int do_backup(pgSetBackupParams *set_backup_params, +extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); @@ -822,7 +825,8 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments, BlockNumber blknum, size_t *result_size, int ptrack_version_num, const char *ptrack_schema); /* in restore.c */ -extern int do_restore_or_validate(time_t target_backup_id, +extern int do_restore_or_validate(InstanceState *instanceState, + time_t target_backup_id, pgRecoveryTarget *rt, pgRestoreParams *params, bool no_sync); @@ -843,7 +847,7 @@ extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetT extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); /* in merge.c */ -extern void do_merge(time_t backup_id); +extern void do_merge(InstanceState *instanceState, time_t backup_id); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); extern void merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup); @@ -872,10 +876,10 @@ extern int do_show(char *backup_catalog_path, const char *instance_name, time_t requested_backup_id, bool show_archive); /* in delete.c */ -extern void do_delete(time_t backup_id); +extern void do_delete(InstanceState *instanceState, time_t backup_id); extern void delete_backup_files(pgBackup *backup); -extern void do_retention(void); -extern int do_delete_instance(void); +extern void do_retention(InstanceState *instanceState); +extern int do_delete_instance(InstanceState *instanceState); extern void do_delete_status(InstanceConfig *instance_config, const char *status); /* in fetch.c */ @@ -893,7 +897,7 @@ extern void help_command(ProbackupSubcmd const subcmd); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); -extern int do_validate_all(char *backup_catalog_path); +extern int do_validate_all(CatalogState *catalogState, InstanceState *instanceState); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); @@ -912,7 +916,7 @@ extern bool validate_tablespace_map(pgBackup *backup); extern pgBackup *read_backup(const char *root_dir); extern void write_backup(pgBackup *backup, bool strict); extern void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name, bool strict); + bool strict); extern void write_backup_data_bytes(pgBackup *backup); extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); diff --git a/src/restore.c b/src/restore.c index 3f0adf7b7..ad7300ba8 100644 --- a/src/restore.c +++ b/src/restore.c @@ -94,7 +94,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", @@ -117,7 +117,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int -do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, +do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pgRecoveryTarget *rt, pgRestoreParams *params, bool no_sync) { int i = 0; @@ -136,7 +136,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, bool backup_has_tblspc = true; /* backup contain tablespace */ XLogRecPtr shift_lsn = InvalidXLogRecPtr; - if (instance_name == NULL) + if (instanceState == NULL) elog(ERROR, "required parameter not specified: --instance"); if (params->is_restore) @@ -216,7 +216,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(LOG, "%s begin.", action); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); /* Find backup range we should restore or validate. */ while ((i < parray_num(backups)) && !dest_backup) @@ -364,7 +364,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(backup->start_time), missing_backup_id); diff --git a/src/validate.c b/src/validate.c index 40c94af67..0049b4f71 100644 --- a/src/validate.c +++ b/src/validate.c @@ -16,7 +16,7 @@ #include "utils/thread.h" static void *pgBackupValidateFiles(void *arg); -static void do_validate_instance(void); +static void do_validate_instance(InstanceState *instanceState); static bool corrupted_backup_found = false; static bool skipped_due_to_lock = false; @@ -75,7 +75,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", base36enc(backup->start_time), status2str(backup->status)); - write_backup_status(backup, BACKUP_STATUS_ERROR, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ERROR, true); corrupted_backup_found = true; return; } @@ -121,7 +121,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time)); backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); return; } @@ -190,7 +190,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK, instance_name, true); + BACKUP_STATUS_OK, true); if (corrupted) elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); @@ -214,7 +214,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) "https://github.com/postgrespro/pg_probackup/issues/132", base36enc(backup->start_time)); backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); } } } @@ -380,25 +380,25 @@ pgBackupValidateFiles(void *arg) /* * Validate all backups in the backup catalog. * If --instance option was provided, validate only backups of this instance. + * + * TODO: split into two functions: do_validate_catalog and do_validate_instance. */ int -do_validate_all(char *backup_catalog_path) +do_validate_all(CatalogState *catalogState, InstanceState *instanceState) { corrupted_backup_found = false; skipped_due_to_lock = false; - if (instance_name == NULL) + if (instanceState == NULL) { /* Show list of instances */ - char path[MAXPGPATH]; DIR *dir; struct dirent *dent; /* open directory and list contents */ - join_path_components(path, backup_catalog_path, BACKUPS_DIR); - dir = opendir(path); + dir = opendir(catalogState->backup_subdir_path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); errno = 0; while ((dent = readdir(dir))) @@ -406,13 +406,15 @@ do_validate_all(char *backup_catalog_path) char conf_path[MAXPGPATH]; char child[MAXPGPATH]; struct stat st; + InstanceState *instance_state; + /* skip entries point current dir or parent dir */ if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) continue; - join_path_components(child, path, dent->d_name); + join_path_components(child, catalogState->backup_subdir_path, dent->d_name); if (lstat(child, &st) == -1) elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); @@ -423,10 +425,20 @@ do_validate_all(char *backup_catalog_path) /* * Initialize instance configuration. */ - instance_name = dent->d_name; + instance_state = pgut_new(InstanceState); + strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); + + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instance_name); + +#ifdef REFACTORE_ME sprintf(backup_instance_path, "%s/%s/%s", - backup_catalog_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_catalog_path, "wal", instance_name); + catalogState->catalog_path, BACKUPS_DIR, instanceState->instance_name); + + sprintf(arclog_path, "%s/%s/%s", catalogState->catalog_path, "wal", instanceState->instance_name); +#endif join_path_components(conf_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); if (config_read_opt(conf_path, instance_options, ERROR, false, @@ -437,12 +449,12 @@ do_validate_all(char *backup_catalog_path) continue; } - do_validate_instance(); + do_validate_instance(instanceState); } } else { - do_validate_instance(); + do_validate_instance(instanceState); } /* TODO: Probably we should have different exit code for every condition @@ -472,17 +484,17 @@ do_validate_all(char *backup_catalog_path) * Validate all backups in the given instance of the backup catalog. */ static void -do_validate_instance(void) +do_validate_instance(InstanceState *instanceState) { int i; int j; parray *backups; pgBackup *current_backup = NULL; - elog(INFO, "Validate backups of the instance '%s'", instance_name); + elog(INFO, "Validate backups of the instance '%s'", instanceState->instance_name); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) @@ -512,7 +524,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", base36enc(current_backup->start_time), parent_backup_id); @@ -536,7 +548,7 @@ do_validate_instance(void) if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(current_backup->start_time), backup_id, status2str(tmp_backup->status)); @@ -609,7 +621,7 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", base36enc(backup->start_time), From 341d7b82746f4b17d807f0d600ac6495090ced21 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 20:00:43 +0300 Subject: [PATCH 1609/2107] Refactoring. Get rid of global variable instance_name --- src/backup.c | 10 ++++++---- src/delete.c | 11 ++++++----- src/merge.c | 11 ++++------- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 3 +-- src/restore.c | 31 ++++++++++++++++--------------- src/validate.c | 8 ++++---- 7 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/backup.c b/src/backup.c index bbd168db0..17bb3345f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -47,7 +47,8 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_pg(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); +static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, + PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); @@ -92,7 +93,8 @@ backup_stopbackup_callback(bool fatal, void *userdata) * Move files from 'pgdata' to a subdirectory in backup catalog. */ static void -do_backup_pg(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) +do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, + PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) { int i; char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ @@ -155,7 +157,7 @@ do_backup_pg(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backu current.backup_mode == BACKUP_MODE_DIFF_DELTA) { /* get list of backups already taken */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) @@ -824,7 +826,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, add_note(¤t, set_backup_params->note); /* backup data */ - do_backup_pg(backup_conn, &nodeInfo, no_sync, backup_logs); + do_backup_pg(instanceState, backup_conn, &nodeInfo, no_sync, backup_logs); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ diff --git a/src/delete.c b/src/delete.c index b020716fa..d3d719506 100644 --- a/src/delete.c +++ b/src/delete.c @@ -18,8 +18,8 @@ static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); -static void do_retention_merge(parray *backup_list, parray *to_keep_list, - parray *to_purge_list); +static void do_retention_merge(InstanceState *instanceState, parray *backup_list, + parray *to_keep_list, parray *to_purge_list); static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); static void do_retention_wal(bool dry_run); @@ -172,7 +172,7 @@ void do_retention(InstanceState *instanceState) do_retention_internal(backup_list, to_keep_list, to_purge_list); if (merge_expired && !dry_run && !backup_list_is_empty) - do_retention_merge(backup_list, to_keep_list, to_purge_list); + do_retention_merge(instanceState, backup_list, to_keep_list, to_purge_list); if (delete_expired && !dry_run && !backup_list_is_empty) do_retention_purge(to_keep_list, to_purge_list); @@ -420,7 +420,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* Merge partially expired incremental chains */ static void -do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list) +do_retention_merge(InstanceState *instanceState, parray *backup_list, + parray *to_keep_list, parray *to_purge_list) { int i; int j; @@ -539,7 +540,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l */ keep_backup = parray_get(merge_list, 0); - merge_chain(merge_list, full_backup, keep_backup); + merge_chain(instanceState, merge_list, full_backup, keep_backup); backup_merged = true; for (j = parray_num(merge_list) - 2; j >= 0; j--) diff --git a/src/merge.c b/src/merge.c index 6dc599fe4..90376c02c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -405,7 +405,7 @@ do_merge(InstanceState *instanceState, time_t backup_id) catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* do actual merge */ - merge_chain(merge_list, full_backup, dest_backup); + merge_chain(instanceState, merge_list, full_backup, dest_backup); pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) @@ -434,7 +434,8 @@ do_merge(InstanceState *instanceState, time_t backup_id) * that chain is ok. */ void -merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) +merge_chain(InstanceState *instanceState, + parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) { int i; char *dest_backup_id; @@ -846,13 +847,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) else { /* Ugly */ - char backups_dir[MAXPGPATH]; - char instance_dir[MAXPGPATH]; char destination_path[MAXPGPATH]; - join_path_components(backups_dir, backup_path, BACKUPS_DIR); - join_path_components(instance_dir, backups_dir, instance_name); - join_path_components(destination_path, instance_dir, + join_path_components(destination_path, instanceState->instance_backup_subdir_path, base36enc(full_backup->merge_dest_backup)); elog(LOG, "Rename %s to %s", full_backup->root_dir, destination_path); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9af364674..423087b4c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -144,7 +144,7 @@ static char *delete_status = NULL; static bool compress_shortcut = false; /* ================ instanceState =========== */ -char *instance_name; +static char *instance_name; static InstanceState *instanceState = NULL; @@ -488,9 +488,9 @@ main(int argc, char *argv[]) strncpy(instanceState->instance_name, instance_name, MAXPGPATH); join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instance_name); + catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instance_name); + catalogState->wal_subdir_path, instanceState->instance_name); } /* ===== instanceState (END) ======*/ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 3125e81ad..756361165 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -793,7 +793,6 @@ extern bool merge_expired; extern bool dry_run; /* ===== instanceState ===== */ -extern char *instance_name; /* ===== instanceState (END) ===== */ @@ -849,7 +848,7 @@ extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); -extern void merge_chain(parray *parent_chain, +extern void merge_chain(InstanceState *instanceState, parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup); extern parray *read_database_map(pgBackup *backup); diff --git a/src/restore.c b/src/restore.c index ad7300ba8..628dad978 100644 --- a/src/restore.c +++ b/src/restore.c @@ -41,22 +41,22 @@ typedef struct static void -print_recovery_settings(FILE *fp, pgBackup *backup, +print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt); static void print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params); #if PG_VERSION_NUM >= 120000 static void -update_recovery_options(pgBackup *backup, +update_recovery_options(InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt); #else static void -update_recovery_options_before_v12(pgBackup *backup, +update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt); #endif -static void create_recovery_conf(time_t backup_id, +static void create_recovery_conf(InstanceState *instanceState, time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params); @@ -673,7 +673,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg //TODO rename and update comment /* Create recovery.conf with given recovery target parameters */ - create_recovery_conf(target_backup_id, rt, dest_backup, params); + create_recovery_conf(instanceState, target_backup_id, rt, dest_backup, params); } /* ssh connection to longer needed */ @@ -1298,7 +1298,7 @@ restore_files(void *arg) * with given recovery target parameters */ static void -create_recovery_conf(time_t backup_id, +create_recovery_conf(InstanceState *instanceState, time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params) @@ -1345,16 +1345,16 @@ create_recovery_conf(time_t backup_id, elog(LOG, "----------------------------------------"); #if PG_VERSION_NUM >= 120000 - update_recovery_options(backup, params, rt); + update_recovery_options(instanceState, backup, params, rt); #else - update_recovery_options_before_v12(backup, params, rt); + update_recovery_options_before_v12(instanceState, backup, params, rt); #endif } -/* TODO get rid of using global variables: instance_config, backup_path, instance_name */ +/* TODO get rid of using global variables: instance_config */ static void -print_recovery_settings(FILE *fp, pgBackup *backup, +print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt) { char restore_command_guc[16384]; @@ -1370,7 +1370,8 @@ print_recovery_settings(FILE *fp, pgBackup *backup, sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " "--wal-file-path=%%p --wal-file-name=%%f", PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, - backup_path, instance_name); + /* TODO What is going on here? Why do we use catalog path as wal-file-path? */ + instanceState->catalog_state->catalog_path, instanceState->instance_name); /* append --remote-* parameters provided via --archive-* settings */ if (instance_config.archive.host) @@ -1455,7 +1456,7 @@ print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *param #if PG_VERSION_NUM < 120000 static void -update_recovery_options_before_v12(pgBackup *backup, +update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt) { FILE *fp; @@ -1486,7 +1487,7 @@ update_recovery_options_before_v12(pgBackup *backup, PROGRAM_VERSION); if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(fp, backup, params, rt); + print_recovery_settings(instanceState, fp, backup, params, rt); if (params->restore_as_replica) { @@ -1508,7 +1509,7 @@ update_recovery_options_before_v12(pgBackup *backup, */ #if PG_VERSION_NUM >= 120000 static void -update_recovery_options(pgBackup *backup, +update_recovery_options(InstanceState *instanceState, pgBackup *backup, pgRestoreParams *params, pgRecoveryTarget *rt) { @@ -1616,7 +1617,7 @@ update_recovery_options(pgBackup *backup, base36enc(backup->start_time), current_time_str); if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(fp, backup, params, rt); + print_recovery_settings(instanceState, fp, backup, params, rt); if (params->restore_as_replica) print_standby_settings_common(fp, backup, params); diff --git a/src/validate.c b/src/validate.c index 0049b4f71..21cf32250 100644 --- a/src/validate.c +++ b/src/validate.c @@ -406,7 +406,7 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) char conf_path[MAXPGPATH]; char child[MAXPGPATH]; struct stat st; - InstanceState *instance_state; + InstanceState *instanceState; /* skip entries point current dir or parent dir */ @@ -425,13 +425,13 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) /* * Initialize instance configuration. */ - instance_state = pgut_new(InstanceState); + instanceState = pgut_new(InstanceState); strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instance_name); + catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instance_name); + catalogState->wal_subdir_path, instanceState->instance_name); #ifdef REFACTORE_ME sprintf(backup_instance_path, "%s/%s/%s", From b956febb7b425ed605504845599b867e576e8859 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 20:50:34 +0300 Subject: [PATCH 1610/2107] Refactoring. Get rid of global variable backup_path --- src/backup.c | 4 +-- src/catalog.c | 61 ++++++++++++++++++++-------------------- src/configure.c | 8 +++--- src/delete.c | 23 +++++++-------- src/merge.c | 2 +- src/pg_probackup.c | 8 +++--- src/pg_probackup.h | 32 +++++++++++++++------ src/pg_probackup_state.h | 13 --------- src/restore.c | 2 +- src/show.c | 54 +++++++++++++++++------------------ src/validate.c | 2 +- 11 files changed, 105 insertions(+), 104 deletions(-) diff --git a/src/backup.c b/src/backup.c index 17bb3345f..56711b206 100644 --- a/src/backup.c +++ b/src/backup.c @@ -157,7 +157,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, current.backup_mode == BACKUP_MODE_DIFF_DELTA) { /* get list of backups already taken */ - backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) @@ -168,7 +168,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, current.tli); /* TODO: use read_timeline_history */ - tli_list = catalog_get_timelines(&instance_config); + tli_list = catalog_get_timelines(instanceState, &instance_config); if (parray_num(tli_list) == 0) elog(WARNING, "Cannot find valid backup on previous timelines, " diff --git a/src/catalog.c b/src/catalog.c index f215e90e2..b32043a0e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -688,13 +688,11 @@ IsDir(const char *dirpath, const char *entry, fio_location location) /* * Create list of instances in given backup catalog. * - * Returns parray of "InstanceConfig" structures, filled with - * actual config of each instance. + * Returns parray of InstanceState structures. */ parray * -catalog_get_instance_list(char *backup_catalog_path) +catalog_get_instance_list(CatalogState *catalogState) { - char path[MAXPGPATH]; DIR *dir; struct dirent *dent; parray *instances; @@ -702,24 +700,23 @@ catalog_get_instance_list(char *backup_catalog_path) instances = parray_new(); /* open directory and list contents */ - join_path_components(path, backup_catalog_path, BACKUPS_DIR); - dir = opendir(path); + dir = opendir(catalogState->backup_subdir_path); if (dir == NULL) elog(ERROR, "Cannot open directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); while (errno = 0, (dent = readdir(dir)) != NULL) { char child[MAXPGPATH]; struct stat st; - InstanceConfig *instance; + InstanceState *instanceState = NULL; /* skip entries point current dir or parent dir */ if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) continue; - join_path_components(child, path, dent->d_name); + join_path_components(child, catalogState->backup_subdir_path, dent->d_name); if (lstat(child, &st) == -1) elog(ERROR, "Cannot stat file \"%s\": %s", @@ -728,9 +725,16 @@ catalog_get_instance_list(char *backup_catalog_path) if (!S_ISDIR(st.st_mode)) continue; - instance = readInstanceConfigFile(dent->d_name); + instanceState = pgut_new(InstanceState); + instanceState->config = readInstanceConfigFile(instanceState); + + strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instanceState->instance_name); - parray_append(instances, instance); + parray_append(instances, instanceState); } /* TODO 3.0: switch to ERROR */ @@ -739,11 +743,11 @@ catalog_get_instance_list(char *backup_catalog_path) if (errno) elog(ERROR, "Cannot read directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); if (closedir(dir)) elog(ERROR, "Cannot close directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); return instances; } @@ -755,22 +759,18 @@ catalog_get_instance_list(char *backup_catalog_path) * If valid backup id is passed only matching backup will be added to the list. */ parray * -catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) +catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id) { DIR *data_dir = NULL; struct dirent *data_ent = NULL; parray *backups = NULL; int i; - char backup_instance_path[MAXPGPATH]; - - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); /* open backup instance backups directory */ - data_dir = fio_opendir(backup_instance_path, FIO_BACKUP_HOST); + data_dir = fio_opendir(instanceState->instance_backup_subdir_path, FIO_BACKUP_HOST); if (data_dir == NULL) { - elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + elog(WARNING, "cannot open directory \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); goto err_proc; } @@ -784,12 +784,12 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) pgBackup *backup = NULL; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, data_ent->d_name, FIO_BACKUP_HOST) + if (!IsDir(instanceState->instance_backup_subdir_path, data_ent->d_name, FIO_BACKUP_HOST) || data_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(data_path, backup_instance_path, data_ent->d_name); + join_path_components(data_path, instanceState->instance_backup_subdir_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); @@ -835,7 +835,7 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) if (errno) { elog(WARNING, "cannot read backup root directory \"%s\": %s", - backup_instance_path, strerror(errno)); + instanceState->instance_backup_subdir_path, strerror(errno)); goto err_proc; } @@ -1345,22 +1345,21 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) * TODO: '.partial' and '.part' segno information should be added to tlinfo. */ parray * -catalog_get_timelines(InstanceConfig *instance) +catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) { int i,j,k; parray *xlog_files_list = parray_new(); parray *timelineinfos; parray *backups; timelineInfo *tlinfo; - char arclog_path[MAXPGPATH]; /* for fancy reporting */ char begin_segno_str[MAXFNAMELEN]; char end_segno_str[MAXFNAMELEN]; /* read all xlog files that belong to this archive */ - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, false, true, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, instanceState->instance_wal_subdir_path, + false, false, false, false, true, 0, FIO_BACKUP_HOST); parray_qsort(xlog_files_list, pgFileCompareName); timelineinfos = parray_new(); @@ -1530,7 +1529,7 @@ catalog_get_timelines(InstanceConfig *instance) TimeLineHistoryEntry *tln; sscanf(file->name, "%08X.history", &tli); - timelines = read_timeline_history(arclog_path, tli, true); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, tli, true); if (!tlinfo || tlinfo->tli != tli) { @@ -1564,7 +1563,7 @@ catalog_get_timelines(InstanceConfig *instance) } /* save information about backups belonging to each timeline */ - backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); for (i = 0; i < parray_num(timelineinfos); i++) { @@ -2037,7 +2036,7 @@ get_oldest_backup(timelineInfo *tlinfo) * Overwrite backup metadata. */ void -do_set_backup(const char *instance_name, time_t backup_id, +do_set_backup(InstanceState *instanceState, time_t backup_id, pgSetBackupParams *set_backup_params) { pgBackup *target_backup = NULL; @@ -2046,7 +2045,7 @@ do_set_backup(const char *instance_name, time_t backup_id, if (!set_backup_params) elog(ERROR, "Nothing to set by 'set-backup' command"); - backup_list = catalog_get_backup_list(instance_name, backup_id); + backup_list = catalog_get_backup_list(instanceState, backup_id); if (parray_num(backup_list) != 1) elog(ERROR, "Failed to find backup %s", base36enc(backup_id)); diff --git a/src/configure.c b/src/configure.c index 1aae3df13..065e01e6e 100644 --- a/src/configure.c +++ b/src/configure.c @@ -374,7 +374,7 @@ init_config(InstanceConfig *config, const char *instance_name) * read instance config from file */ InstanceConfig * -readInstanceConfigFile(const char *instance_name) +readInstanceConfigFile(InstanceState *instanceState) { char path[MAXPGPATH]; InstanceConfig *instance = pgut_new(InstanceConfig); @@ -592,14 +592,14 @@ readInstanceConfigFile(const char *instance_name) }; - init_config(instance, instance_name); + init_config(instance, instanceState->instance_name); sprintf(instance->backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); + instanceState->catalog_state->catalog_path, BACKUPS_DIR, instanceState->instance_name); canonicalize_path(instance->backup_instance_path); sprintf(instance->arclog_path, "%s/%s/%s", - backup_path, "wal", instance_name); + instanceState->catalog_state->catalog_path, "wal", instanceState->instance_name); canonicalize_path(instance->arclog_path); join_path_components(path, instance->backup_instance_path, diff --git a/src/delete.c b/src/delete.c index d3d719506..ebe09dbe5 100644 --- a/src/delete.c +++ b/src/delete.c @@ -21,7 +21,7 @@ static void do_retention_internal(parray *backup_list, parray *to_keep_list, static void do_retention_merge(InstanceState *instanceState, parray *backup_list, parray *to_keep_list, parray *to_purge_list); static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); -static void do_retention_wal(bool dry_run); +static void do_retention_wal(InstanceState *instanceState, bool dry_run); // TODO: more useful messages for dry run. static bool backup_deleted = false; /* At least one backup was deleted */ @@ -39,7 +39,7 @@ do_delete(InstanceState *instanceState, time_t backup_id) char size_to_delete_pretty[20]; /* Get complete list of backups */ - backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); delete_list = parray_new(); @@ -105,7 +105,7 @@ do_delete(InstanceState *instanceState, time_t backup_id) /* Clean WAL segments */ if (delete_wal) - do_retention_wal(dry_run); + do_retention_wal(instanceState, dry_run); /* cleanup */ parray_free(delete_list); @@ -139,7 +139,7 @@ void do_retention(InstanceState *instanceState) MyLocation = FIO_LOCAL_HOST; /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -179,7 +179,7 @@ void do_retention(InstanceState *instanceState) /* TODO: some sort of dry run for delete_wal */ if (delete_wal) - do_retention_wal(dry_run); + do_retention_wal(instanceState, dry_run); /* TODO: consider dry-run flag */ @@ -653,12 +653,13 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) * and delete them. */ static void -do_retention_wal(bool dry_run) +do_retention_wal(InstanceState *instanceState, bool dry_run) { parray *tli_list; int i; - tli_list = catalog_get_timelines(&instance_config); + //TODO check that instanceState is not NULL + tli_list = catalog_get_timelines(instanceState, &instance_config); for (i = 0; i < parray_num(tli_list); i++) { @@ -975,7 +976,7 @@ do_delete_instance(InstanceState *instanceState) /* Delete all backups. */ - backup_list = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true); @@ -1015,7 +1016,7 @@ do_delete_instance(InstanceState *instanceState) /* Delete all backups of given status in instance */ void -do_delete_status(InstanceConfig *instance_config, const char *status) +do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, const char *status) { int i; parray *backup_list, *delete_list; @@ -1038,11 +1039,11 @@ do_delete_status(InstanceConfig *instance_config, const char *status) */ pretty_status = status2str(status_for_delete); - backup_list = catalog_get_backup_list(instance_config->name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) { - elog(WARNING, "Instance '%s' has no backups", instance_config->name); + elog(WARNING, "Instance '%s' has no backups", instanceState->instance_name); return; } diff --git a/src/merge.c b/src/merge.c index 90376c02c..265aa4ad0 100644 --- a/src/merge.c +++ b/src/merge.c @@ -86,7 +86,7 @@ do_merge(InstanceState *instanceState, time_t backup_id) elog(INFO, "Merge started"); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find destination backup first */ for (i = 0; i < parray_num(backups); i++) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 423087b4c..656a778ad 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -63,7 +63,7 @@ const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues /* ================ catalogState =========== */ /* directory options */ /* TODO make it local variable, pass as an argument to all commands that need it. */ -char *backup_path = NULL; +static char *backup_path = NULL; /* * path or to the data files in the backup catalog * $BACKUP_PATH/backups/instance_name @@ -844,7 +844,7 @@ main(int argc, char *argv[]) restore_params, no_sync); case SHOW_CMD: - return do_show(backup_path, instance_name, current.backup_id, show_archive); + return do_show(catalogState, instanceState, current.backup_id, show_archive); case DELETE_CMD: if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); @@ -858,7 +858,7 @@ main(int argc, char *argv[]) if (!backup_id_string) { if (delete_status) - do_delete_status(&instance_config, delete_status); + do_delete_status(instanceState, &instance_config, delete_status); else do_retention(instanceState); } @@ -877,7 +877,7 @@ main(int argc, char *argv[]) case SET_BACKUP_CMD: if (!backup_id_string) elog(ERROR, "You must specify parameter (-i, --backup-id) for 'set-backup' command"); - do_set_backup(instance_name, current.backup_id, set_backup_params); + do_set_backup(instanceState, current.backup_id, set_backup_params); break; case CHECKDB_CMD: do_checkdb(need_amcheck, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 756361165..bcfadcb23 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -758,7 +758,6 @@ typedef struct BackupPageHeader2 /* ====== CatalogState ======= */ /* directory options */ -extern char *backup_path; extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; @@ -794,6 +793,21 @@ extern bool dry_run; /* ===== instanceState ===== */ +typedef struct InstanceState +{ + /* catalog, this instance belongs to */ + CatalogState *catalog_state; + + char instance_name[MAXPGPATH]; //previously global var instance_name + /* $BACKUP_PATH/backups/instance_name */ + char instance_backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/backups/instance_name */ + char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path + + //TODO split into some more meaningdul parts + InstanceConfig *config; +} InstanceState; + /* ===== instanceState (END) ===== */ /* show options */ @@ -868,10 +882,10 @@ extern void do_archive_get(InstanceConfig *instance, const char *prefetch_dir_ar extern void do_show_config(void); extern void do_set_config(bool missing_ok); extern void init_config(InstanceConfig *config, const char *instance_name); -extern InstanceConfig *readInstanceConfigFile(const char *instance_name); +extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); /* in show.c */ -extern int do_show(char *backup_catalog_path, const char *instance_name, +extern int do_show(CatalogState *catalogState, InstanceState *instanceState, time_t requested_backup_id, bool show_archive); /* in delete.c */ @@ -879,7 +893,8 @@ extern void do_delete(InstanceState *instanceState, time_t backup_id); extern void delete_backup_files(pgBackup *backup); extern void do_retention(InstanceState *instanceState); extern int do_delete_instance(InstanceState *instanceState); -extern void do_delete_status(InstanceConfig *instance_config, const char *status); +extern void do_delete_status(InstanceState *instanceState, + InstanceConfig *instance_config, const char *status); /* in fetch.c */ extern char *slurpFile(const char *datadir, @@ -922,8 +937,9 @@ extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color); extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode); -extern parray *catalog_get_instance_list(char *backup_catalog_path); -extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); +extern parray *catalog_get_instance_list(CatalogState *catalogState); + +extern parray *catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict, bool exclusive); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, @@ -933,8 +949,8 @@ extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list TimeLineID current_tli, time_t current_start_time, InstanceConfig *instance); extern void timelineInfoFree(void *tliInfo); -extern parray *catalog_get_timelines(InstanceConfig *instance); -extern void do_set_backup(const char *instance_name, time_t backup_id, +extern parray *catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance); +extern void do_set_backup(InstanceState *instanceState, time_t backup_id, pgSetBackupParams *set_backup_params); extern void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params); diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 19e096328..56d852537 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -25,19 +25,6 @@ typedef struct CatalogState /* ===== instanceState ===== */ -typedef struct InstanceState -{ - /* catalog, this instance belongs to */ - CatalogState *catalog_state; - - char instance_name[MAXPGPATH]; //previously global var instance_name - /* $BACKUP_PATH/backups/instance_name */ - char instance_backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/backups/instance_name */ - char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path - - //TODO add config here -} InstanceState; /* ===== instanceState (END) ===== */ diff --git a/src/restore.c b/src/restore.c index 628dad978..398726772 100644 --- a/src/restore.c +++ b/src/restore.c @@ -216,7 +216,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg elog(LOG, "%s begin.", action); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find backup range we should restore or validate. */ while ((i < parray_num(backups)) && !dest_backup) diff --git a/src/show.c b/src/show.c index c88f4fae9..ed4f43ef3 100644 --- a/src/show.c +++ b/src/show.c @@ -54,14 +54,14 @@ typedef struct ShowArchiveRow static void show_instance_start(void); static void show_instance_end(void); -static void show_instance(const char *instance_name, time_t requested_backup_id, bool show_name); +static void show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name); static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup); -static int show_backup(const char *instance_name, time_t requested_backup_id); +static int show_backup(InstanceState *instanceState, time_t requested_backup_id); static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name); static void show_instance_json(const char *instance_name, parray *backup_list); -static void show_instance_archive(InstanceConfig *instance); +static void show_instance_archive(InstanceState *instanceState, InstanceConfig *instance); static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *timelines_list, bool show_name); static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, @@ -75,12 +75,12 @@ static int32 json_level = 0; * Entry point of pg_probackup SHOW subcommand. */ int -do_show(char *backup_catalog_path, const char *instance_name, +do_show(CatalogState *catalogState, InstanceState *instanceState, time_t requested_backup_id, bool show_archive) { int i; - if (instance_name == NULL && + if (instanceState == NULL && requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use (-i, --backup-id) option"); @@ -89,28 +89,25 @@ do_show(char *backup_catalog_path, const char *instance_name, elog(ERROR, "You cannot specify --archive and (-i, --backup-id) options together"); /* - * if instance_name is not specified, + * if instance is not specified, * show information about all instances in this backup catalog */ - if (instance_name == NULL) + if (instanceState == NULL) { - parray *instances = catalog_get_instance_list(backup_catalog_path); + parray *instances = catalog_get_instance_list(catalogState); show_instance_start(); for (i = 0; i < parray_num(instances); i++) { - InstanceConfig *instance = parray_get(instances, i); - char backup_instance_path[MAXPGPATH]; + InstanceState *instanceState = parray_get(instances, i); if (interrupted) elog(ERROR, "Interrupted during show"); - sprintf(backup_instance_path, "%s/%s/%s", backup_catalog_path, BACKUPS_DIR, instance->name); - if (show_archive) - show_instance_archive(instance); + show_instance_archive(instanceState, instanceState->config); else - show_instance(instance->name, INVALID_BACKUP_ID, true); + show_instance(instanceState, INVALID_BACKUP_ID, true); } show_instance_end(); @@ -124,11 +121,11 @@ do_show(char *backup_catalog_path, const char *instance_name, if (show_archive) { - InstanceConfig *instance = readInstanceConfigFile(instance_name); - show_instance_archive(instance); + InstanceConfig *instance = readInstanceConfigFile(instanceState); + show_instance_archive(instanceState, instance); } else - show_instance(instance_name, requested_backup_id, false); + show_instance(instanceState, requested_backup_id, false); show_instance_end(); @@ -138,11 +135,11 @@ do_show(char *backup_catalog_path, const char *instance_name, { if (show_archive) { - InstanceConfig *instance = readInstanceConfigFile(instance_name); - show_instance_archive(instance); + InstanceConfig *instance = readInstanceConfigFile(instanceState); + show_instance_archive(instanceState, instance); } else - show_backup(instance_name, requested_backup_id); + show_backup(instanceState, requested_backup_id); return 0; } @@ -290,16 +287,16 @@ show_instance_end(void) * Show brief meta information about all backups in the backup instance. */ static void -show_instance(const char *instance_name, time_t requested_backup_id, bool show_name) +show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name) { parray *backup_list; - backup_list = catalog_get_backup_list(instance_name, requested_backup_id); + backup_list = catalog_get_backup_list(instanceState, requested_backup_id); if (show_format == SHOW_PLAIN) - show_instance_plain(instance_name, backup_list, show_name); + show_instance_plain(instanceState->instance_name, backup_list, show_name); else if (show_format == SHOW_JSON) - show_instance_json(instance_name, backup_list); + show_instance_json(instanceState->instance_name, backup_list); else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -451,13 +448,14 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) * Show detailed meta information about specified backup. */ static int -show_backup(const char *instance_name, time_t requested_backup_id) +show_backup(InstanceState *instanceState, time_t requested_backup_id) { int i; pgBackup *backup = NULL; parray *backups; - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + //TODO pass requested_backup_id to the function + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find requested backup */ for (i = 0; i < parray_num(backups); i++) @@ -776,11 +774,11 @@ show_instance_json(const char *instance_name, parray *backup_list) * show information about WAL archive of the instance */ static void -show_instance_archive(InstanceConfig *instance) +show_instance_archive(InstanceState *instanceState, InstanceConfig *instance) { parray *timelineinfos; - timelineinfos = catalog_get_timelines(instance); + timelineinfos = catalog_get_timelines(instanceState, instance); if (show_format == SHOW_PLAIN) show_archive_plain(instance->name, instance->xlog_seg_size, timelineinfos, true); diff --git a/src/validate.c b/src/validate.c index 21cf32250..ce434f027 100644 --- a/src/validate.c +++ b/src/validate.c @@ -494,7 +494,7 @@ do_validate_instance(InstanceState *instanceState) elog(INFO, "Validate backups of the instance '%s'", instanceState->instance_name); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState->instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) From 9940967657c428a7e6a2f6b402088809db7d6e45 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 21:28:34 +0300 Subject: [PATCH 1611/2107] Refactoring. Get rid of global variable backup_instance_path --- src/backup.c | 59 ++++++++++++++++++++++++++-------------------- src/catalog.c | 24 ++----------------- src/configure.c | 27 +++++++++------------ src/delete.c | 11 ++++----- src/init.c | 2 +- src/pg_probackup.c | 7 ++++-- src/pg_probackup.h | 14 +++++++---- src/validate.c | 14 ++++------- 8 files changed, 71 insertions(+), 87 deletions(-) diff --git a/src/backup.c b/src/backup.c index 56711b206..baed44fc5 100644 --- a/src/backup.c +++ b/src/backup.c @@ -50,12 +50,12 @@ static void *backup_files(void *arg); static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); -static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, +static void pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); -static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); +static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); -static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, +static XLogRecPtr wait_wal_lsn(InstanceState *instanceState, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir); @@ -77,14 +77,15 @@ static void set_cfs_datafiles(parray *files, const char *root, char *relative, s static void backup_stopbackup_callback(bool fatal, void *userdata) { - PGconn *pg_startbackup_conn = (PGconn *) userdata; + InstanceState *instanceState = (InstanceState *) userdata; + PGconn *pg_startbackup_conn = instanceState->conn; /* * If backup is in progress, notify stop of backup to PostgreSQL */ if (backup_in_progress) { elog(WARNING, "backup in progress, stop backup"); - pg_stop_backup(NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */ + pg_stop_backup(instanceState, NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */ } } @@ -139,7 +140,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, strlen(" with pg_probackup")); /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); + pg_start_backup(instanceState, label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); /* Obtain current timeline */ #if PG_VERSION_NUM >= 90600 @@ -270,7 +271,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, * Because WAL streaming will start after pg_start_backup() in stream * mode. */ - wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); + wait_wal_lsn(instanceState, current.start_lsn, true, current.tli, false, true, ERROR, false); } /* start stream replication */ @@ -527,7 +528,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, } /* Notify end of backup */ - pg_stop_backup(¤t, backup_conn, nodeInfo); + pg_stop_backup(instanceState, ¤t, backup_conn, nodeInfo); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -1028,7 +1029,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) * Notify start of backup to PostgreSQL server. */ static void -pg_start_backup(const char *label, bool smooth, pgBackup *backup, +pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn) { PGresult *res; @@ -1056,7 +1057,8 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; - pgut_atexit_push(backup_stopbackup_callback, conn); + instanceState->conn = conn; + pgut_atexit_push(backup_stopbackup_callback, instanceState); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1262,7 +1264,7 @@ pg_is_superuser(PGconn *conn) * Returns InvalidXLogRecPtr if 'segment_only' flag is used. */ static XLogRecPtr -wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, +wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir) { @@ -1296,8 +1298,9 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, */ if (in_stream_dir) { - pgBackupGetPath2(¤t, pg_wal_dir, lengthof(pg_wal_dir), - DATABASE_DIR, PG_XLOG_DIR); + snprintf(pg_wal_dir, lengthof(pg_wal_dir), "%s/%s/%s/%s", + instanceState->instance_backup_subdir_path, base36enc(current.start_time), + DATABASE_DIR, PG_XLOG_DIR); join_path_components(wal_segment_path, pg_wal_dir, wal_segment); wal_segment_dir = pg_wal_dir; } @@ -1438,7 +1441,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, * Notify end of backup to PostgreSQL server. */ static void -pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, +pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo) { PGconn *conn; @@ -1566,7 +1569,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, } /* After we have sent pg_stop_backup, we don't need this callback anymore */ - pgut_atexit_pop(backup_stopbackup_callback, pg_startbackup_conn); + instanceState->conn = pg_startbackup_conn; + pgut_atexit_pop(backup_stopbackup_callback, instanceState); /* * Wait for the result of pg_stop_backup(), but no longer than @@ -1671,9 +1675,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (stream_wal) { - pgBackupGetPath2(backup, stream_xlog_path, - lengthof(stream_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); + snprintf(stream_xlog_path, lengthof(stream_xlog_path), + "%s/%s/%s/%s", instanceState->instance_backup_subdir_path, + base36enc(backup->start_time), + DATABASE_DIR, PG_XLOG_DIR); xlog_path = stream_xlog_path; } else @@ -1702,7 +1707,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (stop_backup_lsn_tmp % instance_config.xlog_seg_size == 0) { /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, false, true, WARNING, stream_wal); /* Get the first record in segment with current stop_lsn */ @@ -1730,7 +1735,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, /* Despite looking for previous record there is not guarantee of success * because previous record can be the contrecord. */ - lsn_tmp = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + lsn_tmp = wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, true, false, ERROR, stream_wal); /* sanity */ @@ -1744,7 +1749,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, else if (stop_backup_lsn_tmp % XLOG_BLCKSZ == 0) { /* Wait for segment with current stop_lsn */ - wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, false, true, ERROR, stream_wal); /* Get the next closest record in segment with current stop_lsn */ @@ -1776,7 +1781,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if (!exclusive_backup) { Assert(PQnfields(res) >= 4); - pgBackupGetPath2(backup, path, lengthof(path), DATABASE_DIR, NULL); + snprintf(path, lengthof(path), "%s/%s/%s", instanceState->instance_backup_subdir_path, + base36enc(backup->start_time), DATABASE_DIR); /* Write backup_label */ join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); @@ -1873,7 +1879,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * look for previous record with endpoint >= STOP_LSN. */ if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, + stop_backup_lsn = wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, false, false, ERROR, stream_wal); if (stream_wal) @@ -1883,9 +1889,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, if(wait_WAL_streaming_end(backup_files_list)) elog(ERROR, "WAL streaming failed"); - pgBackupGetPath2(backup, stream_xlog_path, - lengthof(stream_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); + snprintf(stream_xlog_path, lengthof(stream_xlog_path), "%s/%s/%s/%s", + instanceState->instance_backup_subdir_path, base36enc(backup->start_time), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; } else diff --git a/src/catalog.c b/src/catalog.c index b32043a0e..d8b6d109a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -733,7 +733,8 @@ catalog_get_instance_list(CatalogState *catalogState) catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, catalogState->wal_subdir_path, instanceState->instance_name); - + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); parray_append(instances, instanceState); } @@ -2808,27 +2809,6 @@ pgBackupCompareIdDesc(const void *l, const void *r) return -pgBackupCompareId(l, r); } -/* - * Construct absolute path of the backup directory. - * Append "subdir1" and "subdir2" to the backup directory. - */ -void -pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2) -{ - /* If "subdir1" is NULL do not check "subdir2" */ - if (!subdir1) - snprintf(path, len, "%s/%s", backup_instance_path, - base36enc(backup->start_time)); - else if (!subdir2) - snprintf(path, len, "%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1); - /* "subdir1" and "subdir2" is not NULL */ - else - snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1, subdir2); -} - /* * Check if multiple backups consider target backup to be their direct parent */ diff --git a/src/configure.c b/src/configure.c index 065e01e6e..b5ef8a356 100644 --- a/src/configure.c +++ b/src/configure.c @@ -277,18 +277,16 @@ do_show_config(void) * values into the file. */ void -do_set_config(bool missing_ok) +do_set_config(InstanceState *instanceState, bool missing_ok) { - char path[MAXPGPATH]; char path_temp[MAXPGPATH]; FILE *fp; int i; - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); + snprintf(path_temp, sizeof(path_temp), "%s.tmp", instanceState->instance_config_path); - if (!missing_ok && !fileExists(path, FIO_LOCAL_HOST)) - elog(ERROR, "Configuration file \"%s\" doesn't exist", path); + if (!missing_ok && !fileExists(instanceState->instance_config_path, FIO_LOCAL_HOST)) + elog(ERROR, "Configuration file \"%s\" doesn't exist", instanceState->instance_config_path); fp = fopen(path_temp, "wt"); if (fp == NULL) @@ -327,12 +325,12 @@ do_set_config(bool missing_ok) fclose(fp); - if (rename(path_temp, path) < 0) + if (rename(path_temp, instanceState->instance_config_path) < 0) { int errno_temp = errno; unlink(path_temp); elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno_temp)); + path_temp, instanceState->instance_config_path, strerror(errno_temp)); } } @@ -376,7 +374,6 @@ init_config(InstanceConfig *config, const char *instance_name) InstanceConfig * readInstanceConfigFile(InstanceState *instanceState) { - char path[MAXPGPATH]; InstanceConfig *instance = pgut_new(InstanceConfig); char *log_level_console = NULL; char *log_level_file = NULL; @@ -602,21 +599,19 @@ readInstanceConfigFile(InstanceState *instanceState) instanceState->catalog_state->catalog_path, "wal", instanceState->instance_name); canonicalize_path(instance->arclog_path); - join_path_components(path, instance->backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - - if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) { - elog(WARNING, "Control file \"%s\" doesn't exist", path); + elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); pfree(instance); return NULL; } - parsed_options = config_read_opt(path, instance_options, WARNING, true, true); + parsed_options = config_read_opt(instanceState->instance_config_path, + instance_options, WARNING, true, true); if (parsed_options == 0) { - elog(WARNING, "Control file \"%s\" is empty", path); + elog(WARNING, "Control file \"%s\" is empty", instanceState->instance_config_path); pfree(instance); return NULL; } diff --git a/src/delete.c b/src/delete.c index ebe09dbe5..e833487a9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -972,8 +972,6 @@ do_delete_instance(InstanceState *instanceState) { parray *backup_list; int i; - char instance_config_path[MAXPGPATH]; - /* Delete all backups. */ backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); @@ -994,16 +992,15 @@ do_delete_instance(InstanceState *instanceState) pgut_rmtree(arclog_path, false, true); /* Delete backup instance config file */ - join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - if (remove(instance_config_path)) + if (remove(instanceState->instance_config_path)) { - elog(ERROR, "Can't remove \"%s\": %s", instance_config_path, + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_config_path, strerror(errno)); } /* Delete instance root directories */ - if (rmdir(backup_instance_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", backup_instance_path, + if (rmdir(instanceState->instance_backup_subdir_path) != 0) + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); if (rmdir(arclog_path) != 0) diff --git a/src/init.c b/src/init.c index 29506749b..dc821325a 100644 --- a/src/init.c +++ b/src/init.c @@ -119,7 +119,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) SOURCE_DEFAULT); /* pgdata was set through command line */ - do_set_config(true); + do_set_config(instanceState, true); elog(INFO, "Instance '%s' successfully inited", instanceState->instance_name); return 0; diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 656a778ad..6b297e892 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -68,7 +68,7 @@ static char *backup_path = NULL; * path or to the data files in the backup catalog * $BACKUP_PATH/backups/instance_name */ -char backup_instance_path[MAXPGPATH]; +static char backup_instance_path[MAXPGPATH]; /* * path or to the wal files in the backup catalog * $BACKUP_PATH/wal/instance_name @@ -491,6 +491,9 @@ main(int argc, char *argv[]) catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, catalogState->wal_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + } /* ===== instanceState (END) ======*/ @@ -872,7 +875,7 @@ main(int argc, char *argv[]) do_show_config(); break; case SET_CONFIG_CMD: - do_set_config(false); + do_set_config(instanceState, false); break; case SET_BACKUP_CMD: if (!backup_id_string) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bcfadcb23..2731b4558 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -758,7 +758,6 @@ typedef struct BackupPageHeader2 /* ====== CatalogState ======= */ /* directory options */ -extern char backup_instance_path[MAXPGPATH]; extern char arclog_path[MAXPGPATH]; /* ====== CatalogState (END) ======= */ @@ -801,9 +800,17 @@ typedef struct InstanceState char instance_name[MAXPGPATH]; //previously global var instance_name /* $BACKUP_PATH/backups/instance_name */ char instance_backup_subdir_path[MAXPGPATH]; + + /* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ + char instance_config_path[MAXPGPATH]; + /* $BACKUP_PATH/backups/instance_name */ char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path + /* TODO: Make it more specific */ + PGconn *conn; + + //TODO split into some more meaningdul parts InstanceConfig *config; } InstanceState; @@ -880,7 +887,7 @@ extern void do_archive_get(InstanceConfig *instance, const char *prefetch_dir_ar /* in configure.c */ extern void do_show_config(void); -extern void do_set_config(bool missing_ok); +extern void do_set_config(InstanceState *instanceState, bool missing_ok); extern void init_config(InstanceConfig *config, const char *instance_name); extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); @@ -959,8 +966,7 @@ extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2); + extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); diff --git a/src/validate.c b/src/validate.c index ce434f027..0a5397550 100644 --- a/src/validate.c +++ b/src/validate.c @@ -403,7 +403,6 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) errno = 0; while ((dent = readdir(dir))) { - char conf_path[MAXPGPATH]; char child[MAXPGPATH]; struct stat st; InstanceState *instanceState; @@ -432,19 +431,16 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, catalogState->wal_subdir_path, instanceState->instance_name); - + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); #ifdef REFACTORE_ME - sprintf(backup_instance_path, "%s/%s/%s", - catalogState->catalog_path, BACKUPS_DIR, instanceState->instance_name); - sprintf(arclog_path, "%s/%s/%s", catalogState->catalog_path, "wal", instanceState->instance_name); #endif - join_path_components(conf_path, backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - if (config_read_opt(conf_path, instance_options, ERROR, false, + + if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false, true) == 0) { - elog(WARNING, "Configuration file \"%s\" is empty", conf_path); + elog(WARNING, "Configuration file \"%s\" is empty", instanceState->instance_config_path); corrupted_backup_found = true; continue; } From d9b3fb22d60c2f7b5033e5b4a3fced60e04a87d3 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 21:44:32 +0300 Subject: [PATCH 1612/2107] Refactoring. Get rid of global variable archlog_path --- src/backup.c | 15 ++++++++------- src/delete.c | 6 +++--- src/pg_probackup.c | 6 ------ src/pg_probackup.h | 7 ------- src/restore.c | 8 +++++--- src/validate.c | 7 ++----- 6 files changed, 18 insertions(+), 31 deletions(-) diff --git a/src/backup.c b/src/backup.c index baed44fc5..3c8c0c93a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -262,9 +262,9 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !stream_wal) { /* Check that archive_dir can be reached */ - if (fio_access(arclog_path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", - arclog_path, strerror(errno)); + instanceState->instance_wal_subdir_path, strerror(errno)); /* * Do not wait start_lsn for stream backup. @@ -391,7 +391,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, * reading WAL segments present in archives up to the point * where this backup has started. */ - pagemap_isok = extractPageMap(arclog_path, instance_config.xlog_seg_size, + pagemap_isok = extractPageMap(instanceState->instance_wal_subdir_path, + instance_config.xlog_seg_size, prev_backup->start_lsn, prev_backup->tli, current.start_lsn, current.tli, tli_list); } @@ -1306,8 +1307,8 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ } else { - join_path_components(wal_segment_path, arclog_path, wal_segment); - wal_segment_dir = arclog_path; + join_path_components(wal_segment_path, instanceState->instance_wal_subdir_path, wal_segment); + wal_segment_dir = instanceState->instance_wal_subdir_path; } /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ @@ -1682,7 +1683,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb xlog_path = stream_xlog_path; } else - xlog_path = arclog_path; + xlog_path = instanceState->instance_wal_subdir_path; GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); @@ -1896,7 +1897,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb xlog_path = stream_xlog_path; } else - xlog_path = arclog_path; + xlog_path = instanceState->instance_wal_subdir_path; backup->stop_lsn = stop_backup_lsn; backup->recovery_xid = recovery_xid; diff --git a/src/delete.c b/src/delete.c index e833487a9..e66ce0cd4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -989,7 +989,7 @@ do_delete_instance(InstanceState *instanceState) parray_free(backup_list); /* Delete all wal files. */ - pgut_rmtree(arclog_path, false, true); + pgut_rmtree(instanceState->instance_wal_subdir_path, false, true); /* Delete backup instance config file */ if (remove(instanceState->instance_config_path)) @@ -1003,8 +1003,8 @@ do_delete_instance(InstanceState *instanceState) elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); - if (rmdir(arclog_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", arclog_path, + if (rmdir(instanceState->instance_wal_subdir_path) != 0) + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_wal_subdir_path, strerror(errno)); elog(INFO, "Instance '%s' successfully deleted", instanceState->instance_name); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6b297e892..453aab458 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -69,11 +69,6 @@ static char *backup_path = NULL; * $BACKUP_PATH/backups/instance_name */ static char backup_instance_path[MAXPGPATH]; -/* - * path or to the wal files in the backup catalog - * $BACKUP_PATH/wal/instance_name - */ -char arclog_path[MAXPGPATH] = ""; static CatalogState *catalogState = NULL; /* ================ catalogState (END) =========== */ @@ -510,7 +505,6 @@ main(int argc, char *argv[]) */ sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); /* * Fill InstanceConfig structure fields used to generate pathes inside diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2731b4558..d4dc90281 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -755,13 +755,6 @@ typedef struct BackupPageHeader2 #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) -/* ====== CatalogState ======= */ - -/* directory options */ -extern char arclog_path[MAXPGPATH]; - -/* ====== CatalogState (END) ======= */ - /* common options */ extern pid_t my_pid; extern __thread int my_thread_num; diff --git a/src/restore.c b/src/restore.c index 398726772..481aee2b2 100644 --- a/src/restore.c +++ b/src/restore.c @@ -287,7 +287,8 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg // elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ - timelines = read_timeline_history(arclog_path, rt->target_tli, true); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, + rt->target_tli, true); if (!satisfy_timeline(timelines, current_backup)) { @@ -489,7 +490,8 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg elog(ERROR, "Incremental restore in 'lsn' mode require " "data_checksums to be enabled in destination data directory"); - timelines = read_timeline_history(arclog_path, redo.tli, false); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, + redo.tli, false); if (!timelines) elog(WARNING, "Failed to get history for redo timeline %i, " @@ -604,7 +606,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg * We pass base_full_backup timeline as last argument to this function, * because it's needed to form the name of xlog file. */ - validate_wal(dest_backup, arclog_path, rt->target_time, + validate_wal(dest_backup, instanceState->instance_wal_subdir_path, rt->target_time, rt->target_xid, rt->target_lsn, dest_backup->tli, instance_config.xlog_seg_size); } diff --git a/src/validate.c b/src/validate.c index 0a5397550..1c013b92c 100644 --- a/src/validate.c +++ b/src/validate.c @@ -433,9 +433,6 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) catalogState->wal_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_config_path, instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); -#ifdef REFACTORE_ME - sprintf(arclog_path, "%s/%s/%s", catalogState->catalog_path, "wal", instanceState->instance_name); -#endif if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false, true) == 0) @@ -587,7 +584,7 @@ do_validate_instance(InstanceState *instanceState) /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) - validate_wal(current_backup, arclog_path, 0, + validate_wal(current_backup, instanceState->instance_wal_subdir_path, 0, 0, 0, current_backup->tli, instance_config.xlog_seg_size); @@ -684,7 +681,7 @@ do_validate_instance(InstanceState *instanceState) { /* Revalidation successful, validate corresponding WAL files */ - validate_wal(backup, arclog_path, 0, + validate_wal(backup, instanceState->instance_wal_subdir_path, 0, 0, 0, backup->tli, instance_config.xlog_seg_size); } From 6c52147c478dbb2e9a663482051e0a19d0476645 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 21:48:53 +0300 Subject: [PATCH 1613/2107] Refactoring. Get rid of global variable backup_instance_path (2) --- src/pg_probackup.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 453aab458..38f78f931 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -64,11 +64,6 @@ const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues /* directory options */ /* TODO make it local variable, pass as an argument to all commands that need it. */ static char *backup_path = NULL; -/* - * path or to the data files in the backup catalog - * $BACKUP_PATH/backups/instance_name - */ -static char backup_instance_path[MAXPGPATH]; static CatalogState *catalogState = NULL; /* ================ catalogState (END) =========== */ @@ -498,14 +493,6 @@ main(int argc, char *argv[]) */ if ((backup_path != NULL) && instance_name) { - /* - * Fill global variables used to generate pathes inside the instance's - * backup catalog. - * TODO replace global variables with InstanceConfig structure fields - */ - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - /* * Fill InstanceConfig structure fields used to generate pathes inside * the instance's backup catalog. @@ -530,10 +517,11 @@ main(int argc, char *argv[]) { struct stat st; - if (fio_stat(backup_instance_path, &st, true, FIO_BACKUP_HOST) != 0) + if (fio_stat(instanceState->instance_backup_subdir_path, + &st, true, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Failed to access directory \"%s\": %s", - backup_instance_path, strerror(errno)); + instanceState->instance_backup_subdir_path, strerror(errno)); // TODO: redundant message, should we get rid of it? elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -555,7 +543,6 @@ main(int argc, char *argv[]) */ if (instance_name) { - char path[MAXPGPATH]; /* Read environment variables */ config_get_opt_env(instance_options); @@ -563,13 +550,10 @@ main(int argc, char *argv[]) if (backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != ARCHIVE_GET_CMD) { - join_path_components(path, backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - if (backup_subcmd == CHECKDB_CMD) - config_read_opt(path, instance_options, ERROR, true, true); + config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, true); else - config_read_opt(path, instance_options, ERROR, true, false); + config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, false); /* * We can determine our location only after reading the configuration file, From f89793158bca9f8ce9239e844029f1cfc86e36d0 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 3 Feb 2021 22:10:17 +0300 Subject: [PATCH 1614/2107] Refactoring. Remove redundand fields from InstanceConfig --- src/archive.c | 18 +++++++++--------- src/configure.c | 12 ++++++------ src/delete.c | 18 +++++++++--------- src/pg_probackup.c | 20 ++------------------ src/pg_probackup.h | 8 ++------ src/show.c | 4 ++-- 6 files changed, 30 insertions(+), 50 deletions(-) diff --git a/src/archive.c b/src/archive.c index 28622cc57..ef87910f8 100644 --- a/src/archive.c +++ b/src/archive.c @@ -113,7 +113,7 @@ static parray *setup_push_filelist(const char *archive_status_dir, * Where archlog_path is $BACKUP_PATH/wal/instance_name */ void -do_archive_push(InstanceConfig *instance, char *wal_file_path, +do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename) { @@ -156,7 +156,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, if (system_id != instance->system_identifier) elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instance->name, instance->system_identifier, system_id); + wal_file_name, instanceState->instance_name, instance->system_identifier, system_id); if (instance->compress_alg == PGLZ_COMPRESS) elog(ERROR, "Cannot use pglz for WAL compression"); @@ -165,7 +165,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); /* Create 'archlog_path' directory. Do nothing if it already exists. */ - //fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + //fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST); #ifdef HAVE_LIBZ if (instance->compress_alg == ZLIB_COMPRESS) @@ -206,7 +206,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); rc = push_file(xlogfile, archive_status_dir, - pg_xlog_dir, instance->arclog_path, + pg_xlog_dir, instanceState->instance_wal_subdir_path, overwrite, no_sync, instance->archive_timeout, no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false, @@ -231,7 +231,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, archive_push_arg *arg = &(threads_args[i]); arg->first_filename = wal_file_name; - arg->archive_dir = instance->arclog_path; + arg->archive_dir = instanceState->instance_wal_subdir_path; arg->pg_xlog_dir = pg_xlog_dir; arg->archive_status_dir = archive_status_dir; arg->overwrite = overwrite; @@ -1008,7 +1008,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, */ void -do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, +do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, char *wal_file_name, int batch_size, bool validate_wal) { @@ -1047,7 +1047,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, /* full filepath to WAL file in archive directory. * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ - join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); + join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name); INSTR_TIME_SET_CURRENT(start_time); if (num_threads > batch_size) @@ -1098,7 +1098,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, * copy requested file directly from archive. */ if (!next_wal_segment_exists(tli, segno, prefetch_dir, instance->xlog_seg_size)) - n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, tli, segno, num_threads, false, batch_size, instance->xlog_seg_size); @@ -1137,7 +1137,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, // rmtree(prefetch_dir, false); /* prefetch files */ - n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, tli, segno, num_threads, true, batch_size, instance->xlog_seg_size); diff --git a/src/configure.c b/src/configure.c index b5ef8a356..21e6b6662 100644 --- a/src/configure.c +++ b/src/configure.c @@ -339,8 +339,6 @@ init_config(InstanceConfig *config, const char *instance_name) { MemSet(config, 0, sizeof(InstanceConfig)); - config->name = pgut_strdup(instance_name); - /* * Starting from PostgreSQL 11 WAL segment size may vary. Prior to * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. @@ -591,13 +589,15 @@ readInstanceConfigFile(InstanceState *instanceState) init_config(instance, instanceState->instance_name); - sprintf(instance->backup_instance_path, "%s/%s/%s", +#ifdef REFACTORE_ME + sprintf(instanceState->instance_backup_subdir_path, "%s/%s/%s", instanceState->catalog_state->catalog_path, BACKUPS_DIR, instanceState->instance_name); - canonicalize_path(instance->backup_instance_path); + canonicalize_path(instanceState->instance_backup_subdir_path); - sprintf(instance->arclog_path, "%s/%s/%s", + sprintf(instanceState->instance_wal_subdir_path, "%s/%s/%s", instanceState->catalog_state->catalog_path, "wal", instanceState->instance_name); - canonicalize_path(instance->arclog_path); + canonicalize_path(instanceState->instance_wal_subdir_path); +#endif if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) { diff --git a/src/delete.c b/src/delete.c index e66ce0cd4..3eb1a8f61 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,7 +14,7 @@ #include #include -static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, +static void delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); @@ -698,22 +698,22 @@ do_retention_wal(InstanceState *instanceState, bool dry_run) { if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) { - delete_walfiles_in_tli(tlinfo->anchor_lsn, + delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); } else { - delete_walfiles_in_tli(tlinfo->oldest_backup->start_lsn, + delete_walfiles_in_tli(instanceState, tlinfo->oldest_backup->start_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); } } else { if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) - delete_walfiles_in_tli(tlinfo->anchor_lsn, + delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else - delete_walfiles_in_tli(InvalidXLogRecPtr, + delete_walfiles_in_tli(instanceState, InvalidXLogRecPtr, tlinfo, instance_config.xlog_seg_size, dry_run); } } @@ -806,7 +806,7 @@ delete_backup_files(pgBackup *backup) * Q: Maybe we should stop treating partial WAL segments as second-class citizens? */ static void -delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, +delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tlinfo, uint32 xlog_seg_size, bool dry_run) { XLogSegNo FirstToDeleteSegNo; @@ -931,7 +931,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { char wal_fullpath[MAXPGPATH]; - join_path_components(wal_fullpath, instance_config.arclog_path, wal_file->file.name); + join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name); /* save segment from purging */ if (instance_config.wal_depth >= 0 && wal_file->keep) @@ -1099,12 +1099,12 @@ do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, if (!dry_run && n_deleted > 0) elog(INFO, "Successfully deleted %i %s from instance '%s'", n_deleted, n_deleted == 1 ? "backup" : "backups", - instance_config->name); + instanceState->instance_name); if (n_found == 0) elog(WARNING, "Instance '%s' has no backups with status '%s'", - instance_config->name, pretty_status); + instanceState->instance_name, pretty_status); // we don`t do WAL purge here, because it is impossible to correctly handle // dry-run case. diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 38f78f931..046d3989b 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -470,9 +470,6 @@ main(int argc, char *argv[]) } else { - /* Set instance name */ - instance_config.name = pgut_strdup(instance_name); - instanceState = pgut_new(InstanceState); instanceState->catalog_state = catalogState; @@ -493,19 +490,6 @@ main(int argc, char *argv[]) */ if ((backup_path != NULL) && instance_name) { - /* - * Fill InstanceConfig structure fields used to generate pathes inside - * the instance's backup catalog. - * TODO continue refactoring to use these fields instead of global vars - */ - sprintf(instance_config.backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - canonicalize_path(instance_config.backup_instance_path); - - sprintf(instance_config.arclog_path, "%s/%s/%s", - backup_path, "wal", instance_name); - canonicalize_path(instance_config.arclog_path); - /* * Ensure that requested backup instance exists. * for all commands except init, which doesn't take this parameter, @@ -780,11 +764,11 @@ main(int argc, char *argv[]) switch (backup_subcmd) { case ARCHIVE_PUSH_CMD: - do_archive_push(&instance_config, wal_file_path, wal_file_name, + do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name, batch_size, file_overwrite, no_sync, no_ready_rename); break; case ARCHIVE_GET_CMD: - do_archive_get(&instance_config, prefetch_dir, + do_archive_get(instanceState, &instance_config, prefetch_dir, wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d4dc90281..e2a07d795 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -362,10 +362,6 @@ typedef struct ArchiveOptions */ typedef struct InstanceConfig { - char *name; - char arclog_path[MAXPGPATH]; - char backup_instance_path[MAXPGPATH]; - uint64 system_identifier; uint32 xlog_seg_size; @@ -872,10 +868,10 @@ extern int do_init(CatalogState *catalogState); extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance); /* in archive.c */ -extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, +extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename); -extern void do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, +extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ diff --git a/src/show.c b/src/show.c index ed4f43ef3..dc953a3f6 100644 --- a/src/show.c +++ b/src/show.c @@ -781,9 +781,9 @@ show_instance_archive(InstanceState *instanceState, InstanceConfig *instance) timelineinfos = catalog_get_timelines(instanceState, instance); if (show_format == SHOW_PLAIN) - show_archive_plain(instance->name, instance->xlog_seg_size, timelineinfos, true); + show_archive_plain(instanceState->instance_name, instance->xlog_seg_size, timelineinfos, true); else if (show_format == SHOW_JSON) - show_archive_json(instance->name, instance->xlog_seg_size, timelineinfos); + show_archive_json(instanceState->instance_name, instance->xlog_seg_size, timelineinfos); else elog(ERROR, "Invalid show format %d", (int) show_format); } From 736b9c2a0c1abf209d54e556266c08d3e5422d07 Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 4 Feb 2021 13:34:53 +0300 Subject: [PATCH 1615/2107] Refactoring. Fixes of previous changes --- src/catalog.c | 3 ++- src/configure.c | 11 ----------- src/pg_probackup.h | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index d8b6d109a..909a1af29 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -726,7 +726,6 @@ catalog_get_instance_list(CatalogState *catalogState) continue; instanceState = pgut_new(InstanceState); - instanceState->config = readInstanceConfigFile(instanceState); strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); join_path_components(instanceState->instance_backup_subdir_path, @@ -735,6 +734,8 @@ catalog_get_instance_list(CatalogState *catalogState) catalogState->wal_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_config_path, instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + + instanceState->config = readInstanceConfigFile(instanceState); parray_append(instances, instanceState); } diff --git a/src/configure.c b/src/configure.c index 21e6b6662..53ac8ef78 100644 --- a/src/configure.c +++ b/src/configure.c @@ -589,16 +589,6 @@ readInstanceConfigFile(InstanceState *instanceState) init_config(instance, instanceState->instance_name); -#ifdef REFACTORE_ME - sprintf(instanceState->instance_backup_subdir_path, "%s/%s/%s", - instanceState->catalog_state->catalog_path, BACKUPS_DIR, instanceState->instance_name); - canonicalize_path(instanceState->instance_backup_subdir_path); - - sprintf(instanceState->instance_wal_subdir_path, "%s/%s/%s", - instanceState->catalog_state->catalog_path, "wal", instanceState->instance_name); - canonicalize_path(instanceState->instance_wal_subdir_path); -#endif - if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); @@ -632,7 +622,6 @@ readInstanceConfigFile(InstanceState *instanceState) #endif return instance; - } static void diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e2a07d795..13f1c0724 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -371,7 +371,7 @@ typedef struct InstanceConfig ConnectionOptions conn_opt; ConnectionOptions master_conn_opt; - uint32 replica_timeout; + uint32 replica_timeout; //Deprecated. Not used anywhere /* Wait timeout for WAL segment archiving */ uint32 archive_timeout; From 8041e666cf1eb375ce7853ccccace048d2b9776c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 4 Feb 2021 22:05:10 +0300 Subject: [PATCH 1616/2107] tests: fix tests.auth_test.SimpleAuthTest.test_backup_via_unprivileged_user --- tests/auth_test.py | 58 ++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index eca62316b..c84fdb981 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -34,10 +34,8 @@ def test_backup_via_unprivileged_user(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'} - ) + initdb_params=['--data-checksums']) + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -64,7 +62,15 @@ def test_backup_via_unprivileged_user(self): "GRANT EXECUTE ON FUNCTION" " pg_start_backup(text, boolean, boolean) TO backup;") - time.sleep(1) + if self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup") + else: + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") + try: self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -84,8 +90,6 @@ def test_backup_via_unprivileged_user(self): "GRANT EXECUTE ON FUNCTION" " pg_create_restore_point(text) TO backup;") - time.sleep(1) - try: self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -129,50 +133,18 @@ def test_backup_via_unprivileged_user(self): node.stop() node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on clearing ptrack_files.") - except ProbackupException as e: - self.assertIn( - "ERROR: must be superuser or replication role to clear ptrack files\n" - "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - time.sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on clearing ptrack_files.") - except ProbackupException as e: - self.assertIn( - "ERROR: must be superuser or replication role read ptrack files\n" - "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - node.safe_psql( "postgres", "ALTER ROLE backup REPLICATION") - time.sleep(1) - # FULL self.backup_node( - backup_dir, 'node', node, - options=['-U', 'backup']) + backup_dir, 'node', node, options=['-U', 'backup']) # PTRACK - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-U', 'backup']) +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='ptrack', options=['-U', 'backup']) # Clean after yourself self.del_test_dir(module_name, fname) From 46c14304d758d713f7204e9e68c996c9f6457af0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Feb 2021 06:00:51 +0300 Subject: [PATCH 1617/2107] [Issue #320] incorrect crc calculation for pg_filenode.map --- tests/incr_restore.py | 110 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 3aa84121f..6246ae197 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -10,6 +10,7 @@ import shutil import json from testgres import QueryException +from distutils.dir_util import copy_tree module_name = 'incr_restore' @@ -2390,5 +2391,114 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): # Clean after yourself self.del_test_dir(module_name, fname, [node2]) + def test_incremental_pg_filenode_map(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # in node1 restore full backup + self.restore_node(backup_dir, 'node', node1) + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + pgbench = node1.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node1.stop() + + + # incremental restore into node1 + self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum", '--log-level-file=VERBOSE']) + + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + node1.safe_psql( + 'postgres', + 'select 1') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + + def test_incr_backup_filenode_map(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--log-level-file=VERBOSE']) + + # incremental restore into node1 + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. From 17cc612089bd200972a4ec735379e18bc4533a53 Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 8 Feb 2021 15:04:51 +0300 Subject: [PATCH 1618/2107] tests: remove debug messages in module 'archive' --- tests/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 482a2502b..05675065a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1908,7 +1908,7 @@ def test_archive_push_sanity(self): output = self.show_archive( backup_dir, 'node', as_json=False, as_text=True, - options=['--log-level-console=VERBOSE']) + options=['--log-level-console=INFO']) self.assertNotIn('WARNING', output) @@ -2181,7 +2181,7 @@ def test_archive_get_batching_sanity(self): restore_command += ' -j 2 --batch-size=10' - print(restore_command) + # print(restore_command) if node.major_version >= 12: self.set_auto_conf(replica, {'restore_command': restore_command}) @@ -2298,7 +2298,7 @@ def test_archive_get_prefetch_corruption(self): dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) shutil.copyfile(src_file, dst_file) - print(dst_file) + # print(dst_file) # corrupt file if files[-2].endswith('.gz'): From b2ab1dfef8307cbce8ebd5fa9bcd7d031a99d74c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Feb 2021 18:31:00 +0300 Subject: [PATCH 1619/2107] [Issue #320] always backup and always restore pg_filenode.map --- src/data.c | 5 +++-- src/pg_probackup.h | 3 ++- src/restore.c | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/data.c b/src/data.c index d3f67f43c..f7032fb55 100644 --- a/src/data.c +++ b/src/data.c @@ -711,8 +711,9 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, /* * If nonedata file exists in previous backup * and its mtime is less than parent backup start time ... */ - if (prev_file && file->exists_in_prev && - file->mtime <= parent_backup_time) + if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) && + (prev_file && file->exists_in_prev && + file->mtime <= parent_backup_time)) { file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f4adc98cc..a766e705e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -70,7 +70,8 @@ extern const char *PROGRAM_EMAIL; #define BACKUP_RO_LOCK_FILE "backup_ro.pid" #define DATABASE_FILE_LIST "backup_content.control" #define PG_BACKUP_LABEL_FILE "backup_label" -#define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define RELMAPPER_FILENAME "pg_filenode.map" #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" #define HEADER_MAP "page_header_map" diff --git a/src/restore.c b/src/restore.c index 3f0adf7b7..b3b9965ea 100644 --- a/src/restore.c +++ b/src/restore.c @@ -903,6 +903,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal)) redundant = false; + /* pg_filenode.map are always restored, because it's crc cannot be trusted */ + if (file->external_dir_num == 0 && + pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) + redundant = true; + /* do not delete the useful internal directories */ if (S_ISDIR(file->mode) && !redundant) continue; From 7ecb56e7438da41b7f8b6bb6d2a04963daa74c7a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Feb 2021 22:25:58 +0300 Subject: [PATCH 1620/2107] [Issue #320] improve test coverage --- tests/backup.py | 51 ++++++++++++++++++++++++++++++++++++++ tests/incr_restore.py | 57 +++---------------------------------------- tests/merge.py | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 53 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 8ca96609f..2ee5b9ace 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3027,3 +3027,54 @@ def test_issue_231(self): # Clean after yourself self.del_test_dir(module_name, fname) + + def test_incr_backup_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # incremental restore into node1 + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 6246ae197..4595cbb31 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -2392,7 +2392,9 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.del_test_dir(module_name, fname, [node2]) def test_incremental_pg_filenode_map(self): - """""" + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2436,9 +2438,8 @@ def test_incremental_pg_filenode_map(self): node1.stop() - # incremental restore into node1 - self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum", '--log-level-file=VERBOSE']) + self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum"]) self.set_auto_conf(node1, {'port': node1.port}) node1.slow_start() @@ -2450,55 +2451,5 @@ def test_incremental_pg_filenode_map(self): # Clean after yourself self.del_test_dir(module_name, fname) - - def test_incr_backup_filenode_map(self): - """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), - initdb_params=['--data-checksums']) - node1.cleanup() - - node.pgbench_init(scale=5) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.safe_psql( - 'postgres', - 'reindex index pg_type_oid_index') - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--log-level-file=VERBOSE']) - - # incremental restore into node1 - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'select 1') - - # Clean after yourself - self.del_test_dir(module_name, fname) - # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn # incremental restore + partial restore. diff --git a/tests/merge.py b/tests/merge.py index 44652066a..bf21405bc 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -2829,5 +2829,57 @@ def test_merge_remote_mode(self): self.del_test_dir(module_name, fname) + def test_merge_pg_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # Clean after yourself + self.del_test_dir(module_name, fname) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From def4cb4ae27d40881a713eafa7265173f61b525f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 8 Feb 2021 22:29:21 +0300 Subject: [PATCH 1621/2107] [Issue #320] minor cleanup in "incr_restore" module --- tests/incr_restore.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 4595cbb31..1d3a52df8 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -10,7 +10,6 @@ import shutil import json from testgres import QueryException -from distutils.dir_util import copy_tree module_name = 'incr_restore' @@ -2452,4 +2451,3 @@ def test_incremental_pg_filenode_map(self): self.del_test_dir(module_name, fname) # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn -# incremental restore + partial restore. From 02a3665375f5e578f390b6dc4d60ed824d9bfcb2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 9 Feb 2021 17:32:27 +0300 Subject: [PATCH 1622/2107] [Issue #311] Release shared locks at proc exit --- src/catalog.c | 425 +++++++++++++++++++++++++++++++---------------- tests/locking.py | 59 ++++++- 2 files changed, 336 insertions(+), 148 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 8ac3e5799..e1b5d770f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -26,12 +26,22 @@ static pgBackup *readBackupControlFile(const char *path); static time_t create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; -static parray *lock_files = NULL; +static parray *locks = NULL; -static int lock_backup_exclusive(pgBackup *backup, bool strict); -static bool lock_backup_internal(pgBackup *backup, bool exclusive); -static bool lock_backup_read_only(pgBackup *backup); -static bool wait_read_only_owners(pgBackup *backup); +static int grab_excl_lock_file(const char *backup_dir, const char *backup_id, bool strict); +static int grab_shared_lock_file(pgBackup *backup); +static int wait_shared_owners(pgBackup *backup); + +static void unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive); +static void release_excl_lock_file(const char *backup_dir); +static void release_shared_lock_file(const char *backup_dir); + +typedef struct LockInfo +{ + char backup_id[10]; + char backup_dir[MAXPGPATH]; + bool exclusive; +} LockInfo; static timelineInfo * timelineInfoNew(TimeLineID tli) @@ -66,28 +76,24 @@ timelineInfoFree(void *tliInfo) pfree(tliInfo); } -/* Iterate over locked backups and delete locks files */ +/* Iterate over locked backups and unlock them */ static void unlink_lock_atexit(void) { - int i; + int i; - if (lock_files == NULL) + if (locks == NULL) return; - for (i = 0; i < parray_num(lock_files); i++) + for (i = 0; i < parray_num(locks); i++) { - char *lock_file = (char *) parray_get(lock_files, i); - int res; - - res = fio_unlink(lock_file, FIO_BACKUP_HOST); - if (res != 0 && errno != ENOENT) - elog(WARNING, "%s: %s", lock_file, strerror(errno)); + LockInfo *lock = (LockInfo *) parray_get(locks, i); + unlock_backup(lock->backup_dir, lock->backup_dir, lock->exclusive); } - parray_walk(lock_files, pfree); - parray_free(lock_files); - lock_files = NULL; + parray_walk(locks, pg_free); + parray_free(locks); + locks = NULL; } /* @@ -147,39 +153,39 @@ write_backup_status(pgBackup *backup, BackupStatus status, } /* - * Lock backup in either exclusive or non-exclusive (read-only) mode. + * Lock backup in either exclusive or shared mode. * "strict" flag allows to ignore "out of space" errors and should be * used only by DELETE command to free disk space on filled up * filesystem. * - * Only read only tasks (validate, restore) are allowed to take non-exclusive locks. + * Only read only tasks (validate, restore) are allowed to take shared locks. * Changing backup metadata must be done with exclusive lock. * * Only one process can hold exclusive lock at any time. * Exlusive lock - PID of process, holding the lock - is placed in * lock file: BACKUP_LOCK_FILE. * - * Multiple proccess are allowed to take non-exclusive locks simultaneously. - * Non-exclusive locks - PIDs of proccesses, holding the lock - are placed in + * Multiple proccess are allowed to take shared locks simultaneously. + * Shared locks - PIDs of proccesses, holding the lock - are placed in * separate lock file: BACKUP_RO_LOCK_FILE. - * When taking RO lock, a brief exclusive lock is taken. + * When taking shared lock, a brief exclusive lock is taken. + * + * -> exclusive -> grab exclusive lock file and wait until all shared lockers are gone, return + * -> shared -> grab exclusive lock file, grab shared lock file, release exclusive lock file, return * * TODO: lock-timeout as parameter - * TODO: we must think about more fine grain unlock mechanism - separate unlock_backup() function. - * TODO: more accurate naming - * -> exclusive lock -> acquire HW_LATCH and wait until all LW_LATCH`es are clear - * -> shared lock -> acquire HW_LATCH, acquire LW_LATCH, release HW_LATCH */ bool lock_backup(pgBackup *backup, bool strict, bool exclusive) { - int rc; - char lock_file[MAXPGPATH]; - bool enospc_detected = false; + int rc; + char lock_file[MAXPGPATH]; + bool enospc_detected = false; + LockInfo *lock = NULL; join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); - rc = lock_backup_exclusive(backup, strict); + rc = grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict); if (rc == 1) return false; @@ -188,54 +194,62 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) enospc_detected = true; if (strict) return false; + + /* + * If we failed to take exclusive lock due to ENOSPC, + * then in lax mode treat such condition as if lock was taken. + */ } /* * We have exclusive lock, now there are following scenarios: * - * 1. If we are for exlusive lock, then we must open the RO lock file + * 1. If we are for exlusive lock, then we must open the shared lock file * and check if any of the processes listed there are still alive. * If some processes are alive and are not going away in lock_timeout, * then return false. * * 2. If we are here for non-exlusive lock, then write the pid - * into RO lock list and release the exclusive lock. + * into shared lock file and release the exclusive lock. */ - if (lock_backup_internal(backup, exclusive)) + if (exclusive) + rc = wait_shared_owners(backup); + else + rc = grab_shared_lock_file(backup); + + if (rc != 0) { - if (!exclusive) - { - /* release exclusive lock */ - if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) - elog(ERROR, "Could not remove exclusive lock file \"%s\": %s", - lock_file, strerror(errno)); + /* + * Failed to grab shared lock or (in case of exclusive mode) shared lock owners + * are not going away in time, release the exclusive lock file and return in shame. + */ + release_excl_lock_file(backup->root_dir); + return false; + } - /* we are done */ - return true; - } + if (!exclusive) + { + /* Shared lock file is grabbed, now we can release exclusive lock file */ + release_excl_lock_file(backup->root_dir); + } - /* When locking backup in lax exclusive mode, - * we should wait until all RO locks owners are gone. + if (exclusive && !strict && enospc_detected) + { + /* We are in lax exclusive mode and EONSPC was encountered: + * once again try to grab exclusive lock file, + * because there is a chance that release of shared lock file in wait_shared_owners may have + * freed some space on filesystem, thanks to unlinking of BACKUP_RO_LOCK_FILE. + * If somebody concurrently acquired exclusive lock file first, then we should give up. */ - if (!strict && enospc_detected) - { - /* We are in lax mode and EONSPC was encountered: once again try to grab exclusive lock, - * because there is a chance that lock_backup_read_only may have freed some space on filesystem, - * thanks to unlinking of BACKUP_RO_LOCK_FILE. - * If somebody concurrently acquired exclusive lock first, then we should give up. - */ - if (lock_backup_exclusive(backup, strict) == 1) - return false; + if (grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict) == 1) + return false; - return true; - } + return true; } - else - return false; /* - * Arrange to unlink the lock file(s) at proc_exit. + * Arrange the unlocking at proc_exit. */ if (!backup_lock_exit_hook_registered) { @@ -243,10 +257,16 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) backup_lock_exit_hook_registered = true; } - /* Use parray so that the lock files are unlinked in a loop */ - if (lock_files == NULL) - lock_files = parray_new(); - parray_append(lock_files, pgut_strdup(lock_file)); + /* save lock metadata for later unlocking */ + lock = pgut_malloc(sizeof(LockInfo)); + snprintf(lock->backup_id, 10, "%s", base36enc(backup->backup_id)); + snprintf(lock->backup_dir, MAXPGPATH, "%s", backup->root_dir); + lock->exclusive = exclusive; + + /* Use parray for lock release */ + if (locks == NULL) + locks = parray_new(); + parray_append(locks, lock); return true; } @@ -258,7 +278,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) * 2 Failed to acquire lock due to ENOSPC */ int -lock_backup_exclusive(pgBackup *backup, bool strict) +grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) { char lock_file[MAXPGPATH]; int fd = 0; @@ -268,7 +288,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) int len; int encoded_pid; - join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); + join_path_components(lock_file, root_dir, BACKUP_LOCK_FILE); /* * We need a loop here because of race conditions. But don't loop forever @@ -280,8 +300,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) FILE *fp_out = NULL; if (interrupted) - elog(ERROR, "Interrupted while locking backup %s", - base36enc(backup->start_time)); + elog(ERROR, "Interrupted while locking backup %s", backup_id); /* * Try to create the lock file --- O_EXCL makes this atomic. @@ -344,7 +363,7 @@ lock_backup_exclusive(pgBackup *backup, bool strict) if ((empty_tries % LOG_FREQ) == 0) elog(WARNING, "Waiting %u seconds on empty exclusive lock for backup %s", - empty_tries, base36enc(backup->start_time)); + empty_tries, backup_id); sleep(1); /* @@ -372,34 +391,32 @@ lock_backup_exclusive(pgBackup *backup, bool strict) */ if (encoded_pid == my_pid) return 0; - else + + if (kill(encoded_pid, 0) == 0) { - if (kill(encoded_pid, 0) == 0) + /* complain every fifth interval */ + if ((ntries % LOG_FREQ) == 0) { - /* complain every fifth interval */ - if ((ntries % LOG_FREQ) == 0) - { - elog(WARNING, "Process %d is using backup %s, and is still running", - encoded_pid, base36enc(backup->start_time)); + elog(WARNING, "Process %d is using backup %s, and is still running", + encoded_pid, backup_id); - elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", - ntries, base36enc(backup->start_time)); - } + elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", + ntries, backup_id); + } - sleep(1); + sleep(1); - /* try again */ - continue; - } + /* try again */ + continue; + } + else + { + if (errno == ESRCH) + elog(WARNING, "Process %d which used backup %s no longer exists", + encoded_pid, backup_id); else - { - if (errno == ESRCH) - elog(WARNING, "Process %d which used backup %s no longer exists", - encoded_pid, base36enc(backup->start_time)); - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); } grab_lock: @@ -484,11 +501,14 @@ lock_backup_exclusive(pgBackup *backup, bool strict) return 0; } -/* Wait until all read-only lock owners are gone */ -bool -wait_read_only_owners(pgBackup *backup) +/* Wait until all shared lock owners are gone + * 0 - successs + * 1 - fail + */ +int +wait_shared_owners(pgBackup *backup) { - FILE *fp = NULL; + FILE *fp = NULL; char buffer[256]; pid_t encoded_pid; int ntries = LOCK_TIMEOUT; @@ -500,7 +520,7 @@ wait_read_only_owners(pgBackup *backup) if (fp == NULL && errno != ENOENT) elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - /* iterate over pids in lock file */ + /* iterate over pids in lock file */ while (fp && fgets(buffer, sizeof(buffer), fp)) { encoded_pid = atoi(buffer); @@ -510,47 +530,42 @@ wait_read_only_owners(pgBackup *backup) continue; } - /* wait until RO lock owners go away */ + /* wait until shared lock owners go away */ do { if (interrupted) elog(ERROR, "Interrupted while locking backup %s", base36enc(backup->start_time)); - if (encoded_pid != my_pid) + if (encoded_pid == my_pid) + break; + + /* check if lock owner is still alive */ + if (kill(encoded_pid, 0) == 0) { - if (kill(encoded_pid, 0) == 0) + /* complain from time to time */ + if ((ntries % LOG_FREQ) == 0) { - if ((ntries % LOG_FREQ) == 0) - { - elog(WARNING, "Process %d is using backup %s in read only mode, and is still running", - encoded_pid, base36enc(backup->start_time)); + elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", + encoded_pid, base36enc(backup->start_time)); - elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, - base36enc(backup->start_time)); - } + elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, + base36enc(backup->start_time)); + } - sleep(1); + sleep(1); - /* try again */ - continue; - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + /* try again */ + continue; } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); /* locker is dead */ break; } while (ntries--); - - if (ntries <= 0) - { - elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns read-only lock", - base36enc(backup->start_time), encoded_pid); - return false; - } } if (fp && ferror(fp)) @@ -559,22 +574,26 @@ wait_read_only_owners(pgBackup *backup) if (fp) fclose(fp); - /* unlink RO lock list */ - fio_unlink(lock_file, FIO_BACKUP_HOST); - return true; -} + /* some shared owners are still alive */ + if (ntries <= 0) + { + elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", + base36enc(backup->start_time), encoded_pid); + return 1; + } -bool -lock_backup_internal(pgBackup *backup, bool exclusive) -{ - if (exclusive) - return wait_read_only_owners(backup); - else - return lock_backup_read_only(backup); + /* unlink shared lock file */ + fio_unlink(lock_file, FIO_BACKUP_HOST); + return 0; } -bool -lock_backup_read_only(pgBackup *backup) +/* + * Lock backup in shared mode + * 0 - successs + * 1 - fail + */ +int +grab_shared_lock_file(pgBackup *backup) { FILE *fp_in = NULL; FILE *fp_out = NULL; @@ -604,20 +623,20 @@ lock_backup_read_only(pgBackup *backup) continue; } - if (encoded_pid != my_pid) + if (encoded_pid == my_pid) + continue; + + if (kill(encoded_pid, 0) == 0) { - if (kill(encoded_pid, 0) == 0) - { - /* - * Somebody is still using this backup in RO mode, - * copy this pid into a new file. - */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); + /* + * Somebody is still using this backup in shared mode, + * copy this pid into a new file. + */ + buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); } if (fp_in) @@ -647,7 +666,123 @@ lock_backup_read_only(pgBackup *backup) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", lock_file_tmp, lock_file, strerror(errno)); - return true; + return 0; +} + +void +unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive) +{ + if (exclusive) + { + release_excl_lock_file(backup_dir); + return; + } + + /* To remove shared lock, we must briefly obtain exclusive lock, ... */ + if (grab_excl_lock_file(backup_dir, backup_id, false) != 0) + /* ... if it's not possible then leave shared lock */ + return; + + release_shared_lock_file(backup_dir); + release_excl_lock_file(backup_dir); +} + +void +release_excl_lock_file(const char *backup_dir) +{ + char lock_file[MAXPGPATH]; + + join_path_components(lock_file, backup_dir, BACKUP_LOCK_FILE); + + /* TODO Sanity check: maybe we should check, that pid in lock file is my_pid */ + + /* unlink pid file */ + fio_unlink(lock_file, FIO_BACKUP_HOST); +} + +void +release_shared_lock_file(const char *backup_dir) +{ + FILE *fp_in = NULL; + FILE *fp_out = NULL; + char buf_in[256]; + pid_t encoded_pid; + char lock_file[MAXPGPATH]; + + char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ + char lock_file_tmp[MAXPGPATH]; + int buffer_len = 0; + + join_path_components(lock_file, backup_dir, BACKUP_RO_LOCK_FILE); + snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + + /* open lock file */ + fp_in = fopen(lock_file, "r"); + if (fp_in == NULL) + { + if (errno == ENOENT) + return; + else + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + } + + /* read PIDs of owners */ + while (fgets(buf_in, sizeof(buf_in), fp_in)) + { + encoded_pid = atoi(buf_in); + + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); + continue; + } + + /* remove my pid */ + if (encoded_pid == my_pid) + continue; + + if (kill(encoded_pid, 0) == 0) + { + /* + * Somebody is still using this backup in shared mode, + * copy this pid into a new file. + */ + buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } + + if (ferror(fp_in)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_in); + + /* if there is no active pid left, then there is nothing to do */ + if (buffer_len == 0) + { + fio_unlink(lock_file, FIO_BACKUP_HOST); + return; + } + + fp_out = fopen(lock_file_tmp, "w"); + if (fp_out == NULL) + elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + /* write out the collected PIDs to temp lock file */ + fwrite(buffer, 1, buffer_len, fp_out); + + if (ferror(fp_out)) + elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); + + if (fclose(fp_out) != 0) + elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + if (rename(lock_file_tmp, lock_file) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + lock_file_tmp, lock_file, strerror(errno)); + + return; } /* diff --git a/tests/locking.py b/tests/locking.py index 540007838..ef7aa1f25 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -581,6 +581,59 @@ def test_empty_lock_file(self): # Clean after yourself self.del_test_dir(module_name, fname) -# TODO: -# test that concurrent validation and restore are not locking each other -# check that quick exclusive lock, when taking RO-lock, is really quick + def test_shared_lock(self): + """ + Make sure that shared lock leaves no files with pids + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile_excl = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + lockfile_shr = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup_ro.pid') + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + + gdb = self.validate_pb(backup_dir, 'node', backup_id, gdb=True) + + gdb.set_breakpoint('validate_one_page') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + os.path.exists(lockfile_shr), + "File should exist: {0}".format(lockfile_shr)) + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From c30628cd1da24f85da8f9fc9316bd8115348efd2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Feb 2021 14:11:57 +0300 Subject: [PATCH 1623/2107] [Issue #237] Ignore EROFS when locking backup in shared mode --- src/catalog.c | 64 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index e1b5d770f..b9d384b69 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -36,6 +36,11 @@ static void unlock_backup(const char *backup_dir, const char *backup_id, bool ex static void release_excl_lock_file(const char *backup_dir); static void release_shared_lock_file(const char *backup_dir); +#define LOCK_OK 0 +#define LOCK_FAIL_TIMEOUT 1 +#define LOCK_FAIL_ENOSPC 2 +#define LOCK_FAIL_EROFS 3 + typedef struct LockInfo { char backup_id[10]; @@ -187,18 +192,26 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) rc = grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict); - if (rc == 1) + if (rc == LOCK_FAIL_TIMEOUT) return false; - else if (rc == 2) + else if (rc == LOCK_FAIL_ENOSPC) { + /* + * If we failed to take exclusive lock due to ENOSPC, + * then in lax mode treat such condition as if lock was taken. + */ + enospc_detected = true; if (strict) return false; - + } + else if (rc == LOCK_FAIL_EROFS) + { /* - * If we failed to take exclusive lock due to ENOSPC, - * then in lax mode treat such condition as if lock was taken. + * If we failed to take exclusive lock due to EROFS, + * then in shared mode treat such condition as if lock was taken. */ + return !exclusive; } /* @@ -242,7 +255,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) * freed some space on filesystem, thanks to unlinking of BACKUP_RO_LOCK_FILE. * If somebody concurrently acquired exclusive lock file first, then we should give up. */ - if (grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict) == 1) + if (grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict) == LOCK_FAIL_TIMEOUT) return false; return true; @@ -271,18 +284,20 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) return true; } -/* Lock backup in exclusive mode +/* + * Lock backup in exclusive mode * Result codes: - * 0 Success - * 1 Failed to acquire lock in lock_timeout time - * 2 Failed to acquire lock due to ENOSPC + * LOCK_OK Success + * LOCK_FAIL_TIMEOUT Failed to acquire lock in lock_timeout time + * LOCK_FAIL_ENOSPC Failed to acquire lock due to ENOSPC + * LOCK_FAIL_EROFS Failed to acquire lock due to EROFS */ int grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) { char lock_file[MAXPGPATH]; int fd = 0; - char buffer[MAXPGPATH * 2 + 256]; + char buffer[256]; int ntries = LOCK_TIMEOUT; int empty_tries = LOCK_STALE_TIMEOUT; int len; @@ -312,6 +327,14 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) if (fd >= 0) break; /* Success; exit the retry loop */ + /* read-only fs is a special case */ + if (errno == EROFS) + { + elog(WARNING, "Could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + return LOCK_FAIL_EROFS; + } + /* * Couldn't create the pid file. Probably it already exists. * If file already exists or we have some permission problem (???), @@ -390,7 +413,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * exist. */ if (encoded_pid == my_pid) - return 0; + return LOCK_OK; if (kill(encoded_pid, 0) == 0) { @@ -437,7 +460,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) /* Failed to acquire exclusive lock in time */ if (fd <= 0) - return 1; + return LOCK_FAIL_TIMEOUT; /* * Successfully created the file, now fill it. @@ -457,7 +480,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * Only delete command should be run in lax mode. */ if (!strict && save_errno == ENOSPC) - return 2; + return LOCK_FAIL_ENOSPC; else elog(ERROR, "Could not write lock file \"%s\": %s", lock_file, strerror(save_errno)); @@ -475,7 +498,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * Only delete command should be run in lax mode. */ if (!strict && save_errno == ENOSPC) - return 2; + return LOCK_FAIL_ENOSPC; else elog(ERROR, "Could not flush lock file \"%s\": %s", lock_file, strerror(save_errno)); @@ -488,7 +511,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) fio_unlink(lock_file, FIO_BACKUP_HOST); if (!strict && errno == ENOSPC) - return 2; + return LOCK_FAIL_ENOSPC; else elog(ERROR, "Could not close lock file \"%s\": %s", lock_file, strerror(save_errno)); @@ -498,7 +521,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) // base36enc(backup->start_time), // LOCK_TIMEOUT - ntries + LOCK_STALE_TIMEOUT - empty_tries); - return 0; + return LOCK_OK; } /* Wait until all shared lock owners are gone @@ -648,7 +671,12 @@ grab_shared_lock_file(pgBackup *backup) fp_out = fopen(lock_file_tmp, "w"); if (fp_out == NULL) + { + if (errno == EROFS) + return 0; + elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + } /* add my own pid */ buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); @@ -679,7 +707,7 @@ unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive) } /* To remove shared lock, we must briefly obtain exclusive lock, ... */ - if (grab_excl_lock_file(backup_dir, backup_id, false) != 0) + if (grab_excl_lock_file(backup_dir, backup_id, false) != LOCK_OK) /* ... if it's not possible then leave shared lock */ return; From 4a4af6270fc3a06195e604064cccc359080526a4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Feb 2021 14:25:13 +0300 Subject: [PATCH 1624/2107] bugfix: for PG>=12 instance with missing postgresql.auto.conf restore now correctly recreates postgresql.auto.conf --- src/restore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/restore.c b/src/restore.c index b3b9965ea..adc20bb6c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1543,6 +1543,7 @@ update_recovery_options(pgBackup *backup, if (errno != ENOENT) elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path, strerror(errno)); + st.st_size = 0; } /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ From 63d79e2b4fac043e29ea4d54589fbb8788a18ffd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 10 Feb 2021 15:10:15 +0300 Subject: [PATCH 1625/2107] [Issue #323] stable remote agent API --- src/pg_probackup.h | 3 +++ src/utils/remote.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a766e705e..c15e6efd8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -307,7 +307,10 @@ typedef enum ShowFormat #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) #define PROGRAM_VERSION "2.4.9" + +/* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 +#define AGENT_PROTOCOL_VERSION_STR "2.4.9" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" diff --git a/src/utils/remote.c b/src/utils/remote.c index 88b1ce7a6..2bfd24d1e 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -247,8 +247,9 @@ bool launch_agent(void) (agent_version / 100) % 100, agent_version % 100); - elog(ERROR, "Remote agent version %s does not match local program version %s", - agent_version_str, PROGRAM_VERSION); + elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " + "consider to upgrade pg_probackup binary", + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } return true; From 6dcf64a90705411e2764800ff8dd2ccb5fc1a4d3 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 10 Feb 2021 16:22:40 +0300 Subject: [PATCH 1626/2107] Tests. Fix cfs_backup tests compatibility with ptrack 2.1 --- tests/cfs_backup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 5a3665518..3d2fa8de7 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -20,9 +20,9 @@ def setUp(self): self.node = self.make_simple_node( base_dir="{0}/{1}/node".format(module_name, self.fname), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'cfs_encryption': 'off', 'max_wal_senders': '2', 'shared_buffers': '200MB' @@ -35,6 +35,11 @@ def setUp(self): self.node.slow_start() + if self.node.major_version >= 12: + self.node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) tblspace = self.node.safe_psql( From 2305d1fdda20fc16af053a06805416de0d2e69de Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 10 Feb 2021 19:44:38 +0300 Subject: [PATCH 1627/2107] Tests. Expect failure in test_validate_target_lsn, because it requires multi-timeline WAL validation, which is not implemented yet --- tests/validate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/validate.py b/tests/validate.py index 6777b92a1..cfec234d0 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3451,7 +3451,8 @@ def test_validate_corrupt_tablespace_map(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.expectedFailure + #TODO fix the test + @unittest.expectedFailure # @unittest.skip("skip") def test_validate_target_lsn(self): """ From bdcc28d36cd227635d8a6dba8e2498d91069d27b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Feb 2021 09:51:38 +0300 Subject: [PATCH 1628/2107] tests: minor fixes for backup and merge modules --- tests/backup.py | 1 + tests/merge.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index 2ee5b9ace..70c4dc13f 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -5,6 +5,7 @@ import shutil from distutils.dir_util import copy_tree from testgres import ProcessType +import subprocess module_name = 'backup' diff --git a/tests/merge.py b/tests/merge.py index bf21405bc..29d60433c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -7,6 +7,7 @@ import shutil from datetime import datetime, timedelta import time +import subprocess module_name = "merge" From 5207900fb843572398513998b6ec158d74473d23 Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 11 Feb 2021 12:01:13 +0300 Subject: [PATCH 1629/2107] Update tests/Readme.md. Add note about python3. --- tests/Readme.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 3adf0c019..adcf5380e 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,7 +1,11 @@ -[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) +[see wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` -Note: For now these are works on Linux and "kinda" works on Windows +Note: For now these tests work on Linux and "kinda" work on Windows +``` + +``` +Note: tests require python3 to work properly. ``` ``` From 71661abc3f4930d5d960f85dda0deae71a31fc63 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Feb 2021 19:28:42 +0300 Subject: [PATCH 1630/2107] tests: fix "cfs_backup" module --- tests/cfs_backup.py | 76 ++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 3d2fa8de7..b7ebff0fe 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -45,13 +45,17 @@ def setUp(self): tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name) - ) - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, + tblspace_name)) + + self.assertIn( + tblspace_name, str(tblspace), "ERROR: The tablespace not created " - "or it create without compressions" - ) + "or it create without compressions") + + self.assertIn( + "compression=true", str(tblspace), + "ERROR: The tablespace not created " + "or it create without compressions") self.assertTrue( find_by_name( @@ -743,12 +747,14 @@ def test_multiple_segments(self): # CHECK FULL BACKUP self.node.stop() self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_full, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( full_result, @@ -762,8 +768,12 @@ def test_multiple_segments(self): self.get_tblspace_path(self.node, tblspace_name), ignore_errors=True) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_page, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( page_result, @@ -791,8 +801,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1) - ) + 't_heap_1', tblspace_name_1)) self.node.safe_psql( "postgres", @@ -800,8 +809,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2) - ) + 't_heap_2', tblspace_name_2)) full_result_1 = self.node.safe_psql( "postgres", "SELECT * FROM t_heap_1") @@ -869,21 +877,16 @@ def test_multiple_segments_in_multiple_tablespaces(self): # CHECK FULL BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + backup_id=backup_id_full, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( full_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), @@ -895,21 +898,16 @@ def test_multiple_segments_in_multiple_tablespaces(self): # CHECK PAGE BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + backup_id=backup_id_page, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( page_result_1, self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), From f324b081809b2f27c28f66b1cf2f878f55131c25 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 11 Feb 2021 20:27:47 +0300 Subject: [PATCH 1631/2107] tests: more fixes for "cfs_backup" module --- tests/cfs_backup.py | 8 ++++---- tests/helpers/cfs_helpers.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index b7ebff0fe..2e686d46c 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -482,7 +482,7 @@ def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): ) # --- Section: Incremental from fill tablespace --- # - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table(self): @@ -546,7 +546,7 @@ def test_fullbackup_after_create_table_ptrack_after_create_table(self): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): @@ -612,7 +612,7 @@ def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table(self): @@ -917,7 +917,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), 'Lost data after restore') - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table_stream(self): diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py index 67e2b331b..31af76f2e 100644 --- a/tests/helpers/cfs_helpers.py +++ b/tests/helpers/cfs_helpers.py @@ -88,4 +88,6 @@ def corrupt_file(filename): def random_string(n): a = string.ascii_letters + string.digits - return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file + random_str = ''.join([random.choice(a) for i in range(int(n)+1)]) + return str.encode(random_str) +# return ''.join([random.choice(a) for i in range(int(n)+1)]) From 31a8ea3ff04c226a12ed93db3aa82b4fd959367b Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 12 Feb 2021 13:08:12 +0300 Subject: [PATCH 1632/2107] tests: Skip test_incr_lsn_long_xact_1, when testing with PostgresPro Enterprise --- tests/incr_restore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 1d3a52df8..a228bdd79 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1711,6 +1711,9 @@ def test_incr_checksum_long_xact(self): # @unittest.skip("skip") # @unittest.expectedFailure + # This test will pass with Enterprise + # because it has checksums enabled by default + @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """ From 8691e08e220804248dd1c241279a98fb7f050d50 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Feb 2021 15:32:51 +0300 Subject: [PATCH 1633/2107] tests: some minor fixes for "page" module --- tests/page.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/page.py b/tests/page.py index 323c0a6de..c0dbb69f2 100644 --- a/tests/page.py +++ b/tests/page.py @@ -757,8 +757,6 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -782,8 +780,6 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -872,8 +868,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -896,8 +890,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -997,8 +989,6 @@ def test_page_backup_with_alien_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -1020,8 +1010,6 @@ def test_page_backup_with_alien_wal_segment(self): "Output: {0} \n CMD: {1}".format( self.output, self.cmd)) except ProbackupException as e: - self.assertIn('INFO: Wait for WAL segment', e.message) - self.assertIn('to be archived', e.message) self.assertIn('Could not read WAL record at', e.message) self.assertIn('WAL file is from different database system: ' 'WAL file database system identifier is', e.message) From 212fad4f931e1269d0eb290e33c5a57435ff9582 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 12 Feb 2021 16:48:51 +0300 Subject: [PATCH 1634/2107] Version 2.4.10 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c15e6efd8..1e001bd05 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -306,7 +306,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.9" +#define PROGRAM_VERSION "2.4.10" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1481e32db..b5204e46e 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.9 \ No newline at end of file +pg_probackup 2.4.10 \ No newline at end of file From 8e930b79abc2f2a0440eb5e1528fe562f2a971c9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 13 Feb 2021 04:03:04 +0300 Subject: [PATCH 1635/2107] make compiler happy --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index b9d384b69..423e83f7b 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -533,7 +533,7 @@ wait_shared_owners(pgBackup *backup) { FILE *fp = NULL; char buffer[256]; - pid_t encoded_pid; + pid_t encoded_pid = 0; int ntries = LOCK_TIMEOUT; char lock_file[MAXPGPATH]; From b11bd8eb74b829509dc963aa674fedf4148a3fe9 Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Sun, 14 Feb 2021 10:34:49 +0300 Subject: [PATCH 1636/2107] fix: clean node data after test --- tests/helpers/ptrack_helpers.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 833e95a36..9af7cbc1c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -145,6 +145,7 @@ def slow_start(self, replica=False): class ProbackupTest(object): # Class attributes enterprise = is_enterprise() + nodes = [] def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) @@ -402,7 +403,7 @@ def make_simple_node( if node.major_version >= 13: self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) - + self.nodes.append(node) return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): @@ -1521,8 +1522,17 @@ def del_test_dir(self, module_name, fname, nodes=[]): except: pass - for node in nodes: - node.stop() + try: + if not nodes: + for node in list(self.nodes): + node.stop() + self.nodes.remove(node) + else: + for node in list(nodes): + node.stop() + self.nodes.remove(node) + except: + pass shutil.rmtree( os.path.join( @@ -1533,7 +1543,7 @@ def del_test_dir(self, module_name, fname, nodes=[]): ignore_errors=True ) try: - os.rmdir(os.path.join(self.tmp_path, module_name)) + shutil.rmtree(os.path.join(self.tmp_path, module_name), ignore_errors=True) except: pass From 6ff3633d1686a0e5a87d2989a756cdaba12644d9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 14 Feb 2021 23:45:54 +0300 Subject: [PATCH 1637/2107] bump year in license --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index dc4e8b8d5..0ba831507 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2019, Postgres Professional +Copyright (c) 2015-2020, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group From 86b71f934d9a951a2a3d14842705307dd01528b7 Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Mon, 15 Feb 2021 06:32:28 +0300 Subject: [PATCH 1638/2107] clear data nodes after run test --- tests/helpers/ptrack_helpers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 833e95a36..623eac708 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -145,6 +145,7 @@ def slow_start(self, replica=False): class ProbackupTest(object): # Class attributes enterprise = is_enterprise() + nodes = [] def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) @@ -402,7 +403,7 @@ def make_simple_node( if node.major_version >= 13: self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) - + self.nodes.append(node) return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): @@ -1521,8 +1522,16 @@ def del_test_dir(self, module_name, fname, nodes=[]): except: pass - for node in nodes: - node.stop() + if not nodes: + nodes = self.nodes + for node in list(nodes): + try: + if node.status() == 0: + node.stop() + except: + pass + if node in self.nodes: + self.nodes.remove(node) shutil.rmtree( os.path.join( From fb5debda8ab7a9d51feb30b226e1c9f43f431f7b Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Mon, 15 Feb 2021 10:24:04 +0300 Subject: [PATCH 1639/2107] fix for test test_pgpro434_4 (not zero code exit) --- tests/helpers/ptrack_helpers.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 623eac708..e8374c981 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1528,23 +1528,20 @@ def del_test_dir(self, module_name, fname, nodes=[]): try: if node.status() == 0: node.stop() - except: - pass - if node in self.nodes: - self.nodes.remove(node) + except Exception as e: + print("Error stop node ", e) + raise + finally: + if node in self.nodes: + self.nodes.remove(node) shutil.rmtree( os.path.join( self.tmp_path, - module_name, - fname + module_name ), ignore_errors=True ) - try: - os.rmdir(os.path.join(self.tmp_path, module_name)) - except: - pass def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): """ return dict with directory content. " From 5ea3615711e526102f67586cc150c1eb361e9a9a Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Mon, 15 Feb 2021 10:32:32 +0300 Subject: [PATCH 1640/2107] fix --- tests/helpers/ptrack_helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e8374c981..0f8cd2130 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1528,8 +1528,7 @@ def del_test_dir(self, module_name, fname, nodes=[]): try: if node.status() == 0: node.stop() - except Exception as e: - print("Error stop node ", e) + except: raise finally: if node in self.nodes: From 4261d5eba46648943172bb273544618b286dcaa6 Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Mon, 15 Feb 2021 10:49:16 +0300 Subject: [PATCH 1641/2107] fix --- tests/helpers/ptrack_helpers.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b03467c02..0f8cd2130 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1541,13 +1541,6 @@ def del_test_dir(self, module_name, fname, nodes=[]): ), ignore_errors=True ) -<<<<<<< HEAD -======= - try: - shutil.rmtree(os.path.join(self.tmp_path, module_name), ignore_errors=True) - except: - pass ->>>>>>> 7f1d12b457f54ddebc0b6f65a1124e7f75dd50ae def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): """ return dict with directory content. " From 294e7dc2056cf4bfc188b8604d678f1295a352a7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Feb 2021 14:53:52 +0300 Subject: [PATCH 1642/2107] [Issue #326] added tests.archive.ArchiveTest.test_archive_empty_history_file --- tests/archive.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tests/archive.py b/tests/archive.py index 05675065a..e651ee77b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2474,6 +2474,96 @@ def test_archive_show_partial_files_handling(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_empty_history_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/326 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s', + 'autovacuum': 'off'}) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=5) + + # FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote']) + + # '--recovery-target-timeline=2', + # Node in timeline 2 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + # Node in timeline 3 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + # Node in timeline 4 + node.slow_start() + node.pgbench_init(scale=1) + + # Truncate history files + for tli in range(2, 4): + file = os.path.join( + backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) + with open(file, "w+") as f: + f.truncate() + + show = self.show_archive(backup_dir, 'node') + + timelines = show['timelines'] + + # check that all timelines are ok + for timeline in replica_timelines: + print(timeline) + + self.del_test_dir(module_name, fname) + # TODO test with multiple not archived segments. # TODO corrupted file in archive. From 74cd9c54dfc79179f95350c14fcbe6b1f268e478 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Feb 2021 15:49:30 +0300 Subject: [PATCH 1643/2107] [Issue #326] Handle empty history files correctly --- src/catalog.c | 4 ++++ src/restore.c | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/catalog.c b/src/catalog.c index 423e83f7b..15386d426 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1581,6 +1581,10 @@ catalog_get_timelines(InstanceConfig *instance) sscanf(file->name, "%08X.history", &tli); timelines = read_timeline_history(arclog_path, tli, true); + /* History file is empty or corrupted, disregard it */ + if (!timelines) + continue; + if (!tlinfo || tlinfo->tli != tli) { tlinfo = timelineInfoNew(tli); diff --git a/src/restore.c b/src/restore.c index adc20bb6c..f28265ccd 100644 --- a/src/restore.c +++ b/src/restore.c @@ -289,6 +289,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* Read timeline history files from archives */ timelines = read_timeline_history(arclog_path, rt->target_tli, true); + if (!timelines) + elog(WARNING, "Failed to get history file for target timeline %i", rt->target_tli); + if (!satisfy_timeline(timelines, current_backup)) { if (target_backup_id != INVALID_BACKUP_ID) @@ -1778,6 +1781,14 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict if (last_timeline && targetTLI <= last_timeline->tli) elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + /* History file is empty or corrupted */ + if (parray_num(result) != 1) + { + elog(WARNING, "History file is corrupted: \"%s\"", path); + pg_free(result); + return NULL; + } + /* append target timeline */ entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; From 4ab117bd3c0caacef09b68105ec769fb577614a8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 15 Feb 2021 16:19:55 +0300 Subject: [PATCH 1644/2107] [Issue #326] some minor fixes --- src/restore.c | 4 ++-- tests/archive.py | 40 +++++++++++++++++++++------------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/restore.c b/src/restore.c index f28265ccd..1a22a9a28 100644 --- a/src/restore.c +++ b/src/restore.c @@ -290,7 +290,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, timelines = read_timeline_history(arclog_path, rt->target_tli, true); if (!timelines) - elog(WARNING, "Failed to get history file for target timeline %i", rt->target_tli); + elog(ERROR, "Failed to get history file for target timeline %i", rt->target_tli); if (!satisfy_timeline(timelines, current_backup)) { @@ -1782,7 +1782,7 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict elog(ERROR, "Timeline IDs must be less than child timeline's ID."); /* History file is empty or corrupted */ - if (parray_num(result) != 1) + if (parray_num(result) == 0) { elog(WARNING, "History file is corrupted: \"%s\"", path); pg_free(result); diff --git a/tests/archive.py b/tests/archive.py index e651ee77b..0cf2f703f 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2485,23 +2485,13 @@ def test_archive_empty_history_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) - - if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( - 'Skipped because backup from replica is not supported in PG 9.5') + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - node.pgbench_init(scale=5) # FULL @@ -2516,7 +2506,6 @@ def test_archive_empty_history_file(self): '--recovery-target=latest', '--recovery-target-action=promote']) - # '--recovery-target-timeline=2', # Node in timeline 2 node.slow_start() @@ -2545,22 +2534,35 @@ def test_archive_empty_history_file(self): # Node in timeline 4 node.slow_start() - node.pgbench_init(scale=1) + node.pgbench_init(scale=5) # Truncate history files - for tli in range(2, 4): + for tli in range(2, 5): file = os.path.join( backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) with open(file, "w+") as f: f.truncate() - show = self.show_archive(backup_dir, 'node') + timelines = self.show_archive(backup_dir, 'node', options=['--log-level-file=INFO']) - timelines = show['timelines'] + # check that all timelines has zero switchpoint + for timeline in timelines: + self.assertEqual(timeline['switchpoint'], '0/0') - # check that all timelines are ok - for timeline in replica_timelines: - print(timeline) + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + wal_dir = os.path.join(backup_dir, 'wal', 'node') + + self.assertIn( + 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), + log_content) self.del_test_dir(module_name, fname) From 92dfb3a31313a37e6e5e355bdbf51c58d10de914 Mon Sep 17 00:00:00 2001 From: kav23alex Date: Tue, 16 Feb 2021 09:34:01 +0700 Subject: [PATCH 1645/2107] clean data after run test --- tests/helpers/ptrack_helpers.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0f8cd2130..dbd0d7feb 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,5 +1,6 @@ # you need os for unittest to work import os +import gc from sys import exit, argv, version_info import subprocess import shutil @@ -1515,29 +1516,24 @@ def get_version(self, node): def get_bin_path(self, binary): return testgres.get_bin_path(binary) + def clean_all(self): + for o in gc.get_referrers(testgres.PostgresNode): + if o.__class__ is testgres.PostgresNode: + o.cleanup() + def del_test_dir(self, module_name, fname, nodes=[]): """ Del testdir and optimistically try to del module dir""" try: - testgres.clean_all() - except: - pass - - if not nodes: - nodes = self.nodes - for node in list(nodes): - try: - if node.status() == 0: - node.stop() - except: - raise - finally: - if node in self.nodes: - self.nodes.remove(node) + self.clean_all() + except Exception as e: + raise e shutil.rmtree( os.path.join( self.tmp_path, - module_name + module_name, + fname, + "backup" ), ignore_errors=True ) From f69d4a1d1878ed419f7152b0c1bd9d0223c488c9 Mon Sep 17 00:00:00 2001 From: kav23alex Date: Tue, 16 Feb 2021 10:12:46 +0700 Subject: [PATCH 1646/2107] merge From f29ab895b386ad23ccb624fc4f673642b4650323 Mon Sep 17 00:00:00 2001 From: kav23alex Date: Tue, 16 Feb 2021 10:15:27 +0700 Subject: [PATCH 1647/2107] remove self.nodes --- tests/helpers/ptrack_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index dbd0d7feb..028e9cd16 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -146,7 +146,6 @@ def slow_start(self, replica=False): class ProbackupTest(object): # Class attributes enterprise = is_enterprise() - nodes = [] def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) @@ -404,7 +403,6 @@ def make_simple_node( if node.major_version >= 13: self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) - self.nodes.append(node) return node def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): From a59631646e9edb1c9a6a8e64f254a216119073be Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 16 Feb 2021 12:03:16 +0300 Subject: [PATCH 1648/2107] [Issue #310] Detect timeline switch via repprotocol --- src/backup.c | 6 +- src/catalog.c | 5 +- src/pg_probackup.h | 3 + src/stream.c | 203 ++++++++++++++++++++++++++++++++++++++++++++- src/utils/pgut.c | 5 +- src/utils/pgut.h | 4 +- 6 files changed, 217 insertions(+), 9 deletions(-) diff --git a/src/backup.c b/src/backup.c index ca8baa777..62bc2bbab 100644 --- a/src/backup.c +++ b/src/backup.c @@ -165,8 +165,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool "trying to look up on previous timelines", current.tli); - /* TODO: use read_timeline_history */ - tli_list = catalog_get_timelines(&instance_config); + tli_list = get_history_streaming(&instance_config.conn_opt, current.tli, backup_list); + if (!tli_list) + /* fallback to using archive */ + tli_list = catalog_get_timelines(&instance_config); if (parray_num(tli_list) == 0) elog(WARNING, "Cannot find valid backup on previous timelines, " diff --git a/src/catalog.c b/src/catalog.c index 15386d426..b0051f699 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -48,7 +48,7 @@ typedef struct LockInfo bool exclusive; } LockInfo; -static timelineInfo * +timelineInfo * timelineInfoNew(TimeLineID tli) { timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); @@ -74,7 +74,8 @@ timelineInfoFree(void *tliInfo) if (tli->backups) { - parray_walk(tli->backups, pgBackupFree); + /* backups themselves should freed separately */ +// parray_walk(tli->backups, pgBackupFree); parray_free(tli->backups); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1e001bd05..e5f05338b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -891,6 +891,8 @@ extern int validate_one_page(Page page, BlockNumber absolute_blkno, uint32 checksum_version); extern bool validate_tablespace_map(pgBackup *backup); +extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list); + /* return codes for validate_one_page */ /* TODO: use enum */ #define PAGE_IS_VALID (-1) @@ -920,6 +922,7 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list, extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list, TimeLineID current_tli, time_t current_start_time, InstanceConfig *instance); +extern timelineInfo *timelineInfoNew(TimeLineID tli); extern void timelineInfoFree(void *tliInfo); extern parray *catalog_get_timelines(InstanceConfig *instance); extern void do_set_backup(const char *instance_name, time_t backup_id, diff --git a/src/stream.c b/src/stream.c index 825aa0e7d..21204ae2c 100644 --- a/src/stream.c +++ b/src/stream.c @@ -10,6 +10,7 @@ #include "pg_probackup.h" #include "receivelog.h" #include "streamutil.h" +#include "access/timeline.h" #include #include @@ -69,6 +70,7 @@ static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, uint32 xlog_seg_size); static void add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir); +static parray* parse_tli_history_buffer(char *history, TimeLineID tli); /* * Run IDENTIFY_SYSTEM through a given connection and @@ -353,6 +355,204 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* --- External API --- */ +/* + * Maybe add a StreamOptions struct ? + * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. + */ +parray* +get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list) +{ + PGresult *res; + PGconn *conn; + char *history; + char query[128]; + parray *result = NULL; + parray *tli_list = NULL; + timelineInfo *tlinfo = NULL; + int i,j; + + snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); + + /* + * Connect in replication mode to the server. + */ + conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser, + false); + + if (!conn) + return NULL; + + res = PQexec(conn, query); + PQfinish(conn); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING, "Could not send replication command \"%s\": %s", + query, PQresultErrorMessage(res)); + PQclear(res); + return NULL; + } + + /* + * The response to TIMELINE_HISTORY is a single row result set + * with two fields: filename and content + */ + + if (PQnfields(res) != 2 || PQntuples(res) != 1) + { + elog(WARNING, "Unexpected response to TIMELINE_HISTORY command: " + "got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + PQclear(res); + return NULL; + } + + history = pgut_strdup(PQgetvalue(res, 0, 1)); + result = parse_tli_history_buffer(history, tli); + + /* some cleanup */ + pg_free(history); + PQclear(res); + + if (result) + tlinfo = timelineInfoNew(tli); + else + return NULL; + + /* transform TimeLineHistoryEntry into timelineInfo */ + for (i = parray_num(result) -1; i >= 0; i--) + { + TimeLineHistoryEntry *tln = (TimeLineHistoryEntry *) parray_get(result, i); + + tlinfo->parent_tli = tln->tli; + tlinfo->switchpoint = tln->end; + + if (!tli_list) + tli_list = parray_new(); + + parray_append(tli_list, tlinfo); + + /* Next tli */ + tlinfo = timelineInfoNew(tln->tli); + + /* oldest tli */ + if (i == 0) + { + tlinfo->tli = tln->tli; + tlinfo->parent_tli = 0; + tlinfo->switchpoint = 0; + parray_append(tli_list, tlinfo); + } + } + + /* link parent to child */ + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + + for (j = 0; j < parray_num(tli_list); j++) + { + timelineInfo *tlinfo_parent = (timelineInfo *) parray_get(tli_list, j); + + if (tlinfo->parent_tli == tlinfo_parent->tli) + { + tlinfo->parent_link = tlinfo_parent; + break; + } + } + } + + /* add backups to each timeline info */ + for (i = 0; i < parray_num(tli_list); i++) + { + timelineInfo *tlinfo = parray_get(tli_list, i); + for (j = 0; j < parray_num(backup_list); j++) + { + pgBackup *backup = parray_get(backup_list, j); + if (tlinfo->tli == backup->tli) + { + if (tlinfo->backups == NULL) + tlinfo->backups = parray_new(); + parray_append(tlinfo->backups, backup); + } + } + } + + /* cleanup */ + parray_walk(result, pg_free); + pg_free(result); + + return tli_list; +} + +parray* +parse_tli_history_buffer(char *history, TimeLineID tli) +{ + char *curLine = history; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + parray *result = NULL; + + /* Parse timeline history buffer string by string */ + while (curLine) + { + char tempStr[1024]; + char *nextLine = strchr(curLine, '\n'); + int curLineLen = nextLine ? (nextLine-curLine) : strlen(curLine); + + memcpy(tempStr, curLine, curLineLen); + tempStr[curLineLen] = '\0'; // NUL-terminate! + curLine = nextLine ? (nextLine+1) : NULL; + + if (curLineLen > 0) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = tempStr; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(tempStr, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a numeric timeline ID.", tempStr); + } + if (nfields != 3) + elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a transaction log switchpoint location.", tempStr); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be in increasing sequence: \"%s\"", tempStr); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + if (!result) + result = parray_new(); + parray_append(result, entry); + + /* we ignore the remainder of each line */ + } + } + + return result; +} + /* * Maybe add a StreamOptions struct ? * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. @@ -374,7 +574,8 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption stream_thread_arg.conn = pgut_connect_replication(conn_opt->pghost, conn_opt->pgport, conn_opt->pgdatabase, - conn_opt->pguser); + conn_opt->pguser, + true); /* sanity check*/ IdentifySystem(&stream_thread_arg); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index ef3472e26..a1631b106 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -270,7 +270,8 @@ pgut_connect(const char *host, const char *port, PGconn * pgut_connect_replication(const char *host, const char *port, - const char *dbname, const char *username) + const char *dbname, const char *username, + bool strict) { PGconn *tmpconn; int argcount = 7; /* dbname, replication, fallback_app_name, @@ -356,7 +357,7 @@ pgut_connect_replication(const char *host, const char *port, continue; } - elog(ERROR, "could not connect to database %s: %s", + elog(strict ? ERROR : WARNING, "could not connect to database %s: %s", dbname, PQerrorMessage(tmpconn)); PQfinish(tmpconn); free(values); diff --git a/src/utils/pgut.h b/src/utils/pgut.h index d196aad3d..e6ccbf211 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -40,8 +40,8 @@ extern char *pgut_get_conninfo_string(PGconn *conn); extern PGconn *pgut_connect(const char *host, const char *port, const char *dbname, const char *username); extern PGconn *pgut_connect_replication(const char *host, const char *port, - const char *dbname, - const char *username); + const char *dbname, const char *username, + bool strict); extern void pgut_disconnect(PGconn *conn); extern void pgut_disconnect_callback(bool fatal, void *userdata); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, From 04bab3e92702f7e7dc8a0cbfd2aea3a1564ffbe7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 12:50:38 +0300 Subject: [PATCH 1649/2107] [Issue #328] added tests.retention.RetentionTest.test_concurrent_running_backup --- tests/retention.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/retention.py b/tests/retention.py index 6dc8536ca..0122913d3 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2536,3 +2536,42 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') self.del_test_dir(module_name, fname, [node]) + + def test_concurrent_running_full_backup(self): + """ + https://github.com/postgrespro/pg_probackup/issues/328 + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'RUNNING') + + print(self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired', '--log-level-console=VERBOSE'], + return_id=False)) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.del_test_dir(module_name, fname, [node]) From 316f0d267bd2e79d88303b1f6a14d82a22793193 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 12:52:35 +0300 Subject: [PATCH 1650/2107] [Issue #328] Do not delete invalid full backups within retention redundancy range --- src/delete.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/delete.c b/src/delete.c index c2b935d67..ec51374b0 100644 --- a/src/delete.c +++ b/src/delete.c @@ -227,13 +227,14 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Consider only valid FULL backups for Redundancy */ - if (instance_config.retention_redundancy > 0 && - backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) + if (backup->backup_mode == BACKUP_MODE_FULL) { - n_full_backups++; + /* Consider only valid FULL backups for Redundancy fulfillment */ + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) + { + n_full_backups++; + } /* Add every FULL backup that satisfy Redundancy policy to separate list */ if (n_full_backups <= instance_config.retention_redundancy) @@ -413,7 +414,10 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg pinning_window ? pinning_window : instance_config.retention_window, action); - if (backup->backup_mode == BACKUP_MODE_FULL) + /* Only valid full backups are count to something */ + if (backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) cur_full_backup_num++; } } From 00ce713c021eea4ed8c5f74563429555ef42d028 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 12:55:18 +0300 Subject: [PATCH 1651/2107] Do no report meaningless timestamp when deleting backup without valid "recovery-time" attribute. Reported by Roman Zharkov --- src/delete.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index ec51374b0..625d92ab7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -745,7 +745,10 @@ delete_backup_files(pgBackup *backup) return; } - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + if (backup->recovery_time) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + else + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); elog(INFO, "Delete: %s %s", base36enc(backup->start_time), timestamp); From be4528caf1a33377050a349037f332fb82e2da2b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 12:56:23 +0300 Subject: [PATCH 1652/2107] [Issue #326] add special handling for Timeline 1 --- src/restore.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/restore.c b/src/restore.c index 1a22a9a28..8d573286a 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1782,9 +1782,9 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict elog(ERROR, "Timeline IDs must be less than child timeline's ID."); /* History file is empty or corrupted */ - if (parray_num(result) == 0) + if (parray_num(result) == 0 && targetTLI != 1) { - elog(WARNING, "History file is corrupted: \"%s\"", path); + elog(WARNING, "History file is corrupted or missing: \"%s\"", path); pg_free(result); return NULL; } From 42aa2f39ff0c7b14a2f4649d2607fb4b55681b2d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 14:50:34 +0300 Subject: [PATCH 1653/2107] [Issue #328] minor refactoring --- src/delete.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/delete.c b/src/delete.c index 625d92ab7..7458f0100 100644 --- a/src/delete.c +++ b/src/delete.c @@ -229,21 +229,21 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg if (backup->backup_mode == BACKUP_MODE_FULL) { - /* Consider only valid FULL backups for Redundancy fulfillment */ - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - n_full_backups++; - } - /* Add every FULL backup that satisfy Redundancy policy to separate list */ - if (n_full_backups <= instance_config.retention_redundancy) + if (n_full_backups < instance_config.retention_redundancy) { if (!redundancy_full_backup_list) redundancy_full_backup_list = parray_new(); parray_append(redundancy_full_backup_list, backup); } + + /* Consider only valid FULL backups for Redundancy fulfillment */ + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) + { + n_full_backups++; + } } } /* Sort list of full backups to keep */ From be3be870b938b79cba63743f5519d7ae1abac48d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 14:50:49 +0300 Subject: [PATCH 1654/2107] [Issue #328] improve test coverage --- tests/retention.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/retention.py b/tests/retention.py index 0122913d3..bd1d9b796 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2567,11 +2567,51 @@ def test_concurrent_running_full_backup(self): print(self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=['--retention-redundancy=2', '--delete-expired', '--log-level-console=VERBOSE'], + options=['--retention-redundancy=2', '--delete-expired'], return_id=False)) self.assertTrue( self.show_pb(backup_dir, 'node')[1]['status'], 'RUNNING') + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + out = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired'], + return_id=False) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'OK') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[2]['status'], + 'OK') + + self.assertEqual( + len(self.show_pb(backup_dir, 'node')), + 6) + self.del_test_dir(module_name, fname, [node]) From 7b2f46cbc07dac2b3ec0e06f8bb2aa6e8b7bf6f2 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 14:58:11 +0300 Subject: [PATCH 1655/2107] tests: minor fixes --- tests/retention.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/retention.py b/tests/retention.py index bd1d9b796..18023751a 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -2565,10 +2565,9 @@ def test_concurrent_running_full_backup(self): self.show_pb(backup_dir, 'node')[0]['status'], 'RUNNING') - print(self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=['--retention-redundancy=2', '--delete-expired'], - return_id=False)) + options=['--retention-redundancy=2', '--delete-expired']) self.assertTrue( self.show_pb(backup_dir, 'node')[1]['status'], @@ -2593,7 +2592,7 @@ def test_concurrent_running_full_backup(self): gdb.run_until_break() gdb.kill() - out = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--retention-redundancy=2', '--delete-expired'], return_id=False) From 8a3dfd5e67e7f704e9311fb2d9c03f4141e1a35e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 17 Feb 2021 16:07:25 +0300 Subject: [PATCH 1656/2107] merge: honor the --no-fsync and --no-validate flags --- src/pg_probackup.c | 9 +++++++++ src/pg_probackup.h | 4 ++++ src/utils/file.c | 3 +++ src/validate.c | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 90202fa84..ac927965c 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -128,6 +128,10 @@ bool compress_shortcut = false; /* other options */ char *instance_name; +/* TODO: quick hack */ +bool merge_no_validate = false; +bool merge_no_sync = false; + /* archive push options */ int batch_size = 1; static char *wal_file_path; @@ -830,6 +834,9 @@ main(int argc, char *argv[]) case SHOW_CMD: return do_show(instance_name, current.backup_id, show_archive); case DELETE_CMD: + merge_no_validate = no_validate; + merge_no_sync = no_sync; + if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); if (merge_expired && backup_id_string) @@ -850,6 +857,8 @@ main(int argc, char *argv[]) do_delete(current.backup_id); break; case MERGE_CMD: + merge_no_validate = no_validate; + merge_no_sync = no_sync; do_merge(current.backup_id); break; case SHOW_CONFIG_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1e001bd05..d9c0f3c24 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -784,6 +784,10 @@ extern bool compress_shortcut; /* other options */ extern char *instance_name; +/* temp merge options */ +extern bool merge_no_validate; +extern bool merge_no_sync; + /* show options */ extern ShowFormat show_format; diff --git a/src/utils/file.c b/src/utils/file.c index 15a7085ec..26892fedd 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1169,6 +1169,9 @@ int fio_sync(char const* path, fio_location location) { int fd; + if (merge_no_sync) + return 0; + fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); if (fd < 0) return -1; diff --git a/src/validate.c b/src/validate.c index 21900c8e4..23819f6b4 100644 --- a/src/validate.c +++ b/src/validate.c @@ -129,6 +129,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // params->partial_restore_type); + if (merge_no_validate) + goto skip_validation; + /* setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -180,6 +183,8 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pfree(threads); pfree(threads_args); +skip_validation: + /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 2193cd76fc4b4cdbfe8383a9c3201dc7e08c8e9c Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Thu, 18 Feb 2021 03:01:28 +0000 Subject: [PATCH 1657/2107] clear pgdata and stop node after tests --- tests/helpers/ptrack_helpers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 028e9cd16..76eac6920 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1521,17 +1521,14 @@ def clean_all(self): def del_test_dir(self, module_name, fname, nodes=[]): """ Del testdir and optimistically try to del module dir""" - try: - self.clean_all() - except Exception as e: - raise e + + self.clean_all() shutil.rmtree( os.path.join( self.tmp_path, module_name, - fname, - "backup" + fname ), ignore_errors=True ) From dd8c23c9b12e7831ec3669b2b52e2f4582d76a62 Mon Sep 17 00:00:00 2001 From: "a.kozhemyakin" Date: Thu, 18 Feb 2021 05:13:11 +0000 Subject: [PATCH 1658/2107] delete nodes in del_test_dir --- tests/archive.py | 5 ++--- tests/backup.py | 10 +++++----- tests/checkdb.py | 2 +- tests/compression.py | 2 +- tests/delta.py | 2 +- tests/helpers/ptrack_helpers.py | 2 +- tests/incr_restore.py | 22 +++++++++++----------- tests/merge.py | 14 +++++++------- tests/page.py | 2 +- tests/ptrack.py | 2 +- tests/replica.py | 2 +- tests/retention.py | 6 +++--- tests/validate.py | 8 ++++---- 13 files changed, 39 insertions(+), 40 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 05675065a..e76ca5d3f 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -981,7 +981,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.backup_node(backup_dir, 'master', replica) # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[master, replica]) + self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") @@ -2016,8 +2016,7 @@ def test_archive_pg_receivexlog_partial_handling(self): # Clean after yourself pg_receivexlog.kill() - self.del_test_dir( - module_name, fname, [node, replica, node_restored]) + self.del_test_dir(module_name, fname) @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): diff --git a/tests/backup.py b/tests/backup.py index 70c4dc13f..65851d0f3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1065,7 +1065,7 @@ def test_basic_tablespace_handling(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_tablespace_handling_1(self): @@ -1599,7 +1599,7 @@ def test_basic_temp_slot_for_stream_backup(self): options=['--stream', '--slot=slot_1', '--temp-slot']) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): @@ -1645,7 +1645,7 @@ def test_backup_concurrent_drop_table(self): self.assertEqual(show_backup['status'], "OK") # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): @@ -1930,7 +1930,7 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 700) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): @@ -1973,7 +1973,7 @@ def test_basic_missing_dir_permissions(self): os.rmdir(full_path) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): diff --git a/tests/checkdb.py b/tests/checkdb.py index 3349ad2ef..5b7a156cc 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -349,7 +349,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): diff --git a/tests/compression.py b/tests/compression.py index 321461d6e..c10a59489 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -117,7 +117,7 @@ def test_basic_compression_stream_zlib(self): self.assertEqual(delta_result, delta_result_new) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) def test_compression_archive_zlib(self): """ diff --git a/tests/delta.py b/tests/delta.py index daa423d49..e18b8fb63 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -80,7 +80,7 @@ def test_basic_delta_vacuum_truncate(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 76eac6920..a84caba4d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1519,7 +1519,7 @@ def clean_all(self): if o.__class__ is testgres.PostgresNode: o.cleanup() - def del_test_dir(self, module_name, fname, nodes=[]): + def del_test_dir(self, module_name, fname): """ Del testdir and optimistically try to del module dir""" self.clean_all() diff --git a/tests/incr_restore.py b/tests/incr_restore.py index a228bdd79..885a88c2e 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -720,7 +720,7 @@ def test_basic_incr_restore_sanity(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_incr_checksum_restore(self): @@ -809,7 +809,7 @@ def test_incr_checksum_restore(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_1]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -898,7 +898,7 @@ def test_incr_lsn_restore(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_1]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_incr_lsn_sanity(self): @@ -967,7 +967,7 @@ def test_incr_lsn_sanity(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname, [node_1]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_incr_checksum_sanity(self): @@ -1026,7 +1026,7 @@ def test_incr_checksum_sanity(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname, [node_1]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -1561,7 +1561,7 @@ def test_make_replica_via_incr_checksum_restore(self): pgbench.wait() # Clean after yourself - self.del_test_dir(module_name, fname, [new_master, old_master]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_make_replica_via_incr_lsn_restore(self): @@ -1634,7 +1634,7 @@ def test_make_replica_via_incr_lsn_restore(self): pgbench.wait() # Clean after yourself - self.del_test_dir(module_name, fname, [new_master, old_master]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -1878,7 +1878,7 @@ def test_incr_lsn_long_xact_2(self): '1') # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -2137,7 +2137,7 @@ def test_incremental_partial_restore_exclude_checksum(self): self.assertNotIn('PANIC', output) # Clean after yourself - self.del_test_dir(module_name, fname, [node, node2]) + self.del_test_dir(module_name, fname) def test_incremental_partial_restore_exclude_lsn(self): """""" @@ -2247,7 +2247,7 @@ def test_incremental_partial_restore_exclude_lsn(self): self.assertNotIn('PANIC', output) # Clean after yourself - self.del_test_dir(module_name, fname, [node2]) + self.del_test_dir(module_name, fname) def test_incremental_partial_restore_exclude_tablespace_checksum(self): """""" @@ -2391,7 +2391,7 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.assertNotIn('PANIC', output) # Clean after yourself - self.del_test_dir(module_name, fname, [node2]) + self.del_test_dir(module_name, fname) def test_incremental_pg_filenode_map(self): """ diff --git a/tests/merge.py b/tests/merge.py index 29d60433c..186b2f203 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -101,7 +101,7 @@ def test_basic_merge_full_page(self): self.assertEqual(count1, count2) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) def test_merge_compressed_backups(self): """ @@ -2245,7 +2245,7 @@ def test_merge_multiple_descendants(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_smart_merge(self): @@ -2305,7 +2305,7 @@ def test_smart_merge(self): logfile_content = f.read() # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) def test_idempotent_merge(self): """ @@ -2380,7 +2380,7 @@ def test_idempotent_merge(self): self.assertEqual( page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) def test_merge_correct_inheritance(self): """ @@ -2435,7 +2435,7 @@ def test_merge_correct_inheritance(self): page_meta['expire-time'], self.show_pb(backup_dir, 'node', page_id)['expire-time']) - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) def test_merge_correct_inheritance_1(self): """ @@ -2485,7 +2485,7 @@ def test_merge_correct_inheritance_1(self): 'expire-time', self.show_pb(backup_dir, 'node', page_id)) - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -2604,7 +2604,7 @@ def test_multi_timeline_merge(self): '-d', 'postgres', '-p', str(node_restored.port)]) # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure diff --git a/tests/page.py b/tests/page.py index c0dbb69f2..8208e8319 100644 --- a/tests/page.py +++ b/tests/page.py @@ -100,7 +100,7 @@ def test_basic_page_vacuum_truncate(self): self.assertEqual(result1, result2) # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_page_vacuum_truncate_1(self): diff --git a/tests/ptrack.py b/tests/ptrack.py index c45ecd6ec..de76d1d36 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3148,7 +3148,7 @@ def test_basic_ptrack_truncate_replica(self): 'select 1') # Clean after yourself - self.del_test_dir(module_name, fname, [master, replica, node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure diff --git a/tests/replica.py b/tests/replica.py index f664ca886..345f8a7dc 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -418,7 +418,7 @@ def test_basic_make_replica_via_restore(self): options=['--archive-timeout=30s', '--stream']) # Clean after yourself - self.del_test_dir(module_name, fname, [master, replica]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_take_backup_from_delayed_replica(self): diff --git a/tests/retention.py b/tests/retention.py index 6dc8536ca..24884574a 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1008,7 +1008,7 @@ def test_basic_window_merge_multiple_descendants(self): 'FULL') # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): @@ -1275,7 +1275,7 @@ def test_basic_window_merge_multiple_descendants_1(self): '--delete-expired', '--log-level-console=log']) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_window_chains(self): @@ -2535,4 +2535,4 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) diff --git a/tests/validate.py b/tests/validate.py index cfec234d0..62116be89 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -298,7 +298,7 @@ def test_basic_validate_corrupted_intermediate_backup(self): 'Backup STATUS should be "ORPHAN"') # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): @@ -3843,7 +3843,7 @@ def test_validate_corrupt_page_header_map(self): self.assertIn("WARNING: Some backups are not valid", e.message) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") @@ -3906,7 +3906,7 @@ def test_validate_truncated_page_header_map(self): self.assertIn("WARNING: Some backups are not valid", e.message) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") @@ -3966,7 +3966,7 @@ def test_validate_missing_page_header_map(self): self.assertIn("WARNING: Some backups are not valid", e.message) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.del_test_dir(module_name, fname) # validate empty backup list # page from future during validate From 2cf0de22573411d36fc6a0f83e0b4bd812d4e096 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Thu, 18 Feb 2021 12:40:51 +0300 Subject: [PATCH 1659/2107] [DOC] Added/removed tags surrounding 'PostgreSQL' for correct import into PostgresPro documentation --- doc/pgprobackup.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index b0a0f6763..3ac4fdcd3 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -155,7 +155,7 @@ doc/src/sgml/pgprobackup.sgml recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. - pg_probackup supports PostgreSQL 9.5 or higher. + pg_probackup supports PostgreSQL 9.5 or higher. @@ -389,7 +389,7 @@ doc/src/sgml/pgprobackup.sgml - pg_probackup only supports PostgreSQL 9.5 and higher. + pg_probackup only supports PostgreSQL 9.5 and higher. @@ -410,7 +410,7 @@ doc/src/sgml/pgprobackup.sgml - For PostgreSQL 9.5, functions + For PostgreSQL 9.5, functions pg_create_restore_point(text) and pg_switch_xlog() can be executed only if the backup role is a superuser, so backup of a @@ -599,7 +599,7 @@ pg_probackup add-instance -B backup_dir -D to the PostgreSQL server: - For PostgreSQL 9.5: + For PostgreSQL 9.5: BEGIN; @@ -1711,7 +1711,7 @@ pg_probackup restore -B backup_dir --instance The speed of restore from backup can be significantly improved by replacing only invalid and changed pages in already - existing PostgreSQL data directory using + existing PostgreSQL data directory using incremental restore options with the command. @@ -1874,11 +1874,11 @@ pg_probackup restore -B backup_dir --instance - Due to recovery specifics of PostgreSQL versions earlier than 12, + Due to recovery specifics of PostgreSQL versions earlier than 12, it is advisable that you set the hot_standby parameter to off when running partial - restore of a PostgreSQL cluster of version earlier than 12. + restore of a PostgreSQL cluster of version earlier than 12. Otherwise the recovery may fail. From 3c0cc47b6d435e3de95c0c8efd4f1ba5a0b84583 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Fri, 19 Feb 2021 16:30:07 +0300 Subject: [PATCH 1660/2107] [DOC] Recommendation on setting ptrack.map_size updated with ptrack documentation --- doc/pgprobackup.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 3ac4fdcd3..e0a733bde 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1202,8 +1202,9 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack.map_size to a higher value - does not affect PTRACK operation. The maximum allowed value is 1024. + Setting ptrack.map_size to a higher value does not + affect PTRACK operation, but it is not recommended to set this parameter + to a value higher than 1024. From 1fd1c8d35c8a02ef6f3d443080cb46f0bbf640fb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Feb 2021 23:20:03 +0300 Subject: [PATCH 1661/2107] [Issue #327] add test coverage --- tests/archive.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/archive.py b/tests/archive.py index a70893958..ad2320a3c 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -983,6 +983,83 @@ def test_basic_master_and_replica_concurrent_archiving(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_concurrent_archiving(self): + """ + Concurrent archiving from master, replica and cascade replica + https://github.com/postgrespro/pg_probackup/issues/327 + + For PG >= 11 it is expected to pass this test + """ + + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + master.pgbench_init(scale=10) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'node', master) + + # Settings for Replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + replica.slow_start(replica=True) + + # create cascade replicas + replica1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica1')) + replica1.cleanup() + + # Settings for casaced replica + self.restore_node(backup_dir, 'node', replica1) + self.set_replica(replica, replica1, synchronous=False) + self.set_auto_conf(replica1, {'port': replica1.port}) + replica1.slow_start(replica=True) + + # Take full backup from master + self.backup_node(backup_dir, 'node', master) + + pgbench = master.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '30', '-c', '1']) + + # Take several incremental backups from master + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + pgbench.wait() + pgbench.stdout.close() + + with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + + self.assertNotIn('different checksum', log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog(self): From 2974c03e7df0b5279d94eb24903a2e1b2679211e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Feb 2021 23:22:35 +0300 Subject: [PATCH 1662/2107] tests: minor fix --- tests/archive.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index ad2320a3c..ac1b2a0d4 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1052,9 +1052,16 @@ def test_concurrent_archiving(self): pgbench.wait() pgbench.stdout.close() - with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: + with open(os.path.join(master.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: log_content = f.read() + self.assertNotIn('different checksum', log_content) + with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() self.assertNotIn('different checksum', log_content) # Clean after yourself From e0dfc9c89e26e9b9e18826dbc0ac36cac7d33359 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 5 Mar 2021 17:01:15 +0300 Subject: [PATCH 1663/2107] [Issue #342] check error when writing to pg_probackup.conf and sync it to disk before rename --- src/configure.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/configure.c b/src/configure.c index 1aae3df13..cf172242a 100644 --- a/src/configure.c +++ b/src/configure.c @@ -299,6 +299,7 @@ do_set_config(bool missing_ok) for (i = 0; instance_options[i].type; i++) { + int rc = 0; ConfigOption *opt = &instance_options[i]; char *value; @@ -319,13 +320,25 @@ do_set_config(bool missing_ok) } if (strchr(value, ' ')) - fprintf(fp, "%s = '%s'\n", opt->lname, value); + rc = fprintf(fp, "%s = '%s'\n", opt->lname, value); else - fprintf(fp, "%s = %s\n", opt->lname, value); + rc = fprintf(fp, "%s = %s\n", opt->lname, value); + + if (rc < 0) + elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); + pfree(value); } - fclose(fp); + if (ferror(fp) || fflush(fp)) + elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); + + if (fclose(fp)) + elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp); + + if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0) + elog(ERROR, "Failed to sync temp configuration file \"%s\": %s", + path_temp, strerror(errno)); if (rename(path_temp, path) < 0) { From 916ffb6d91b162b018cf29ce755fa5235241e8aa Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 7 Mar 2021 18:34:45 +0300 Subject: [PATCH 1664/2107] [Issue #343] redundant errno check when reading "/backup_dir/backups/instance_name" directory --- src/catalog.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 15386d426..995dd8fad 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -972,17 +972,11 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) continue; } parray_append(backups, backup); - - if (errno && errno != ENOENT) - { - elog(WARNING, "cannot read data directory \"%s\": %s", - data_ent->d_name, strerror(errno)); - goto err_proc; - } } + if (errno) { - elog(WARNING, "cannot read backup root directory \"%s\": %s", + elog(WARNING, "Cannot read backup root directory \"%s\": %s", backup_instance_path, strerror(errno)); goto err_proc; } From 79a98911a40ea7bfcade730d152e1903e53659c3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Mar 2021 22:23:28 +0300 Subject: [PATCH 1665/2107] fix achive_timeout=0 due to old bug from 2.0.x version --- src/backup.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index ca8baa777..be0147e91 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1572,8 +1572,13 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, */ if (pg_stop_backup_is_sent && !in_cleanup) { + int timeout = ARCHIVE_TIMEOUT_DEFAULT; res = NULL; + /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ + if (instance_config.archive_timeout > 0) + timeout = instance_config.archive_timeout; + while (1) { if (!PQconsumeInput(conn)) @@ -1598,11 +1603,10 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * If postgres haven't answered in archive_timeout seconds, * send an interrupt. */ - if (pg_stop_backup_timeout > instance_config.archive_timeout) + if (pg_stop_backup_timeout > timeout) { pgut_cancel(conn); - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", - instance_config.archive_timeout); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); } } else From 69754925b2a952c65f743eaa6fbd41ff69beca11 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Mar 2021 23:26:19 +0300 Subject: [PATCH 1666/2107] [Issue #348] added tests.compatibility.CompatibilityTest.test_compatibility_tablespace --- tests/compatibility.py | 82 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index 18f601506..da9d72f83 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -1419,3 +1419,85 @@ def test_hidden_files(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_compatibility_tablespace(self): + """ + https://github.com/postgrespro/pg_probackup/issues/348 + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"], old_binary=True) + + tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old') + + self.create_tblspace_in_node( + node, 'tblspace', + tblspc_path=tblspace_old_path) + + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace tblspace " + "as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"], old_binary=True) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 58d02b27071980023bf7f2157b577b31e62d96c0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 20 Mar 2021 23:27:34 +0300 Subject: [PATCH 1667/2107] [Issue #248] Use crc32 when validating backup of version "2.0.21 < version <= 2.0.25" --- src/validate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/validate.c b/src/validate.c index 21900c8e4..a91fe817f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -716,6 +716,8 @@ validate_tablespace_map(pgBackup *backup) pgFile **tablespace_map = NULL; pg_crc32 crc; parray *files = get_backup_filelist(backup, true); + bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || + parse_program_version(backup->program_version) >= 20025; parray_qsort(files, pgFileCompareRelPathWithExternal); join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); @@ -738,7 +740,7 @@ validate_tablespace_map(pgBackup *backup) map_path, base36enc(backup->backup_id)); /* check tablespace map checksumms */ - crc = pgFileGetCRC(map_path, true, false); + crc = pgFileGetCRC(map_path, use_crc32c, false); if ((*tablespace_map)->crc != crc) elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " From f4ab4b9d2b46f337b601af9b6ee640d5f03abf47 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 21 Mar 2021 01:33:36 +0300 Subject: [PATCH 1668/2107] Version 2.4.11 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1e001bd05..f28f96e2e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -306,7 +306,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.10" +#define PROGRAM_VERSION "2.4.11" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index b5204e46e..35f065a13 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.10 \ No newline at end of file +pg_probackup 2.4.11 \ No newline at end of file From 0876dd61900a0ef5a733f3025c7a732dc6e208b9 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Mar 2021 20:01:53 +0300 Subject: [PATCH 1669/2107] [Issue #346] set "interrupted" in elog, wait for streamed segments --- src/backup.c | 32 +++++++++++++++++++------------- src/utils/logger.c | 1 + 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index be0147e91..4ec790644 100644 --- a/src/backup.c +++ b/src/backup.c @@ -56,7 +56,7 @@ static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNode static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir); + int timeout_elevel, bool in_stream_dir, pgBackup *backup); static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); @@ -268,7 +268,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool * Because WAL streaming will start after pg_start_backup() in stream * mode. */ - wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); + wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false, ¤t); } /* start stream replication */ @@ -279,6 +279,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool start_WAL_streaming(backup_conn, dst_backup_path, &instance_config.conn_opt, current.start_lsn, current.tli); + + /* Make sure that WAL streaming is working + * PAGE backup in stream mode is waited twice, first for + * segment in WAL archive and then for streamed segment + */ + wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, true, ¤t); } /* initialize backup's file list */ @@ -1262,7 +1268,7 @@ pg_is_superuser(PGconn *conn) static XLogRecPtr wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir) + int timeout_elevel, bool in_stream_dir, pgBackup *backup) { XLogSegNo targetSegNo; char pg_wal_dir[MAXPGPATH]; @@ -1294,15 +1300,14 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, */ if (in_stream_dir) { - pgBackupGetPath2(¤t, pg_wal_dir, lengthof(pg_wal_dir), - DATABASE_DIR, PG_XLOG_DIR); + join_path_components(pg_wal_dir, backup->database_dir, PG_XLOG_DIR); join_path_components(wal_segment_path, pg_wal_dir, wal_segment); wal_segment_dir = pg_wal_dir; } else { join_path_components(wal_segment_path, arclog_path, wal_segment); - wal_segment_dir = arclog_path; + wal_segment_dir = arclog_path; /* global var */ } /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ @@ -1394,7 +1399,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, sleep(1); if (interrupted) - elog(ERROR, "Interrupted during waiting for WAL archiving"); + elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving"); try_count++; /* Inform user if WAL segment is absent in first attempt */ @@ -1418,9 +1423,10 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, { if (file_exists) elog(timeout_elevel, "WAL segment %s was %s, " - "but target LSN %X/%X could not be archived in %d seconds", + "but target LSN %X/%X could not be %s in %d seconds", wal_segment, wal_delivery_str, - (uint32) (target_lsn >> 32), (uint32) target_lsn, timeout); + (uint32) (target_lsn >> 32), (uint32) target_lsn, + wal_delivery_str, timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(timeout_elevel, @@ -1705,7 +1711,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, { /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, true, WARNING, stream_wal); + false, true, WARNING, stream_wal, backup); /* Get the first record in segment with current stop_lsn */ lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, @@ -1733,7 +1739,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * because previous record can be the contrecord. */ lsn_tmp = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - true, false, ERROR, stream_wal); + true, false, ERROR, stream_wal, backup); /* sanity */ if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) @@ -1747,7 +1753,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, { /* Wait for segment with current stop_lsn */ wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, true, ERROR, stream_wal); + false, true, ERROR, stream_wal, backup); /* Get the next closest record in segment with current stop_lsn */ lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, @@ -1876,7 +1882,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, */ if (!stop_lsn_exists) stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, false, ERROR, stream_wal); + false, false, ERROR, stream_wal, backup); if (stream_wal) { diff --git a/src/utils/logger.c b/src/utils/logger.c index f039d4a5d..584b937e7 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -169,6 +169,7 @@ exit_if_necessary(int elevel) { /* Interrupt other possible routines */ thread_interrupted = true; + interrupted = true; #ifdef WIN32 ExitThread(elevel); #else From 9fcf99034dc46749f260bcbd30bd30a0075ff65e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Mar 2021 21:24:14 +0300 Subject: [PATCH 1670/2107] tests: store gdb output internally --- tests/helpers/ptrack_helpers.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a84caba4d..b0da8abbb 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1757,6 +1757,7 @@ def __str__(self): class GDBobj(ProbackupTest): def __init__(self, cmd, verbose, attach=False): self.verbose = verbose + self.output = '' # Check gdb presense try: @@ -1801,7 +1802,8 @@ def __init__(self, cmd, verbose, attach=False): # discard data from pipe, # is there a way to do it a less derpy way? while True: - line = self.proc.stdout.readline() +# line = self.proc.stdout.readline() + line = self.get_line() if 'No such process' in line: raise GdbException(line) @@ -1811,6 +1813,12 @@ def __init__(self, cmd, verbose, attach=False): else: break + def get_line(self): + line = self.proc.stdout.readline() +# self.output += repr(line) + '\n' + self.output += line + return line + def kill(self): self.proc.kill() self.proc.wait() @@ -1932,10 +1940,8 @@ def continue_execution_until_break(self, ignore_count=0): 'Failed to continue execution until break.\n') def stopped_in_breakpoint(self): - output = [] while True: - line = self.proc.stdout.readline() - output += [line] + line = self.get_line() if self.verbose: print(line) if line.startswith('*stopped,reason="breakpoint-hit"'): @@ -1952,7 +1958,7 @@ def _execute(self, cmd, running=True): # look for command we just send while True: - line = self.proc.stdout.readline() + line = self.get_line() if self.verbose: print(repr(line)) @@ -1962,7 +1968,7 @@ def _execute(self, cmd, running=True): break while True: - line = self.proc.stdout.readline() + line = self.get_line() output += [line] if self.verbose: print(repr(line)) From c36ba06c350d7baab3113aa937a0b9e3da8ba80b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Mar 2021 21:26:32 +0300 Subject: [PATCH 1671/2107] [Issue #346] tests: added tests.backup.BackupTest.test_missing_wal_segment --- tests/backup.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index 65851d0f3..f14e61745 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2365,7 +2365,7 @@ def test_parent_choosing_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.skip("skip") + # @unittest.skip("skip") def test_backup_with_less_privileges_role(self): """ check permissions correctness from documentation: @@ -3079,3 +3079,80 @@ def test_incr_backup_filenode_map(self): # Clean after yourself self.del_test_dir(module_name, fname) + + + + # @unittest.skip("skip") + def test_missing_wal_segment(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # get segments in pg_wal, sort then and remove all but the latest + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + + if node.major_version >= 10: + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + else: + pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog') + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', '--log-level-file=INFO'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('start_WAL_streaming') + gdb.run_until_break() + + # generate some more data + node.pgbench_init(scale=3) + + # remove redundant WAL segments in pg_wal + files = os.listdir(pg_wal_dir) + files.sort(reverse=True) + + # leave first two files in list + del files[:2] + for filename in files: + os.remove(os.path.join(pg_wal_dir, filename)) + + gdb.continue_execution_until_exit() + + self.assertIn( + 'unexpected termination of replication stream: ERROR: requested WAL segment', + gdb.output) + + self.assertIn( + 'has already been removed', + gdb.output) + + self.assertIn( + 'ERROR: Interrupted during waiting for WAL streaming', + gdb.output) + + self.assertIn( + 'WARNING: backup in progress, stop backup', + gdb.output) + + # TODO: check the same for PAGE backup + + # Clean after yourself + self.del_test_dir(module_name, fname) From afcb00a04e466e547145f6a7f2707eed0a9d2eda Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Mar 2021 21:32:38 +0300 Subject: [PATCH 1672/2107] tests: remove some old comments --- tests/helpers/ptrack_helpers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b0da8abbb..5b4adedcc 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1799,10 +1799,7 @@ def __init__(self, cmd, verbose, attach=False): ) self.gdb_pid = self.proc.pid - # discard data from pipe, - # is there a way to do it a less derpy way? while True: -# line = self.proc.stdout.readline() line = self.get_line() if 'No such process' in line: @@ -1815,7 +1812,6 @@ def __init__(self, cmd, verbose, attach=False): def get_line(self): line = self.proc.stdout.readline() -# self.output += repr(line) + '\n' self.output += line return line From 59dc864483917ec4bd620bb63d3c30488ae8b990 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 22 Mar 2021 21:32:52 +0300 Subject: [PATCH 1673/2107] open header file in append mode --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index f7032fb55..4db674611 100644 --- a/src/data.c +++ b/src/data.c @@ -2268,7 +2268,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->fp = fopen(map_path, PG_BINARY_W); + hdr_map->fp = fopen(map_path, 'a'); if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); From 1647d1145698792c8583aaee8e6743233cb15167 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 25 Mar 2021 16:45:24 +0300 Subject: [PATCH 1674/2107] [Issue #351] fix data type for header offset --- src/catalog.c | 2 +- src/data.c | 10 +++++----- src/pg_probackup.h | 16 +++++++++------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 995dd8fad..25e08e6ff 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2456,7 +2456,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, { len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); - len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off); + len += sprintf(line+len, ",\"hdr_off\":\"%llu\"", file->hdr_off); len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size); } diff --git a/src/data.c b/src/data.c index 4db674611..544497aca 100644 --- a/src/data.c +++ b/src/data.c @@ -2160,7 +2160,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (fseek(in, file->hdr_off, SEEK_SET)) { - elog(strict ? ERROR : WARNING, "Cannot seek to position %lu in page header map \"%s\": %s", + elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); goto cleanup; } @@ -2177,7 +2177,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) { - elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %li len: %i \"%s\": %s", + elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); goto cleanup; } @@ -2208,7 +2208,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (hdr_crc != file->hdr_crc) { elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " - "offset: %lu, len: %lu, current: %u, expected: %u", + "offset: %llu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); goto cleanup; } @@ -2268,7 +2268,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->fp = fopen(map_path, 'a'); + hdr_map->fp = fopen(map_path, "a"); if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); @@ -2297,7 +2297,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->rel_path, z_len); } - elog(VERBOSE, "Writing headers for file \"%s\" offset: %li, len: %i, crc: %u", + elog(VERBOSE, "Writing headers for file \"%s\" offset: %llu, len: %i, crc: %u", file->rel_path, file->hdr_off, z_len, file->hdr_crc); if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f28f96e2e..4cde5e180 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -208,6 +208,8 @@ do { \ FIN_TRADITIONAL_CRC32(crc); \ } while (0) +#define pg_off_t unsigned long long + /* Information about single file (or dir) in backup */ typedef struct pgFile @@ -249,8 +251,8 @@ typedef struct pgFile /* Coordinates in header map */ int n_headers; /* number of blocks in the data file in backup */ pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ - off_t hdr_off; /* offset in header map */ - int hdr_size; /* offset in header map */ + pg_off_t hdr_off; /* offset in header map */ + int hdr_size; /* length of headers */ } pgFile; typedef struct page_map_entry @@ -406,11 +408,11 @@ typedef struct PGNodeInfo /* structure used for access to block header map */ typedef struct HeaderMap { - char path[MAXPGPATH]; - char path_tmp[MAXPGPATH]; /* used only in merge */ - FILE *fp; /* used only for writing */ - char *buf; /* buffer */ - off_t offset; /* current position in fp */ + char path[MAXPGPATH]; + char path_tmp[MAXPGPATH]; /* used only in merge */ + FILE *fp; /* used only for writing */ + char *buf; /* buffer */ + pg_off_t offset; /* current position in fp */ pthread_mutex_t mutex; } HeaderMap; From 731d19089d7b97c35556e0b4588006db11ea1e91 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 26 Mar 2021 00:30:29 +0300 Subject: [PATCH 1675/2107] [Issue #310] test coverage --- tests/backup.py | 261 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index f14e61745..ef3928d5c 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3080,8 +3080,6 @@ def test_incr_backup_filenode_map(self): # Clean after yourself self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_missing_wal_segment(self): """""" @@ -3156,3 +3154,262 @@ def test_missing_wal_segment(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_missing_replication_permission(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) +# self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # >= 10 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + + sleep(2) + replica.promote() + + # Delta backup + try: + self.backup_node( + backup_dir, 'node', replica, backup_type='delta', + data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "FATAL: must be superuser or replication role to start walsender", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_missing_replication_permission_1(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # >= 10 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + + replica.promote() + + # PAGE + output = self.backup_node( + backup_dir, 'node', replica, backup_type='page', + data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'], + return_id=False) + + self.assertIn( + 'WARNING: Valid backup on current timeline 2 is not found, trying to look up on previous timelines', + output) + + self.assertIn( + 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender', + output) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 7329256b95a2549a8364d746d5aebc8a7b45fb00 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Mar 2021 16:22:40 +0300 Subject: [PATCH 1676/2107] [Issue #324] "--no-validate" and "--no-sync" flags for merge and delete commands --- src/backup.c | 2 +- src/delete.c | 15 +++++------ src/help.c | 11 ++++++-- src/merge.c | 63 +++++++++++++++++++++++++++------------------- src/pg_probackup.c | 12 ++------- src/pg_probackup.h | 12 +++------ src/utils/file.c | 3 --- src/validate.c | 5 ---- 8 files changed, 61 insertions(+), 62 deletions(-) diff --git a/src/backup.c b/src/backup.c index ac4f1013f..de5c0cee1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -887,7 +887,7 @@ do_backup(pgSetBackupParams *set_backup_params, * which are expired according to retention policies */ if (delete_expired || merge_expired || delete_wal) - do_retention(); + do_retention(no_validate, no_sync); return 0; } diff --git a/src/delete.c b/src/delete.c index 7458f0100..d1afa2874 100644 --- a/src/delete.c +++ b/src/delete.c @@ -19,7 +19,7 @@ static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); static void do_retention_merge(parray *backup_list, parray *to_keep_list, - parray *to_purge_list); + parray *to_purge_list, bool no_validate, bool no_sync); static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); static void do_retention_wal(bool dry_run); @@ -123,7 +123,7 @@ do_delete(time_t backup_id) * which FULL backup should be keeped for redundancy obligation(only valid do), * but if invalid backup is not guarded by retention - it is removed */ -void do_retention(void) +void do_retention(bool no_validate, bool no_sync) { parray *backup_list = NULL; parray *to_keep_list = parray_new(); @@ -172,7 +172,7 @@ void do_retention(void) do_retention_internal(backup_list, to_keep_list, to_purge_list); if (merge_expired && !dry_run && !backup_list_is_empty) - do_retention_merge(backup_list, to_keep_list, to_purge_list); + do_retention_merge(backup_list, to_keep_list, to_purge_list, no_validate, no_sync); if (delete_expired && !dry_run && !backup_list_is_empty) do_retention_purge(to_keep_list, to_purge_list); @@ -424,7 +424,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* Merge partially expired incremental chains */ static void -do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list) +do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list, + bool no_validate, bool no_sync) { int i; int j; @@ -543,7 +544,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l */ keep_backup = parray_get(merge_list, 0); - merge_chain(merge_list, full_backup, keep_backup); + merge_chain(merge_list, full_backup, keep_backup, no_validate, no_sync); backup_merged = true; for (j = parray_num(merge_list) - 2; j >= 0; j--) @@ -554,8 +555,8 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l parray_rm(to_purge_list, tmp_backup, pgBackupCompareId); parray_set(to_keep_list, i, NULL); } - - pgBackupValidate(full_backup, NULL); + if (!no_validate) + pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time)); diff --git a/src/help.c b/src/help.c index 2b5bcd06e..f72dc90dc 100644 --- a/src/help.c +++ b/src/help.c @@ -197,11 +197,12 @@ help_pg_probackup(void) printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n")); printf(_(" [--delete-wal]\n")); - printf(_(" [--dry-run]\n")); + printf(_(" [--dry-run] [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--progress] [-j num-threads]\n")); + printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); @@ -631,13 +632,16 @@ help_delete(void) printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n\n")); + printf(_(" [--wal-depth=wal-depth]\n")); + printf(_(" [--no-validate] [--no-sync]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to delete\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); + printf(_(" --no-validate disable validation during retention merge\n")); + printf(_(" --no-sync do not sync merged files to disk\n")); printf(_("\n Retention options:\n")); printf(_(" --delete-expired delete backups expired according to current\n")); @@ -681,6 +685,7 @@ help_merge(void) { printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); printf(_(" [--log-filename=log-filename]\n")); @@ -695,6 +700,8 @@ help_merge(void) printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); + printf(_(" --no-validate disable validation during retention merge\n")); + printf(_(" --no-sync do not sync merged files to disk\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/merge.c b/src/merge.c index 3c51a1fae..3fd5b13ae 100644 --- a/src/merge.c +++ b/src/merge.c @@ -30,6 +30,7 @@ typedef struct bool program_version_match; bool use_bitmap; bool is_retry; + bool no_sync; /* * Return value from the thread. @@ -50,13 +51,13 @@ static void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, const char *to_root, bool use_bitmap, - bool is_retry); + bool is_retry, bool no_sync); static void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, const char *full_database_dir, - const char *full_external_prefix); + const char *full_external_prefix, bool no_sync); static bool is_forward_compatible(parray *parent_chain); @@ -68,7 +69,7 @@ static bool is_forward_compatible(parray *parent_chain); * - Remove unnecessary files, which doesn't exist in the target backup anymore */ void -do_merge(time_t backup_id) +do_merge(time_t backup_id, bool no_validate, bool no_sync) { parray *backups; parray *merge_list = parray_new(); @@ -405,9 +406,10 @@ do_merge(time_t backup_id) catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* do actual merge */ - merge_chain(merge_list, full_backup, dest_backup); + merge_chain(merge_list, full_backup, dest_backup, no_validate, no_sync); - pgBackupValidate(full_backup, NULL); + if (!no_validate) + pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); @@ -434,7 +436,8 @@ do_merge(time_t backup_id) * that chain is ok. */ void -merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) +merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, + bool no_validate, bool no_sync) { int i; char *dest_backup_id; @@ -554,25 +557,28 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * with sole exception of FULL backup. If it has MERGING status * then it isn't valid backup until merging is finished. */ - elog(INFO, "Validate parent chain for backup %s", - base36enc(dest_backup->start_time)); - - for (i = parray_num(parent_chain) - 1; i >= 0; i--) + if (!no_validate) { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + elog(INFO, "Validate parent chain for backup %s", + base36enc(dest_backup->start_time)); - /* FULL backup is not to be validated if its status is MERGING */ - if (backup->backup_mode == BACKUP_MODE_FULL && - backup->status == BACKUP_STATUS_MERGING) + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { - continue; - } + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - pgBackupValidate(backup, NULL); + /* FULL backup is not to be validated if its status is MERGING */ + if (backup->backup_mode == BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_MERGING) + { + continue; + } - if (backup->status != BACKUP_STATUS_OK) - elog(ERROR, "Backup %s has status %s, merge is aborted", - base36enc(backup->start_time), status2str(backup->status)); + pgBackupValidate(backup, NULL); + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status %s, merge is aborted", + base36enc(backup->start_time), status2str(backup->status)); + } } /* @@ -665,6 +671,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) arg->program_version_match = program_version_match; arg->use_bitmap = use_bitmap; arg->is_retry = is_retry; + arg->no_sync = no_sync; /* By default there are some error */ arg->ret = 1; @@ -1102,14 +1109,16 @@ merge_files(void *arg) dest_file, tmp_file, arguments->full_database_dir, arguments->use_bitmap, - arguments->is_retry); + arguments->is_retry, + arguments->no_sync); else merge_non_data_file(arguments->parent_chain, arguments->full_backup, arguments->dest_backup, dest_file, tmp_file, arguments->full_database_dir, - arguments->full_external_prefix); + arguments->full_external_prefix, + arguments->no_sync); done: parray_append(arguments->merge_filelist, tmp_file); @@ -1202,7 +1211,8 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, bool use_bitmap, bool is_retry) + const char *full_database_dir, bool use_bitmap, bool is_retry, + bool no_sync) { FILE *out = NULL; char *buffer = pgut_malloc(STDIO_BUFSIZE); @@ -1273,7 +1283,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, return; /* sync second temp file to disk */ - if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); @@ -1294,7 +1304,8 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, const char *to_external_prefix) + const char *full_database_dir, const char *to_external_prefix, + bool no_sync) { int i; char to_fullpath[MAXPGPATH]; @@ -1378,7 +1389,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); /* sync temp file to disk */ - if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp, strerror(errno)); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ac927965c..854493bdc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -128,10 +128,6 @@ bool compress_shortcut = false; /* other options */ char *instance_name; -/* TODO: quick hack */ -bool merge_no_validate = false; -bool merge_no_sync = false; - /* archive push options */ int batch_size = 1; static char *wal_file_path; @@ -834,8 +830,6 @@ main(int argc, char *argv[]) case SHOW_CMD: return do_show(instance_name, current.backup_id, show_archive); case DELETE_CMD: - merge_no_validate = no_validate; - merge_no_sync = no_sync; if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); @@ -851,15 +845,13 @@ main(int argc, char *argv[]) if (delete_status) do_delete_status(&instance_config, delete_status); else - do_retention(); + do_retention(no_validate, no_sync); } else do_delete(current.backup_id); break; case MERGE_CMD: - merge_no_validate = no_validate; - merge_no_sync = no_sync; - do_merge(current.backup_id); + do_merge(current.backup_id, no_validate, no_sync); break; case SHOW_CONFIG_CMD: do_show_config(); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 31fb66235..6df6d0f0e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -786,10 +786,6 @@ extern bool compress_shortcut; /* other options */ extern char *instance_name; -/* temp merge options */ -extern bool merge_no_validate; -extern bool merge_no_sync; - /* show options */ extern ShowFormat show_format; @@ -843,10 +839,10 @@ extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetT extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); /* in merge.c */ -extern void do_merge(time_t backup_id); +extern void do_merge(time_t backup_id, bool no_validate, bool no_sync); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); -extern void merge_chain(parray *parent_chain, - pgBackup *full_backup, pgBackup *dest_backup); +extern void merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, + bool no_validate, bool no_sync); extern parray *read_database_map(pgBackup *backup); @@ -873,7 +869,7 @@ extern int do_show(const char *instance_name, time_t requested_backup_id, bool s /* in delete.c */ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); -extern void do_retention(void); +extern void do_retention(bool no_validate, bool no_sync); extern int do_delete_instance(void); extern void do_delete_status(InstanceConfig *instance_config, const char *status); diff --git a/src/utils/file.c b/src/utils/file.c index 26892fedd..15a7085ec 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1169,9 +1169,6 @@ int fio_sync(char const* path, fio_location location) { int fd; - if (merge_no_sync) - return 0; - fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); if (fd < 0) return -1; diff --git a/src/validate.c b/src/validate.c index fc35bf2cb..a91fe817f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -129,9 +129,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) // dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, // params->partial_restore_type); - if (merge_no_validate) - goto skip_validation; - /* setup threads */ for (i = 0; i < parray_num(files); i++) { @@ -183,8 +180,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pfree(threads); pfree(threads_args); -skip_validation: - /* cleanup */ parray_walk(files, pgFileFree); parray_free(files); From 0751b89626f618a49ef1e63186213890cdbbac12 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Mar 2021 16:45:07 +0300 Subject: [PATCH 1677/2107] [Issue #324] update documentation --- doc/pgprobackup.xml | 53 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index e0a733bde..ae9e5f8bb 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4108,7 +4108,7 @@ pg_probackup validate -B backup_dir merge pg_probackup merge -B backup_dir --instance instance_name -i backup_id -[--help] [-j num_threads] [--progress] +[--help] [-j num_threads] [--progress] [--no-validate] [--no-sync] [logging_options] @@ -4119,6 +4119,30 @@ pg_probackup merge -B backup_dir --instance + + + + + + + + Skips automatic validation before and after merge. + + + + + + + + Do not sync merged files to disk. You can use this flag to speed + up the merge process. Using this flag can result in data + corruption in case of operating system or hardware crash. + + + + + + For details, see the section Merging Backups. @@ -4131,13 +4155,38 @@ pg_probackup delete -B backup_dir --instance num_threads] [--progress] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status=backup_status} -[--dry-run] [logging_options] +[--dry-run] [--no-validate] [--no-sync] [logging_options] Deletes backup with specified backup_id or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. + + + + + + + + Skips automatic validation before and after retention merge. + + + + + + + + + Do not sync merged files to disk. You can use this flag to speed + up the retention merge process. Using this flag can result in data + corruption in case of operating system or hardware crash. + + + + + + For details, see the sections Deleting Backups, From 8dbc90a2eb2cf6898c3b644ef678410b18409de8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 28 Mar 2021 17:57:09 +0300 Subject: [PATCH 1678/2107] Version 2.4.12 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6df6d0f0e..d072ab715 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.11" +#define PROGRAM_VERSION "2.4.12" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 35f065a13..7813360e8 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.11 \ No newline at end of file +pg_probackup 2.4.12 \ No newline at end of file From 3813726bbdb20d16ae6d09834ba41144607bd438 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Mar 2021 13:06:09 +0300 Subject: [PATCH 1679/2107] tests: some fixes --- tests/archive.py | 6 +++--- tests/backup.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index ac1b2a0d4..2d0bb5037 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2638,13 +2638,13 @@ def test_archive_empty_history_file(self): wal_dir = os.path.join(backup_dir, 'wal', 'node') self.assertIn( - 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), log_content) self.assertIn( - 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), log_content) self.assertIn( - 'WARNING: History file is corrupted: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), log_content) self.del_test_dir(module_name, fname) diff --git a/tests/backup.py b/tests/backup.py index ef3928d5c..ec11bdc85 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -4,7 +4,7 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import shutil from distutils.dir_util import copy_tree -from testgres import ProcessType +from testgres import ProcessType, QueryException import subprocess @@ -1576,7 +1576,7 @@ def test_basic_temp_slot_for_stream_backup(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_size': '40MB'}) + 'max_wal_size': '40MB', 'default_transaction_read_only': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3413,3 +3413,51 @@ def test_missing_replication_permission_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_backup_default_transaction_read_only(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'default_transaction_read_only': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + node.safe_psql( + 'postgres', + 'create temp table t1()') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertIn( + "cannot execute CREATE TABLE in a read-only transaction", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--temp-slot']) + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Clean after yourself + self.del_test_dir(module_name, fname) From 15c90460fd3e2fdad89c0b022ca9a6a4778759e1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Mar 2021 14:02:13 +0300 Subject: [PATCH 1680/2107] use 0 instead of BUFSIZ --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 544497aca..a4fea7782 100644 --- a/src/data.c +++ b/src/data.c @@ -2156,7 +2156,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b return NULL; } /* disable buffering for header file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); + setvbuf(in, NULL, _IONBF, 0); if (fseek(in, file->hdr_off, SEEK_SET)) { From b794fbc723b73fc3409f0a9adc715b250e09c132 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 29 Mar 2021 15:11:36 +0300 Subject: [PATCH 1681/2107] use PG_BINARY_A flag when opening file with map headers --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index a4fea7782..9ff54ed4b 100644 --- a/src/data.c +++ b/src/data.c @@ -2268,7 +2268,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->fp = fopen(map_path, "a"); + hdr_map->fp = fopen(map_path, PG_BINARY_A); if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); From 5d348641cdec2b231aada3fe76b5e4d863732596 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 1 Apr 2021 12:02:37 +0300 Subject: [PATCH 1682/2107] use fseeko instead of fseek when accessing the map of page headers --- src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.c b/src/data.c index 9ff54ed4b..999de867d 100644 --- a/src/data.c +++ b/src/data.c @@ -2158,7 +2158,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b /* disable buffering for header file */ setvbuf(in, NULL, _IONBF, 0); - if (fseek(in, file->hdr_off, SEEK_SET)) + if (fseeko(in, file->hdr_off, SEEK_SET)) { elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); From 1d4d293947de69a2cff1159d472d250b9f4b59bf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 1 Apr 2021 14:25:30 +0300 Subject: [PATCH 1683/2107] Version 2.4.13 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d072ab715..950c71343 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.12" +#define PROGRAM_VERSION "2.4.13" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 7813360e8..752e96a8c 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.12 \ No newline at end of file +pg_probackup 2.4.13 \ No newline at end of file From 91da77b16f864f57bf57777ae556e4094216686b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 10 Apr 2021 15:08:35 +0300 Subject: [PATCH 1684/2107] add elog message about waiting for pg_start_backup() execution --- src/backup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backup.c b/src/backup.c index de5c0cee1..112d8b365 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1044,6 +1044,8 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, params[0] = label; + elog(INFO, "wait for pg_start_backup()"); + /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; if (!exclusive_backup) From 0e6e9a45e9c49bc15cb76687f5098a92288ccb3e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 13:20:54 +0300 Subject: [PATCH 1685/2107] [Issue #364] honor the "--no-validate" flag when working with tablespace_map --- src/dir.c | 6 ++++-- src/pg_probackup.h | 4 ++-- src/restore.c | 9 ++++++++- src/validate.c | 15 +++++++++------ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/dir.c b/src/dir.c index d07a4d2f5..e1d9e580a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1116,6 +1116,8 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba elog(VERBOSE, "Create directory \"%s\"", dir->rel_path); join_path_components(to_path, data_dir, dir->rel_path); + + // TODO check exit code fio_mkdir(to_path, dir->mode, location); } @@ -1191,7 +1193,7 @@ read_tablespace_map(parray *links, const char *backup_dir) * 3. backup has tablespaces and some of them are not empty */ int -check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty) +check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate) { parray *links = parray_new(); size_t i; @@ -1205,7 +1207,7 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg /* validate tablespace map, * if there are no tablespaces, then there is nothing left to do */ - if (!validate_tablespace_map(backup)) + if (!validate_tablespace_map(backup, no_validate)) { /* * Sanity check diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 950c71343..f237250b9 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -891,7 +891,7 @@ extern int do_validate_all(void); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); -extern bool validate_tablespace_map(pgBackup *backup); +extern bool validate_tablespace_map(pgBackup *backup, bool no_validate); extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list); @@ -987,7 +987,7 @@ extern void create_data_directories(parray *dest_files, extern void read_tablespace_map(parray *links, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); -extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty); +extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate); extern void check_external_dir_mapping(pgBackup *backup, bool incremental); extern char *get_external_remap(char *current_dir); diff --git a/src/restore.c b/src/restore.c index 8d573286a..f088ca36c 100644 --- a/src/restore.c +++ b/src/restore.c @@ -410,7 +410,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { int rc = check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, params->force, - pgdata_is_empty); + pgdata_is_empty, params->no_validate); /* backup contain no tablespaces */ if (rc == NoTblspc) @@ -2173,6 +2173,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, postmaster_is_up = true; } + /* check that PG_VERSION is the same */ + /* slurp pg_control and check that system ID is the same * check that instance is not running * if lsn_based, check that there is no backup_label files is around AND @@ -2182,6 +2184,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, * data files content, because based on pg_control information we will * choose a backup suitable for lsn based incremental restore. */ + elog(INFO, "Trying to read pg_control file in destination direstory"); system_id_pgdata = get_system_identifier(pgdata); @@ -2214,6 +2217,10 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, if (postmaster_is_up) return POSTMASTER_IS_RUNNING; + /* PG_CONTROL MISSING */ + + /* PG_CONTROL unreadable */ + if (!system_id_match) return SYSTEM_ID_MISMATCH; diff --git a/src/validate.c b/src/validate.c index a91fe817f..6bedd7269 100644 --- a/src/validate.c +++ b/src/validate.c @@ -709,7 +709,7 @@ do_validate_instance(void) * already filled pgBackup.files */ bool -validate_tablespace_map(pgBackup *backup) +validate_tablespace_map(pgBackup *backup, bool no_validate) { char map_path[MAXPGPATH]; pgFile *dummy = NULL; @@ -740,12 +740,15 @@ validate_tablespace_map(pgBackup *backup) map_path, base36enc(backup->backup_id)); /* check tablespace map checksumms */ - crc = pgFileGetCRC(map_path, use_crc32c, false); + if (!no_validate) + { + crc = pgFileGetCRC(map_path, use_crc32c, false); - if ((*tablespace_map)->crc != crc) - elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " - "probably backup %s is corrupt, validate it", - map_path, crc, (*tablespace_map)->crc, base36enc(backup->backup_id)); + if ((*tablespace_map)->crc != crc) + elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " + "probably backup %s is corrupt, validate it", + map_path, crc, (*tablespace_map)->crc, base36enc(backup->backup_id)); + } pgFileFree(dummy); parray_walk(files, pgFileFree); From 6e2e78c494f58bfe87c4b0ece48f1744b6413fd0 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 15:12:25 +0300 Subject: [PATCH 1686/2107] parse tablespace_map correctly --- src/dir.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index e1d9e580a..f150039e1 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1153,7 +1153,7 @@ read_tablespace_map(parray *links, const char *backup_dir) path[MAXPGPATH]; pgFile *file; - if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) + if (sscanf(buf, "%s %n", link_name, path) != 2) elog(ERROR, "invalid format found in \"%s\"", map_path); file = pgut_new(pgFile); @@ -1279,13 +1279,19 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg TablespaceListCell *cell; bool remapped = false; + elog(INFO, "Linked path: \"%s\"", linked_path); + for (cell = tablespace_dirs.head; cell; cell = cell->next) + { + elog(INFO, "Remap dir: \"%s\"", cell->old_dir); + if (strcmp(link->linked, cell->old_dir) == 0) { linked_path = cell->new_dir; remapped = true; break; } + } if (!is_absolute_path(linked_path)) elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", From ac1e9a7da95bbfe75fdb452d5bbf729d4cab070a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 15:33:33 +0300 Subject: [PATCH 1687/2107] fix tablespace_map parsing --- src/dir.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dir.c b/src/dir.c index f150039e1..ef23f6842 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1149,11 +1149,11 @@ read_tablespace_map(parray *links, const char *backup_dir) while (fgets(buf, lengthof(buf), fp)) { - char link_name[MAXPGPATH], - path[MAXPGPATH]; - pgFile *file; + char link_name[MAXPGPATH]; + char path[MAXPGPATH]; + pgFile *file; - if (sscanf(buf, "%s %n", link_name, path) != 2) + if (sscanf(buf, "%s %s", link_name, path) != 2) elog(ERROR, "invalid format found in \"%s\"", map_path); file = pgut_new(pgFile); From 7a464232ed5c03a2b560d1bd53c487c23151b634 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 16:01:17 +0300 Subject: [PATCH 1688/2107] another fix for tablespace_map parsing --- src/dir.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/dir.c b/src/dir.c index ef23f6842..bd91c108d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1150,12 +1150,22 @@ read_tablespace_map(parray *links, const char *backup_dir) while (fgets(buf, lengthof(buf), fp)) { char link_name[MAXPGPATH]; - char path[MAXPGPATH]; + char *path; + int n = 0; pgFile *file; + int i = 0; - if (sscanf(buf, "%s %s", link_name, path) != 2) + if (sscanf(buf, "%s %n", link_name, &n) != 1) elog(ERROR, "invalid format found in \"%s\"", map_path); + path = buf + n; + + /* remove '\n' */ + i = strlen(path) - 1; + path[i] = '\0'; + + elog(INFO, "STR: '%s'", path); + file = pgut_new(pgFile); memset(file, 0, sizeof(pgFile)); From b7b3bb728dd6210e0316452b352309dbc90f7232 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 16:34:31 +0300 Subject: [PATCH 1689/2107] minor cleanup --- src/dir.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/dir.c b/src/dir.c index bd91c108d..e7fe9ffa8 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1160,12 +1160,10 @@ read_tablespace_map(parray *links, const char *backup_dir) path = buf + n; - /* remove '\n' */ + /* Remove newline character at the end of string */ i = strlen(path) - 1; path[i] = '\0'; - elog(INFO, "STR: '%s'", path); - file = pgut_new(pgFile); memset(file, 0, sizeof(pgFile)); @@ -1289,12 +1287,8 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg TablespaceListCell *cell; bool remapped = false; - elog(INFO, "Linked path: \"%s\"", linked_path); - for (cell = tablespace_dirs.head; cell; cell = cell->next) { - elog(INFO, "Remap dir: \"%s\"", cell->old_dir); - if (strcmp(link->linked, cell->old_dir) == 0) { linked_path = cell->new_dir; @@ -1303,6 +1297,11 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg } } + if (remapped) + elog(INFO, "Tablespace %s will be remapped from \"%s\" to \"%s\"", link->name, cell->old_dir, cell->new_dir); + else + elog(INFO, "Tablespace %s will be restored using old path \"%s\"", link->name, linked_path); + if (!is_absolute_path(linked_path)) elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", linked_path); From 2a3c90a65ffe18ca75fd22fff1da60db28a7a750 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Apr 2021 16:39:15 +0300 Subject: [PATCH 1690/2107] Version 2.4.14 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f237250b9..bbd516e4d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.13" +#define PROGRAM_VERSION "2.4.14" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 752e96a8c..c2748505e 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.13 \ No newline at end of file +pg_probackup 2.4.14 \ No newline at end of file From a6fabdb1585e1d937d5c44b85bbf641265d843d6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 16 Apr 2021 01:40:44 +0300 Subject: [PATCH 1691/2107] [Issue #366] escape paths in restore_command with double quotes when running restore --- src/restore.c | 2 +- tests/archive.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/restore.c b/src/restore.c index f088ca36c..9594ef0b0 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1375,7 +1375,7 @@ print_recovery_settings(FILE *fp, pgBackup *backup, else { /* default cmdline, ok for local restore */ - sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " + sprintf(restore_command_guc, "\"%s\" archive-get -B \"%s\" --instance \"%s\" " "--wal-file-path=%%p --wal-file-name=%%f", PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, backup_path, instance_name); diff --git a/tests/archive.py b/tests/archive.py index 2d0bb5037..70a86393a 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1715,7 +1715,7 @@ def test_archive_options(self): recovery_content = f.read() self.assertIn( - "restore_command = '{0} archive-get -B {1} --instance {2} " + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " "--remote-port=22 --remote-user={3}'".format( self.probackup_path, backup_dir, 'node', self.user), @@ -1782,7 +1782,7 @@ def test_archive_options_1(self): self.restore_node( backup_dir, 'node', node, options=[ - '--restore-command=none'.format(wal_dir), + '--restore-command=none', '--archive-host=localhost1', '--archive-port=23', '--archive-user={0}'.format(self.user) @@ -1792,7 +1792,7 @@ def test_archive_options_1(self): recovery_content = f.read() self.assertIn( - "restore_command = '{0} archive-get -B {1} --instance {2} " + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " "--remote-port=23 --remote-user={3}'".format( self.probackup_path, backup_dir, 'node', self.user), From 34741de0e1a17f6d8c5021ecc474055dcdebd663 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 00:52:07 +0300 Subject: [PATCH 1692/2107] [Issue #364] added tests.validate.ValidateTest.test_no_validate_tablespace_map --- tests/validate.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index 62116be89..b5e7b685a 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3968,6 +3968,71 @@ def test_validate_missing_page_header_map(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_no_validate_tablespace_map(self): + """ + Check that --no-validate is propagated to tablespace_map + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'external_dir') + + node.safe_psql( + 'postgres', + 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') + + tblspace_new = self.get_tblspace_path(node, 'external_dir_new') + + oid = node.safe_psql( + 'postgres', + "select oid from pg_tablespace where spcname = 'external_dir'").decode('utf-8').rstrip() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'tablespace_map') + + # overwrite tablespace_map file + with open(tablespace_map, "w") as f: + f.write("{0} {1}".format(oid, tblspace_new)) + f.close + + node.cleanup() + + self.restore_node(backup_dir, 'node', node, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that tablespace restore as symlink + tablespace_link = os.path.join(node.data_dir, 'pg_tblspc', oid) + self.assertTrue( + os.path.islink(tablespace_link), + "'%s' is not a symlink" % tablespace_link) + + self.assertEqual( + os.readlink(tablespace_link), + tblspace_new, + "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # validate empty backup list # page from future during validate # page from future during backup From b1442f46dc66d29a2f929c3902164f4c65e0872d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 02:02:40 +0300 Subject: [PATCH 1693/2107] [Issue #364] correctly set null-termination char during tablespace_map parsing --- src/dir.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dir.c b/src/dir.c index e7fe9ffa8..368d40832 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1160,9 +1160,10 @@ read_tablespace_map(parray *links, const char *backup_dir) path = buf + n; - /* Remove newline character at the end of string */ - i = strlen(path) - 1; - path[i] = '\0'; + /* Remove newline character at the end of string if any */ + i = strcspn(path, "\n"); + if (strlen(path) > i) + path[i] = '\0'; file = pgut_new(pgFile); memset(file, 0, sizeof(pgFile)); From 36f21a963ec1b4d33f7bb8fd14694af161582bab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 17:35:26 +0300 Subject: [PATCH 1694/2107] [Issue #369] add elog message about failure of obtaining history file via replication protocol --- src/backup.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 112d8b365..ce8e51228 100644 --- a/src/backup.c +++ b/src/backup.c @@ -161,14 +161,17 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool if (prev_backup == NULL) { /* try to setup multi-timeline backup chain */ - elog(WARNING, "Valid backup on current timeline %u is not found, " + elog(WARNING, "Valid full backup on current timeline %u is not found, " "trying to look up on previous timelines", current.tli); tli_list = get_history_streaming(&instance_config.conn_opt, current.tli, backup_list); if (!tli_list) + { + elog(WARNING, "Failed to obtain current timeline history file via replication protocol"); /* fallback to using archive */ tli_list = catalog_get_timelines(&instance_config); + } if (parray_num(tli_list) == 0) elog(WARNING, "Cannot find valid backup on previous timelines, " From b14dda4d49c9fb67fb56b3fef7a5ef096bd7e236 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 17:36:04 +0300 Subject: [PATCH 1695/2107] [Issue #369] fix tests --- tests/backup.py | 12 +++++------- tests/false_positive.py | 6 ++---- tests/retention.py | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index ec11bdc85..e3bfc84e4 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -149,13 +149,11 @@ def test_incremental_backup_without_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - sleep(1) - try: self.backup_node(backup_dir, 'node', node, backup_type="ptrack") # we should die here because exception is what we expect to happen @@ -166,7 +164,7 @@ def test_incremental_backup_without_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -230,7 +228,7 @@ def test_incremental_backup_corrupt_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -2352,7 +2350,7 @@ def test_parent_choosing_2(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'WARNING: Valid backup on current timeline 1 is not found' in e.message and + 'WARNING: Valid full backup on current timeline 1 is not found' in e.message and 'ERROR: Create new full backup before an incremental one' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3404,7 +3402,7 @@ def test_missing_replication_permission_1(self): return_id=False) self.assertIn( - 'WARNING: Valid backup on current timeline 2 is not found, trying to look up on previous timelines', + 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', output) self.assertIn( diff --git a/tests/false_positive.py b/tests/false_positive.py index fc9ee4b62..d4e7ccf0d 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -83,12 +83,10 @@ def test_incremental_backup_corrupt_full_1(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: Valid backup on current timeline is not found. ' + 'ERROR: Valid full backup on current timeline is not found. ' 'Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - sleep(1) self.assertFalse( True, "Expecting Error because page backup should not be " @@ -98,7 +96,7 @@ def test_incremental_backup_corrupt_full_1(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: Valid backup on current timeline is not found. ' + 'ERROR: Valid full backup on current timeline is not found. ' 'Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/retention.py b/tests/retention.py index 0a439c9e7..75b19c28a 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1712,7 +1712,7 @@ def test_wal_purge_victim(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) From bd284a7a41d8f5832b62a2c88ab7eea8db3b7ce5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 17:36:34 +0300 Subject: [PATCH 1696/2107] Readme: allow tracing for gdb tests --- tests/Readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Readme.md b/tests/Readme.md index adcf5380e..f8dd91db0 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -38,6 +38,7 @@ Run ptrack tests: Usage: + sudo echo 0 > /proc/sys/kernel/yama/ptrace_scope pip install testgres export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] From ed897d45e38367cdbdda214d1b6d472db1c30c99 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 17:49:06 +0300 Subject: [PATCH 1697/2107] [Issue #310] fix tests.replica.ReplicaTest.test_parent_choosing --- tests/replica.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 345f8a7dc..ff9f09fa0 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1660,23 +1660,9 @@ def test_parent_choosing(self): # failing, because without archving, it is impossible to # take multi-timeline backup. - try: - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of timeline switch " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Cannot find valid backup on previous timelines, ' - 'WAL archive is not available' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) # Clean after yourself self.del_test_dir(module_name, fname) From 278d433194553b3113daae1df5bfd6c6ab1aa1d4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 18:02:46 +0300 Subject: [PATCH 1698/2107] [Issue #310] fix tests.replica.ReplicaTest.test_replica_promote_3 --- tests/replica.py | 79 ------------------------------------------------ 1 file changed, 79 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index ff9f09fa0..ce90ef96e 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -1278,85 +1278,6 @@ def test_replica_promote_2(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_replica_promote_3(self): - """ - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.add_instance(backup_dir, 'replica', replica) - - full_id = self.backup_node( - backup_dir, 'replica', - replica, options=['--stream']) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - - replica.promote() - - # failing, because without archving, it is impossible to - # take multi-timeline backup. - try: - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of timeline switch " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Cannot find valid backup on previous timelines, ' - 'WAL archive is not available' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_archive_delta(self): """ From 588c4cd04bfdd3dc95e8ac6d3495150b5e05a2bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 18:31:34 +0300 Subject: [PATCH 1699/2107] tests: fix tests.restore.RestoreTest.test_restore_chain --- tests/restore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index 9db885f09..f42d4fdb9 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1616,7 +1616,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1624,7 +1624,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1636,7 +1636,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass From 4390ad297b832bf18d796f8b6c61a51116a7231d Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 18:37:29 +0300 Subject: [PATCH 1700/2107] tests: fix tests.restore.RestoreTest.test_restore_chain_with_corrupted_backup --- tests/restore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/restore.py b/tests/restore.py index f42d4fdb9..2a11a27a4 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1704,7 +1704,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=0s']) + backup_type='page', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1716,7 +1716,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1728,7 +1728,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass From 09accf7adcc5dbc27fbe0fe8ddd2609612de1060 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 22 Apr 2021 22:59:49 +0300 Subject: [PATCH 1701/2107] fix instance validation so the validation carry on after corrupt backup detected --- src/parsexlog.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index 41a410d30..43b26d1e6 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1290,6 +1290,8 @@ RunXLogThreads(const char *archivedir, time_t target_time, if (thread_args[i].ret == 1) result = false; } + thread_interrupted = false; + interrupted = false; /* Release threads here, use thread_args only below */ pfree(threads); From fd73b9ccb6c5bc38fcc11f9c9d9f6bf3e2df5208 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 23 Apr 2021 13:19:59 +0300 Subject: [PATCH 1702/2107] add TODO for WAL parsing --- src/parsexlog.c | 5 ++++- src/utils/thread.c | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 43b26d1e6..4a0f38642 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1291,7 +1291,10 @@ RunXLogThreads(const char *archivedir, time_t target_time, result = false; } thread_interrupted = false; - interrupted = false; + +// TODO: we must detect difference between actual error (failed to read WAL) and interrupt signal +// if (interrupted) +// elog(ERROR, "Interrupted during WAL parsing"); /* Release threads here, use thread_args only below */ pfree(threads); diff --git a/src/utils/thread.c b/src/utils/thread.c index 5ceee068d..1c469bd29 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -11,6 +11,10 @@ #include "thread.h" +/* + * Global var used to detect error condition (not signal interrupt!) in threads, + * so if one thread errored out, then others may abort + */ bool thread_interrupted = false; #ifdef WIN32 From 82af8f35a6d74c8c312a9d4683dcbde5f1255ae1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 23 Apr 2021 14:09:11 +0300 Subject: [PATCH 1703/2107] [Issue #346] detect failure of streaming thread --- src/backup.c | 2 +- src/data.c | 4 ++-- src/dir.c | 2 ++ src/stream.c | 6 ++++++ src/utils/logger.c | 1 - 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index ce8e51228..3815900b9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1405,7 +1405,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, } sleep(1); - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving"); try_count++; diff --git a/src/data.c b/src/data.c index 999de867d..4370bcbbc 100644 --- a/src/data.c +++ b/src/data.c @@ -1851,7 +1851,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, if (feof(in)) break; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during page reading"); } @@ -1914,7 +1914,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, if (feof(in)) break; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during page reading"); } diff --git a/src/dir.c b/src/dir.c index 368d40832..c5c5b3297 100644 --- a/src/dir.c +++ b/src/dir.c @@ -781,6 +781,8 @@ dir_check_file(pgFile *file, bool backup_logs) * List files in parent->path directory. If "exclude" is true do not add into * "files" files from pgdata_exclude_files and directories from * pgdata_exclude_dir. + * + * TODO: should we check for interrupt here ? */ static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, diff --git a/src/stream.c b/src/stream.c index 21204ae2c..01161f720 100644 --- a/src/stream.c +++ b/src/stream.c @@ -233,7 +233,10 @@ StreamLog(void *arg) ctl.mark_done = false; if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + { + interrupted = true; elog(ERROR, "Problem in receivexlog"); + } #if PG_VERSION_NUM >= 100000 if (!ctl.walmethod->finish()) @@ -245,7 +248,10 @@ StreamLog(void *arg) if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, NULL, (char *) stream_arg->basedir, stop_streaming, standby_message_timeout, NULL, false, false) == false) + { + interrupted = true; elog(ERROR, "Problem in receivexlog"); + } #endif /* be paranoid and sort xlog_files_list, diff --git a/src/utils/logger.c b/src/utils/logger.c index 584b937e7..f039d4a5d 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -169,7 +169,6 @@ exit_if_necessary(int elevel) { /* Interrupt other possible routines */ thread_interrupted = true; - interrupted = true; #ifdef WIN32 ExitThread(elevel); #else From a4ddadd0c3db31cfe0caee1bc9530b697da9c75f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 23 Apr 2021 15:31:39 +0300 Subject: [PATCH 1704/2107] [Issue #355] documentation update --- doc/pgprobackup.xml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index ae9e5f8bb..d5cce129a 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -794,7 +794,7 @@ ALTER ROLE backup WITH REPLICATION; parameter, as follows: -archive_command = 'install_dir/pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-name=%f [remote_options]' +archive_command = '"install_dir/pg_probackup" archive-push -B "backup_dir" --instance instance_name --wal-file-name=%f [remote_options]' @@ -1588,7 +1588,7 @@ pg_probackup validate -B backup_dir --instance -pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' +pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time="2017-05-18 14:18:11+03" If you specify the backup_id of an incremental backup, @@ -1915,7 +1915,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time="2017-05-18 14:18:11+03" @@ -1942,7 +1942,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-name option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' +pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name="before_app_upgrade" @@ -1952,7 +1952,7 @@ pg_probackup restore -B backup_dir --instance latest value: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' +pg_probackup restore -B backup_dir --instance instance_name --recovery-target="latest" @@ -2079,14 +2079,14 @@ pg_probackup restore -B backup_dir --instance restore_command: -restore_command = 'install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +restore_command = '"install_dir/pg_probackup" archive-get -B "backup_dir" --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' Alternatively, you can use the option to provide the entire restore_command: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='"install_dir/pg_probackup" archive-get -B "backup_dir" --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' @@ -3134,7 +3134,7 @@ pg_probackup set-backup -B backup_dir --instance --expire-time option. For example: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' +pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time="2020-01-01 00:00:00+03" Alternatively, you can use the and @@ -3144,7 +3144,7 @@ pg_probackup set-backup -B backup_dir --instance pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d -pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' +pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time="2020-01-01 00:00:00+03" To check if the backup is pinned, @@ -3868,7 +3868,7 @@ pg_probackup restore -B backup_dir --instance -R flag is specified. - Example: --primary-conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' + Example: --primary-conninfo="host=192.168.1.50 port=5432 user=foo password=foopass" @@ -4469,7 +4469,7 @@ pg_probackup archive-get -B backup_dir --instance - Example: --recovery-target-time='2020-01-01 00:00:00+03' + Example: --recovery-target-time="2020-01-01 00:00:00+03" @@ -4659,7 +4659,7 @@ pg_probackup archive-get -B backup_dir --instance - Example: --expire-time='2020-01-01 00:00:00+03' + Example: --expire-time="2020-01-01 00:00:00+03" @@ -5202,7 +5202,7 @@ pg_probackup archive-get -B backup_dir --instance keep-alive for SSH connections opened by pg_probackup: - --ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'. + --ssh-options="-o ServerAliveCountMax=5 -o ServerAliveInterval=60". For the full list of possible options, see ssh_config manual page. @@ -5555,14 +5555,14 @@ INFO: Backup catalog '/mnt/backups' successfully inited Add instance <literal>pg-11</literal> to the backup catalog: -[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main +[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main INFO: Instance 'node' successfully inited Take a FULL backup: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Start transferring data files INFO: Data files are transferred @@ -5577,7 +5577,7 @@ INFO: Backup PZ7YK2 completed Let's take a look at the backup catalog: -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' +[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance pg-11 BACKUP INSTANCE 'pg-11' ================================================================================================================================== @@ -5589,7 +5589,7 @@ BACKUP INSTANCE 'pg-11' Take an incremental backup in the DELTA mode: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b delta --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Parent backup: PZ7YK2 INFO: Start transferring data files @@ -5606,14 +5606,14 @@ INFO: Backup PZ7YMP completed Let's add some parameters to <application>pg_probackup</application> configuration file, so that you can omit them from the command line: -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb +[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb Take another incremental backup in the DELTA mode, omitting some of the previous parameters: -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream +[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b delta --stream INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Parent backup: PZ7YMP INFO: Start transferring data files @@ -5629,7 +5629,7 @@ INFO: Backup PZ7YR5 completed Let's take a look at the instance configuration: -[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' +[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance pg-11 # Backup instance information pgdata = /var/lib/postgresql/11/main @@ -5668,7 +5668,7 @@ remote-host = postgres_host Let's take a look at the backup catalog: -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' +[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance pg-11 ==================================================================================================================================== Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status From 637f1d1c059b8fd20899571865edee84a326c64b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 00:26:40 +0300 Subject: [PATCH 1705/2107] tests: fix archive.ArchiveTest.test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 70a86393a..7f5c75879 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2090,11 +2090,11 @@ def test_archive_pg_receivexlog_partial_handling(self): result = node.safe_psql( "postgres", - "select sum(id) from t_heap") + "select sum(id) from t_heap").decode('utf-8').rstrip() result_new = node_restored.safe_psql( "postgres", - "select sum(id) from t_heap") + "select sum(id) from t_heap").decode('utf-8').rstrip() self.assertEqual(result, result_new) From 898d5f329273d50e91c88bb66ee18b318dac598a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 01:10:36 +0300 Subject: [PATCH 1706/2107] sleep in slow_start --- tests/helpers/ptrack_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 5b4adedcc..793a0b147 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -142,6 +142,7 @@ def slow_start(self, replica=False): else: raise e + sleep(1) class ProbackupTest(object): # Class attributes From 4bcdda4346fc84bdfba0d620faf2c2d06d585702 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 02:22:30 +0300 Subject: [PATCH 1707/2107] tests: fix test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 7f5c75879..ac5a6569e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2008,8 +2008,7 @@ def test_archive_pg_receivexlog_partial_handling(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '10s'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 8291582b4a546629df06d343f5aaa99aad8e7ba5 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 02:30:39 +0300 Subject: [PATCH 1708/2107] tests: another fix for test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/archive.py b/tests/archive.py index ac5a6569e..40be72aab 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2032,8 +2032,10 @@ def test_archive_pg_receivexlog_partial_handling(self): replica.slow_start(replica=True) if self.get_version(replica) < 100000: + app_name = 'pg_receivexlog' pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: + app_name = 'pg_receivewal' pg_receivexlog_path = self.get_bin_path('pg_receivewal') cmdline = [ @@ -2079,6 +2081,7 @@ def test_archive_pg_receivexlog_partial_handling(self): node_restored.data_dir, options=['--recovery-target=latest', '--recovery-target-action=promote']) self.set_auto_conf(node_restored, {'port': node_restored.port}) self.set_auto_conf(node_restored, {'hot_standby': 'off'}) + self.set_auto_conf(node_restored, {'synchronous_standby_names': app_name}) # it will set node_restored as warm standby. # with open(os.path.join(node_restored.data_dir, "standby.signal"), 'w') as f: From b6e948be69f5e91f1eb5fa5c27aa1bf354731b88 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 14:04:37 +0300 Subject: [PATCH 1709/2107] tests: another fix for test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 40be72aab..cbb5059c8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2015,23 +2015,7 @@ def test_archive_pg_receivexlog_partial_handling(self): node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node( - backup_dir, 'node', replica, replica.data_dir, options=['-R']) - self.set_auto_conf(replica, {'port': replica.port}) - self.set_replica(node, replica) - - self.add_instance(backup_dir, 'replica', replica) - # self.set_archiving(backup_dir, 'replica', replica, replica=True) - - replica.slow_start(replica=True) - - if self.get_version(replica) < 100000: + if self.get_version(node) < 100000: app_name = 'pg_receivexlog' pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: @@ -2039,8 +2023,8 @@ def test_archive_pg_receivexlog_partial_handling(self): pg_receivexlog_path = self.get_bin_path('pg_receivewal') cmdline = [ - pg_receivexlog_path, '-p', str(replica.port), '--synchronous', - '-D', os.path.join(backup_dir, 'wal', 'replica')] + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node')] if self.archive_compress and node.major_version >= 10: cmdline += ['-Z', '1'] @@ -2053,8 +2037,12 @@ def test_archive_pg_receivexlog_partial_handling(self): 'Failed to start pg_receivexlog: {0}'.format( pg_receivexlog.communicate()[1])) + self.set_auto_conf(node, {'synchronous_standby_names': app_name}) + self.set_auto_conf(node, {'synchronous_commit': 'on'}) + node.reload() + # FULL - self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", @@ -2064,7 +2052,7 @@ def test_archive_pg_receivexlog_partial_handling(self): # PAGE self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', options=['--stream']) + backup_dir, 'node', node, backup_type='page', options=['--stream']) node.safe_psql( "postgres", @@ -2077,16 +2065,10 @@ def test_archive_pg_receivexlog_partial_handling(self): node_restored.cleanup() self.restore_node( - backup_dir, 'replica', node_restored, - node_restored.data_dir, options=['--recovery-target=latest', '--recovery-target-action=promote']) + backup_dir, 'node', node_restored, node_restored.data_dir, + options=['--recovery-target=latest', '--recovery-target-action=promote']) self.set_auto_conf(node_restored, {'port': node_restored.port}) self.set_auto_conf(node_restored, {'hot_standby': 'off'}) - self.set_auto_conf(node_restored, {'synchronous_standby_names': app_name}) - - # it will set node_restored as warm standby. -# with open(os.path.join(node_restored.data_dir, "standby.signal"), 'w') as f: -# f.flush() -# f.close() node_restored.slow_start() From 7a3f26fbbad3bbf6b7ac22331ab9689b7d5106fd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 14:15:48 +0300 Subject: [PATCH 1710/2107] tests: disable autovacuum in test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index cbb5059c8..6bb85e558 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2008,7 +2008,8 @@ def test_archive_pg_receivexlog_partial_handling(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From df8aadfe2df44d4530d6dc4549e47996284f3ab6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 14:22:51 +0300 Subject: [PATCH 1711/2107] tests: kill pg_receivexlog after test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index 6bb85e558..7f11b808c 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2061,6 +2061,8 @@ def test_archive_pg_receivexlog_partial_handling(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(1000000,2000000) i") + pg_receivexlog.kill() + node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) node_restored.cleanup() @@ -2084,7 +2086,6 @@ def test_archive_pg_receivexlog_partial_handling(self): self.assertEqual(result, result_new) # Clean after yourself - pg_receivexlog.kill() self.del_test_dir(module_name, fname) @unittest.skip("skip") From 13492bf8ca135f8770601be7976612fd3c8cc75b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 15:04:06 +0300 Subject: [PATCH 1712/2107] tests: disable autovacuum in test_checksum_corruption_detection --- tests/incr_restore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 885a88c2e..4838fefc9 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -136,7 +136,8 @@ def test_checksum_corruption_detection(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 9f29fb9b59fcb070376b00e559edf4f9f7986d75 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 15:33:49 +0300 Subject: [PATCH 1713/2107] tests: sleep in slow_start --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 793a0b147..d06527887 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -138,11 +138,11 @@ def slow_start(self, replica=False): except testgres.QueryException as e: if 'database system is starting up' in e.message: - continue + pass else: raise e - sleep(1) + sleep(0.5) class ProbackupTest(object): # Class attributes From e387e8e0d508fbab620f29bdc84f3a2a0e0d5ab3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 16:58:32 +0300 Subject: [PATCH 1714/2107] tests: update expected help --- tests/expected/option_help.out | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 2170e2773..c2b15e7ac 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -122,11 +122,12 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--wal-depth=wal-depth] [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] [--delete-wal] - [--dry-run] + [--dry-run] [--no-validate] [--no-sync] [--help] pg_probackup merge -B backup-path --instance=instance_name -i backup-id [--progress] [-j num-threads] + [--no-validate] [--no-sync] [--help] pg_probackup add-instance -B backup-path -D pgdata-path From 931e0a451c39aaa45c0d89af832800659d2ebdcd Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 17:00:53 +0300 Subject: [PATCH 1715/2107] Version 2.4.15 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bbd516e4d..f5bac64a0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.14" +#define PROGRAM_VERSION "2.4.15" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index c2748505e..05a8660ab 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.14 \ No newline at end of file +pg_probackup 2.4.15 \ No newline at end of file From 2bbecfd06c43be5dc2721a0adee129a7792994eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 19:13:23 +0300 Subject: [PATCH 1716/2107] tests: added test_validate_instance_with_several_corrupt_backups and test_validate_instance_with_several_corrupt_backups_interrupt --- tests/validate.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/tests/validate.py b/tests/validate.py index b5e7b685a..eee990ad6 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -982,6 +982,204 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups(self): + """ + make archive node, take FULL1, PAGE1_1, FULL2, PAGE2_1 backups, FULL3 + corrupt file in FULL and FULL2 and run validate on instance, + expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN + FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_2, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_4, + 'database', rel_path)) + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups_interrupt(self): + """ + check that interrupt during validation is handled correctly + """ + fname = self.id().split('.')[3]q + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_1, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_3, + 'database', rel_path)) + + # Validate Instance + gdb = self.validate_pb( + backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) + + gdb.set_breakpoint('validate_file_pages') + gdb.run_until_break() + gdb.continue_execution_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'Interrupted while locking backup', log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_validate_instance_with_corrupted_page(self): """ From 1c860ff9bceadc7f2b66925e292fe5a79204b24b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 25 Apr 2021 21:39:56 +0300 Subject: [PATCH 1717/2107] tests: fix typo in test_validate_instance_with_several_corrupt_backups_interrupt --- tests/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/validate.py b/tests/validate.py index eee990ad6..c5cc80733 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1088,7 +1088,7 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ check that interrupt during validation is handled correctly """ - fname = self.id().split('.')[3]q + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums']) From 23b00b1ddfec8a262b200228fc06f8e77bb9e2c8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 26 Apr 2021 12:52:37 +0300 Subject: [PATCH 1718/2107] tests: set env for run_binary --- tests/archive.py | 4 +++- tests/helpers/ptrack_helpers.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 7f11b808c..c506ccbf5 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2030,7 +2030,9 @@ def test_archive_pg_receivexlog_partial_handling(self): if self.archive_compress and node.major_version >= 10: cmdline += ['-Z', '1'] - pg_receivexlog = self.run_binary(cmdline, asynchronous=True) + env = self.test_env + env["PGAPPNAME"] = app_name + pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env) if pg_receivexlog.returncode: self.assertFalse( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d06527887..b0400a72d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -779,7 +779,11 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur except subprocess.CalledProcessError as e: raise ProbackupException(e.output.decode('utf-8'), self.cmd) - def run_binary(self, command, asynchronous=False): + def run_binary(self, command, asynchronous=False, env=None): + + if not env: + env = self.test_env + if self.verbose: print([' '.join(map(str, command))]) try: @@ -789,13 +793,13 @@ def run_binary(self, command, asynchronous=False): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.test_env + env=env ) else: self.output = subprocess.check_output( command, stderr=subprocess.STDOUT, - env=self.test_env + env=env ).decode('utf-8') return self.output except subprocess.CalledProcessError as e: From 3837a62dcace3736d24225b80e02409d292ed59f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 26 Apr 2021 12:56:44 +0300 Subject: [PATCH 1719/2107] tests: fix env in test_archive_pg_receivexlog_partial_handling --- tests/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index c506ccbf5..2ebe09b39 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -2032,7 +2032,7 @@ def test_archive_pg_receivexlog_partial_handling(self): env = self.test_env env["PGAPPNAME"] = app_name - pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env) + pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env=env) if pg_receivexlog.returncode: self.assertFalse( From b1ee3a9dc36bc07689f5c4c69d92428e1af2ed07 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 27 Apr 2021 22:19:29 +0300 Subject: [PATCH 1720/2107] DOC: update --- doc/pgprobackup.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index d5cce129a..740517313 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -761,10 +761,9 @@ ALTER ROLE backup WITH REPLICATION; Setting up Continuous WAL Archiving Making backups in PAGE backup mode, performing - PITR, + PITR and making backups with - ARCHIVE WAL delivery mode and - running incremental backup after timeline switch + ARCHIVE WAL delivery mode require continuous WAL archiving to be enabled. To set up continuous From 17251d6677baac69117647541e8ee4087dcddad1 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 28 Apr 2021 11:04:48 +0300 Subject: [PATCH 1721/2107] [Issue #376] follow symlinks when walking the WAL archive directory --- src/catalog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 6ee7b93d4..981841747 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1404,7 +1404,7 @@ catalog_get_timelines(InstanceConfig *instance) /* read all xlog files that belong to this archive */ sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, false, true, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, arclog_path, false, true, false, false, true, 0, FIO_BACKUP_HOST); parray_qsort(xlog_files_list, pgFileCompareName); timelineinfos = parray_new(); From 30f7f78b086846cd823e9a1b9d8a9cff486fc0e6 Mon Sep 17 00:00:00 2001 From: dld-r00f Date: Mon, 3 May 2021 20:07:54 +0300 Subject: [PATCH 1722/2107] Update README.md Error in ubuntu repo installation command for not super user. Sudo command works only for echo command not for redirection and plain user have no rights to write to /etc/apt/sources.list.d/ folder. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ecaf9695..d4833c8de 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #### pg_probackup for vanilla PostgreSQL ```shell #DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list +sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5} sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}-dbg From 89188e81cfd28b6f3395258ff1ea0dee6bb2aa6b Mon Sep 17 00:00:00 2001 From: dld-r00f Date: Tue, 4 May 2021 11:14:09 +0300 Subject: [PATCH 1723/2107] Update README.md Change repo setup for Ubuntu/Deb for PostgresPro section. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4833c8de..ab6af6685 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #### pg_probackup for PostgresPro Standard and Enterprise ```shell #DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list +sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg From 46bf00d60a3f65359d9153be0db7fe6e704ce538 Mon Sep 17 00:00:00 2001 From: dld-r00f Date: Tue, 4 May 2021 11:50:08 +0300 Subject: [PATCH 1724/2107] Update README.md Change repo setup for all sections where the `sudo echo` commands are located. --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ab6af6685..49b9351df 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,8 @@ sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5} sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}-dbg #DEB-SRC Packages -sudo echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ - /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update +sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update sudo apt-get source pg-probackup-{13,12,11,10,9.6,9.5} #RPM Centos Packages @@ -109,19 +109,19 @@ zypper install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo zypper si pg_probackup-{13,12,11,10,9.6,9.5} #RPM ALT Linux 7 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux 8 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #RPM ALT Linux 9 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo @@ -151,19 +151,19 @@ yum install pg_probackup-{std,ent}-{12,11,10,9.6} yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM ALT Linux 7 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM ALT Linux 8 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo #RPM ALT Linux 9 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo ``` From 33c28fdd7f5fe7349b7e4cc9ebb8f1e81fa11bb9 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 12 May 2021 20:39:05 +0300 Subject: [PATCH 1725/2107] Accept ptrack v2.2 as well --- src/ptrack.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ptrack.c b/src/ptrack.c index 06ba44eeb..dc1a2c74c 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -201,6 +201,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = 20; else if (strcmp(ptrack_version_str, "2.1") == 0) nodeInfo->ptrack_version_num = 21; + else if (strcmp(ptrack_version_str, "2.2") == 0) + nodeInfo->ptrack_version_num = 22; else elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s", ptrack_version_str); From 0e4b3a970a811fc02bd9830317d2a0d9776607e6 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 May 2021 12:12:42 +0300 Subject: [PATCH 1726/2107] [Issue #385] pg_stop_backup refactoring, part 1 --- src/backup.c | 899 ++++++++++++++++++++++++--------------------- src/pg_probackup.h | 5 + src/utils/pgut.c | 16 +- src/utils/pgut.h | 3 +- tests/archive.py | 11 +- tests/replica.py | 6 +- 6 files changed, 517 insertions(+), 423 deletions(-) diff --git a/src/backup.c b/src/backup.c index a4b88e02f..46e3ba482 100644 --- a/src/backup.c +++ b/src/backup.c @@ -32,13 +32,25 @@ static parray *backup_files_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; - +// TODO: move to PGnodeInfo bool exclusive_backup = false; /* Is pg_start_backup() was executed */ static bool backup_in_progress = false; -/* Is pg_stop_backup() was sent */ -static bool pg_stop_backup_is_sent = false; + +struct pg_stop_backup_result { + /* + * We will use values of snapshot_xid and invocation_time if there are + * no transactions between start_lsn and stop_lsn. + */ + TransactionId snapshot_xid; + time_t invocation_time; + XLogRecPtr lsn; + size_t backup_label_content_len; + char *backup_label_content; + size_t tablespace_map_content_len; + char *tablespace_map_content; +}; /* * Backup routines @@ -53,7 +65,11 @@ static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, static void pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); +static void pg_silent_client_messages(PGconn *conn); +static void pg_create_restore_point(PGconn *conn, time_t backup_start_time); + static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); +static void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); static XLogRecPtr wait_wal_lsn(InstanceState *instanceState, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, @@ -74,18 +90,20 @@ static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +static StopBackupCallbackState stop_callback_state; + static void backup_stopbackup_callback(bool fatal, void *userdata) { - InstanceState *instanceState = (InstanceState *) userdata; - PGconn *pg_startbackup_conn = instanceState->conn; + StopBackupCallbackState *st = (StopBackupCallbackState *) userdata; /* * If backup is in progress, notify stop of backup to PostgreSQL */ if (backup_in_progress) { elog(WARNING, "backup in progress, stop backup"); - pg_stop_backup(instanceState, NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */ + /* don't care about stop_lsn in case of error */ + pg_stop_backup_send(st->conn, st->server_version, current.from_replica, exclusive_backup, NULL); } } @@ -1048,7 +1066,6 @@ pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pg const char *params[2]; uint32 lsn_hi; uint32 lsn_lo; - params[0] = label; elog(INFO, "wait for pg_start_backup()"); @@ -1071,8 +1088,9 @@ pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pg * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; - instanceState->conn = conn; - pgut_atexit_push(backup_stopbackup_callback, instanceState); + stop_callback_state.conn = conn; + stop_callback_state.server_version = nodeInfo->server_version; + pgut_atexit_push(backup_stopbackup_callback, &stop_callback_state); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1103,9 +1121,7 @@ pg_switch_wal(PGconn *conn) { PGresult *res; - /* Remove annoying NOTICE messages generated by backend */ - res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); - PQclear(res); + pg_silent_client_messages(conn); #if PG_VERSION_NUM >= 100000 res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL); @@ -1450,70 +1466,101 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ } } -/* - * Notify end of backup to PostgreSQL server. - */ +/* Remove annoying NOTICE messages generated by backend */ static void -pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, - PGNodeInfo *nodeInfo) +pg_silent_client_messages(PGconn *conn) { - PGconn *conn; - PGresult *res; - PGresult *tablespace_map_content = NULL; - uint32 lsn_hi; - uint32 lsn_lo; - //XLogRecPtr restore_lsn = InvalidXLogRecPtr; - int pg_stop_backup_timeout = 0; - char path[MAXPGPATH]; - char backup_label[MAXPGPATH]; - FILE *fp; - pgFile *file; - size_t len; - char *val = NULL; - char *stop_backup_query = NULL; - bool stop_lsn_exists = false; - XLogRecPtr stop_backup_lsn_tmp = InvalidXLogRecPtr; - - /* - * We will use this values if there are no transactions between start_lsn - * and stop_lsn. - */ - time_t recovery_time; - TransactionId recovery_xid; - - if (!backup_in_progress) - elog(ERROR, "backup is not in progress"); - - conn = pg_startbackup_conn; - - /* Remove annoying NOTICE messages generated by backend */ + PGresult *res; res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); +} - /* Make proper timestamp format for parse_time() */ - res = pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); - PQclear(res); +static void +pg_create_restore_point(PGconn *conn, time_t backup_start_time) +{ + PGresult *res; + const char *params[1]; + char name[1024]; - /* Create restore point - * Only if backup is from master. - * For PG 9.5 create restore point only if pguser is superuser. - */ - if (backup != NULL && !backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) - { - const char *params[1]; - char name[1024]; + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup_start_time)); + params[0] = name; - snprintf(name, lengthof(name), "pg_probackup, backup_id %s", - base36enc(backup->start_time)); - params[0] = name; + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); +} - res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", - 1, params); - PQclear(res); - } +void +pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text) +{ + static const char + stop_exlusive_backup_query[] = + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + * TODO, question: add NULLs as backup_label and tablespace_map? + */ + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn", + stop_backup_on_master_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false, false)", + stop_backup_on_master_before10_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)", + /* + * In case of backup from replica >= 9.6 we do not trust minRecPoint + * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. + */ + stop_backup_on_replica_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_wal_replay_lsn()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false, false)", + stop_backup_on_replica_before10_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_xlog_replay_location()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)"; + + const char * const stop_backup_query = + is_exclusive ? + stop_exlusive_backup_query : + server_version >= 100000 ? + (is_started_on_replica ? + stop_backup_on_replica_query : + stop_backup_on_master_query + ) : + (is_started_on_replica ? + stop_backup_on_replica_before10_query : + stop_backup_on_master_before10_query + ); + bool sent = false; + + /* Make proper timestamp format for parse_time(recovery_time) */ + pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); + // TODO: check result /* * send pg_stop_backup asynchronously because we could came @@ -1521,415 +1568,437 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb * postgres archive_command problem and in this case we will * wait for pg_stop_backup() forever. */ - - if (!pg_stop_backup_is_sent) - { - bool sent = false; - - if (!exclusive_backup) - { - /* - * Stop the non-exclusive backup. Besides stop_lsn it returns from - * pg_stop_backup(false) copy of the backup label and tablespace map - * so they can be written to disk by the caller. - * In case of backup from replica >= 9.6 we do not trust minRecPoint - * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. - */ - - /* current is used here because of cleanup */ - if (current.from_replica) - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," -#if PG_VERSION_NUM >= 100000 - " pg_catalog.pg_last_wal_replay_lsn()," -#else - " pg_catalog.pg_last_xlog_replay_location()," -#endif - " labelfile," - " spcmapfile" -#if PG_VERSION_NUM >= 100000 - " FROM pg_catalog.pg_stop_backup(false, false)"; -#else - " FROM pg_catalog.pg_stop_backup(false)"; -#endif - else - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" -#if PG_VERSION_NUM >= 100000 - " FROM pg_catalog.pg_stop_backup(false, false)"; -#else - " FROM pg_catalog.pg_stop_backup(false)"; -#endif - - } - else - { - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn"; - } - - sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); - pg_stop_backup_is_sent = true; - if (!sent) - elog(ERROR, "Failed to send pg_stop_backup query"); - } + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + if (!sent) + elog(ERROR, "Failed to send pg_stop_backup query"); /* After we have sent pg_stop_backup, we don't need this callback anymore */ - instanceState->conn = pg_startbackup_conn; - pgut_atexit_pop(backup_stopbackup_callback, instanceState); + pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_state); - /* - * Wait for the result of pg_stop_backup(), but no longer than - * archive_timeout seconds - */ - if (pg_stop_backup_is_sent && !in_cleanup) - { - int timeout = ARCHIVE_TIMEOUT_DEFAULT; - res = NULL; + if (query_text) + *query_text = pgut_strdup(stop_backup_query); +} - /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ - if (instance_config.archive_timeout > 0) - timeout = instance_config.archive_timeout; +/* + * pg_stop_backup_consume -- get 'pg_stop_backup' query results + * side effects: + * - allocates memory for tablespace_map and backup_label contents, so it must freed by caller (if its not null) + * parameters: + * - + */ +static void +pg_stop_backup_consume(PGconn *conn, int server_version, + bool is_exclusive, uint32 timeout, const char *query_text, + struct pg_stop_backup_result *result) +{ + PGresult *query_result; + uint32 pg_stop_backup_timeout = 0; + enum stop_backup_query_result_column_numbers { + recovery_xid_colno = 0, + recovery_time_colno, + lsn_colno, + backup_label_colno, + tablespace_map_colno + }; + + /* and now wait */ + while (1) + { + if (!PQconsumeInput(conn)) + elog(ERROR, "pg_stop backup() failed: %s", + PQerrorMessage(conn)); - while (1) + if (PQisBusy(conn)) { - if (!PQconsumeInput(conn)) - elog(ERROR, "pg_stop backup() failed: %s", - PQerrorMessage(conn)); + pg_stop_backup_timeout++; + sleep(1); - if (PQisBusy(conn)) + if (interrupted) { - pg_stop_backup_timeout++; - sleep(1); - - if (interrupted) - { - pgut_cancel(conn); - elog(ERROR, "interrupted during waiting for pg_stop_backup"); - } + pgut_cancel(conn); + elog(ERROR, "interrupted during waiting for pg_stop_backup"); + } - if (pg_stop_backup_timeout == 1) - elog(INFO, "wait for pg_stop_backup()"); + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); - /* - * If postgres haven't answered in archive_timeout seconds, - * send an interrupt. - */ - if (pg_stop_backup_timeout > timeout) - { - pgut_cancel(conn); - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); - } - } - else + /* + * If postgres haven't answered in archive_timeout seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > timeout) { - res = PQgetResult(conn); - break; + pgut_cancel(conn); + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); } } - - /* Check successfull execution of pg_stop_backup() */ - if (!res) - elog(ERROR, "pg_stop backup() failed"); else { - switch (PQresultStatus(res)) - { - /* - * We should expect only PGRES_TUPLES_OK since pg_stop_backup - * returns tuples. - */ - case PGRES_TUPLES_OK: - break; - default: - elog(ERROR, "query failed: %s query was: %s", - PQerrorMessage(conn), stop_backup_query); - } - elog(INFO, "pg_stop backup() successfully executed"); + query_result = PQgetResult(conn); + break; } + } + /* Check successfull execution of pg_stop_backup() */ + if (!query_result) + elog(ERROR, "pg_stop_backup() failed"); + else + { + switch (PQresultStatus(query_result)) + { + /* + * We should expect only PGRES_TUPLES_OK since pg_stop_backup + * returns tuples. + */ + case PGRES_TUPLES_OK: + break; + default: + elog(ERROR, "query failed: %s query was: %s", + PQerrorMessage(conn), query_text); + } backup_in_progress = false; + elog(INFO, "pg_stop backup() successfully executed"); + } + + /* get results and fill result structure */ + /* get&check recovery_xid */ + if (sscanf(PQgetvalue(query_result, 0, recovery_xid_colno), XID_FMT, &result->snapshot_xid) != 1) + elog(ERROR, + "result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(query_result, 0, recovery_xid_colno)); + + /* get&check recovery_time */ + if (!parse_time(PQgetvalue(query_result, 0, recovery_time_colno), &result->invocation_time, true)) + elog(ERROR, + "result of current_timestamp is invalid: %s", + PQgetvalue(query_result, 0, recovery_time_colno)); + + /* get stop_backup_lsn */ + { + uint32 lsn_hi; + uint32 lsn_lo; // char *target_lsn = "2/F578A000"; // XLogDataFromLSN(target_lsn, &lsn_hi, &lsn_lo); /* Extract timeline and LSN from results of pg_stop_backup() */ - XLogDataFromLSN(PQgetvalue(res, 0, 2), &lsn_hi, &lsn_lo); + XLogDataFromLSN(PQgetvalue(query_result, 0, lsn_colno), &lsn_hi, &lsn_lo); /* Calculate LSN */ - stop_backup_lsn_tmp = ((uint64) lsn_hi) << 32 | lsn_lo; + result->lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + } - /* It is ok for replica to return invalid STOP LSN - * UPD: Apparently it is ok even for a master. - */ - if (!XRecOffIsValid(stop_backup_lsn_tmp)) - { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; + /* get backup_label_content */ + result->backup_label_content = NULL; + // if (!PQgetisnull(query_result, 0, backup_label_colno)) + if (!is_exclusive) + { + result->backup_label_content_len = PQgetlength(query_result, 0, backup_label_colno); + if (result->backup_label_content_len > 0) + result->backup_label_content = pgut_strndup(PQgetvalue(query_result, 0, backup_label_colno), + result->backup_label_content_len); + } else { + result->backup_label_content_len = 0; + } - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* get tablespace_map_content */ + result->tablespace_map_content = NULL; + // if (!PQgetisnull(query_result, 0, tablespace_map_colno)) + if (!is_exclusive) + { + result->tablespace_map_content_len = PQgetlength(query_result, 0, tablespace_map_colno); + if (result->tablespace_map_content_len > 0) + result->tablespace_map_content = pgut_strndup(PQgetvalue(query_result, 0, tablespace_map_colno), + result->tablespace_map_content_len); + } else { + result->tablespace_map_content_len = 0; + } +} - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + invalid LSN, so emulate it for manual testing. - */ - //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); +/* + * helper routine used to write backup_label and tablespace_map in pg_stop_backup() + */ +static void +pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, + const void *data, size_t len, parray *file_list) +{ + FILE *fp; + pgFile *file; + char full_filename[MAXPGPATH]; + + join_path_components(full_filename, path, filename); + fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); + if (fp == NULL) + elog(ERROR, "can't open %s file \"%s\": %s", + error_msg_filename, full_filename, strerror(errno)); + + if (fio_fwrite(fp, data, len) != len || + fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "can't write %s file \"%s\": %s", + error_msg_filename, full_filename, strerror(errno)); - if (stream_wal) - { - snprintf(stream_xlog_path, lengthof(stream_xlog_path), - "%s/%s/%s/%s", instanceState->instance_backup_subdir_path, - base36enc(backup->start_time), - DATABASE_DIR, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = instanceState->instance_wal_subdir_path; + /* + * It's vital to check if backup_files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (file_list) + { + file = pgFileNew(full_filename, filename, true, 0, + FIO_BACKUP_HOST); - GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); + if (S_ISREG(file->mode)) + { + file->crc = pgFileGetCRC(full_filename, true, false); - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return invalid LSN. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid LSN on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN - * on write-idle system. If that fails too, error out. - */ + file->write_size = file->size; + file->uncompressed_size = file->size; + } + parray_append(file_list, file); + } +} - /* stop_lsn is pointing to a 0 byte of xlog segment */ - if (stop_backup_lsn_tmp % instance_config.xlog_seg_size == 0) - { - /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, - false, true, WARNING, stream_wal, backup); - - /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout); - - /* Check that returned LSN is valid and greater than stop_lsn */ - if (XLogRecPtrIsInvalid(lsn_tmp) || - !XRecOffIsValid(lsn_tmp) || - lsn_tmp < stop_backup_lsn_tmp) - { - /* Backup from master should error out here */ - if (!backup->from_replica) - elog(ERROR, "Failed to get next WAL record after %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - - /* No luck, falling back to looking up for previous record */ - elog(WARNING, "Failed to get next WAL record after %X/%X, " - "looking for previous WAL record", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - - /* Despite looking for previous record there is not guarantee of success - * because previous record can be the contrecord. - */ - lsn_tmp = wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, - true, false, ERROR, stream_wal, backup); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record prior to %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - } - } - /* stop lsn is aligned to xlog block size, just find next lsn */ - else if (stop_backup_lsn_tmp % XLOG_BLCKSZ == 0) - { - /* Wait for segment with current stop_lsn */ - wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, - false, true, ERROR, stream_wal, backup); +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, + PGNodeInfo *nodeInfo) +{ + PGconn *conn; + bool stop_lsn_exists = false; + struct pg_stop_backup_result stop_backup_result; + char *xlog_path,stream_xlog_path[MAXPGPATH]; + /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ + int timeout = (instance_config.archive_timeout > 0) ? + instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; + char *query_text = NULL; + + /* Remove it ? */ + if (!backup_in_progress) + elog(ERROR, "backup is not in progress"); - /* Get the next closest record in segment with current stop_lsn */ - lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout, - stop_backup_lsn_tmp); + conn = pg_startbackup_conn; - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record next to %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - } - /* PostgreSQL returned something very illegal as STOP_LSN, error out */ - else - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + pg_silent_client_messages(conn); - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; - } + /* Create restore point + * Only if backup is from master. + * For PG 9.5 create restore point only if pguser is superuser. + */ + if (!backup->from_replica && + !(nodeInfo->server_version < 90600 && + !nodeInfo->is_superuser)) //TODO: check correctness + pg_create_restore_point(conn, backup->start_time); - elog(LOG, "stop_lsn: %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* Execute pg_stop_backup using PostgreSQL connection */ + pg_stop_backup_send(conn, nodeInfo->server_version, current.from_replica, exclusive_backup, &query_text); - /* Write backup_label and tablespace_map */ - if (!exclusive_backup) - { - Assert(PQnfields(res) >= 4); - snprintf(path, lengthof(path), "%s/%s/%s", instanceState->instance_backup_subdir_path, - base36enc(backup->start_time), DATABASE_DIR); - - /* Write backup_label */ - join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fio_fopen(backup_label, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "can't open backup label file \"%s\": %s", - backup_label, strerror(errno)); - - len = strlen(PQgetvalue(res, 0, 3)); - if (fio_fwrite(fp, PQgetvalue(res, 0, 3), len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "can't write backup label file \"%s\": %s", - backup_label, strerror(errno)); + /* + * Wait for the result of pg_stop_backup(), but no longer than + * archive_timeout seconds + */ + pg_stop_backup_consume(conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); - /* - * It's vital to check if backup_files_list is initialized, - * because we could get here because the backup was interrupted - */ - if (backup_files_list) - { - file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0, - FIO_BACKUP_HOST); + /* It is ok for replica to return invalid STOP LSN + * UPD: Apparently it is ok even for a master. + */ + if (!XRecOffIsValid(stop_backup_result.lsn)) + { + char *xlog_path, + stream_xlog_path[MAXPGPATH]; + XLogSegNo segno = 0; + XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - file->crc = pgFileGetCRC(backup_label, true, false); + /* + * Even though the value is invalid, it's expected postgres behaviour + * and we're trying to fix it below. + */ + elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", + (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); - file->write_size = file->size; - file->uncompressed_size = file->size; - parray_append(backup_files_list, file); - } + /* + * Note: even with gdb it is very hard to produce automated tests for + * contrecord + invalid LSN, so emulate it for manual testing. + */ + //stop_backup_result.lsn = stop_backup_result.lsn - XLOG_SEG_SIZE; + //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", + // (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); + + if (stream_wal) + { + snprintf(stream_xlog_path, lengthof(stream_xlog_path), + "%s/%s/%s/%s", instanceState->instance_backup_subdir_path, + base36enc(backup->start_time), + DATABASE_DIR, PG_XLOG_DIR); + xlog_path = stream_xlog_path; } + else + xlog_path = instanceState->instance_wal_subdir_path; + + GetXLogSegNo(stop_backup_result.lsn, segno, instance_config.xlog_seg_size); - if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) - elog(ERROR, - "result of txid_snapshot_xmax() is invalid: %s", - PQgetvalue(res, 0, 0)); - if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) - elog(ERROR, - "result of current_timestamp is invalid: %s", - PQgetvalue(res, 0, 1)); - - /* Get content for tablespace_map from stop_backup results - * in case of non-exclusive backup + /* + * Note, that there is no guarantee that corresponding WAL file even exists. + * Replica may return LSN from future and keep staying in present. + * Or it can return invalid LSN. + * + * That's bad, since we want to get real LSN to save it in backup label file + * and to use it in WAL validation. + * + * So we try to do the following: + * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and + * look for the first valid record in it. + * It solves the problem of occasional invalid LSN on write-busy system. + * 2. Failing that, look for record in previous segment with endpoint + * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN + * on write-idle system. If that fails too, error out. */ - if (!exclusive_backup) - val = PQgetvalue(res, 0, 4); - /* Write tablespace_map */ - if (!exclusive_backup && val && strlen(val) > 0) + /* stop_lsn is pointing to a 0 byte of xlog segment */ + if (stop_backup_result.lsn % instance_config.xlog_seg_size == 0) { - char tablespace_map[MAXPGPATH]; - - join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fio_fopen(tablespace_map, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "can't open tablespace map file \"%s\": %s", - tablespace_map, strerror(errno)); - - len = strlen(val); - if (fio_fwrite(fp, val, len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "can't write tablespace map file \"%s\": %s", - tablespace_map, strerror(errno)); - - if (backup_files_list) + /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ + wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, + false, true, WARNING, stream_wal, backup); + + /* Get the first record in segment with current stop_lsn */ + lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout); + + /* Check that returned LSN is valid and greater than stop_lsn */ + if (XLogRecPtrIsInvalid(lsn_tmp) || + !XRecOffIsValid(lsn_tmp) || + lsn_tmp < stop_backup_result.lsn) { - file = pgFileNew(tablespace_map, PG_TABLESPACE_MAP_FILE, true, 0, - FIO_BACKUP_HOST); - if (S_ISREG(file->mode)) - { - file->crc = pgFileGetCRC(tablespace_map, true, false); - file->write_size = file->size; - } + /* Backup from master should error out here */ + if (!backup->from_replica) + elog(ERROR, "Failed to get next WAL record after %X/%X", + (uint32) (stop_backup_result.lsn >> 32), + (uint32) (stop_backup_result.lsn)); + + /* No luck, falling back to looking up for previous record */ + elog(WARNING, "Failed to get next WAL record after %X/%X, " + "looking for previous WAL record", + (uint32) (stop_backup_result.lsn >> 32), + (uint32) (stop_backup_result.lsn)); + + /* Despite looking for previous record there is not guarantee of success + * because previous record can be the contrecord. + */ + lsn_tmp = wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, + true, false, ERROR, stream_wal, backup); - parray_append(backup_files_list, file); + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record prior to %X/%X", + (uint32) (stop_backup_result.lsn >> 32), + (uint32) (stop_backup_result.lsn)); } } + /* stop lsn is aligned to xlog block size, just find next lsn */ + else if (stop_backup_result.lsn % XLOG_BLCKSZ == 0) + { + /* Wait for segment with current stop_lsn */ + wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, + false, true, ERROR, stream_wal, backup); + + /* Get the next closest record in segment with current stop_lsn */ + lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout, + stop_backup_result.lsn); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record next to %X/%X", + (uint32) (stop_backup_result.lsn >> 32), + (uint32) (stop_backup_result.lsn)); + } + /* PostgreSQL returned something very illegal as STOP_LSN, error out */ + else + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); - if (tablespace_map_content) - PQclear(tablespace_map_content); - PQclear(res); + /* Setting stop_backup_lsn will set stop point for streaming */ + stop_backup_lsn = lsn_tmp; + stop_lsn_exists = true; } - /* Fill in fields if that is the correct end of backup. */ - if (backup != NULL) + elog(LOG, "stop_lsn: %X/%X", + (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); + + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; + char path[MAXPGPATH]; - /* - * Wait for stop_lsn to be archived or streamed. - * If replica returned valid STOP_LSN of not actually existing record, - * look for previous record with endpoint >= STOP_LSN. - */ - if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(instanceState, stop_backup_lsn_tmp, false, backup->tli, - false, false, ERROR, stream_wal, backup); + Assert(stop_backup_result.backup_label_content != NULL); + snprintf(path, lengthof(path), "%s/%s/%s", instanceState->instance_backup_subdir_path, + base36enc(backup->start_time), DATABASE_DIR); - if (stream_wal) + /* Write backup_label */ + pg_stop_backup_write_file_helper(path, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, + backup_files_list); + free(stop_backup_result.backup_label_content); + stop_backup_result.backup_label_content = NULL; + stop_backup_result.backup_label_content_len = 0; + + /* Write tablespace_map */ + if (stop_backup_result.tablespace_map_content != NULL) { - /* This function will also add list of xlog files - * to the passed filelist */ - if(wait_WAL_streaming_end(backup_files_list)) - elog(ERROR, "WAL streaming failed"); + pg_stop_backup_write_file_helper(path, PG_TABLESPACE_MAP_FILE, "tablespace map", + stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, + backup_files_list); + free(stop_backup_result.tablespace_map_content); + stop_backup_result.tablespace_map_content = NULL; + stop_backup_result.tablespace_map_content_len = 0; + } + } + + /* + * Wait for stop_lsn to be archived or streamed. + * If replica returned valid STOP_LSN of not actually existing record, + * look for previous record with endpoint >= STOP_LSN. + */ + if (!stop_lsn_exists) + stop_backup_lsn = wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, + false, false, ERROR, stream_wal, backup); - snprintf(stream_xlog_path, lengthof(stream_xlog_path), "%s/%s/%s/%s", - instanceState->instance_backup_subdir_path, base36enc(backup->start_time), - DATABASE_DIR, PG_XLOG_DIR); + if (stream_wal) + { + /* This function will also add list of xlog files + * to the passed filelist */ + if(wait_WAL_streaming_end(backup_files_list)) + elog(ERROR, "WAL streaming failed"); - xlog_path = stream_xlog_path; - } - else - xlog_path = instanceState->instance_wal_subdir_path; + snprintf(stream_xlog_path, lengthof(stream_xlog_path), "%s/%s/%s/%s", + instanceState->instance_backup_subdir_path, base36enc(backup->start_time), + DATABASE_DIR, PG_XLOG_DIR); - backup->stop_lsn = stop_backup_lsn; - backup->recovery_xid = recovery_xid; + xlog_path = stream_xlog_path; + } + else + xlog_path = instanceState->instance_wal_subdir_path; - elog(LOG, "Getting the Recovery Time from WAL"); + backup->stop_lsn = stop_backup_lsn; + backup->recovery_xid = stop_backup_result.snapshot_xid; - /* iterate over WAL from stop_backup lsn to start_backup lsn */ - if (!read_recovery_info(xlog_path, backup->tli, - instance_config.xlog_seg_size, - backup->start_lsn, backup->stop_lsn, - &backup->recovery_time)) - { - elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); - backup->recovery_time = recovery_time; - } + elog(LOG, "Getting the Recovery Time from WAL"); + + /* iterate over WAL from stop_backup lsn to start_backup lsn */ + if (!read_recovery_info(xlog_path, backup->tli, + instance_config.xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time)) + { + elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); + backup->recovery_time = stop_backup_result.invocation_time; } + + /* Cleanup */ + pg_free(query_text); } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4a97cfd3e..d02bbb033 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -679,6 +679,11 @@ typedef struct BackupPageHeader2 uint16 checksum; } BackupPageHeader2; +typedef struct StopBackupCallbackState { + PGconn *conn; + int server_version; +} StopBackupCallbackState; + /* Special value for compressed_size field */ #define PageIsOk 0 #define SkipCurrentPage -1 diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a1631b106..72f8a2705 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2019, Postgres Professional + * Portions Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -902,6 +902,20 @@ pgut_strdup(const char *str) return ret; } +char * +pgut_strndup(const char *str, size_t n) +{ + char *ret; + + if (str == NULL) + return NULL; + + if ((ret = strndup(str, n)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + return ret; +} + FILE * pgut_fopen(const char *path, const char *mode, bool missing_ok) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index e6ccbf211..6b9e7d740 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2019, Postgres Professional + * Portions Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -61,6 +61,7 @@ extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); extern void *pgut_malloc(size_t size); extern void *pgut_realloc(void *p, size_t size); extern char *pgut_strdup(const char *str); +extern char *pgut_strndup(const char *str, size_t n); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) diff --git a/tests/archive.py b/tests/archive.py index 329c5d676..0eabe5b0c 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -733,7 +733,7 @@ def test_replica_archive(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -768,7 +768,7 @@ def test_replica_archive(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,80680) i") @@ -911,6 +911,11 @@ def test_basic_master_and_replica_concurrent_archiving(self): 'autovacuum': 'off', 'archive_timeout': '10s'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() @@ -956,7 +961,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") diff --git a/tests/replica.py b/tests/replica.py index ce90ef96e..bab5b563b 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -149,7 +149,7 @@ def test_replica_stream_ptrack_backup(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") before = master.safe_psql("postgres", "SELECT * FROM t_heap") @@ -185,7 +185,7 @@ def test_replica_stream_ptrack_backup(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") @@ -279,7 +279,7 @@ def test_replica_archive_page_backup(self): # equal to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,25120) i") From d8050e5ce4e90fbf648877ce0f567479dcb22f5e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Tue, 18 May 2021 14:59:00 +0300 Subject: [PATCH 1727/2107] [Issue #385] improve test coverage --- tests/backup.py | 112 ++++++++++++++++++++++++++++++++ tests/helpers/ptrack_helpers.py | 53 +++++++++++++++ 2 files changed, 165 insertions(+) diff --git a/tests/backup.py b/tests/backup.py index e3bfc84e4..72daaa544 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3459,3 +3459,115 @@ def test_basic_backup_default_transaction_read_only(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_backup_atexit(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=VERBOSE'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + sleep(1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node')[0]['status'], 'ERROR') + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + #print(log_content) + self.assertIn( + 'WARNING: backup in progress, stop backup', + log_content) + + self.assertIn( + 'FROM pg_catalog.pg_stop_backup', + log_content) + + self.assertIn( + 'setting its status to ERROR', + log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_pg_stop_backup_missing_permissions(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + self.simple_bootstrap(node, 'backup') + + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') + else: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') + + # Full backup in streaming mode + try: + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_stop_backup " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: permission denied for function pg_stop_backup", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + self.assertIn( + "query was: SELECT pg_catalog.txid_snapshot_xmax", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b0400a72d..8204ca3d1 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -405,6 +405,59 @@ def make_simple_node( self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) return node + + def simple_bootstrap(self, node, role) -> None: + + node.safe_psql( + 'postgres', + 'CREATE ROLE {0} WITH LOGIN REPLICATION'.format(role)) + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0};'.format(role)) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) + # >= 10 + else: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( From ac2e7ccf1c5f72fbe452bb6110c98cf03077d575 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 24 May 2021 06:22:11 +0300 Subject: [PATCH 1728/2107] PGPRO-5018: Passing prev_backup_start_lsn (also known as horizonLsn) into fio_send_pages() and send_pages() in case of ptrack backup for additional verification of block changes on the server side (without test yet). --- src/data.c | 6 +++--- src/utils/file.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/data.c b/src/data.c index 4370bcbbc..0b02ff15b 100644 --- a/src/data.c +++ b/src/data.c @@ -459,7 +459,7 @@ prepare_page(ConnectionArgs *conn_arg, * Skip page if page lsn is less than START_LSN of parent backup. * Nullified pages must be copied by DELTA backup, just to be safe. */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA && + if ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev && page_st->lsn > 0 && page_st->lsn < prev_backup_start_lsn) @@ -603,7 +603,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, rc = fio_send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, calg, clevel, checksum_version, /* send pagemap if any */ @@ -616,7 +616,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* TODO: stop handling errors internally */ rc = send_pages(conn_arg, to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, calg, clevel, checksum_version, use_pagemap, &headers, backup_mode, ptrack_version_num, ptrack_schema); diff --git a/src/utils/file.c b/src/utils/file.c index 634ddfba0..0bc4622c2 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1992,13 +1992,13 @@ static void fio_send_pages_impl(int out, char* buf) n_blocks_read++; /* - * horizonLsn is not 0 only in case of delta backup. + * horizonLsn is not 0 only in case of delta and ptrack backup. * As far as unsigned number are always greater or equal than zero, * there is no sense to add more checks. */ - if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page, ptrack */ + if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page */ (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta */ + (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta, ptrack */ { int compressed_size = 0; char write_buffer[BLCKSZ*2]; From 012719d28633a53a45a1c41cde6d226354bed037 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 May 2021 02:16:31 +0300 Subject: [PATCH 1729/2107] fix valgrind alerts valgrind detected some uninitialized memory usage. Looks like shift_lsn one is a real bug. --- src/data.c | 41 ++++++++++++++++++++++++++--------------- src/restore.c | 2 +- src/util.c | 2 +- src/utils/pgut.c | 11 +++++++++++ src/utils/pgut.h | 2 ++ 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/data.c b/src/data.c index 4370bcbbc..d70aae8fd 100644 --- a/src/data.c +++ b/src/data.c @@ -2001,13 +2001,14 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f { FILE *in = NULL; FILE *out = NULL; - int hdr_num = -1; off_t cur_pos_out = 0; char curr_page[BLCKSZ]; int n_blocks_read = 0; BlockNumber blknum = 0; datapagemap_iterator_t *iter = NULL; int compressed_size = 0; + BackupPageHeader2 *header = NULL; + parray *harray = NULL; /* stdio buffers */ char *in_buf = NULL; @@ -2046,6 +2047,8 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); } + harray = parray_new(); + while (blknum < file->n_blocks) { PageState page_st; @@ -2063,17 +2066,15 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f if (!out) out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - hdr_num++; - - if (!*headers) - *headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+1) * sizeof(BackupPageHeader2)); + header = pgut_new0(BackupPageHeader2); + *header = (BackupPageHeader2){ + .block = blknum, + .pos = cur_pos_out, + .lsn = page_st.lsn, + .checksum = page_st.checksum, + }; - (*headers)[hdr_num].block = blknum; - (*headers)[hdr_num].pos = cur_pos_out; - (*headers)[hdr_num].lsn = page_st.lsn; - (*headers)[hdr_num].checksum = page_st.checksum; + parray_append(harray, header); compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), rc, curr_page, calg, clevel, @@ -2098,12 +2099,22 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f * Add dummy header, so we can later extract the length of last header * as difference between their offsets. */ - if (*headers) + if (parray_num(harray) > 0) { - file->n_headers = hdr_num +1; - *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+2) * sizeof(BackupPageHeader2)); - (*headers)[hdr_num+1].pos = cur_pos_out; + size_t hdr_num = parray_num(harray); + size_t i; + + file->n_headers = (int) hdr_num; /* is it valid? */ + *headers = (BackupPageHeader2 *) pgut_malloc0((hdr_num + 1) * sizeof(BackupPageHeader2)); + for (i = 0; i < hdr_num; i++) + { + header = (BackupPageHeader2 *)parray_get(harray, i); + (*headers)[i] = *header; + pg_free(header); + } + (*headers)[hdr_num] = (BackupPageHeader2){.pos=cur_pos_out}; } + parray_free(harray); /* cleanup */ if (in && fclose(in)) diff --git a/src/restore.c b/src/restore.c index 9594ef0b0..86317596e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -557,8 +557,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, elog(INFO, "shift LSN: %X/%X", (uint32) (shift_lsn >> 32), (uint32) shift_lsn); - params->shift_lsn = shift_lsn; } + params->shift_lsn = shift_lsn; /* for validation or restore with enabled validation */ if (!params->is_restore || !params->no_validate) diff --git a/src/util.c b/src/util.c index 946957819..87ec36713 100644 --- a/src/util.c +++ b/src/util.c @@ -136,7 +136,7 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo #endif /* copy controlFileSize */ - buffer = pg_malloc(ControlFileSize); + buffer = pg_malloc0(ControlFileSize); memcpy(buffer, ControlFile, sizeof(ControlFileData)); /* Write pg_control */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index a1631b106..eba31faa6 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -877,6 +877,17 @@ pgut_malloc(size_t size) return ret; } +void * +pgut_malloc0(size_t size) +{ + char *ret; + + ret = pgut_malloc(size); + memset(ret, 0, size); + + return ret; +} + void * pgut_realloc(void *p, size_t size) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index e6ccbf211..77337a945 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -59,10 +59,12 @@ extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); * memory allocators */ extern void *pgut_malloc(size_t size); +extern void *pgut_malloc0(size_t size); extern void *pgut_realloc(void *p, size_t size); extern char *pgut_strdup(const char *str); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) /* From d3bbb74d01b2a78f6db79845b667c43bf53cdaab Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 25 May 2021 17:09:42 +0300 Subject: [PATCH 1730/2107] tests: disable autovacuum by default With autovacuum enabled tests are unstable. Especially they are unstable if postgresql is running under valgrind and therefore is severely inhibited (and vacuum has time to be triggered). --- tests/archive.py | 29 ++++--------- tests/backup.py | 20 +++------ tests/compatibility.py | 46 ++++++-------------- tests/delta.py | 19 +-------- tests/exclude.py | 1 - tests/external.py | 73 +++++++++---------------------- tests/helpers/ptrack_helpers.py | 1 + tests/incr_restore.py | 57 +++++++++---------------- tests/merge.py | 76 +++++++++------------------------ tests/page.py | 16 ++----- tests/ptrack.py | 68 +++++++++-------------------- tests/replica.py | 10 ++--- tests/restore.py | 23 +++------- tests/retention.py | 15 +++---- tests/show.py | 9 ++-- tests/time_stamp.py | 6 +-- 16 files changed, 134 insertions(+), 335 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 2ebe09b39..a7bc04e13 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -903,7 +903,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off', 'archive_timeout': '10s'}) replica = self.make_simple_node( @@ -1002,8 +1001,7 @@ def test_concurrent_archiving(self): master = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', master) @@ -1235,8 +1233,7 @@ def test_archive_catalog(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) @@ -1558,8 +1555,7 @@ def test_archive_catalog_1(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1614,8 +1610,7 @@ def test_archive_catalog_2(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1811,8 +1806,7 @@ def test_hexadecimal_timeline(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1875,7 +1869,6 @@ def test_archiving_and_slots(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'checkpoint_timeout': '30s', 'max_wal_size': '64MB'}) @@ -2008,8 +2001,7 @@ def test_archive_pg_receivexlog_partial_handling(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2098,8 +2090,7 @@ def test_multi_timeline_recovery_prefetching(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2214,8 +2205,7 @@ def test_archive_get_batching_sanity(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) @@ -2287,8 +2277,7 @@ def test_archive_get_prefetch_corruption(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/backup.py b/tests/backup.py index e3bfc84e4..d713263c3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1401,8 +1401,7 @@ def test_drop_rel_during_backup_page(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1660,8 +1659,7 @@ def test_pg_11_adjusted_wal_segment_size(self): '--data-checksums', '--wal-segsize=64'], pg_options={ - 'min_wal_size': '128MB', - 'autovacuum': 'off'}) + 'min_wal_size': '128MB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2576,9 +2574,7 @@ def test_issue_132(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2616,9 +2612,7 @@ def test_issue_132_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 @@ -2963,8 +2957,7 @@ def test_issue_203(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3004,8 +2997,7 @@ def test_issue_231(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/compatibility.py b/tests/compatibility.py index da9d72f83..d0fae2528 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -19,9 +19,7 @@ def test_backward_compatibility_page(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -156,8 +154,7 @@ def test_backward_compatibility_delta(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -296,9 +293,7 @@ def test_backward_compatibility_ptrack(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -408,9 +403,7 @@ def test_backward_compatibility_compression(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -572,9 +565,7 @@ def test_backward_compatibility_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -630,8 +621,7 @@ def test_backward_compatibility_merge_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -703,8 +693,7 @@ def test_backward_compatibility_merge_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -826,8 +815,7 @@ def test_backward_compatibility_merge_3(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -953,8 +941,7 @@ def test_backward_compatibility_merge_4(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1036,8 +1023,7 @@ def test_backward_compatibility_merge_5(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1112,8 +1098,7 @@ def test_page_vacuum_truncate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1217,8 +1202,7 @@ def test_page_vacuum_truncate_compression(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1297,8 +1281,7 @@ def test_page_vacuum_truncate_compressed_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1403,8 +1386,7 @@ def test_hidden_files(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) diff --git a/tests/delta.py b/tests/delta.py index e18b8fb63..c2f58d10f 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -26,9 +26,7 @@ def test_basic_delta_vacuum_truncate(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -96,9 +94,6 @@ def test_delta_vacuum_truncate_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored'), @@ -183,9 +178,6 @@ def test_delta_vacuum_truncate_2(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored'), @@ -408,7 +400,6 @@ def test_delta_multiple_segments(self): 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', - 'autovacuum': 'off', 'full_page_writes': 'off' } ) @@ -566,7 +557,6 @@ def test_create_db(self): initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'autovacuum': 'off' } ) @@ -693,7 +683,6 @@ def test_exists_in_previous_backup(self): pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', - 'autovacuum': 'off' } ) @@ -798,7 +787,6 @@ def test_alter_table_set_tablespace_delta(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -884,9 +872,6 @@ def test_alter_database_set_tablespace_delta(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -976,7 +961,6 @@ def test_delta_delete(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -1184,7 +1168,6 @@ def test_delta_pg_resetxlog(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) diff --git a/tests/exclude.py b/tests/exclude.py index 7ee315fa5..83743bf0b 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -156,7 +156,6 @@ def test_exclude_unlogged_tables_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', "shared_buffers": "10MB"}) self.init_pb(backup_dir) diff --git a/tests/external.py b/tests/external.py index 5658de2bf..5c970f57b 100644 --- a/tests/external.py +++ b/tests/external.py @@ -378,8 +378,7 @@ def test_external_backward_compatibility(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -480,8 +479,7 @@ def test_external_backward_compatibility_merge_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -573,8 +571,7 @@ def test_external_backward_compatibility_merge_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -695,8 +692,7 @@ def test_external_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -783,9 +779,7 @@ def test_external_merge_skip_external_dirs(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -883,9 +877,7 @@ def test_external_merge_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -965,8 +957,7 @@ def test_external_merge_3(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1059,9 +1050,7 @@ def test_external_merge_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1155,9 +1144,7 @@ def test_restore_external_changed_data(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1257,7 +1244,6 @@ def test_restore_external_changed_data_1(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'max_wal_size': '32MB'}) self.init_pb(backup_dir) @@ -1365,7 +1351,6 @@ def test_merge_external_changed_data(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'max_wal_size': '32MB'}) self.init_pb(backup_dir) @@ -1469,9 +1454,7 @@ def test_restore_skip_external(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1549,9 +1532,7 @@ def test_external_dir_is_symlink(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1802,9 +1783,7 @@ def test_external_dir_is_tablespace(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1853,9 +1832,7 @@ def test_restore_external_dir_not_empty(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1936,9 +1913,7 @@ def test_restore_external_dir_is_missing(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2023,9 +1998,7 @@ def test_merge_external_dir_is_missing(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2111,9 +2084,7 @@ def test_restore_external_dir_is_empty(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2177,9 +2148,7 @@ def test_merge_external_dir_is_empty(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2246,9 +2215,7 @@ def test_restore_external_dir_string_order(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2328,9 +2295,7 @@ def test_merge_external_dir_string_order(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b0400a72d..3caba25df 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -375,6 +375,7 @@ def make_simple_node( options['log_connections'] = 'on' options['log_disconnections'] = 'on' options['restart_after_crash'] = 'off' + options['autovacuum'] = 'off' # Allow replication in pg_hba.conf if set_replication: diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 4838fefc9..cb684a23a 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -23,8 +23,7 @@ def test_basic_incr_restore(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -86,8 +85,7 @@ def test_basic_incr_restore_into_missing_directory(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -136,8 +134,7 @@ def test_checksum_corruption_detection(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -237,8 +234,7 @@ def test_incr_restore_with_tablespace_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={'autovacuum': 'off'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -299,8 +295,7 @@ def test_incr_restore_with_tablespace_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={'autovacuum': 'off'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -662,8 +657,7 @@ def test_basic_incr_restore_sanity(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - set_replication=True, - pg_options={'autovacuum': 'off'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -735,7 +729,7 @@ def test_incr_checksum_restore(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -825,7 +819,7 @@ def test_incr_lsn_restore(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -914,7 +908,7 @@ def test_incr_lsn_sanity(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -981,8 +975,7 @@ def test_incr_checksum_sanity(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1039,7 +1032,7 @@ def test_incr_checksum_corruption_detection(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), # initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1097,7 +1090,7 @@ def test_incr_lsn_corruption_detection(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1154,8 +1147,7 @@ def test_incr_restore_multiple_external(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1226,8 +1218,7 @@ def test_incr_lsn_restore_multiple_external(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1300,7 +1291,7 @@ def test_incr_lsn_restore_backward(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on', 'hot_standby': 'on'}) + pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1409,7 +1400,6 @@ def test_incr_checksum_restore_backward(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'hot_standby': 'on'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1645,10 +1635,7 @@ def test_incr_checksum_long_xact(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, -# initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1721,10 +1708,7 @@ def test_incr_lsn_long_xact_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, -# initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + set_replication=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1804,7 +1788,6 @@ def test_incr_lsn_long_xact_2(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'off', 'wal_log_hints': 'off'}) @@ -1890,8 +1873,7 @@ def test_incr_restore_zero_size_file_checksum(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1964,8 +1946,7 @@ def test_incr_restore_zero_size_file_lsn(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/merge.py b/tests/merge.py index 186b2f203..668691fc8 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -175,8 +175,7 @@ def test_merge_compressed_backups_1(self): # Initialize instance and backup directory node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - pg_options={'autovacuum': 'off'}) + set_replication=True, initdb_params=["--data-checksums"]) self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) @@ -248,9 +247,6 @@ def test_merge_compressed_and_uncompressed_backups(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -323,9 +319,6 @@ def test_merge_compressed_and_uncompressed_backups_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -400,9 +393,6 @@ def test_merge_compressed_and_uncompressed_backups_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -478,9 +468,6 @@ def test_merge_tablespaces(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -556,9 +543,6 @@ def test_merge_tablespaces_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -640,8 +624,7 @@ def test_merge_page_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -730,8 +713,7 @@ def test_merge_delta_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -822,8 +804,7 @@ def test_merge_ptrack_truncate(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - ptrack_enable=True, - pg_options={'autovacuum': 'off'}) + ptrack_enable=True) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -918,7 +899,6 @@ def test_merge_delta_delete(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -1435,8 +1415,7 @@ def test_crash_after_opening_backup_control_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1488,8 +1467,7 @@ def test_crash_after_opening_backup_control_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1580,8 +1558,7 @@ def test_losing_file_after_failed_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1668,8 +1645,7 @@ def test_failed_merge_after_delete(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1750,8 +1726,7 @@ def test_failed_merge_after_delete_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1827,8 +1802,7 @@ def test_failed_merge_after_delete_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1890,8 +1864,7 @@ def test_failed_merge_after_delete_3(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1981,8 +1954,7 @@ def test_merge_backup_from_future(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2315,8 +2287,7 @@ def test_idempotent_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2393,8 +2364,7 @@ def test_merge_correct_inheritance(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2448,8 +2418,7 @@ def test_merge_correct_inheritance_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2506,8 +2475,7 @@ def test_multi_timeline_merge(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2618,8 +2586,7 @@ def test_merge_page_header_map_retry(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2665,8 +2632,7 @@ def test_missing_data_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2724,8 +2690,7 @@ def test_missing_non_data_file(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2782,8 +2747,7 @@ def test_merge_remote_mode(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/page.py b/tests/page.py index 8208e8319..c1cba6b40 100644 --- a/tests/page.py +++ b/tests/page.py @@ -27,8 +27,7 @@ def test_basic_page_vacuum_truncate(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node_restored')) @@ -115,8 +114,7 @@ def test_page_vacuum_truncate_1(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -373,7 +371,6 @@ def test_page_multiple_segments(self): 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', - 'autovacuum': 'off', 'full_page_writes': 'off'}) self.init_pb(backup_dir) @@ -447,7 +444,6 @@ def test_page_delete(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -521,7 +517,6 @@ def test_page_delete_1(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -1074,7 +1069,6 @@ def test_page_create_db(self): pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', - 'autovacuum': 'off' } ) @@ -1190,8 +1184,7 @@ def test_multi_timeline_page(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1330,7 +1323,7 @@ def test_multitimeline_page_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1403,7 +1396,6 @@ def test_page_pg_resetxlog(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) diff --git a/tests/ptrack.py b/tests/ptrack.py index de76d1d36..aa0bbadc1 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -66,8 +66,7 @@ def test_ptrack_multi_timeline_backup(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -149,8 +148,7 @@ def test_ptrack_multi_timeline_backup_1(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -301,8 +299,7 @@ def test_ptrack_simple(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -666,8 +663,7 @@ def test_ptrack_uncommitted_xact(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'autovacuum': 'off'}) + 'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -815,8 +811,7 @@ def test_ptrack_vacuum_truncate(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -974,8 +969,7 @@ def test_ptrack_stream(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1066,8 +1060,7 @@ def test_ptrack_archive(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1183,8 +1176,7 @@ def test_ptrack_pgpro417(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1266,8 +1258,7 @@ def test_page_pgpro417(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1337,8 +1328,7 @@ def test_full_pgpro417(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1410,8 +1400,7 @@ def test_create_db(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_size': '10GB', - 'autovacuum': 'off'}) + 'max_wal_size': '10GB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1530,8 +1519,7 @@ def test_create_db_on_replica(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1630,8 +1618,7 @@ def test_alter_table_set_tablespace_ptrack(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1724,8 +1711,7 @@ def test_alter_database_set_tablespace_ptrack(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1796,8 +1782,7 @@ def test_drop_tablespace(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1893,8 +1878,7 @@ def test_ptrack_alter_tablespace(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2011,7 +1995,6 @@ def test_ptrack_multiple_segments(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'off'}) self.init_pb(backup_dir) @@ -2768,9 +2751,7 @@ def test_ptrack_empty(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -2851,8 +2832,7 @@ def test_ptrack_empty_replica(self): base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], - ptrack_enable=True, - pg_options={'autovacuum': 'off'}) + ptrack_enable=True) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3031,8 +3011,7 @@ def test_basic_ptrack_truncate_replica(self): pg_options={ 'max_wal_size': '32MB', 'archive_timeout': '10s', - 'checkpoint_timeout': '5min', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '5min'}) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3692,8 +3671,7 @@ def test_ptrack_vacuum_full_replica(self): base_dir=os.path.join(module_name, fname, 'master'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3982,9 +3960,7 @@ def test_ptrack_recovery(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -4048,7 +4024,6 @@ def test_ptrack_recovery_1(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) @@ -4181,7 +4156,6 @@ def test_ptrack_pg_resetxlog(self): ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) diff --git a/tests/replica.py b/tests/replica.py index ce90ef96e..d59b11dbf 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -983,7 +983,6 @@ def test_replica_toast(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'checkpoint_timeout': '1h', 'wal_level': 'replica', 'shared_buffers': '128MB'}) @@ -1084,7 +1083,6 @@ def test_start_stop_lsn_in_the_same_segno(self): set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'checkpoint_timeout': '1h', 'wal_level': 'replica', 'shared_buffers': '128MB'}) @@ -1293,8 +1291,7 @@ def test_replica_promote_archive_delta(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'archive_timeout': '30s', - 'autovacuum': 'off'}) + 'archive_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) @@ -1414,8 +1411,7 @@ def test_replica_promote_archive_page(self): initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'archive_timeout': '30s', - 'autovacuum': 'off'}) + 'archive_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) @@ -1645,7 +1641,7 @@ def test_replica_via_basebackup(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'hot_standby': 'on'}) + pg_options={'hot_standby': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/restore.py b/tests/restore.py index 2a11a27a4..61aae9285 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1028,8 +1028,7 @@ def test_restore_with_missing_or_corrupted_tablespace_map(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1488,7 +1487,6 @@ def test_zags_block_corrupt_1(self): base_dir=os.path.join(module_name, fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'on'} ) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1864,9 +1862,7 @@ def test_restore_backup_from_future(self): node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -3177,8 +3173,7 @@ def test_missing_database_map(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3580,8 +3575,7 @@ def test_issue_249(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3659,8 +3653,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3729,8 +3722,7 @@ def test_drop_postgresql_auto_conf(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3775,8 +3767,7 @@ def test_truncate_postgresql_auto_conf(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/retention.py b/tests/retention.py index 75b19c28a..19204807b 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -619,8 +619,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum':'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -765,8 +764,7 @@ def test_basic_window_merge_multiple_descendants(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1026,8 +1024,7 @@ def test_basic_window_merge_multiple_descendants_1(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1292,8 +1289,7 @@ def test_window_chains(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1855,8 +1851,7 @@ def test_wal_depth_1(self): initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/show.py b/tests/show.py index 3fdd85d04..2a13a768b 100644 --- a/tests/show.py +++ b/tests/show.py @@ -216,8 +216,7 @@ def test_corrupt_correctness(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -323,8 +322,7 @@ def test_corrupt_correctness_1(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -434,8 +432,7 @@ def test_corrupt_correctness_2(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index 4a4198c27..c49d183da 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -115,8 +115,7 @@ def test_dst_timezone_handling(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -191,8 +190,7 @@ def test_dst_timezone_handling_backward_compatibilty(self): backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) From 4188a699f220b78e76d5398c4a4f62044d7f37dc Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Wed, 26 May 2021 12:57:10 +0300 Subject: [PATCH 1731/2107] Version 2.4.16 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f5bac64a0..746b0f5a5 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.15" +#define PROGRAM_VERSION "2.4.16" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 05a8660ab..1330acb5a 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.15 \ No newline at end of file +pg_probackup 2.4.16 \ No newline at end of file From 3c66873f57ec89a78cde126df9f61560b0936a0e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 May 2021 02:25:56 +0300 Subject: [PATCH 1732/2107] [PR #386] fix --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index 601e4f674..2a8a42764 100644 --- a/src/data.c +++ b/src/data.c @@ -335,8 +335,8 @@ prepare_page(ConnectionArgs *conn_arg, return PageIsOk; case PAGE_IS_VALID: - /* in DELTA mode we must compare lsn */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA) + /* in DELTA or PTRACK modes we must compare lsn */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) page_is_valid = true; else return PageIsOk; From 151d4999b0f78994dbfbb8f040aa86426d9fad98 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 27 May 2021 02:27:19 +0300 Subject: [PATCH 1733/2107] [PR #386] test coverage --- tests/ptrack.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index aa0bbadc1..011f8754a 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4436,3 +4436,78 @@ def test_corrupt_ptrack_map(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_horizon_lsn_ptrack(self): + """ + https://github.com/postgrespro/pg_probackup/pull/386 + """ + + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip("You need PostgreSQL >= 11 for this test") + + self.assertLessEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num('2.4.15'), + 'You need pg_probackup old_binary =< 2.4.15 for this test') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + if node.major_version >= 11: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # TODO: ptrack version must be 2.1 + ptrack_version = node.safe_psql( + "postgres", + "SELECT extversion " + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() + + self.assertEqual( + ptrack_version, + "2.1", + "You need ptrack 2.1 for this test") + + # set map_size to a minimal value + self.set_auto_conf(node, {'ptrack.map_size': '1'}) + node.restart() + + node.pgbench_init(scale=100) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node, options=['--stream'], old_binary=True) + + # enable archiving so the WAL size to do interfere with data bytes comparison later + self.set_archiving(backup_dir, 'node', node) + node.restart() + + # change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA is exemplar + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + delta_bytes = self.show_pb(backup_dir, 'node', backup_id=delta_id)["data-bytes"] + self.delete_pb(backup_dir, 'node', backup_id=delta_id) + + # PTRACK with current binary + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + ptrack_bytes = self.show_pb(backup_dir, 'node', backup_id=ptrack_id)["data-bytes"] + + # make sure that backup size is exactly the same + self.assertEqual(delta_bytes, ptrack_bytes) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 3013a783f33a10d13911d2a9ae79cff433384adf Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 27 May 2021 03:14:15 +0300 Subject: [PATCH 1734/2107] Release 2 5 pg stop backup decomposition2 (#387) * Rename pg_checksum_enable() to pg_is_checksum_enabled * Remove unused instanceState from pg_start_backup() * Refactor wait_wal_lsn(): remove unused pgBackup * parameter and replace InstanceState * with simple directory string * Refactor pg_stop_backup(): remove useless conn variable * Make some functions and variables (from backup.c) accessible from other compilation units * Remove some references to global stream_wal variable * Remove unused variable externaldir * Yet another split of pg_stop_backup(): separate verification of stop_lsn into wait_wal_and_calculate_stop_lsn() * Create pfilearray_clear_locks() helper function --- src/backup.c | 425 ++++++++++++++++++++------------------------- src/checkdb.c | 3 +- src/dir.c | 16 ++ src/pg_probackup.c | 2 - src/pg_probackup.h | 45 ++++- src/restore.c | 8 +- src/utils/parray.c | 2 +- src/validate.c | 6 +- 8 files changed, 257 insertions(+), 250 deletions(-) diff --git a/src/backup.c b/src/backup.c index 46e3ba482..21df1d95e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -27,7 +27,7 @@ //const char *progname = "pg_probackup"; /* list of files contained in backup */ -static parray *backup_files_list = NULL; +parray *backup_files_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -36,21 +36,7 @@ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; bool exclusive_backup = false; /* Is pg_start_backup() was executed */ -static bool backup_in_progress = false; - -struct pg_stop_backup_result { - /* - * We will use values of snapshot_xid and invocation_time if there are - * no transactions between start_lsn and stop_lsn. - */ - TransactionId snapshot_xid; - time_t invocation_time; - XLogRecPtr lsn; - size_t backup_label_content_len; - char *backup_label_content; - size_t tablespace_map_content_len; - char *tablespace_map_content; -}; +bool backup_in_progress = false; /* * Backup routines @@ -62,18 +48,9 @@ static void *backup_files(void *arg); static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); -static void pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); -static void pg_silent_client_messages(PGconn *conn); -static void pg_create_restore_point(PGconn *conn, time_t backup_start_time); static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); -static void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); - -static XLogRecPtr wait_wal_lsn(InstanceState *instanceState, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir, pgBackup *backup); static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); @@ -83,19 +60,19 @@ static parray *get_database_map(PGconn *pg_startbackup_conn); static bool pgpro_support(PGconn *conn); /* Check functions */ -static bool pg_checksum_enable(PGconn *conn); +static bool pg_is_checksum_enabled(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); -static StopBackupCallbackState stop_callback_state; +static StopBackupCallbackParams stop_callback_params; static void backup_stopbackup_callback(bool fatal, void *userdata) { - StopBackupCallbackState *st = (StopBackupCallbackState *) userdata; + StopBackupCallbackParams *st = (StopBackupCallbackParams *) userdata; /* * If backup is in progress, notify stop of backup to PostgreSQL */ @@ -158,7 +135,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, strlen(" with pg_probackup")); /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(instanceState, label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); + pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); /* Obtain current timeline */ #if PG_VERSION_NUM >= 90600 @@ -214,11 +191,11 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, if (prev_backup) { - if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) - elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " - "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); + if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); @@ -282,7 +259,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, write_backup(¤t, true); /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !stream_wal) + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !current.stream) { /* Check that archive_dir can be reached */ if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) @@ -294,11 +271,11 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, * Because WAL streaming will start after pg_start_backup() in stream * mode. */ - wait_wal_lsn(instanceState, current.start_lsn, true, current.tli, false, true, ERROR, false, ¤t); + wait_wal_lsn(instanceState->instance_wal_subdir_path, current.start_lsn, true, current.tli, false, true, ERROR, false); } /* start stream replication */ - if (stream_wal) + if (current.stream) { join_path_components(dst_backup_path, current.database_dir, PG_XLOG_DIR); fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); @@ -310,7 +287,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, * PAGE backup in stream mode is waited twice, first for * segment in WAL archive and then for streamed segment */ - wait_wal_lsn(instanceState, current.start_lsn, true, current.tli, false, true, ERROR, true, ¤t); + wait_wal_lsn(dst_backup_path, current.start_lsn, true, current.tli, false, true, ERROR, true); } /* initialize backup's file list */ @@ -453,7 +430,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, } /* - * Make directories before backup and setup threads at the same time + * Make directories before backup */ for (i = 0; i < parray_num(backup_files_list); i++) { @@ -478,10 +455,11 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } - /* setup threads */ - pg_atomic_clear_flag(&file->lock); } + /* setup thread locks */ + pfilearray_clear_locks(backup_files_list); + /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); /* Sort the array for binary search */ @@ -728,7 +706,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) /* Confirm that this server version is supported */ check_server_version(cur_conn, nodeInfo); - if (pg_checksum_enable(cur_conn)) + if (pg_is_checksum_enabled(cur_conn)) current.checksum_version = 1; else current.checksum_version = 0; @@ -1058,8 +1036,8 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) /* * Notify start of backup to PostgreSQL server. */ -static void -pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pgBackup *backup, +void +pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn) { PGresult *res; @@ -1088,9 +1066,9 @@ pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pg * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; - stop_callback_state.conn = conn; - stop_callback_state.server_version = nodeInfo->server_version; - pgut_atexit_push(backup_stopbackup_callback, &stop_callback_state); + stop_callback_params.conn = conn; + stop_callback_params.server_version = nodeInfo->server_version; + pgut_atexit_push(backup_stopbackup_callback, &stop_callback_params); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1099,7 +1077,7 @@ pg_start_backup(InstanceState *instanceState, const char *label, bool smooth, pg PQclear(res); - if ((!stream_wal || current.backup_mode == BACKUP_MODE_DIFF_PAGE) && + if ((!backup->stream || backup->backup_mode == BACKUP_MODE_DIFF_PAGE) && !backup->from_replica && !(nodeInfo->server_version < 90600 && !nodeInfo->is_superuser)) @@ -1218,7 +1196,7 @@ get_database_map(PGconn *conn) /* Check if ptrack is enabled in target instance */ static bool -pg_checksum_enable(PGconn *conn) +pg_is_checksum_enabled(PGconn *conn) { PGresult *res_db; @@ -1284,7 +1262,7 @@ pg_is_superuser(PGconn *conn) * previous segment. * * Flag 'in_stream_dir' determine whether we looking for WAL in 'pg_wal' directory or - * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' because, + * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' (current.stream) because, * for example, PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. * * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than @@ -1293,15 +1271,13 @@ pg_is_superuser(PGconn *conn) * Returns target LSN if such is found, failing that returns LSN of record prior to target LSN. * Returns InvalidXLogRecPtr if 'segment_only' flag is used. */ -static XLogRecPtr -wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, +XLogRecPtr +wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir, pgBackup *backup) + int timeout_elevel, bool in_stream_dir) { XLogSegNo targetSegNo; - char pg_wal_dir[MAXPGPATH]; char wal_segment_path[MAXPGPATH], - *wal_segment_dir, wal_segment[MAXFNAMELEN]; bool file_exists = false; uint32 try_count = 0, @@ -1319,6 +1295,7 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ GetXLogFileName(wal_segment, tli, targetSegNo, instance_config.xlog_seg_size); + join_path_components(wal_segment_path, wal_segment_dir, wal_segment); /* * In pg_start_backup we wait for 'target_lsn' in 'pg_wal' directory if it is * stream and non-page backup. Page backup needs archived WAL files, so we @@ -1326,17 +1303,6 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ * * In pg_stop_backup it depends only on stream_wal. */ - if (in_stream_dir) - { - join_path_components(pg_wal_dir, backup->database_dir, PG_XLOG_DIR); - join_path_components(wal_segment_path, pg_wal_dir, wal_segment); - wal_segment_dir = pg_wal_dir; - } - else - { - join_path_components(wal_segment_path, instanceState->instance_wal_subdir_path, wal_segment); - wal_segment_dir = instanceState->instance_wal_subdir_path; - } /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ if (instance_config.archive_timeout > 0) @@ -1442,7 +1408,7 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ wal_delivery_str, wal_segment_path); } - if (!stream_wal && is_start_lsn && try_count == 30) + if (!current.stream && is_start_lsn && try_count == 30) elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " "If continuous archiving is not set up, use '--stream' option to make autonomous backup. " "Otherwise check that continuous archiving works correctly."); @@ -1466,8 +1432,144 @@ wait_wal_lsn(InstanceState *instanceState, XLogRecPtr target_lsn, bool is_start_ } } +/* + * Check stop_lsn (returned from pg_stop_backup()) and update backup->stop_lsn + */ +void +wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup) +{ + bool stop_lsn_exists = false; + + /* It is ok for replica to return invalid STOP LSN + * UPD: Apparently it is ok even for a master. + */ + if (!XRecOffIsValid(stop_lsn)) + { + XLogSegNo segno = 0; + XLogRecPtr lsn_tmp = InvalidXLogRecPtr; + + /* + * Even though the value is invalid, it's expected postgres behaviour + * and we're trying to fix it below. + */ + elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* + * Note: even with gdb it is very hard to produce automated tests for + * contrecord + invalid LSN, so emulate it for manual testing. + */ + //lsn = lsn - XLOG_SEG_SIZE; + //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", + // (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + GetXLogSegNo(stop_lsn, segno, instance_config.xlog_seg_size); + + /* + * Note, that there is no guarantee that corresponding WAL file even exists. + * Replica may return LSN from future and keep staying in present. + * Or it can return invalid LSN. + * + * That's bad, since we want to get real LSN to save it in backup label file + * and to use it in WAL validation. + * + * So we try to do the following: + * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and + * look for the first valid record in it. + * It solves the problem of occasional invalid LSN on write-busy system. + * 2. Failing that, look for record in previous segment with endpoint + * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN + * on write-idle system. If that fails too, error out. + */ + + /* stop_lsn is pointing to a 0 byte of xlog segment */ + if (stop_lsn % instance_config.xlog_seg_size == 0) + { + /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ + wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, true, WARNING, backup->stream); + + /* Get the first record in segment with current stop_lsn */ + lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout); + + /* Check that returned LSN is valid and greater than stop_lsn */ + if (XLogRecPtrIsInvalid(lsn_tmp) || + !XRecOffIsValid(lsn_tmp) || + lsn_tmp < stop_lsn) + { + /* Backup from master should error out here */ + if (!backup->from_replica) + elog(ERROR, "Failed to get next WAL record after %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + + /* No luck, falling back to looking up for previous record */ + elog(WARNING, "Failed to get next WAL record after %X/%X, " + "looking for previous WAL record", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + + /* Despite looking for previous record there is not guarantee of success + * because previous record can be the contrecord. + */ + lsn_tmp = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + true, false, ERROR, backup->stream); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record prior to %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + } + } + /* stop lsn is aligned to xlog block size, just find next lsn */ + else if (stop_lsn % XLOG_BLCKSZ == 0) + { + /* Wait for segment with current stop_lsn */ + wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, true, ERROR, backup->stream); + + /* Get the next closest record in segment with current stop_lsn */ + lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout, + stop_lsn); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record next to %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + } + /* PostgreSQL returned something very illegal as STOP_LSN, error out */ + else + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* Setting stop_backup_lsn will set stop point for streaming */ + stop_backup_lsn = lsn_tmp; + stop_lsn_exists = true; + } + + elog(LOG, "stop_lsn: %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* + * Wait for stop_lsn to be archived or streamed. + * If replica returned valid STOP_LSN of not actually existing record, + * look for previous record with endpoint >= STOP_LSN. + */ + if (!stop_lsn_exists) + stop_backup_lsn = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, false, ERROR, backup->stream); + + backup->stop_lsn = stop_backup_lsn; +} + /* Remove annoying NOTICE messages generated by backend */ -static void +void pg_silent_client_messages(PGconn *conn) { PGresult *res; @@ -1476,7 +1578,7 @@ pg_silent_client_messages(PGconn *conn) PQclear(res); } -static void +void pg_create_restore_point(PGconn *conn, time_t backup_start_time) { PGresult *res; @@ -1573,7 +1675,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica elog(ERROR, "Failed to send pg_stop_backup query"); /* After we have sent pg_stop_backup, we don't need this callback anymore */ - pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_state); + pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); if (query_text) *query_text = pgut_strdup(stop_backup_query); @@ -1586,10 +1688,10 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica * parameters: * - */ -static void +void pg_stop_backup_consume(PGconn *conn, int server_version, bool is_exclusive, uint32 timeout, const char *query_text, - struct pg_stop_backup_result *result) + PGStopBackupResult *result) { PGresult *query_result; uint32 pg_stop_backup_timeout = 0; @@ -1717,7 +1819,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* * helper routine used to write backup_label and tablespace_map in pg_stop_backup() */ -static void +void pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, const void *data, size_t len, parray *file_list) { @@ -1738,7 +1840,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c error_msg_filename, full_filename, strerror(errno)); /* - * It's vital to check if backup_files_list is initialized, + * It's vital to check if files_list is initialized, * because we could get here because the backup was interrupted */ if (file_list) @@ -1764,10 +1866,8 @@ static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo) { - PGconn *conn; - bool stop_lsn_exists = false; - struct pg_stop_backup_result stop_backup_result; - char *xlog_path,stream_xlog_path[MAXPGPATH]; + PGStopBackupResult stop_backup_result; + char *xlog_path, stream_xlog_path[MAXPGPATH]; /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ int timeout = (instance_config.archive_timeout > 0) ? instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; @@ -1777,9 +1877,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb if (!backup_in_progress) elog(ERROR, "backup is not in progress"); - conn = pg_startbackup_conn; - - pg_silent_client_messages(conn); + pg_silent_client_messages(pg_startbackup_conn); /* Create restore point * Only if backup is from master. @@ -1788,157 +1886,34 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb if (!backup->from_replica && !(nodeInfo->server_version < 90600 && !nodeInfo->is_superuser)) //TODO: check correctness - pg_create_restore_point(conn, backup->start_time); + pg_create_restore_point(pg_startbackup_conn, backup->start_time); /* Execute pg_stop_backup using PostgreSQL connection */ - pg_stop_backup_send(conn, nodeInfo->server_version, current.from_replica, exclusive_backup, &query_text); + pg_stop_backup_send(pg_startbackup_conn, nodeInfo->server_version, backup->from_replica, exclusive_backup, &query_text); /* * Wait for the result of pg_stop_backup(), but no longer than * archive_timeout seconds */ - pg_stop_backup_consume(conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); + pg_stop_backup_consume(pg_startbackup_conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); - /* It is ok for replica to return invalid STOP LSN - * UPD: Apparently it is ok even for a master. - */ - if (!XRecOffIsValid(stop_backup_result.lsn)) + if (backup->stream) { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", - (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); - - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + invalid LSN, so emulate it for manual testing. - */ - //stop_backup_result.lsn = stop_backup_result.lsn - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); - - if (stream_wal) - { - snprintf(stream_xlog_path, lengthof(stream_xlog_path), - "%s/%s/%s/%s", instanceState->instance_backup_subdir_path, - base36enc(backup->start_time), - DATABASE_DIR, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = instanceState->instance_wal_subdir_path; - - GetXLogSegNo(stop_backup_result.lsn, segno, instance_config.xlog_seg_size); - - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return invalid LSN. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid LSN on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN - * on write-idle system. If that fails too, error out. - */ - - /* stop_lsn is pointing to a 0 byte of xlog segment */ - if (stop_backup_result.lsn % instance_config.xlog_seg_size == 0) - { - /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, - false, true, WARNING, stream_wal, backup); - - /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout); - - /* Check that returned LSN is valid and greater than stop_lsn */ - if (XLogRecPtrIsInvalid(lsn_tmp) || - !XRecOffIsValid(lsn_tmp) || - lsn_tmp < stop_backup_result.lsn) - { - /* Backup from master should error out here */ - if (!backup->from_replica) - elog(ERROR, "Failed to get next WAL record after %X/%X", - (uint32) (stop_backup_result.lsn >> 32), - (uint32) (stop_backup_result.lsn)); - - /* No luck, falling back to looking up for previous record */ - elog(WARNING, "Failed to get next WAL record after %X/%X, " - "looking for previous WAL record", - (uint32) (stop_backup_result.lsn >> 32), - (uint32) (stop_backup_result.lsn)); - - /* Despite looking for previous record there is not guarantee of success - * because previous record can be the contrecord. - */ - lsn_tmp = wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, - true, false, ERROR, stream_wal, backup); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record prior to %X/%X", - (uint32) (stop_backup_result.lsn >> 32), - (uint32) (stop_backup_result.lsn)); - } - } - /* stop lsn is aligned to xlog block size, just find next lsn */ - else if (stop_backup_result.lsn % XLOG_BLCKSZ == 0) - { - /* Wait for segment with current stop_lsn */ - wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, - false, true, ERROR, stream_wal, backup); - - /* Get the next closest record in segment with current stop_lsn */ - lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout, - stop_backup_result.lsn); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record next to %X/%X", - (uint32) (stop_backup_result.lsn >> 32), - (uint32) (stop_backup_result.lsn)); - } - /* PostgreSQL returned something very illegal as STOP_LSN, error out */ - else - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); - - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; + join_path_components(stream_xlog_path, backup->database_dir, PG_XLOG_DIR); + xlog_path = stream_xlog_path; } + else + xlog_path = instanceState->instance_wal_subdir_path; - elog(LOG, "stop_lsn: %X/%X", - (uint32) (stop_backup_result.lsn >> 32), (uint32) (stop_backup_result.lsn)); + wait_wal_and_calculate_stop_lsn(xlog_path, stop_backup_result.lsn, backup); /* Write backup_label and tablespace_map */ if (!exclusive_backup) { - char path[MAXPGPATH]; - Assert(stop_backup_result.backup_label_content != NULL); - snprintf(path, lengthof(path), "%s/%s/%s", instanceState->instance_backup_subdir_path, - base36enc(backup->start_time), DATABASE_DIR); /* Write backup_label */ - pg_stop_backup_write_file_helper(path, PG_BACKUP_LABEL_FILE, "backup label", + pg_stop_backup_write_file_helper(backup->database_dir, PG_BACKUP_LABEL_FILE, "backup label", stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, backup_files_list); free(stop_backup_result.backup_label_content); @@ -1948,7 +1923,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb /* Write tablespace_map */ if (stop_backup_result.tablespace_map_content != NULL) { - pg_stop_backup_write_file_helper(path, PG_TABLESPACE_MAP_FILE, "tablespace map", + pg_stop_backup_write_file_helper(backup->database_dir, PG_TABLESPACE_MAP_FILE, "tablespace map", stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, backup_files_list); free(stop_backup_result.tablespace_map_content); @@ -1957,32 +1932,14 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb } } - /* - * Wait for stop_lsn to be archived or streamed. - * If replica returned valid STOP_LSN of not actually existing record, - * look for previous record with endpoint >= STOP_LSN. - */ - if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(instanceState, stop_backup_result.lsn, false, backup->tli, - false, false, ERROR, stream_wal, backup); - - if (stream_wal) + if (backup->stream) { /* This function will also add list of xlog files * to the passed filelist */ if(wait_WAL_streaming_end(backup_files_list)) elog(ERROR, "WAL streaming failed"); - - snprintf(stream_xlog_path, lengthof(stream_xlog_path), "%s/%s/%s/%s", - instanceState->instance_backup_subdir_path, base36enc(backup->start_time), - DATABASE_DIR, PG_XLOG_DIR); - - xlog_path = stream_xlog_path; } - else - xlog_path = instanceState->instance_wal_subdir_path; - backup->stop_lsn = stop_backup_lsn; backup->recovery_xid = stop_backup_result.snapshot_xid; elog(LOG, "Getting the Recovery Time from WAL"); diff --git a/src/checkdb.c b/src/checkdb.c index 4ea1d0800..5d7d6652b 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -455,7 +455,6 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, ind->heapallindexed_is_supported = heapallindexed_is_supported; ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); strcpy(ind->amcheck_nspname, amcheck_nspname); - pg_atomic_clear_flag(&ind->lock); if (index_list == NULL) index_list = parray_new(); @@ -463,6 +462,8 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, parray_append(index_list, ind); } + pfilearray_clear_locks(index_list); + PQclear(res); return index_list; diff --git a/src/dir.c b/src/dir.c index 86848d8d5..dfcddd7d0 100644 --- a/src/dir.c +++ b/src/dir.c @@ -222,6 +222,8 @@ pgFileInit(const char *rel_path) /* Number of blocks backed up during backup */ file->n_headers = 0; + // May be add? + // pg_atomic_clear_flag(file->lock); return file; } @@ -1859,3 +1861,17 @@ cleanup_tablespace(const char *path) parray_walk(files, pgFileFree); parray_free(files); } + +/* + * Clear the synchronisation locks in a parray of (pgFile *)'s + */ +void +pfilearray_clear_locks(parray *file_list) +{ + int i; + for (i = 0; i < parray_num(file_list); i++) + { + pgFile *file = (pgFile *) parray_get(file_list, i); + pg_atomic_clear_flag(&file->lock); + } +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1b2e7f751..3150900b6 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -68,8 +68,6 @@ static char *backup_path = NULL; static CatalogState *catalogState = NULL; /* ================ catalogState (END) =========== */ -/* colon separated external directories list ("/path1:/path2") */ -char *externaldir = NULL; /* common options */ int num_threads = 1; bool stream_wal = false; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index d02bbb033..a7979ed27 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -600,7 +600,6 @@ typedef struct int ret; } backup_files_arg; - typedef struct timelineInfo timelineInfo; /* struct to collect info about timelines in WAL archive */ @@ -679,10 +678,11 @@ typedef struct BackupPageHeader2 uint16 checksum; } BackupPageHeader2; -typedef struct StopBackupCallbackState { +typedef struct StopBackupCallbackParams +{ PGconn *conn; int server_version; -} StopBackupCallbackState; +} StopBackupCallbackParams; /* Special value for compressed_size field */ #define PageIsOk 0 @@ -1061,6 +1061,7 @@ extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); extern int pgCompareOid(const void *f1, const void *f2); +extern void pfilearray_clear_locks(parray *file_list); /* in data.c */ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, @@ -1259,4 +1260,42 @@ extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, XLogRecPtr startpos, TimeLineID starttli); extern int wait_WAL_streaming_end(parray *backup_files_list); + +/* external variables and functions, implemented in backup.c */ +typedef struct PGStopBackupResult +{ + /* + * We will use values of snapshot_xid and invocation_time if there are + * no transactions between start_lsn and stop_lsn. + */ + TransactionId snapshot_xid; + time_t invocation_time; + /* + * Fields that store pg_catalog.pg_stop_backup() result + */ + XLogRecPtr lsn; + size_t backup_label_content_len; + char *backup_label_content; + size_t tablespace_map_content_len; + char *tablespace_map_content; +} PGStopBackupResult; + +extern bool backup_in_progress; +extern parray *backup_files_list; + +extern void pg_start_backup(const char *label, bool smooth, pgBackup *backup, + PGNodeInfo *nodeInfo, PGconn *conn); +extern void pg_silent_client_messages(PGconn *conn); +extern void pg_create_restore_point(PGconn *conn, time_t backup_start_time); +extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); +extern void pg_stop_backup_consume(PGconn *conn, int server_version, + bool is_exclusive, uint32 timeout, const char *query_text, + PGStopBackupResult *result); +extern void pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, + const void *data, size_t len, parray *file_list); +extern XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, + bool in_prev_segment, bool segment_only, + int timeout_elevel, bool in_stream_dir); +extern void wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup); + #endif /* PG_PROBACKUP_H */ diff --git a/src/restore.c b/src/restore.c index 6aa4c5345..81ee9a6f7 100644 --- a/src/restore.c +++ b/src/restore.c @@ -824,7 +824,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } /* - * Setup directory structure for external directories and file locks + * Setup directory structure for external directories */ for (i = 0; i < parray_num(dest_files); i++) { @@ -848,11 +848,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(VERBOSE, "Create external directory \"%s\"", dirpath); fio_mkdir(dirpath, file->mode, FIO_DB_HOST); } - - /* setup threads */ - pg_atomic_clear_flag(&file->lock); } + /* setup threads */ + pfilearray_clear_locks(dest_files); + /* Get list of files in destination directory and remove redundant files */ if (params->incremental_mode != INCR_NONE || cleanup_pgdata) { diff --git a/src/utils/parray.c b/src/utils/parray.c index 31148ee9a..95b83365d 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -175,7 +175,7 @@ parray_rm(parray *array, const void *key, int(*compare)(const void *, const void size_t parray_num(const parray *array) { - return array->used; + return array!= NULL ? array->used : (size_t) 0; } void diff --git a/src/validate.c b/src/validate.c index f000698d0..4044ac158 100644 --- a/src/validate.c +++ b/src/validate.c @@ -130,11 +130,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) // params->partial_restore_type); /* setup threads */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - pg_atomic_clear_flag(&file->lock); - } + pfilearray_clear_locks(files); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); From 908a5ad65a61ec874b9a425d81308123b497db1f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 3 Jun 2021 18:28:22 +0300 Subject: [PATCH 1735/2107] introduce ptrack_parse_version_string() and new ptrack version numbering schema --- src/backup.c | 10 +++++----- src/data.c | 4 ++-- src/ptrack.c | 56 +++++++++++++++++++++++++++++++--------------------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/backup.c b/src/backup.c index 3815900b9..6f1aa867a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -219,7 +219,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); - if (nodeInfo->ptrack_version_num < 20) + if (nodeInfo->ptrack_version_num < 200) { // backward compatibility kludge: use Stop LSN for ptrack 1.x, if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) @@ -408,14 +408,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num >= 20) + if (nodeInfo->ptrack_version_num >= 200) make_pagemap_from_ptrack_2(backup_files_list, backup_conn, nodeInfo->ptrack_schema, nodeInfo->ptrack_version_num, prev_backup_start_lsn); - else if (nodeInfo->ptrack_version_num == 15 || - nodeInfo->ptrack_version_num == 16 || - nodeInfo->ptrack_version_num == 17) + else if (nodeInfo->ptrack_version_num == 105 || + nodeInfo->ptrack_version_num == 106 || + nodeInfo->ptrack_version_num == 107) make_pagemap_from_ptrack_1(backup_files_list, backup_conn); } diff --git a/src/data.c b/src/data.c index d70aae8fd..280ede5c8 100644 --- a/src/data.c +++ b/src/data.c @@ -300,7 +300,7 @@ prepare_page(ConnectionArgs *conn_arg, * Under high write load it's possible that we've read partly * flushed page, so try several times before throwing an error. */ - if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 20) + if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 200) { int rc = 0; while (!page_is_valid && try_again--) @@ -400,7 +400,7 @@ prepare_page(ConnectionArgs *conn_arg, * We do this only in the cases of PTRACK 1.x versions backup */ if (backup_mode == BACKUP_MODE_DIFF_PTRACK - && (ptrack_version_num >= 15 && ptrack_version_num < 20)) + && (ptrack_version_num >= 105 && ptrack_version_num < 200)) { int rc = 0; size_t page_size = 0; diff --git a/src/ptrack.c b/src/ptrack.c index dc1a2c74c..5a2b9f046 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -139,6 +139,23 @@ make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) } } +/* + * Parse a string like "2.1" into int + * result: int by formula major_number * 100 + minor_number + * or -1 if string cannot be parsed + */ +static int +ptrack_parse_version_string(const char *version_str) +{ + int ma, mi; + int sscanf_readed_count; + if (sscanf(version_str, "%u.%2u%n", &ma, &mi, &sscanf_readed_count) != 2) + return -1; + if (sscanf_readed_count != strlen(version_str)) + return -1; + return ma * 100 + mi; +} + /* Check if the instance supports compatible version of ptrack, * fill-in version number if it does. * Also for ptrack 2.x save schema namespace. @@ -148,6 +165,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) { PGresult *res_db; char *ptrack_version_str; + int ptrack_version_num; res_db = pgut_execute(backup_conn, "SELECT extnamespace::regnamespace, extversion " @@ -191,24 +209,16 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) ptrack_version_str = PQgetvalue(res_db, 0, 0); } - if (strcmp(ptrack_version_str, "1.5") == 0) - nodeInfo->ptrack_version_num = 15; - else if (strcmp(ptrack_version_str, "1.6") == 0) - nodeInfo->ptrack_version_num = 16; - else if (strcmp(ptrack_version_str, "1.7") == 0) - nodeInfo->ptrack_version_num = 17; - else if (strcmp(ptrack_version_str, "2.0") == 0) - nodeInfo->ptrack_version_num = 20; - else if (strcmp(ptrack_version_str, "2.1") == 0) - nodeInfo->ptrack_version_num = 21; - else if (strcmp(ptrack_version_str, "2.2") == 0) - nodeInfo->ptrack_version_num = 22; - else - elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s", + ptrack_version_num = ptrack_parse_version_string(ptrack_version_str); + if (ptrack_version_num == -1) + /* leave default nodeInfo->ptrack_version_num = 0 from pgNodeInit() */ + elog(WARNING, "Cannot parse ptrack version string \"%s\"", ptrack_version_str); + else + nodeInfo->ptrack_version_num = ptrack_version_num; /* ptrack 1.X is buggy, so fall back to DELTA backup strategy for safety */ - if (nodeInfo->ptrack_version_num >= 15 && nodeInfo->ptrack_version_num < 20) + if (nodeInfo->ptrack_version_num >= 105 && nodeInfo->ptrack_version_num < 200) { if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -231,12 +241,12 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) PGresult *res_db; bool result = false; - if (ptrack_version_num < 20) + if (ptrack_version_num < 200) { res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; } - else if (ptrack_version_num == 20) + else if (ptrack_version_num == 200) { res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; @@ -270,7 +280,7 @@ pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) char *params[2]; // FIXME Perform this check on caller's side - if (ptrack_version_num >= 20) + if (ptrack_version_num >= 200) return; params[0] = palloc(64); @@ -472,14 +482,14 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) uint32 lsn_lo; XLogRecPtr lsn; - if (nodeInfo->ptrack_version_num < 20) + if (nodeInfo->ptrack_version_num < 200) res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_control_lsn()", 0, NULL); else { char query[128]; - if (nodeInfo->ptrack_version_num == 20) + if (nodeInfo->ptrack_version_num == 200) sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); else sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); @@ -537,7 +547,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - if (ptrack_version_num < 20) + if (ptrack_version_num < 200) res = pgut_execute_parallel(arguments->conn, arguments->cancel_conn, "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", @@ -550,7 +560,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - if (ptrack_version_num == 20) + if (ptrack_version_num == 200) sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); else elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); @@ -614,7 +624,7 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - if (ptrack_version_num == 20) + if (ptrack_version_num == 200) sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", ptrack_schema); else From 0dcfb06ec7e8f397b32f65de8562c82aec458f60 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 7 Jun 2021 15:48:53 +0300 Subject: [PATCH 1736/2107] s:snprintf(..., MAXPGPATH, "%s/%s", ...):join_path_components(...):g --- src/catalog.c | 4 ++-- src/parsexlog.c | 2 +- src/restore.c | 13 ++++++------- src/util.c | 4 ++-- src/utils/pgut.c | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 981841747..3ea4d9bca 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -829,7 +829,7 @@ IsDir(const char *dirpath, const char *entry, fio_location location) char path[MAXPGPATH]; struct stat st; - snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + join_path_components(path, dirpath, entry); return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); } @@ -941,7 +941,7 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) join_path_components(data_path, backup_instance_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ - snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + join_path_components(backup_conf_path, data_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); if (!backup) diff --git a/src/parsexlog.c b/src/parsexlog.c index 4a0f38642..8dfb2c78c 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1017,7 +1017,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); - snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, xlogfname); + join_path_components(reader_data->xlogpath, wal_archivedir, xlogfname); snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath); /* We fall back to using .partial segment in case if we are running diff --git a/src/restore.c b/src/restore.c index 86317596e..9c0b059e9 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1480,7 +1480,7 @@ update_recovery_options_before_v12(pgBackup *backup, } elog(LOG, "update recovery settings in recovery.conf"); - snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); + join_path_components(path, instance_config.pgdata, "recovery.conf"); fp = fio_fopen(path, "w", FIO_DB_HOST); if (fp == NULL) @@ -1537,8 +1537,7 @@ update_recovery_options(pgBackup *backup, time2iso(current_time_str, lengthof(current_time_str), current_time, false); - snprintf(postgres_auto_path, lengthof(postgres_auto_path), - "%s/postgresql.auto.conf", instance_config.pgdata); + join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) { @@ -1648,7 +1647,7 @@ update_recovery_options(pgBackup *backup, if (params->recovery_settings_mode == PITR_REQUESTED) { elog(LOG, "creating recovery.signal file"); - snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); + join_path_components(path, instance_config.pgdata, "recovery.signal"); fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); if (fp == NULL) @@ -1664,7 +1663,7 @@ update_recovery_options(pgBackup *backup, if (params->restore_as_replica) { elog(LOG, "creating standby.signal file"); - snprintf(path, lengthof(path), "%s/standby.signal", instance_config.pgdata); + join_path_components(path, instance_config.pgdata, "standby.signal"); fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); if (fp == NULL) @@ -2160,7 +2159,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, { char pid_file[MAXPGPATH]; - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); + join_path_components(pid_file, pgdata, "postmaster.pid"); elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", pid_file); success = false; @@ -2201,7 +2200,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ if (incremental_mode == INCR_LSN) { - snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata); + join_path_components(backup_label, pgdata, "backup_label"); if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " diff --git a/src/util.c b/src/util.c index 87ec36713..8fcec6189 100644 --- a/src/util.c +++ b/src/util.c @@ -418,7 +418,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, FIN_CRC32C(ControlFile.crc); /* overwrite pg_control */ - snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); + join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE); writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST); /* Update pg_control checksum in backup_list */ @@ -569,7 +569,7 @@ check_postmaster(const char *pgdata) pid_t pid; char pid_file[MAXPGPATH]; - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); + join_path_components(pid_file, pgdata, "postmaster.pid"); fp = fopen(pid_file, "r"); if (fp == NULL) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index eba31faa6..e1e52b24b 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1184,7 +1184,7 @@ pgut_rmtree(const char *path, bool rmtopdir, bool strict) /* now we have the names we can start removing things */ for (filename = filenames; *filename; filename++) { - snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename); + join_path_components(pathbuf, path, *filename); if (lstat(pathbuf, &statbuf) != 0) { From 477e5bcb4fcb0ea74cfb49d865e21e4c7bff5f0a Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Sat, 12 Jun 2021 20:21:14 +0300 Subject: [PATCH 1737/2107] Some minor fixes (#397) * Reformat fio_*() definitions for easier grep'ping * typo * move check_postmaster() into src/utils/file.c (from src/util.c), rename it to local_check_postmaster() and make it static/private to src/utils/file.c --- src/data.c | 2 +- src/pg_probackup.h | 2 - src/util.c | 48 --------- src/utils/file.c | 238 ++++++++++++++++++++++++++++++++------------- src/utils/file.h | 1 - 5 files changed, 174 insertions(+), 117 deletions(-) diff --git a/src/data.c b/src/data.c index 280ede5c8..60986fd5c 100644 --- a/src/data.c +++ b/src/data.c @@ -1516,7 +1516,7 @@ validate_one_page(Page page, BlockNumber absolute_blkno, } /* - * Valiate pages of datafile in PGDATA one by one. + * Validate pages of datafile in PGDATA one by one. * * returns true if the file is valid * also returns true if the file was not found diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 746b0f5a5..fca08bdac 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1071,8 +1071,6 @@ extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); -extern pid_t check_postmaster(const char *pgdata); - extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); diff --git a/src/util.c b/src/util.c index 8fcec6189..9fd0114bb 100644 --- a/src/util.c +++ b/src/util.c @@ -556,51 +556,3 @@ datapagemap_print_debug(datapagemap_t *map) pg_free(iter); } - -/* - * Return pid of postmaster process running in given pgdata. - * Return 0 if there is none. - * Return 1 if postmaster.pid is mangled. - */ -pid_t -check_postmaster(const char *pgdata) -{ - FILE *fp; - pid_t pid; - char pid_file[MAXPGPATH]; - - join_path_components(pid_file, pgdata, "postmaster.pid"); - - fp = fopen(pid_file, "r"); - if (fp == NULL) - { - /* No pid file, acceptable*/ - if (errno == ENOENT) - return 0; - else - elog(ERROR, "Cannot open file \"%s\": %s", - pid_file, strerror(errno)); - } - - if (fscanf(fp, "%i", &pid) != 1) - { - /* something is wrong with the file content */ - pid = 1; - } - - if (pid > 1) - { - if (kill(pid, 0) != 0) - { - /* process no longer exists */ - if (errno == ESRCH) - pid = 0; - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - pid, strerror(errno)); - } - } - - fclose(fp); - return pid; -} diff --git a/src/utils/file.c b/src/utils/file.c index 15a7085ec..6dcf2288e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -30,7 +30,6 @@ typedef struct int path_len; } fio_send_request; - typedef struct { char path[MAXPGPATH]; @@ -85,14 +84,16 @@ typedef struct #endif /* Use specified file descriptors as stdin/stdout for FIO functions */ -void fio_redirect(int in, int out, int err) +void +fio_redirect(int in, int out, int err) { fio_stdin = in; fio_stdout = out; fio_stderr = err; } -void fio_error(int rc, int size, char const* file, int line) +void +fio_error(int rc, int size, char const* file, int line) { if (remote_agent) { @@ -115,7 +116,8 @@ void fio_error(int rc, int size, char const* file, int line) } /* Check if file descriptor is local or remote (created by FIO) */ -static bool fio_is_remote_fd(int fd) +static bool +fio_is_remote_fd(int fd) { return (fd & FIO_PIPE_MARKER) != 0; } @@ -157,14 +159,17 @@ fio_safestat(const char *path, struct stat *buf) #define stat(x, y) fio_safestat(x, y) /* TODO: use real pread on Linux */ -static ssize_t pread(int fd, void* buf, size_t size, off_t off) +static ssize_t +pread(int fd, void* buf, size_t size, off_t off) { off_t rc = lseek(fd, off, SEEK_SET); if (rc != off) return -1; return read(fd, buf, size); } -static int remove_file_or_dir(char const* path) + +static int +remove_file_or_dir(char const* path) { int rc = remove(path); #ifdef WIN32 @@ -178,7 +183,8 @@ static int remove_file_or_dir(char const* path) #endif /* Check if specified location is local for current node */ -bool fio_is_remote(fio_location location) +bool +fio_is_remote(fio_location location) { bool is_remote = MyLocation != FIO_LOCAL_HOST && location != FIO_LOCAL_HOST @@ -189,7 +195,8 @@ bool fio_is_remote(fio_location location) } /* Check if specified location is local for current node */ -bool fio_is_remote_simple(fio_location location) +bool +fio_is_remote_simple(fio_location location) { bool is_remote = MyLocation != FIO_LOCAL_HOST && location != FIO_LOCAL_HOST @@ -198,7 +205,8 @@ bool fio_is_remote_simple(fio_location location) } /* Try to read specified amount of bytes unless error or EOF are encountered */ -static ssize_t fio_read_all(int fd, void* buf, size_t size) +static ssize_t +fio_read_all(int fd, void* buf, size_t size) { size_t offs = 0; while (offs < size) @@ -220,7 +228,8 @@ static ssize_t fio_read_all(int fd, void* buf, size_t size) } /* Try to write specified amount of bytes unless error is encountered */ -static ssize_t fio_write_all(int fd, void const* buf, size_t size) +static ssize_t +fio_write_all(int fd, void const* buf, size_t size) { size_t offs = 0; while (offs < size) @@ -241,7 +250,8 @@ static ssize_t fio_write_all(int fd, void const* buf, size_t size) } /* Get version of remote agent */ -int fio_get_agent_version(void) +int +fio_get_agent_version(void) { fio_header hdr; hdr.cop = FIO_AGENT_VERSION; @@ -254,7 +264,8 @@ int fio_get_agent_version(void) } /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ -FILE* fio_open_stream(char const* path, fio_location location) +FILE* +fio_open_stream(char const* path, fio_location location) { FILE* f; if (fio_is_remote(location)) @@ -294,7 +305,8 @@ FILE* fio_open_stream(char const* path, fio_location location) } /* Close input stream */ -int fio_close_stream(FILE* f) +int +fio_close_stream(FILE* f) { if (fio_stdin_buffer) { @@ -305,7 +317,8 @@ int fio_close_stream(FILE* f) } /* Open directory */ -DIR* fio_opendir(char const* path, fio_location location) +DIR* +fio_opendir(char const* path, fio_location location) { DIR* dir; if (fio_is_remote(location)) @@ -346,7 +359,8 @@ DIR* fio_opendir(char const* path, fio_location location) } /* Get next directory entry */ -struct dirent* fio_readdir(DIR *dir) +struct dirent* +fio_readdir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) { @@ -374,7 +388,8 @@ struct dirent* fio_readdir(DIR *dir) } /* Close directory */ -int fio_closedir(DIR *dir) +int +fio_closedir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) { @@ -394,7 +409,8 @@ int fio_closedir(DIR *dir) } /* Open file */ -int fio_open(char const* path, int mode, fio_location location) +int +fio_open(char const* path, int mode, fio_location location) { int fd; if (fio_is_remote(location)) @@ -461,7 +477,8 @@ fio_disconnect(void) } /* Open stdio file */ -FILE* fio_fopen(char const* path, char const* mode, fio_location location) +FILE* +fio_fopen(char const* path, char const* mode, fio_location location) { FILE *f = NULL; @@ -506,7 +523,8 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) } /* Format output to file stream */ -int fio_fprintf(FILE* f, char const* format, ...) +int +fio_fprintf(FILE* f, char const* format, ...) { int rc; va_list args; @@ -532,7 +550,8 @@ int fio_fprintf(FILE* f, char const* format, ...) } /* Flush stream data (does nothing for remote file) */ -int fio_fflush(FILE* f) +int +fio_fflush(FILE* f) { int rc = 0; if (!fio_is_remote_file(f)) @@ -541,13 +560,15 @@ int fio_fflush(FILE* f) } /* Sync file to the disk (does nothing for remote file) */ -int fio_flush(int fd) +int +fio_flush(int fd) { return fio_is_remote_fd(fd) ? 0 : fsync(fd); } /* Close output stream */ -int fio_fclose(FILE* f) +int +fio_fclose(FILE* f) { return fio_is_remote_file(f) ? fio_close(fio_fileno(f)) @@ -555,7 +576,8 @@ int fio_fclose(FILE* f) } /* Close file */ -int fio_close(int fd) +int +fio_close(int fd) { if (fio_is_remote_fd(fd)) { @@ -578,7 +600,8 @@ int fio_close(int fd) } /* Truncate stdio file */ -int fio_ftruncate(FILE* f, off_t size) +int +fio_ftruncate(FILE* f, off_t size) { return fio_is_remote_file(f) ? fio_truncate(fio_fileno(f), size) @@ -588,7 +611,8 @@ int fio_ftruncate(FILE* f, off_t size) /* Truncate file * TODO: make it synchronous */ -int fio_truncate(int fd, off_t size) +int +fio_truncate(int fd, off_t size) { if (fio_is_remote_fd(fd)) { @@ -613,7 +637,8 @@ int fio_truncate(int fd, off_t size) /* * Read file from specified location. */ -int fio_pread(FILE* f, void* buf, off_t offs) +int +fio_pread(FILE* f, void* buf, off_t offs) { if (fio_is_remote_file(f)) { @@ -649,7 +674,8 @@ int fio_pread(FILE* f, void* buf, off_t offs) } /* Set position in stdio file */ -int fio_fseek(FILE* f, off_t offs) +int +fio_fseek(FILE* f, off_t offs) { return fio_is_remote_file(f) ? fio_seek(fio_fileno(f), offs) @@ -658,7 +684,8 @@ int fio_fseek(FILE* f, off_t offs) /* Set position in file */ /* TODO: make it synchronous or check async error */ -int fio_seek(int fd, off_t offs) +int +fio_seek(int fd, off_t offs) { if (fio_is_remote_fd(fd)) { @@ -699,7 +726,8 @@ fio_seek_impl(int fd, off_t offs) } /* Write data to stdio file */ -size_t fio_fwrite(FILE* f, void const* buf, size_t size) +size_t +fio_fwrite(FILE* f, void const* buf, size_t size) { if (fio_is_remote_file(f)) return fio_write(fio_fileno(f), buf, size); @@ -708,7 +736,8 @@ size_t fio_fwrite(FILE* f, void const* buf, size_t size) } /* Write data to the file synchronously */ -ssize_t fio_write(int fd, void const* buf, size_t size) +ssize_t +fio_write(int fd, void const* buf, size_t size) { if (fio_is_remote_fd(fd)) { @@ -759,7 +788,8 @@ fio_write_impl(int fd, void const* buf, size_t size, int out) return; } -size_t fio_fwrite_async(FILE* f, void const* buf, size_t size) +size_t +fio_fwrite_async(FILE* f, void const* buf, size_t size) { return fio_is_remote_file(f) ? fio_write_async(fio_fileno(f), buf, size) @@ -768,7 +798,8 @@ size_t fio_fwrite_async(FILE* f, void const* buf, size_t size) /* Write data to the file */ /* TODO: support async report error */ -ssize_t fio_write_async(int fd, void const* buf, size_t size) +ssize_t +fio_write_async(int fd, void const* buf, size_t size) { if (size == 0) return 0; @@ -836,7 +867,8 @@ fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char * } /* Write data to the file */ -ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) +ssize_t +fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) { if (fio_is_remote_file(f)) { @@ -976,7 +1008,8 @@ fio_get_async_error_impl(int out) } /* Read data from stdio file */ -ssize_t fio_fread(FILE* f, void* buf, size_t size) +ssize_t +fio_fread(FILE* f, void* buf, size_t size) { size_t rc; if (fio_is_remote_file(f)) @@ -986,7 +1019,8 @@ ssize_t fio_fread(FILE* f, void* buf, size_t size) } /* Read data from file */ -ssize_t fio_read(int fd, void* buf, size_t size) +ssize_t +fio_read(int fd, void* buf, size_t size) { if (fio_is_remote_fd(fd)) { @@ -1012,7 +1046,8 @@ ssize_t fio_read(int fd, void* buf, size_t size) } /* Get information about file */ -int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) +int +fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) { if (fio_is_remote(location)) { @@ -1045,7 +1080,8 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_locatio } /* Check presence of the file */ -int fio_access(char const* path, int mode, fio_location location) +int +fio_access(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -1076,7 +1112,8 @@ int fio_access(char const* path, int mode, fio_location location) } /* Create symbolic link */ -int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) +int +fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) { if (fio_is_remote(location)) { @@ -1103,7 +1140,8 @@ int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_l } } -static void fio_symlink_impl(int out, char *buf, bool overwrite) +static void +fio_symlink_impl(int out, char *buf, bool overwrite) { char *linked_path = buf; char *link_path = buf + strlen(buf) + 1; @@ -1117,7 +1155,8 @@ static void fio_symlink_impl(int out, char *buf, bool overwrite) } /* Rename file */ -int fio_rename(char const* old_path, char const* new_path, fio_location location) +int +fio_rename(char const* old_path, char const* new_path, fio_location location) { if (fio_is_remote(location)) { @@ -1143,7 +1182,8 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location } /* Sync file to disk */ -int fio_sync(char const* path, fio_location location) +int +fio_sync(char const* path, fio_location location) { if (fio_is_remote(location)) { @@ -1185,7 +1225,8 @@ int fio_sync(char const* path, fio_location location) } /* Get crc32 of file */ -pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress) +pg_crc32 +fio_get_crc32(const char *file_path, fio_location location, bool decompress) { if (fio_is_remote(location)) { @@ -1216,7 +1257,8 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decomp } /* Remove file */ -int fio_unlink(char const* path, fio_location location) +int +fio_unlink(char const* path, fio_location location) { if (fio_is_remote(location)) { @@ -1241,7 +1283,8 @@ int fio_unlink(char const* path, fio_location location) /* Create directory * TODO: add strict flag */ -int fio_mkdir(char const* path, int mode, fio_location location) +int +fio_mkdir(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -1267,7 +1310,8 @@ int fio_mkdir(char const* path, int mode, fio_location location) } /* Change file mode */ -int fio_chmod(char const* path, int mode, fio_location location) +int +fio_chmod(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -1562,7 +1606,8 @@ fio_gzclose(gzFile f) } } -int fio_gzeof(gzFile f) +int +fio_gzeof(gzFile f) { if ((size_t)f & FIO_GZ_REMOTE_MARKER) { @@ -1575,7 +1620,8 @@ int fio_gzeof(gzFile f) } } -const char* fio_gzerror(gzFile f, int *errnum) +const char* +fio_gzerror(gzFile f, int *errnum) { if ((size_t)f & FIO_GZ_REMOTE_MARKER) { @@ -1590,7 +1636,8 @@ const char* fio_gzerror(gzFile f, int *errnum) } } -z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) +z_off_t +fio_gzseek(gzFile f, z_off_t offset, int whence) { Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); return gzseek(f, offset, whence); @@ -1602,7 +1649,8 @@ z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) /* Send file content * Note: it should not be used for large files. */ -static void fio_load_file(int out, char const* path) +static void +fio_load_file(int out, char const* path) { int fd = open(path, O_RDONLY); fio_header hdr; @@ -1644,7 +1692,8 @@ static void fio_load_file(int out, char const* path) * In case of DELTA mode horizonLsn must be a valid lsn, * otherwise it should be set to InvalidXLogRecPtr. */ -int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, +int +fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers) @@ -1804,7 +1853,8 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f * FIO_SEND_FILE_CORRUPTION * FIO_SEND_FILE_EOF */ -static void fio_send_pages_impl(int out, char* buf) +static void +fio_send_pages_impl(int out, char* buf) { FILE *in = NULL; BlockNumber blknum = 0; @@ -2074,7 +2124,8 @@ static void fio_send_pages_impl(int out, char* buf) * ZLIB_ERROR (-5) * REMOTE_ERROR (-6) */ -int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg) +int +fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; @@ -2234,7 +2285,8 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o * OPEN_FAILED and READ_FAIL should also set errormsg. * If pgFile is not NULL then we must calculate crc and read_size for it. */ -int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, +int +fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, pgFile *file, char **errormsg) { fio_header hdr; @@ -2315,7 +2367,8 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, * FIO_SEND_FILE_EOF * */ -static void fio_send_file_impl(int out, char const* path) +static void +fio_send_file_impl(int out, char const* path) { FILE *fp; fio_header hdr; @@ -2406,7 +2459,8 @@ static void fio_send_file_impl(int out, char const* path) } /* Compile the array of files located on remote machine in directory root */ -static void fio_list_dir_internal(parray *files, const char *root, bool exclude, +static void +fio_list_dir_internal(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num) { @@ -2499,7 +2553,8 @@ static void fio_list_dir_internal(parray *files, const char *root, bool exclude, * * TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages */ -static void fio_list_dir_impl(int out, char* buf) +static void +fio_list_dir_impl(int out, char* buf) { int i; fio_header hdr; @@ -2565,7 +2620,8 @@ static void fio_list_dir_impl(int out, char* buf) } /* Wrapper for directory listing */ -void fio_list_dir(parray *files, const char *root, bool exclude, +void +fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num) { @@ -2620,7 +2676,8 @@ fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks } } -static void fio_get_checksum_map_impl(int out, char *buf) +static void +fio_get_checksum_map_impl(int out, char *buf) { fio_header hdr; PageState *checksum_map = NULL; @@ -2687,7 +2744,8 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } -static void fio_get_lsn_map_impl(int out, char *buf) +static void +fio_get_lsn_map_impl(int out, char *buf) { fio_header hdr; datapagemap_t *lsn_map = NULL; @@ -2713,11 +2771,60 @@ static void fio_get_lsn_map_impl(int out, char *buf) } } +/* + * Return pid of postmaster process running in given pgdata on local machine. + * Return 0 if there is none. + * Return 1 if postmaster.pid is mangled. + */ +static pid_t +local_check_postmaster(const char *pgdata) +{ + FILE *fp; + pid_t pid; + char pid_file[MAXPGPATH]; + + join_path_components(pid_file, pgdata, "postmaster.pid"); + + fp = fopen(pid_file, "r"); + if (fp == NULL) + { + /* No pid file, acceptable*/ + if (errno == ENOENT) + return 0; + else + elog(ERROR, "Cannot open file \"%s\": %s", + pid_file, strerror(errno)); + } + + if (fscanf(fp, "%i", &pid) != 1) + { + /* something is wrong with the file content */ + pid = 1; + } + + if (pid > 1) + { + if (kill(pid, 0) != 0) + { + /* process no longer exists */ + if (errno == ESRCH) + pid = 0; + else + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + pid, strerror(errno)); + } + } + + fclose(fp); + return pid; +} + /* * Go to the remote host and get postmaster pid from file postmaster.pid * and check that process is running, if process is running, return its pid number. */ -pid_t fio_check_postmaster(const char *pgdata, fio_location location) +pid_t +fio_check_postmaster(const char *pgdata, fio_location location) { if (fio_is_remote(location)) { @@ -2734,16 +2841,17 @@ pid_t fio_check_postmaster(const char *pgdata, fio_location location) return hdr.arg; } else - return check_postmaster(pgdata); + return local_check_postmaster(pgdata); } -static void fio_check_postmaster_impl(int out, char *buf) +static void +fio_check_postmaster_impl(int out, char *buf) { fio_header hdr; pid_t postmaster_pid; char *pgdata = (char*) buf; - postmaster_pid = check_postmaster(pgdata); + postmaster_pid = local_check_postmaster(pgdata); /* send arrays of checksums to main process */ hdr.arg = postmaster_pid; @@ -2782,7 +2890,8 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ -void fio_communicate(int in, int out) +void +fio_communicate(int in, int out) { /* * Map of file and directory descriptors. @@ -2990,4 +3099,3 @@ void fio_communicate(int in, int out) exit(EXIT_FAILURE); } } - diff --git a/src/utils/file.h b/src/utils/file.h index 1eafe543d..ad65b9901 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -145,4 +145,3 @@ extern const char* fio_gzerror(gzFile file, int *errnum); #endif #endif - From 8ae217bae5ab6a6b02943f3279f87f914d685acf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Jun 2021 13:34:05 +0300 Subject: [PATCH 1738/2107] [Issue #394] correctly detect ENOSPC when using write(): durable_write is implemented --- src/archive.c | 1 + src/utils/file.c | 42 ++++++++++++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/archive.c b/src/archive.c index 2d858a64c..6ac1062b8 100644 --- a/src/archive.c +++ b/src/archive.c @@ -568,6 +568,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d } /* copy content */ + errno = 0; for (;;) { size_t read_len = 0; diff --git a/src/utils/file.c b/src/utils/file.c index 6dcf2288e..19e71fab2 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -796,6 +796,34 @@ fio_fwrite_async(FILE* f, void const* buf, size_t size) : fwrite(buf, 1, size, f); } +/* + * Write buffer to descriptor by calling write(), + * If size of written data is less than buffer size, + * then try to write what is left. + * We do this to get honest errno if there are some problems + * with filesystem, since writing less than buffer size + * is not considered an error. + */ +static ssize_t +durable_write(int fd, const char* buf, size_t size) +{ + off_t current_pos = 0; + size_t bytes_left = size; + + while (bytes_left > 0) + { + int rc = write(fd, buf + current_pos, bytes_left); + + if (rc <= 0) + return rc; + + bytes_left -= rc; + current_pos += rc; + } + + return size; +} + /* Write data to the file */ /* TODO: support async report error */ ssize_t @@ -814,27 +842,21 @@ fio_write_async(int fd, void const* buf, size_t size) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - - return size; } else - { - return write(fd, buf, size); - } + return durable_write(fd, buf, size); + + return size; } static void fio_write_async_impl(int fd, void const* buf, size_t size, int out) { - int rc; - /* Quick exit if agent is tainted */ if (async_errormsg) return; - rc = write(fd, buf, size); - - if (rc <= 0) + if (durable_write(fd, buf, size) <= 0) { async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); From da2cbcb2e789154f33cea3e409bd41b454a28566 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Jun 2021 13:59:49 +0300 Subject: [PATCH 1739/2107] [Issue #394] fio_close is now synchronous --- src/utils/file.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index ed72d6dc0..f341b7a37 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -607,7 +607,15 @@ fio_close(int fd) fio_fdset &= ~(1 << hdr.handle); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - /* Note, that file is closed without waiting for confirmation */ + + /* Wait for response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } return 0; } @@ -617,6 +625,22 @@ fio_close(int fd) } } +/* Close remote file implementation */ +static void +fio_close_impl(int fd, int out) +{ + fio_header hdr; + + hdr.cop = FIO_CLOSE; + hdr.arg = 0; + + if (close(fd) != 0) + hdr.arg = errno; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Truncate stdio file */ int fio_ftruncate(FILE* f, off_t size) @@ -3000,7 +3024,7 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CLOSE: /* Close file */ - SYS_CHECK(close(fd[hdr.handle])); + fio_close_impl(fd[hdr.handle], out); break; case FIO_WRITE: /* Write to the current position in file */ // IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); From d1106e5571c8f2e13d03759fc32fa774467bc8bb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 14 Jun 2021 14:16:07 +0300 Subject: [PATCH 1740/2107] [Issue #394] use durable_write in fio_write --- src/utils/file.c | 81 +++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 19e71fab2..ef322997f 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -735,6 +735,34 @@ fio_fwrite(FILE* f, void const* buf, size_t size) return fwrite(buf, 1, size, f); } +/* + * Write buffer to descriptor by calling write(), + * If size of written data is less than buffer size, + * then try to write what is left. + * We do this to get honest errno if there are some problems + * with filesystem, since writing less than buffer size + * is not considered an error. + */ +static ssize_t +durable_write(int fd, const char* buf, size_t size) +{ + off_t current_pos = 0; + size_t bytes_left = size; + + while (bytes_left > 0) + { + int rc = write(fd, buf + current_pos, bytes_left); + + if (rc <= 0) + return rc; + + bytes_left -= rc; + current_pos += rc; + } + + return size; +} + /* Write data to the file synchronously */ ssize_t fio_write(int fd, void const* buf, size_t size) @@ -764,7 +792,7 @@ fio_write(int fd, void const* buf, size_t size) } else { - return write(fd, buf, size); + return durable_write(fd, buf, size); } } @@ -774,7 +802,7 @@ fio_write_impl(int fd, void const* buf, size_t size, int out) int rc; fio_header hdr; - rc = write(fd, buf, size); + rc = durable_write(fd, buf, size); hdr.arg = 0; hdr.size = 0; @@ -796,34 +824,6 @@ fio_fwrite_async(FILE* f, void const* buf, size_t size) : fwrite(buf, 1, size, f); } -/* - * Write buffer to descriptor by calling write(), - * If size of written data is less than buffer size, - * then try to write what is left. - * We do this to get honest errno if there are some problems - * with filesystem, since writing less than buffer size - * is not considered an error. - */ -static ssize_t -durable_write(int fd, const char* buf, size_t size) -{ - off_t current_pos = 0; - size_t bytes_left = size; - - while (bytes_left > 0) - { - int rc = write(fd, buf + current_pos, bytes_left); - - if (rc <= 0) - return rc; - - bytes_left -= rc; - current_pos += rc; - } - - return size; -} - /* Write data to the file */ /* TODO: support async report error */ ssize_t @@ -908,23 +908,22 @@ fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_ } else { - char uncompressed_buf[BLCKSZ]; char *errormsg = NULL; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &errormsg); + char decompressed_buf[BLCKSZ]; + int32 decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &errormsg); - if (uncompressed_size < 0) + if (decompressed_size < 0) elog(ERROR, "%s", errormsg); - return fwrite(uncompressed_buf, 1, uncompressed_size, f); + return fwrite(decompressed_buf, 1, decompressed_size, f); } } static void fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) { - int rc; - int32 uncompressed_size; - char uncompressed_buf[BLCKSZ]; + int32 decompressed_size; + char decompressed_buf[BLCKSZ]; /* If the previous command already have failed, * then there is no point in bashing a head against the wall @@ -933,14 +932,12 @@ fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg return; /* decompress chunk */ - uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg, &async_errormsg); + decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &async_errormsg); - if (uncompressed_size < 0) + if (decompressed_size < 0) return; - rc = write(fd, uncompressed_buf, uncompressed_size); - - if (rc <= 0) + if (durable_write(fd, decompressed_buf, decompressed_size) <= 0) { async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); From 6e8e948fd6d784b58b90f4f05bdb9eccee7b104e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 02:32:36 +0300 Subject: [PATCH 1741/2107] [Issue #313] added test coverage --- tests/helpers/ptrack_helpers.py | 3 ++ tests/restore.py | 66 ++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 3caba25df..b6dc6d028 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1950,6 +1950,9 @@ def stopped_in_breakpoint(self): return True return False + def quit(self): + self.proc.terminate() + # use for breakpoint, run, continue def _execute(self, cmd, running=True): output = [] diff --git a/tests/restore.py b/tests/restore.py index 61aae9285..8ccffa44c 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -9,7 +9,8 @@ import shutil import json from shutil import copyfile -from testgres import QueryException +from testgres import QueryException, StartNodeException +from stat import S_ISDIR module_name = 'restore' @@ -3856,3 +3857,66 @@ def test_concurrent_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_issue_313(self): + """ + Check that partially restored PostgreSQL instance cannot be started + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + node.cleanup() + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: + count += 1 + + node_restored = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node_restored')) + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 2) + gdb.quit() + + # emulate the user or HA taking care of PG configuration + for fname in os.listdir(node_restored.data_dir): + if fname.endswith('.conf'): + os.rename( + os.path.join(node_restored.data_dir, fname), + os.path.join(node.data_dir, fname)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) From cce59bc824262656b4ad1396465dcfd40a56c3de Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 17 Jun 2021 19:11:36 +0300 Subject: [PATCH 1742/2107] Remove ptrack-1.* support (#398) * Remove ptrack-1.* support * [PR #398] review feedback * [PR #398] remove conn_arg from backup_files_arg Co-authored-by: Grigory Smolkin --- doc/pgprobackup.xml | 30 +-- src/backup.c | 71 ++---- src/data.c | 239 +++++++----------- src/dir.c | 10 - src/merge.c | 2 +- src/pg_probackup.h | 27 +- src/ptrack.c | 448 +-------------------------------- src/utils/file.c | 3 - tests/auth_test.py | 4 +- tests/backup.py | 1 + tests/false_positive.py | 186 -------------- tests/ptrack.py | 536 +++++++++++++++++----------------------- tests/restore.py | 2 +- 13 files changed, 352 insertions(+), 1207 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2bf197814..b1ddd0032 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1162,7 +1162,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - PTRACK versions lower than 2.0 are deprecated. Postgres Pro Standard and Postgres Pro Enterprise + PTRACK versions lower than 2.0 are deprecated and not supported. Postgres Pro Standard and Postgres Pro Enterprise versions starting with 11.9.1 contain PTRACK 2.0. Upgrade your server to avoid issues in backups that you will take in future and be sure to take fresh backups of your clusters with the upgraded PTRACK since the backups taken with PTRACK 1.x might be corrupt. @@ -1218,34 +1218,6 @@ CREATE EXTENSION ptrack; - - For older PostgreSQL versions, - PTRACK required taking backups in the exclusive mode - to provide exclusive access to bitmaps with changed blocks. - To set up PTRACK backups for PostgreSQL 10 - or lower, do the following: - - - - - Set the ptrack_enable parameter to - on. - - - - - Grant the right to execute PTRACK - functions to the backup role - in every database of the - cluster: - - -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; - - - - diff --git a/src/backup.c b/src/backup.c index 46e4f1ea7..738b6dcf2 100644 --- a/src/backup.c +++ b/src/backup.c @@ -125,10 +125,6 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, check_external_for_tablespaces(external_dirs, backup_conn); } - /* Clear ptrack files for not PTRACK backups */ - if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable) - pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); - /* notify start of backup to PostgreSQL server */ time2iso(label, lengthof(label), current.start_time, false); strncat(label, " with pg_probackup", lengthof(label) - @@ -217,29 +213,14 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); - if (nodeInfo->ptrack_version_num < 200) + // new ptrack (>=2.0) is more robust and checks Start LSN + if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) { - // backward compatibility kludge: use Stop LSN for ptrack 1.x, - if (ptrack_lsn > prev_backup->stop_lsn || ptrack_lsn == InvalidXLogRecPtr) - { - elog(ERROR, "LSN from ptrack_control %X/%X differs from Stop LSN of previous backup %X/%X.\n" - "Create new full backup before an incremental one.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (prev_backup->stop_lsn >> 32), - (uint32) (prev_backup->stop_lsn)); - } - } - else - { - // new ptrack is more robust and checks Start LSN - if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) - { - elog(ERROR, "LSN from ptrack_control %X/%X is greater than Start LSN of previous backup %X/%X.\n" - "Create new full backup before an incremental one.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (prev_backup->start_lsn >> 32), - (uint32) (prev_backup->start_lsn)); - } + elog(ERROR, "LSN from ptrack_control %X/%X is greater than Start LSN of previous backup %X/%X.\n" + "Create new full backup before an incremental one.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (prev_backup->start_lsn >> 32), + (uint32) (prev_backup->start_lsn)); } } @@ -407,15 +388,10 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num >= 200) - make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - nodeInfo->ptrack_version_num, - prev_backup_start_lsn); - else if (nodeInfo->ptrack_version_num == 105 || - nodeInfo->ptrack_version_num == 106 || - nodeInfo->ptrack_version_num == 107) - make_pagemap_from_ptrack_1(backup_files_list, backup_conn); + make_pagemap_from_ptrack_2(backup_files_list, backup_conn, + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); } time(&end_time); @@ -490,8 +466,6 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; - arg->conn_arg.conn = NULL; - arg->conn_arg.cancel_conn = NULL; arg->hdr_map = &(current.hdr_map); arg->thread_num = i+1; /* By default there are some error */ @@ -816,6 +790,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { + /* ptrack_version_num < 2.0 was already checked in get_ptrack_version() */ if (nodeInfo.ptrack_version_num == 0) elog(ERROR, "This PostgreSQL instance does not support ptrack"); else @@ -2085,15 +2060,15 @@ backup_files(void *arg) /* backup file */ if (file->is_datafile && !file->is_cfs) { - backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, - arguments->prev_start_lsn, - current.backup_mode, - instance_config.compress_alg, - instance_config.compress_level, - arguments->nodeInfo->checksum_version, - arguments->nodeInfo->ptrack_version_num, - arguments->nodeInfo->ptrack_schema, - arguments->hdr_map, false); + backup_data_file(file, from_fullpath, to_fullpath, + arguments->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arguments->nodeInfo->checksum_version, + arguments->nodeInfo->ptrack_version_num, + arguments->nodeInfo->ptrack_schema, + arguments->hdr_map, false); } else { @@ -2117,10 +2092,6 @@ backup_files(void *arg) /* ssh connection to longer needed */ fio_disconnect(); - /* Close connection */ - if (arguments->conn_arg.conn) - pgut_disconnect(arguments->conn_arg.conn); - /* Data files transferring is successful */ arguments->ret = 0; diff --git a/src/data.c b/src/data.c index 9d8bfc584..314490585 100644 --- a/src/data.c +++ b/src/data.c @@ -276,8 +276,7 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) * return it to the caller */ static int32 -prepare_page(ConnectionArgs *conn_arg, - pgFile *file, XLogRecPtr prev_backup_start_lsn, +prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, FILE *in, BackupMode backup_mode, Page page, bool strict, @@ -290,6 +289,7 @@ prepare_page(ConnectionArgs *conn_arg, int try_again = PAGE_READ_ATTEMPTS; bool page_is_valid = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + int rc = 0; /* check for interrupt */ if (interrupted || thread_interrupted) @@ -300,161 +300,97 @@ prepare_page(ConnectionArgs *conn_arg, * Under high write load it's possible that we've read partly * flushed page, so try several times before throwing an error. */ - if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 200) + while (!page_is_valid && try_again--) { - int rc = 0; - while (!page_is_valid && try_again--) - { - /* read the block */ - int read_len = fio_pread(in, page, blknum * BLCKSZ); + /* read the block */ + int read_len = fio_pread(in, page, blknum * BLCKSZ); - /* The block could have been truncated. It is fine. */ - if (read_len == 0) - { - elog(VERBOSE, "Cannot read block %u of \"%s\": " - "block truncated", blknum, from_fullpath); - return PageIsTruncated; - } - else if (read_len < 0) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - else if (read_len != BLCKSZ) - elog(WARNING, "Cannot read block %u of \"%s\": " - "read %i of %d, try again", - blknum, from_fullpath, read_len, BLCKSZ); - else + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(VERBOSE, "Cannot read block %u of \"%s\": " + "block truncated", blknum, from_fullpath); + return PageIsTruncated; + } + else if (read_len < 0) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + blknum, from_fullpath, strerror(errno)); + else if (read_len != BLCKSZ) + elog(WARNING, "Cannot read block %u of \"%s\": " + "read %i of %d, try again", + blknum, from_fullpath, read_len, BLCKSZ); + else + { + /* We have BLCKSZ of raw data, validate it */ + rc = validate_one_page(page, absolute_blknum, + InvalidXLogRecPtr, page_st, + checksum_version); + switch (rc) { - /* We have BLCKSZ of raw data, validate it */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - switch (rc) - { - case PAGE_IS_ZEROED: - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + case PAGE_IS_ZEROED: + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + return PageIsOk; + + case PAGE_IS_VALID: + /* in DELTA or PTRACK modes we must compare lsn */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) + page_is_valid = true; + else return PageIsOk; - - case PAGE_IS_VALID: - /* in DELTA or PTRACK modes we must compare lsn */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) - page_is_valid = true; - else - return PageIsOk; - break; - - case PAGE_HEADER_IS_INVALID: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", - from_fullpath, blknum); - break; - - case PAGE_CHECKSUM_MISMATCH: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", - from_fullpath, blknum); - break; - default: - Assert(false); - } + break; + + case PAGE_HEADER_IS_INVALID: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", + from_fullpath, blknum); + break; + + case PAGE_CHECKSUM_MISMATCH: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", + from_fullpath, blknum); + break; + default: + Assert(false); } } - - /* - * If page is not valid after 100 attempts to read it - * throw an error. - */ - if (!page_is_valid) - { - int elevel = ERROR; - char *errormsg = NULL; - - /* Get the details of corruption */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(page, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(page, &errormsg, - file->segno * RELSEG_SIZE + blknum); - - /* Error out in case of merge or backup without ptrack support; - * issue warning in case of checkdb or backup with ptrack support - */ - if (!strict) - elevel = WARNING; - - if (errormsg) - elog(elevel, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - else - elog(elevel, "Corruption detected in file \"%s\", block %u", - from_fullpath, blknum); - - pg_free(errormsg); - return PageIsCorrupted; - } - - /* Checkdb not going futher */ - if (!strict) - return PageIsOk; } /* - * Get page via ptrack interface from PostgreSQL shared buffer. - * We do this only in the cases of PTRACK 1.x versions backup + * If page is not valid after PAGE_READ_ATTEMPTS attempts to read it + * throw an error. */ - if (backup_mode == BACKUP_MODE_DIFF_PTRACK - && (ptrack_version_num >= 105 && ptrack_version_num < 200)) + if (!page_is_valid) { - int rc = 0; - size_t page_size = 0; - Page ptrack_page = NULL; - ptrack_page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, - file->relOid, absolute_blknum, &page_size, - ptrack_version_num, ptrack_schema); - - if (ptrack_page == NULL) - /* This block was truncated.*/ - return PageIsTruncated; - - if (page_size != BLCKSZ) - elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", - from_fullpath, blknum, BLCKSZ, page_size); - - /* - * We need to copy the page that was successfully - * retrieved from ptrack into our output "page" parameter. - */ - memcpy(page, ptrack_page, BLCKSZ); - pg_free(ptrack_page); - - /* - * UPD: It apprears that is possible to get zeroed page or page with invalid header - * from shared buffer. - * Note, that getting page with wrong checksumm from shared buffer is - * acceptable. - */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - - /* It is ok to get zeroed page */ - if (rc == PAGE_IS_ZEROED) - return PageIsOk; + int elevel = ERROR; + char *errormsg = NULL; - /* Getting page with invalid header from shared buffers is unacceptable */ + /* Get the details of corruption */ if (rc == PAGE_HEADER_IS_INVALID) - { - char *errormsg = NULL; get_header_errormsg(page, &errormsg); - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - } + else if (rc == PAGE_CHECKSUM_MISMATCH) + get_checksum_errormsg(page, &errormsg, + file->segno * RELSEG_SIZE + blknum); - /* - * We must set checksum here, because it is outdated - * in the block recieved from shared buffers. + /* Error out in case of merge or backup without ptrack support; + * issue warning in case of checkdb or backup with ptrack support */ - if (checksum_version) - page_st->checksum = ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + if (!strict) + elevel = WARNING; + + if (errormsg) + elog(elevel, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, blknum, errormsg); + else + elog(elevel, "Corruption detected in file \"%s\", block %u", + from_fullpath, blknum); + + pg_free(errormsg); + return PageIsCorrupted; } + /* Checkdb not going futher */ + if (!strict) + return PageIsOk; + /* * Skip page if page lsn is less than START_LSN of parent backup. * Nullified pages must be copied by DELTA backup, just to be safe. @@ -531,8 +467,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, * backup with special header. */ void -backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, +backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, int ptrack_version_num, const char *ptrack_schema, @@ -614,7 +549,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, else { /* TODO: stop handling errors internally */ - rc = send_pages(conn_arg, to_fullpath, from_fullpath, file, + rc = send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, @@ -1563,10 +1498,10 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, for (blknum = 0; blknum < nblocks; blknum++) { PageState page_st; - page_state = prepare_page(NULL, file, InvalidXLogRecPtr, - blknum, in, BACKUP_MODE_FULL, - curr_page, false, checksum_version, - 0, NULL, from_fullpath, &page_st); + page_state = prepare_page(file, InvalidXLogRecPtr, + blknum, in, BACKUP_MODE_FULL, + curr_page, false, checksum_version, + 0, NULL, from_fullpath, &page_st); if (page_state == PageIsTruncated) break; @@ -1994,7 +1929,7 @@ open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) /* backup local file */ int -send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, +send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) @@ -2052,11 +1987,11 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f while (blknum < file->n_blocks) { PageState page_st; - int rc = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath, &page_st); + int rc = prepare_page(file, prev_backup_start_lsn, + blknum, in, backup_mode, curr_page, + true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath, &page_st); if (rc == PageIsTruncated) break; diff --git a/src/dir.c b/src/dir.c index dfcddd7d0..ce255d0ad 100644 --- a/src/dir.c +++ b/src/dir.c @@ -677,26 +677,16 @@ dir_check_file(pgFile *file, bool backup_logs) */ if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) return CHECK_FALSE; - - if (sscanf_res == 3 && S_ISDIR(file->mode) && - strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) - file->is_database = true; } else if (path_is_prefix_of_path("global", file->rel_path)) { file->tblspcOid = GLOBALTABLESPACE_OID; - - if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) - file->is_database = true; } else if (path_is_prefix_of_path("base", file->rel_path)) { file->tblspcOid = DEFAULTTABLESPACE_OID; sscanf(file->rel_path, "base/%u/", &(file->dbOid)); - - if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) - file->is_database = true; } /* Do not backup ptrack_init files */ diff --git a/src/merge.c b/src/merge.c index 6e0e74940..e59b359fe 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1253,7 +1253,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, where n_blocks is missing. */ - backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, dest_backup->checksum_version, 0, NULL, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f5fd0f672..ccbf803fd 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -264,7 +264,6 @@ typedef struct pgFile int segno; /* Segment number for ptrack */ int n_blocks; /* number of blocks in the data file in data directory */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ - bool is_database; /* Flag used strictly by ptrack 1.x backup */ int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -589,7 +588,6 @@ typedef struct parray *external_dirs; XLogRecPtr prev_start_lsn; - ConnectionArgs conn_arg; int thread_num; HeaderMap *hdr_map; @@ -842,10 +840,6 @@ extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern char *pg_ptrack_get_block(ConnectionArgs *arguments, - Oid dbOid, Oid tblsOid, Oid relOid, - BlockNumber blknum, size_t *result_size, - int ptrack_version_num, const char *ptrack_schema); /* in restore.c */ extern int do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, @@ -1067,12 +1061,11 @@ extern void pfilearray_clear_locks(parray *file_list); extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, const char *from_fullpath, uint32 checksum_version); -extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, - HeaderMap *hdr_map, bool missing_ok); +extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, + HeaderMap *hdr_map, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, @@ -1172,20 +1165,12 @@ extern void check_system_identifiers(PGconn *conn, char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ -extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); -extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); -extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); -extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, - Oid db_oid, - Oid rel_oid, - size_t *result_size, - PGconn *backup_conn); extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); @@ -1193,7 +1178,7 @@ extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack /* open local file to writing */ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size); -extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, +extern int send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); diff --git a/src/ptrack.c b/src/ptrack.c index 5a2b9f046..6825686c6 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -2,7 +2,7 @@ * * ptrack.c: support functions for ptrack backups * - * Copyright (c) 2019 Postgres Professional + * Copyright (c) 2021 Postgres Professional * *------------------------------------------------------------------------- */ @@ -21,124 +21,6 @@ #define PTRACK_BITS_PER_HEAPBLOCK 1 #define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) -/* - * Given a list of files in the instance to backup, build a pagemap for each - * data file that has ptrack. Result is saved in the pagemap field of pgFile. - * NOTE we rely on the fact that provided parray is sorted by file->rel_path. - */ -void -make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) -{ - size_t i; - Oid dbOid_with_ptrack_init = 0; - Oid tblspcOid_with_ptrack_init = 0; - char *ptrack_nonparsed = NULL; - size_t ptrack_nonparsed_size = 0; - - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - size_t start_addr; - - /* - * If there is a ptrack_init file in the database, - * we must backup all its files, ignoring ptrack files for relations. - */ - if (file->is_database) - { - /* - * The function pg_ptrack_get_and_clear_db returns true - * if there was a ptrack_init file. - * Also ignore ptrack files for global tablespace, - * to avoid any possible specific errors. - */ - if ((file->tblspcOid == GLOBALTABLESPACE_OID) || - pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid, backup_conn)) - { - dbOid_with_ptrack_init = file->dbOid; - tblspcOid_with_ptrack_init = file->tblspcOid; - } - } - - if (file->is_datafile) - { - if (file->tblspcOid == tblspcOid_with_ptrack_init && - file->dbOid == dbOid_with_ptrack_init) - { - /* ignore ptrack if ptrack_init exists */ - elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->rel_path); - file->pagemap_isabsent = true; - continue; - } - - /* get ptrack bitmap once for all segments of the file */ - if (file->segno == 0) - { - /* release previous value */ - pg_free(ptrack_nonparsed); - ptrack_nonparsed_size = 0; - - ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, - file->relOid, &ptrack_nonparsed_size, backup_conn); - } - - if (ptrack_nonparsed != NULL) - { - /* - * pg_ptrack_get_and_clear() returns ptrack with VARHDR cut out. - * Compute the beginning of the ptrack map related to this segment - * - * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 - * RELSEG_SIZE. Number of Pages per segment: 131072 - * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed - * to keep track on one relsegment: 16384 - */ - start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; - - /* - * If file segment was created after we have read ptrack, - * we won't have a bitmap for this segment. - */ - if (start_addr > ptrack_nonparsed_size) - { - elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); - file->pagemap_isabsent = true; - } - else - { - - if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) - { - file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - else - { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - - file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); - memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); - } - } - else - { - /* - * If ptrack file is missing, try to copy the entire file. - * It can happen in two cases: - * - files were created by commands that bypass buffer manager - * and, correspondingly, ptrack mechanism. - * i.e. CREATE DATABASE - * - target relation was deleted. - */ - elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); - file->pagemap_isabsent = true; - } - } - } -} - /* * Parse a string like "2.1" into int * result: int by formula major_number * 100 + minor_number @@ -218,7 +100,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) nodeInfo->ptrack_version_num = ptrack_version_num; /* ptrack 1.X is buggy, so fall back to DELTA backup strategy for safety */ - if (nodeInfo->ptrack_version_num >= 105 && nodeInfo->ptrack_version_num < 200) + if (nodeInfo->ptrack_version_num < 200) { if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -241,12 +123,7 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) PGresult *res_db; bool result = false; - if (ptrack_version_num < 200) - { - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; - } - else if (ptrack_version_num == 200) + if (ptrack_version_num == 200) { res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; @@ -262,214 +139,6 @@ pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) return result; } - -/* ---------------------------- - * Ptrack 1.* support functions - * ---------------------------- - */ - -/* Clear ptrack files in all databases of the instance we connected to */ -void -pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) -{ - PGresult *res_db, - *res; - const char *dbname; - int i; - Oid dbOid, tblspcOid; - char *params[2]; - - // FIXME Perform this check on caller's side - if (ptrack_version_num >= 200) - return; - - params[0] = palloc(64); - params[1] = palloc(64); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", - 0, NULL); - - for(i = 0; i < PQntuples(res_db); i++) - { - PGconn *tmp_conn; - - dbname = PQgetvalue(res_db, i, 0); - if (strcmp(dbname, "template0") == 0) - continue; - - dbOid = atoll(PQgetvalue(res_db, i, 1)); - tblspcOid = atoll(PQgetvalue(res_db, i, 2)); - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", - 0, NULL); - PQclear(res); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - PQclear(res); - - pgut_disconnect(tmp_conn); - } - - pfree(params[0]); - pfree(params[1]); - PQclear(res_db); -} - -bool -pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) -{ - char *params[2]; - char *dbname; - PGresult *res_db; - PGresult *res; - bool result; - - params[0] = palloc(64); - params[1] = palloc(64); - - sprintf(params[0], "%i", dbOid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return false; - - dbname = PQgetvalue(res_db, 0, 0); - - /* Always backup all files from template0 database */ - if (strcmp(dbname, "template0") == 0) - { - PQclear(res_db); - return true; - } - PQclear(res_db); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); - - if (!parse_bool(PQgetvalue(res, 0, 0), &result)) - elog(ERROR, - "result of pg_ptrack_get_and_clear_db() is invalid: %s", - PQgetvalue(res, 0, 0)); - - PQclear(res); - pfree(params[0]); - pfree(params[1]); - - return result; -} - -/* Read and clear ptrack files of the target relation. - * Result is a bytea ptrack map of all segments of the target relation. - * case 1: we know a tablespace_oid, db_oid, and rel_filenode - * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) - * case 3: we know only rel_filenode (because file in pg_global) - */ -char * -pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, - size_t *result_size, PGconn *backup_conn) -{ - PGconn *tmp_conn; - PGresult *res_db, - *res; - char *params[2]; - char *result; - char *val; - - params[0] = palloc(64); - params[1] = palloc(64); - - /* regular file (not in directory 'global') */ - if (db_oid != 0) - { - char *dbname; - - sprintf(params[0], "%i", db_oid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return NULL; - - dbname = PQgetvalue(res_db, 0, 0); - - if (strcmp(dbname, "template0") == 0) - { - PQclear(res_db); - return NULL; - } - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", - dbname, tablespace_oid, rel_filenode); - PQclear(res_db); - pgut_disconnect(tmp_conn); - } - /* file in directory 'global' */ - else - { - /* - * execute ptrack_get_and_clear for relation in pg_global - * Use backup_conn, cause we can do it from any database. - */ - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", - rel_filenode); - } - - val = PQgetvalue(res, 0, 0); - - /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. - * It should be fixed in future ptrack releases, but till then we - * can parse it. - */ - if (strcmp("x", val+1) == 0) - { - /* Ptrack file is missing */ - return NULL; - } - - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - PQclear(res); - pfree(params[0]); - pfree(params[1]); - - return result; -} - /* * Get lsn of the moment when ptrack was enabled the last time. */ @@ -482,20 +151,14 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) uint32 lsn_lo; XLogRecPtr lsn; - if (nodeInfo->ptrack_version_num < 200) - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_control_lsn()", - 0, NULL); - else - { - char query[128]; + char query[128]; - if (nodeInfo->ptrack_version_num == 200) - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); - else - sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 200) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); - res = pgut_execute(backup_conn, query, 0, NULL); - } + res = pgut_execute(backup_conn, query, 0, NULL); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -506,99 +169,6 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) return lsn; } -char * -pg_ptrack_get_block(ConnectionArgs *arguments, - Oid dbOid, - Oid tblsOid, - Oid relOid, - BlockNumber blknum, - size_t *result_size, - int ptrack_version_num, - const char *ptrack_schema) -{ - PGresult *res; - char *params[4]; - char *result; - - params[0] = palloc(64); - params[1] = palloc(64); - params[2] = palloc(64); - params[3] = palloc(64); - - /* - * Use tmp_conn, since we may work in parallel threads. - * We can connect to any database. - */ - sprintf(params[0], "%i", tblsOid); - sprintf(params[1], "%i", dbOid); - sprintf(params[2], "%i", relOid); - sprintf(params[3], "%u", blknum); - - if (arguments->conn == NULL) - { - arguments->conn = pgut_connect(instance_config.conn_opt.pghost, - instance_config.conn_opt.pgport, - instance_config.conn_opt.pgdatabase, - instance_config.conn_opt.pguser); - } - - if (arguments->cancel_conn == NULL) - arguments->cancel_conn = PQgetCancel(arguments->conn); - - // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - - if (ptrack_version_num < 200) - res = pgut_execute_parallel(arguments->conn, - arguments->cancel_conn, - "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", - 4, (const char **)params, true, false, false); - else - { - char query[128]; - - /* sanity */ - if (!ptrack_schema) - elog(ERROR, "Schema name of ptrack extension is missing"); - - if (ptrack_version_num == 200) - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); - else - elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); - // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); - - res = pgut_execute_parallel(arguments->conn, - arguments->cancel_conn, - query, 4, (const char **)params, - true, false, false); - } - - if (PQnfields(res) != 1) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - if (PQgetisnull(res, 0, 0)) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - - PQclear(res); - - pfree(params[0]); - pfree(params[1]); - pfree(params[2]); - pfree(params[3]); - - return result; -} - /* ---------------------------- * Ptrack 2.* support functions * ---------------------------- diff --git a/src/utils/file.c b/src/utils/file.c index d40817aed..e9792dd9c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -48,7 +48,6 @@ typedef struct size_t size; time_t mtime; bool is_datafile; - bool is_database; Oid tblspcOid; Oid dbOid; Oid relOid; @@ -2571,7 +2570,6 @@ fio_list_dir_internal(parray *files, const char *root, bool exclude, file->size = fio_file.size; file->mtime = fio_file.mtime; file->is_datafile = fio_file.is_datafile; - file->is_database = fio_file.is_database; file->tblspcOid = fio_file.tblspcOid; file->dbOid = fio_file.dbOid; file->relOid = fio_file.relOid; @@ -2645,7 +2643,6 @@ fio_list_dir_impl(int out, char* buf) fio_file.size = file->size; fio_file.mtime = file->mtime; fio_file.is_datafile = file->is_datafile; - fio_file.is_database = file->is_database; fio_file.tblspcOid = file->tblspcOid; fio_file.dbOid = file->dbOid; fio_file.relOid = file->relOid; diff --git a/tests/auth_test.py b/tests/auth_test.py index c84fdb981..78af21be9 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -190,9 +190,7 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') @classmethod diff --git a/tests/backup.py b/tests/backup.py index 53790ad03..8c537dbc3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2106,6 +2106,7 @@ def test_backup_with_least_privileges_role(self): if self.ptrack: if node.major_version < 12: + # Reviewer, NB: skip this test in case of old ptrack? for fname in [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', diff --git a/tests/false_positive.py b/tests/false_positive.py index d4e7ccf0d..a101f8107 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -107,192 +107,6 @@ def test_incremental_backup_corrupt_full_1(self): # Clean after yourself self.del_test_dir(module_name, fname) - @unittest.expectedFailure - def test_ptrack_concurrent_get_and_clear_1(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'], - gdb=True - ) - - gdb.set_breakpoint('make_pagemap_from_ptrack') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - tablespace_oid = node.safe_psql( - "postgres", - "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() - - relfilenode = node.safe_psql( - "postgres", - "select 't_heap'::regclass::oid").rstrip() - - node.safe_psql( - "postgres", - "SELECT pg_ptrack_get_and_clear({0}, {1})".format( - tablespace_oid, relfilenode)) - - gdb.continue_execution_until_exit() - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - # Logical comparison - self.assertEqual( - result, - node.safe_psql("postgres", "SELECT * FROM t_heap")) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.expectedFailure - def test_ptrack_concurrent_get_and_clear_2(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'], - gdb=True - ) - - gdb.set_breakpoint('pthread_create') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - tablespace_oid = node.safe_psql( - "postgres", - "select oid from pg_tablespace " - "where spcname = 'pg_default'").rstrip() - - relfilenode = node.safe_psql( - "postgres", - "select 't_heap'::regclass::oid").rstrip() - - node.safe_psql( - "postgres", - "SELECT pg_ptrack_get_and_clear({0}, {1})".format( - tablespace_oid, relfilenode)) - - gdb._execute("delete breakpoints") - gdb.continue_execution_until_exit() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup ptrack_lsn.\n" - " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - # Logical comparison - self.assertEqual( - result, - node.safe_psql("postgres", "SELECT * FROM t_heap") - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_pg_10_waldir(self): diff --git a/tests/ptrack.py b/tests/ptrack.py index 011f8754a..fb530a691 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -14,6 +14,10 @@ class PtrackTest(ProbackupTest, unittest.TestCase): + def setUp(self): + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + self.fname = self.id().split('.')[3] # @unittest.skip("skip") def test_ptrack_stop_pg(self): @@ -22,10 +26,9 @@ def test_ptrack_stop_pg(self): restart node, check that ptrack backup can be taken """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -52,7 +55,7 @@ def test_ptrack_stop_pg(self): backup_type='ptrack', options=['--stream']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_multi_timeline_backup(self): @@ -60,10 +63,9 @@ def test_ptrack_multi_timeline_backup(self): t2 /------P2 t1 ------F---*-----P1 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -130,7 +132,7 @@ def test_ptrack_multi_timeline_backup(self): self.assertEqual('0', balance) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_multi_timeline_backup_1(self): @@ -142,10 +144,9 @@ def test_ptrack_multi_timeline_backup_1(self): t2 /------P2 t1 ---F--------* """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -206,17 +207,16 @@ def test_ptrack_multi_timeline_backup_1(self): self.assertEqual('0', balance) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_eat_my_data(self): """ PGPRO-4051 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -236,7 +236,7 @@ def test_ptrack_eat_my_data(self): self.backup_node(backup_dir, 'node', node) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) @@ -287,16 +287,15 @@ def test_ptrack_eat_my_data(self): 'Data loss') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_simple(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -335,7 +334,7 @@ def test_ptrack_simple(self): result = node.safe_psql("postgres", "SELECT * FROM t_heap") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -358,15 +357,14 @@ def test_ptrack_simple(self): node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_unprivileged(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -484,7 +482,8 @@ def test_ptrack_unprivileged(self): ) if node.major_version < 11: - fnames = [ + # Reviewer, NB: skip this test in case of old ptrack? + self.fnames = [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', 'pg_catalog.pg_ptrack_clear()', @@ -494,7 +493,7 @@ def test_ptrack_unprivileged(self): 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' ] - for fname in fnames: + for self.fname in self.fnames: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) @@ -536,10 +535,9 @@ def test_ptrack_unprivileged(self): # @unittest.expectedFailure def test_ptrack_enable(self): """make ptrack without full backup, should result in error""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -577,7 +575,7 @@ def test_ptrack_enable(self): ) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -587,10 +585,9 @@ def test_ptrack_disable(self): enable ptrack, restart postgresql, take ptrack backup which should fail """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -650,15 +647,14 @@ def test_ptrack_disable(self): ) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_uncommitted_xact(self): """make ptrack backup while there is uncommitted open transaction""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -689,7 +685,7 @@ def test_ptrack_uncommitted_xact(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -710,16 +706,15 @@ def test_ptrack_uncommitted_xact(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_vacuum_full(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -773,7 +768,7 @@ def test_ptrack_vacuum_full(self): process.join() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -797,7 +792,7 @@ def test_ptrack_vacuum_full(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_vacuum_truncate(self): @@ -805,10 +800,9 @@ def test_ptrack_vacuum_truncate(self): delete last 3 pages, vacuum relation, take ptrack backup, take second ptrack backup, restore last ptrack backup and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -856,7 +850,7 @@ def test_ptrack_vacuum_truncate(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -882,16 +876,17 @@ def test_ptrack_vacuum_truncate(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_get_block(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """ + make node, make full and ptrack stream backups, + restore them and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -900,11 +895,9 @@ def test_ptrack_get_block(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.safe_psql( "postgres", @@ -917,10 +910,7 @@ def test_ptrack_get_block(self): options=['--stream'], gdb=True) - if node.major_version > 11: - gdb.set_breakpoint('make_pagemap_from_ptrack_2') - else: - gdb.set_breakpoint('make_pagemap_from_ptrack_1') + gdb.set_breakpoint('make_pagemap_from_ptrack_2') gdb.run_until_break() node.safe_psql( @@ -950,21 +940,18 @@ def test_ptrack_get_block(self): # Logical comparison self.assertEqual( result, - node.safe_psql("postgres", "SELECT * FROM t_heap") - ) + node.safe_psql("postgres", "SELECT * FROM t_heap")) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_stream(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -975,10 +962,9 @@ def test_ptrack_stream(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql("postgres", "create sequence t_seq") @@ -1045,17 +1031,15 @@ def test_ptrack_stream(self): self.assertEqual(ptrack_result, ptrack_result_new) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_archive(self): """make archive node, make full and ptrack backups, check data correctness in restored instance""" - self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1067,10 +1051,9 @@ def test_ptrack_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -1158,20 +1141,18 @@ def test_ptrack_archive(self): node.cleanup() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_ptrack_pgpro417(self): - """Make node, take full backup, take ptrack backup, - delete ptrack backup. Try to take ptrack backup, - which should fail. Actual only for PTRACK 1.x""" - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """ + Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail. Actual only for PTRACK 1.x + """ + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1238,22 +1219,18 @@ def test_ptrack_pgpro417(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_page_pgpro417(self): """ Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail. Actual only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1308,22 +1285,18 @@ def test_page_pgpro417(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") + @unittest.skip("skip") def test_full_pgpro417(self): """ Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1384,7 +1357,7 @@ def test_full_pgpro417(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_create_db(self): @@ -1392,10 +1365,9 @@ def test_create_db(self): Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1406,10 +1378,9 @@ def test_create_db(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -1439,7 +1410,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1501,7 +1472,7 @@ def test_create_db(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_create_db_on_replica(self): @@ -1511,10 +1482,9 @@ def test_create_db_on_replica(self): create database db1, take ptrack backup from replica, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1537,7 +1507,7 @@ def test_create_db_on_replica(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.backup_node( @@ -1590,7 +1560,7 @@ def test_create_db_on_replica(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1604,16 +1574,15 @@ def test_create_db_on_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_alter_table_set_tablespace_ptrack(self): """Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database.""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1661,7 +1630,7 @@ def test_alter_table_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1696,17 +1665,16 @@ def test_alter_table_set_tablespace_ptrack(self): # self.assertEqual(result, result_new, 'lost some data after restore') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_alter_database_set_tablespace_ptrack(self): """Make node, create tablespace with database," " take full backup, alter tablespace location," " take ptrack backup, restore database.""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1744,7 +1712,7 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1766,7 +1734,7 @@ def test_alter_database_set_tablespace_ptrack(self): node_restored.slow_start() # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_drop_tablespace(self): @@ -1774,10 +1742,9 @@ def test_drop_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1862,7 +1829,7 @@ def test_drop_tablespace(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): @@ -1870,10 +1837,9 @@ def test_ptrack_alter_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1920,7 +1886,7 @@ def test_ptrack_alter_tablespace(self): # Restore ptrack backup restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') @@ -1979,7 +1945,7 @@ def test_ptrack_alter_tablespace(self): self.assertEqual(result, result_new) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_multiple_segments(self): @@ -1987,10 +1953,9 @@ def test_ptrack_multiple_segments(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -2056,7 +2021,7 @@ def test_ptrack_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -2090,28 +2055,23 @@ def test_ptrack_multiple_segments(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_atexit_fail(self): """ Take backups of every available types and check that PTRACK is clean. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'max_connections': '15'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2147,26 +2107,22 @@ def test_atexit_fail(self): "f") # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") + @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): """ Take backups of every available types and check that PTRACK is clean Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2259,29 +2215,24 @@ def test_ptrack_clean(self): self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_ptrack_clean_replica(self): """ Take backups of every available types from master and check that PTRACK on replica is clean. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2289,7 +2240,7 @@ def test_ptrack_clean_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2402,18 +2353,17 @@ def test_ptrack_clean_replica(self): self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_on_btree(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2467,18 +2417,17 @@ def test_ptrack_cluster_on_btree(self): self.check_ptrack_map_sanity(node, idx_ptrack) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_cluster_on_gist(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2540,18 +2489,17 @@ def test_ptrack_cluster_on_gist(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_cluster_on_btree_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2564,7 +2512,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2628,7 +2576,7 @@ def test_ptrack_cluster_on_btree_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node) @@ -2637,17 +2585,16 @@ def test_ptrack_cluster_on_btree_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_cluster_on_gist_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2660,7 +2607,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2730,7 +2677,7 @@ def test_ptrack_cluster_on_gist_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node) @@ -2740,20 +2687,19 @@ def test_ptrack_cluster_on_gist_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2792,7 +2738,7 @@ def test_ptrack_empty(self): node.safe_psql('postgres', 'checkpoint') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') @@ -2818,7 +2764,7 @@ def test_ptrack_empty(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -2827,14 +2773,13 @@ def test_ptrack_empty_replica(self): Take backups of every available types from master and check that PTRACK on replica is clean """ - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2847,7 +2792,7 @@ def test_ptrack_empty_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2903,7 +2848,7 @@ def test_ptrack_empty_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2915,19 +2860,18 @@ def test_ptrack_empty_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_truncate(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2998,13 +2942,12 @@ def test_ptrack_truncate(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_basic_ptrack_truncate_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -3013,7 +2956,7 @@ def test_basic_ptrack_truncate_replica(self): 'archive_timeout': '10s', 'checkpoint_timeout': '5min'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3026,7 +2969,7 @@ def test_basic_ptrack_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3108,7 +3051,7 @@ def test_basic_ptrack_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) @@ -3127,19 +3070,18 @@ def test_basic_ptrack_truncate_replica(self): 'select 1') # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3215,20 +3157,19 @@ def test_ptrack_vacuum(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_vacuum_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3241,7 +3182,7 @@ def test_ptrack_vacuum_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3314,7 +3255,7 @@ def test_ptrack_vacuum_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) @@ -3323,19 +3264,18 @@ def test_ptrack_vacuum_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3403,18 +3343,17 @@ def test_ptrack_vacuum_bits_frozen(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_ptrack_vacuum_bits_frozen_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3427,7 +3366,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3502,19 +3441,18 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3582,18 +3520,17 @@ def test_ptrack_vacuum_bits_visibility(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3661,19 +3598,18 @@ def test_ptrack_vacuum_full(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3685,7 +3621,7 @@ def test_ptrack_vacuum_full_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3763,19 +3699,18 @@ def test_ptrack_vacuum_full_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3832,7 +3767,7 @@ def test_ptrack_vacuum_truncate(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3841,19 +3776,18 @@ def test_ptrack_vacuum_truncate(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3866,7 +3800,7 @@ def test_ptrack_vacuum_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3938,7 +3872,7 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'replica', node_restored) @@ -3947,22 +3881,21 @@ def test_ptrack_vacuum_truncate_replica(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_ptrack_recovery(self): - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] + """ + Check that ptrack map contain correct bits after recovery. + Actual only for PTRACK 1.x + """ node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4009,17 +3942,13 @@ def test_ptrack_recovery(self): self.check_ptrack_recovery(idx_ptrack[i]) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery_1(self): - if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -4027,7 +3956,7 @@ def test_ptrack_recovery_1(self): 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4067,9 +3996,9 @@ def test_ptrack_recovery_1(self): 'postgres', "create extension pg_buffercache") - print(node.safe_psql( - 'postgres', - "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + #print(node.safe_psql( + # 'postgres', + # "SELECT count(*) FROM pg_buffercache WHERE isdirty")) if self.verbose: print('Killing postmaster. Losing Ptrack changes') @@ -4088,7 +4017,7 @@ def test_ptrack_recovery_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -4098,19 +4027,18 @@ def test_ptrack_recovery_1(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_zero_changes(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4144,14 +4072,13 @@ def test_ptrack_zero_changes(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -4159,7 +4086,7 @@ def test_ptrack_pg_resetxlog(self): 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4259,7 +4186,7 @@ def test_ptrack_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( @@ -4269,23 +4196,18 @@ def test_ptrack_pg_resetxlog(self): # self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_ptrack_map(self): - - if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4388,11 +4310,8 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) - node.slow_start() - sleep(1) - try: self.backup_node( backup_dir, 'node', node, @@ -4410,8 +4329,6 @@ def test_corrupt_ptrack_map(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) - sleep(1) - self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream']) @@ -4435,26 +4352,21 @@ def test_corrupt_ptrack_map(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") def test_horizon_lsn_ptrack(self): """ https://github.com/postgrespro/pg_probackup/pull/386 """ - - if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip("You need PostgreSQL >= 11 for this test") - self.assertLessEqual( self.version_to_num(self.old_probackup_version), self.version_to_num('2.4.15'), 'You need pg_probackup old_binary =< 2.4.15 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -4510,4 +4422,4 @@ def test_horizon_lsn_ptrack(self): self.assertEqual(delta_bytes, ptrack_bytes) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(module_name, self.fname) diff --git a/tests/restore.py b/tests/restore.py index 8ccffa44c..4e4b5c926 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3300,6 +3300,7 @@ def test_missing_database_map(self): if self.ptrack: fnames = [] if node.major_version < 12: + # Reviewer, NB: skip this test in case of old ptrack? fnames += [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', @@ -3314,7 +3315,6 @@ def test_missing_database_map(self): # fnames += [ # 'pg_ptrack_get_pagemapset(pg_lsn)', # 'pg_ptrack_control_lsn()', -# 'pg_ptrack_get_block(oid, oid, oid, bigint)' # ] node.safe_psql( "backupdb", From 7c3f49a7eb992d1de510b549206116b6d9e33bcf Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 19:28:48 +0300 Subject: [PATCH 1743/2107] travis: run on every branch, do not tolerate job failures --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 457835d3c..eb01d7ae4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,11 +42,11 @@ env: - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -jobs: - allow_failures: - - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) +#jobs: +# allow_failures: +# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage -branches: - only: - - master +#branches: +# only: +# - master From 4c27a93e8dd8a442afc350fcc70ced375bfada48 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 19:44:08 +0300 Subject: [PATCH 1744/2107] tests: minor fix in ptrack.py --- tests/ptrack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index fb530a691..611c6c8b0 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -483,7 +483,7 @@ def test_ptrack_unprivileged(self): if node.major_version < 11: # Reviewer, NB: skip this test in case of old ptrack? - self.fnames = [ + fnames = [ 'pg_catalog.oideq(oid, oid)', 'pg_catalog.ptrack_version()', 'pg_catalog.pg_ptrack_clear()', @@ -493,7 +493,7 @@ def test_ptrack_unprivileged(self): 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' ] - for self.fname in self.fnames: + for fname in fnames: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) From 5f54e623c957f0798be1763a9a6b414155600398 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:23:34 +0300 Subject: [PATCH 1745/2107] travis: start sshd in container --- travis/run_tests.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 1bb3a6fde..9aa49bae4 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -4,6 +4,13 @@ # Copyright (c) 2019-2020, Postgres Professional # +sudo su -c 'apt-get update -y' +sudo su -c 'apt-get install openssh-client openssh-server -y' +/etc/init.d/ssh start + +ssh-keygen -t rsa -q -N "" +cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys +ssh-keyscan -H localhost >> ~/.ssh/known_hosts PG_SRC=$PWD/postgres From 88bea549d33b6837275efb95027ad67e57bf1a25 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:31:30 +0300 Subject: [PATCH 1746/2107] travis: move to python3 --- travis/Dockerfile.in | 4 ++-- travis/run_tests.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index a3c858ee2..a1f30d7f6 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,11 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip +RUN apt-get -yq install python3 python3-pip # RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py # RUN python2 get-pip.py -RUN python2 -m pip install virtualenv +RUN python3 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 9aa49bae4..149111c60 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -67,17 +67,17 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -python2 -m virtualenv pyenv +python3 -m virtualenv pyenv source pyenv/bin/activate -pip install testgres==1.8.2 +pip3 install testgres echo "############### Testing:" if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON - python -m unittest -v tests - python -m unittest -v tests.init + python3 -m unittest -v tests + python3 -m unittest -v tests.init else - python -m unittest -v tests.$MODE + python3 -m unittest -v tests.$MODE fi # Generate *.gcov files From 365c959f58e1fb509019b2bd7400bf1f6fd3f9eb Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:40:46 +0300 Subject: [PATCH 1747/2107] travis: some more fixes --- travis/run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 149111c60..b985208ab 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -3,7 +3,9 @@ # # Copyright (c) 2019-2020, Postgres Professional # +set -xe +sudo su -c 'mkdir /run/sshd' sudo su -c 'apt-get update -y' sudo su -c 'apt-get install openssh-client openssh-server -y' /etc/init.d/ssh start From 423ffcaf38e272b689c2a9a360db375a98f49f72 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:23:34 +0300 Subject: [PATCH 1748/2107] travis: start sshd in container --- travis/run_tests.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 1bb3a6fde..9aa49bae4 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -4,6 +4,13 @@ # Copyright (c) 2019-2020, Postgres Professional # +sudo su -c 'apt-get update -y' +sudo su -c 'apt-get install openssh-client openssh-server -y' +/etc/init.d/ssh start + +ssh-keygen -t rsa -q -N "" +cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys +ssh-keyscan -H localhost >> ~/.ssh/known_hosts PG_SRC=$PWD/postgres From 6506eade264700c2525e8cbf920b8322b584e6ba Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:31:30 +0300 Subject: [PATCH 1749/2107] travis: move to python3 --- travis/Dockerfile.in | 4 ++-- travis/run_tests.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index a3c858ee2..a1f30d7f6 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -2,11 +2,11 @@ FROM ololobus/postgres-dev:stretch USER root RUN apt-get update -RUN apt-get -yq install python python-pip +RUN apt-get -yq install python3 python3-pip # RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py # RUN python2 get-pip.py -RUN python2 -m pip install virtualenv +RUN python3 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 9aa49bae4..149111c60 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -67,17 +67,17 @@ make USE_PGXS=1 top_srcdir=$PG_SRC install # Setup python environment echo "############### Setting up python env:" -python2 -m virtualenv pyenv +python3 -m virtualenv pyenv source pyenv/bin/activate -pip install testgres==1.8.2 +pip3 install testgres echo "############### Testing:" if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON - python -m unittest -v tests - python -m unittest -v tests.init + python3 -m unittest -v tests + python3 -m unittest -v tests.init else - python -m unittest -v tests.$MODE + python3 -m unittest -v tests.$MODE fi # Generate *.gcov files From 318cb6e5b84cf0d16c828a6a9cc97dca48ad6dab Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:40:46 +0300 Subject: [PATCH 1750/2107] travis: some more fixes --- travis/run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 149111c60..b985208ab 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -3,7 +3,9 @@ # # Copyright (c) 2019-2020, Postgres Professional # +set -xe +sudo su -c 'mkdir /run/sshd' sudo su -c 'apt-get update -y' sudo su -c 'apt-get install openssh-client openssh-server -y' /etc/init.d/ssh start From cf8fb8c9deea49d045e756f551bf7d33f6d1986e Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 21:59:04 +0300 Subject: [PATCH 1751/2107] travis: run only smoke suit --- .travis.yml | 21 +++++++++++---------- travis/run_tests.sh | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb01d7ae4..462534a7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,21 +26,22 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: + - PG_VERSION=12 PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore - PG_VERSION=11 PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore #jobs: # allow_failures: diff --git a/travis/run_tests.sh b/travis/run_tests.sh index b985208ab..748a06d59 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -8,9 +8,9 @@ set -xe sudo su -c 'mkdir /run/sshd' sudo su -c 'apt-get update -y' sudo su -c 'apt-get install openssh-client openssh-server -y' -/etc/init.d/ssh start +sudo su -c '/etc/init.d/ssh start' -ssh-keygen -t rsa -q -N "" +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N "" cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys ssh-keyscan -H localhost >> ~/.ssh/known_hosts From dbbfee4a635a5688521a9d35398d7b2801a9fa50 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 17 Jun 2021 22:29:55 +0300 Subject: [PATCH 1752/2107] tests: disable ptrack testing for PG < 11 --- tests/helpers/ptrack_helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 0a87eb1b6..5eeeeaa91 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -312,6 +312,12 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" + if self.ptrack: + self.assertGreaterEqual( + self.pg_config_version, + self.version_to_num('11.0'), + "ptrack testing require PostgreSQL >= 11") + @property def pg_config_version(self): return self.version_to_num( From d65ae6ccbdbe98cde690c88f64e87f47fc50c775 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Jun 2021 01:15:43 +0300 Subject: [PATCH 1753/2107] travis: build only required PG objects to speed up test --- travis/run_tests.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 748a06d59..fd5779e1f 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -36,7 +36,10 @@ git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 echo "############### Compiling Postgres:" cd postgres # Go to postgres dir ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests -make -s -j$(nproc) install +#make -s -j$(nproc) install +make -s -j$(nproc) -C 'src/common' install +make -s -j$(nproc) -C 'src/port' install +make -s -j$(nproc) -C 'src/interfaces' install make -s -j$(nproc) -C contrib/ install # Override default Postgres instance From 469c5a1736cf909d1b92ff07b55954cca0379af4 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Jun 2021 10:38:15 +0300 Subject: [PATCH 1754/2107] travis: some more fixes --- .travis.yml | 2 +- travis/run_tests.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 462534a7f..c66cf6439 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=12 PG_BRANCH=REL_13_STABLE + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE - PG_VERSION=11 PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE diff --git a/travis/run_tests.sh b/travis/run_tests.sh index fd5779e1f..635b9f422 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -36,10 +36,10 @@ git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 echo "############### Compiling Postgres:" cd postgres # Go to postgres dir ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests -#make -s -j$(nproc) install -make -s -j$(nproc) -C 'src/common' install -make -s -j$(nproc) -C 'src/port' install -make -s -j$(nproc) -C 'src/interfaces' install +make -s -j$(nproc) install +#make -s -j$(nproc) -C 'src/common' install +#make -s -j$(nproc) -C 'src/port' install +#make -s -j$(nproc) -C 'src/interfaces' install make -s -j$(nproc) -C contrib/ install # Override default Postgres instance From 7ca590c6cc39df44e2e73d3e363447aca24a0fef Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Jun 2021 12:12:37 +0300 Subject: [PATCH 1755/2107] tests: drop ptrack1 suppport --- tests/backup.py | 231 +++---------------- tests/cfs_backup.py | 7 +- tests/compatibility.py | 7 +- tests/delete.py | 7 +- tests/helpers/ptrack_helpers.py | 10 +- tests/merge.py | 7 +- tests/ptrack.py | 381 +++++++++++++++++++------------- tests/restore.py | 68 ++---- 8 files changed, 296 insertions(+), 422 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 8c537dbc3..45fd137eb 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -18,9 +18,6 @@ class BackupTest(ProbackupTest, unittest.TestCase): # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -33,12 +30,7 @@ def test_backup_modes_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - backup_id = self.backup_node(backup_dir, 'node', node) + full_backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] self.assertEqual(show_backup['status'], "OK") @@ -47,7 +39,7 @@ def test_backup_modes_archive(self): # postmaster.pid and postmaster.opts shouldn't be copied excluded = True db_dir = os.path.join( - backup_dir, "backups", 'node', backup_id, "database") + backup_dir, "backups", 'node', full_backup_id, "database") for f in os.listdir(db_dir): if ( @@ -64,31 +56,30 @@ def test_backup_modes_archive(self): page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - # print self.show_pb(node) - show_backup = self.show_pb(backup_dir, 'node')[1] + show_backup_1 = self.show_pb(backup_dir, 'node')[1] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "PAGE") + # delta backup mode + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + show_backup_2 = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "DELTA") + # Check parent backup self.assertEqual( - backup_id, + full_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['id'])["parent-backup-id"]) - - # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + backup_id=show_backup_1['id'])["parent-backup-id"]) - show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PTRACK") - - # Check parent backup self.assertEqual( page_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['id'])["parent-backup-id"]) + backup_id=show_backup_2['id'])["parent-backup-id"]) # Clean after yourself self.del_test_dir(module_name, fname) @@ -118,10 +109,7 @@ def test_smooth_checkpoint(self): # @unittest.skip("skip") def test_incremental_backup_without_full(self): - """page-level backup without validated full backup""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - + """page backup without validated full backup""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -134,11 +122,6 @@ def test_incremental_backup_without_full(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - try: self.backup_node(backup_dir, 'node', node, backup_type="page") # we should die here because exception is what we expect to happen @@ -154,29 +137,10 @@ def test_incremental_backup_without_full(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - try: - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - self.assertEqual( self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], - "ERROR") - # Clean after yourself self.del_test_dir(module_name, fname) @@ -242,64 +206,19 @@ def test_incremental_backup_corrupt_full(self): self.del_test_dir(module_name, fname) # @unittest.skip("skip") - def test_ptrack_threads(self): - """ptrack multi thread backup mode""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_threads_stream(self): - """ptrack multi thread backup mode and stream""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - + def test_delta_threads_stream(self): + """delta multi thread backup mode and stream""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - ptrack_enable=True) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -307,7 +226,7 @@ def test_ptrack_threads_stream(self): self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) + backup_type="delta", options=["-j", "4", "--stream"]) self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") # Clean after yourself @@ -1459,76 +1378,6 @@ def test_drop_rel_during_backup_page(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_drop_rel_during_backup_ptrack(self): - """""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - absolute_path = os.path.join(node.data_dir, relative_path) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # PTRACK backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - gdb=True, options=['--log-level-file=LOG']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - os.remove(absolute_path) - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( - 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_persistent_slot_for_stream_backup(self): """""" @@ -1992,10 +1841,11 @@ def test_backup_with_least_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') - if self.ptrack and node.major_version >= 12: + if self.ptrack: node.safe_psql( "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + "CREATE SCHEMA ptrack; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") # PG 9.5 if self.get_version(node) < 90600: @@ -2105,33 +1955,14 @@ def test_backup_with_least_privileges_role(self): ) if self.ptrack: - if node.major_version < 12: - # Reviewer, NB: skip this test in case of old ptrack? - for fname in [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', - 'pg_catalog.pg_stop_backup()']: - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) - else: - fnames = [ - 'pg_catalog.ptrack_get_pagemapset(pg_lsn)', - 'pg_catalog.ptrack_init_lsn()' - ] - - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " + "GRANT EXECUTE ON FUNCTION 'ptrack.ptrack_init_lsn()' TO backup; ") if ProbackupTest.enterprise: node.safe_psql( @@ -2391,7 +2222,7 @@ def test_backup_with_less_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') - if self.ptrack and node.major_version >= 12: + if self.ptrack: node.safe_psql( 'backupdb', 'CREATE EXTENSION ptrack') diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 2e686d46c..d820360fe 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -35,10 +35,9 @@ def setUp(self): self.node.slow_start() - if self.node.major_version >= 12: - self.node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + self.node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) diff --git a/tests/compatibility.py b/tests/compatibility.py index d0fae2528..e274c22be 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -304,10 +304,9 @@ def test_backward_compatibility_ptrack(self): self.set_archiving(backup_dir, 'node', node, old_binary=True) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=10) diff --git a/tests/delete.py b/tests/delete.py index 8ebd7d13a..345a70284 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -203,10 +203,9 @@ def test_delete_increment_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - 'postgres', - 'CREATE EXTENSION ptrack') + node.safe_psql( + 'postgres', + 'CREATE EXTENSION ptrack') # full backup mode self.backup_node(backup_dir, 'node', node) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 5eeeeaa91..ea661d158 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -388,11 +388,8 @@ def make_simple_node( options['max_wal_senders'] = 10 if ptrack_enable: - if node.major_version >= 11: - options['ptrack.map_size'] = '128' - options['shared_preload_libraries'] = 'ptrack' - else: - options['ptrack_enable'] = 'on' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' if node.major_version >= 13: options['wal_keep_size'] = '200MB' @@ -622,9 +619,6 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): return ptrack_bits_for_fork def check_ptrack_map_sanity(self, node, idx_ptrack): - if node.major_version >= 12: - return - success = True for i in idx_ptrack: # get new size of heap and indexes. size calculated in pages diff --git a/tests/merge.py b/tests/merge.py index 668691fc8..fe0927f49 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -811,10 +811,9 @@ def test_merge_ptrack_truncate(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') diff --git a/tests/ptrack.py b/tests/ptrack.py index 611c6c8b0..bcc8dc20a 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -19,6 +19,136 @@ def setUp(self): return unittest.skip('You need PostgreSQL >= 11 for this test') self.fname = self.id().split('.')[3] + # @unittest.skip("skip") + def test_drop_rel_during_backup_ptrack(self): + """ + drop relation during ptrack backup + """ + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PTRACK backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + gdb=True, options=['--log-level-file=LOG']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + self.del_test_dir(module_name, self.fname) + + # @unittest.skip("skip") + def test_ptrack_without_full(self): + """ptrack backup without validated full backup""" + node = self.make_simple_node( + base_dir=os.path.join(module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # Clean after yourself + self.del_test_dir(module_name, self.fname) + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + node = self.make_simple_node( + base_dir=os.path.join(module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Clean after yourself + self.del_test_dir(module_name, self.fname) + # @unittest.skip("skip") def test_ptrack_stop_pg(self): """ @@ -37,10 +167,9 @@ def test_ptrack_stop_pg(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=1) @@ -75,10 +204,9 @@ def test_ptrack_multi_timeline_backup(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=5) @@ -156,10 +284,9 @@ def test_ptrack_multi_timeline_backup_1(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=5) @@ -226,10 +353,9 @@ def test_ptrack_eat_my_data(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=50) @@ -304,10 +430,9 @@ def test_ptrack_simple(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -481,33 +606,15 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - if node.major_version < 11: - # Reviewer, NB: skip this test in case of old ptrack? - fnames = [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' - ] - - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) - - else: - node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack") - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - node.safe_psql( - "backupdb", - "GRANT USAGE ON SCHEMA ptrack TO backup") + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack") + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") node.safe_psql( "backupdb", @@ -547,10 +654,9 @@ def test_ptrack_enable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # PTRACK BACKUP try: @@ -597,28 +703,21 @@ def test_ptrack_disable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) # DISABLE PTRACK - if node.major_version >= 11: - node.safe_psql('postgres', "alter system set ptrack.map_size to 0") - else: - node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") node.stop() node.slow_start() # ENABLE PTRACK - if node.major_version >= 11: - node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") - node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") - else: - node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") node.stop() node.slow_start() @@ -665,10 +764,9 @@ def test_ptrack_uncommitted_xact(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -725,10 +823,9 @@ def test_ptrack_vacuum_full(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -813,10 +910,9 @@ def test_ptrack_vacuum_truncate(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.safe_psql( "postgres", @@ -1495,10 +1591,9 @@ def test_create_db_on_replica(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -1593,10 +1688,9 @@ def test_alter_table_set_tablespace_ptrack(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.create_tblspace_in_node(node, 'somedata') @@ -1685,10 +1779,9 @@ def test_alter_database_set_tablespace_ptrack(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1755,10 +1848,9 @@ def test_drop_tablespace(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -1850,10 +1942,9 @@ def test_ptrack_alter_tablespace(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') tblspc_path = self.get_tblspace_path(node, 'somedata') @@ -1966,10 +2057,9 @@ def test_ptrack_multiple_segments(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2368,10 +2458,9 @@ def test_ptrack_cluster_on_btree(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2432,10 +2521,9 @@ def test_ptrack_cluster_on_gist(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table and indexes node.safe_psql( @@ -2704,10 +2792,9 @@ def test_ptrack_empty(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2876,10 +2963,9 @@ def test_ptrack_truncate(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3086,10 +3172,9 @@ def test_ptrack_vacuum(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3280,10 +3365,9 @@ def test_ptrack_vacuum_bits_frozen(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3457,10 +3541,9 @@ def test_ptrack_vacuum_bits_visibility(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3535,10 +3618,9 @@ def test_ptrack_vacuum_full(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3715,10 +3797,9 @@ def test_ptrack_vacuum_truncate(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table and indexes res = node.safe_psql( @@ -4043,10 +4124,9 @@ def test_ptrack_zero_changes(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table node.safe_psql( @@ -4091,10 +4171,9 @@ def test_ptrack_pg_resetxlog(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table node.safe_psql( @@ -4212,10 +4291,9 @@ def test_corrupt_ptrack_map(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table node.safe_psql( @@ -4375,10 +4453,9 @@ def test_horizon_lsn_ptrack(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 11: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # TODO: ptrack version must be 2.1 ptrack_version = node.safe_psql( diff --git a/tests/restore.py b/tests/restore.py index 4e4b5c926..e59a1b0ec 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -512,10 +512,9 @@ def test_restore_full_ptrack_archive(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -567,10 +566,9 @@ def test_restore_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -630,10 +628,9 @@ def test_restore_full_ptrack_stream(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -689,10 +686,9 @@ def test_restore_full_ptrack_under_load(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -759,10 +755,9 @@ def test_restore_full_under_load_ptrack(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # wal_segment_size = self.guc_wal_segment_size(node) node.pgbench_init(scale=2) @@ -3298,32 +3293,13 @@ def test_missing_database_map(self): ) if self.ptrack: - fnames = [] - if node.major_version < 12: - # Reviewer, NB: skip this test in case of old ptrack? - fnames += [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' - ] - else: - # TODO why backup works without these grants ? -# fnames += [ -# 'pg_ptrack_get_pagemapset(pg_lsn)', -# 'pg_ptrack_control_lsn()', -# ] - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) + # TODO why backup works without these grants ? + # 'pg_ptrack_get_pagemapset(pg_lsn)', + # 'pg_ptrack_control_lsn()', + # because PUBLIC + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") if ProbackupTest.enterprise: node.safe_psql( From e68132d10c0b2fe604e3a0c95b6d6d9e02a22778 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Jun 2021 12:20:58 +0300 Subject: [PATCH 1756/2107] tests: fix tests.archive.ArchiveTest.test_basic_master_and_replica_concurrent_archiving --- tests/archive.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/archive.py b/tests/archive.py index a7bc04e13..44fd7bcfb 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -895,6 +895,9 @@ def test_basic_master_and_replica_concurrent_archiving(self): set replica with archiving, make sure that archiving on both node is working. """ + if self.pg_config_version < self.version_to_num('9.6.0'): + return unittest.skip('You need PostgreSQL >= 9.6 for this test') + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( From d55838e80de51743ea3b644deeccfa34132f5efd Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 18 Jun 2021 13:39:11 +0300 Subject: [PATCH 1757/2107] fix test restore.RestoreTest.test_missing_database_map for PGPROEE11 compatibility --- tests/restore.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/restore.py b/tests/restore.py index e59a1b0ec..a76272b12 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3299,7 +3299,9 @@ def test_missing_database_map(self): # because PUBLIC node.safe_psql( "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + "CREATE SCHEMA ptrack; " + "GRANT USAGE ON SCHEMA ptrack TO backup; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") if ProbackupTest.enterprise: node.safe_psql( From ad4fc967d6a2b36a91ef12bf849dbc4a6632d89f Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 18 Jun 2021 20:04:39 +0300 Subject: [PATCH 1758/2107] fix segfault in get_index_list --- src/checkdb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 5d7d6652b..4ea1d0800 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -455,6 +455,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, ind->heapallindexed_is_supported = heapallindexed_is_supported; ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); strcpy(ind->amcheck_nspname, amcheck_nspname); + pg_atomic_clear_flag(&ind->lock); if (index_list == NULL) index_list = parray_new(); @@ -462,8 +463,6 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, parray_append(index_list, ind); } - pfilearray_clear_locks(index_list); - PQclear(res); return index_list; From e27a6a91e0444bf70c3203b1d7efef6585917df7 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 20 Jun 2021 02:13:35 +0300 Subject: [PATCH 1759/2107] [Issue #400] PostgreSQL 14 support --- src/backup.c | 4 ++-- src/catalog.c | 4 ++-- src/merge.c | 2 +- src/parsexlog.c | 12 ++++++++++++ src/show.c | 4 ++-- src/utils/pgut.c | 15 ++++++++++++++- tests/helpers/ptrack_helpers.py | 4 ++++ 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6f1aa867a..83785c1cb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -725,7 +725,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) elog(WARNING, "Current PostgreSQL role is superuser. " "It is not recommended to run backup or checkdb as superuser."); - StrNCpy(current.server_version, nodeInfo->server_version_str, + strlcpy(current.server_version, nodeInfo->server_version_str, sizeof(current.server_version)); return cur_conn; @@ -761,7 +761,7 @@ do_backup(pgSetBackupParams *set_backup_params, current.status = BACKUP_STATUS_RUNNING; current.start_time = current.backup_id; - StrNCpy(current.program_version, PROGRAM_VERSION, + strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); current.compress_alg = instance_config.compress_alg; diff --git a/src/catalog.c b/src/catalog.c index 3ea4d9bca..3ba17e9fd 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2648,14 +2648,14 @@ readBackupControlFile(const char *path) if (program_version) { - StrNCpy(backup->program_version, program_version, + strlcpy(backup->program_version, program_version, sizeof(backup->program_version)); pfree(program_version); } if (server_version) { - StrNCpy(backup->server_version, server_version, + strlcpy(backup->server_version, server_version, sizeof(backup->server_version)); pfree(server_version); } diff --git a/src/merge.c b/src/merge.c index 3fd5b13ae..f351975d3 100644 --- a/src/merge.c +++ b/src/merge.c @@ -735,7 +735,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, * We cannot set backup status to OK just yet, * because it still has old start_time. */ - StrNCpy(full_backup->program_version, PROGRAM_VERSION, + strlcpy(full_backup->program_version, PROGRAM_VERSION, sizeof(full_backup->program_version)); full_backup->parent_backup = INVALID_BACKUP_ID; full_backup->start_lsn = dest_backup->start_lsn; diff --git a/src/parsexlog.c b/src/parsexlog.c index 8dfb2c78c..19078fb64 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1798,6 +1798,18 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, * source system. */ } + else if (rmid == RM_XACT_ID && + ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED)) + { + /* + * These records can include "dropped rels". We can safely ignore + * them, we will see that they are missing and copy them from the + * source. + */ + } else if (info & XLR_SPECIAL_REL_UPDATE) { /* diff --git a/src/show.c b/src/show.c index c1482772e..496c9d833 100644 --- a/src/show.c +++ b/src/show.c @@ -552,7 +552,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na time2iso(row->recovery_time, lengthof(row->recovery_time), backup->recovery_time, false); else - StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time)); + strlcpy(row->recovery_time, "----", sizeof(row->recovery_time)); widths[cur] = Max(widths[cur], strlen(row->recovery_time)); cur++; @@ -587,7 +587,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na pretty_time_interval(difftime(backup->end_time, backup->start_time), row->duration, lengthof(row->duration)); else - StrNCpy(row->duration, "----", sizeof(row->duration)); + strlcpy(row->duration, "----", sizeof(row->duration)); widths[cur] = Max(widths[cur], strlen(row->duration)); cur++; diff --git a/src/utils/pgut.c b/src/utils/pgut.c index e1e52b24b..1d8845c23 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -16,6 +16,10 @@ #include "libpq/pqsignal.h" #include "pqexpbuffer.h" +#if PG_VERSION_NUM >= 140000 +#include "common/string.h" +#endif + #include #include "pgut.h" @@ -75,7 +79,16 @@ prompt_for_password(const char *username) password = NULL; } -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 140000 + if (username == NULL) + password = simple_prompt("Password: ", false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message , false); + } +#elif PG_VERSION_NUM >= 100000 password = (char *) pgut_malloc(sizeof(char) * 100 + 1); if (username == NULL) simple_prompt("Password: ", password, 100, false); diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b6dc6d028..bf84f266e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -139,6 +139,10 @@ def slow_start(self, replica=False): except testgres.QueryException as e: if 'database system is starting up' in e.message: pass + elif 'FATAL: the database system is not accepting connections' in e.message: + pass + elif replica and 'Hot standby mode is disabled' in e.message: + raise e else: raise e From b395b3c942ce6cf49ed7d299fa1764eef2a9b0f8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 20 Jun 2021 02:28:59 +0300 Subject: [PATCH 1760/2107] Readme: update --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 49b9351df..b7e170cf5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.5, 9.6, 10, 11, 12, 13; +* PostgreSQL 9.5, 9.6, 10, 11, 12, 13, 14; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 12,13 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 9.6, 10, 11, 12 -* Postgres Pro Enterprise 9.6, 10, 11, 12 +* vanilla PostgreSQL 11, 12, 13, 14 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 9.6, 10, 11, 12, 13 +* Postgres Pro Enterprise 9.6, 10, 11, 12, 13 ## Limitations From 7de728496d1080cd34d957af254eed556fc163d3 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Mon, 21 Jun 2021 11:45:10 +0300 Subject: [PATCH 1761/2107] Catchup command implementation (#392) [ PR #392] New command "catchup" is added, it allows fallen-behind standby to "catch up" with master, or create standby from scratch without resorting to restore from backup Co-authored-by: Grigory Smolkin Co-authored-by: anastasia Co-authored-by: Elena Indrupskaya --- Makefile | 2 +- doc/pgprobackup.xml | 282 ++++++++- src/archive.c | 2 +- src/backup.c | 78 ++- src/catalog.c | 2 +- src/catchup.c | 1020 +++++++++++++++++++++++++++++++ src/data.c | 314 +++++++++- src/dir.c | 9 +- src/help.c | 63 +- src/init.c | 2 +- src/pg_probackup.c | 47 +- src/pg_probackup.h | 42 +- src/ptrack.c | 2 +- src/restore.c | 16 +- src/stream.c | 30 +- src/util.c | 38 +- src/utils/configuration.c | 1 + src/utils/configuration.h | 3 +- src/utils/file.c | 256 +++++++- src/utils/file.h | 4 +- src/utils/parray.c | 7 + src/utils/parray.h | 1 + tests/__init__.py | 4 +- tests/catchup.py | 977 +++++++++++++++++++++++++++++ tests/helpers/ptrack_helpers.py | 46 +- 25 files changed, 3146 insertions(+), 102 deletions(-) create mode 100644 src/catchup.c create mode 100644 tests/catchup.py diff --git a/Makefile b/Makefile index 1431be4ef..5173aa38f 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ - src/util.o src/validate.o src/datapagemap.o + src/util.o src/validate.o src/datapagemap.o src/catchup.o # borrowed files OBJS += src/pg_crc.o src/receivelog.o src/streamutil.o \ diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index b1ddd0032..f7814c2d2 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -143,6 +143,14 @@ doc/src/sgml/pgprobackup.sgml wal_file_name option + + pg_probackup + + catchup_mode + =path_to_pgdata_on_remote_server + =path_to_local_dir + option + @@ -283,6 +291,11 @@ doc/src/sgml/pgprobackup.sgml Partial restore: restoring only the specified databases. + + + Catchup: cloning a PostgreSQL instance for a fallen-behind standby server to catch up with master. + + To manage backup data, pg_probackup creates a @@ -1076,7 +1089,8 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; mode: , , , - , + , + , and . @@ -1431,6 +1445,7 @@ pg_probackup backup -B backup_dir --instance + Performing Cluster Verification @@ -1506,6 +1521,7 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ higher cost of CPU, memory, and I/O consumption. + Validating a Backup @@ -2073,6 +2089,7 @@ pg_probackup restore -B backup_dir --instance , , , + , and processes can be executed on several parallel threads. This can significantly @@ -3390,6 +3407,148 @@ pg_probackup delete -B backup_dir --instance + + + Cloning <productname>PostgreSQL</productname> Instance + + pg_probackup can create a copy of a PostgreSQL + instance directly, without using the backup catalog. This allows you + to add a new standby server in a parallel mode or to have a standby + server that has fallen behind catch up with master. + + + + Cloning a PostgreSQL instance is different from other pg_probackup + operations: + + + + The backup catalog is not required. + + + + + STREAM WAL delivery mode is only supported. + + + + + Copying external directories + is not supported. + + + + + No SQL commands involving tablespaces, such as + CREATE TABLESPACE/DROP TABLESPACE, + can be run simultaneously with catchup. + + + + + catchup takes configuration files, such as + postgresql.conf, postgresql.auto.conf, + or pg_hba.conf, from the source server and overwrites them + on the target server. + + + + + + + Before cloning a PostgreSQL instance, set up the source database server as follows: + + + + Configure + the database cluster for the instance to copy. + + + + + To copy from a remote server, configure the remote mode. + + + + + To use the PTRACK catchup mode, set up PTRACK backups. + + + + + + + To clone a PostgreSQL instance, ensure that the source + database server is running and accepting connections and + on the server with the destination database, run the following command: + + +pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream [connection_options] [remote_options] + + + Where catchup_mode can take one of the + following values: FULL, DELTA, or PTRACK. + + + + + FULL — creates a full copy of the PostgreSQL instance. + The destination directory must be empty for this mode. + + + + + DELTA — reads all data files in the data directory and + creates an incremental copy for pages that have changed + since the destination database was shut down cleanly. + For this mode, the destination directory must contain a previous + copy of the database that was shut down cleanly. + + + + + PTRACK — tracking page changes on the fly, + only copies pages that have changed since the point of divergence + of the source and destination databases. + For this mode, the destination directory must contain a previous + copy of the database that was shut down cleanly. + + + + + You can use connection_options to specify + the connection to the source database cluster. If it is located on a different server, + also specify remote_options. + If the source database contains tablespaces that must be located in + a different directory, additionally specify the + option: + +pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --tablespace-mapping=OLDDIR=NEWDIR + + To run the catchup command on parallel threads, specify the number + of threads with the option: + +pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --threads=num_threads + + + + For example, assume that a remote standby server with the PostgreSQL instance having /replica-pgdata data directory has fallen behind. To sync this instance with the one in /master-pgdata data directory, you can run + the catchup command in the PTRACK mode on four parallel threads as follows: + +pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=PTRACK --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 + + + + Another example shows how you can add a new remote standby server with the PostgreSQL data directory /replica-pgdata by running the catchup command in the FULL mode + on four parallel threads: + +pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=FULL --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 + + + @@ -4262,6 +4421,121 @@ pg_probackup archive-get -B backup_dir --instance Archiving Options. + + + catchup + +pg_probackup catchup -b catchup_mode +--source-pgdata=path_to_pgdata_on_remote_server +--destination-pgdata=path_to_local_dir +[--help] [--stream] [-j num_threads] +[-T OLDDIR=NEWDIR] +[connection_options] [remote_options] + + + Creates a copy of a PostgreSQL + instance without using the backup catalog. + + + + + + + + Specifies the catchup mode to use. Possible values are: + + + + + FULL — creates a full copy of the PostgreSQL instance. + + + + + DELTA — reads all data files in the data directory and + creates an incremental copy for pages that have changed + since the destination database was shut down cleanly. + + + + + PTRACK — tracking page changes on the fly, + only copies pages that have changed since the point of divergence + of the source and destination databases. + + + + + + + + + + + + Specifies the path to the data directory of the instance to be copied. The path can be local or remote. + + + + + + + + + Specifies the path to the local data directory to copy to. + + + + + + + + + Makes a STREAM backup, which + includes all the necessary WAL files by streaming them from + the database server via replication protocol. + + + + + + + + + + Sets the number of parallel threads for + catchup process. + + + + + + + + + + Relocates the tablespace from the OLDDIR to the NEWDIR + directory at the time of recovery. Both OLDDIR and NEWDIR must + be absolute paths. If the path contains the equals sign (=), + escape it with a backslash. This option can be specified + multiple times for multiple tablespaces. + + + + + + + + + Additionally, connection + options, remote + mode options can be used. + + + For details on usage, see the section + Cloning PostgreSQL Instance. + + Options @@ -4651,7 +4925,7 @@ pg_probackup archive-get -B backup_dir --instance - Disable the coloring for console log messages of warning and error levels. + Disable coloring for console log messages of warning and error levels. @@ -4804,7 +5078,8 @@ pg_probackup archive-get -B backup_dir --instance Connection Options You can use these options together with - and + + , , and commands. @@ -5095,6 +5370,7 @@ pg_probackup archive-get -B backup_dir --instance , , , + , , , and commands. diff --git a/src/archive.c b/src/archive.c index 4058cd0d4..7bb8c1c03 100644 --- a/src/archive.c +++ b/src/archive.c @@ -148,7 +148,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa elog(ERROR, "getcwd() error"); /* verify that archive-push --instance parameter is valid */ - system_id = get_system_identifier(current_dir); + system_id = get_system_identifier(current_dir, FIO_DB_HOST); if (instance->pgdata == NULL) elog(ERROR, "Cannot read pg_probackup.conf for this instance"); diff --git a/src/backup.c b/src/backup.c index 688afefca..2d834410a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -94,7 +94,6 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { int i; char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ - char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -137,7 +136,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); #else - current.tli = get_current_timeline_from_control(false); + current.tli = get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); #endif /* @@ -258,17 +257,19 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, /* start stream replication */ if (current.stream) { - join_path_components(dst_backup_path, current.database_dir, PG_XLOG_DIR); - fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); + char stream_xlog_path[MAXPGPATH]; - start_WAL_streaming(backup_conn, dst_backup_path, &instance_config.conn_opt, + join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR); + fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + + start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, current.start_lsn, current.tli); /* Make sure that WAL streaming is working * PAGE backup in stream mode is waited twice, first for * segment in WAL archive and then for streamed segment */ - wait_wal_lsn(dst_backup_path, current.start_lsn, true, current.tli, false, true, ERROR, true); + wait_wal_lsn(stream_xlog_path, current.start_lsn, true, current.tli, false, true, ERROR, true); } /* initialize backup's file list */ @@ -315,23 +316,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, elog(ERROR, "PGDATA is almost empty. Either it was concurrently deleted or " "pg_probackup do not possess sufficient permissions to list PGDATA content"); - /* Calculate pgdata_bytes */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - if (file->external_dir_num != 0) - continue; - - if (S_ISDIR(file->mode)) - { - current.pgdata_bytes += 4096; - continue; - } - - current.pgdata_bytes += file->size; - } - + current.pgdata_bytes += calculate_datasize_of_filelist(backup_files_list); pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); elog(INFO, "PGDATA size: %s", pretty_bytes); @@ -697,7 +682,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) if (nodeInfo->is_superuser) elog(WARNING, "Current PostgreSQL role is superuser. " - "It is not recommended to run backup or checkdb as superuser."); + "It is not recommended to run pg_probackup under superuser."); strlcpy(current.server_version, nodeInfo->server_version_str, sizeof(current.server_version)); @@ -786,7 +771,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); + nodeInfo.is_ptrack_enabled = pg_is_ptrack_enabled(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { @@ -795,7 +780,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, elog(ERROR, "This PostgreSQL instance does not support ptrack"); else { - if (!nodeInfo.is_ptrack_enable) + if (!nodeInfo.is_ptrack_enabled) elog(ERROR, "Ptrack is disabled"); } } @@ -953,12 +938,12 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) * All system identifiers must be equal. */ void -check_system_identifiers(PGconn *conn, char *pgdata) +check_system_identifiers(PGconn *conn, const char *pgdata) { uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata); + system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST); system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ @@ -1069,7 +1054,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, * Switch to a new WAL segment. It should be called only for master. * For PG 9.5 it should be called only if pguser is superuser. */ -static void +void pg_switch_wal(PGconn *conn) { PGresult *res; @@ -2282,7 +2267,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) } -static void +void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) { PGresult *res; @@ -2346,3 +2331,36 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) } } } + +/* + * Calculate pgdata_bytes + * accepts (parray *) of (pgFile *) + */ +int64 +calculate_datasize_of_filelist(parray *filelist) +{ + int64 bytes = 0; + int i; + + /* parray_num don't check for NULL */ + if (filelist == NULL) + return 0; + + for (i = 0; i < parray_num(filelist); i++) + { + pgFile *file = (pgFile *) parray_get(filelist, i); + + if (file->external_dir_num != 0) + continue; + + if (S_ISDIR(file->mode)) + { + // TODO is a dir always 4K? + bytes += 4096; + continue; + } + + bytes += file->size; + } + return bytes; +} diff --git a/src/catalog.c b/src/catalog.c index f8af2b72e..9775968b8 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2883,7 +2883,7 @@ pgNodeInit(PGNodeInfo *node) node->server_version_str[0] = '\0'; node->ptrack_version_num = 0; - node->is_ptrack_enable = false; + node->is_ptrack_enabled = false; node->ptrack_schema = NULL; } diff --git a/src/catchup.c b/src/catchup.c new file mode 100644 index 000000000..f80a0f0f9 --- /dev/null +++ b/src/catchup.c @@ -0,0 +1,1020 @@ +/*------------------------------------------------------------------------- + * + * catchup.c: sync DB cluster + * + * Copyright (c) 2021, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#if PG_VERSION_NUM < 110000 +#include "catalog/catalog.h" +#endif +#include "catalog/pg_tablespace.h" +#include "access/timeline.h" +#include "pgtar.h" +#include "streamutil.h" + +#include +#include +#include + +#include "utils/thread.h" +#include "utils/file.h" + +/* + * Catchup routines + */ +static PGconn *catchup_collect_info(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); +static void catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, + const char *dest_pgdata); +static void catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn); +static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli); + +//REVIEW The name of this function looks strange to me. +//Maybe catchup_init_state() or catchup_setup() will do better? +//I'd also suggest to wrap all these fields into some CatchupState, but it isn't urgent. +/* + * Prepare for work: fill some globals, open connection to source database + */ +static PGconn * +catchup_collect_info(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) +{ + PGconn *source_conn; + + /* Initialize PGInfonode */ + pgNodeInit(source_node_info); + + /* Get WAL segments size and system ID of source PG instance */ + instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); + instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST); + current.start_time = time(NULL); + + StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); + + /* Do some compatibility checks and fill basic info about PG instance */ + source_conn = pgdata_basic_setup(instance_config.conn_opt, source_node_info); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(source_conn)) + elog(ERROR, "Failed to retrieve wal_segment_size"); +#endif + + get_ptrack_version(source_conn, source_node_info); + if (source_node_info->ptrack_version_num > 0) + source_node_info->is_ptrack_enabled = pg_is_ptrack_enabled(source_conn, source_node_info->ptrack_version_num); + + /* Obtain current timeline */ +#if PG_VERSION_NUM >= 90600 + current.tli = get_current_timeline(source_conn); +#else + instance_config.pgdata = source_pgdata; + current.tli = get_current_timeline_from_control(source_pgdata, FIO_DB_HOST, false); +#endif + + elog(INFO, "Catchup start, pg_probackup version: %s, " + "PostgreSQL version: %s, " + "remote: %s, source-pgdata: %s, destination-pgdata: %s", + PROGRAM_VERSION, source_node_info->server_version_str, + IsSshProtocol() ? "true" : "false", + source_pgdata, dest_pgdata); + + if (current.from_replica) + elog(INFO, "Running catchup from standby"); + + return source_conn; +} + +/* + * Check that catchup can be performed on source and dest + * this function is for checks, that can be performed without modification of data on disk + */ +static void +catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, + const char *source_pgdata, const char *dest_pgdata) +{ + /* TODO + * gsmol - fallback to FULL mode if dest PGDATA is empty + * kulaginm -- I think this is a harmful feature. If user requested an incremental catchup, then + * he expects that this will be done quickly and efficiently. If, for example, he made a mistake + * with dest_dir, then he will receive a second full copy instead of an error message, and I think + * that in some cases he would prefer the error. + * I propose in future versions to offer a backup_mode auto, in which we will look to the dest_dir + * and decide which of the modes will be the most effective. + * I.e.: + * if(requested_backup_mode == BACKUP_MODE_DIFF_AUTO) + * { + * if(dest_pgdata_is_empty) + * backup_mode = BACKUP_MODE_FULL; + * else + * if(ptrack supported and applicable) + * backup_mode = BACKUP_MODE_DIFF_PTRACK; + * else + * backup_mode = BACKUP_MODE_DIFF_DELTA; + * } + */ + + if (dir_is_empty(dest_pgdata, FIO_LOCAL_HOST)) + { + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + elog(ERROR, "\"%s\" is empty, but incremental catchup mode requested.", + dest_pgdata); + } + else /* dest dir not empty */ + { + if (current.backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Can't perform full catchup into non-empty directory \"%s\".", + dest_pgdata); + } + + /* check that postmaster is not running in destination */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + pid_t pid; + pid = fio_check_postmaster(dest_pgdata, FIO_LOCAL_HOST); + if (pid == 1) /* postmaster.pid is mangled */ + { + char pid_filename[MAXPGPATH]; + join_path_components(pid_filename, dest_pgdata, "postmaster.pid"); + elog(ERROR, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", + pid_filename); + } + else if (pid > 1) /* postmaster is up */ + { + elog(ERROR, "Postmaster with pid %u is running in destination directory \"%s\"", + pid, dest_pgdata); + } + } + + /* check backup_label absence in dest */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char backup_label_filename[MAXPGPATH]; + + join_path_components(backup_label_filename, dest_pgdata, PG_BACKUP_LABEL_FILE); + if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0) + elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); + } + + /* check that destination database is shutdowned cleanly */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + DBState state; + state = get_system_dbstate(dest_pgdata, FIO_LOCAL_HOST); + /* see states in postgres sources (src/include/catalog/pg_control.h) */ + if (state != DB_SHUTDOWNED && state != DB_SHUTDOWNED_IN_RECOVERY) + elog(ERROR, "Postmaster in destination directory \"%s\" must be stopped cleanly", + dest_pgdata); + } + + /* Check that connected PG instance, source and destination PGDATA are the same */ + { + uint64 source_conn_id, source_id, dest_id; + + source_conn_id = get_remote_system_identifier(source_conn); + source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */ + + if (source_conn_id != source_id) + elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", + source_conn_id, source_pgdata, source_id); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST); + if (source_conn_id != dest_id) + elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", + source_conn_id, dest_pgdata, dest_id); + } + } + + /* check PTRACK version */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (source_node_info->ptrack_version_num == 0) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else if (source_node_info->ptrack_version_num < 200) + elog(ERROR, "ptrack extension is too old.\n" + "Upgrade ptrack to version >= 2"); + else if (!source_node_info->is_ptrack_enabled) + elog(ERROR, "Ptrack is disabled"); + } + + if (current.from_replica && exclusive_backup) + elog(ERROR, "Catchup from standby is only available for PostgreSQL >= 9.6"); + + /* check that we don't overwrite tablespace in source pgdata */ + catchup_check_tablespaces_existance_in_tbsmapping(source_conn); + + /* check timelines */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; + + /* fill dest_redo.lsn and dest_redo.tli */ + get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + + if (current.tli != 1) + { + parray *source_timelines; /* parray* of TimeLineHistoryEntry* */ + source_timelines = catchup_get_tli_history(&instance_config.conn_opt, current.tli); + + if (source_timelines == NULL) + elog(ERROR, "Cannot get source timeline history"); + + if (!satisfy_timeline(source_timelines, dest_redo.tli, dest_redo.lsn)) + elog(ERROR, "Destination is not in source timeline history"); + + parray_walk(source_timelines, pfree); + parray_free(source_timelines); + } + else /* special case -- no history files in source */ + { + if (dest_redo.tli != 1) + elog(ERROR, "Source is behind destination in timeline history"); + } + } +} + +/* + * Check that all tablespaces exists in tablespace mapping (--tablespace-mapping option) + * Check that all local mapped directories is empty if it is local FULL catchup + * Emit fatal error if that (not existent in map or not empty) tablespace found + */ +static void +catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) +{ + PGresult *res; + int i; + char *tablespace_path = NULL; + const char *linked_path = NULL; + char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " + "FROM pg_catalog.pg_tablespace " + "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; + + res = pgut_execute(conn, query, 0, NULL); + + if (!res) + elog(ERROR, "Failed to get list of tablespaces"); + + for (i = 0; i < res->ntups; i++) + { + tablespace_path = PQgetvalue(res, i, 0); + Assert (strlen(tablespace_path) > 0); + + canonicalize_path(tablespace_path); + linked_path = get_tablespace_mapping(tablespace_path); + + if (strcmp(tablespace_path, linked_path) == 0) + /* same result -> not found in mapping */ + { + if (!fio_is_remote(FIO_DB_HOST)) + elog(ERROR, "Local catchup executed, but source database contains " + "tablespace (\"%s\"), that is not listed in the map", tablespace_path); + else + elog(WARNING, "Remote catchup executed and source database contains " + "tablespace (\"%s\"), that is not listed in the map", tablespace_path); + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "Tablespace directory path must be an absolute path: \"%s\"", + linked_path); + + if (current.backup_mode == BACKUP_MODE_FULL + && !dir_is_empty(linked_path, FIO_LOCAL_HOST)) + elog(ERROR, "Target mapped tablespace directory (\"%s\") is not empty in FULL catchup", + linked_path); + } + PQclear(res); +} + +/* + * Get timeline history via replication connection + * returns parray* of TimeLineHistoryEntry* + */ +static parray* +catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli) +{ + PGresult *res; + PGconn *conn; + char *history; + char query[128]; + parray *result = NULL; + + snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); + + /* + * Connect in replication mode to the server. + */ + conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser, + false); + + if (!conn) + return NULL; + + res = PQexec(conn, query); + PQfinish(conn); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING, "Could not send replication command \"%s\": %s", + query, PQresultErrorMessage(res)); + PQclear(res); + return NULL; + } + + /* + * The response to TIMELINE_HISTORY is a single row result set + * with two fields: filename and content + */ + if (PQnfields(res) != 2 || PQntuples(res) != 1) + { + elog(ERROR, "Unexpected response to TIMELINE_HISTORY command: " + "got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + PQclear(res); + return NULL; + } + + history = pgut_strdup(PQgetvalue(res, 0, 1)); + result = parse_tli_history_buffer(history, tli); + + /* some cleanup */ + pg_free(history); + PQclear(res); + + return result; +} + +/* + * catchup multithreaded copy rountine and helper structure and function + */ + +/* parameters for catchup_thread_runner() passed from catchup_multithreaded_copy() */ +typedef struct +{ + PGNodeInfo *nodeInfo; + const char *from_root; + const char *to_root; + parray *source_filelist; + parray *dest_filelist; + XLogRecPtr sync_lsn; + BackupMode backup_mode; + int thread_num; + bool completed; +} catchup_thread_runner_arg; + +/* Catchup file copier executed in separate thread */ +static void * +catchup_thread_runner(void *arg) +{ + int i; + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + + catchup_thread_runner_arg *arguments = (catchup_thread_runner_arg *) arg; + int n_files = parray_num(arguments->source_filelist); + + /* catchup a file */ + for (i = 0; i < n_files; i++) + { + pgFile *file = (pgFile *) parray_get(arguments->source_filelist, i); + pgFile *dest_file = NULL; + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during catchup"); + + if (progress) + elog(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_files, file->rel_path); + + /* construct destination filepath */ + Assert(file->external_dir_num == 0); + join_path_components(from_fullpath, arguments->from_root, file->rel_path); + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); + + /* Check that file exist in dest pgdata */ + if (arguments->backup_mode != BACKUP_MODE_FULL) + { + pgFile **dest_file_tmp = NULL; + dest_file_tmp = (pgFile **) parray_bsearch(arguments->dest_filelist, + file, pgFileCompareRelPathWithExternal); + if (dest_file_tmp) + { + /* File exists in destination PGDATA */ + file->exists_in_prev = true; + dest_file = *dest_file_tmp; + } + } + + /* Do actual work */ + if (file->is_datafile && !file->is_cfs) + { + catchup_data_file(file, from_fullpath, to_fullpath, + arguments->sync_lsn, + arguments->backup_mode, + NONE_COMPRESS, + 0, + arguments->nodeInfo->checksum_version, + arguments->nodeInfo->ptrack_version_num, + arguments->nodeInfo->ptrack_schema, + false, + dest_file != NULL ? dest_file->size : 0); + } + else + { + backup_non_data_file(file, dest_file, from_fullpath, to_fullpath, + arguments->backup_mode, current.parent_backup, true); + } + + if (file->write_size == FILE_NOT_FOUND) + continue; + + if (file->write_size == BYTES_INVALID) + { + elog(VERBOSE, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); + continue; + } + + elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + from_fullpath, file->write_size); + } + + /* ssh connection to longer needed */ + fio_disconnect(); + + /* Data files transferring is successful */ + arguments->completed = true; + + return NULL; +} + +/* + * main multithreaded copier + */ +static bool +catchup_multithreaded_copy(int num_threads, + PGNodeInfo *source_node_info, + const char *source_pgdata_path, + const char *dest_pgdata_path, + parray *source_filelist, + parray *dest_filelist, + XLogRecPtr sync_lsn, + BackupMode backup_mode) +{ + /* arrays with meta info for multi threaded catchup */ + catchup_thread_runner_arg *threads_args; + pthread_t *threads; + + bool all_threads_successful = true; + int i; + + /* init thread args */ + threads_args = (catchup_thread_runner_arg *) palloc(sizeof(catchup_thread_runner_arg) * num_threads); + for (i = 0; i < num_threads; i++) + threads_args[i] = (catchup_thread_runner_arg){ + .nodeInfo = source_node_info, + .from_root = source_pgdata_path, + .to_root = dest_pgdata_path, + .source_filelist = source_filelist, + .dest_filelist = dest_filelist, + .sync_lsn = sync_lsn, + .backup_mode = backup_mode, + .thread_num = i + 1, + .completed = false, + }; + + /* Run threads */ + thread_interrupted = false; + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + for (i = 0; i < num_threads; i++) + { + elog(VERBOSE, "Start thread num: %i", i); + pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i])); + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + pthread_join(threads[i], NULL); + all_threads_successful &= threads_args[i].completed; + } + + free(threads); + free(threads_args); + return all_threads_successful; +} + +/* + * + */ +static void +catchup_sync_destination_files(const char* pgdata_path, fio_location location, parray *filelist, pgFile *pg_control_file) +{ + char fullpath[MAXPGPATH]; + time_t start_time, end_time; + char pretty_time[20]; + int i; + + elog(INFO, "Syncing copied files to disk"); + time(&start_time); + + for (i = 0; i < parray_num(filelist); i++) + { + pgFile *file = (pgFile *) parray_get(filelist, i); + + /* TODO: sync directory ? */ + if (S_ISDIR(file->mode)) + continue; + + Assert(file->external_dir_num == 0); + join_path_components(fullpath, pgdata_path, file->rel_path); + if (fio_sync(fullpath, location) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); + } + + /* + * sync pg_control file + */ + join_path_components(fullpath, pgdata_path, pg_control_file->rel_path); + if (fio_sync(fullpath, location) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Files are synced, time elapsed: %s", pretty_time); +} + +/* + * Entry point of pg_probackup CATCHUP subcommand. + */ +int +do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files) +{ + PGconn *source_conn = NULL; + PGNodeInfo source_node_info; + bool backup_logs = false; + parray *source_filelist = NULL; + pgFile *source_pg_control_file = NULL; + parray *dest_filelist = NULL; + char dest_xlog_path[MAXPGPATH]; + + RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; + PGStopBackupResult stop_backup_result; + bool catchup_isok = true; + + int i; + + /* for fancy reporting */ + time_t start_time, end_time; + char pretty_time[20]; + char pretty_bytes[20]; + + source_conn = catchup_collect_info(&source_node_info, source_pgdata, dest_pgdata); + catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); + + elog(LOG, "Database catchup start"); + + { + char label[1024]; + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time, false); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + + /* Call pg_start_backup function in PostgreSQL connect */ + pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); + elog(LOG, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + //REVIEW I wonder, if we can move this piece above and call before pg_start backup()? + //It seems to be a part of setup phase. + if (current.backup_mode != BACKUP_MODE_FULL) + { + dest_filelist = parray_new(); + dir_list_file(dest_filelist, dest_pgdata, + true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + + // fill dest_redo.lsn and dest_redo.tli + get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn); + + /* + * Future improvement to catch partial catchup: + * 1. rename dest pg_control into something like pg_control.pbk + * (so user can't start partial catchup'ed instance from this point) + * 2. try to read by get_redo() pg_control and pg_control.pbk (to detect partial catchup) + * 3. at the end (after copy of correct pg_control), remove pg_control.pbk + */ + } + + //REVIEW I wonder, if we can move this piece above and call before pg_start backup()? + //It seems to be a part of setup phase. + /* + * TODO: move to separate function to use in both backup.c and catchup.c + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, &source_node_info); + + // new ptrack is more robust and checks Start LSN + if (ptrack_lsn > dest_redo.lsn || ptrack_lsn == InvalidXLogRecPtr) + elog(ERROR, "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" + "You can perform only FULL catchup.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (dest_redo.lsn >> 32), + (uint32) (dest_redo.lsn)); + } + + /* Check that dest_redo.lsn is less than current.start_lsn */ + if (current.backup_mode != BACKUP_MODE_FULL && + dest_redo.lsn > current.start_lsn) + elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " + "it may indicate that we are trying to catchup with PostgreSQL instance from the past", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn)); + + /* Start stream replication */ + join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); + fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); + start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, + current.start_lsn, current.tli); + + source_filelist = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(source_filelist, source_pgdata, + true, true, false, backup_logs, true, 0); + else + dir_list_file(source_filelist, source_pgdata, + true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + + //REVIEW FIXME. Let's fix that before release. + // TODO filter pg_xlog/wal? + // TODO what if wal is not a dir (symlink to a dir)? + + /* close ssh session in main thread */ + fio_disconnect(); + + //REVIEW Do we want to do similar calculation for dest? + current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); + pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); + elog(INFO, "Source PGDATA size: %s", pretty_bytes); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(source_filelist, pgFileCompareRelPathWithExternal); + + /* Extract information about files in source_filelist parsing their names:*/ + parse_filelist_filenames(source_filelist, source_pgdata); + + elog(LOG, "Start LSN (source): %X/%X, TLI: %X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + current.tli); + if (current.backup_mode != BACKUP_MODE_FULL) + elog(LOG, "LSN in destination: %X/%X, TLI: %X", + (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn), + dest_redo.tli); + + /* Build page mapping in PTRACK mode */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + time(&start_time); + elog(INFO, "Extracting pagemap of changed blocks"); + + /* Build the page map from ptrack information */ + make_pagemap_from_ptrack_2(source_filelist, source_conn, + source_node_info.ptrack_schema, + source_node_info.ptrack_version_num, + dest_redo.lsn); + time(&end_time); + elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", + difftime(end_time, start_time)); + } + + /* + * Make directories before catchup + */ + /* + * We iterate over source_filelist and for every directory with parent 'pg_tblspc' + * we must lookup this directory name in tablespace map. + * If we got a match, we treat this directory as tablespace. + * It means that we create directory specified in tablespace_map and + * original directory created as symlink to it. + */ + for (i = 0; i < parray_num(source_filelist); i++) + { + pgFile *file = (pgFile *) parray_get(source_filelist, i); + char parent_dir[MAXPGPATH]; + + if (!S_ISDIR(file->mode)) + continue; + + /* + * check if it is fake "directory" and is a tablespace link + * this is because we passed the follow_symlink when building the list + */ + /* get parent dir of rel_path */ + strncpy(parent_dir, file->rel_path, MAXPGPATH); + get_parent_directory(parent_dir); + + /* check if directory is actually link to tablespace */ + if (strcmp(parent_dir, PG_TBLSPC_DIR) != 0) + { + /* if the entry is a regular directory, create it in the destination */ + char dirpath[MAXPGPATH]; + + join_path_components(dirpath, dest_pgdata, file->rel_path); + + elog(VERBOSE, "Create directory '%s'", dirpath); + fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); + } + else + { + /* this directory located in pg_tblspc */ + const char *linked_path = NULL; + char to_path[MAXPGPATH]; + + // TODO perform additional check that this is actually symlink? + { /* get full symlink path and map this path to new location */ + char source_full_path[MAXPGPATH]; + char symlink_content[MAXPGPATH]; + join_path_components(source_full_path, source_pgdata, file->rel_path); + fio_readlink(source_full_path, symlink_content, sizeof(symlink_content), FIO_DB_HOST); + /* we checked that mapping exists in preflight_checks for local catchup */ + linked_path = get_tablespace_mapping(symlink_content); + elog(INFO, "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", + source_full_path, + symlink_content, + linked_path); + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", + linked_path); + + join_path_components(to_path, dest_pgdata, file->rel_path); + + elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + linked_path, to_path); + + /* create tablespace directory */ + if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) + elog(ERROR, "Could not create tablespace directory \"%s\": %s", + linked_path, strerror(errno)); + + /* create link to linked_path */ + if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) + elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", + linked_path, to_path, strerror(errno)); + } + } + + /* + * find pg_control file (in already sorted source_filelist) + * and exclude it from list for future special processing + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(source_filelist, &search_key, pgFileCompareRelPathWithExternal); + if(control_file_elem_index < 0) + elog(ERROR, "\"%s\" not found in \"%s\"\n", XLOG_CONTROL_FILE, source_pgdata); + source_pg_control_file = parray_remove(source_filelist, control_file_elem_index); + } + + /* + * remove absent source files in dest (dropped tables, etc...) + * note: global/pg_control will also be deleted here + */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(INFO, "Removing redundant files in destination directory"); + parray_qsort(dest_filelist, pgFileCompareRelPathWithExternalDesc); + for (i = 0; i < parray_num(dest_filelist); i++) + { + bool redundant = true; + pgFile *file = (pgFile *) parray_get(dest_filelist, i); + + //TODO optimize it and use some merge-like algorithm + //instead of bsearch for each file. + if (parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal)) + redundant = false; + + /* pg_filenode.map are always restored, because it's crc cannot be trusted */ + Assert(file->external_dir_num == 0); + if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) + redundant = true; + + //REVIEW This check seems unneded. Anyway we delete only redundant stuff below. + /* do not delete the useful internal directories */ + if (S_ISDIR(file->mode) && !redundant) + continue; + + /* if file does not exists in destination list, then we can safely unlink it */ + if (redundant) + { + char fullpath[MAXPGPATH]; + + join_path_components(fullpath, dest_pgdata, file->rel_path); + + fio_delete(file->mode, fullpath, FIO_DB_HOST); + elog(VERBOSE, "Deleted file \"%s\"", fullpath); + + /* shrink pgdata list */ + pgFileFree(file); + parray_remove(dest_filelist, i); + i--; + } + } + } + + /* clear file locks */ + pfilearray_clear_locks(source_filelist); + + /* Sort by size for load balancing */ + parray_qsort(source_filelist, pgFileCompareSizeDesc); + + /* Sort the array for binary search */ + if (dest_filelist) + parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal); + + /* run copy threads */ + elog(INFO, "Start transferring data files"); + time(&start_time); + catchup_isok = catchup_multithreaded_copy(num_threads, &source_node_info, + source_pgdata, dest_pgdata, + source_filelist, dest_filelist, + dest_redo.lsn, current.backup_mode); + + /* at last copy control file */ + if (catchup_isok) + { + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + join_path_components(from_fullpath, source_pgdata, source_pg_control_file->rel_path); + join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); + copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, + to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); + } + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + if (catchup_isok) + elog(INFO, "Data files are transferred, time elapsed: %s", + pretty_time); + else + elog(ERROR, "Data files transferring failed, time elapsed: %s", + pretty_time); + + /* Notify end of backup */ + { + //REVIEW Is it relevant to catchup? I suppose it isn't, since catchup is a new code. + //If we do need it, please write a comment explaining that. + /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ + int timeout = (instance_config.archive_timeout > 0) ? + instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; + char *stop_backup_query_text = NULL; + + pg_silent_client_messages(source_conn); + + //REVIEW. Do we want to support pg 9.5? I suppose we never test it... + //Maybe check it and error out early? + /* Create restore point + * Only if backup is from master. + * For PG 9.5 create restore point only if pguser is superuser. + */ + if (!current.from_replica && + !(source_node_info.server_version < 90600 && + !source_node_info.is_superuser)) //TODO: check correctness + pg_create_restore_point(source_conn, current.start_time); + + /* Execute pg_stop_backup using PostgreSQL connection */ + pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, exclusive_backup, &stop_backup_query_text); + + /* + * Wait for the result of pg_stop_backup(), but no longer than + * archive_timeout seconds + */ + pg_stop_backup_consume(source_conn, source_node_info.server_version, exclusive_backup, timeout, stop_backup_query_text, &stop_backup_result); + + /* Cleanup */ + pg_free(stop_backup_query_text); + } + + wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); + +#if PG_VERSION_NUM >= 90600 + /* Write backup_label */ + Assert(stop_backup_result.backup_label_content != NULL); + pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, + NULL); + free(stop_backup_result.backup_label_content); + stop_backup_result.backup_label_content = NULL; + stop_backup_result.backup_label_content_len = 0; + + /* tablespace_map */ + if (stop_backup_result.tablespace_map_content != NULL) + { + // TODO what if tablespace is created during catchup? + /* Because we have already created symlinks in pg_tblspc earlier, + * we do not need to write the tablespace_map file. + * So this call is unnecessary: + * pg_stop_backup_write_file_helper(dest_pgdata, PG_TABLESPACE_MAP_FILE, "tablespace map", + * stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, + * NULL); + */ + free(stop_backup_result.tablespace_map_content); + stop_backup_result.tablespace_map_content = NULL; + stop_backup_result.tablespace_map_content_len = 0; + } +#endif + + if(wait_WAL_streaming_end(NULL)) + elog(ERROR, "WAL streaming failed"); + + //REVIEW Please add a comment about these lsns. It is a crutial part of the algorithm. + current.recovery_xid = stop_backup_result.snapshot_xid; + + elog(LOG, "Getting the Recovery Time from WAL"); + + /* iterate over WAL from stop_backup lsn to start_backup lsn */ + if (!read_recovery_info(dest_xlog_path, current.tli, + instance_config.xlog_seg_size, + current.start_lsn, current.stop_lsn, + ¤t.recovery_time)) + { + elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); + current.recovery_time = stop_backup_result.invocation_time; + } + + /* + * In case of backup from replica >= 9.6 we must fix minRecPoint + */ + if (current.from_replica && !exclusive_backup) + { + set_min_recovery_point(source_pg_control_file, dest_pgdata, current.stop_lsn); + } + + /* close ssh session in main thread */ + fio_disconnect(); + + /* Sync all copied files unless '--no-sync' flag is used */ + if (catchup_isok) + { + if (sync_dest_files) + catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); + else + elog(WARNING, "Files are not synced to disk"); + } + + /* Cleanup */ + if (dest_filelist) + { + parray_walk(dest_filelist, pgFileFree); + parray_free(dest_filelist); + } + parray_walk(source_filelist, pgFileFree); + parray_free(source_filelist); + pgFileFree(source_pg_control_file); + + //REVIEW: Are we going to do that before release? + /* TODO: show the amount of transfered data in bytes and calculate incremental ratio */ + + return 0; +} diff --git a/src/data.c b/src/data.c index 314490585..49b696059 100644 --- a/src/data.c +++ b/src/data.c @@ -268,7 +268,7 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) * PageIsOk(0) if page was successfully retrieved * PageIsTruncated(-1) if the page was truncated * SkipCurrentPage(-2) if we need to skip this page, - * only used for DELTA backup + * only used for DELTA and PTRACK backup * PageIsCorrupted(-3) if the page checksum mismatch * or header corruption, * only used for checkdb @@ -400,7 +400,12 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, page_st->lsn > 0 && page_st->lsn < prev_backup_start_lsn) { - elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, from_fullpath); + elog(VERBOSE, "Skipping blknum %u in file: \"%s\", file->exists_in_prev: %s, page_st->lsn: %X/%X, prev_backup_start_lsn: %X/%X", + blknum, from_fullpath, + file->exists_in_prev ? "true" : "false", + (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, + (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn + ); return SkipCurrentPage; } @@ -458,6 +463,23 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, return compressed_size; } +/* взята из compress_and_backup_page, но выпилена вся магия заголовков и компрессии, просто копирование 1-в-1 */ +static int +copy_page(pgFile *file, BlockNumber blknum, + FILE *in, FILE *out, Page page, + const char *to_fullpath) +{ + /* write data page */ + if (fio_fwrite(out, page, BLCKSZ) != BLCKSZ) + elog(ERROR, "File: \"%s\", cannot write at block %u: %s", + to_fullpath, blknum, strerror(errno)); + + file->write_size += BLCKSZ; + file->uncompressed_size += BLCKSZ; + + return BLCKSZ; +} + /* * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with @@ -623,6 +645,169 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat pg_free(headers); } +/* + * Backup data file in the from_root directory to the to_root directory with + * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental backup), validate checksum, optionally compress and write to + * backup with special header. + */ +void +catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, + bool is_merge, size_t prev_size) +{ + int rc; + bool use_pagemap; + char *errmsg = NULL; + BlockNumber err_blknum = 0; + /* page headers */ + BackupPageHeader2 *headers = NULL; + + /* sanity */ + if (file->size % BLCKSZ != 0) + elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + file->n_blocks = file->size/BLCKSZ; + + /* + * Skip unchanged file only if it exists in previous backup. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && file->size == prev_size && !file->pagemap_isabsent) + { + /* + * There are no changed blocks since last backup. We want to make + * incremental backup, so we should exit. + */ + file->write_size = BYTES_INVALID; + return; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + file->uncompressed_size = 0; + INIT_FILE_CRC32(true, file->crc); + + /* + * Read each page, verify checksum and write it to backup. + * If page map is empty or file is not present in previous backup + * backup all pages of the relation. + * + * In PTRACK 1.x there was a problem + * of data files with missing _ptrack map. + * Such files should be fully copied. + */ + + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev || + !file->pagemap.bitmap) + use_pagemap = false; + else + use_pagemap = true; + + if (use_pagemap) + elog(VERBOSE, "Using pagemap for file \"%s\"", file->rel_path); + + /* Remote mode */ + if (fio_is_remote(FIO_DB_HOST)) + { + rc = fio_copy_pages(to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + calg, clevel, checksum_version, + /* send pagemap if any */ + use_pagemap, + /* variables for error reporting */ + &err_blknum, &errmsg, &headers); + } + else + { + /* TODO: stop handling errors internally */ + rc = copy_pages(to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, + checksum_version, use_pagemap, + backup_mode, ptrack_version_num, ptrack_schema); + } + + /* check for errors */ + if (rc == FILE_MISSING) + { + elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + err_blknum, to_fullpath, strerror(errno)); + + else if (rc == PAGE_CORRUPTION) + { + if (errmsg) + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, err_blknum, errmsg); + else + elog(ERROR, "Corruption detected in file \"%s\", block %u", + from_fullpath, err_blknum); + } + /* OPEN_FAILED and READ_FAILED */ + else if (rc == OPEN_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot open file \"%s\"", from_fullpath); + } + else if (rc == READ_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot read file \"%s\"", from_fullpath); + } + + file->read_size = rc * BLCKSZ; + + /* refresh n_blocks for FULL and DELTA */ + if (backup_mode == BACKUP_MODE_FULL || + backup_mode == BACKUP_MODE_DIFF_DELTA) + file->n_blocks = file->read_size / BLCKSZ; + + /* Determine that file didn`t changed in case of incremental catchup */ + if (backup_mode != BACKUP_MODE_FULL && + file->exists_in_prev && + file->write_size == 0 && + file->n_blocks > 0) + { + file->write_size = BYTES_INVALID; + } + +cleanup: + + /* finish CRC calculation */ + FIN_FILE_CRC32(true, file->crc); + + pg_free(errmsg); + pg_free(file->pagemap.bitmap); + pg_free(headers); +} + /* * Backup non data file * We do not apply compression to this file. @@ -1992,6 +2177,7 @@ send_pages(const char *to_fullpath, const char *from_fullpath, true, checksum_version, ptrack_version_num, ptrack_schema, from_fullpath, &page_st); + if (rc == PageIsTruncated) break; @@ -2068,6 +2254,130 @@ send_pages(const char *to_fullpath, const char *from_fullpath, return n_blocks_read; } +/* copy local file (взята из send_pages, но используется простое копирование странички, без добавления заголовков и компрессии) */ +int +copy_pages(const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr sync_lsn, + uint32 checksum_version, bool use_pagemap, + BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) +{ + FILE *in = NULL; + FILE *out = NULL; + char curr_page[BLCKSZ]; + int n_blocks_read = 0; + BlockNumber blknum = 0; + datapagemap_iterator_t *iter = NULL; + + /* stdio buffers */ + char *in_buf = NULL; + char *out_buf = NULL; + + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + return FILE_MISSING; + + elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); + } + + /* + * Enable stdio buffering for local input file, + * unless the pagemap is involved, which + * imply a lot of random access. + */ + + if (use_pagemap) + { + iter = datapagemap_iterate(&file->pagemap); + datapagemap_next(iter, &blknum); /* set first block */ + + setvbuf(in, NULL, _IONBF, BUFSIZ); + } + else + { + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + } + + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + if (out == NULL) + elog(ERROR, "Cannot open destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + elog(VERBOSE, "ftruncate file \"%s\" to size %lu", + to_fullpath, file->size); + if (fio_ftruncate(out, file->size) == -1) + elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + to_fullpath, file->size, strerror(errno)); + + if (!fio_is_remote_file(out)) + { + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + } + + while (blknum < file->n_blocks) + { + PageState page_st; + int rc = prepare_page(file, sync_lsn, + blknum, in, backup_mode, curr_page, + true, checksum_version, + ptrack_version_num, ptrack_schema, + from_fullpath, &page_st); + if (rc == PageIsTruncated) + break; + + else if (rc == PageIsOk) + { + if (fio_fseek(out, blknum * BLCKSZ) < 0) + { + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + copy_page(file, blknum, in, out, curr_page, to_fullpath); + } + + n_blocks_read++; + + /* next block */ + if (use_pagemap) + { + /* exit if pagemap is exhausted */ + if (!datapagemap_next(iter, &blknum)) + break; + } + else + blknum++; + } + + /* cleanup */ + if (in && fclose(in)) + elog(ERROR, "Cannot close the source file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* close local output file */ + if (out && fio_fclose(out)) + elog(ERROR, "Cannot close the destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + pg_free(iter); + pg_free(in_buf); + pg_free(out_buf); + + return n_blocks_read; +} + /* * Attempt to open header file, read content and return as * array of headers. diff --git a/src/dir.c b/src/dir.c index ce255d0ad..473534c8b 100644 --- a/src/dir.c +++ b/src/dir.c @@ -485,6 +485,13 @@ pgFileCompareSize(const void *f1, const void *f2) return 0; } +/* Compare two pgFile with their size in descending order */ +int +pgFileCompareSizeDesc(const void *f1, const void *f2) +{ + return -1 * pgFileCompareSize(f1, f2); +} + static int pgCompareString(const void *str1, const void *str2) { @@ -887,7 +894,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, * * Copy of function get_tablespace_mapping() from pg_basebackup.c. */ -static const char * +const char * get_tablespace_mapping(const char *dir) { TablespaceListCell *cell; diff --git a/src/help.c b/src/help.c index e1c8d6833..921feaec0 100644 --- a/src/help.c +++ b/src/help.c @@ -2,7 +2,7 @@ * * help.c * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -29,6 +29,7 @@ static void help_archive_get(void); static void help_checkdb(void); static void help_help(void); static void help_version(void); +static void help_catchup(void); void help_print_version(void) @@ -70,6 +71,7 @@ help_command(ProbackupSubcmd const subcmd) &help_internal, // AGENT_CMD &help_help, &help_version, + &help_catchup, }; Assert((int)subcmd < sizeof(help_functions) / sizeof(help_functions[0])); @@ -246,6 +248,19 @@ help_pg_probackup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); + printf(_("\n %s catchup -b catchup-mode\n"), PROGRAM_NAME); + printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); + printf(_(" --destination-pgdata=path_to_local_dir\n")); + printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [-j num-threads]\n")); + printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n")); + if ((PROGRAM_URL || PROGRAM_EMAIL)) { printf("\n"); @@ -1009,3 +1024,49 @@ help_version(void) printf(_("\n%s version\n"), PROGRAM_NAME); printf(_("%s --version\n\n"), PROGRAM_NAME); } + +static void +help_catchup(void) +{ + printf(_("\n%s catchup -b catchup-mode\n"), PROGRAM_NAME); + printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); + printf(_(" --destination-pgdata=path_to_local_dir\n")); + printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [-j num-threads]\n")); + printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--help]\n\n")); + + printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n")); + printf(_(" --stream stream the transaction log (only supported mode)\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --temp-slot use temporary replication slot\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); +} diff --git a/src/init.c b/src/init.c index dc821325a..a4911cb5c 100644 --- a/src/init.c +++ b/src/init.c @@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata); + instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 3150900b6..00796be04 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -88,6 +88,9 @@ bool backup_logs = false; bool smooth_checkpoint; char *remote_agent; static char *backup_note = NULL; +/* catchup options */ +static char *catchup_source_pgdata = NULL; +static char *catchup_destination_pgdata = NULL; /* restore options */ static char *target_time = NULL; static char *target_xid = NULL; @@ -201,6 +204,9 @@ static ConfigOption cmd_options[] = { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, + /* catchup options */ + { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, + { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -445,11 +451,12 @@ main(int argc, char *argv[]) catalogState->catalog_path, WAL_SUBDIR); } - /* backup_path is required for all pg_probackup commands except help, version and checkdb */ + /* backup_path is required for all pg_probackup commands except help, version, checkdb and catchup */ if (backup_path == NULL && backup_subcmd != CHECKDB_CMD && backup_subcmd != HELP_CMD && - backup_subcmd != VERSION_CMD) + backup_subcmd != VERSION_CMD && + backup_subcmd != CATCHUP_CMD) elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); /* ===== catalogState (END) ======*/ @@ -458,12 +465,12 @@ main(int argc, char *argv[]) /* * Option --instance is required for all commands except - * init, show, checkdb and validate + * init, show, checkdb, validate and catchup */ if (instance_name == NULL) { if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) + backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD && backup_subcmd != CATCHUP_CMD) elog(ERROR, "required parameter not specified: --instance"); } else @@ -545,6 +552,10 @@ main(int argc, char *argv[]) setMyLocation(backup_subcmd); } } + else if (backup_subcmd == CATCHUP_CMD) + { + config_get_opt_env(instance_options); + } /* * Disable logging into file for archive-push and archive-get. @@ -587,6 +598,13 @@ main(int argc, char *argv[]) "You must specify --log-directory option when running checkdb with " "--log-level-file option enabled."); + if (backup_subcmd == CATCHUP_CMD && + instance_config.logger.log_level_file != LOG_OFF && + instance_config.logger.log_directory == NULL) + elog(ERROR, "Cannot save catchup logs to a file. " + "You must specify --log-directory option when running catchup with " + "--log-level-file option enabled."); + /* Initialize logger */ init_logger(backup_path, &instance_config.logger); @@ -745,6 +763,25 @@ main(int argc, char *argv[]) } } + /* checking required options */ + if (backup_subcmd == CATCHUP_CMD) + { + if (catchup_source_pgdata == NULL) + elog(ERROR, "You must specify \"--source-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (catchup_destination_pgdata == NULL) + elog(ERROR, "You must specify \"--destination-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "Required parameter not specified: BACKUP_MODE (-b, --backup-mode)"); + if (current.backup_mode != BACKUP_MODE_FULL && current.backup_mode != BACKUP_MODE_DIFF_PTRACK && current.backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(ERROR, "Only \"FULL\", \"PTRACK\" and \"DELTA\" modes are supported with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (!stream_wal) + elog(INFO, "--stream is required, forcing stream mode"); + current.stream = stream_wal = true; + if (instance_config.external_dir_str) + elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); + // TODO проверить instance_config.conn_opt + } + /* sanity */ if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", @@ -787,6 +824,8 @@ main(int argc, char *argv[]) return do_backup(instanceState, set_backup_params, no_validate, no_sync, backup_logs); } + case CATCHUP_CMD: + return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync); case RESTORE_CMD: return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index ccbf803fd..1cad526dd 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -17,6 +17,7 @@ #include "access/xlog_internal.h" #include "utils/pg_crc.h" +#include "catalog/pg_control.h" #if PG_VERSION_NUM >= 120000 #include "common/logging.h" @@ -420,7 +421,7 @@ typedef struct PGNodeInfo char server_version_str[100]; int ptrack_version_num; - bool is_ptrack_enable; + bool is_ptrack_enabled; const char *ptrack_schema; /* used only for ptrack 2.x */ } PGNodeInfo; @@ -840,13 +841,16 @@ extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); +/* in catchup.c */ +extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files); + /* in restore.c */ extern int do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pgRecoveryTarget *rt, pgRestoreParams *params, bool no_sync); -extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); extern pgRecoveryTarget *parseRecoveryTargetOptions( @@ -861,6 +865,8 @@ extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); +extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, + IncrRestoreMode incremental_mode); /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); @@ -1002,6 +1008,7 @@ extern void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location); +extern const char *get_tablespace_mapping(const char *dir); extern void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, @@ -1054,6 +1061,7 @@ extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); +extern int pgFileCompareSizeDesc(const void *f1, const void *f2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); @@ -1061,6 +1069,12 @@ extern void pfilearray_clear_locks(parray *file_list); extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, const char *from_fullpath, uint32 checksum_version); + +extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + int ptrack_version_num, const char *ptrack_schema, + bool is_merge, size_t prev_size); extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, @@ -1129,14 +1143,15 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(bool safe); +extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path); +extern uint64 get_system_identifier(const char *pgdata_path, fio_location location); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern uint32 get_xlog_seg_size(char *pgdata_path); -extern void get_redo(const char *pgdata_path, RedoParams *redo); +extern DBState get_system_dbstate(const char *pgdata_path, fio_location location); +extern uint32 get_xlog_seg_size(const char *pgdata_path); +extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, @@ -1161,7 +1176,7 @@ extern void pretty_size(int64 size, char *buf, size_t len); extern void pretty_time_interval(double time, char *buf, size_t len); extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); -extern void check_system_identifiers(PGconn *conn, char *pgdata); +extern void check_system_identifiers(PGconn *conn, const char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ @@ -1170,7 +1185,8 @@ extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, int ptrack_version_num, XLogRecPtr lsn); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); +extern bool pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num); + extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); @@ -1182,6 +1198,10 @@ extern int send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); +extern int copy_pages(const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + uint32 checksum_version, bool use_pagemap, + BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); /* FIO */ extern void setMyLocation(ProbackupSubcmd const subcmd); @@ -1190,6 +1210,10 @@ extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pg XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber *err_blknum, char **errormsg, BackupPageHeader2 **headers); +extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber *err_blknum, char **errormsg, + BackupPageHeader2 **headers); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, @@ -1243,6 +1267,7 @@ extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, XLogRecPtr startpos, TimeLineID starttli); extern int wait_WAL_streaming_end(parray *backup_files_list); +extern parray* parse_tli_history_buffer(char *history, TimeLineID tli); /* external variables and functions, implemented in backup.c */ typedef struct PGStopBackupResult @@ -1280,5 +1305,6 @@ extern XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr lsn, bool bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir); extern void wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup); +extern int64 calculate_datasize_of_filelist(parray *filelist); #endif /* PG_PROBACKUP_H */ diff --git a/src/ptrack.c b/src/ptrack.c index 6825686c6..c631d7386 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -118,7 +118,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) +pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; bool result = false; diff --git a/src/restore.c b/src/restore.c index 7f5df1a00..005984aed 100644 --- a/src/restore.c +++ b/src/restore.c @@ -67,8 +67,6 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, const char *pgdata_path, bool no_sync, bool cleanup_pgdata, bool backup_has_tblspc); -static DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -293,7 +291,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (!timelines) elog(ERROR, "Failed to get history file for target timeline %i", rt->target_tli); - if (!satisfy_timeline(timelines, current_backup)) + if (!satisfy_timeline(timelines, current_backup->tli, current_backup->stop_lsn)) { if (target_backup_id != INVALID_BACKUP_ID) elog(ERROR, "target backup %s does not satisfy target timeline", @@ -487,7 +485,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg { RedoParams redo; parray *timelines = NULL; - get_redo(instance_config.pgdata, &redo); + get_redo(instance_config.pgdata, FIO_DB_HOST, &redo); if (redo.checksum_version == 0) elog(ERROR, "Incremental restore in 'lsn' mode require " @@ -1819,7 +1817,7 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) /* TODO description */ bool -satisfy_timeline(const parray *timelines, const pgBackup *backup) +satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn) { int i; @@ -1828,9 +1826,9 @@ satisfy_timeline(const parray *timelines, const pgBackup *backup) TimeLineHistoryEntry *timeline; timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); - if (backup->tli == timeline->tli && + if (tli == timeline->tli && (XLogRecPtrIsInvalid(timeline->end) || - backup->stop_lsn <= timeline->end)) + lsn <= timeline->end)) return true; } return false; @@ -2186,9 +2184,9 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, * data files content, because based on pg_control information we will * choose a backup suitable for lsn based incremental restore. */ - elog(INFO, "Trying to read pg_control file in destination direstory"); + elog(INFO, "Trying to read pg_control file in destination directory"); - system_id_pgdata = get_system_identifier(pgdata); + system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST); if (system_id_pgdata == instance_config.system_identifier) system_id_match = true; diff --git a/src/stream.c b/src/stream.c index 01161f720..615d25281 100644 --- a/src/stream.c +++ b/src/stream.c @@ -70,7 +70,6 @@ static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, uint32 xlog_seg_size); static void add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir); -static parray* parse_tli_history_buffer(char *history, TimeLineID tli); /* * Run IDENTIFY_SYSTEM through a given connection and @@ -173,7 +172,7 @@ StreamLog(void *arg) */ stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; - xlog_files_list = parray_new(); + xlog_files_list = parray_new(); /* Initialize timeout */ stream_stop_begin = 0; @@ -308,14 +307,14 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) /* we assume that we get called once at the end of each segment */ if (segment_finished) - { - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), - (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + { + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - add_walsegment_to_filelist(xlog_files_list, timeline, xlogpos, - (char*) stream_thread_arg.basedir, - instance_config.xlog_seg_size); - } + add_walsegment_to_filelist(xlog_files_list, timeline, xlogpos, + (char*) stream_thread_arg.basedir, + instance_config.xlog_seg_size); + } /* * Note that we report the previous, not current, position here. After a @@ -588,20 +587,25 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption /* Set error exit code as default */ stream_thread_arg.ret = 1; /* we must use startpos as start_lsn from start_backup */ - stream_thread_arg.startpos = current.start_lsn; - stream_thread_arg.starttli = current.tli; + stream_thread_arg.startpos = startpos; + stream_thread_arg.starttli = starttli; thread_interrupted = false; pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); } -/* Wait for the completion of stream */ +/* + * Wait for the completion of stream + * append list of streamed xlog files + * into backup_files_list (if it is not NULL) + */ int wait_WAL_streaming_end(parray *backup_files_list) { pthread_join(stream_thread, NULL); - parray_concat(backup_files_list, xlog_files_list); + if(backup_files_list != NULL) + parray_concat(backup_files_list, xlog_files_list); parray_free(xlog_files_list); return stream_thread_arg.ret; } diff --git a/src/util.c b/src/util.c index c0a1dc9e4..4e32e0639 100644 --- a/src/util.c +++ b/src/util.c @@ -10,8 +10,6 @@ #include "pg_probackup.h" -#include "catalog/pg_control.h" - #include #include @@ -174,7 +172,7 @@ get_current_timeline(PGconn *conn) if (PQresultStatus(res) == PGRES_TUPLES_OK) val = PQgetvalue(res, 0, 0); else - return get_current_timeline_from_control(false); + return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); if (!parse_uint32(val, &tli, 0)) { @@ -182,7 +180,7 @@ get_current_timeline(PGconn *conn) elog(WARNING, "Invalid value of timeline_id %s", val); /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(false); + return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); } return tli; @@ -190,15 +188,15 @@ get_current_timeline(PGconn *conn) /* Get timeline from pg_control file */ TimeLineID -get_current_timeline_from_control(bool safe) +get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, - safe, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, + safe, location); if (safe && buffer == NULL) return 0; @@ -249,14 +247,14 @@ get_checkpoint_location(PGconn *conn) } uint64 -get_system_identifier(const char *pgdata_path) +get_system_identifier(const char *pgdata_path, fio_location location) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -299,7 +297,7 @@ get_remote_system_identifier(PGconn *conn) } uint32 -get_xlog_seg_size(char *pgdata_path) +get_xlog_seg_size(const char *pgdata_path) { #if PG_VERSION_NUM >= 110000 ControlFileData ControlFile; @@ -351,15 +349,31 @@ get_pgcontrol_checksum(const char *pgdata_path) return ControlFile.crc; } +DBState +get_system_dbstate(const char *pgdata_path, fio_location location) +{ + ControlFileData ControlFile; + char *buffer; + size_t size; + + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); + if (buffer == NULL) + return 0; + digestControlFile(&ControlFile, buffer, size); + pg_free(buffer); + + return ControlFile.state; +} + void -get_redo(const char *pgdata_path, RedoParams *redo) +get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index afc1bc056..04bfbbe3b 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -110,6 +110,7 @@ static char const * const subcmd_names[] = "agent", "help", "version", + "catchup", }; ProbackupSubcmd diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 4ed4e0e61..3a5de4b83 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -38,7 +38,8 @@ typedef enum ProbackupSubcmd SSH_CMD, AGENT_CMD, HELP_CMD, - VERSION_CMD + VERSION_CMD, + CATCHUP_CMD, } ProbackupSubcmd; typedef enum OptionSource diff --git a/src/utils/file.c b/src/utils/file.c index e9792dd9c..b808d6293 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -94,7 +94,7 @@ setMyLocation(ProbackupSubcmd const subcmd) MyLocation = IsSshProtocol() ? (subcmd == ARCHIVE_PUSH_CMD || subcmd == ARCHIVE_GET_CMD) ? FIO_DB_HOST - : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD) + : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD || subcmd == CATCHUP_CMD) ? FIO_BACKUP_HOST : FIO_LOCAL_HOST : FIO_LOCAL_HOST; @@ -1139,6 +1139,46 @@ fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location lo } } +/* + * Read value of a symbolic link + * this is a wrapper about readlink() syscall + * side effects: string truncation occur (and it + * can be checked by caller by comparing + * returned value >= valsiz) + */ +ssize_t +fio_readlink(const char *path, char *value, size_t valsiz, fio_location location) +{ + if (!fio_is_remote(location)) + { + /* readlink don't place trailing \0 */ + ssize_t len = readlink(path, value, valsiz); + value[len < valsiz ? len : valsiz] = '\0'; + return len; + } + else + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + + hdr.cop = FIO_READLINK; + hdr.handle = -1; + Assert(valsiz <= UINT_MAX); /* max value of fio_header.arg */ + hdr.arg = valsiz; + hdr.size = path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_READLINK); + Assert(hdr.size <= valsiz); + IO_CHECK(fio_read_all(fio_stdin, value, hdr.size), hdr.size); + value[hdr.size < valsiz ? hdr.size : valsiz] = '\0'; + return hdr.size; + } +} + /* Check presence of the file */ int fio_access(char const* path, int mode, fio_location location) @@ -1769,7 +1809,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, /* send message with header - 8bytes 24bytes var var + 16bytes 24bytes var var -------------------------------------------------------------- | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | -------------------------------------------------------------- @@ -1903,6 +1943,198 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, return n_blocks_read; } +/* + * Return number of actually(!) readed blocks, attempts or + * half-readed block are not counted. + * Return values in case of error: + * FILE_MISSING + * OPEN_FAILED + * READ_ERROR + * PAGE_CORRUPTION + * WRITE_FAILED + * + * If none of the above, this function return number of blocks + * readed by remote agent. + * + * In case of DELTA mode horizonLsn must be a valid lsn, + * otherwise it should be set to InvalidXLogRecPtr. + * Взято из fio_send_pages + */ +int +fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber* err_blknum, char **errormsg, + BackupPageHeader2 **headers) +{ + FILE *out = NULL; + char *out_buf = NULL; + struct { + fio_header hdr; + fio_send_request arg; + } req; + BlockNumber n_blocks_read = 0; + BlockNumber blknum = 0; + + /* send message with header + + 16bytes 24bytes var var + -------------------------------------------------------------- + | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | + -------------------------------------------------------------- + */ + + req.hdr.cop = FIO_SEND_PAGES; + + if (use_pagemap) + { + req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; + req.arg.bitmapsize = (*file).pagemap.bitmapsize; + + /* TODO: add optimization for the case of pagemap + * containing small number of blocks with big serial numbers: + * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 + */ + } + else + { + req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; + req.arg.bitmapsize = 0; + } + + req.arg.nblocks = file->size/BLCKSZ; + req.arg.segmentno = file->segno * RELSEG_SIZE; + req.arg.horizonLsn = horizonLsn; + req.arg.checksumVersion = checksum_version; + req.arg.calg = calg; + req.arg.clevel = clevel; + req.arg.path_len = strlen(from_fullpath) + 1; + + file->compress_alg = calg; /* TODO: wtf? why here? */ + +//<----- +// datapagemap_iterator_t *iter; +// BlockNumber blkno; +// iter = datapagemap_iterate(pagemap); +// while (datapagemap_next(iter, &blkno)) +// elog(INFO, "block %u", blkno); +// pg_free(iter); +//<----- + + /* send header */ + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + /* send file path */ + IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); + + /* send pagemap if any */ + if (use_pagemap) + IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); + + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + if (out == NULL) + elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); + + /* update file permission */ + if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + elog(VERBOSE, "ftruncate file \"%s\" to size %lu", + to_fullpath, file->size); + if (fio_ftruncate(out, file->size) == -1) + elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + to_fullpath, file->size, strerror(errno)); + + if (!fio_is_remote_file(out)) + { + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + } + + while (true) + { + fio_header hdr; + char buf[BLCKSZ + sizeof(BackupPageHeader)]; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (interrupted) + elog(ERROR, "Interrupted during page reading"); + + if (hdr.cop == FIO_ERROR) + { + /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); + } + + return hdr.arg; + } + else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) + { + *err_blknum = hdr.arg; + + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); + } + return PAGE_CORRUPTION; + } + else if (hdr.cop == FIO_SEND_FILE_EOF) + { + /* n_blocks_read reported by EOF */ + n_blocks_read = hdr.arg; + + /* receive headers if any */ + if (hdr.size > 0) + { + *headers = pgut_malloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); + file->n_headers = (hdr.size / sizeof(BackupPageHeader2)) -1; + } + + break; + } + else if (hdr.cop == FIO_PAGE) + { + blknum = hdr.arg; + + Assert(hdr.size <= sizeof(buf)); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + + if (fio_fseek(out, blknum * BLCKSZ) < 0) + { + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + // должен прилетать некомпрессированный блок с заголовком + // Вставить assert? + if (fio_fwrite(out, buf + sizeof(BackupPageHeader), hdr.size - sizeof(BackupPageHeader)) != BLCKSZ) + { + fio_fclose(out); + *err_blknum = blknum; + return WRITE_FAILED; + } + file->write_size += BLCKSZ; + file->uncompressed_size += BLCKSZ; + } + else + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + } + + if (out) + fclose(out); + pg_free(out_buf); + + return n_blocks_read; +} + /* TODO: read file using large buffer * Return codes: * FIO_ERROR: @@ -3147,6 +3379,26 @@ fio_communicate(int in, int out) case FIO_GET_ASYNC_ERROR: fio_get_async_error_impl(out); break; + case FIO_READLINK: /* Read content of a symbolic link */ + { + /* + * We need a buf for a arguments and for a result at the same time + * hdr.size = strlen(symlink_name) + 1 + * hdr.arg = bufsize for a answer (symlink content) + */ + size_t filename_size = (size_t)hdr.size; + if (filename_size + hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); + } + rc = readlink(buf, buf + filename_size, hdr.arg); + hdr.cop = FIO_READLINK; + hdr.size = rc > 0 ? rc : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf + filename_size, hdr.size), hdr.size); + } + break; default: Assert(false); } diff --git a/src/utils/file.h b/src/utils/file.h index ad65b9901..edb5ea0f9 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -55,7 +55,8 @@ typedef enum FIO_LIST_DIR, FIO_CHECK_POSTMASTER, FIO_GET_ASYNC_ERROR, - FIO_WRITE_ASYNC + FIO_WRITE_ASYNC, + FIO_READLINK } fio_operations; typedef enum @@ -128,6 +129,7 @@ extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); extern int fio_access(char const* path, int mode, fio_location location); extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); +extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); extern DIR* fio_opendir(char const* path, fio_location location); extern struct dirent * fio_readdir(DIR *dirp); extern int fio_closedir(DIR *dirp); diff --git a/src/utils/parray.c b/src/utils/parray.c index 95b83365d..792e26907 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -198,6 +198,13 @@ parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const return bsearch(&key, array->data, array->used, sizeof(void *), compare); } +int +parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + void **elem = parray_bsearch(array, key, compare); + return elem != NULL ? elem - array->data : -1; +} + /* checks that parray contains element */ bool parray_contains(parray *array, void *elem) { diff --git a/src/utils/parray.h b/src/utils/parray.h index 85d7383f3..e92ad728c 100644 --- a/src/utils/parray.h +++ b/src/utils/parray.h @@ -29,6 +29,7 @@ extern bool parray_rm(parray *array, const void *key, int(*compare)(const void * extern size_t parray_num(const parray *array); extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern int parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)); extern void parray_walk(parray *array, void (*action)(void *)); extern bool parray_contains(parray *array, void *elem); diff --git a/tests/__init__.py b/tests/__init__.py index dbf84feea..080512760 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,8 @@ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external, config, checkdb, set_backup, incr_restore + locking, remote, external, config, checkdb, set_backup, incr_restore, \ + catchup def load_tests(loader, tests, pattern): @@ -23,6 +24,7 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) + suite.addTests(loader.loadTestsFromModule(catchup)) suite.addTests(loader.loadTestsFromModule(compatibility)) suite.addTests(loader.loadTestsFromModule(checkdb)) suite.addTests(loader.loadTestsFromModule(config)) diff --git a/tests/catchup.py b/tests/catchup.py new file mode 100644 index 000000000..5df538e42 --- /dev/null +++ b/tests/catchup.py @@ -0,0 +1,977 @@ +import os +import signal +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'catchup' + +class CatchupTest(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + +######################################### +# Basic tests +######################################### + def test_basic_full_catchup(self): + """ + Test 'multithreaded basebackup' mode (aka FULL catchup) + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do full catchup + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_full_catchup_with_tablespace(self): + """ + Test tablespace transfers + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') + self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path) + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do full catchup with tablespace mapping + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # make changes in master tablespace + src_pg.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + src_pg.stop() + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_basic_delta_catchup(self): + """ + Test delta catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_basic_ptrack_catchup(self): + """ + Test ptrack catchup + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tli_delta_catchup(self): + """ + Test that we correctly follow timeline change with delta catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: promote source + src_pg.stop() + self.set_replica(dst_pg, src_pg) # fake replication + src_pg.slow_start(replica = True) + src_pg.promote() + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + src_pg.stop() + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tli_ptrack_catchup(self): + """ + Test that we correctly follow timeline change with ptrack catchup + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: promote source + src_pg.stop() + self.set_replica(dst_pg, src_pg) # fake replication + src_pg.slow_start(replica = True) + src_pg.promote() + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + src_pg.stop() + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + +######################################### +# Test various corner conditions +######################################### + def test_table_drop_with_delta(self): + """ + Test that dropped table in source will be dropped in delta catchup'ed instance too + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + # perform checkpoint twice to ensure, that datafile is actually deleted on filesystem + src_pg.safe_psql("postgres", "DROP TABLE ultimate_question") + src_pg.safe_psql("postgres", "CHECKPOINT") + src_pg.safe_psql("postgres", "CHECKPOINT") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_table_drop_with_ptrack(self): + """ + Test that dropped table in source will be dropped in ptrack catchup'ed instance too + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + # perform checkpoint twice to ensure, that datafile is actually deleted on filesystem + src_pg.safe_psql("postgres", "DROP TABLE ultimate_question") + src_pg.safe_psql("postgres", "CHECKPOINT") + src_pg.safe_psql("postgres", "CHECKPOINT") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tablefile_truncation_with_delta(self): + """ + Test that truncated table in source will be truncated in delta catchup'ed instance too + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE SEQUENCE t_seq; " + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 1024) i") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dest_options = {} + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.safe_psql("postgres", "DELETE FROM t_heap WHERE ctid >= '(11,0)'") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tablefile_truncation_with_ptrack(self): + """ + Test that truncated table in source will be truncated in ptrack catchup'ed instance too + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE SEQUENCE t_seq; " + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 1024) i") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dest_options = {} + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.safe_psql("postgres", "DELETE FROM t_heap WHERE ctid >= '(11,0)'") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + +######################################### +# Test reaction on user errors +######################################### + def test_local_tablespace_without_mapping(self): + """ + Test that we detect absence of needed --tablespace-mapping option + """ + if self.remote: + return unittest.skip('Skipped because this test tests local catchup error handling') + + src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'src')) + src_pg.slow_start() + + tblspace_path = self.get_tblspace_path(src_pg, 'tblspace') + self.create_tblspace_in_node( + src_pg, 'tblspace', + tblspc_path = tblspace_path) + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace AS SELECT 42 AS answer") + + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + ] + ) + self.assertEqual(1, 0, "Expecting Error because '-T' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Local catchup executed, but source database contains tablespace', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_running_dest_postmaster(self): + """ + Test that we detect running postmaster in destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + # leave running destination postmaster + # so don't call dst_pg.stop() + + # try delta catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because postmaster in destination is running.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Postmaster with pid ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_same_db_id(self): + """ + Test that we detect different id's of source and destination + """ + # preparation: + # source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + # destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + # fake destination + fake_dst_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'fake_dst')) + # fake source + fake_src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'fake_src')) + + # try delta catchup (src (with correct src conn), fake_dst) + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = fake_dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Database identifiers mismatch: ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try delta catchup (fake_src (with wrong src conn), dst) + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = fake_src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Database identifiers mismatch: ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_destination_dbstate(self): + """ + Test that we detect that destination pg is not cleanly shutdowned + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # try #1 + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination directory contains "backup_label" file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try #2 + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") + os.kill(dst_pg.pid, signal.SIGKILL) + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'must be stopped cleanly', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tli_destination_mismatch(self): + """ + Test that we detect TLI mismatch in destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + self.set_replica(src_pg, dst_pg) + dst_pg.slow_start(replica = True) + dst_pg.promote() + dst_pg.stop() + + # preparation 3: "useful" changes + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # try catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.stop() + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + except ProbackupException as e: + self.assertIn( + 'ERROR: Source is behind destination in timeline history', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_tli_source_mismatch(self): + """ + Test that we detect TLI mismatch in source history + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: fake source (promouted copy) + fake_src_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'fake_src')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = fake_src_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + fake_src_options = {} + fake_src_options['port'] = str(fake_src_pg.port) + self.set_auto_conf(fake_src_pg, fake_src_options) + self.set_replica(src_pg, fake_src_pg) + fake_src_pg.slow_start(replica = True) + fake_src_pg.promote() + self.switch_wal_segment(fake_src_pg) + fake_src_pg.safe_psql( + "postgres", + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 256) i") + self.switch_wal_segment(fake_src_pg) + fake_src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 'trash' AS garbage") + + # preparation 3: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 4: "useful" changes + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # try catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = fake_src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(fake_src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.stop() + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination is not in source timeline history', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + fake_src_pg.stop() + self.del_test_dir(module_name, self.fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index af27669b1..1de004250 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -345,14 +345,9 @@ def pg_config_version(self): # print('PGPROBACKUP_SSH_USER is not set') # exit(1) - def make_simple_node( + def make_empty_node( self, - base_dir=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}): - + base_dir=None): real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) os.makedirs(real_base_dir) @@ -361,6 +356,17 @@ def make_simple_node( # bound method slow_start() to 'node' class instance node.slow_start = slow_start.__get__(node) node.should_rm_dirs = True + return node + + def make_simple_node( + self, + base_dir=None, + set_replication=False, + ptrack_enable=False, + initdb_params=[], + pg_options={}): + + node = self.make_empty_node(base_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) @@ -1036,6 +1042,28 @@ def restore_node( return self.run_pb(cmd_list + options, gdb=gdb, old_binary=old_binary) + def catchup_node( + self, + backup_mode, source_pgdata, destination_node, + options = [] + ): + + cmd_list = [ + 'catchup', + '--backup-mode={0}'.format(backup_mode), + '--source-pgdata={0}'.format(source_pgdata), + '--destination-pgdata={0}'.format(destination_node.data_dir) + ] + if self.remote: + cmd_list += ['--remote-proto=ssh', '--remote-host=localhost'] + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(destination_node.logs_dir) + ] + + return self.run_pb(cmd_list + options) + def show_pb( self, backup_dir, instance=None, backup_id=None, options=[], as_text=False, as_json=True, old_binary=False, @@ -1736,10 +1764,10 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): ): fail = True error_message += '\nFile permissions mismatch:\n' - error_message += ' File_old: {0} Permissions: {1}\n'.format( + error_message += ' File_old: {0} Permissions: {1:o}\n'.format( os.path.join(original_pgdata['pgdata'], file), original_pgdata['files'][file]['mode']) - error_message += ' File_new: {0} Permissions: {1}\n'.format( + error_message += ' File_new: {0} Permissions: {1:o}\n'.format( os.path.join(restored_pgdata['pgdata'], file), restored_pgdata['files'][file]['mode']) From 07127b8eb5dad28b9dd48ff2a8c51321006a1958 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Mon, 21 Jun 2021 23:55:29 +0300 Subject: [PATCH 1762/2107] [Issue #400] some leftovers --- src/catchup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catchup.c b/src/catchup.c index f80a0f0f9..58ce13c10 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -52,7 +52,7 @@ catchup_collect_info(PGNodeInfo *source_node_info, const char *source_pgdata, co instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST); current.start_time = time(NULL); - StrNCpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); + strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); /* Do some compatibility checks and fill basic info about PG instance */ source_conn = pgdata_basic_setup(instance_config.conn_opt, source_node_info); From 57f871accce26046430e7ec3c54bdd7d13563907 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Thu, 15 Jul 2021 00:48:21 +0300 Subject: [PATCH 1763/2107] Version 2.4.17 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fca08bdac..c9792ba1f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -308,7 +308,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.16" +#define PROGRAM_VERSION "2.4.17" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20409 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 1330acb5a..5c86262a8 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.4.16 \ No newline at end of file +pg_probackup 2.4.17 \ No newline at end of file From f63faad5e77a3293f6510045da40ea75f69957e1 Mon Sep 17 00:00:00 2001 From: AndrewBille <83072690+AndrewBille@users.noreply.github.com> Date: Thu, 15 Jul 2021 14:44:00 +0700 Subject: [PATCH 1764/2107] tests: Run compatibility tests only if PGPROBACKUPBIN_OLD set (#408) --- tests/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index dbf84feea..5c8231ffb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -23,7 +23,8 @@ def load_tests(loader, tests, pattern): # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) - suite.addTests(loader.loadTestsFromModule(compatibility)) + if 'PGPROBACKUPBIN_OLD' in os.environ and os.environ['PGPROBACKUPBIN_OLD']: + suite.addTests(loader.loadTestsFromModule(compatibility)) suite.addTests(loader.loadTestsFromModule(checkdb)) suite.addTests(loader.loadTestsFromModule(config)) # suite.addTests(loader.loadTestsFromModule(cfs_backup)) From 817b79b00f5dfa45ab676c8acaf6da6e6e4c5762 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:25:32 +0300 Subject: [PATCH 1765/2107] Add compatibility with postgres master (upcoming PG-15). (#410) Upstream commit cda03cfed6b changed CreateReplicationSlot() signature --- src/stream.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index 01161f720..3b84236c3 100644 --- a/src/stream.c +++ b/src/stream.c @@ -185,7 +185,12 @@ StreamLog(void *arg) #endif -#if PG_VERSION_NUM >= 110000 +#if PG_VERSION_NUM >= 150000 + /* Create temp repslot */ + if (temp_slot) + CreateReplicationSlot(stream_arg->conn, replication_slot, + NULL, temp_slot, true, true, false, false); +#elif PG_VERSION_NUM >= 110000 /* Create temp repslot */ if (temp_slot) CreateReplicationSlot(stream_arg->conn, replication_slot, From 0f3ae09bfd3b648dc6617a5fea58882f9b1a2ca0 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 2 Aug 2021 23:39:08 +0300 Subject: [PATCH 1766/2107] add REL_14_STABLE and master PG branches to travis --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c66cf6439..6d98d8ca3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE - PG_VERSION=13 PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE - PG_VERSION=11 PG_BRANCH=REL_11_STABLE @@ -42,9 +43,11 @@ env: # - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica # - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention # - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore + - PG_VERSION=15 PG_BRANCH=master -#jobs: -# allow_failures: +jobs: + allow_failures: + - env: PG_BRANCH=master # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage From 4de1607e0805eef162d9164b3ecc128fd897c2ff Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Tue, 3 Aug 2021 13:20:07 +0300 Subject: [PATCH 1767/2107] "fix" unstable backup.BackupTest.test_backup_with_less_privileges_role (disable tests in archive mode from replica) (#414) --- tests/backup.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index d713263c3..60e70cc28 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2518,45 +2518,47 @@ def test_backup_with_less_privileges_role(self): replica.slow_start(replica=True) + # Archive backups from replica in this test are disabled, + # because WAL archiving on replica in idle DB in PostgreSQL is broken: + # replica will not archive the previous WAL until it receives new records in the next WAL file, + # this "lazy" archiving can be seen in src/backend/replication/walreceiver.c:XLogWalRcvWrite() + # (see !XLByteInSeg checking and XLogArchiveNotify() calling). + # # self.switch_wal_segment(node) - # self.switch_wal_segment(node) - - self.backup_node( - backup_dir, 'replica', replica, - datname='backupdb', options=['-U', 'backup']) + #self.backup_node( + # backup_dir, 'replica', replica, + # datname='backupdb', options=['-U', 'backup']) # stream full backup from replica self.backup_node( backup_dir, 'replica', replica, datname='backupdb', options=['--stream', '-U', 'backup']) -# self.switch_wal_segment(node) - # PAGE backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + #self.switch_wal_segment(node) + #self.backup_node( + # backup_dir, 'replica', replica, backup_type='page', + # datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) + #self.switch_wal_segment(node) + #self.backup_node( + # backup_dir, 'replica', replica, backup_type='delta', + # datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK backup from replica if self.ptrack: - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) + #self.switch_wal_segment(node) + #self.backup_node( + # backup_dir, 'replica', replica, backup_type='ptrack', + # datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) From 475513996eee38bc3747b1e120b4a0a4a464ae43 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Tue, 3 Aug 2021 18:54:21 +0300 Subject: [PATCH 1768/2107] travis: refine allow_failures condition (allow fail with postgresql master branch) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6d98d8ca3..b6b8fd217 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ env: jobs: allow_failures: - - env: PG_BRANCH=master + - if: env(PG_BRANCH) = master # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage From 8846e1997a7299aa9e11e7ea74805a3801258cf1 Mon Sep 17 00:00:00 2001 From: AndrewBille <83072690+AndrewBille@users.noreply.github.com> Date: Wed, 4 Aug 2021 01:59:49 +0700 Subject: [PATCH 1769/2107] Stabilizy tests. (#411) * tests: Introduced a new flag for tests -- PGPROBACKUP_GDB * tests: Do travis builds with CFLAGS="-O0" (stabilization of gdb tests) * tests: Run compatibility tests only if PGPROBACKUPBIN_OLD set * tests: Running some tests now depends on PGPROBACKUP_SSH_REMOTE --- tests/Readme.md | 5 +++ tests/archive.py | 11 +++--- tests/checkdb.py | 5 +++ tests/delta.py | 7 +++- tests/external.py | 12 +++++++ tests/helpers/ptrack_helpers.py | 18 +++++----- tests/pgpro2068.py | 5 +++ tests/replica.py | 20 +++++++++++ tests/restore.py | 16 +++++++-- tests/show.py | 59 +++++++++++---------------------- tests/snapfs.py | 3 +- tests/validate.py | 5 +++ travis/run_tests.sh | 2 +- 13 files changed, 109 insertions(+), 59 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index f8dd91db0..668552c94 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -30,6 +30,11 @@ Specify path to pg_probackup binary file. By default tests use = 12 for this test') diff --git a/tests/show.py b/tests/show.py index 2a13a768b..b1ebebf18 100644 --- a/tests/show.py +++ b/tests/show.py @@ -212,6 +212,9 @@ def test_corrupt_control_file(self): # @unittest.expectedFailure def test_corrupt_correctness(self): """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -232,12 +235,7 @@ def test_corrupt_correctness(self): output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - if self.remote: - backup_remote_id = self.backup_node(backup_dir, 'node', node) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node(backup_dir, 'node', node) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -260,13 +258,8 @@ def test_corrupt_correctness(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -290,13 +283,8 @@ def test_corrupt_correctness(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -318,6 +306,9 @@ def test_corrupt_correctness(self): # @unittest.expectedFailure def test_corrupt_correctness_1(self): """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -338,12 +329,7 @@ def test_corrupt_correctness_1(self): output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - if self.remote: - backup_remote_id = self.backup_node(backup_dir, 'node', node) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node(backup_dir, 'node', node) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -370,13 +356,8 @@ def test_corrupt_correctness_1(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -400,13 +381,8 @@ def test_corrupt_correctness_1(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -428,6 +404,9 @@ def test_corrupt_correctness_1(self): # @unittest.expectedFailure def test_corrupt_correctness_2(self): """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/snapfs.py b/tests/snapfs.py index a7f926c4c..991741952 100644 --- a/tests/snapfs.py +++ b/tests/snapfs.py @@ -10,9 +10,10 @@ class SnapFSTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_snapfs_simple(self): """standart backup modes with ARCHIVE WAL method""" + if not self.enterprise: + self.skipTest('This test must be run on enterprise') fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/validate.py b/tests/validate.py index c5cc80733..0b04d92fe 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1088,6 +1088,11 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ check that interrupt during validation is handled correctly """ + if not self.gdb: + self.skipTest( + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 635b9f422..325b89060 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -35,7 +35,7 @@ git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 # Compile and install Postgres echo "############### Compiling Postgres:" cd postgres # Go to postgres dir -./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests +CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests make -s -j$(nproc) install #make -s -j$(nproc) -C 'src/common' install #make -s -j$(nproc) -C 'src/port' install From 384cf6dcfd87060a0a705fe5ac647dc2d223555a Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 12 Aug 2021 14:50:07 +0300 Subject: [PATCH 1770/2107] CVE-2018-1058 fix (#415) * CVE-2018-1058 fix --- src/backup.c | 6 +- src/checkdb.c | 10 +-- src/ptrack.c | 8 +-- src/util.c | 2 +- src/utils/pgut.c | 30 ++++++++- tests/CVE_2018_1058.py | 143 +++++++++++++++++++++++++++++++++++++++++ tests/__init__.py | 4 +- 7 files changed, 188 insertions(+), 15 deletions(-) create mode 100644 tests/CVE_2018_1058.py diff --git a/src/backup.c b/src/backup.c index 83785c1cb..71fd9670e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -928,7 +928,7 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) nodeInfo->server_version_str, "9.6"); if (nodeInfo->pgpro_support) - res = pgut_execute(conn, "SELECT pgpro_edition()", 0, NULL); + res = pgut_execute(conn, "SELECT pg_catalog.pgpro_edition()", 0, NULL); /* * Check major version of connected PostgreSQL and major version of @@ -1120,7 +1120,7 @@ pgpro_support(PGconn *conn) PGresult *res; res = pgut_execute(conn, - "SELECT proname FROM pg_proc WHERE proname='pgpro_edition'", + "SELECT proname FROM pg_catalog.pg_proc WHERE proname='pgpro_edition'::name AND pronamespace='pg_catalog'::regnamespace::oid", 0, NULL); if (PQresultStatus(res) == PGRES_TUPLES_OK && @@ -1159,7 +1159,7 @@ get_database_map(PGconn *conn) */ res = pgut_execute_extended(conn, "SELECT oid, datname FROM pg_catalog.pg_database " - "WHERE datname NOT IN ('template1', 'template0')", + "WHERE datname NOT IN ('template1'::name, 'template0'::name)", 0, NULL, true, true); /* Don't error out, simply return NULL. See comment above. */ diff --git a/src/checkdb.c b/src/checkdb.c index 4ea1d0800..e3f2df538 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -357,10 +357,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, res = pgut_execute(db_conn, "SELECT " "extname, nspname, extversion " - "FROM pg_namespace n " - "JOIN pg_extension e " + "FROM pg_catalog.pg_namespace n " + "JOIN pg_catalog.pg_extension e " "ON n.oid=e.extnamespace " - "WHERE e.extname IN ('amcheck', 'amcheck_next') " + "WHERE e.extname IN ('amcheck'::name, 'amcheck_next'::name) " "ORDER BY extversion DESC " "LIMIT 1", 0, NULL); @@ -556,8 +556,8 @@ do_amcheck(ConnectionOptions conn_opt, PGconn *conn) res_db = pgut_execute(conn, "SELECT datname, oid, dattablespace " - "FROM pg_database " - "WHERE datname NOT IN ('template0', 'template1')", + "FROM pg_catalog.pg_database " + "WHERE datname NOT IN ('template0'::name, 'template1'::name)", 0, NULL); /* we don't need this connection anymore */ diff --git a/src/ptrack.c b/src/ptrack.c index 5a2b9f046..b5f3a88a6 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -169,7 +169,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) res_db = pgut_execute(backup_conn, "SELECT extnamespace::regnamespace, extversion " - "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'", + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'::name", 0, NULL); if (PQntuples(res_db) > 0) @@ -187,7 +187,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* ptrack 1.x is supported, save version */ PQclear(res_db); res_db = pgut_execute(backup_conn, - "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + "SELECT proname FROM pg_catalog.pg_proc WHERE proname='ptrack_version'::name", 0, NULL); if (PQntuples(res_db) == 0) @@ -285,7 +285,7 @@ pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) params[0] = palloc(64); params[1] = palloc(64); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", + res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_catalog.pg_database", 0, NULL); for(i = 0; i < PQntuples(res_db); i++) @@ -335,7 +335,7 @@ pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) sprintf(params[0], "%i", dbOid); res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", + "SELECT datname FROM pg_catalog.pg_database WHERE oid=$1", 1, (const char **) params); /* * If database is not found, it's not an error. diff --git a/src/util.c b/src/util.c index 9fd0114bb..1e540a974 100644 --- a/src/util.c +++ b/src/util.c @@ -169,7 +169,7 @@ get_current_timeline(PGconn *conn) char *val; res = pgut_execute_extended(conn, - "SELECT timeline_id FROM pg_control_checkpoint()", 0, NULL, true, true); + "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()", 0, NULL, true, true); if (PQresultStatus(res) == PGRES_TUPLES_OK) val = PQgetvalue(res, 0, 0); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 1d8845c23..e9f902c0e 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -20,6 +20,12 @@ #include "common/string.h" #endif +#if PG_VERSION_NUM >= 100000 +#include "common/connect.h" +#else +#include "fe_utils/connect.h" +#endif + #include #include "pgut.h" @@ -257,7 +263,7 @@ pgut_connect(const char *host, const char *port, pthread_lock(&atexit_callback_disconnect_mutex); pgut_atexit_push(pgut_disconnect_callback, conn); pthread_mutex_unlock(&atexit_callback_disconnect_mutex); - return conn; + break; } if (conn && PQconnectionNeedsPassword(conn) && prompt_password) @@ -279,6 +285,28 @@ pgut_connect(const char *host, const char *port, PQfinish(conn); return NULL; } + + /* + * Fix for CVE-2018-1058. This code was taken with small modification from + * src/bin/pg_basebackup/streamutil.c:GetConnection() + */ + if (dbname != NULL) + { + PGresult *res; + + res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(ERROR, "could not clear search_path: %s", + PQerrorMessage(conn)); + PQclear(res); + PQfinish(conn); + return NULL; + } + PQclear(res); + } + + return conn; } PGconn * diff --git a/tests/CVE_2018_1058.py b/tests/CVE_2018_1058.py new file mode 100644 index 000000000..3da41f116 --- /dev/null +++ b/tests/CVE_2018_1058.py @@ -0,0 +1,143 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'CVE-2018-1058' + +class CVE_2018_1058(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_default_search_path(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pgpro_edition() " + "RETURNS text " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_backup_modified_search_path(self): + """""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_control_checkpoint(OUT timeline_id integer, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_proc(OUT proname name, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertFalse( + 'pg_probackup vulnerable!' in log_content) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_basic_checkdb_modified_search_path(self): + """""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_database(OUT datname name, OUT oid oid, OUT dattablespace oid) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_database AS SELECT * FROM public.pg_database()") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_extension(OUT extname name, OUT extnamespace oid, OUT extversion text) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE FUNCTION public.pg_namespace(OUT oid oid, OUT nspname name) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_extension AS SELECT * FROM public.pg_extension();" + "CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();" + ) + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + self.assertEqual( + 1, 0, + "Expecting Error because amcheck{,_next} not installed\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/__init__.py b/tests/__init__.py index 5c8231ffb..3a297c45e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,8 @@ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external, config, checkdb, set_backup, incr_restore + locking, remote, external, config, checkdb, set_backup, incr_restore, \ + CVE_2018_1058 def load_tests(loader, tests, pattern): @@ -55,6 +56,7 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(snapfs)) suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(validate)) + suite.addTests(loader.loadTestsFromModule(CVE_2018_1058)) return suite From 002d7b53b982bf06567272c0a89f5cbeda1025d7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Fri, 13 Aug 2021 02:59:20 +0300 Subject: [PATCH 1771/2107] [PGPRO-5378] fix various tests (#420) * [PGPRO-5378] tests.replica.ReplicaTest.test_replica_archive_page_backup stabilization * Skip some tests on PG-9.5 (test_replica_switchover, test_replica_promote_archive_delta, test_replica_promote_archive_page, test_parent_choosing) * travis: Fix compatibility issues with GDB --- tests/replica.py | 43 ++++++++++++++++++++++++++++++++++++--- travis/Dockerfile.in | 1 + travis/docker-compose.yml | 19 +++++++++++++++-- travis/make_dockerfile.sh | 6 ++++++ travis/run_tests.sh | 6 ++++++ 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index 828305da7..383a4979a 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -21,7 +21,6 @@ def test_replica_switchover(self): over the course of several switchovers https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru """ - fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node1 = self.make_simple_node( @@ -29,6 +28,11 @@ def test_replica_switchover(self): set_replication=True, initdb_params=['--data-checksums']) + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) @@ -287,6 +291,16 @@ def test_replica_archive_page_backup(self): self.wait_until_replica_catch_with_master(master, replica) + master.pgbench_init(scale=5) + # Continuous making some changes on master, + # because WAL archiving on replica in idle DB in PostgreSQL is broken: + # replica will not archive the previous WAL until it receives new records in the next WAL file, + # this "lazy" archiving can be seen in src/backend/replication/walreceiver.c:XLogWalRcvWrite() + # (see !XLByteInSeg checking and XLogArchiveNotify() calling). + pgbench = master.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '3', '-c', '1', '--no-vacuum']) + backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -295,6 +309,9 @@ def test_replica_archive_page_backup(self): '--master-db=postgres', '--master-port={0}'.format(master.port)]) + pgbench.wait() + pgbench.stdout.close() + self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -317,8 +334,6 @@ def test_replica_archive_page_backup(self): # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal # to original data - master.pgbench_init(scale=5) - pgbench = master.pgbench( options=['-T', '30', '-c', '2', '--no-vacuum']) @@ -535,6 +550,11 @@ def test_replica_promote(self): start backup from replica, during backup promote replica check that backup is failed """ + if not self.gdb: + self.skipTest( + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -706,6 +726,7 @@ def test_replica_stop_lsn_null_offset(self): output) # Clean after yourself + gdb_checkpointer.kill() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -1085,6 +1106,7 @@ def test_replica_toast(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself + gdb_checkpointer.kill() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -1313,6 +1335,11 @@ def test_replica_promote_archive_delta(self): 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) self.set_config( @@ -1433,6 +1460,11 @@ def test_replica_promote_archive_page(self): 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) self.set_archiving(backup_dir, 'node', node1) @@ -1550,6 +1582,11 @@ def test_parent_choosing(self): set_replication=True, initdb_params=['--data-checksums']) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index a1f30d7f6..3e451e24f 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -10,6 +10,7 @@ RUN python3 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} +ENV PGPROBACKUP_GDB=${PGPROBACKUP_GDB} ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin # Make directories diff --git a/travis/docker-compose.yml b/travis/docker-compose.yml index 471ab779f..fc6545567 100644 --- a/travis/docker-compose.yml +++ b/travis/docker-compose.yml @@ -1,2 +1,17 @@ -tests: - build: . +version: "3.7" +services: + tests: + build: + context: . + + cap_add: + - SYS_PTRACE + + security_opt: + - seccomp=unconfined + + # don't work + #sysctls: + # kernel.yama.ptrace_scope: 0 + privileged: true + diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh index 3e6938bd9..2e8ccd5a3 100755 --- a/travis/make_dockerfile.sh +++ b/travis/make_dockerfile.sh @@ -14,12 +14,18 @@ if [ -z ${MODE+x} ]; then MODE=basic fi +if [ -z ${PGPROBACKUP_GDB+x} ]; then + PGPROBACKUP_GDB=ON +fi + echo PG_VERSION=${PG_VERSION} echo PG_BRANCH=${PG_BRANCH} echo MODE=${MODE} +echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} sed \ -e 's/${PG_VERSION}/'${PG_VERSION}/g \ -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ -e 's/${MODE}/'${MODE}/g \ + -e 's/${PGPROBACKUP_GDB}/'${PGPROBACKUP_GDB}/g \ Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 325b89060..488d8ee45 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -65,6 +65,12 @@ which pg_config echo "############### pg_config:" pg_config +# Show kernel parameters +echo "############### kernel params:" +cat /proc/sys/kernel/yama/ptrace_scope +sudo sysctl kernel.yama.ptrace_scope=0 +cat /proc/sys/kernel/yama/ptrace_scope + # Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) echo "############### Compiling and installing pg_probackup:" # make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install From c00ffe27906824b1110c86625ff4ef6a6fef35d0 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:05:42 +0300 Subject: [PATCH 1772/2107] catchup functionality expansion (#419) [PR #419] New command 'catchup' for synchronizing stale standby with master by transfering only changes blocks, or creating standby from scratch. Co-authored-by: Elena Indrupskaya Co-authored-by: Grigory Smolkin --- .travis.yml | 33 ++- doc/pgprobackup.xml | 179 +++++++++--- src/backup.c | 6 +- src/catchup.c | 301 ++++++++++++-------- src/data.c | 332 ++++++++++------------ src/dir.c | 39 ++- src/help.c | 14 +- src/merge.c | 2 +- src/pg_probackup.c | 73 +++-- src/pg_probackup.h | 34 ++- src/ptrack.c | 15 +- src/stream.c | 166 +++++++---- src/utils/configuration.h | 14 +- src/utils/file.c | 9 +- tests/backup.py | 13 +- tests/catchup.py | 581 +++++++++++++++++++++++++++++++++----- travis/Dockerfile.in | 1 + travis/make_dockerfile.sh | 6 + travis/run_tests.sh | 17 ++ 19 files changed, 1311 insertions(+), 524 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6b8fd217..873dd8f20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,24 +26,26 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE + - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_VERSION=13 + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_VERSION=13 + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_VERSION=12 + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_VERSION=11 - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention -# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore - - PG_VERSION=15 PG_BRANCH=master +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=archive +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=backup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=catchup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=compression +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=delta +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=locking +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=merge +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=page +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=ptrack +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=replica +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=retention +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=restore jobs: allow_failures: @@ -54,3 +56,4 @@ jobs: #branches: # only: # - master + diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index f7814c2d2..7178cb14c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3409,16 +3409,29 @@ pg_probackup delete -B backup_dir --instance - Cloning <productname>PostgreSQL</productname> Instance + Cloning and Synchronizing <productname>PostgreSQL</productname> Instance pg_probackup can create a copy of a PostgreSQL - instance directly, without using the backup catalog. This allows you - to add a new standby server in a parallel mode or to have a standby - server that has fallen behind catch up with master. + instance directly, without using the backup catalog. To do this, you can run the command. + It can be useful in the following cases: + + + To add a new standby server. + Usually, pg_basebackup + is used to create a copy of a PostgreSQL instance. If the data directory of the destination instance + is empty, the catchup command works similarly, but it can be faster if run in parallel mode. + + + To have a fallen-behind standby server catch up with master. + Under high write load, replicas may fail to replay WAL fast enough to keep up with master and hence may lag behind. + A usual solution to create a new replica and switch to it requires a lot of extra space and data transfer. The catchup + command allows you to update an existing replica much faster by bringing differences from master. + + - Cloning a PostgreSQL instance is different from other pg_probackup + catchup is different from other pg_probackup operations: @@ -3439,12 +3452,12 @@ pg_probackup delete -B backup_dir --instance - No SQL commands involving tablespaces, such as + DDL commands CREATE TABLESPACE/DROP TABLESPACE, - can be run simultaneously with catchup. + >DROP TABLESPACE + cannot be run simultaneously with catchup. @@ -3452,14 +3465,16 @@ pg_probackup delete -B backup_dir --instance catchup takes configuration files, such as postgresql.conf, postgresql.auto.conf, or pg_hba.conf, from the source server and overwrites them - on the target server. + on the target server. The option allows you to keep + the configuration files intact. - Before cloning a PostgreSQL instance, set up the source database server as follows: + To prepare for cloning/synchronizing a PostgreSQL instance, + set up the source instance server as follows: @@ -3481,9 +3496,10 @@ pg_probackup delete -B backup_dir --instance - To clone a PostgreSQL instance, ensure that the source - database server is running and accepting connections and - on the server with the destination database, run the following command: + Before cloning/synchronizing a PostgreSQL instance, ensure that the source + instance server is running and accepting connections. To clone/sync a PostgreSQL instance, + on the server with the destination instance, you can run + the command as follows: pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream [connection_options] [remote_options] @@ -3496,33 +3512,43 @@ pg_probackup catchup -b catchup-mode --source-pgdata= FULL — creates a full copy of the PostgreSQL instance. - The destination directory must be empty for this mode. + The data directory of the destination instance must be empty for this mode. DELTA — reads all data files in the data directory and creates an incremental copy for pages that have changed - since the destination database was shut down cleanly. - For this mode, the destination directory must contain a previous - copy of the database that was shut down cleanly. + since the destination instance was shut down. PTRACK — tracking page changes on the fly, - only copies pages that have changed since the point of divergence - of the source and destination databases. - For this mode, the destination directory must contain a previous - copy of the database that was shut down cleanly. + only reads and copies pages that have changed since the point of divergence + of the source and destination instances. + + + PTRACK catchup mode requires PTRACK + not earlier than 2.0 and hence, PostgreSQL not earlier than 11. + + + + By specifying the option, you can set + STREAM WAL delivery mode + of copying, which will include all the necessary WAL files by streaming them from + the instance server via replication protocol. + You can use connection_options to specify the connection to the source database cluster. If it is located on a different server, also specify remote_options. - If the source database contains tablespaces that must be located in + + + If the source database cluster contains tablespaces that must be located in a different directory, additionally specify the option: @@ -3538,8 +3564,9 @@ pg_probackup catchup -b catchup-mode --source-pgdata= For example, assume that a remote standby server with the PostgreSQL instance having /replica-pgdata data directory has fallen behind. To sync this instance with the one in /master-pgdata data directory, you can run the catchup command in the PTRACK mode on four parallel threads as follows: -pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=PTRACK --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 +pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=PTRACK --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 --exclude-path=postgresql.conf --exclude-path=postgresql.auto.conf --exclude-path=pg_hba.conf --exclude-path=pg_ident.conf + Note that in this example, the configuration files will not be overwritten during synchronization. Another example shows how you can add a new remote standby server with the PostgreSQL data directory /replica-pgdata by running the catchup command in the FULL mode @@ -4428,7 +4455,9 @@ pg_probackup archive-get -B backup_dir --instance catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir -[--help] [--stream] [-j num_threads] +[--help] [-j | --threads=num_threads] [--stream] +[--temp-slot] [-P | --perm-slot] [-S | --slot=slot_name] +[--exclude-path=PATHNAME] [-T OLDDIR=NEWDIR] [connection_options] [remote_options] @@ -4454,14 +4483,20 @@ pg_probackup catchup -b catchup_mode DELTA — reads all data files in the data directory and creates an incremental copy for pages that have changed - since the destination database was shut down cleanly. + since the destination instance was shut down. PTRACK — tracking page changes on the fly, - only copies pages that have changed since the point of divergence - of the source and destination databases. + only reads and copies pages that have changed since the point of divergence + of the source and destination instances. + + + PTRACK catchup mode requires PTRACK + not earlier than 2.0 and hence, PostgreSQL not earlier than 11. + + @@ -4487,24 +4522,98 @@ pg_probackup catchup -b catchup_mode + + + + + + Sets the number of parallel threads for + catchup process. + + + + - Makes a STREAM backup, which - includes all the necessary WAL files by streaming them from - the database server via replication protocol. + Copies the instance in STREAM WAL delivery mode, + including all the necessary WAL files by streaming them from + the instance server via replication protocol. - - +=path_prefix +=path_prefix - Sets the number of parallel threads for - catchup process. + Specifies a prefix for files to exclude from the synchronization of PostgreSQL + instances during copying. The prefix must contain a path relative to the data directory of an instance. + If the prefix specifies a directory, + all files in this directory will not be synchronized. + + + This option is dangerous since excluding files from synchronization can result in + incomplete synchronization; use with care. + + + + + + + + + + + Copies the instance in STREAM WAL delivery mode, + including all the necessary WAL files by streaming them from + the instance server via replication protocol. + + + + + + + + + Creates a temporary physical replication slot for streaming + WAL from the PostgreSQL instance being copied. It ensures that + all the required WAL segments remain available if WAL is + rotated while the backup is in progress. This flag can only be + used together with the flag and + cannot be used together with the flag. + The default slot name is pg_probackup_slot, + which can be changed using the / option. + + + + + + + + + + Creates a permanent physical replication slot for streaming + WAL from the PostgreSQL instance being copied. This flag can only be + used together with the flag and + cannot be used together with the flag. + The default slot name is pg_probackup_perm_slot, + which can be changed using the / option. + + + + + + + + + + Specifies the replication slot for WAL streaming. This option + can only be used together with the + flag. @@ -4533,7 +4642,7 @@ pg_probackup catchup -b catchup_mode For details on usage, see the section - Cloning PostgreSQL Instance. + Cloning and Synchronizing PostgreSQL Instance. diff --git a/src/backup.c b/src/backup.c index e9c8a22d1..1d08c3828 100644 --- a/src/backup.c +++ b/src/backup.c @@ -263,7 +263,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli); + current.start_lsn, current.tli, true); /* Make sure that WAL streaming is working * PAGE backup in stream mode is waited twice, first for @@ -2051,8 +2051,6 @@ backup_files(void *arg) instance_config.compress_alg, instance_config.compress_level, arguments->nodeInfo->checksum_version, - arguments->nodeInfo->ptrack_version_num, - arguments->nodeInfo->ptrack_schema, arguments->hdr_map, false); } else @@ -2350,7 +2348,7 @@ calculate_datasize_of_filelist(parray *filelist) { pgFile *file = (pgFile *) parray_get(filelist, i); - if (file->external_dir_num != 0) + if (file->external_dir_num != 0 || file->excluded) continue; if (S_ISDIR(file->mode)) diff --git a/src/catchup.c b/src/catchup.c index 58ce13c10..5a0c8e45a 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -27,20 +27,19 @@ /* * Catchup routines */ -static PGconn *catchup_collect_info(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); +static PGconn *catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); static void catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, const char *dest_pgdata); static void catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn); static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli); -//REVIEW The name of this function looks strange to me. -//Maybe catchup_init_state() or catchup_setup() will do better? -//I'd also suggest to wrap all these fields into some CatchupState, but it isn't urgent. +//REVIEW I'd also suggest to wrap all these fields into some CatchupState, but it isn't urgent. +//REVIEW_ANSWER what for? /* * Prepare for work: fill some globals, open connection to source database */ static PGconn * -catchup_collect_info(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) +catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) { PGconn *source_conn; @@ -159,17 +158,6 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); } - /* check that destination database is shutdowned cleanly */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - DBState state; - state = get_system_dbstate(dest_pgdata, FIO_LOCAL_HOST); - /* see states in postgres sources (src/include/catalog/pg_control.h) */ - if (state != DB_SHUTDOWNED && state != DB_SHUTDOWNED_IN_RECOVERY) - elog(ERROR, "Postmaster in destination directory \"%s\" must be stopped cleanly", - dest_pgdata); - } - /* Check that connected PG instance, source and destination PGDATA are the same */ { uint64 source_conn_id, source_id, dest_id; @@ -366,6 +354,7 @@ typedef struct XLogRecPtr sync_lsn; BackupMode backup_mode; int thread_num; + size_t transfered_bytes; bool completed; } catchup_thread_runner_arg; @@ -390,6 +379,9 @@ catchup_thread_runner(void *arg) if (S_ISDIR(file->mode)) continue; + if (file->excluded) + continue; + if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -431,12 +423,7 @@ catchup_thread_runner(void *arg) catchup_data_file(file, from_fullpath, to_fullpath, arguments->sync_lsn, arguments->backup_mode, - NONE_COMPRESS, - 0, arguments->nodeInfo->checksum_version, - arguments->nodeInfo->ptrack_version_num, - arguments->nodeInfo->ptrack_schema, - false, dest_file != NULL ? dest_file->size : 0); } else @@ -445,6 +432,7 @@ catchup_thread_runner(void *arg) arguments->backup_mode, current.parent_backup, true); } + /* file went missing during catchup */ if (file->write_size == FILE_NOT_FOUND) continue; @@ -454,6 +442,7 @@ catchup_thread_runner(void *arg) continue; } + arguments->transfered_bytes += file->write_size; elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", from_fullpath, file->write_size); } @@ -469,8 +458,10 @@ catchup_thread_runner(void *arg) /* * main multithreaded copier + * returns size of transfered data file + * or -1 in case of error */ -static bool +static ssize_t catchup_multithreaded_copy(int num_threads, PGNodeInfo *source_node_info, const char *source_pgdata_path, @@ -485,6 +476,7 @@ catchup_multithreaded_copy(int num_threads, pthread_t *threads; bool all_threads_successful = true; + ssize_t transfered_bytes_result = 0; int i; /* init thread args */ @@ -499,6 +491,7 @@ catchup_multithreaded_copy(int num_threads, .sync_lsn = sync_lsn, .backup_mode = backup_mode, .thread_num = i + 1, + .transfered_bytes = 0, .completed = false, }; @@ -516,15 +509,16 @@ catchup_multithreaded_copy(int num_threads, { pthread_join(threads[i], NULL); all_threads_successful &= threads_args[i].completed; + transfered_bytes_result += threads_args[i].transfered_bytes; } free(threads); free(threads_args); - return all_threads_successful; + return all_threads_successful ? transfered_bytes_result : -1; } /* - * + * Sync every file in destination directory to disk */ static void catchup_sync_destination_files(const char* pgdata_path, fio_location location, parray *filelist, pgFile *pg_control_file) @@ -541,8 +535,13 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p { pgFile *file = (pgFile *) parray_get(filelist, i); - /* TODO: sync directory ? */ - if (S_ISDIR(file->mode)) + /* TODO: sync directory ? + * - at first glance we can rely on fs journaling, + * which is enabled by default on most platforms + * - but PG itself is not relying on fs, its durable_sync + * includes directory sync + */ + if (S_ISDIR(file->mode) || file->excluded) continue; Assert(file->external_dir_num == 0); @@ -564,11 +563,50 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p elog(INFO, "Files are synced, time elapsed: %s", pretty_time); } +/* + * Filter filelist helper function (used to process --exclude-path's) + * filelist -- parray of pgFile *, can't be NULL + * exclude_absolute_paths_list -- sorted parray of char * (absolute paths, starting with '/'), can be NULL + * exclude_relative_paths_list -- sorted parray of char * (relative paths), can be NULL + * logging_string -- helper parameter, used for generating verbose log messages ("Source" or "Destination") + */ +static void +filter_filelist(parray *filelist, const char *pgdata, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list, + const char *logging_string) +{ + int i; + + if (exclude_absolute_paths_list == NULL && exclude_relative_paths_list == NULL) + return; + + for (i = 0; i < parray_num(filelist); ++i) + { + char full_path[MAXPGPATH]; + pgFile *file = (pgFile *) parray_get(filelist, i); + join_path_components(full_path, pgdata, file->rel_path); + + if ( + (exclude_absolute_paths_list != NULL + && parray_bsearch(exclude_absolute_paths_list, full_path, pgPrefixCompareString)!= NULL + ) || ( + exclude_relative_paths_list != NULL + && parray_bsearch(exclude_relative_paths_list, file->rel_path, pgPrefixCompareString)!= NULL) + ) + { + elog(LOG, "%s file \"%s\" excluded with --exclude-path option", logging_string, full_path); + file->excluded = true; + } + } +} + /* * Entry point of pg_probackup CATCHUP subcommand. + * exclude_*_paths_list are parray's of char * */ int -do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files) +do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list) { PGconn *source_conn = NULL; PGNodeInfo source_node_info; @@ -586,33 +624,27 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* for fancy reporting */ time_t start_time, end_time; - char pretty_time[20]; - char pretty_bytes[20]; + ssize_t transfered_datafiles_bytes = 0; + ssize_t transfered_walfiles_bytes = 0; + char pretty_source_bytes[20]; - source_conn = catchup_collect_info(&source_node_info, source_pgdata, dest_pgdata); + source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata); catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); - elog(LOG, "Database catchup start"); + /* we need to sort --exclude_path's for future searching */ + if (exclude_absolute_paths_list != NULL) + parray_qsort(exclude_absolute_paths_list, pgCompareString); + if (exclude_relative_paths_list != NULL) + parray_qsort(exclude_relative_paths_list, pgCompareString); - { - char label[1024]; - /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time, false); - strncat(label, " with pg_probackup", lengthof(label) - - strlen(" with pg_probackup")); - - /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); - elog(LOG, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); - } + elog(LOG, "Database catchup start"); - //REVIEW I wonder, if we can move this piece above and call before pg_start backup()? - //It seems to be a part of setup phase. if (current.backup_mode != BACKUP_MODE_FULL) { dest_filelist = parray_new(); dir_list_file(dest_filelist, dest_pgdata, true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination"); // fill dest_redo.lsn and dest_redo.tli get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); @@ -627,16 +659,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, */ } - //REVIEW I wonder, if we can move this piece above and call before pg_start backup()? - //It seems to be a part of setup phase. /* + * Make sure that sync point is withing ptrack tracking range * TODO: move to separate function to use in both backup.c and catchup.c */ if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, &source_node_info); - // new ptrack is more robust and checks Start LSN if (ptrack_lsn > dest_redo.lsn || ptrack_lsn == InvalidXLogRecPtr) elog(ERROR, "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" "You can perform only FULL catchup.", @@ -645,7 +675,19 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, (uint32) (dest_redo.lsn)); } - /* Check that dest_redo.lsn is less than current.start_lsn */ + { + char label[1024]; + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time, false); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + + /* Call pg_start_backup function in PostgreSQL connect */ + pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); + elog(LOG, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* Sanity: source cluster must be "in future" relatively to dest cluster */ if (current.backup_mode != BACKUP_MODE_FULL && dest_redo.lsn > current.start_lsn) elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " @@ -657,7 +699,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli); + current.start_lsn, current.tli, false); source_filelist = parray_new(); @@ -670,17 +712,16 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); //REVIEW FIXME. Let's fix that before release. - // TODO filter pg_xlog/wal? // TODO what if wal is not a dir (symlink to a dir)? + // - Currently backup/restore transform pg_wal symlink to directory + // so the problem is not only with catchup. + // if we want to make it right - we must provide the way + // for symlink remapping during restore and catchup. + // By default everything must be left as it is. /* close ssh session in main thread */ fio_disconnect(); - //REVIEW Do we want to do similar calculation for dest? - current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); - pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Source PGDATA size: %s", pretty_bytes); - /* * Sort pathname ascending. It is necessary to create intermediate * directories sequentially. @@ -694,8 +735,24 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, */ parray_qsort(source_filelist, pgFileCompareRelPathWithExternal); - /* Extract information about files in source_filelist parsing their names:*/ - parse_filelist_filenames(source_filelist, source_pgdata); + //REVIEW Do we want to do similar calculation for dest? + //REVIEW_ANSWER what for? + { + ssize_t source_bytes = 0; + char pretty_bytes[20]; + + source_bytes += calculate_datasize_of_filelist(source_filelist); + + /* Extract information about files in source_filelist parsing their names:*/ + parse_filelist_filenames(source_filelist, source_pgdata); + filter_filelist(source_filelist, source_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Source"); + + current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); + + pretty_size(current.pgdata_bytes, pretty_source_bytes, lengthof(pretty_source_bytes)); + pretty_size(source_bytes - current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); + elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, pretty_bytes); + } elog(LOG, "Start LSN (source): %X/%X, TLI: %X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), @@ -728,7 +785,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, * We iterate over source_filelist and for every directory with parent 'pg_tblspc' * we must lookup this directory name in tablespace map. * If we got a match, we treat this directory as tablespace. - * It means that we create directory specified in tablespace_map and + * It means that we create directory specified in tablespace map and * original directory created as symlink to it. */ for (i = 0; i < parray_num(source_filelist); i++) @@ -736,7 +793,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, pgFile *file = (pgFile *) parray_get(source_filelist, i); char parent_dir[MAXPGPATH]; - if (!S_ISDIR(file->mode)) + if (!S_ISDIR(file->mode) || file->excluded) continue; /* @@ -816,9 +873,22 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, source_pg_control_file = parray_remove(source_filelist, control_file_elem_index); } + /* TODO before public release: must be more careful with pg_control. + * when running catchup or incremental restore + * cluster is actually in two states + * simultaneously - old and new, so + * it must contain both pg_control files + * describing those states: global/pg_control_old, global/pg_control_new + * 1. This approach will provide us with means of + * robust detection of previos failures and thus correct operation retrying (or forbidding). + * 2. We will have the ability of preventing instance from starting + * in the middle of our operations. + */ + /* * remove absent source files in dest (dropped tables, etc...) * note: global/pg_control will also be deleted here + * mark dest files (that excluded with source --exclude-path) also for exclusion */ if (current.backup_mode != BACKUP_MODE_FULL) { @@ -828,33 +898,33 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, { bool redundant = true; pgFile *file = (pgFile *) parray_get(dest_filelist, i); + pgFile **src_file = NULL; //TODO optimize it and use some merge-like algorithm //instead of bsearch for each file. - if (parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal)) + src_file = (pgFile **) parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal); + + if (src_file!= NULL && !(*src_file)->excluded && file->excluded) + (*src_file)->excluded = true; + + if (src_file!= NULL || file->excluded) redundant = false; - /* pg_filenode.map are always restored, because it's crc cannot be trusted */ + /* pg_filenode.map are always copied, because it's crc cannot be trusted */ Assert(file->external_dir_num == 0); if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) redundant = true; - //REVIEW This check seems unneded. Anyway we delete only redundant stuff below. - /* do not delete the useful internal directories */ - if (S_ISDIR(file->mode) && !redundant) - continue; - /* if file does not exists in destination list, then we can safely unlink it */ if (redundant) { char fullpath[MAXPGPATH]; join_path_components(fullpath, dest_pgdata, file->rel_path); - fio_delete(file->mode, fullpath, FIO_DB_HOST); elog(VERBOSE, "Deleted file \"%s\"", fullpath); - /* shrink pgdata list */ + /* shrink dest pgdata list */ pgFileFree(file); parray_remove(dest_filelist, i); i--; @@ -875,10 +945,11 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* run copy threads */ elog(INFO, "Start transferring data files"); time(&start_time); - catchup_isok = catchup_multithreaded_copy(num_threads, &source_node_info, + transfered_datafiles_bytes = catchup_multithreaded_copy(num_threads, &source_node_info, source_pgdata, dest_pgdata, source_filelist, dest_filelist, dest_redo.lsn, current.backup_mode); + catchup_isok = transfered_datafiles_bytes != -1; /* at last copy control file */ if (catchup_isok) @@ -889,17 +960,22 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); + transfered_datafiles_bytes += source_pg_control_file->size; } - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), + if (!catchup_isok) + { + char pretty_time[20]; + char pretty_transfered_data_bytes[20]; + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); - if (catchup_isok) - elog(INFO, "Data files are transferred, time elapsed: %s", - pretty_time); - else - elog(ERROR, "Data files transferring failed, time elapsed: %s", - pretty_time); + pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); + + elog(ERROR, "Catchup failed. Transfered: %s, time elapsed: %s", + pretty_transfered_data_bytes, pretty_time); + } /* Notify end of backup */ { @@ -912,17 +988,6 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, pg_silent_client_messages(source_conn); - //REVIEW. Do we want to support pg 9.5? I suppose we never test it... - //Maybe check it and error out early? - /* Create restore point - * Only if backup is from master. - * For PG 9.5 create restore point only if pguser is superuser. - */ - if (!current.from_replica && - !(source_node_info.server_version < 90600 && - !source_node_info.is_superuser)) //TODO: check correctness - pg_create_restore_point(source_conn, current.start_time); - /* Execute pg_stop_backup using PostgreSQL connection */ pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, exclusive_backup, &stop_backup_query_text); @@ -965,22 +1030,23 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, } #endif - if(wait_WAL_streaming_end(NULL)) - elog(ERROR, "WAL streaming failed"); + /* wait for end of wal streaming and calculate wal size transfered */ + { + parray *wal_files_list = NULL; + wal_files_list = parray_new(); - //REVIEW Please add a comment about these lsns. It is a crutial part of the algorithm. - current.recovery_xid = stop_backup_result.snapshot_xid; + if (wait_WAL_streaming_end(wal_files_list)) + elog(ERROR, "WAL streaming failed"); - elog(LOG, "Getting the Recovery Time from WAL"); + for (i = 0; i < parray_num(wal_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(wal_files_list, i); + transfered_walfiles_bytes += file->size; + } - /* iterate over WAL from stop_backup lsn to start_backup lsn */ - if (!read_recovery_info(dest_xlog_path, current.tli, - instance_config.xlog_seg_size, - current.start_lsn, current.stop_lsn, - ¤t.recovery_time)) - { - elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); - current.recovery_time = stop_backup_result.invocation_time; + parray_walk(wal_files_list, pgFileFree); + parray_free(wal_files_list); + wal_files_list = NULL; } /* @@ -994,15 +1060,33 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* close ssh session in main thread */ fio_disconnect(); - /* Sync all copied files unless '--no-sync' flag is used */ - if (catchup_isok) + /* fancy reporting */ { - if (sync_dest_files) - catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); - else - elog(WARNING, "Files are not synced to disk"); + char pretty_transfered_data_bytes[20]; + char pretty_transfered_wal_bytes[20]; + char pretty_time[20]; + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); + pretty_size(transfered_walfiles_bytes, pretty_transfered_wal_bytes, lengthof(pretty_transfered_wal_bytes)); + + elog(INFO, "Databases synchronized. Transfered datafiles size: %s, transfered wal size: %s, time elapsed: %s", + pretty_transfered_data_bytes, pretty_transfered_wal_bytes, pretty_time); + + if (current.backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Catchup incremental ratio (less is better): %.f%% (%s/%s)", + ((float) transfered_datafiles_bytes / current.pgdata_bytes) * 100, + pretty_transfered_data_bytes, pretty_source_bytes); } + /* Sync all copied files unless '--no-sync' flag is used */ + if (sync_dest_files) + catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); + else + elog(WARNING, "Files are not synced to disk"); + /* Cleanup */ if (dest_filelist) { @@ -1013,8 +1097,5 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, parray_free(source_filelist); pgFileFree(source_pg_control_file); - //REVIEW: Are we going to do that before release? - /* TODO: show the amount of transfered data in bytes and calculate incremental ratio */ - return 0; } diff --git a/src/data.c b/src/data.c index 49b696059..f02e3fd14 100644 --- a/src/data.c +++ b/src/data.c @@ -28,10 +28,10 @@ typedef struct DataPage { BackupPageHeader bph; - char data[BLCKSZ]; + char data[BLCKSZ]; } DataPage; -static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, +static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, pg_crc32 *crc, bool use_crc32c); #ifdef HAVE_LIBZ @@ -40,9 +40,9 @@ static int32 zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, int level) { - uLongf compressed_size = dst_size; - int rc = compress2(dst, &compressed_size, src, src_size, - level); + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); return rc == Z_OK ? compressed_size : rc; } @@ -51,8 +51,8 @@ zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, static int32 zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) { - uLongf dest_len = dst_size; - int rc = uncompress(dst, &dest_len, src, src_size); + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); return rc == Z_OK ? dest_len : rc; } @@ -63,7 +63,7 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) * written in the destination buffer, or -1 if compression fails. */ int32 -do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, +do_compress(void *dst, size_t dst_size, void const *src, size_t src_size, CompressAlg alg, int level, const char **errormsg) { switch (alg) @@ -73,13 +73,13 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_compress(dst, dst_size, src, src_size, level); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } + { + int32 ret; + ret = zlib_compress(dst, dst_size, src, src_size, level); + if (ret < Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -93,25 +93,25 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, * decompressed in the destination buffer, or -1 if decompression fails. */ int32 -do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, +do_decompress(void *dst, size_t dst_size, void const *src, size_t src_size, CompressAlg alg, const char **errormsg) { switch (alg) { case NONE_COMPRESS: case NOT_DEFINED_COMPRESS: - if (errormsg) + if (errormsg) *errormsg = "Invalid compression algorithm"; return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_decompress(dst, dst_size, src, src_size); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } + { + int32 ret; + ret = zlib_decompress(dst, dst_size, src, src_size); + if (ret < Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: @@ -125,7 +125,6 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; } - #define ZLIB_MAGIC 0x78 /* @@ -162,7 +161,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) /* For zlib we can check page magic: * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like */ - if (alg == ZLIB_COMPRESS && *(char*)page != ZLIB_MAGIC) + if (alg == ZLIB_COMPRESS && *(char *)page != ZLIB_MAGIC) { return false; } @@ -281,8 +280,6 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, Page page, bool strict, uint32 checksum_version, - int ptrack_version_num, - const char *ptrack_schema, const char *from_fullpath, PageState *page_st) { @@ -404,8 +401,7 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, blknum, from_fullpath, file->exists_in_prev ? "true" : "false", (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, - (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn - ); + (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn); return SkipCurrentPage; } @@ -422,7 +418,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, { int compressed_size = 0; size_t write_buffer_size = 0; - char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ + char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ BackupPageHeader* bph = (BackupPageHeader*)write_buffer; const char *errormsg = NULL; @@ -463,16 +459,13 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, return compressed_size; } -/* взята из compress_and_backup_page, но выпилена вся магия заголовков и компрессии, просто копирование 1-в-1 */ +/* Write page as-is. TODO: make it fastpath option in compress_and_backup_page() */ static int -copy_page(pgFile *file, BlockNumber blknum, - FILE *in, FILE *out, Page page, - const char *to_fullpath) +write_page(pgFile *file, FILE *out, Page page) { /* write data page */ if (fio_fwrite(out, page, BLCKSZ) != BLCKSZ) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); + return -1; file->write_size += BLCKSZ; file->uncompressed_size += BLCKSZ; @@ -492,13 +485,12 @@ void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, HeaderMap *hdr_map, bool is_merge) { int rc; bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; + char *errmsg = NULL; + BlockNumber err_blknum = 0; /* page headers */ BackupPageHeader2 *headers = NULL; @@ -547,7 +539,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat * Such files should be fully copied. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev || !file->pagemap.bitmap) use_pagemap = false; @@ -557,7 +549,6 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat /* Remote mode */ if (fio_is_remote(FIO_DB_HOST)) { - rc = fio_send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && @@ -576,7 +567,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, calg, clevel, checksum_version, use_pagemap, - &headers, backup_mode, ptrack_version_num, ptrack_schema); + &headers, backup_mode); } /* check for errors */ @@ -646,30 +637,21 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat } /* - * Backup data file in the from_root directory to the to_root directory with - * same relative path. If prev_backup_start_lsn is not NULL, only pages with + * Catchup data file in the from_root directory to the to_root directory with + * same relative path. If sync_lsn is not NULL, only pages with equal or * higher lsn will be copied. * Not just copy file, but read it block by block (use bitmap in case of - * incremental backup), validate checksum, optionally compress and write to - * backup with special header. + * incremental catchup), validate page checksum. */ void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, - bool is_merge, size_t prev_size) + XLogRecPtr sync_lsn, BackupMode backup_mode, + uint32 checksum_version, size_t prev_size) { int rc; bool use_pagemap; char *errmsg = NULL; BlockNumber err_blknum = 0; - /* page headers */ - BackupPageHeader2 *headers = NULL; - - /* sanity */ - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); /* * Compute expected number of blocks in the file. @@ -679,7 +661,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa file->n_blocks = file->size/BLCKSZ; /* - * Skip unchanged file only if it exists in previous backup. + * Skip unchanged file only if it exists in destination directory. * This way we can correctly handle null-sized files which are * not tracked by pagemap and thus always marked as unchanged. */ @@ -688,8 +670,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa file->exists_in_prev && file->size == prev_size && !file->pagemap_isabsent) { /* - * There are no changed blocks since last backup. We want to make - * incremental backup, so we should exit. + * There are none changed pages. */ file->write_size = BYTES_INVALID; return; @@ -699,16 +680,10 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); /* - * Read each page, verify checksum and write it to backup. - * If page map is empty or file is not present in previous backup - * backup all pages of the relation. - * - * In PTRACK 1.x there was a problem - * of data files with missing _ptrack map. - * Such files should be fully copied. + * If page map is empty or file is not present in destination directory, + * then copy backup all pages of the relation. */ if (file->pagemap.bitmapsize == PageBitmapIsEmpty || @@ -726,29 +701,28 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa { rc = fio_copy_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, + ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, + NONE_COMPRESS, 1, checksum_version, /* send pagemap if any */ use_pagemap, /* variables for error reporting */ - &err_blknum, &errmsg, &headers); + &err_blknum, &errmsg); } else { /* TODO: stop handling errors internally */ rc = copy_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - checksum_version, use_pagemap, - backup_mode, ptrack_version_num, ptrack_schema); + ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, + checksum_version, use_pagemap, backup_mode); } /* check for errors */ if (rc == FILE_MISSING) { - elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath); + elog(LOG, "File not found: \"%s\"", from_fullpath); file->write_size = FILE_NOT_FOUND; goto cleanup; } @@ -784,11 +758,6 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa file->read_size = rc * BLCKSZ; - /* refresh n_blocks for FULL and DELTA */ - if (backup_mode == BACKUP_MODE_FULL || - backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; - /* Determine that file didn`t changed in case of incremental catchup */ if (backup_mode != BACKUP_MODE_FULL && file->exists_in_prev && @@ -799,13 +768,8 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa } cleanup: - - /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); - pg_free(errmsg); pg_free(file->pagemap.bitmap); - pg_free(headers); } /* @@ -816,9 +780,9 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa */ void backup_non_data_file(pgFile *file, pgFile *prev_file, - const char *from_fullpath, const char *to_fullpath, - BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok) + const char *from_fullpath, const char *to_fullpath, + BackupMode backup_mode, time_t parent_backup_time, + bool missing_ok) { /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) @@ -891,7 +855,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* page headers */ BackupPageHeader2 *headers = NULL; - pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); + pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); if (use_bitmap) backup_seq++; @@ -899,7 +863,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, backup_seq--; /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); + res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); tmp_file = (res_file) ? *res_file : NULL; /* Destination file is not exists yet at this moment */ @@ -951,13 +915,13 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, * copy the file from backup. */ total_write_len += restore_data_file_internal(in, out, tmp_file, - parse_program_version(backup->program_version), - from_fullpath, to_fullpath, dest_file->n_blocks, - use_bitmap ? &(dest_file)->pagemap : NULL, - checksum_map, backup->checksum_version, - /* shiftmap can be used only if backup state precedes the shift */ - backup->stop_lsn <= shift_lsn ? lsn_map : NULL, - headers); + parse_program_version(backup->program_version), + from_fullpath, to_fullpath, dest_file->n_blocks, + use_bitmap ? &(dest_file)->pagemap : NULL, + checksum_map, backup->checksum_version, + /* shiftmap can be used only if backup state precedes the shift */ + backup->stop_lsn <= shift_lsn ? lsn_map : NULL, + headers); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -983,15 +947,15 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map, BackupPageHeader2 *headers) + const char *from_fullpath, const char *to_fullpath, int nblocks, + datapagemap_t *map, PageState *checksum_map, int checksum_version, + datapagemap_t *lsn_map, BackupPageHeader2 *headers) { BlockNumber blknum = 0; - int n_hdr = -1; - size_t write_len = 0; - off_t cur_pos_out = 0; - off_t cur_pos_in = 0; + int n_hdr = -1; + size_t write_len = 0; + off_t cur_pos_out = 0; + off_t cur_pos_in = 0; /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); @@ -1007,7 +971,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * but should never happen in case of blocks from FULL backup. */ if (fio_fseek(out, cur_pos_out) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", + elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); for (;;) @@ -1020,7 +984,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers bool is_compressed = false; /* incremental restore vars */ - uint16 page_crc = 0; + uint16 page_crc = 0; XLogRecPtr page_lsn = InvalidXLogRecPtr; /* check for interrupt */ @@ -1072,7 +1036,6 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * Now we have to deal with backward compatibility. */ read_len = MAXALIGN(compressed_size); - } else break; @@ -1183,8 +1146,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * page_may_be_compressed() function. */ if (compressed_size != BLCKSZ - || page_may_be_compressed(page.data, file->compress_alg, - backup_version)) + || page_may_be_compressed(page.data, file->compress_alg, backup_version)) { is_compressed = true; } @@ -1244,10 +1206,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers */ void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath) + const char *from_fullpath, const char *to_fullpath) { - size_t read_len = 0; - char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ + size_t read_len = 0; + char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ /* copy content */ for (;;) @@ -1310,7 +1272,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, tmp_backup = dest_backup->parent_backup_link; while (tmp_backup) { - pgFile **res_file = NULL; + pgFile **res_file = NULL; /* lookup file in intermediate backup */ res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); @@ -1420,10 +1382,10 @@ backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok) { - FILE *in = NULL; - FILE *out = NULL; - ssize_t read_len = 0; - char *buf = NULL; + FILE *in = NULL; + FILE *out = NULL; + ssize_t read_len = 0; + char *buf = NULL; INIT_FILE_CRC32(true, file->crc); @@ -1553,7 +1515,7 @@ backup_non_data_file_internal(const char *from_fullpath, */ bool create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file) + fio_location to_location, pgFile *file) { char to_path[MAXPGPATH]; FILE *out; @@ -1650,7 +1612,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, BlockNumber nblocks = 0; int page_state; char curr_page[BLCKSZ]; - bool is_valid = true; + bool is_valid = true; in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) @@ -1686,7 +1648,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, page_state = prepare_page(file, InvalidXLogRecPtr, blknum, in, BACKUP_MODE_FULL, curr_page, false, checksum_version, - 0, NULL, from_fullpath, &page_st); + from_fullpath, &page_st); if (page_state == PageIsTruncated) break; @@ -1744,9 +1706,9 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, while (true) { int rc = 0; - size_t len = 0; + size_t len = 0; DataPage compressed_page; /* used as read buffer */ - int compressed_size = 0; + int compressed_size = 0; DataPage page; BlockNumber blknum = 0; PageState page_st; @@ -1834,7 +1796,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) { - int32 uncompressed_size = 0; + int32 uncompressed_size = 0; const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, @@ -1862,13 +1824,13 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } rc = validate_one_page(page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); } else rc = validate_one_page(compressed_page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); switch (rc) { @@ -1986,11 +1948,11 @@ datapagemap_t * get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) { - FILE *in = NULL; - BlockNumber blknum = 0; - char read_buffer[BLCKSZ]; - char in_buf[STDIO_BUFSIZE]; - datapagemap_t *lsn_map = NULL; + FILE *in = NULL; + BlockNumber blknum = 0; + char read_buffer[BLCKSZ]; + char in_buf[STDIO_BUFSIZE]; + datapagemap_t *lsn_map = NULL; Assert(shift_lsn > 0); @@ -2069,10 +2031,10 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, else if (read_len != 0 && feof(in)) elog(ERROR, "Odd size page found at offset %lu of \"%s\"", - ftell(in), fullpath); + ftello(in), fullpath); else elog(ERROR, "Cannot read header at offset %lu of \"%s\": %s", - ftell(in), fullpath, strerror(errno)); + ftello(in), fullpath, strerror(errno)); } /* In older versions < 2.4.0, when crc for file was calculated, header was @@ -2117,7 +2079,7 @@ int send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) + BackupMode backup_mode) { FILE *in = NULL; FILE *out = NULL; @@ -2175,7 +2137,6 @@ send_pages(const char *to_fullpath, const char *from_fullpath, int rc = prepare_page(file, prev_backup_start_lsn, blknum, in, backup_mode, curr_page, true, checksum_version, - ptrack_version_num, ptrack_schema, from_fullpath, &page_st); if (rc == PageIsTruncated) @@ -2254,17 +2215,19 @@ send_pages(const char *to_fullpath, const char *from_fullpath, return n_blocks_read; } -/* copy local file (взята из send_pages, но используется простое копирование странички, без добавления заголовков и компрессии) */ +/* + * Copy local data file just as send_pages but without attaching additional header and compression + */ int copy_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr sync_lsn, - uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) + pgFile *file, XLogRecPtr sync_lsn, + uint32 checksum_version, bool use_pagemap, + BackupMode backup_mode) { FILE *in = NULL; FILE *out = NULL; - char curr_page[BLCKSZ]; - int n_blocks_read = 0; + char curr_page[BLCKSZ]; + int n_blocks_read = 0; BlockNumber blknum = 0; datapagemap_iterator_t *iter = NULL; @@ -2308,44 +2271,36 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); if (out == NULL) elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); + to_fullpath, strerror(errno)); /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) + if (chmod(to_fullpath, file->mode) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + strerror(errno)); - elog(VERBOSE, "ftruncate file \"%s\" to size %lu", - to_fullpath, file->size); - if (fio_ftruncate(out, file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); - - if (!fio_is_remote_file(out)) - { - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); - } + /* Enable buffering for output file */ + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); while (blknum < file->n_blocks) { PageState page_st; int rc = prepare_page(file, sync_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath, &page_st); + blknum, in, backup_mode, curr_page, + true, checksum_version, + from_fullpath, &page_st); if (rc == PageIsTruncated) break; else if (rc == PageIsOk) { - if (fio_fseek(out, blknum * BLCKSZ) < 0) - { - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - } - copy_page(file, blknum, in, out, curr_page, to_fullpath); + if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) + elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", + blknum * BLCKSZ, to_fullpath, strerror(errno)); + + if (write_page(file, out, curr_page) != BLCKSZ) + elog(ERROR, "File: \"%s\", cannot write at block %u: %s", + to_fullpath, blknum, strerror(errno)); } n_blocks_read++; @@ -2361,13 +2316,36 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, blknum++; } + /* truncate output file if required */ + if (fseek(out, 0, SEEK_END) != 0) + elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", + to_fullpath, strerror(errno)); + { + size_t pos = ftell(out); + + if (pos < 0) + elog(ERROR, "Cannot get position in destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + if (pos != file->size) + { + if (fflush(out) != 0) + elog(ERROR, "Cannot flush destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + if (ftruncate(fileno(out), file->size) == -1) + elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + to_fullpath, file->size, strerror(errno)); + } + } + /* cleanup */ - if (in && fclose(in)) + if (fclose(in)) elog(ERROR, "Cannot close the source file \"%s\": %s", to_fullpath, strerror(errno)); - /* close local output file */ - if (out && fio_fclose(out)) + /* close output file */ + if (fclose(out)) elog(ERROR, "Cannot close the destination file \"%s\": %s", to_fullpath, strerror(errno)); @@ -2503,19 +2481,19 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, /* when running merge we must write headers into temp map */ map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; - read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); /* calculate checksums */ INIT_FILE_CRC32(true, file->hdr_crc); COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); FIN_FILE_CRC32(true, file->hdr_crc); - zheaders = pgut_malloc(read_len*2); - memset(zheaders, 0, read_len*2); + zheaders = pgut_malloc(read_len * 2); + memset(zheaders, 0, read_len * 2); /* compress headers */ - z_len = do_compress(zheaders, read_len*2, headers, - read_len, ZLIB_COMPRESS, 1, &errormsg); + z_len = do_compress(zheaders, read_len * 2, headers, + read_len, ZLIB_COMPRESS, 1, &errormsg); /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ @@ -2559,7 +2537,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); - file->hdr_size = z_len; /* save the length of compressed headers */ + file->hdr_size = z_len; /* save the length of compressed headers */ hdr_map->offset += z_len; /* update current offset in map */ /* End critical section */ diff --git a/src/dir.c b/src/dir.c index 473534c8b..bac583b4d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -121,8 +121,6 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; -static int pgCompareString(const void *str1, const void *str2); - static char dir_check_file(pgFile *file, bool backup_logs); static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, @@ -224,6 +222,7 @@ pgFileInit(const char *rel_path) // May be add? // pg_atomic_clear_flag(file->lock); + file->excluded = false; return file; } @@ -426,6 +425,26 @@ pgFileCompareName(const void *f1, const void *f2) return strcmp(f1p->name, f2p->name); } +/* Compare pgFile->name with string in ascending order of ASCII code. */ +int +pgFileCompareNameWithString(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + char *f2s = *(char **)f2; + + return strcmp(f1p->name, f2s); +} + +/* Compare pgFile->rel_path with string in ascending order of ASCII code. */ +int +pgFileCompareRelPathWithString(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + char *f2s = *(char **)f2; + + return strcmp(f1p->rel_path, f2s); +} + /* * Compare two pgFile with their relative path and external_dir_num in ascending * order of ASСII code. @@ -492,12 +511,26 @@ pgFileCompareSizeDesc(const void *f1, const void *f2) return -1 * pgFileCompareSize(f1, f2); } -static int +int pgCompareString(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); } +/* + * From bsearch(3): "The compar routine is expected to have two argu‐ + * ments which point to the key object and to an array member, in that order" + * But in practice this is opposite, so we took strlen from second string (search key) + * This is checked by tests.catchup.CatchupTest.test_catchup_with_exclude_path + */ +int +pgPrefixCompareString(const void *str1, const void *str2) +{ + const char *s1 = *(char **) str1; + const char *s2 = *(char **) str2; + return strncmp(s1, s2, strlen(s2)); +} + /* Compare two Oids */ int pgCompareOid(const void *f1, const void *f2) diff --git a/src/help.c b/src/help.c index 921feaec0..1515359e4 100644 --- a/src/help.c +++ b/src/help.c @@ -124,7 +124,7 @@ help_pg_probackup(void) printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); @@ -251,9 +251,10 @@ help_pg_probackup(void) printf(_("\n %s catchup -b catchup-mode\n"), PROGRAM_NAME); printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); printf(_(" --destination-pgdata=path_to_local_dir\n")); - printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot | --perm-slot]]\n")); printf(_(" [-j num-threads]\n")); printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--exclude-path=path_prefix]\n")); printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -295,7 +296,7 @@ help_backup(void) { printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-E external-directories-paths]\n")); @@ -1031,9 +1032,10 @@ help_catchup(void) printf(_("\n%s catchup -b catchup-mode\n"), PROGRAM_NAME); printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); printf(_(" --destination-pgdata=path_to_local_dir\n")); - printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name]] [--temp-slot | --perm-slot]\n")); printf(_(" [-j num-threads]\n")); printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--exclude-path=path_prefix]\n")); printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); printf(_(" [-w --no-password] [-W --password]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -1045,11 +1047,15 @@ help_catchup(void) printf(_(" --stream stream the transaction log (only supported mode)\n")); printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); printf(_(" --temp-slot use temporary replication slot\n")); + printf(_(" -P --perm-slot create permanent replication slot\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + printf(_(" -x, --exclude-path=path_prefix files with path_prefix (relative to pgdata) will be\n")); + printf(_(" excluded from catchup (can be used multiple times)\n")); + printf(_(" Dangerous option! Use at your own risk!\n")); printf(_("\n Connection options:\n")); printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); diff --git a/src/merge.c b/src/merge.c index cd070fce4..ff39c2510 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1256,7 +1256,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, - dest_backup->checksum_version, 0, NULL, + dest_backup->checksum_version, &(full_backup->hdr_map), true); /* drop restored temp file */ diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 00796be04..d629d838d 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -80,8 +80,9 @@ bool progress = false; bool no_sync = false; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; -#endif bool temp_slot = false; +#endif +bool perm_slot = false; /* backup options */ bool backup_logs = false; @@ -118,6 +119,9 @@ bool skip_external_dirs = false; /* array for datnames, provided via db-include and db-exclude */ static parray *datname_exclude_list = NULL; static parray *datname_include_list = NULL; +/* arrays for --exclude-path's */ +static parray *exclude_absolute_paths_list = NULL; +static parray *exclude_relative_paths_list = NULL; /* checkdb options */ bool need_amcheck = false; @@ -176,6 +180,7 @@ static void compress_init(ProbackupSubcmd const subcmd); static void opt_datname_exclude_list(ConfigOption *opt, const char *arg); static void opt_datname_include_list(ConfigOption *opt, const char *arg); +static void opt_exclude_path(ConfigOption *opt, const char *arg); /* * Short name should be non-printable ASCII character. @@ -198,7 +203,10 @@ static ConfigOption cmd_options[] = { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, +#if PG_VERSION_NUM >= 100000 { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, +#endif + { 'b', 'P', "perm-slot", &perm_slot, SOURCE_CMD_STRICT }, { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, @@ -207,6 +215,7 @@ static ConfigOption cmd_options[] = /* catchup options */ { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, + { 'f', 'x', "exclude-path", opt_exclude_path, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, @@ -787,6 +796,17 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); +#if PG_VERSION_NUM >= 100000 + if (temp_slot && perm_slot) + elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option"); + + /* if slot name was not provided for temp slot, use default slot name */ + if (!replication_slot && temp_slot) + replication_slot = DEFAULT_TEMP_SLOT_NAME; +#endif + if (!replication_slot && perm_slot) + replication_slot = DEFAULT_PERMANENT_SLOT_NAME; + if (num_threads < 1) num_threads = 1; @@ -825,7 +845,8 @@ main(int argc, char *argv[]) no_validate, no_sync, backup_logs); } case CATCHUP_CMD: - return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync); + return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync, + exclude_absolute_paths_list, exclude_relative_paths_list); case RESTORE_CMD: return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, @@ -990,39 +1011,45 @@ compress_init(ProbackupSubcmd const subcmd) } } -/* Construct array of datnames, provided by user via db-exclude option */ -void -opt_datname_exclude_list(ConfigOption *opt, const char *arg) +static void +opt_parser_add_to_parray_helper(parray **list, const char *str) { - char *dbname = NULL; + char *elem = NULL; - if (!datname_exclude_list) - datname_exclude_list = parray_new(); + if (*list == NULL) + *list = parray_new(); - dbname = pgut_malloc(strlen(arg) + 1); + elem = pgut_malloc(strlen(str) + 1); + strcpy(elem, str); - /* TODO add sanity for database name */ - strcpy(dbname, arg); + parray_append(*list, elem); +} - parray_append(datname_exclude_list, dbname); +/* Construct array of datnames, provided by user via db-exclude option */ +void +opt_datname_exclude_list(ConfigOption *opt, const char *arg) +{ + /* TODO add sanity for database name */ + opt_parser_add_to_parray_helper(&datname_exclude_list, arg); } /* Construct array of datnames, provided by user via db-include option */ void opt_datname_include_list(ConfigOption *opt, const char *arg) { - char *dbname = NULL; - - if (!datname_include_list) - datname_include_list = parray_new(); - - dbname = pgut_malloc(strlen(arg) + 1); - - if (strcmp(dbname, "tempate0") == 0 || - strcmp(dbname, "tempate1") == 0) + if (strcmp(arg, "tempate0") == 0 || + strcmp(arg, "tempate1") == 0) elog(ERROR, "Databases 'template0' and 'template1' cannot be used for partial restore or validation"); - strcpy(dbname, arg); + opt_parser_add_to_parray_helper(&datname_include_list, arg); +} - parray_append(datname_include_list, dbname); +/* Parse --exclude-path option */ +void +opt_exclude_path(ConfigOption *opt, const char *arg) +{ + if (is_absolute_path(arg)) + opt_parser_add_to_parray_helper(&exclude_absolute_paths_list, arg); + else + opt_parser_add_to_parray_helper(&exclude_relative_paths_list, arg); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1cad526dd..19f6feff0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -86,6 +86,10 @@ extern const char *PROGRAM_EMAIL; #define HEADER_MAP "page_header_map" #define HEADER_MAP_TMP "page_header_map_tmp" +/* default replication slot names */ +#define DEFAULT_TEMP_SLOT_NAME "pg_probackup_slot"; +#define DEFAULT_PERMANENT_SLOT_NAME "pg_probackup_perm_slot"; + /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 @@ -278,6 +282,7 @@ typedef struct pgFile pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ pg_off_t hdr_off; /* offset in header map */ int hdr_size; /* length of headers */ + bool excluded; /* excluded via --exclude-path option */ } pgFile; typedef struct page_map_entry @@ -771,11 +776,12 @@ extern bool stream_wal; extern bool show_color; extern bool progress; extern bool is_archive_cmd; /* true for archive-{get,push} */ -#if PG_VERSION_NUM >= 100000 /* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; -#endif +#if PG_VERSION_NUM >= 100000 extern bool temp_slot; +#endif +extern bool perm_slot; /* backup options */ extern bool smooth_checkpoint; @@ -842,7 +848,8 @@ extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); /* in catchup.c */ -extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files); +extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list); /* in restore.c */ extern int do_restore_or_validate(InstanceState *instanceState, @@ -1057,11 +1064,15 @@ extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool miss extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); +extern int pgFileCompareNameWithString(const void *f1, const void *f2); +extern int pgFileCompareRelPathWithString(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); extern int pgFileCompareSizeDesc(const void *f1, const void *f2); +extern int pgCompareString(const void *str1, const void *str2); +extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); @@ -1071,14 +1082,11 @@ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, - bool is_merge, size_t prev_size); + XLogRecPtr sync_lsn, BackupMode backup_mode, + uint32 checksum_version, size_t prev_size); extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, HeaderMap *hdr_map, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, @@ -1197,11 +1205,11 @@ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 extern int send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); + BackupMode backup_mode); extern int copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); + BackupMode backup_mode); /* FIO */ extern void setMyLocation(ProbackupSubcmd const subcmd); @@ -1212,8 +1220,7 @@ extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pg BackupPageHeader2 **headers); extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber *err_blknum, char **errormsg, - BackupPageHeader2 **headers); + bool use_pagemap, BlockNumber *err_blknum, char **errormsg); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, @@ -1265,7 +1272,8 @@ datapagemap_print_debug(datapagemap_t *map); extern XLogRecPtr stop_backup_lsn; extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, - XLogRecPtr startpos, TimeLineID starttli); + XLogRecPtr startpos, TimeLineID starttli, + bool is_backup); extern int wait_WAL_streaming_end(parray *backup_files_list); extern parray* parse_tli_history_buffer(char *history, TimeLineID tli); diff --git a/src/ptrack.c b/src/ptrack.c index 191f988a3..3f395b286 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -123,19 +123,24 @@ pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num) PGresult *res_db; bool result = false; - if (ptrack_version_num == 200) + if (ptrack_version_num > 200) + { + res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && + strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; + PQclear(res_db); + } + else if (ptrack_version_num == 200) { res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; + PQclear(res_db); } else { - res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && - strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; + result = false; } - PQclear(res_db); return result; } diff --git a/src/stream.c b/src/stream.c index 5912ff44b..570108cde 100644 --- a/src/stream.c +++ b/src/stream.c @@ -59,6 +59,7 @@ static pthread_t stream_thread; static StreamThreadArg stream_thread_arg = {"", NULL, 1}; static parray *xlog_files_list = NULL; +static bool do_crc = true; static void IdentifySystem(StreamThreadArg *stream_thread_arg); static int checkpoint_timeout(PGconn *backup_conn); @@ -159,6 +160,56 @@ checkpoint_timeout(PGconn *backup_conn) return val_int; } +/* + * CreateReplicationSlot_compat() -- wrapper for CreateReplicationSlot() used in StreamLog() + * src/bin/pg_basebackup/streamutil.c + * CreateReplicationSlot() has different signatures on different PG versions: + * PG 15 + * bool + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_temporary, bool is_physical, bool reserve_wal, + * bool slot_exists_ok, bool two_phase) + * PG 11-14 + * bool + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_temporary, bool is_physical, bool reserve_wal, + * bool slot_exists_ok) + * PG 9.5-10 + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_physical, bool slot_exists_ok) + * NOTE: PG 9.6 and 10 support reserve_wal in + * pg_catalog.pg_create_physical_replication_slot(slot_name name [, immediately_reserve boolean]) + * and + * CREATE_REPLICATION_SLOT slot_name { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin } + * replication protocol command, but CreateReplicationSlot() C function doesn't + */ +static bool +CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *plugin, + bool is_temporary, bool is_physical, + bool slot_exists_ok) +{ +#if PG_VERSION_NUM >= 150000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); +#elif PG_VERSION_NUM >= 110000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok); +#elif PG_VERSION_NUM >= 100000 + /* + * PG-10 doesn't support creating temp_slot by calling CreateReplicationSlot(), but + * it will be created by setting StreamCtl.temp_slot later in StreamLog() + */ + if (!is_temporary) + return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); + else + return true; +#else + /* these parameters not supported in PG < 10 */ + Assert(!is_temporary); + return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); +#endif +} + /* * Start the log streaming */ @@ -177,31 +228,36 @@ StreamLog(void *arg) /* Initialize timeout */ stream_stop_begin = 0; + /* Create repslot */ #if PG_VERSION_NUM >= 100000 - /* if slot name was not provided for temp slot, use default slot name */ - if (!replication_slot && temp_slot) - replication_slot = "pg_probackup_slot"; -#endif - - -#if PG_VERSION_NUM >= 150000 - /* Create temp repslot */ - if (temp_slot) - CreateReplicationSlot(stream_arg->conn, replication_slot, - NULL, temp_slot, true, true, false, false); -#elif PG_VERSION_NUM >= 110000 - /* Create temp repslot */ - if (temp_slot) - CreateReplicationSlot(stream_arg->conn, replication_slot, - NULL, temp_slot, true, true, false); + if (temp_slot || perm_slot) + if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, temp_slot, true, false)) +#else + if (perm_slot) + if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, false, true, false)) #endif + { + interrupted = true; + elog(ERROR, "Couldn't create physical replication slot %s", replication_slot); + } /* * Start the replication */ - elog(LOG, "started streaming WAL at %X/%X (timeline %u)", - (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, - stream_arg->starttli); + if (replication_slot) + elog(LOG, "started streaming WAL at %X/%X (timeline %u) using%s slot %s", + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli, +#if PG_VERSION_NUM >= 100000 + temp_slot ? " temporary" : "", +#else + "", +#endif + replication_slot); + else + elog(LOG, "started streaming WAL at %X/%X (timeline %u)", + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli); #if PG_VERSION_NUM >= 90600 { @@ -212,6 +268,11 @@ StreamLog(void *arg) ctl.startpos = stream_arg->startpos; ctl.timeline = stream_arg->starttli; ctl.sysidentifier = NULL; + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; #if PG_VERSION_NUM >= 100000 ctl.walmethod = CreateWalDirectoryMethod( @@ -224,19 +285,14 @@ StreamLog(void *arg) ctl.do_sync = false; /* We sync all files at the end of backup */ // ctl.mark_done /* for future use in s3 */ #if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 + /* StreamCtl.temp_slot used only for PG-10, in PG>10, temp_slots are created by calling CreateReplicationSlot() */ ctl.temp_slot = temp_slot; -#endif -#else +#endif /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 */ +#else /* PG_VERSION_NUM < 100000 */ ctl.basedir = (char *) stream_arg->basedir; -#endif - - ctl.stream_stop = stop_streaming; - ctl.standby_message_timeout = standby_message_timeout; - ctl.partial_suffix = NULL; - ctl.synchronous = false; - ctl.mark_done = false; +#endif /* PG_VERSION_NUM >= 100000 */ - if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) + if (ReceiveXlogStream(stream_arg->conn, &ctl) == false) { interrupted = true; elog(ERROR, "Problem in receivexlog"); @@ -244,38 +300,42 @@ StreamLog(void *arg) #if PG_VERSION_NUM >= 100000 if (!ctl.walmethod->finish()) + { + interrupted = true; elog(ERROR, "Could not finish writing WAL files: %s", strerror(errno)); -#endif + } +#endif /* PG_VERSION_NUM >= 100000 */ } -#else - if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, +#else /* PG_VERSION_NUM < 90600 */ + /* PG-9.5 */ + if (ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, NULL, (char *) stream_arg->basedir, stop_streaming, standby_message_timeout, NULL, false, false) == false) { interrupted = true; elog(ERROR, "Problem in receivexlog"); } -#endif +#endif /* PG_VERSION_NUM >= 90600 */ - /* be paranoid and sort xlog_files_list, - * so if stop_lsn segno is already in the list, - * then list must be sorted to detect duplicates. - */ - parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); + /* be paranoid and sort xlog_files_list, + * so if stop_lsn segno is already in the list, + * then list must be sorted to detect duplicates. + */ + parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); - /* Add the last segment to the list */ - add_walsegment_to_filelist(xlog_files_list, stream_arg->starttli, + /* Add the last segment to the list */ + add_walsegment_to_filelist(xlog_files_list, stream_arg->starttli, stop_stream_lsn, (char *) stream_arg->basedir, instance_config.xlog_seg_size); - /* append history file to walsegment filelist */ - add_history_file_to_filelist(xlog_files_list, stream_arg->starttli, (char *) stream_arg->basedir); + /* append history file to walsegment filelist */ + add_history_file_to_filelist(xlog_files_list, stream_arg->starttli, (char *) stream_arg->basedir); - /* - * TODO: remove redundant WAL segments - * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 - */ + /* + * TODO: remove redundant WAL segments + * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 + */ elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); @@ -569,8 +629,10 @@ parse_tli_history_buffer(char *history, TimeLineID tli) */ void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, - XLogRecPtr startpos, TimeLineID starttli) + XLogRecPtr startpos, TimeLineID starttli, bool is_backup) { + /* calculate crc only when running backup, catchup has no need for it */ + do_crc = is_backup; /* How long we should wait for streaming end after pg_stop_backup */ stream_stop_timeout = checkpoint_timeout(backup_conn); //TODO Add a comment about this calculation @@ -654,15 +716,16 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos if (existing_file) { - (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + if (do_crc) + (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); (*existing_file)->write_size = xlog_seg_size; (*existing_file)->uncompressed_size = xlog_seg_size; return; } - /* calculate crc */ - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + if (do_crc) + file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); /* Should we recheck it using stat? */ file->write_size = xlog_seg_size; @@ -692,7 +755,8 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); /* calculate crc */ - file->crc = pgFileGetCRC(fullpath, true, false); + if (do_crc) + file->crc = pgFileGetCRC(fullpath, true, false); file->write_size = file->size; file->uncompressed_size = file->size; diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 3a5de4b83..2c6ea3eec 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -61,14 +61,14 @@ typedef char *(*option_get_fn) (ConfigOption *opt); /* * type: - * b: bool (true) - * B: bool (false) + * b: bool (true) + * B: bool (false) * f: option_fn - * i: 32bit signed integer - * u: 32bit unsigned integer - * I: 64bit signed integer - * U: 64bit unsigned integer - * s: string + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string * t: time_t */ struct ConfigOption diff --git a/src/utils/file.c b/src/utils/file.c index b808d6293..f86e605cb 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1963,8 +1963,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg, - BackupPageHeader2 **headers) + bool use_pagemap, BlockNumber* err_blknum, char **errormsg) { FILE *out = NULL; char *out_buf = NULL; @@ -2092,9 +2091,9 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, /* receive headers if any */ if (hdr.size > 0) { - *headers = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); - file->n_headers = (hdr.size / sizeof(BackupPageHeader2)) -1; + char *tmp = pgut_malloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, tmp, hdr.size), hdr.size); + pg_free(tmp); } break; diff --git a/tests/backup.py b/tests/backup.py index 0bfd0c1b9..558c62de3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1421,8 +1421,10 @@ def test_basic_temp_slot_for_stream_backup(self): base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '40MB', 'default_transaction_read_only': 'on'}) + pg_options={'max_wal_size': '40MB'}) + + if self.get_version(node) < self.version_to_num('10.0'): + return unittest.skip('You need PostgreSQL >= 10 for this test') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1434,11 +1436,6 @@ def test_basic_temp_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--temp-slot']) - if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - # FULL backup self.backup_node( backup_dir, 'node', node, @@ -3274,7 +3271,7 @@ def test_basic_backup_default_transaction_read_only(self): # FULL backup self.backup_node( backup_dir, 'node', node, - options=['--stream', '--temp-slot']) + options=['--stream']) # DELTA backup self.backup_node( diff --git a/tests/catchup.py b/tests/catchup.py index 5df538e42..45d999629 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import signal import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException @@ -55,6 +56,7 @@ def test_basic_full_catchup(self): # Cleanup dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') self.del_test_dir(module_name, self.fname) def test_full_catchup_with_tablespace(self): @@ -180,6 +182,7 @@ def test_basic_delta_catchup(self): # Cleanup dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') self.del_test_dir(module_name, self.fname) def test_basic_ptrack_catchup(self): @@ -252,6 +255,7 @@ def test_basic_ptrack_catchup(self): # Cleanup dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') self.del_test_dir(module_name, self.fname) def test_tli_delta_catchup(self): @@ -776,69 +780,6 @@ def test_same_db_id(self): src_pg.stop() self.del_test_dir(module_name, self.fname) - def test_destination_dbstate(self): - """ - Test that we detect that destination pg is not cleanly shutdowned - """ - # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), - set_replication = True, - pg_options = { 'wal_log_hints': 'on' } - ) - src_pg.slow_start() - - # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] - ) - - # try #1 - try: - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] - ) - self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Destination directory contains "backup_label" file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # try #2 - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") - os.kill(dst_pg.pid, signal.SIGKILL) - try: - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] - ) - self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'must be stopped cleanly', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # Cleanup - src_pg.stop() - self.del_test_dir(module_name, self.fname) - def test_tli_destination_mismatch(self): """ Test that we detect TLI mismatch in destination @@ -975,3 +916,517 @@ def test_tli_source_mismatch(self): src_pg.stop() fake_src_pg.stop() self.del_test_dir(module_name, self.fname) + +######################################### +# Test unclean destination +######################################### + def test_unclean_delta_catchup(self): + """ + Test that we correctly recover uncleanly shutdowned destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # try #1 + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination directory contains "backup_label" file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try #2 + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") + os.kill(dst_pg.pid, signal.SIGKILL) + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_unclean_ptrack_catchup(self): + """ + Test that we correctly recover uncleanly shutdowned destination + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # try #1 + try: + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination directory contains "backup_label" file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try #2 + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") + os.kill(dst_pg.pid, signal.SIGKILL) + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + self.del_test_dir(module_name, self.fname) + +######################################### +# Test replication slot logic +# +# -S, --slot=SLOTNAME replication slot to use +# --temp-slot use temporary replication slot +# -P --perm-slot create permanent replication slot +# --primary-slot-name=SLOTNAME value for primary_slot_name parameter +# +# 1. if "--slot" is used - try to use already existing slot with given name +# 2. if "--slot" and "--perm-slot" used - try to create permanent slot and use it. +# 3. If "--perm-slot " flag is used without "--slot" option - use generic slot name like "pg_probackup_perm_slot" +# 4. If "--perm-slot " flag is used and permanent slot already exists - fail with error. +# 5. "--perm-slot" and "--temp-slot" flags cannot be used together. +######################################### + def test_catchup_with_replication_slot(self): + """ + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # 1a. --slot option + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_1a')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=nonexistentslot_1a' + ] + ) + self.assertEqual(1, 0, "Expecting Error because replication slot does not exist.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: replication slot "nonexistentslot_1a" does not exist', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # 1b. --slot option + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_1b')) + src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_1b')") + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=existentslot_1b' + ] + ) + + # 2a. --slot --perm-slot + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_2a')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=nonexistentslot_2a', + '--perm-slot' + ] + ) + + # 2b. and 4. --slot --perm-slot + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_2b')) + src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_2b')") + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=existentslot_2b', + '--perm-slot' + ] + ) + self.assertEqual(1, 0, "Expecting Error because replication slot already exist.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: replication slot "existentslot_2b" already exists', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # 3. --perm-slot --slot + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_3')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot' + ] + ) + slot_name = src_pg.safe_psql( + "postgres", + "SELECT slot_name FROM pg_catalog.pg_replication_slots " + "WHERE slot_name NOT LIKE '%existentslot%' " + "AND slot_type = 'physical'" + ).decode('utf-8').rstrip() + self.assertEqual(slot_name, 'pg_probackup_perm_slot', 'Slot name mismatch') + + # 5. --perm-slot --temp-slot (PG>=10) + if self.get_version(src_pg) >= self.version_to_num('10.0'): + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_5')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot' + ] + ) + self.assertEqual(1, 0, "Expecting Error because conflicting options --perm-slot and --temp-slot used together\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + #self.assertEqual(1, 0, 'Stop test') + self.del_test_dir(module_name, self.fname) + +######################################### +# --exclude-path +######################################### + def test_catchup_with_exclude_path(self): + """ + various syntetic tests for --exclude-path option + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # test 1 + os.mkdir(os.path.join(src_pg.data_dir, 'src_usefull_dir')) + with open(os.path.join(os.path.join(src_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')), 'w') as f: + f.write('garbage') + f.flush() + f.close + os.mkdir(os.path.join(src_pg.data_dir, 'src_garbage_dir')) + with open(os.path.join(os.path.join(src_pg.data_dir, 'src_garbage_dir', 'src_garbage_file')), 'w') as f: + f.write('garbage') + f.flush() + f.close + + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}'.format(os.path.join(src_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')), + '-x', '{0}'.format(os.path.join(src_pg.data_dir, 'src_garbage_dir')), + ] + ) + + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_garbage_dir')).exists()) + + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # test 2 + os.mkdir(os.path.join(dst_pg.data_dir, 'dst_garbage_dir')) + os.mkdir(os.path.join(dst_pg.data_dir, 'dst_usefull_dir')) + with open(os.path.join(os.path.join(dst_pg.data_dir, 'dst_usefull_dir', 'dst_usefull_file')), 'w') as f: + f.write('gems') + f.flush() + f.close + + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path=src_usefull_dir/src_garbage_file', + '--exclude-path=src_garbage_dir', + '--exclude-path={0}'.format(os.path.join(dst_pg.data_dir, 'dst_usefull_dir')), + ] + ) + + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_garbage_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'dst_garbage_dir')).exists()) + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'dst_usefull_dir', 'dst_usefull_file')).exists()) + + #self.assertEqual(1, 0, 'Stop test') + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_config_exclusion(self): + """ + Test that catchup can preserve dest replication config + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 1: do delta catchup with relative exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path=postgresql.conf', + '--exclude-path=postgresql.auto.conf', + '--exclude-path=recovery.conf', + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # preparation 4: make changes on master (source) + dst_pg.stop() + #src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 2: do delta catchup with absolute source exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}/postgresql.conf'.format(src_pg.data_dir), + '--exclude-path={0}/postgresql.auto.conf'.format(src_pg.data_dir), + '--exclude-path={0}/recovery.conf'.format(src_pg.data_dir), + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # preparation 5: make changes on master (source) + dst_pg.stop() + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 3: do delta catchup with absolute destination exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}/postgresql.conf'.format(dst_pg.data_dir), + '--exclude-path={0}/postgresql.auto.conf'.format(dst_pg.data_dir), + '--exclude-path={0}/recovery.conf'.format(dst_pg.data_dir), + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + src_pg.stop() + dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') + self.del_test_dir(module_name, self.fname) diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index 3e451e24f..e6bbedb61 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -10,6 +10,7 @@ RUN python3 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} +ENV PTRACK_PATCH_PG_VERSION=${PTRACK_PATCH_PG_VERSION} ENV PGPROBACKUP_GDB=${PGPROBACKUP_GDB} ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh index 2e8ccd5a3..fc2742cdb 100755 --- a/travis/make_dockerfile.sh +++ b/travis/make_dockerfile.sh @@ -14,6 +14,10 @@ if [ -z ${MODE+x} ]; then MODE=basic fi +if [ -z ${PTRACK_PATCH_PG_VERSION+x} ]; then + PTRACK_PATCH_PG_VERSION=off +fi + if [ -z ${PGPROBACKUP_GDB+x} ]; then PGPROBACKUP_GDB=ON fi @@ -21,11 +25,13 @@ fi echo PG_VERSION=${PG_VERSION} echo PG_BRANCH=${PG_BRANCH} echo MODE=${MODE} +echo PTRACK_PATCH_PG_VERSION=${PTRACK_PATCH_PG_VERSION} echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} sed \ -e 's/${PG_VERSION}/'${PG_VERSION}/g \ -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ -e 's/${MODE}/'${MODE}/g \ + -e 's/${PTRACK_PATCH_PG_VERSION}/'${PTRACK_PATCH_PG_VERSION}/g \ -e 's/${PGPROBACKUP_GDB}/'${PGPROBACKUP_GDB}/g \ Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 488d8ee45..4a64fed80 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -32,9 +32,21 @@ PG_SRC=$PWD/postgres echo "############### Getting Postgres sources:" git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 +# Clone ptrack +if [ "$PTRACK_PATCH_PG_VERSION" != "off" ]; then + git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 + export PG_PROBACKUP_PTRACK=on +else + export PG_PROBACKUP_PTRACK=off +fi + + # Compile and install Postgres echo "############### Compiling Postgres:" cd postgres # Go to postgres dir +if [ "$PG_PROBACKUP_PTRACK" = "on" ]; then + git apply -3 ../ptrack/patches/REL_${PTRACK_PATCH_PG_VERSION}_STABLE-ptrack-core.diff +fi CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests make -s -j$(nproc) install #make -s -j$(nproc) -C 'src/common' install @@ -47,6 +59,11 @@ export PATH=$PGHOME/bin:$PATH export LD_LIBRARY_PATH=$PGHOME/lib export PG_CONFIG=$(which pg_config) +if [ "$PG_PROBACKUP_PTRACK" = "on" ]; then + echo "############### Compiling Ptrack:" + make USE_PGXS=1 -C ../ptrack install +fi + # Get amcheck if missing if [ ! -d "contrib/amcheck" ]; then echo "############### Getting missing amcheck:" From 83c0cbb61b36539529ffca085e86ed53cdb48ca3 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Fri, 13 Aug 2021 15:23:03 +0300 Subject: [PATCH 1773/2107] [DOC] [skip travis] Removed version tags 12+ from documentation --- doc/pgprobackup.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7178cb14c..36628ce46 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -143,7 +143,7 @@ doc/src/sgml/pgprobackup.sgml wal_file_name option - + pg_probackup catchup_mode @@ -291,7 +291,7 @@ doc/src/sgml/pgprobackup.sgml Partial restore: restoring only the specified databases. - + Catchup: cloning a PostgreSQL instance for a fallen-behind standby server to catch up with master. @@ -1089,7 +1089,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; mode: , , , - , + , , and . @@ -2089,7 +2089,7 @@ pg_probackup restore -B backup_dir --instance , , , - , + , and processes can be executed on several parallel threads. This can significantly @@ -3408,7 +3408,7 @@ pg_probackup delete -B backup_dir --instance - + Cloning and Synchronizing <productname>PostgreSQL</productname> Instance pg_probackup can create a copy of a PostgreSQL @@ -4449,7 +4449,7 @@ pg_probackup archive-get -B backup_dir --instance - + catchup pg_probackup catchup -b catchup_mode @@ -5188,7 +5188,7 @@ pg_probackup catchup -b catchup_mode You can use these options together with - , , and + , , and commands. @@ -5479,7 +5479,7 @@ pg_probackup catchup -b catchup_mode used with , , , - , + , , , and commands. From 328eea196a4a4aa6fca2d6dd76e1647945bafc0b Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 13 Aug 2021 16:14:04 +0300 Subject: [PATCH 1774/2107] Version 2.5.1 --- src/pg_probackup.h | 8 ++++---- tests/expected/option_version.out | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 19f6feff0..dfa7051a3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,14 +338,14 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.0" +#define PROGRAM_VERSION "2.5.1" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20500 -#define AGENT_PROTOCOL_VERSION_STR "2.5.0" +#define AGENT_PROTOCOL_VERSION 20501 +#define AGENT_PROTOCOL_VERSION_STR "2.5.1" /* update only when changing storage format */ -#define STORAGE_FORMAT_VERSION "2.5.0" +#define STORAGE_FORMAT_VERSION "2.4.4" typedef struct ConnectionOptions { diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 560b6b592..36e5d4c7a 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.0 +pg_probackup 2.5.1 \ No newline at end of file From 97fe5b83723fde64e93163b6efee2a1042c8013f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 13 Aug 2021 19:14:22 +0300 Subject: [PATCH 1775/2107] fix for windows: conditional strndup() replacement --- src/utils/pgut.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/utils/pgut.c b/src/utils/pgut.c index db7c7fd95..52599848d 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -962,9 +962,18 @@ pgut_strndup(const char *str, size_t n) if (str == NULL) return NULL; +#if _POSIX_C_SOURCE >= 200809L if ((ret = strndup(str, n)) == NULL) elog(ERROR, "could not duplicate string \"%s\": %s", str, strerror(errno)); +#else /* WINDOWS doesn't have strndup() */ + if ((ret = malloc(n + 1)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + + memcpy(ret, str, n); + ret[n] = '\0'; +#endif return ret; } From e5714bc625693dc50270245de2bd18194a29ce92 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Tue, 17 Aug 2021 18:23:59 +0300 Subject: [PATCH 1776/2107] [DOC] [skip travis] Fixed misplaced comma in documentation --- doc/pgprobackup.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 36628ce46..9b236b129 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -5187,8 +5187,8 @@ pg_probackup catchup -b catchup_mode Connection Options You can use these options together with - - , , and + , + , and commands. From 14035b271d9a8b6d6e1a4b5bfafa67880c0cfd36 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Thu, 19 Aug 2021 10:01:18 +0300 Subject: [PATCH 1777/2107] [DOC] [skip travis] Incorporated feedback on documentation from translator --- doc/pgprobackup.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 9b236b129..a1eb2bc50 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2090,7 +2090,7 @@ pg_probackup restore -B backup_dir --instance , , , - and + , and processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU @@ -3502,7 +3502,7 @@ pg_probackup delete -B backup_dir --instance command as follows: -pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream [connection_options] [remote_options] +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream [connection_options] [remote_options] Where catchup_mode can take one of the @@ -3552,12 +3552,12 @@ pg_probackup catchup -b catchup-mode --source-pgdata= a different directory, additionally specify the option: -pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --tablespace-mapping=OLDDIR=NEWDIR +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --tablespace-mapping=OLDDIR=NEWDIR To run the catchup command on parallel threads, specify the number of threads with the option: -pg_probackup catchup -b catchup-mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --threads=num_threads +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --threads=num_threads @@ -4349,7 +4349,7 @@ pg_probackup delete -B backup_dir --instance For details, see the sections Deleting Backups, - Retention Options and + Retention Options, and Configuring Retention Policy. From 52840d78995c057df4647d597f3ec7861edf4791 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 19 Aug 2021 19:29:45 +0300 Subject: [PATCH 1778/2107] [PGPRO-5454] copying basedir to exclude the situation of overwriting memory in another thread --- src/stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stream.c b/src/stream.c index 570108cde..a53077391 100644 --- a/src/stream.c +++ b/src/stream.c @@ -42,7 +42,7 @@ static time_t stream_stop_begin = 0; */ typedef struct { - const char *basedir; + char basedir[MAXPGPATH]; PGconn *conn; /* @@ -638,7 +638,7 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption //TODO Add a comment about this calculation stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; - stream_thread_arg.basedir = stream_dst_path; + strncpy(stream_thread_arg.basedir, stream_dst_path, sizeof(stream_thread_arg.basedir)); /* * Connect in replication mode to the server. From 986e9ab958aef12b16cf113cebc3b94cb6f26e8c Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Mon, 30 Aug 2021 14:33:51 +0300 Subject: [PATCH 1779/2107] [DOC] [skip travis] Incorporated more feedback on probackup documentation from translator --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a1eb2bc50..78abf0a86 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3423,7 +3423,7 @@ pg_probackup delete -B backup_dir --instance To have a fallen-behind standby server catch up with master. - Under high write load, replicas may fail to replay WAL fast enough to keep up with master and hence may lag behind. + Under write-intensive load, replicas may fail to replay WAL fast enough to keep up with master and hence may lag behind. A usual solution to create a new replica and switch to it requires a lot of extra space and data transfer. The catchup command allows you to update an existing replica much faster by bringing differences from master. From 19a3efe07f0336194df2591ffaffa32a71b37542 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 4 Sep 2021 17:08:44 +0300 Subject: [PATCH 1780/2107] [Issue #193] added instructions for Astra Linux installation --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b7e170cf5..cc369c096 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,11 @@ sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update sudo apt-get source pg-probackup-{13,12,11,10,9.6,9.5} +#DEB Astra Linix Orel +sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' +sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update +sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}{-dbg,} + #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm yum install pg_probackup-{13,12,11,10,9.6,9.5} From ce32d19f360b76126341a3a34fe800abb57f95c8 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 4 Sep 2021 17:11:05 +0300 Subject: [PATCH 1781/2107] [Issue #193] added instructions std|ent package installation for Astra Linux Orel --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index cc369c096..344b03fb3 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,12 @@ sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PR sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg +#DEB Astra Linix Orel +sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' +sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update +sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} + + #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm yum install pg_probackup-{std,ent}-{12,11,10,9.6} From 4a7db4cf5e682441d05a8983d32c1c0a7fadc335 Mon Sep 17 00:00:00 2001 From: gsmol Date: Sun, 26 Sep 2021 01:13:04 +0300 Subject: [PATCH 1782/2107] built-in package infrastructure (#418) [PR #418] built-in package infrastructure --- .gitignore | 6 + Makefile | 7 + packaging/Dockerfiles/Dockerfile-altlinux_8 | 5 + packaging/Dockerfiles/Dockerfile-altlinux_9 | 5 + packaging/Dockerfiles/Dockerfile-astra_1.11 | 7 + packaging/Dockerfiles/Dockerfile-centos_7 | 5 + packaging/Dockerfiles/Dockerfile-centos_8 | 5 + packaging/Dockerfiles/Dockerfile-createrepo1C | 4 + packaging/Dockerfiles/Dockerfile-debian_10 | 7 + packaging/Dockerfiles/Dockerfile-debian_11 | 7 + packaging/Dockerfiles/Dockerfile-debian_8 | 7 + packaging/Dockerfiles/Dockerfile-debian_9 | 7 + .../Dockerfiles/Dockerfile-oraclelinux_6 | 5 + .../Dockerfiles/Dockerfile-oraclelinux_7 | 5 + .../Dockerfiles/Dockerfile-oraclelinux_8 | 5 + packaging/Dockerfiles/Dockerfile-rhel_7 | 7 + packaging/Dockerfiles/Dockerfile-rhel_8 | 5 + packaging/Dockerfiles/Dockerfile-rosa_6 | 5 + packaging/Dockerfiles/Dockerfile-suse_15.1 | 4 + packaging/Dockerfiles/Dockerfile-suse_15.2 | 4 + packaging/Dockerfiles/Dockerfile-ubuntu_14.04 | 7 + packaging/Dockerfiles/Dockerfile-ubuntu_16.04 | 7 + packaging/Dockerfiles/Dockerfile-ubuntu_18.04 | 7 + packaging/Dockerfiles/Dockerfile-ubuntu_18.10 | 7 + packaging/Dockerfiles/Dockerfile-ubuntu_20.04 | 8 + packaging/Makefile.pkg | 191 ++++++++++++++++++ packaging/Makefile.repo | 167 +++++++++++++++ packaging/Makefile.test | 145 +++++++++++++ packaging/Readme.md | 21 ++ packaging/pkg/Makefile.alt | 74 +++++++ packaging/pkg/Makefile.centos | 49 +++++ packaging/pkg/Makefile.debian | 99 +++++++++ packaging/pkg/Makefile.oraclelinux | 74 +++++++ packaging/pkg/Makefile.rhel | 49 +++++ packaging/pkg/Makefile.suse | 49 +++++ packaging/pkg/Makefile.ubuntu | 99 +++++++++ packaging/pkg/scripts/alt.sh | 123 +++++++++++ packaging/pkg/scripts/deb.sh | 148 ++++++++++++++ packaging/pkg/scripts/rpm.sh | 148 ++++++++++++++ packaging/pkg/scripts/suse.sh | 95 +++++++++ .../specs/deb/pg_probackup/debian/changelog | 11 + .../pkg/specs/deb/pg_probackup/debian/compat | 1 + .../pkg/specs/deb/pg_probackup/debian/control | 29 +++ .../pg_probackup/debian/pg_probackup.install | 1 + .../pkg/specs/deb/pg_probackup/debian/rules | 29 +++ .../deb/pg_probackup/debian/source/format | 1 + .../rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP | 52 +++++ .../SOURCES/GPG-KEY-PG_PROBACKUP-FORKS | 52 +++++ .../rpmbuild/SOURCES/pg_probackup-forks.repo | 6 + .../rpm/rpmbuild/SOURCES/pg_probackup.repo | 13 ++ .../rpmbuild/SPECS/pg_probackup-pgpro.spec | 71 +++++++ .../SPECS/pg_probackup-repo-forks.spec | 49 +++++ .../rpm/rpmbuild/SPECS/pg_probackup-repo.spec | 58 ++++++ .../SPECS/pg_probackup.alt.forks.spec | 67 ++++++ .../rpm/rpmbuild/SPECS/pg_probackup.alt.spec | 48 +++++ .../rpm/rpmbuild/SPECS/pg_probackup.spec | 48 +++++ packaging/pkg/tarballs/.gitkeep | 0 packaging/repo/scripts/alt.sh | 62 ++++++ packaging/repo/scripts/deb.sh | 51 +++++ packaging/repo/scripts/rpm.sh | 65 ++++++ packaging/repo/scripts/suse.sh | 72 +++++++ packaging/test/Makefile.alt | 20 ++ packaging/test/Makefile.centos | 41 ++++ packaging/test/Makefile.debian | 41 ++++ packaging/test/Makefile.oraclelinux | 41 ++++ packaging/test/Makefile.rhel | 41 ++++ packaging/test/Makefile.suse | 41 ++++ packaging/test/Makefile.ubuntu | 62 ++++++ packaging/test/scripts/alt.sh | 72 +++++++ packaging/test/scripts/alt_forks.sh | 75 +++++++ packaging/test/scripts/deb.sh | 136 +++++++++++++ packaging/test/scripts/deb_forks.sh | 149 ++++++++++++++ packaging/test/scripts/rpm.sh | 166 +++++++++++++++ packaging/test/scripts/rpm_forks.sh | 173 ++++++++++++++++ packaging/test/scripts/suse.sh | 128 ++++++++++++ packaging/test/scripts/suse_forks.sh | 5 + 76 files changed, 3656 insertions(+) create mode 100644 packaging/Dockerfiles/Dockerfile-altlinux_8 create mode 100644 packaging/Dockerfiles/Dockerfile-altlinux_9 create mode 100644 packaging/Dockerfiles/Dockerfile-astra_1.11 create mode 100644 packaging/Dockerfiles/Dockerfile-centos_7 create mode 100644 packaging/Dockerfiles/Dockerfile-centos_8 create mode 100644 packaging/Dockerfiles/Dockerfile-createrepo1C create mode 100644 packaging/Dockerfiles/Dockerfile-debian_10 create mode 100644 packaging/Dockerfiles/Dockerfile-debian_11 create mode 100644 packaging/Dockerfiles/Dockerfile-debian_8 create mode 100644 packaging/Dockerfiles/Dockerfile-debian_9 create mode 100644 packaging/Dockerfiles/Dockerfile-oraclelinux_6 create mode 100644 packaging/Dockerfiles/Dockerfile-oraclelinux_7 create mode 100644 packaging/Dockerfiles/Dockerfile-oraclelinux_8 create mode 100644 packaging/Dockerfiles/Dockerfile-rhel_7 create mode 100644 packaging/Dockerfiles/Dockerfile-rhel_8 create mode 100644 packaging/Dockerfiles/Dockerfile-rosa_6 create mode 100644 packaging/Dockerfiles/Dockerfile-suse_15.1 create mode 100644 packaging/Dockerfiles/Dockerfile-suse_15.2 create mode 100644 packaging/Dockerfiles/Dockerfile-ubuntu_14.04 create mode 100644 packaging/Dockerfiles/Dockerfile-ubuntu_16.04 create mode 100644 packaging/Dockerfiles/Dockerfile-ubuntu_18.04 create mode 100644 packaging/Dockerfiles/Dockerfile-ubuntu_18.10 create mode 100644 packaging/Dockerfiles/Dockerfile-ubuntu_20.04 create mode 100644 packaging/Makefile.pkg create mode 100644 packaging/Makefile.repo create mode 100644 packaging/Makefile.test create mode 100644 packaging/Readme.md create mode 100644 packaging/pkg/Makefile.alt create mode 100644 packaging/pkg/Makefile.centos create mode 100644 packaging/pkg/Makefile.debian create mode 100644 packaging/pkg/Makefile.oraclelinux create mode 100644 packaging/pkg/Makefile.rhel create mode 100644 packaging/pkg/Makefile.suse create mode 100644 packaging/pkg/Makefile.ubuntu create mode 100755 packaging/pkg/scripts/alt.sh create mode 100755 packaging/pkg/scripts/deb.sh create mode 100755 packaging/pkg/scripts/rpm.sh create mode 100755 packaging/pkg/scripts/suse.sh create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/changelog create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/compat create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/control create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/rules create mode 100644 packaging/pkg/specs/deb/pg_probackup/debian/source/format create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec create mode 100644 packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec create mode 100644 packaging/pkg/tarballs/.gitkeep create mode 100755 packaging/repo/scripts/alt.sh create mode 100755 packaging/repo/scripts/deb.sh create mode 100755 packaging/repo/scripts/rpm.sh create mode 100755 packaging/repo/scripts/suse.sh create mode 100644 packaging/test/Makefile.alt create mode 100644 packaging/test/Makefile.centos create mode 100644 packaging/test/Makefile.debian create mode 100644 packaging/test/Makefile.oraclelinux create mode 100644 packaging/test/Makefile.rhel create mode 100644 packaging/test/Makefile.suse create mode 100644 packaging/test/Makefile.ubuntu create mode 100755 packaging/test/scripts/alt.sh create mode 100755 packaging/test/scripts/alt_forks.sh create mode 100755 packaging/test/scripts/deb.sh create mode 100755 packaging/test/scripts/deb_forks.sh create mode 100755 packaging/test/scripts/rpm.sh create mode 100755 packaging/test/scripts/rpm_forks.sh create mode 100755 packaging/test/scripts/suse.sh create mode 100644 packaging/test/scripts/suse_forks.sh diff --git a/.gitignore b/.gitignore index 474df1c73..c0b4de331 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,12 @@ /make_dockerfile.sh /backup_restore.sh +# Packaging +/build +/packaging/pkg/tarballs/pgpro.tar.bz2 +/packaging/repo/pg_probackup +/packaging/repo/pg_probackup-forks + # Misc .python-version .vscode diff --git a/Makefile b/Makefile index 1431be4ef..4e463bf7c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ PROGRAM = pg_probackup +WORKDIR ?= $(CURDIR) +BUILDDIR = $(WORKDIR)/build/ +PBK_GIT_REPO = http://github.com/postgrespro/pg_probackup # utils OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ @@ -80,3 +83,7 @@ src/walmethods.h: $(srchome)/src/bin/pg_basebackup/walmethods.h ifeq ($(PORTNAME), aix) CC=xlc_r endif + +include packaging/Makefile.pkg +include packaging/Makefile.repo +include packaging/Makefile.test diff --git a/packaging/Dockerfiles/Dockerfile-altlinux_8 b/packaging/Dockerfiles/Dockerfile-altlinux_8 new file mode 100644 index 000000000..961aa43dd --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-altlinux_8 @@ -0,0 +1,5 @@ +FROM alt:p8 +RUN ulimit -n 1024 && apt-get update -y && apt-get install -y tar wget rpm-build +RUN ulimit -n 1024 && apt-get install -y make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && apt-get install -y git perl-devel readline-devel libxml2-devel libxslt-devel python-devel zlib-devel openssl-devel libkrb5 libkrb5-devel +RUN ulimit -n 1024 && apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-altlinux_9 b/packaging/Dockerfiles/Dockerfile-altlinux_9 new file mode 100644 index 000000000..a75728720 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-altlinux_9 @@ -0,0 +1,5 @@ +FROM alt:p9 +RUN ulimit -n 1024 && apt-get update -y && apt-get install -y tar wget rpm-build +RUN ulimit -n 1024 && apt-get install -y make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && apt-get install -y git perl-devel readline-devel libxml2-devel libxslt-devel python-devel zlib-devel openssl-devel libkrb5 libkrb5-devel +RUN ulimit -n 1024 && apt-get dist-upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-astra_1.11 b/packaging/Dockerfiles/Dockerfile-astra_1.11 new file mode 100644 index 000000000..7db4999cd --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-astra_1.11 @@ -0,0 +1,7 @@ +FROM pgpro/astra:1.11 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install reprepro -y +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-centos_7 b/packaging/Dockerfiles/Dockerfile-centos_7 new file mode 100644 index 000000000..363440e85 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-centos_7 @@ -0,0 +1,5 @@ +FROM centos:7 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-centos_8 b/packaging/Dockerfiles/Dockerfile-centos_8 new file mode 100644 index 000000000..9de1d31b1 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-centos_8 @@ -0,0 +1,5 @@ +FROM centos:8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-createrepo1C b/packaging/Dockerfiles/Dockerfile-createrepo1C new file mode 100644 index 000000000..d987c4f5f --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-createrepo1C @@ -0,0 +1,4 @@ +FROM ubuntu:17.10 +RUN apt-get -qq update -y +RUN apt-get -qq install -y reprepro rpm createrepo gnupg rsync perl less wget expect rsync dpkg-dev +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_10 b/packaging/Dockerfiles/Dockerfile-debian_10 new file mode 100644 index 000000000..f25ceeac5 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_10 @@ -0,0 +1,7 @@ +FROM debian:10 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_11 b/packaging/Dockerfiles/Dockerfile-debian_11 new file mode 100644 index 000000000..db736c193 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_11 @@ -0,0 +1,7 @@ +FROM debian:11 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_8 b/packaging/Dockerfiles/Dockerfile-debian_8 new file mode 100644 index 000000000..0be9528bb --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_8 @@ -0,0 +1,7 @@ +FROM debian:8 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_9 b/packaging/Dockerfiles/Dockerfile-debian_9 new file mode 100644 index 000000000..6ca10faa8 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_9 @@ -0,0 +1,7 @@ +FROM debian:9 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_6 b/packaging/Dockerfiles/Dockerfile-oraclelinux_6 new file mode 100644 index 000000000..04325e869 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_6 @@ -0,0 +1,5 @@ +FROM oraclelinux:6 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_7 b/packaging/Dockerfiles/Dockerfile-oraclelinux_7 new file mode 100644 index 000000000..871d920eb --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_7 @@ -0,0 +1,5 @@ +FROM oraclelinux:7 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_8 b/packaging/Dockerfiles/Dockerfile-oraclelinux_8 new file mode 100644 index 000000000..32e7cb03f --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_8 @@ -0,0 +1,5 @@ +FROM oraclelinux:8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-rhel_7 b/packaging/Dockerfiles/Dockerfile-rhel_7 new file mode 100644 index 000000000..322c44b59 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rhel_7 @@ -0,0 +1,7 @@ +FROM registry.access.redhat.com/ubi7 +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/elfutils-0.176-5.el7.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/rpm-build-4.11.3-45.el7.x86_64.rpm +RUN yum install -y tar wget yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-rhel_8 b/packaging/Dockerfiles/Dockerfile-rhel_8 new file mode 100644 index 000000000..c8e1e225e --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rhel_8 @@ -0,0 +1,5 @@ +FROM registry.access.redhat.com/ubi8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-rosa_6 b/packaging/Dockerfiles/Dockerfile-rosa_6 new file mode 100644 index 000000000..42fa913e1 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rosa_6 @@ -0,0 +1,5 @@ +FROM pgpro/rosa-6 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-suse_15.1 b/packaging/Dockerfiles/Dockerfile-suse_15.1 new file mode 100644 index 000000000..afc9434a2 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-suse_15.1 @@ -0,0 +1,4 @@ +FROM opensuse/leap:15.1 +RUN ulimit -n 1024 && zypper install -y tar wget rpm-build +RUN ulimit -n 1024 && zypper install -y gcc make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && zypper install -y git rsync diff --git a/packaging/Dockerfiles/Dockerfile-suse_15.2 b/packaging/Dockerfiles/Dockerfile-suse_15.2 new file mode 100644 index 000000000..7e56e299a --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-suse_15.2 @@ -0,0 +1,4 @@ +FROM opensuse/leap:15.2 +RUN ulimit -n 1024 && zypper install -y tar wget rpm-build +RUN ulimit -n 1024 && zypper install -y gcc make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && zypper install -y git rsync diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 new file mode 100644 index 000000000..10132f826 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 @@ -0,0 +1,7 @@ +FROM ubuntu:14.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 new file mode 100644 index 000000000..d511829c0 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 @@ -0,0 +1,7 @@ +FROM ubuntu:16.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 new file mode 100644 index 000000000..20a8567e0 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 @@ -0,0 +1,7 @@ +FROM ubuntu:18.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 b/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 new file mode 100644 index 000000000..66cefff16 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 @@ -0,0 +1,7 @@ +FROM ubuntu:18.10 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 new file mode 100644 index 000000000..79288d308 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND noninteractive +RUN ulimit -n 1024 && apt-get update -y +RUN ulimit -n 1024 && apt-get install -y devscripts +RUN ulimit -n 1024 && apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN ulimit -n 1024 && apt-get install -y cmake bison flex libboost-all-dev +RUN ulimit -n 1024 && apt-get install -y reprepro +RUN ulimit -n 1024 && apt-get upgrade -y diff --git a/packaging/Makefile.pkg b/packaging/Makefile.pkg new file mode 100644 index 000000000..bfe2043c3 --- /dev/null +++ b/packaging/Makefile.pkg @@ -0,0 +1,191 @@ +ifeq ($(PBK_EDITION),std) + PBK_PKG_REPO = pg_probackup-forks + PBK_EDITION_FULL = Standart + PKG_NAME_SUFFIX = std- +else ifeq ($(PBK_EDITION),ent) + PBK_PKG_REPO = pg_probackup-forks + PBK_EDITION_FULL = Enterprise + PKG_NAME_SUFFIX = ent- +else + PBK_PKG_REPO = pg_probackup + PBK_EDITION_FULL = + PBK_EDITION = + PKG_NAME_SUFFIX = +endif + +check_env: + @if [ -z ${PBK_VERSION} ] ; then \ + echo "Env variable PBK_VERSION is not set" ; \ + false ; \ + fi + + @if [ -z ${PBK_RELEASE} ] ; then \ + echo "Env variable PBK_RELEASE is not set" ; \ + false ; \ + fi + + @if [ -z ${PBK_HASH} ] ; then \ + echo "Env variable PBK_HASH is not set" ; \ + false ; \ + fi + +pkg: check_env build/prepare build/all + @echo Build for all platform: done + +build/prepare: + mkdir -p build + +build/clean: build/prepare + find $(BUILDDIR) -maxdepth 1 -type f -exec rm -f {} \; + +build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt build/suse # build/rhel + @echo Packaging is done + +### DEBIAN +build/debian: build/debian_8 build/debian_9 build/debian_10 build/debian_11 + @echo Debian: done + +build/debian_8: build/debian_8_9.5 build/debian_8_9.6 build/debian_8_10 build/debian_8_11 build/debian_8_12 build/debian_8_13 + @echo Debian 8: done + +build/debian_9: build/debian_9_9.5 build/debian_9_9.6 build/debian_9_10 build/debian_9_11 build/debian_9_12 build/debian_9_13 + @echo Debian 9: done + +build/debian_10: build/debian_10_9.5 build/debian_10_9.6 build/debian_10_10 build/debian_10_11 build/debian_10_12 build/debian_10_13 + @echo Debian 10: done + +build/debian_11: build/debian_11_9.5 build/debian_11_9.6 build/debian_11_10 build/debian_11_11 build/debian_11_12 build/debian_11_13 + @echo Debian 11: done + +### UBUNTU +build/ubuntu: build/ubuntu_14.04 build/ubuntu_16.04 build/ubuntu_18.04 build/ubuntu_20.04 + @echo Ubuntu: done + +build/ubuntu_14.04: build/ubuntu_14.04_9.5 build/ubuntu_14.04_9.6 build/ubuntu_14.04_10 build/ubuntu_14.04_11 build/ubuntu_14.04_12 build/ubuntu_14.04_13 + @echo Ubuntu 14.04: done + +build/ubuntu_16.04: build/ubuntu_16.04_9.5 build/ubuntu_16.04_9.6 build/ubuntu_16.04_10 build/ubuntu_16.04_11 build/ubuntu_16.04_12 build/ubuntu_16.04_13 + @echo Ubuntu 16.04: done + +build/ubuntu_18.04: build/ubuntu_18.04_9.5 build/ubuntu_18.04_9.6 build/ubuntu_18.04_10 build/ubuntu_18.04_11 build/ubuntu_18.04_12 build/ubuntu_18.04_13 + @echo Ubuntu 18.04: done + +build/ubuntu_20.04: build/ubuntu_20.04_9.5 build/ubuntu_20.04_9.6 build/ubuntu_20.04_10 build/ubuntu_20.04_11 build/ubuntu_20.04_12 build/ubuntu_20.04_13 + @echo Ubuntu 20.04: done + +define build_deb + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg-probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg-probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/deb.sh +endef + +include packaging/pkg/Makefile.debian +include packaging/pkg/Makefile.ubuntu + +# CENTOS +build/centos: build/centos_7 build/centos_8 #build/rpm_repo_package_centos + @echo Centos: done + +build/centos_7: build/centos_7_9.5 build/centos_7_9.6 build/centos_7_10 build/centos_7_11 build/centos_7_12 build/centos_7_13 + @echo Centos 7: done + +build/centos_8: build/centos_8_9.5 build/centos_8_9.6 build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 + @echo Centos 8: done + +# Oracle Linux +build/oraclelinux: build/oraclelinux_6 build/oraclelinux_7 build/oraclelinux_8 #build/rpm_repo_package_oraclelinux + @echo Oraclelinux: done + +build/oraclelinux_6: build/oraclelinux_6_9.5 build/oraclelinux_6_9.6 build/oraclelinux_6_10 build/oraclelinux_6_11 build/oraclelinux_6_12 build/oraclelinux_6_13 + @echo Oraclelinux 6: done + +build/oraclelinux_7: build/oraclelinux_7_9.5 build/oraclelinux_7_9.6 build/oraclelinux_7_10 build/oraclelinux_7_11 build/oraclelinux_7_12 build/oraclelinux_7_13 + @echo Oraclelinux 7: done + +build/oraclelinux_8: build/oraclelinux_8_9.5 build/oraclelinux_8_9.6 build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 + @echo Oraclelinux 8: done + +# RHEL +build/rhel: build/rhel_7 build/rhel_8 #build/rpm_repo_package_rhel + @echo Rhel: done + +build/rhel_7: build/rhel_7_9.5 build/rhel_7_9.6 build/rhel_7_10 build/rhel_7_11 build/rhel_7_12 build/rhel_7_13 + @echo Rhel 7: done + +build/rhel_8: build/rhel_8_9.5 build/rhel_8_9.6 build/rhel_8_10 build/rhel_8_11 build/rhel_8_12 build/rhel_8_13 + @echo Rhel 8: done + + +define build_rpm + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/rpm.sh +endef + +include packaging/pkg/Makefile.centos +include packaging/pkg/Makefile.rhel +include packaging/pkg/Makefile.oraclelinux + + +# Alt Linux +build/alt: build/alt_7 build/alt_8 build/alt_9 + @echo Alt Linux: done + +build/alt_7: build/alt_7_9.5 build/alt_7_9.6 build/alt_7_10 build/alt_7_11 build/alt_7_12 build/alt_7_13 + @echo Alt Linux 7: done + +build/alt_8: build/alt_8_9.5 build/alt_8_9.6 build/alt_8_10 build/alt_8_11 build/alt_8_12 build/alt_8_13 + @echo Alt Linux 8: done + +build/alt_9: build/alt_9_9.5 build/alt_9_9.6 build/alt_9_10 build/alt_9_11 build/alt_9_12 build/alt_9_13 + @echo Alt Linux 9: done + +define build_alt + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/alt.sh +endef + +include packaging/pkg/Makefile.alt + +# SUSE Linux +build/suse: build/suse_15.1 build/suse_15.2 + @echo Suse: done + +build/suse_15.1: build/suse_15.1_9.5 build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 + @echo Rhel 15.1: done + +build/suse_15.2: build/suse_15.2_9.5 build/suse_15.2_9.6 build/suse_15.2_10 build/suse_15.2_11 build/suse_15.2_12 build/suse_15.2_13 + @echo Rhel 15.1: done + +define build_suse + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/suse.sh +endef + +include packaging/pkg/Makefile.suse diff --git a/packaging/Makefile.repo b/packaging/Makefile.repo new file mode 100644 index 000000000..986c827e9 --- /dev/null +++ b/packaging/Makefile.repo @@ -0,0 +1,167 @@ +#### REPO BUILD #### +repo: check_env repo/debian repo/ubuntu repo/centos repo/oraclelinux repo/alt repo/suse repo_finish #repo/rhel + @echo Build repo for all platform: done + +# Debian +repo/debian: build/repo_debian_8 build/repo_debian_9 build/repo_debian_10 build/repo_debian_11 + @echo Build repo for debian platforms: done + +build/repo_debian_8: + $(call build_repo_deb,debian,8,jessie) + touch build/repo_debian_8 + +build/repo_debian_9: + $(call build_repo_deb,debian,9,stretch) + touch build/repo_debian_9 + +build/repo_debian_10: + $(call build_repo_deb,debian,10,buster) + touch build/repo_debian_10 + +build/repo_debian_11: + $(call build_repo_deb,debian,11,bullseye) + touch build/repo_debian_11 + +# Ubuntu +repo/ubuntu: build/repo_ubuntu_14.04 build/repo_ubuntu_16.04 build/repo_ubuntu_18.04 build/repo_ubuntu_20.04 + @echo Build repo for ubuntu platforms: done + +build/repo_ubuntu_14.04: + $(call build_repo_deb,ubuntu,14.04,trusty) + touch build/repo_ubuntu_14.04 + +build/repo_ubuntu_16.04: + $(call build_repo_deb,ubuntu,16.04,xenial) + touch build/repo_ubuntu_16.04 + +build/repo_ubuntu_18.04: + $(call build_repo_deb,ubuntu,18.04,bionic) + touch build/repo_ubuntu_18.04 + +build/repo_ubuntu_20.04: + $(call build_repo_deb,ubuntu,20.04,focal) + touch build/repo_ubuntu_20.04 + +# Centos +repo/centos: build/repo_centos_7 build/repo_centos_8 + @echo Build repo for centos platforms: done + +build/repo_centos_7: + $(call build_repo_rpm,centos,7,,) + touch build/repo_centos_7 + +build/repo_centos_8: + $(call build_repo_rpm,centos,8,,) + touch build/repo_centos_8 + +# Oraclelinux +repo/oraclelinux: build/repo_oraclelinux_6 build/repo_oraclelinux_7 build/repo_oraclelinux_8 + @echo Build repo for oraclelinux platforms: done + +build/repo_oraclelinux_6: + $(call build_repo_rpm,oraclelinux,6,6Server) + touch build/repo_oraclelinux_6 + +build/repo_oraclelinux_7: + $(call build_repo_rpm,oraclelinux,7,7Server) + touch build/repo_oraclelinux_7 + +build/repo_oraclelinux_8: + $(call build_repo_rpm,oraclelinux,8,,) + touch build/repo_oraclelinux_8 + +# RHEL +repo/rhel: build/repo_rhel_7 build/repo_rhel_8 + @echo Build repo for rhel platforms: done + +build/repo_rhel_7: + $(call build_repo_rpm,rhel,7,7Server) + touch build/repo_rhel_7 + +build/repo_rhel_8: + $(call build_repo_rpm,rhel,8,,) + touch build/repo_rhel_8 + +# ALT +repo/alt: build/repo_alt_7 build/repo_alt_8 build/repo_alt_9 + @echo Build repo for alt platforms: done + +build/repo_alt_7: + $(call build_repo_alt,alt,7,,) + touch build/repo_alt_7 + +build/repo_alt_8: + $(call build_repo_alt,alt,8,,) + touch build/repo_alt_8 + +build/repo_alt_9: + $(call build_repo_alt,alt,9,,) + touch build/repo_alt_9 + +# SUSE +repo/suse: build/repo_suse_15.1 build/repo_suse_15.2 + @echo Build repo for suse platforms: done + +build/repo_suse_15.1: + $(call build_repo_suse,suse,15.1,,) + touch build/repo_suse_15.1 + +build/repo_suse_15.2: + $(call build_repo_suse,suse,15.2,,) + touch build/repo_suse_15.2 + +repo_finish: +# cd build/data/www/$(PBK_PKG_REPO)/ + cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/rpm && sudo ln -nsf $(PBK_VERSION) latest + cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/srpm && sudo ln -nsf $(PBK_VERSION) latest + +# sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/rpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/rpm/latest +# sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/srpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/srpm/latest + +define build_repo_deb + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/repo /app/repo/scripts/deb.sh +endef + +define build_repo_rpm + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/repo /app/repo/scripts/rpm.sh +endef + +define build_repo_alt + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/$1:$2 /app/repo/scripts/alt.sh +endef + +define build_repo_suse + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/$1:$2 /app/repo/scripts/suse.sh +endef diff --git a/packaging/Makefile.test b/packaging/Makefile.test new file mode 100644 index 000000000..fbb415c46 --- /dev/null +++ b/packaging/Makefile.test @@ -0,0 +1,145 @@ +ifeq ($(PBK_EDITION),std) + SCRIPT_SUFFIX = _forks +else ifeq ($(PBK_EDITION),ent) + SCRIPT_SUFFIX = _forks +else + SCRIPT_SUFFIX = +endif + +test: build/test_all + @echo Test for all platform: done + +build/test_all: build/test_debian build/test_ubuntu build/test_centos build/test_oraclelinux build/test_alt build/test_suse # build/test_rhel + @echo Package testing is done + +### DEBIAN +build/test_debian: build/test_debian_9 build/test_debian_10 build/test_debian_11 + @echo Debian: done + +build/test_debian_9: build/test_debian_9_9.6 build/test_debian_9_10 build/test_debian_9_11 build/test_debian_9_12 build/test_debian_9_13 + @echo Debian 9: done + +build/test_debian_10: build/test_debian_10_9.6 build/test_debian_10_10 build/test_debian_10_11 build/test_debian_10_12 build/test_debian_10_13 + @echo Debian 10: done + +build/test_debian_11: build/test_debian_11_9.6 build/test_debian_11_10 build/test_debian_11_11 build/test_debian_11_12 build/test_debian_11_13 + @echo Debian 11: done + +### UBUNTU +build/test_ubuntu: build/test_ubuntu_16.04 build/test_ubuntu_18.04 build/test_ubuntu_20.04 + @echo Ubuntu: done + +build/test_ubuntu_16.04: build/test_ubuntu_16.04_9.6 build/test_ubuntu_16.04_10 build/test_ubuntu_16.04_11 build/test_ubuntu_16.04_12 build/test_ubuntu_16.04_13 + @echo Ubuntu 16.04: done + +build/test_ubuntu_18.04: build/test_ubuntu_18.04_9.6 build/test_ubuntu_18.04_10 build/test_ubuntu_18.04_11 build/test_ubuntu_18.04_12 build/test_ubuntu_18.04_13 + @echo Ubuntu 18.04: done + +build/test_ubuntu_20.04: build/test_ubuntu_20.04_9.6 build/test_ubuntu_20.04_10 build/test_ubuntu_20.04_11 build/test_ubuntu_20.04_12 build/test_ubuntu_20.04_13 + @echo Ubuntu 20.04: done + +define test_deb + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg-probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/deb$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.debian +include packaging/test/Makefile.ubuntu + +# CENTOS +build/test_centos: build/test_centos_7 build/test_centos_8 + @echo Centos: done + +build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 build/test_centos_7_13 + @echo Centos 7: done + +build/test_centos_8: build/test_centos_8_9.6 build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 build/test_centos_8_13 + @echo Centos 8: done + +# Oracle Linux +build/test_oraclelinux: build/test_oraclelinux_7 build/test_oraclelinux_8 + @echo Oraclelinux: done + +build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 build/test_oraclelinux_7_13 + @echo Oraclelinux 7: done + +build/test_oraclelinux_8: build/test_oraclelinux_8_9.6 build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 build/test_oraclelinux_8_13 + @echo Oraclelinux 8: done + +# RHEL +build/test_rhel: build/test_rhel_7 build/test_rhel_8 + @echo Rhel: done + +build/test_rhel_7: build/test_rhel_7_9.5 build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 build/test_rhel_7_13 + @echo Rhel 7: done + +build/test_rhel_8: build/test_rhel_8_9.5 build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 build/test_rhel_8_13 + @echo Rhel 8: done + +define test_rpm + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/rpm$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.centos +include packaging/test/Makefile.rhel +include packaging/test/Makefile.oraclelinux + +# Alt Linux +build/test_alt: build/test_alt_9 + @echo Alt Linux: done + +build/test_alt_9: build/test_alt_9_9.6 build/test_alt_9_10 build/test_alt_9_11 build/test_alt_9_12 build/test_alt_9_13 + @echo Alt Linux 9: done + +define test_alt + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/alt$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.alt + +# SUSE Linux +build/test_suse: build/test_suse_15.1 build/test_suse_15.2 + @echo Suse: done + +build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13 + @echo Rhel 15.1: done + +build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 + @echo Rhel 15.1: done + +define test_suse + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/suse$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.suse diff --git a/packaging/Readme.md b/packaging/Readme.md new file mode 100644 index 000000000..c6cbf16b5 --- /dev/null +++ b/packaging/Readme.md @@ -0,0 +1,21 @@ +Example: +``` +export PBK_VERSION=2.4.17 +export PBK_HASH=57f871accce2604 +export PBK_RELEASE=1 +export PBK_EDITION=std|ent +make pkg +``` + +To build binaries for PostgresPro Standart or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/tarballs` directory: +``` +cd packaging/tarballs +git clone pgpro_repo pgpro +tar -cjSf pgpro.tar.bz2 pgpro +``` + +To build repo the gpg keys for package signing must be present ... +Repo must be build using 1 thread (due to debian bullshit): +``` +make repo -j1 +``` diff --git a/packaging/pkg/Makefile.alt b/packaging/pkg/Makefile.alt new file mode 100644 index 000000000..e3fbae26e --- /dev/null +++ b/packaging/pkg/Makefile.alt @@ -0,0 +1,74 @@ +# ALT 7 +build/alt_7_9.5: + $(call build_alt,alt,7,,9.5,9.5.25) + touch build/alt_7_9.5 + +build/alt_7_9.6: + $(call build_alt,alt,7,,9.6,9.6.21) + touch build/alt_7_9.6 + +build/alt_7_10: + $(call build_alt,alt,7,,10,10.17) + touch build/alt_7_10 + +build/alt_7_11: + $(call build_alt,alt,7,,11,11.11) + touch build/alt_7_11 + +build/alt_7_12: + $(call build_alt,alt,7,,12,12.6) + touch build/alt_7_12 + +build/alt_7_13: + $(call build_alt,alt,7,,13,13.2) + touch build/alt_7_13 + +# ALT 8 +build/alt_8_9.5: + $(call build_alt,alt,8,,9.5,9.5.25) + touch build/alt_8_9.5 + +build/alt_8_9.6: + $(call build_alt,alt,8,,9.6,9.6.21) + touch build/alt_8_9.6 + +build/alt_8_10: + $(call build_alt,alt,8,,10,10.17) + touch build/alt_8_10 + +build/alt_8_11: + $(call build_alt,alt,8,,11,11.11) + touch build/alt_8_11 + +build/alt_8_12: + $(call build_alt,alt,8,,12,12.6) + touch build/alt_8_12 + +build/alt_8_13: + $(call build_alt,alt,8,,13,13.2) + touch build/alt_8_13 + +# ALT 9 +build/alt_9_9.5: + $(call build_alt,alt,9,,9.5,9.5.25) + touch build/alt_9_9.5 + +build/alt_9_9.6: + $(call build_alt,alt,9,,9.6,9.6.21) + touch build/alt_9_9.6 + +build/alt_9_10: + $(call build_alt,alt,9,,10,10.17) + touch build/alt_9_10 + +build/alt_9_11: + $(call build_alt,alt,9,,11,11.11) + touch build/alt_9_11 + +build/alt_9_12: + $(call build_alt,alt,9,,12,12.6) + touch build/alt_9_12 + +build/alt_9_13: + $(call build_alt,alt,9,,13,13.2) + touch build/alt_9_13 diff --git a/packaging/pkg/Makefile.centos b/packaging/pkg/Makefile.centos new file mode 100644 index 000000000..9353b2cde --- /dev/null +++ b/packaging/pkg/Makefile.centos @@ -0,0 +1,49 @@ +# CENTOS 7 +build/centos_7_9.5: + $(call build_rpm,centos,7,,9.5,9.5.25) + touch build/centos_7_9.5 + +build/centos_7_9.6: + $(call build_rpm,centos,7,,9.6,9.6.21) + touch build/centos_7_9.6 + +build/centos_7_10: + $(call build_rpm,centos,7,,10,10.16) + touch build/centos_7_10 + +build/centos_7_11: + $(call build_rpm,centos,7,,11,11.11) + touch build/centos_7_11 + +build/centos_7_12: + $(call build_rpm,centos,7,,12,12.6) + touch build/centos_7_12 + +build/centos_7_13: + $(call build_rpm,centos,7,,13,13.2) + touch build/centos_7_13 + +# CENTOS 8 +build/centos_8_9.5: + $(call build_rpm,centos,8,,9.5,9.5.25) + touch build/centos_8_9.5 + +build/centos_8_9.6: + $(call build_rpm,centos,8,,9.6,9.6.21) + touch build/centos_8_9.6 + +build/centos_8_10: + $(call build_rpm,centos,8,,10,10.16) + touch build/centos_8_10 + +build/centos_8_11: + $(call build_rpm,centos,8,,11,11.11) + touch build/centos_8_11 + +build/centos_8_12: + $(call build_rpm,centos,8,,12,12.6) + touch build/centos_8_12 + +build/centos_8_13: + $(call build_rpm,centos,8,,13,13.2) + touch build/centos_8_13 diff --git a/packaging/pkg/Makefile.debian b/packaging/pkg/Makefile.debian new file mode 100644 index 000000000..9625a14e9 --- /dev/null +++ b/packaging/pkg/Makefile.debian @@ -0,0 +1,99 @@ +# DEBIAN 8 +build/debian_8_9.5: + $(call build_deb,debian,8,jessie,9.5,9.5.25) + touch build/debian_8_9.5 + +build/debian_8_9.6: + $(call build_deb,debian,8,jessie,9.6,9.6.21) + touch build/debian_8_9.6 + +build/debian_8_10: + $(call build_deb,debian,8,jessie,10,10.16) + touch build/debian_8_10 + +build/debian_8_11: + $(call build_deb,debian,8,jessie,11,11.11) + touch build/debian_8_11 + +build/debian_8_12: + $(call build_deb,debian,8,jessie,12,12.6) + touch build/debian_8_12 + +build/debian_8_13: + $(call build_deb,debian,8,jessie,13,13.2) + touch build/debian_8_13 + +# DEBIAN 9 +build/debian_9_9.5: + $(call build_deb,debian,9,stretch,9.5,9.5.25) + touch build/debian_9_9.5 + +build/debian_9_9.6: + $(call build_deb,debian,9,stretch,9.6,9.6.21) + touch build/debian_9_9.6 + +build/debian_9_10: + $(call build_deb,debian,9,stretch,10,10.16) + touch build/debian_9_10 + +build/debian_9_11: + $(call build_deb,debian,9,stretch,11,11.11) + touch build/debian_9_11 + +build/debian_9_12: + $(call build_deb,debian,9,stretch,12,12.6) + touch build/debian_9_12 + +build/debian_9_13: + $(call build_deb,debian,9,stretch,13,13.2) + touch build/debian_9_13 + +# DEBIAN 10 +build/debian_10_9.5: + $(call build_deb,debian,10,buster,9.5,9.5.25) + touch build/debian_10_9.5 + +build/debian_10_9.6: + $(call build_deb,debian,10,buster,9.6,9.6.21) + touch build/debian_10_9.6 + +build/debian_10_10: + $(call build_deb,debian,10,buster,10,10.16) + touch build/debian_10_10 + +build/debian_10_11: + $(call build_deb,debian,10,buster,11,11.11) + touch build/debian_10_11 + +build/debian_10_12: + $(call build_deb,debian,10,buster,12,12.6) + touch build/debian_10_12 + +build/debian_10_13: + $(call build_deb,debian,10,buster,13,13.2) + touch build/debian_10_13 + +# DEBIAN 11 +build/debian_11_9.5: + $(call build_deb,debian,11,bullseye,9.5,9.5.25) + touch build/debian_11_9.5 + +build/debian_11_9.6: + $(call build_deb,debian,11,bullseye,9.6,9.6.21) + touch build/debian_11_9.6 + +build/debian_11_10: + $(call build_deb,debian,11,bullseye,10,10.16) + touch build/debian_11_10 + +build/debian_11_11: + $(call build_deb,debian,11,bullseye,11,11.11) + touch build/debian_11_11 + +build/debian_11_12: + $(call build_deb,debian,11,bullseye,12,12.6) + touch build/debian_11_12 + +build/debian_11_13: + $(call build_deb,debian,11,bullseye,13,13.2) + touch build/debian_11_13 diff --git a/packaging/pkg/Makefile.oraclelinux b/packaging/pkg/Makefile.oraclelinux new file mode 100644 index 000000000..f4eada23f --- /dev/null +++ b/packaging/pkg/Makefile.oraclelinux @@ -0,0 +1,74 @@ +# ORACLE LINUX 6 +build/oraclelinux_6_9.5: + $(call build_rpm,oraclelinux,6,,9.5,9.5.25) + touch build/oraclelinux_6_9.5 + +build/oraclelinux_6_9.6: + $(call build_rpm,oraclelinux,6,,9.6,9.6.21) + touch build/oraclelinux_6_9.6 + +build/oraclelinux_6_10: + $(call build_rpm,oraclelinux,6,,10,10.16) + touch build/oraclelinux_6_10 + +build/oraclelinux_6_11: + $(call build_rpm,oraclelinux,6,,11,11.11) + touch build/oraclelinux_6_11 + +build/oraclelinux_6_12: + $(call build_rpm,oraclelinux,6,,12,12.6) + touch build/oraclelinux_6_12 + +build/oraclelinux_6_13: + $(call build_rpm,oraclelinux,6,,13,13.2) + touch build/oraclelinux_6_13 + +# ORACLE LINUX 7 +build/oraclelinux_7_9.5: + $(call build_rpm,oraclelinux,7,,9.5,9.5.25) + touch build/oraclelinux_7_9.5 + +build/oraclelinux_7_9.6: + $(call build_rpm,oraclelinux,7,,9.6,9.6.21) + touch build/oraclelinux_7_9.6 + +build/oraclelinux_7_10: + $(call build_rpm,oraclelinux,7,,10,10.16) + touch build/oraclelinux_7_10 + +build/oraclelinux_7_11: + $(call build_rpm,oraclelinux,7,,11,11.11) + touch build/oraclelinux_7_11 + +build/oraclelinux_7_12: + $(call build_rpm,oraclelinux,7,,12,12.6) + touch build/oraclelinux_7_12 + +build/oraclelinux_7_13: + $(call build_rpm,oraclelinux,7,,13,13.2) + touch build/oraclelinux_7_13 + +# ORACLE LINUX 8 +build/oraclelinux_8_9.5: + $(call build_rpm,oraclelinux,8,,9.5,9.5.25) + touch build/oraclelinux_8_9.5 + +build/oraclelinux_8_9.6: + $(call build_rpm,oraclelinux,8,,9.6,9.6.21) + touch build/oraclelinux_8_9.6 + +build/oraclelinux_8_10: + $(call build_rpm,oraclelinux,8,,10,10.16) + touch build/oraclelinux_8_10 + +build/oraclelinux_8_11: + $(call build_rpm,oraclelinux,8,,11,11.11) + touch build/oraclelinux_8_11 + +build/oraclelinux_8_12: + $(call build_rpm,oraclelinux,8,,12,12.6) + touch build/oraclelinux_8_12 + +build/oraclelinux_8_13: + $(call build_rpm,oraclelinux,8,,13,13.2) + touch build/oraclelinux_8_13 diff --git a/packaging/pkg/Makefile.rhel b/packaging/pkg/Makefile.rhel new file mode 100644 index 000000000..f266966cf --- /dev/null +++ b/packaging/pkg/Makefile.rhel @@ -0,0 +1,49 @@ +# RHEL 7 +build/rhel_7_9.5: + $(call build_rpm,rhel,7,7Server,9.5,9.5.25) + touch build/rhel_7_9.5 + +build/rhel_7_9.6: + $(call build_rpm,rhel,7,7Server,9.6,9.6.21) + touch build/rhel_7_9.6 + +build/rhel_7_10: + $(call build_rpm,rhel,7,7Server,10,10.16) + touch build/rhel_7_10 + +build/rhel_7_11: + $(call build_rpm,rhel,7,7Server,11,11.11) + touch build/rhel_7_11 + +build/rhel_7_12: + $(call build_rpm,rhel,7,7Server,12,12.6) + touch build/rhel_7_12 + +build/rhel_7_13: + $(call build_rpm,rhel,7,7Server,13,13.2) + touch build/rhel_7_13 + +# RHEL 8 +build/rhel_8_9.5: + $(call build_rpm,rhel,8,8Server,9.5,9.5.25) + touch build/rhel_8_9.5 + +build/rhel_8_9.6: + $(call build_rpm,rhel,8,8Server,9.6,9.6.21) + touch build/rhel_8_9.6 + +build/rhel_8_10: + $(call build_rpm,rhel,8,8Server,10,10.16) + touch build/rhel_8_10 + +build/rhel_8_11: + $(call build_rpm,rhel,8,8Server,11,11.11) + touch build/rhel_8_11 + +build/rhel_8_12: + $(call build_rpm,rhel,8,8Server,12,12.6) + touch build/rhel_8_12 + +build/rhel_8_13: + $(call build_rpm,rhel,8,8Server,13,13.2) + touch build/rhel_8_13 diff --git a/packaging/pkg/Makefile.suse b/packaging/pkg/Makefile.suse new file mode 100644 index 000000000..a9f1eaa36 --- /dev/null +++ b/packaging/pkg/Makefile.suse @@ -0,0 +1,49 @@ +# Suse 15.1 +build/suse_15.1_9.5: + $(call build_suse,suse,15.1,,9.5,9.5.25) + touch build/suse_15.1_9.5 + +build/suse_15.1_9.6: + $(call build_suse,suse,15.1,,9.6,9.6.21) + touch build/suse_15.1_9.6 + +build/suse_15.1_10: + $(call build_suse,suse,15.1,,10,10.16) + touch build/suse_15.1_10 + +build/suse_15.1_11: + $(call build_suse,suse,15.1,,11,11.11) + touch build/suse_15.1_11 + +build/suse_15.1_12: + $(call build_suse,suse,15.1,,12,12.6) + touch build/suse_15.1_12 + +build/suse_15.1_13: + $(call build_suse,suse,15.1,,13,13.2) + touch build/suse_15.1_13 + +# Suse 15.2 +build/suse_15.2_9.5: + $(call build_suse,suse,15.2,,9.5,9.5.25) + touch build/suse_15.2_9.5 + +build/suse_15.2_9.6: + $(call build_suse,suse,15.2,,9.6,9.6.21) + touch build/suse_15.2_9.6 + +build/suse_15.2_10: + $(call build_suse,suse,15.2,,10,10.16) + touch build/suse_15.2_10 + +build/suse_15.2_11: + $(call build_suse,suse,15.2,,11,11.11) + touch build/suse_15.2_11 + +build/suse_15.2_12: + $(call build_suse,suse,15.2,,12,12.6) + touch build/suse_15.2_12 + +build/suse_15.2_13: + $(call build_suse,suse,15.2,,13,13.2) + touch build/suse_15.2_13 diff --git a/packaging/pkg/Makefile.ubuntu b/packaging/pkg/Makefile.ubuntu new file mode 100644 index 000000000..3f76de516 --- /dev/null +++ b/packaging/pkg/Makefile.ubuntu @@ -0,0 +1,99 @@ +# UBUNTU 20.04 +build/ubuntu_20.04_9.5: + $(call build_deb,ubuntu,20.04,focal,9.5,9.5.25) + touch build/ubuntu_20.04_9.5 + +build/ubuntu_20.04_9.6: + $(call build_deb,ubuntu,20.04,focal,9.6,9.6.21) + touch build/ubuntu_20.04_9.6 + +build/ubuntu_20.04_10: + $(call build_deb,ubuntu,20.04,focal,10,10.16) + touch build/ubuntu_20.04_10 + +build/ubuntu_20.04_11: + $(call build_deb,ubuntu,20.04,focal,11,11.11) + touch build/ubuntu_20.04_11 + +build/ubuntu_20.04_12: + $(call build_deb,ubuntu,20.04,focal,12,12.6) + touch build/ubuntu_20.04_12 + +build/ubuntu_20.04_13: + $(call build_deb,ubuntu,20.04,focal,13,13.2) + touch build/ubuntu_20.04_13 + +# UBUNTU 18.04 +build/ubuntu_18.04_9.5: + $(call build_deb,ubuntu,18.04,bionic,9.5,9.5.25) + touch build/ubuntu_18.04_9.5 + +build/ubuntu_18.04_9.6: + $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.21) + touch build/ubuntu_18.04_9.6 + +build/ubuntu_18.04_10: + $(call build_deb,ubuntu,18.04,bionic,10,10.16) + touch build/ubuntu_18.04_10 + +build/ubuntu_18.04_11: + $(call build_deb,ubuntu,18.04,bionic,11,11.11) + touch build/ubuntu_18.04_11 + +build/ubuntu_18.04_12: + $(call build_deb,ubuntu,18.04,bionic,12,12.6) + touch build/ubuntu_18.04_12 + +build/ubuntu_18.04_13: + $(call build_deb,ubuntu,18.04,bionic,13,13.2) + touch build/ubuntu_18.04_13 + +# UBUNTU 16.04 +build/ubuntu_16.04_9.5: + $(call build_deb,ubuntu,16.04,xenial,9.5,9.5.25) + touch build/ubuntu_16.04_9.5 + +build/ubuntu_16.04_9.6: + $(call build_deb,ubuntu,16.04,xenial,9.6,9.6.21) + touch build/ubuntu_16.04_9.6 + +build/ubuntu_16.04_10: + $(call build_deb,ubuntu,16.04,xenial,10,10.16) + touch build/ubuntu_16.04_10 + +build/ubuntu_16.04_11: + $(call build_deb,ubuntu,16.04,xenial,11,11.11) + touch build/ubuntu_16.04_11 + +build/ubuntu_16.04_12: + $(call build_deb,ubuntu,16.04,xenial,12,12.6) + touch build/ubuntu_16.04_12 + +build/ubuntu_16.04_13: + $(call build_deb,ubuntu,16.04,xenial,13,13.2) + touch build/ubuntu_16.04_13 + +# UBUNTU 14.04 +build/ubuntu_14.04_9.5: + $(call build_deb,ubuntu,14.04,trusty,9.5,9.5.25) + touch build/ubuntu_14.04_9.5 + +build/ubuntu_14.04_9.6: + $(call build_deb,ubuntu,14.04,trusty,9.6,9.6.21) + touch build/ubuntu_14.04_9.6 + +build/ubuntu_14.04_10: + $(call build_deb,ubuntu,14.04,trusty,10,10.16) + touch build/ubuntu_14.04_10 + +build/ubuntu_14.04_11: + $(call build_deb,ubuntu,14.04,trusty,11,11.11) + touch build/ubuntu_14.04_11 + +build/ubuntu_14.04_12: + $(call build_deb,ubuntu,14.04,trusty,12,12.6) + touch build/ubuntu_14.04_12 + +build/ubuntu_14.04_13: + $(call build_deb,ubuntu,14.04,trusty,13,13.2) + touch build/ubuntu_14.04_13 diff --git a/packaging/pkg/scripts/alt.sh b/packaging/pkg/scripts/alt.sh new file mode 100755 index 000000000..ae3c713fa --- /dev/null +++ b/packaging/pkg/scripts/alt.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# THere is no std/ent packages for PG 9.5 +if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 +apt-get update -y + +mkdir /root/build +cd /root/build + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +# tarball it +if [[ ${PBK_EDITION} == '' ]] ; then + tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} + mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES + rm -rf pg_probackup-${PKG_VERSION} +else + mv pg_probackup-${PKG_VERSION} /root/rpmbuild/SOURCES +fi + + +if [[ ${PBK_EDITION} == '' ]] ; then + # Download PostgreSQL source + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O postgresql-${PG_VERSION}.tar.bz2 + mv postgresql-${PG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES/ + +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/rpmbuild/SOURCES/ + cd /root/rpmbuild/SOURCES/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + rm -rf .git + + cd /root/rpmbuild/SOURCES/ + mv pgpro postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + chown -R root:root postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + + +#cd /root/rpmbuild/SOURCES +#sed -i "s/@PG_VERSION@/${PKG_VERSION}/" pg_probackup.repo + +# build postgresql +echo '%_allow_root_build yes' > /root/.rpmmacros +echo '%_topdir %{getenv:HOME}/rpmbuild' >> /root/.rpmmacros + +cd /root/rpmbuild/SPECS +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.alt.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.alt.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.alt.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.alt.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.alt.spec +else + sed -i "s/@EDITION@/${PBK_EDITION}/" pg_probackup.alt.forks.spec + sed -i "s/@EDITION_FULL@/${PBK_EDITION_FULL}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.alt.forks.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.alt.forks.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.alt.forks.spec + + if [ ${PG_VERSION} != '9.6' ]; then + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup.alt.forks.spec + fi +fi + +# ALT Linux suck as detecting dependecies, so the manual hint is required +if [ ${DISTRIB_VERSION} == '7' ]; then + apt-get install libpq5.10 + +elif [ ${DISTRIB_VERSION} == '8' ]; then + apt-get install libpq5.12 + +else + apt-get install libpq5 +fi + +# install dependencies +#stolen from postgrespro +apt-get install -y flex libldap-devel libpam-devel libreadline-devel libssl-devel + +if [[ ${PBK_EDITION} == '' ]] ; then + + # build pg_probackup + rpmbuild -bs pg_probackup.alt.spec + rpmbuild -ba pg_probackup.alt.spec #2>&1 | tee -ai /app/out/build.log + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out +else + rpmbuild -ba pg_probackup.alt.forks.spec #2>&1 | tee -ai /app/out/build.log + # write artefacts to out directory + rm -rf /app/out/* + # cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out + cp -arv /root/rpmbuild/RPMS /app/out +fi diff --git a/packaging/pkg/scripts/deb.sh b/packaging/pkg/scripts/deb.sh new file mode 100755 index 000000000..2fe2018b6 --- /dev/null +++ b/packaging/pkg/scripts/deb.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# THere is no std/ent packages for PG 9.5 +if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# PACKAGES NEEDED +apt-get update -y && apt-get install -y git wget bzip2 devscripts equivs + +# Prepare +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +if [ ${CODENAME} == 'jessie' ]; then +printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list +fi + +apt-get -qq update -y + +# download PKG_URL if PKG_HASH is omitted +mkdir /root/build +cd /root/build + +# clone pbk repo +git clone $PKG_URL ${PKG_NAME}_${PKG_VERSION} +cd ${PKG_NAME}_${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +PG_TOC=$(echo ${PG_VERSION} | sed 's|\.||g') +# Download PostgreSQL source if building for vanilla +if [[ ${PBK_EDITION} == '' ]] ; then + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 +fi + +cd /root/build/${PKG_NAME}_${PKG_VERSION} +cp -av /app/in/specs/deb/pg_probackup/debian ./ +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/changelog + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/g" debian/changelog + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/g" debian/changelog + sed -i "s/@PKG_HASH@/${PKG_HASH}/g" debian/changelog + sed -i "s/@CODENAME@/${CODENAME}/g" debian/changelog + + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/control + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/control + + sed -i "s/@PG_VERSION@/${PG_VERSION}/" debian/pg_probackup.install + mv debian/pg_probackup.install debian/${PKG_NAME}.install + + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/rules + sed -i "s/@PG_TOC@/${PG_TOC}/g" debian/rules + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/rules + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/g" debian/rules + sed -i "s|@PREFIX@|/stump|g" debian/rules +else + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/changelog + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/g" debian/changelog + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/g" debian/changelog + sed -i "s/@PKG_HASH@/${PKG_HASH}/g" debian/changelog + sed -i "s/@CODENAME@/${CODENAME}/g" debian/changelog + + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/control + sed -i "s/pg-probackup-@PG_VERSION@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/control + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/control + sed -i "s/PostgreSQL/PostgresPro ${PBK_EDITION_FULL}/g" debian/control + + sed -i "s/pg_probackup-@PG_VERSION@/pg_probackup-${PBK_EDITION}-${PG_VERSION}/" debian/pg_probackup.install + mv debian/pg_probackup.install debian/pg-probackup-${PBK_EDITION}-${PG_VERSION}.install + + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/rules + sed -i "s/@PG_TOC@/${PG_TOC}/g" debian/rules + sed -i "s/pg_probackup-@PG_VERSION@/pg_probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/rules + sed -i "s/postgresql-@PG_FULL_VERSION@/postgrespro-${PBK_EDITION}-${PG_FULL_VERSION}/g" debian/rules + + if [ ${PG_VERSION} == '9.6' ]; then + sed -i "s|@PREFIX@|/stump|g" debian/rules + else + sed -i "s|@PREFIX@|/opt/pgpro/${PBK_EDITION}-${PG_VERSION}|g" debian/rules + fi +fi + +# Build dependencies +mk-build-deps --install --remove --tool 'apt-get --no-install-recommends --yes' debian/control +rm -rf ./*.deb + +# Pack source to orig.tar.gz +mkdir -p /root/build/dsc +if [[ ${PBK_EDITION} == '' ]] ; then + mv /root/build/postgresql-${PG_FULL_VERSION}.tar.bz2 \ + /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 + + cd /root/build/${PKG_NAME}_${PKG_VERSION} + tar -xf /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 + cd /root/build + + tar -czf ${PKG_NAME}_${PKG_VERSION}.orig.tar.gz \ + ${PKG_NAME}_${PKG_VERSION} + + mv /root/build/${PKG_NAME}_${PKG_VERSION}.orig.tar.gz /root/build/dsc + + cd /root/build/${PKG_NAME}_${PKG_VERSION} + tar -xf /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/build/dsc/ + cd /root/build/dsc/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + + mv /root/build/dsc/pgpro /root/build/${PKG_NAME}_${PKG_VERSION}/postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + +# BUILD: SOURCE PKG +if [[ ${PBK_EDITION} == '' ]] ; then + cd /root/build/dsc + dpkg-source -b /root/build/${PKG_NAME}_${PKG_VERSION} +fi + +# BUILD: DEB PKG +cd /root/build/${PKG_NAME}_${PKG_VERSION} +dpkg-buildpackage -b #&> /app/out/build.log + +# COPY ARTEFACTS +rm -rf /app/out/* +cd /root/build +cp -v *.deb /app/out +cp -v *.changes /app/out + +if [[ ${PBK_EDITION} == '' ]] ; then + cp -arv dsc /app/out +fi diff --git a/packaging/pkg/scripts/rpm.sh b/packaging/pkg/scripts/rpm.sh new file mode 100755 index 000000000..fc95bf7dd --- /dev/null +++ b/packaging/pkg/scripts/rpm.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + + +#yum upgrade -y || echo "some packages in docker fail to install" +#if [ -f /etc/rosa-release ]; then +# # Avoids old yum bugs on rosa-6 +# yum upgrade -y || echo "some packages in docker fail to install" +#fi + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# THere is no std/ent packages for PG 9.5 +if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# PACKAGES NEEDED +yum install -y git wget bzip2 rpm-build + +mkdir /root/build +cd /root/build +rpm --rebuilddb && yum clean all + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} + +# move it to source +cd /root/build +if [[ ${PBK_EDITION} == '' ]] ; then + tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} + mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES + rm -rf pg_probackup-${PKG_VERSION} +else + mv pg_probackup-${PKG_VERSION} /root/rpmbuild/SOURCES +fi + +if [[ ${PBK_EDITION} == '' ]] ; then + + # Download PostgreSQL source + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O /root/rpmbuild/SOURCES/postgresql-${PG_VERSION}.tar.bz2 + + cd /root/rpmbuild/SOURCES/ + sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup.repo + if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup.repo + elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup.repo + elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup.repo + fi +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/rpmbuild/SOURCES/ + cd /root/rpmbuild/SOURCES/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + rm -rf .git + + cd /root/rpmbuild/SOURCES/ + sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup-forks.repo + if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup-forks.repo + elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup-forks.repo + elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup-forks.repo + fi + + mv pgpro postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + chown -R root:root postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + +# tar -cjf postgrespro-${PBK_EDITION}-${PG_FULL_VERSION}.tar.bz2 postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + +cd /root/rpmbuild/SPECS +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.spec + + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo.spec +else + sed -i "s/@EDITION@/${PBK_EDITION}/" pg_probackup-pgpro.spec + sed -i "s/@EDITION_FULL@/${PBK_EDITION_FULL}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup-pgpro.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup-pgpro.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup-pgpro.spec + + if [ ${PG_VERSION} != '9.6' ]; then + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup-pgpro.spec + fi + + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo-forks.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo-forks.spec +fi + +if [[ ${PBK_EDITION} == '' ]] ; then + + # install dependencies + yum-builddep -y pg_probackup.spec + + # build pg_probackup + rpmbuild -bs pg_probackup.spec + rpmbuild -ba pg_probackup.spec + + # build repo files + rpmbuild -bs pg_probackup-repo.spec + rpmbuild -ba pg_probackup-repo.spec + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out +else + # install dependencies + yum-builddep -y pg_probackup-pgpro.spec + # build pg_probackup + rpmbuild -ba pg_probackup-pgpro.spec + + # build repo files + rpmbuild -ba pg_probackup-repo-forks.spec + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/RPMS /app/out +fi diff --git a/packaging/pkg/scripts/suse.sh b/packaging/pkg/scripts/suse.sh new file mode 100755 index 000000000..76b444b5b --- /dev/null +++ b/packaging/pkg/scripts/suse.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + + +#yum upgrade -y || echo "some packages in docker fail to install" +#if [ -f /etc/rosa-release ]; then +# # Avoids old yum bugs on rosa-6 +# yum upgrade -y || echo "some packages in docker fail to install" +#fi + +set -xe +set -o pipefail + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 +zypper clean + +# PACKAGES NEEDED +zypper install -y git wget bzip2 rpm-build + +mkdir /root/build +cd /root/build + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +# tarball it +tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} +mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES +rm -rf pg_probackup-${PKG_VERSION} + +# Download PostgreSQL source +wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O /root/rpmbuild/SOURCES/postgresql-${PG_VERSION}.tar.bz2 + +rm -rf /usr/src/packages +ln -s /root/rpmbuild /usr/src/packages + +cd /root/rpmbuild/SOURCES +sed -i "s/@PG_VERSION@/${PKG_VERSION}/" pg_probackup.repo + + +# change to build dir +cd /root/rpmbuild/SOURCES +sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup.repo +if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup.repo +elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup.repo +elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup.repo +elif [ $DISTRIB == 'suse' ] + then sed -i "s/@SHORT_CODENAME@/SUSE/" pg_probackup.repo +fi + +cd /root/rpmbuild/SPECS +sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.spec +sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.spec +sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.spec +sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.spec +sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.spec + +sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup-repo.spec +sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo.spec +sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo.spec + +# install dependencies +zypper -n install \ + $(rpmspec --parse pg_probackup.spec | grep BuildRequires | cut -d':' -f2 | xargs) + +# build pg_probackup +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log + +# build repo files, TODO: move to separate repo +rpmbuild -ba pg_probackup-repo.spec + +# write artefacts to out directory +rm -rf /app/out/* + +cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/changelog b/packaging/pkg/specs/deb/pg_probackup/debian/changelog new file mode 100644 index 000000000..5b9160220 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/changelog @@ -0,0 +1,11 @@ +@PKG_NAME@ (@PKG_VERSION@-@PKG_RELEASE@.@PKG_HASH@.@CODENAME@) @CODENAME@; urgency=medium + + * @PKG_VERSION@ + + -- Grigory Smolkin Wed, 9 Feb 2018 10:22:08 +0300 + +@PKG_NAME@ (2.0.14-1.@CODENAME@) @CODENAME@; urgency=medium + + * Initial package + + -- Grigory Smolkin Fri, 29 Jan 2018 10:22:08 +0300 diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/compat b/packaging/pkg/specs/deb/pg_probackup/debian/compat new file mode 100644 index 000000000..ec635144f --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/compat @@ -0,0 +1 @@ +9 diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/control b/packaging/pkg/specs/deb/pg_probackup/debian/control new file mode 100644 index 000000000..8f1d42007 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/control @@ -0,0 +1,29 @@ +Source: @PKG_NAME@ +Section: database +Priority: optional +Maintainer: PostgresPro DBA +Uploaders: Grigory Smolkin +Build-Depends: + debhelper (>= 9), + bison, + dpkg-dev, + flex, + gettext, + zlib1g-dev | libz-dev, + libpq5 +Standards-Version: 3.9.6 +Homepage: https://github.com/postgrespro/pg_probackup + +Package: @PKG_NAME@ +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: Backup tool for PostgreSQL. + . + This package provides pg_probackup binary for PostgreSQL @PG_VERSION@. + +Package: @PKG_NAME@-dbg +Depends: @PKG_NAME@ +Architecture: any +Description: Backup tool for PostgreSQL. + . + This package provides detached debugging symbols for pg_probackup diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install b/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install new file mode 100644 index 000000000..ed904ca40 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install @@ -0,0 +1 @@ +pg_probackup-@PG_VERSION@ /usr/bin/ \ No newline at end of file diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/rules b/packaging/pkg/specs/deb/pg_probackup/debian/rules new file mode 100644 index 000000000..309a9a1d4 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/rules @@ -0,0 +1,29 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ + +override_dh_auto_clean: + # skip + +override_dh_auto_build: + cd postgresql-@PG_FULL_VERSION@ && ./configure --enable-debug --without-readline --prefix=@PREFIX@ &&\ + make MAKELEVEL=0 install DESTDIR=$(CURDIR)/debian/tmp && cd .. &&\ + make USE_PGXS=1 top_srcdir=$(CURDIR)/postgresql-@PG_FULL_VERSION@ PG_CONFIG=$(CURDIR)/debian/tmp/@PREFIX@/bin/pg_config &&\ + mv pg_probackup pg_probackup-@PG_VERSION@ + +override_dh_auto_test: + # skip + +override_dh_auto_install: + # skip + +override_dh_strip: + dh_strip --dbg-package=@PKG_NAME@-dbg + +override_dh_auto_clean: + # skip + #make clean top_srcdir=$(CURDIR)/pg@PG_TOC@-source PG_CONFIG=$(CURDIR)/debian/tmp/stump/bin/pg_config diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/source/format b/packaging/pkg/specs/deb/pg_probackup/debian/source/format new file mode 100644 index 000000000..163aaf8d8 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP new file mode 100644 index 000000000..c11d9c015 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFpy9DABEADd44hR3o4i4DrUephrr7iHPHcRH0Zego3A36NdOf0ymP94H8Bi +U8C6YyKFbltShh18IC3QZJK04hLRQEs6sPKC2XHwlz+Tndi49Z45pfV54xEVKmBS +IZ5AM9y1FxwQAOzu6pZGu32DWDXZzhI7nLuY8rqAMMuzKeRcGm3sQ6ZcAwYOLT+e +ZAxkUL05MBGDaLc91HtKiurRlHuMySiVdkNu9ebTGV4zZv+ocBK8iC5rJjTJCv78 +eLkrRgjp7/MuLQ7mmiwfZx5lUIO9S87HDeH940mcYWRGUsdCbj0791wHY0PXlqhH +6lCLGur9/5yM88pGU79uahCblxsYdue6fdgIZR0hQBUxnLphI2SCshP89VDEwaP2 +dlC/qESJ3xyULkdJz67wlhOrIPC9T1d2pa5MUurOK0yTFH7j4JLWjBgU59h31ZEF +NMHde+Fwv+lL/yRht2Xz7HG5Rt8ogn4/rPBloXr1v83iN34aZnnqanyhSbE9xUhP +RNK3fBxXmX9IjFsBhRelPcv5NWNnxnnMkEfhoZvrAy+ykUGLP+J+Rj+d5v/8nAUc +taxqAXlUz1VabR0BVISBsRY+ket4O2dJ1WbZ8KXG6q/F9UMpS0v9aRdb1JyzrWCw +wT/l3q9x89i27SgDZgAfEFhvbMN6hUmFyVoMBgk8kqvi4b3lZZGCeuLX5wARAQAB +tCxQb3N0Z3JlU1FMIFByb2Zlc3Npb25hbCA8ZGJhQHBvc3RncmVzcHJvLnJ1PokC +OQQTAQIAIwUCWnL0MAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEKeJ +efZjbXF+zDUP/RfYxlq3erzP/cG6/LghZlJy6hGuUgyDFj2zUVAbpoFhqCAmaNLc ++bBYMCyNRhS8/oXushCSxUV8D7LRIRIRdtbNAnd4MNl6U4ORF6JcdPPNLROzwMik +3TmIVACMdjb9IRF5+8jVrIgDPI/FVtf5qp0Ot6OBtpD5oWQ7ubZ31RPR3pacdujK +jlbzL5Y6HsonhMbSJU/d0d9DylMvX4Gcxdw7M2Pfe3E6mjPJmcHiKuCKln2eLOsg +53HA/RWKy+uYDo+vdefUsQCIdnC5VghnXG8FTuvVqeqiSeU2XdyuzjndqxKZNrMw +YK1POK7R55R1aKJaOKEwnfd5oN02p77U+R/vb/mDcfZWbXI8JrHwPKVOQiEl0S+G +ePPW57EmX9yFuWAzcOPp9yCt/+roVry1ICifrFaLOhtT+/vle0j3+rbn31BMPsjf +QbREVetHfWB0N78k/hKC8SivDdrXsdqovcGgSAjFzPEdznvx9vKopwz2CQ6DK25Q +3M4j79Akcaa08k5Wphyx48PbhzSeE/d4xVzey7ge0BwYMdNGXKeyBjT6h9e+iySE +UTZ3/3c7O1D8p2EfPUMT/aI5fWlLBXlT5fDp2yX0HMTt/NUIXAiTHb5BDnZ+4ld3 +KXjHw4WzaOfHBfGDjJDtHPgdTEJTsQbH8//D+wwU3ueNS1ho4DpLqc+YuQINBFpy +9DABEADJMkgQ2m4g4LX7FNnmQbRgDcuhL8Y0VRGST+5x6xvb2em1boQHUaTt7/3z +DnaIRrZqrFP09O6xblSjEu9FZE+JuQGNyC4TH9fjvKnkRlqTF6X87nRVGByRmrdL +lPp9XPJY2Mc7c0PisncI/j7d9PmUHOSmaWeLG/WqMbzZA+s1IWjC0tqIN2k5ivTN +PfRm+9ebEHMUN+D7yZQMBlCmFexwy6h5pAioyj4tAOHqxfNDE33qezaeBn/E1BpW +NyegKwNtPUL0t2kXTO5tspKKCcny4HJ7K60gak0fWp42qVygwSPR54ztFM+6XjCh +0MmZ/mAdzLd6OJiP8RfMCfXbXpK4793+Cw0AK3Mu+mnJ26kz1KEZ9DKiAEhBhK3r +Z3/isUc8LcVYLHIduH9b/K50FjgR0T1Lm4r6Hpf6nTROlfiFSMXJU0HepAzMPHRq +EWqTJ49UgI7Llf+aBP7fGLqRPvWJpAJaQkMiUxfP5JYYCb+45d7I54iXQCD6ToK1 +bDnh+zZIrwyUIxPfFQh1xPYyFWRELJpeOFzm+espqiVFPXpBoimVlytwNrGdbxbY +SO0eEVlE41AjD8cgk+ibAvt/moT2+Mps/t083LR+J92kj+iX/D4NHVy4CjJTrhwO +rI3FrxtdU+NFXULyj0KslOKuyG5WuHLQvfL5P3JGuTkP4iJOTQARAQABiQIfBBgB +AgAJBQJacvQwAhsMAAoJEKeJefZjbXF+8JgQAJqlO1ftIsJvZ/+4ZVVOTPx5ZmYs +ABp4/2gaiLdhajN8ynbZqtCyjtQwSCLJFf2CcDL8XUooJzdQECkqdiI7ouYSFBzO +ui3jjCuFz5oHv88OtX2cIRxHqlZQmXEHvk0gH61xDV5CWBJmjxdRcsC7n1I8DSVg +Qmuq06S+xIX6rHf2CRxYKahBip71u7OIH4BRV44y26xf1a8an+8BkqF9+mYt7zqO +vyMCJ1UftXcuE5SxY54jnNAavF7Kq/2Yp7v3aYqFREngxtbWudyo7QW5EuToSvY2 +qY6tpInahWjuXxeARsFzp4fB0Eo/yH+iqG30zkQCuxLyxzbMMcNQP4if3yV6uO14 +LqapZLrMp6IMTfHDKmbbtDQ2RpRRut3K4khXRQ1LjGKziOU4ZCEazrXEijm2AlKw +7JS3POGvM+VAiaGNBqfdHpTwXXT7zkxJjfJC3Et/6fHy1xuCtUtMs41PjHS/HWi8 +w70T8XpKub+ewVElxq2D83rx07w3HuBtVUyqG0XgcACwqQA1vMLJaR3VoX1024ho +sf2PtZqQ7SCgt0hkZAT72j05nz4bIxUIcDkAGtd9FDPQ4Ixi6fRfTJpZ7lIEV5as +Zs9C0hrxmWgJwSGgQa2Waylvw47fMwfMn+gUNRqwanyOjVYfpSJafLc6Ol43bQN/ +jCKs4enncezhjcAh +=TVZj +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS new file mode 100644 index 000000000..c11d9c015 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFpy9DABEADd44hR3o4i4DrUephrr7iHPHcRH0Zego3A36NdOf0ymP94H8Bi +U8C6YyKFbltShh18IC3QZJK04hLRQEs6sPKC2XHwlz+Tndi49Z45pfV54xEVKmBS +IZ5AM9y1FxwQAOzu6pZGu32DWDXZzhI7nLuY8rqAMMuzKeRcGm3sQ6ZcAwYOLT+e +ZAxkUL05MBGDaLc91HtKiurRlHuMySiVdkNu9ebTGV4zZv+ocBK8iC5rJjTJCv78 +eLkrRgjp7/MuLQ7mmiwfZx5lUIO9S87HDeH940mcYWRGUsdCbj0791wHY0PXlqhH +6lCLGur9/5yM88pGU79uahCblxsYdue6fdgIZR0hQBUxnLphI2SCshP89VDEwaP2 +dlC/qESJ3xyULkdJz67wlhOrIPC9T1d2pa5MUurOK0yTFH7j4JLWjBgU59h31ZEF +NMHde+Fwv+lL/yRht2Xz7HG5Rt8ogn4/rPBloXr1v83iN34aZnnqanyhSbE9xUhP +RNK3fBxXmX9IjFsBhRelPcv5NWNnxnnMkEfhoZvrAy+ykUGLP+J+Rj+d5v/8nAUc +taxqAXlUz1VabR0BVISBsRY+ket4O2dJ1WbZ8KXG6q/F9UMpS0v9aRdb1JyzrWCw +wT/l3q9x89i27SgDZgAfEFhvbMN6hUmFyVoMBgk8kqvi4b3lZZGCeuLX5wARAQAB +tCxQb3N0Z3JlU1FMIFByb2Zlc3Npb25hbCA8ZGJhQHBvc3RncmVzcHJvLnJ1PokC +OQQTAQIAIwUCWnL0MAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEKeJ +efZjbXF+zDUP/RfYxlq3erzP/cG6/LghZlJy6hGuUgyDFj2zUVAbpoFhqCAmaNLc ++bBYMCyNRhS8/oXushCSxUV8D7LRIRIRdtbNAnd4MNl6U4ORF6JcdPPNLROzwMik +3TmIVACMdjb9IRF5+8jVrIgDPI/FVtf5qp0Ot6OBtpD5oWQ7ubZ31RPR3pacdujK +jlbzL5Y6HsonhMbSJU/d0d9DylMvX4Gcxdw7M2Pfe3E6mjPJmcHiKuCKln2eLOsg +53HA/RWKy+uYDo+vdefUsQCIdnC5VghnXG8FTuvVqeqiSeU2XdyuzjndqxKZNrMw +YK1POK7R55R1aKJaOKEwnfd5oN02p77U+R/vb/mDcfZWbXI8JrHwPKVOQiEl0S+G +ePPW57EmX9yFuWAzcOPp9yCt/+roVry1ICifrFaLOhtT+/vle0j3+rbn31BMPsjf +QbREVetHfWB0N78k/hKC8SivDdrXsdqovcGgSAjFzPEdznvx9vKopwz2CQ6DK25Q +3M4j79Akcaa08k5Wphyx48PbhzSeE/d4xVzey7ge0BwYMdNGXKeyBjT6h9e+iySE +UTZ3/3c7O1D8p2EfPUMT/aI5fWlLBXlT5fDp2yX0HMTt/NUIXAiTHb5BDnZ+4ld3 +KXjHw4WzaOfHBfGDjJDtHPgdTEJTsQbH8//D+wwU3ueNS1ho4DpLqc+YuQINBFpy +9DABEADJMkgQ2m4g4LX7FNnmQbRgDcuhL8Y0VRGST+5x6xvb2em1boQHUaTt7/3z +DnaIRrZqrFP09O6xblSjEu9FZE+JuQGNyC4TH9fjvKnkRlqTF6X87nRVGByRmrdL +lPp9XPJY2Mc7c0PisncI/j7d9PmUHOSmaWeLG/WqMbzZA+s1IWjC0tqIN2k5ivTN +PfRm+9ebEHMUN+D7yZQMBlCmFexwy6h5pAioyj4tAOHqxfNDE33qezaeBn/E1BpW +NyegKwNtPUL0t2kXTO5tspKKCcny4HJ7K60gak0fWp42qVygwSPR54ztFM+6XjCh +0MmZ/mAdzLd6OJiP8RfMCfXbXpK4793+Cw0AK3Mu+mnJ26kz1KEZ9DKiAEhBhK3r +Z3/isUc8LcVYLHIduH9b/K50FjgR0T1Lm4r6Hpf6nTROlfiFSMXJU0HepAzMPHRq +EWqTJ49UgI7Llf+aBP7fGLqRPvWJpAJaQkMiUxfP5JYYCb+45d7I54iXQCD6ToK1 +bDnh+zZIrwyUIxPfFQh1xPYyFWRELJpeOFzm+espqiVFPXpBoimVlytwNrGdbxbY +SO0eEVlE41AjD8cgk+ibAvt/moT2+Mps/t083LR+J92kj+iX/D4NHVy4CjJTrhwO +rI3FrxtdU+NFXULyj0KslOKuyG5WuHLQvfL5P3JGuTkP4iJOTQARAQABiQIfBBgB +AgAJBQJacvQwAhsMAAoJEKeJefZjbXF+8JgQAJqlO1ftIsJvZ/+4ZVVOTPx5ZmYs +ABp4/2gaiLdhajN8ynbZqtCyjtQwSCLJFf2CcDL8XUooJzdQECkqdiI7ouYSFBzO +ui3jjCuFz5oHv88OtX2cIRxHqlZQmXEHvk0gH61xDV5CWBJmjxdRcsC7n1I8DSVg +Qmuq06S+xIX6rHf2CRxYKahBip71u7OIH4BRV44y26xf1a8an+8BkqF9+mYt7zqO +vyMCJ1UftXcuE5SxY54jnNAavF7Kq/2Yp7v3aYqFREngxtbWudyo7QW5EuToSvY2 +qY6tpInahWjuXxeARsFzp4fB0Eo/yH+iqG30zkQCuxLyxzbMMcNQP4if3yV6uO14 +LqapZLrMp6IMTfHDKmbbtDQ2RpRRut3K4khXRQ1LjGKziOU4ZCEazrXEijm2AlKw +7JS3POGvM+VAiaGNBqfdHpTwXXT7zkxJjfJC3Et/6fHy1xuCtUtMs41PjHS/HWi8 +w70T8XpKub+ewVElxq2D83rx07w3HuBtVUyqG0XgcACwqQA1vMLJaR3VoX1024ho +sf2PtZqQ7SCgt0hkZAT72j05nz4bIxUIcDkAGtd9FDPQ4Ixi6fRfTJpZ7lIEV5as +Zs9C0hrxmWgJwSGgQa2Waylvw47fMwfMn+gUNRqwanyOjVYfpSJafLc6Ol43bQN/ +jCKs4enncezhjcAh +=TVZj +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo new file mode 100644 index 000000000..fcef58a9c --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo @@ -0,0 +1,6 @@ +[pg_probackup-forks] +name=PG_PROBACKUP @SHORT_CODENAME@ packages for PostgresPro Standart and Enterprise - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo new file mode 100644 index 000000000..33dc31a24 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo @@ -0,0 +1,13 @@ +[pg_probackup] +name=PG_PROBACKUP Packages for @SHORT_CODENAME@ Linux - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup/rpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +[pg_probackup-sources] +name=PG_PROBACKUP Source Packages for @SHORT_CODENAME@ Linux - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup/srpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec new file mode 100644 index 000000000..d5811171d --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec @@ -0,0 +1,71 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%global edition @EDITION@ +%global edition_full @EDITION_FULL@ +%global prefix @PREFIX@ + +Name: pg_probackup-%{edition}-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgresPro %{edition_full} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +#Source0: postgrespro-%{edition}-%{pgsql_full}.tar.bz2 +#Source1: pg_probackup-%{version}.tar.bz2 +Source0: postgrespro-%{edition}-%{pgsql_full} +Source1: pg_probackup-%{version} +BuildRequires: gcc make perl glibc-devel +BuildRequires: openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgresPro %{edition_full}. + +%prep +#%setup -q -b1 -n pg_probackup-%{version}.tar.bz2 +mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} %{_topdir}/BUILD +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup + +mkdir %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +mkdir %{_topdir}/SOURCES/pg_probackup-%{version} + +%build +#cd %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +#mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} ./ +#cd postgrespro-%{edition}-%{pgsql_full} +#mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} + +%if "%{pgsql_major}" == "9.6" +./configure --enable-debug +%else +./configure --enable-debug --without-readline --prefix=%{prefix} +%endif +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Wed Feb 9 2018 Grigory Smolkin - %{version}-%{release}.%{hash} +- @PKG_VERSION@ + +* Fri Jan 29 2018 Grigory Smolkin - 2.0.14-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec new file mode 100644 index 000000000..fd4a99f2c --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec @@ -0,0 +1,49 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ + +Summary: pg_probackup repo RPM +Name: pg_probackup-repo-forks +Version: %{version} +Release: %{release} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ + +Source0: http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP +Source1: pg_probackup-forks.repo + +BuildArch: noarch + +%description +This package contains yum configuration for @SHORT_CODENAME@, and also the GPG key +for pg_probackup RPMs for PostgresPro Standart and Enterprise. + +%prep +%setup -q -c -T +install -pm 644 %{SOURCE0} . +install -pm 644 %{SOURCE1} . + +%build + +%install +rm -rf $RPM_BUILD_ROOT + +#GPG Key +install -Dpm 644 %{SOURCE0} \ + $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +# yum +install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d +install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%config(noreplace) /etc/yum.repos.d/* +/etc/pki/rpm-gpg/* + +%changelog +* Fri Oct 26 2019 Grigory Smolkin +- Initial package diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec new file mode 100644 index 000000000..da54bc7b1 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec @@ -0,0 +1,58 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ + +Summary: PG_PROBACKUP RPMs +Name: pg_probackup-repo +Version: %{version} +Release: %{release} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ + +Source0: http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP +Source1: pg_probackup.repo + +BuildArch: noarch + +%description +This package contains yum configuration for Centos, and also the GPG key for PG_PROBACKUP RPMs. + +%prep +%setup -q -c -T +install -pm 644 %{SOURCE0} . +install -pm 644 %{SOURCE1} . + +%build + +%install +rm -rf $RPM_BUILD_ROOT + +#GPG Key +install -Dpm 644 %{SOURCE0} \ + $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +# yum /etc/zypp/repos.d/repo-update.repo + +%if 0%{?suse_version} + install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d + install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d +%else + install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d + install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d +%endif + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%if 0%{?suse_version} + %config(noreplace) /etc/zypp/repos.d/* +%else + %config(noreplace) /etc/yum.repos.d/* +%endif +/etc/pki/rpm-gpg/* + +%changelog +* Mon Jun 29 2020 Grigory Smolkin +- release update diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec new file mode 100644 index 000000000..cbfd61a0f --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec @@ -0,0 +1,67 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%global edition @EDITION@ +%global edition_full @EDITION_FULL@ +%global prefix @PREFIX@ + +#%set_verify_elf_method unresolved=relaxed, rpath=relaxed +%set_verify_elf_method rpath=relaxed,unresolved=relaxed + +Name: pg_probackup-%{edition}-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgresPro %{edition_full} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +#Source0: postgrespro-%{edition}-%{pgsql_full}.tar.bz2 +#Source1: pg_probackup-%{edition}-%{version}.tar.bz2 +Source0: postgrespro-%{edition}-%{pgsql_full} +Source1: pg_probackup-%{version} +BuildRequires: gcc make perl glibc-devel bison flex +BuildRequires: readline-devel openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgresPro %{edition_full}. + +%prep +#%setup -q -b1 -n postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} %{_topdir}/BUILD +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup + +mkdir %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +mkdir %{_topdir}/SOURCES/pg_probackup-%{edition}-%{version} +mkdir %{_topdir}/SOURCES/pg_probackup-%{version} + +%build +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%if "%{pgsql_major}" == "9.6" +./configure --enable-debug +%else +./configure --enable-debug --prefix=%{prefix} +%endif +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Mon Nov 17 2019 Grigory Smolkin - 2.2.6-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec new file mode 100644 index 000000000..3105ffa67 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec @@ -0,0 +1,48 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%set_verify_elf_method rpath=relaxed + +Name: pg_probackup-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgreSQL +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +Source0: http://ftp.postgresql.org/pub/source/v%{pgsql_full}/postgresql-%{pgsql_major}.tar.bz2 +Source1: pg_probackup-%{version}.tar.bz2 +BuildRequires: gcc make perl glibc-devel bison flex +BuildRequires: readline-devel openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgreSQL. + +%prep +%setup -q -b1 -n postgresql-%{pgsql_full} + +%build +mv %{_builddir}/pg_probackup-%{version} contrib/pg_probackup +./configure --enable-debug --without-readline +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Mon Nov 17 2019 Grigory Smolkin - 2.2.6-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec new file mode 100644 index 000000000..e5fb5ad48 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec @@ -0,0 +1,48 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ + +Name: pg_probackup-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgreSQL +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +Source0: http://ftp.postgresql.org/pub/source/v%{pgsql_full}/postgresql-%{pgsql_major}.tar.bz2 +Source1: pg_probackup-%{version}.tar.bz2 +BuildRequires: gcc make perl glibc-devel openssl-devel gettext zlib-devel + +%description +Backup tool for PostgreSQL. + +%prep +%setup -q -b1 -n postgresql-%{pgsql_full} + +%build +mv %{_builddir}/pg_probackup-%{version} contrib/pg_probackup +./configure --enable-debug --without-readline +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Wed Feb 9 2018 Grigory Smolkin - %{version}-%{release}.%{hash} +- @PKG_VERSION@ + +* Fri Jan 29 2018 Grigory Smolkin - 2.0.14-1 +- Initial release. diff --git a/packaging/pkg/tarballs/.gitkeep b/packaging/pkg/tarballs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/packaging/repo/scripts/alt.sh b/packaging/repo/scripts/alt.sh new file mode 100755 index 000000000..4cda313ef --- /dev/null +++ b/packaging/repo/scripts/alt.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -exu +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} + +apt-get update -y +apt-get install -qq -y apt-repo-tools gnupg rsync perl less wget + +if [[ ${PBK_EDITION} == '' ]] ; then + REPO_SUFFIX='vanilla' + FORK='PostgreSQL' +else + REPO_SUFFIX='forks' + FORK='PostgresPro' +fi + +cd $INPUT_DIR + +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg +for pkg in $(ls); do + for pkg_full_version in $(ls ./$pkg); do + + # THere is no std/ent packages for PG 9.5 + if [[ ${pkg} == 'pg_probackup-std-9.5' ]] || [[ ${pkg} == 'pg_probackup-ent-9.5' ]] ; then + continue; + fi + + RPM_DIR=${OUT_DIR}/rpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION}/x86_64/RPMS.${REPO_SUFFIX} + mkdir -p "$RPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + + genbasedir --architecture=x86_64 --architectures=x86_64 --origin=repo.postgrespro.ru \ + --label="${FORK} backup utility pg_probackup" --description "${FORK} pg_probackup repo" \ + --version=$pkg_full_version --bloat --progress --create \ + --topdir=${OUT_DIR}/rpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION} x86_64 ${REPO_SUFFIX} + + # SRPM is available only for vanilla + if [[ ${PBK_EDITION} == '' ]] ; then + SRPM_DIR=${OUT_DIR}/srpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION}/x86_64/SRPMS.${REPO_SUFFIX} + mkdir -p "$SRPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + + genbasedir --architecture=x86_64 --architectures=x86_64 --origin=repo.postgrespro.ru \ + --label="${FORK} backup utility pg_probackup sources" --description "${FORK} pg_probackup repo" \ + --version=$pkg_full_version --bloat --progress --create \ + --topdir=${OUT_DIR}/srpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION} x86_64 ${REPO_SUFFIX} + fi + done +done diff --git a/packaging/repo/scripts/deb.sh b/packaging/repo/scripts/deb.sh new file mode 100755 index 000000000..6515e6b42 --- /dev/null +++ b/packaging/repo/scripts/deb.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -exu +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in # dir with builded deb +export OUT_DIR=/app/www/${PBK_PKG_REPO} +#export REPO_DIR=/app/repo + +cd $INPUT_DIR + +export DEB_DIR=$OUT_DIR/deb +export KEYS_DIR=$OUT_DIR/keys +export CONF=/app/repo/${PBK_PKG_REPO}/conf +mkdir -p "$KEYS_DIR" +cp -av /app/repo/${PBK_PKG_REPO}/gnupg /root/.gnupg + +rsync /app/repo/${PBK_PKG_REPO}/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +mkdir -p $DEB_DIR +cd $DEB_DIR +cp -av $CONF ./ + +# make remove-debpkg tool +echo -n "#!" > remove-debpkg +echo "/bin/sh" >> remove-debpkg +echo "CODENAME=\$1" >> remove-debpkg +echo "DEBFILE=\$2" >> remove-debpkg +echo "DEBNAME=\`basename \$DEBFILE | sed -e 's/_.*//g'\`" >> remove-debpkg +echo "reprepro --waitforlock 5 remove \$CODENAME \$DEBNAME" >> remove-debpkg +chmod +x remove-debpkg + +#find $INPUT_DIR/ -name '*.changes' -exec reprepro -P optional -Vb . include ${CODENAME} {} \; +find $INPUT_DIR -name "*${CODENAME}*.deb" -exec ./remove-debpkg $CODENAME {} \; +find $INPUT_DIR -name "*${CODENAME}*.dsc" -exec reprepro --waitforlock 5 -i undefinedtarget --ignore=missingfile -P optional -S main -Vb . includedsc $CODENAME {} \; +find $INPUT_DIR -name "*${CODENAME}*.deb" -exec reprepro --waitforlock 5 -i undefinedtarget --ignore=missingfile -P optional -Vb . includedeb $CODENAME {} \; +reprepro export $CODENAME + +rm -f remove-debpkg +rm -rf ./conf +rm -rf /root/.gnupg diff --git a/packaging/repo/scripts/rpm.sh b/packaging/repo/scripts/rpm.sh new file mode 100755 index 000000000..d4e621c3e --- /dev/null +++ b/packaging/repo/scripts/rpm.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -ex +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} +export KEYS_DIR=$OUT_DIR/keys + +# deploy keys +mkdir -p "$KEYS_DIR" +rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP +chmod 755 $KEYS_DIR +chmod +x /app/repo/$PBK_PKG_REPO/autosign.sh +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +cd $INPUT_DIR + +cp -arv /app/repo/$PBK_PKG_REPO/rpmmacros /root/.rpmmacros +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg +chown -R root:root /root/.gnupg + +for pkg in $(ls ${INPUT_DIR}); do + for pkg_full_version in $(ls ${INPUT_DIR}/$pkg); do + + # THere is no std/ent packages for PG 9.5 + if [[ ${pkg} == 'pg_probackup-std-9.5' ]] || [[ ${pkg} == 'pg_probackup-ent-9.5' ]] ; then + continue; + fi + + if [[ ${PBK_EDITION} == '' ]] ; then + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-$DISTRIB.noarch.rpm + else + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-forks-$DISTRIB.noarch.rpm + fi + + [ ! -z "$CODENAME" ] && export DISTRIB_VERSION=$CODENAME + RPM_DIR=$OUT_DIR/rpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + mkdir -p "$RPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + for f in $(ls $RPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + createrepo $RPM_DIR/ + + if [[ ${PBK_EDITION} == '' ]] ; then + SRPM_DIR=$OUT_DIR/srpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + mkdir -p "$SRPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + for f in $(ls $SRPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + createrepo $SRPM_DIR/ + fi + + done +done diff --git a/packaging/repo/scripts/suse.sh b/packaging/repo/scripts/suse.sh new file mode 100755 index 000000000..7253df700 --- /dev/null +++ b/packaging/repo/scripts/suse.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -ex +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} +export KEYS_DIR=$OUT_DIR/keys +# deploy keys + +zypper install -y createrepo +rm -rf /root/.gnupg + +cd $INPUT_DIR + +mkdir -p $KEYS_DIR +chmod 755 $KEYS_DIR +rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP + +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +cp -arv /app/repo/$PBK_PKG_REPO/rpmmacros /root/.rpmmacros +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg + +for pkg in $(ls); do + for pkg_full_version in $(ls ./$pkg); do + + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-$DISTRIB.noarch.rpm + [ ! -z "$CODENAME" ] && export DISTRIB_VERSION=$CODENAME + RPM_DIR=$OUT_DIR/rpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + SRPM_DIR=$OUT_DIR/srpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + + # rm -rf "$RPM_DIR" && mkdir -p "$RPM_DIR" + # rm -rf "$SRPM_DIR" && mkdir -p "$SRPM_DIR" + mkdir -p "$RPM_DIR" + mkdir -p "$SRPM_DIR" + + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + + for f in $(ls $RPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + for f in $(ls $SRPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + + createrepo $RPM_DIR/ + createrepo $SRPM_DIR/ + + # rpm --addsign $RPM_DIR/repodata/repomd.xml + # rpm --addsign $SRPM_DIR/repodata/repomd.xml + + gpg --batch --yes -a --detach-sign $RPM_DIR/repodata/repomd.xml + gpg --batch --yes -a --detach-sign $SRPM_DIR/repodata/repomd.xml + + cp -a /root/.gnupg/key.public $RPM_DIR/repodata/repomd.xml.key + cp -a /root/.gnupg/key.public $SRPM_DIR/repodata/repomd.xml.key + done +done diff --git a/packaging/test/Makefile.alt b/packaging/test/Makefile.alt new file mode 100644 index 000000000..3c1899cb9 --- /dev/null +++ b/packaging/test/Makefile.alt @@ -0,0 +1,20 @@ +# ALT 9 +build/test_alt_9_9.6: + $(call test_alt,alt,9,,9.6,9.6.21) + touch build/test_alt_9_9.6 + +build/test_alt_9_10: + $(call test_alt,alt,9,,10,10.17) + touch build/test_alt_9_10 + +build/test_alt_9_11: + $(call test_alt,alt,9,,11,11.11) + touch build/test_alt_9_11 + +build/test_alt_9_12: + $(call test_alt,alt,9,,12,12.6) + touch build/test_alt_9_12 + +build/test_alt_9_13: + $(call test_alt,alt,9,,13,13.2) + touch build/test_alt_9_13 diff --git a/packaging/test/Makefile.centos b/packaging/test/Makefile.centos new file mode 100644 index 000000000..e3787c612 --- /dev/null +++ b/packaging/test/Makefile.centos @@ -0,0 +1,41 @@ +# CENTOS 7 +build/test_centos_7_9.6: + $(call test_rpm,centos,7,,9.6,9.6.21) + touch build/test_centos_7_9.6 + +build/test_centos_7_10: + $(call test_rpm,centos,7,,10,10.16) + touch build/test_centos_7_10 + +build/test_centos_7_11: + $(call test_rpm,centos,7,,11,11.11) + touch build/test_centos_7_11 + +build/test_centos_7_12: + $(call test_rpm,centos,7,,12,12.6) + touch build/test_centos_7_12 + +build/test_centos_7_13: + $(call test_rpm,centos,7,,13,13.2) + touch build/test_centos_7_13 + +# CENTOS 8 +build/test_centos_8_9.6: + $(call test_rpm,centos,8,,9.6,9.6.21) + touch build/test_centos_8_9.6 + +build/test_centos_8_10: + $(call test_rpm,centos,8,,10,10.16) + touch build/test_centos_8_10 + +build/test_centos_8_11: + $(call test_rpm,centos,8,,11,11.11) + touch build/test_centos_8_11 + +build/test_centos_8_12: + $(call test_rpm,centos,8,,12,12.6) + touch build/test_centos_8_12 + +build/test_centos_8_13: + $(call test_rpm,centos,8,,13,13.2) + touch build/test_centos_8_13 diff --git a/packaging/test/Makefile.debian b/packaging/test/Makefile.debian new file mode 100644 index 000000000..f540f9205 --- /dev/null +++ b/packaging/test/Makefile.debian @@ -0,0 +1,41 @@ +# DEBIAN 9 +build/test_debian_9_9.6: + $(call test_deb,debian,9,stretch,9.6,9.6.21) + touch build/test_debian_9_9.6 + +build/test_debian_9_10: + $(call test_deb,debian,9,stretch,10,10.16) + touch build/test_debian_9_10 + +build/test_debian_9_11: + $(call test_deb,debian,9,stretch,11,11.11) + touch build/test_debian_9_11 + +build/test_debian_9_12: + $(call test_deb,debian,9,stretch,12,12.6) + touch build/test_debian_9_12 + +build/test_debian_9_13: + $(call test_deb,debian,9,stretch,13,13.2) + touch build/test_debian_9_13 + +# DEBIAN 10 +build/test_debian_10_9.6: + $(call test_deb,debian,10,buster,9.6,9.6.21) + touch build/test_debian_10_9.6 + +build/test_debian_10_10: + $(call test_deb,debian,10,buster,10,10.16) + touch build/test_debian_10_10 + +build/test_debian_10_11: + $(call test_deb,debian,10,buster,11,11.11) + touch build/test_debian_10_11 + +build/test_debian_10_12: + $(call test_deb,debian,10,buster,12,12.6) + touch build/test_debian_10_12 + +build/test_debian_10_13: + $(call test_deb,debian,10,buster,13,13.2) + touch build/test_debian_10_13 diff --git a/packaging/test/Makefile.oraclelinux b/packaging/test/Makefile.oraclelinux new file mode 100644 index 000000000..fdf44de8b --- /dev/null +++ b/packaging/test/Makefile.oraclelinux @@ -0,0 +1,41 @@ +# ORACLE LINUX 7 +build/test_oraclelinux_7_9.6: + $(call test_rpm,oraclelinux,7,,9.6,9.6.21) + touch build/test_oraclelinux_7_9.6 + +build/test_oraclelinux_7_10: + $(call test_rpm,oraclelinux,7,,10,10.16) + touch build/test_oraclelinux_7_10 + +build/test_oraclelinux_7_11: + $(call test_rpm,oraclelinux,7,,11,11.11) + touch build/test_oraclelinux_7_11 + +build/test_oraclelinux_7_12: + $(call test_rpm,oraclelinux,7,,12,12.6) + touch build/test_oraclelinux_7_12 + +build/test_oraclelinux_7_13: + $(call test_rpm,oraclelinux,7,,13,13.2) + touch build/test_oraclelinux_7_13 + +# ORACLE LINUX 8 +build/test_oraclelinux_8_9.6: + $(call test_rpm,oraclelinux,8,,9.6,9.6.21) + touch build/test_oraclelinux_8_9.6 + +build/test_oraclelinux_8_10: + $(call test_rpm,oraclelinux,8,,10,10.16) + touch build/test_oraclelinux_8_10 + +build/test_oraclelinux_8_11: + $(call test_rpm,oraclelinux,8,,11,11.11) + touch build/test_oraclelinux_8_11 + +build/test_oraclelinux_8_12: + $(call test_rpm,oraclelinux,8,,12,12.6) + touch build/test_oraclelinux_8_12 + +build/test_oraclelinux_8_13: + $(call test_rpm,oraclelinux,8,,13,13.2) + touch build/test_oraclelinux_8_13 diff --git a/packaging/test/Makefile.rhel b/packaging/test/Makefile.rhel new file mode 100644 index 000000000..3169d11c9 --- /dev/null +++ b/packaging/test/Makefile.rhel @@ -0,0 +1,41 @@ +# RHEL 7 +build/test_rhel_7_9.6: + $(call test_rpm,rhel,7,7Server,9.6,9.6.21) + touch build/test_rhel_7_9.6 + +build/test_rhel_7_10: + $(call test_rpm,rhel,7,7Server,10,10.16) + touch build/test_rhel_7_10 + +build/test_rhel_7_11: + $(call test_rpm,rhel,7,7Server,11,11.11) + touch build/test_rhel_7_11 + +build/test_rhel_7_12: + $(call test_rpm,rhel,7,7Server,12,12.6) + touch build/test_rhel_7_12 + +build/test_rhel_7_13: + $(call test_rpm,rhel,7,7Server,13,13.2) + touch build/test_rhel_7_13 + +# RHEL 8 +build/test_rhel_8_9.6: + $(call test_rpm,rhel,8,8Server,9.6,9.6.21) + touch build/test_rhel_8_9.6 + +build/test_rhel_8_10: + $(call test_rpm,rhel,8,8Server,10,10.16) + touch build/test_rhel_8_10 + +build/test_rhel_8_11: + $(call test_rpm,rhel,8,8Server,11,11.11) + touch build/test_rhel_8_11 + +build/test_rhel_8_12: + $(call test_rpm,rhel,8,8Server,12,12.6) + touch build/test_rhel_8_12 + +build/test_rhel_8_13: + $(call test_rpm,rhel,8,8Server,13,13.2) + touch build/test_rhel_8_13 diff --git a/packaging/test/Makefile.suse b/packaging/test/Makefile.suse new file mode 100644 index 000000000..9257bdbfd --- /dev/null +++ b/packaging/test/Makefile.suse @@ -0,0 +1,41 @@ +# Suse 15.1 +build/test_suse_15.1_9.6: + $(call test_suse,suse,15.1,,9.6,9.6.21) + touch build/test_suse_15.1_9.6 + +build/test_suse_15.1_10: + $(call test_suse,suse,15.1,,10,10.16) + touch build/test_suse_15.1_10 + +build/test_suse_15.1_11: + $(call test_suse,suse,15.1,,11,11.11) + touch build/test_suse_15.1_11 + +build/test_suse_15.1_12: + $(call test_suse,suse,15.1,,12,12.6) + touch build/test_suse_15.1_12 + +build/test_suse_15.1_13: + $(call test_suse,suse,15.1,,13,13.2) + touch build/test_suse_15.1_13 + +# Suse 15.2 +build/test_suse_15.2_9.6: + $(call test_suse,suse,15.2,,9.6,9.6.21) + touch build/test_suse_15.2_9.6 + +build/test_suse_15.2_10: + $(call test_suse,suse,15.2,,10,10.16) + touch build/test_suse_15.2_10 + +build/test_suse_15.2_11: + $(call test_suse,suse,15.2,,11,11.11) + touch build/test_suse_15.2_11 + +build/test_suse_15.2_12: + $(call test_suse,suse,15.2,,12,12.6) + touch build/test_suse_15.2_12 + +build/test_suse_15.2_13: + $(call test_suse,suse,15.2,,13,13.2) + touch build/test_suse_15.2_13 diff --git a/packaging/test/Makefile.ubuntu b/packaging/test/Makefile.ubuntu new file mode 100644 index 000000000..9e201a30b --- /dev/null +++ b/packaging/test/Makefile.ubuntu @@ -0,0 +1,62 @@ +# UBUNTU 16.04 +build/test_ubuntu_16.04_9.6: + $(call test_deb,ubuntu,16.04,xenial,9.6,9.6.21) + touch build/test_ubuntu_16.04_9.6 + +build/test_ubuntu_16.04_10: + $(call test_deb,ubuntu,16.04,xenial,10,10.16) + touch build/test_ubuntu_16.04_10 + +build/test_ubuntu_16.04_11: + $(call test_deb,ubuntu,16.04,xenial,11,11.11) + touch build/test_ubuntu_16.04_11 + +build/test_ubuntu_16.04_12: + $(call test_deb,ubuntu,16.04,xenial,12,12.6) + touch build/test_ubuntu_16.04_12 + +build/test_ubuntu_16.04_13: + $(call test_deb,ubuntu,16.04,xenial,13,13.2) + touch build/test_ubuntu_16.04_13 + +# UBUNTU 18.04 +build/test_ubuntu_18.04_9.6: + $(call test_deb,ubuntu,18.04,bionic,9.6,9.6.21) + touch build/test_ubuntu_18.04_9.6 + +build/test_ubuntu_18.04_10: + $(call test_deb,ubuntu,18.04,bionic,10,10.16) + touch build/test_ubuntu_18.04_10 + +build/test_ubuntu_18.04_11: + $(call test_deb,ubuntu,18.04,bionic,11,11.11) + touch build/test_ubuntu_18.04_11 + +build/test_ubuntu_18.04_12: + $(call test_deb,ubuntu,18.04,bionic,12,12.6) + touch build/test_ubuntu_18.04_12 + +build/test_ubuntu_18.04_13: + $(call test_deb,ubuntu,18.04,bionic,13,13.2) + touch build/test_ubuntu_18.04_13 + +# UBUNTU 20.04 +build/test_ubuntu_20.04_9.6: + $(call test_deb,ubuntu,20.04,focal,9.6,9.6.21) + touch build/test_ubuntu_20.04_9.6 + +build/test_ubuntu_20.04_10: + $(call test_deb,ubuntu,20.04,focal,10,10.16) + touch build/test_ubuntu_20.04_10 + +build/test_ubuntu_20.04_11: + $(call test_deb,ubuntu,20.04,focal,11,11.11) + touch build/test_ubuntu_20.04_11 + +build/test_ubuntu_20.04_12: + $(call test_deb,ubuntu,20.04,focal,12,12.6) + touch build/test_ubuntu_20.04_12 + +build/test_ubuntu_20.04_13: + $(call test_deb,ubuntu,20.04,focal,13,13.2) + touch build/test_ubuntu_20.04_13 diff --git a/packaging/test/scripts/alt.sh b/packaging/test/scripts/alt.sh new file mode 100755 index 000000000..262864474 --- /dev/null +++ b/packaging/test/scripts/alt.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -xe +set -o pipefail + +ulimit -n 1024 + +apt-get clean -y +apt-get update -y +apt-get install nginx su -y + +adduser nginx + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF + +/etc/init.d/nginx start + +# install POSTGRESQL + +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +apt-get update +apt-get install ${PKG_NAME} -y +${PKG_NAME} --help +${PKG_NAME} --version + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p${DISTRIB_VERSION} x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +echo "rpm [p${DISTRIB_VERSION}] http://mirror.yandex.ru/altlinux p${DISTRIB_VERSION}/branch/x86_64 debuginfo" > /etc/apt/sources.list.d/debug.list + +apt-get update -y +apt-get install ${PKG_NAME} -y +${PKG_NAME} --help +${PKG_NAME} --version + +exit 0 + +# TODO: run init, add-instance, backup and restore +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync --compress" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt diff --git a/packaging/test/scripts/alt_forks.sh b/packaging/test/scripts/alt_forks.sh new file mode 100755 index 000000000..c406e5358 --- /dev/null +++ b/packaging/test/scripts/alt_forks.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +ulimit -n 1024 + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +apt-get clean -y +apt-get update -y +apt-get install nginx su -y + +adduser nginx + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF + +/etc/init.d/nginx start + +# install POSTGRESQL + +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p${DISTRIB_VERSION} x86_64 forks" > /etc/apt/sources.list.d/pg_probackup.list +echo "rpm [p${DISTRIB_VERSION}] http://mirror.yandex.ru/altlinux p${DISTRIB_VERSION}/branch/x86_64 debuginfo" > /etc/apt/sources.list.d/debug.list + +apt-get update -y +apt-get install ${PKG_NAME} ${PKG_NAME}-debuginfo -y +${PKG_NAME} --help +${PKG_NAME} --version + +exit 0 + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt diff --git a/packaging/test/scripts/deb.sh b/packaging/test/scripts/deb.sh new file mode 100755 index 000000000..76e3bb043 --- /dev/null +++ b/packaging/test/scripts/deb.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +# upgrade and utils +# export parameters +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +apt-get -qq update +apt-get -qq install -y wget nginx gnupg lsb-release +#apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps + +# echo -e 'Package: *\nPin: origin test.postgrespro.ru\nPin-Priority: 800' >\ +# /etc/apt/preferences.d/pgpro-800 + +# install nginx +echo "127.0.0.1 test.postgrespro.ru" >> /etc/hosts +cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + apt-get update -y + apt-get install -y postgresql-${PG_VERSION} +#fi + +# install pg_probackup from current public repo +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-old.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get install -y pg-probackup-${PG_VERSION} +pg_probackup-${PG_VERSION} --help +pg_probackup-${PG_VERSION} --version + +# Artful do no have PostgreSQL packages at all, Precise do not have PostgreSQL 10 +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + export PGDATA=/var/lib/postgresql/${PG_VERSION}/data + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/initdb -k -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} init -B /tmp/backup" + su postgres -c "pg_probackup-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='pg_probackup-${PG_VERSION} archive-push --no-sync -B /tmp/backup compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" + sleep 5 + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b full -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup --archive -D ${PGDATA}" + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pgbench --no-vacuum -i -s 5" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +#fi + +# install new packages +echo "deb [arch=amd64] http://test.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-new.list +wget -O - http://test.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - +apt-get update +apt-get install -y pg-probackup-${PG_VERSION} +pg_probackup-${PG_VERSION} --help +pg_probackup-${PG_VERSION} --version + +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl restart -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + +# su postgres -c "pg_probackup-${PG_VERSION} init -B /tmp/backup" +# su postgres -c "pg_probackup-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node --compress -b delta -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" + rm -rf ${PGDATA} + + su postgres -c "pg_probackup-${PG_VERSION} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl start -w -t 60 -D ${PGDATA}" + +sleep 5 +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/psql" || exit 1 +#fi + +# CHECK SRC package +apt-get install -y dpkg-dev +echo "deb-src [arch=amd64] http://test.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list + +wget -O - http://test.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get update -y + +cd /mnt +apt-get source pg-probackup-${PG_VERSION} +exit 0 + +cd pg-probackup-${PG_VERSION}-${PKG_VERSION} +#mk-build-deps --install --remove --tool 'apt-get --no-install-recommends --yes' debian/control +#rm -rf ./*.deb +apt-get install -y debhelper bison flex gettext zlib1g-dev +dpkg-buildpackage -us -uc diff --git a/packaging/test/scripts/deb_forks.sh b/packaging/test/scripts/deb_forks.sh new file mode 100755 index 000000000..5175f38db --- /dev/null +++ b/packaging/test/scripts/deb_forks.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# TODO: remove after release +exit 0 + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then + exit 0 +fi + +# upgrade and utils +# export parameters +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +#if [ ${CODENAME} == 'jessie' ]; then +#printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list +#fi + +apt-get -qq update +apt-get -qq install -y wget nginx gnupg lsb-release apt-transport-https +#apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps + +# echo -e 'Package: *\nPin: origin test.postgrespro.ru\nPin-Priority: 800' >\ +# /etc/apt/preferences.d/pgpro-800 + +# install nginx +echo "127.0.0.1 test.postgrespro.ru" >> /etc/hosts +cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESPRO +if [ ${PBK_EDITION} == 'std' ]; then + sh -c 'echo "deb https://repo.postgrespro.ru/pgpro-${PG_VERSION}/${DISTRIB}/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/pgpro.list' + wget --quiet -O - https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/GPG-KEY-POSTGRESPRO | apt-key add - + apt-get update -y + + apt-get install -y postgrespro-std-${PG_VERSION} + BINDIR="/opt/pgpro/std-${PG_VERSION}/bin" + export LD_LIBRARY_PATH=/opt/pgpro/std-${PG_VERSION}/lib/ +fi + +# install pg_probackup from current public repo +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-old.list +wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} +pg_probackup-${PBK_EDITION}-${PG_VERSION} --help +pg_probackup-${PBK_EDITION}-${PG_VERSION} --version + + +if [ ${PBK_EDITION} == 'std' ]; then + export PGDATA=/tmp/data + su postgres -c "${BINDIR}/initdb -k -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} init -B /tmp/backup" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='pg_probackup-${PBK_EDITION}-${PG_VERSION} archive-push --no-sync -B /tmp/backup compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + + su postgres -c "${BINDIR}/pg_ctl stop -w -t 60 -D /var/lib/pgpro/std-${PG_VERSION}/data" || echo "it is all good" + su postgres -c "${BINDIR}/pg_ctl start -D ${PGDATA}" + sleep 5 + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b full -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + + su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +fi + +# install new packages +echo "deb [arch=amd64] http://test.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-new.list +wget -O - http://test.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | apt-key add - +apt-get update -y + +#if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then +# apt-get install -y libpq5 pg-probackup-${PBK_EDITION}-${PG_VERSION} +#else +# apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} +#fi + +apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} + +# in Ent 11 and 10 because of PQselect vanilla libpq5 is incompatible with Ent pg_probackup +if [ ${PBK_EDITION} == 'ent' ]; then + if [ ${PG_VERSION} == '11' ] || [ ${PG_VERSION} == '10' ] || [ ${PG_VERSION} == '9.6' ]; then + exit 0 + fi +fi + +pg_probackup-${PBK_EDITION}-${PG_VERSION} --help +pg_probackup-${PBK_EDITION}-${PG_VERSION} --version + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then + exit 0 +fi + + +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + su postgres -c "${BINDIR}/pgbench --no-vacuum -t 1000 -c 1" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + + su postgres -c "${BINDIR}/pg_ctl stop -w -t 60 -D ${PGDATA}" + rm -rf ${PGDATA} + + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} restore --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "${BINDIR}/pg_ctl start -w -t 60 -D ${PGDATA}" + +sleep 5 +echo "select count(*) from pgbench_accounts;" | su postgres -c "${BINDIR}/psql" || exit 1 + +exit 0 diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh new file mode 100755 index 000000000..3f24cc7e5 --- /dev/null +++ b/packaging/test/scripts/rpm.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +# yum upgrade -y || echo 'some packages in docker failed to upgrade' +# yum install -y sudo +if [ ${DISTRIB} == 'rhel' ] && [ ${PG_TOG} == '13' ]; then # no packages for PG13 on PGDG + exit 0 +fi + +#if [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ] && [ ${PG_TOG} == '13' ]; then # no packages for PG13 on PGDG +# exit 0 +#fi + +if [ ${DISTRIB_VERSION} == '6' ]; then + yum install -y https://nginx.org/packages/rhel/6/x86_64/RPMS/nginx-1.8.1-1.el6.ngx.x86_64.rpm +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then + yum install -y nginx +elif [ ${DISTRIB_VERSION} == '7' ]; then + yum install -y https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.8.1-1.el7.ngx.x86_64.rpm +else + yum install epel-release -y + yum install -y nginx +fi + +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-${DISTRIB_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm + +if [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then + dnf -qy module disable postgresql +fi + +if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then + dnf -qy module disable postgresql +fi + +yum install -y postgresql${PG_TOG}-server.x86_64 +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages +yum install -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +yum install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf +echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +echo "archive_command='${PKG_NAME} archive-push --no-sync -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +sleep 5 + +su postgres -c "${PKG_NAME} init -B /tmp/backup" +su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts + +# yum remove -y pg_probackup-repo +#yum install -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +yum clean all -y + +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup.repo + +yum update -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +#else +# echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +# rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +# yum install -y ${PKG_NAME} +# ${PKG_NAME} --help +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 10" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +# rm -rf ${PGDATA} +# su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 10 +# echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 +#fi + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt +yum install yum-utils rpm-build -y +yumdownloader --source ${PKG_NAME} +rpm -ivh ./*.rpm +cd /root/rpmbuild/SPECS +exit 0 + +# build pg_probackup +yum-builddep -y pg_probackup.spec +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log diff --git a/packaging/test/scripts/rpm_forks.sh b/packaging/test/scripts/rpm_forks.sh new file mode 100755 index 000000000..8596f6656 --- /dev/null +++ b/packaging/test/scripts/rpm_forks.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +if [ ${PBK_PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +# yum upgrade -y || echo 'some packages in docker failed to upgrade' +# yum install -y sudo + +if [ ${DISTRIB} == 'rhel' ] && [ ${DISTRIB_VERSION} == '6' ]; then + exit 0; +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ]; then + exit 0; +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then + yum install -y nginx +elif [ ${DISTRIB_VERSION} == '7' ]; then + yum install -y https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.8.1-1.el7.ngx.x86_64.rpm +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ]; then + yum install -y https://nginx.org/packages/rhel/6/x86_64/RPMS/nginx-1.8.1-1.el6.ngx.x86_64.rpm +else + yum install epel-release -y + yum install -y nginx +fi + +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# if [ ${DISTRIB} == 'centos' ]; then + +# install old packages +yum install -y http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-${DISTRIB}.noarch.rpm +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup-forks.repo + +yum install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +if [ $PBK_EDITION == 'std' ] ; then + + # install POSTGRESQL + # rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-${DISTRIB_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm + if [[ ${PG_VERSION} == '11' ]] || [[ ${PG_VERSION} == '12' ]]; then + rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + else + rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + fi + + if [[ ${PG_VERSION} == '9.6' ]]; then + yum install -y postgrespro${PG_TOG}-server.x86_64 + BINDIR="/usr/pgpro-${PG_VERSION}/bin" + else + yum install -y postgrespro-std-${PG_TOG}-server.x86_64 + BINDIR="/opt/pgpro/std-${PG_VERSION}/bin" + export LD_LIBRARY_PATH=/opt/pgpro/std-${PG_VERSION}/lib/ + fi + + export PGDATA=/tmp/data + + su postgres -c "${BINDIR}/initdb -k -D ${PGDATA}" + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + su postgres -c "${BINDIR}/pg_ctl start -D ${PGDATA}" + sleep 5 + + su postgres -c "${PKG_NAME} init -B /tmp/backup" + su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" + su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + + su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" + su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +fi + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts + +# yum remove -y pg_probackup-repo +#yum install -y http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-${DISTRIB}.noarch.rpm +#yum clean all -y + +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup-forks.repo + +# yum update -y ${PKG_NAME} +yum install -y ${PKG_NAME} + +${PKG_NAME} --help +${PKG_NAME} --version + +if [ $PBK_EDITION == 'ent' ]; then + exit 0 +fi + +# su postgres -c "${BINDIR}/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b full -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "${BINDIR}/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${BINDIR}/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "${BINDIR}/psql" || exit 1 + +#else +# echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +# rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +# yum install -y ${PKG_NAME} +# ${PKG_NAME} --help +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 10" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +# rm -rf ${PGDATA} +# su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 10 +# echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 +#fi + +exit 0 diff --git a/packaging/test/scripts/suse.sh b/packaging/test/scripts/suse.sh new file mode 100755 index 000000000..ff630b479 --- /dev/null +++ b/packaging/test/scripts/suse.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +if [ ${PG_TOG} == '13' ]; then # no packages for PG13 + exit 0 +fi + +if [ ${PG_TOG} == '11' ]; then # no packages for PG11 + exit 0 +fi + +if [ ${PG_TOG} == '95' ]; then # no packages for PG95 + exit 0 +fi + +zypper install -y nginx +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +useradd postgres + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +zypper install -y postgresql${PG_TOG} postgresql${PG_TOG}-server postgresql${PG_TOG}-contrib +export PGDATA=/tmp/data + +# install old packages +zypper install --allow-unsigned-rpm -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +zypper --gpg-auto-import-keys install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/initdb -k -D ${PGDATA}" +echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf +echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +echo "archive_command='${PKG_NAME} archive-push --no-sync -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl start -D ${PGDATA}" +sleep 5 + +su postgres -c "${PKG_NAME} init -B /tmp/backup" +su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +zypper clean all -y + +sed -i "s/https/http/g" /etc/zypp/repos.d/pg_probackup.repo + +zypper update -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/psql" || exit 1 + +exit 0 + +# SRC PACKAGE +cd /mnt +yum install yum-utils rpm-build -y +yumdownloader --source ${PKG_NAME} +rpm -ivh ./*.rpm +cd /root/rpmbuild/SPECS +exit 0 + +# build pg_probackup +yum-builddep -y pg_probackup.spec +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log diff --git a/packaging/test/scripts/suse_forks.sh b/packaging/test/scripts/suse_forks.sh new file mode 100644 index 000000000..b83f1ddd9 --- /dev/null +++ b/packaging/test/scripts/suse_forks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -xe +set -o pipefail +exit 0 From 2e2a8b8dca14f6118c650b6eab8db5a53897e1d3 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Sep 2021 15:40:00 +0300 Subject: [PATCH 1783/2107] [Issue #360] add test coverage --- tests/exclude.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/exclude.py b/tests/exclude.py index 83743bf0b..b98a483d0 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -181,8 +181,7 @@ def test_exclude_unlogged_tables_1(self): self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=['--stream'] - ) + options=['--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -201,6 +200,67 @@ def test_exclude_unlogged_tables_1(self): # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_exclude_unlogged_tables_2(self): + """ + make node, create unlogged, take FULL, check + that unlogged was not backed up + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + "shared_buffers": "10MB"}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for backup_type in ['full', 'delta', 'page']: + + if backup_type == 'full': + node.safe_psql( + 'postgres', + 'create unlogged table test as select generate_series(0,20050000)::text') + else: + node.safe_psql( + 'postgres', + 'insert into test select generate_series(0,20050000)::text') + + rel_path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('test')").decode('utf-8').rstrip() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type=backup_type, options=['--stream']) + + filelist = self.get_backup_filelist( + backup_dir, 'node', backup_id) + + self.assertNotIn( + rel_path, filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.1', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.2', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.3', filelist, + "Unlogged table was not excluded") + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_exclude_log_dir(self): """ From 7f690abdf67d02c2c12ac3359cdbe6aed8a0ed59 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sun, 26 Sep 2021 15:46:02 +0300 Subject: [PATCH 1784/2107] [Issue #360] correctly exclude unlogged relations from backup --- src/dir.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/dir.c b/src/dir.c index c5c5b3297..00a4c4f82 100644 --- a/src/dir.c +++ b/src/dir.c @@ -730,21 +730,32 @@ dir_check_file(pgFile *file, bool backup_logs) if (fork_name) { /* Auxiliary fork of the relfile */ - if (strcmp(fork_name, "vm") == 0) + if (strcmp(fork_name, "_vm") == 0) file->forkName = vm; - else if (strcmp(fork_name, "fsm") == 0) + else if (strcmp(fork_name, "_fsm") == 0) file->forkName = fsm; - else if (strcmp(fork_name, "cfm") == 0) + else if (strcmp(fork_name, "_cfm") == 0) file->forkName = cfm; - else if (strcmp(fork_name, "ptrack") == 0) + else if (strcmp(fork_name, "_ptrack") == 0) file->forkName = ptrack; - else if (strcmp(fork_name, "init") == 0) + else if (strcmp(fork_name, "_init") == 0) file->forkName = init; + // extract relOid for certain forks + if (file->forkName == vm || + file->forkName == fsm || + file->forkName == init || + file->forkName == cfm) + { + // sanity + if (sscanf(file->name, "%u_*", &(file->relOid)) != 1) + file->relOid = 0; + } + /* Do not backup ptrack files */ if (file->forkName == ptrack) return CHECK_FALSE; From 6081c08f6128d18273fc10ad56796881178a6498 Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 2 Oct 2021 12:52:21 +0300 Subject: [PATCH 1785/2107] Fix link to latest windows installers --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 344b03fb3..e8d25c5a6 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ For detailed release plans check [Milestones](https://github.com/postgrespro/pg_ ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.9). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.15). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From 7feb7489053efe267d717f8efea51be4b0a6969e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 13 Oct 2021 05:30:20 +0300 Subject: [PATCH 1786/2107] Prerelease test stabilization ptrack.PtrackTest.test_ptrack_threads backup.BackupTest.test_backup_with_least_privileges_role ptrack.PtrackTest.test_ptrack_without_full option.OptionTest.test_help_1 --- tests/backup.py | 2 +- tests/expected/option_help.out | 20 +++++++++++++++++--- tests/ptrack.py | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 558c62de3..3548fa56b 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1959,7 +1959,7 @@ def test_backup_with_least_privileges_role(self): node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " - "GRANT EXECUTE ON FUNCTION 'ptrack.ptrack_init_lsn()' TO backup; ") + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") if ProbackupTest.enterprise: node.safe_psql( diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index c2b15e7ac..01384a893 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -42,7 +42,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup backup -B backup-path -b backup-mode --instance=instance_name [-D pgdata-path] [-C] - [--stream [-S slot-name]] [--temp-slot] + [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] [--no-validate] [--skip-block-validation] [--external-dirs=external-directories-paths] @@ -53,7 +53,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--error-log-filename=error-log-filename] [--log-directory=log-directory] [--log-rotation-size=log-rotation-size] - [--log-rotation-age=log-rotation-age] + [--log-rotation-age=log-rotation-age] [--no-color] [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -113,7 +113,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup show -B backup-path [--instance=instance_name [-i backup-id]] [--format=format] [--archive] - [--help] + [--no-color] [--help] pg_probackup delete -B backup-path --instance=instance_name [-j num-threads] [--progress] @@ -165,5 +165,19 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--help] + pg_probackup catchup -b catchup-mode + --source-pgdata=path_to_pgdata_on_remote_server + --destination-pgdata=path_to_local_dir + [--stream [-S slot-name] [--temp-slot | --perm-slot]] + [-j num-threads] + [-T OLDDIR=NEWDIR] + [--exclude-path=path_prefix] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + Read the website for details. Report bugs to . diff --git a/tests/ptrack.py b/tests/ptrack.py index bcc8dc20a..a20be54b8 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -100,6 +100,10 @@ def test_ptrack_without_full(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + try: self.backup_node(backup_dir, 'node', node, backup_type="ptrack") # we should die here because exception is what we expect to happen @@ -136,6 +140,10 @@ def test_ptrack_threads(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4"]) From 396155e5bcc69a6ec21598012592a0d5cf31b2eb Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Wed, 13 Oct 2021 05:56:44 +0300 Subject: [PATCH 1787/2107] Issue 439 (#440) * [Issue #439] skip unsupported tests in 9.5 (tests with backups from replica and with pg_control_checkpoint() calling) --- tests/archive.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index 0ade2d66a..4b07c1dbd 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -83,6 +83,12 @@ def test_pgpro434_2(self): pg_options={ 'checkpoint_timeout': '30s'} ) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -693,6 +699,11 @@ def test_replica_archive(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) @@ -818,6 +829,12 @@ def test_master_and_replica_parallel_archiving(self): pg_options={ 'archive_timeout': '10s'} ) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() @@ -908,6 +925,11 @@ def test_basic_master_and_replica_concurrent_archiving(self): 'checkpoint_timeout': '30s', 'archive_timeout': '10s'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + replica = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'replica')) replica.cleanup() @@ -2009,6 +2031,11 @@ def test_archive_pg_receivexlog_partial_handling(self): set_replication=True, initdb_params=['--data-checksums']) + if self.get_version(node) < self.version_to_num('9.6.0'): + self.del_test_dir(module_name, fname) + return unittest.skip( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2655,4 +2682,4 @@ def test_archive_empty_history_file(self): #t2 ---------------- # / #t1 -A-------- -# \ No newline at end of file +# From f7a81aa5fd07e6dc38192d1af6c896e4c7d61a9c Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 13 Oct 2021 13:48:53 +0300 Subject: [PATCH 1788/2107] [Issue #413] packaging: bump postgres versions, add 14, remove 9.5, some workarounds (caused by old docker images) --- Makefile | 2 +- packaging/Makefile.pkg | 42 +++++++++++----------- packaging/pkg/Makefile.alt | 43 ++++++++++++++-------- packaging/pkg/Makefile.centos | 28 +++++++++------ packaging/pkg/Makefile.debian | 56 ++++++++++++++++++----------- packaging/pkg/Makefile.oraclelinux | 43 ++++++++++++++-------- packaging/pkg/Makefile.rhel | 28 +++++++++------ packaging/pkg/Makefile.suse | 28 +++++++++------ packaging/pkg/Makefile.ubuntu | 57 +++++++++++++++++++----------- packaging/pkg/scripts/deb.sh | 6 ++-- packaging/pkg/scripts/rpm.sh | 19 ++++++---- 11 files changed, 220 insertions(+), 132 deletions(-) diff --git a/Makefile b/Makefile index 4e463bf7c..aca1df356 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PROGRAM = pg_probackup WORKDIR ?= $(CURDIR) BUILDDIR = $(WORKDIR)/build/ -PBK_GIT_REPO = http://github.com/postgrespro/pg_probackup +PBK_GIT_REPO = https://github.com/postgrespro/pg_probackup # utils OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ diff --git a/packaging/Makefile.pkg b/packaging/Makefile.pkg index bfe2043c3..fc92ae408 100644 --- a/packaging/Makefile.pkg +++ b/packaging/Makefile.pkg @@ -38,39 +38,39 @@ build/prepare: build/clean: build/prepare find $(BUILDDIR) -maxdepth 1 -type f -exec rm -f {} \; -build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt build/suse # build/rhel +build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt build/suse build/rhel @echo Packaging is done ### DEBIAN build/debian: build/debian_8 build/debian_9 build/debian_10 build/debian_11 @echo Debian: done -build/debian_8: build/debian_8_9.5 build/debian_8_9.6 build/debian_8_10 build/debian_8_11 build/debian_8_12 build/debian_8_13 +build/debian_8: build/debian_8_9.6 build/debian_8_10 build/debian_8_11 build/debian_8_12 build/debian_8_13 build/debian_8_14 @echo Debian 8: done -build/debian_9: build/debian_9_9.5 build/debian_9_9.6 build/debian_9_10 build/debian_9_11 build/debian_9_12 build/debian_9_13 +build/debian_9: build/debian_9_9.6 build/debian_9_10 build/debian_9_11 build/debian_9_12 build/debian_9_13 build/debian_9_14 @echo Debian 9: done -build/debian_10: build/debian_10_9.5 build/debian_10_9.6 build/debian_10_10 build/debian_10_11 build/debian_10_12 build/debian_10_13 +build/debian_10: build/debian_10_9.6 build/debian_10_10 build/debian_10_11 build/debian_10_12 build/debian_10_13 build/debian_10_14 @echo Debian 10: done -build/debian_11: build/debian_11_9.5 build/debian_11_9.6 build/debian_11_10 build/debian_11_11 build/debian_11_12 build/debian_11_13 +build/debian_11: build/debian_11_9.6 build/debian_11_10 build/debian_11_11 build/debian_11_12 build/debian_11_13 build/debian_11_14 @echo Debian 11: done ### UBUNTU build/ubuntu: build/ubuntu_14.04 build/ubuntu_16.04 build/ubuntu_18.04 build/ubuntu_20.04 @echo Ubuntu: done -build/ubuntu_14.04: build/ubuntu_14.04_9.5 build/ubuntu_14.04_9.6 build/ubuntu_14.04_10 build/ubuntu_14.04_11 build/ubuntu_14.04_12 build/ubuntu_14.04_13 +build/ubuntu_14.04: build/ubuntu_14.04_9.6 build/ubuntu_14.04_10 build/ubuntu_14.04_11 build/ubuntu_14.04_12 build/ubuntu_14.04_13 build/ubuntu_14.04_14 @echo Ubuntu 14.04: done -build/ubuntu_16.04: build/ubuntu_16.04_9.5 build/ubuntu_16.04_9.6 build/ubuntu_16.04_10 build/ubuntu_16.04_11 build/ubuntu_16.04_12 build/ubuntu_16.04_13 +build/ubuntu_16.04: build/ubuntu_16.04_9.6 build/ubuntu_16.04_10 build/ubuntu_16.04_11 build/ubuntu_16.04_12 build/ubuntu_16.04_13 build/ubuntu_16.04_14 @echo Ubuntu 16.04: done -build/ubuntu_18.04: build/ubuntu_18.04_9.5 build/ubuntu_18.04_9.6 build/ubuntu_18.04_10 build/ubuntu_18.04_11 build/ubuntu_18.04_12 build/ubuntu_18.04_13 +build/ubuntu_18.04: build/ubuntu_18.04_9.6 build/ubuntu_18.04_10 build/ubuntu_18.04_11 build/ubuntu_18.04_12 build/ubuntu_18.04_13 build/ubuntu_18.04_14 @echo Ubuntu 18.04: done -build/ubuntu_20.04: build/ubuntu_20.04_9.5 build/ubuntu_20.04_9.6 build/ubuntu_20.04_10 build/ubuntu_20.04_11 build/ubuntu_20.04_12 build/ubuntu_20.04_13 +build/ubuntu_20.04: build/ubuntu_20.04_9.6 build/ubuntu_20.04_10 build/ubuntu_20.04_11 build/ubuntu_20.04_12 build/ubuntu_20.04_13 build/ubuntu_20.04_14 @echo Ubuntu 20.04: done define build_deb @@ -92,33 +92,33 @@ include packaging/pkg/Makefile.ubuntu build/centos: build/centos_7 build/centos_8 #build/rpm_repo_package_centos @echo Centos: done -build/centos_7: build/centos_7_9.5 build/centos_7_9.6 build/centos_7_10 build/centos_7_11 build/centos_7_12 build/centos_7_13 +build/centos_7: build/centos_7_9.6 build/centos_7_10 build/centos_7_11 build/centos_7_12 build/centos_7_13 build/centos_7_14 @echo Centos 7: done -build/centos_8: build/centos_8_9.5 build/centos_8_9.6 build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 +build/centos_8: build/centos_8_9.6 build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 build/centos_8_14 @echo Centos 8: done # Oracle Linux build/oraclelinux: build/oraclelinux_6 build/oraclelinux_7 build/oraclelinux_8 #build/rpm_repo_package_oraclelinux @echo Oraclelinux: done -build/oraclelinux_6: build/oraclelinux_6_9.5 build/oraclelinux_6_9.6 build/oraclelinux_6_10 build/oraclelinux_6_11 build/oraclelinux_6_12 build/oraclelinux_6_13 +build/oraclelinux_6: build/oraclelinux_6_9.6 build/oraclelinux_6_10 build/oraclelinux_6_11 build/oraclelinux_6_12 build/oraclelinux_6_13 build/oraclelinux_6_14 @echo Oraclelinux 6: done -build/oraclelinux_7: build/oraclelinux_7_9.5 build/oraclelinux_7_9.6 build/oraclelinux_7_10 build/oraclelinux_7_11 build/oraclelinux_7_12 build/oraclelinux_7_13 +build/oraclelinux_7: build/oraclelinux_7_9.6 build/oraclelinux_7_10 build/oraclelinux_7_11 build/oraclelinux_7_12 build/oraclelinux_7_13 build/oraclelinux_7_14 @echo Oraclelinux 7: done -build/oraclelinux_8: build/oraclelinux_8_9.5 build/oraclelinux_8_9.6 build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 +build/oraclelinux_8: build/oraclelinux_8_9.6 build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 build/oraclelinux_8_14 @echo Oraclelinux 8: done # RHEL build/rhel: build/rhel_7 build/rhel_8 #build/rpm_repo_package_rhel @echo Rhel: done -build/rhel_7: build/rhel_7_9.5 build/rhel_7_9.6 build/rhel_7_10 build/rhel_7_11 build/rhel_7_12 build/rhel_7_13 +build/rhel_7: build/rhel_7_9.6 build/rhel_7_10 build/rhel_7_11 build/rhel_7_12 build/rhel_7_13 build/rhel_7_14 @echo Rhel 7: done -build/rhel_8: build/rhel_8_9.5 build/rhel_8_9.6 build/rhel_8_10 build/rhel_8_11 build/rhel_8_12 build/rhel_8_13 +build/rhel_8: build/rhel_8_9.6 build/rhel_8_10 build/rhel_8_11 build/rhel_8_12 build/rhel_8_13 build/rhel_8_14 @echo Rhel 8: done @@ -143,13 +143,13 @@ include packaging/pkg/Makefile.oraclelinux build/alt: build/alt_7 build/alt_8 build/alt_9 @echo Alt Linux: done -build/alt_7: build/alt_7_9.5 build/alt_7_9.6 build/alt_7_10 build/alt_7_11 build/alt_7_12 build/alt_7_13 +build/alt_7: build/alt_7_9.6 build/alt_7_10 build/alt_7_11 build/alt_7_12 build/alt_7_13 build/alt_7_14 @echo Alt Linux 7: done -build/alt_8: build/alt_8_9.5 build/alt_8_9.6 build/alt_8_10 build/alt_8_11 build/alt_8_12 build/alt_8_13 +build/alt_8: build/alt_8_9.6 build/alt_8_10 build/alt_8_11 build/alt_8_12 build/alt_8_13 build/alt_8_14 @echo Alt Linux 8: done -build/alt_9: build/alt_9_9.5 build/alt_9_9.6 build/alt_9_10 build/alt_9_11 build/alt_9_12 build/alt_9_13 +build/alt_9: build/alt_9_9.6 build/alt_9_10 build/alt_9_11 build/alt_9_12 build/alt_9_13 build/alt_9_14 @echo Alt Linux 9: done define build_alt @@ -170,10 +170,10 @@ include packaging/pkg/Makefile.alt build/suse: build/suse_15.1 build/suse_15.2 @echo Suse: done -build/suse_15.1: build/suse_15.1_9.5 build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 +build/suse_15.1: build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 build/suse_15.1_14 @echo Rhel 15.1: done -build/suse_15.2: build/suse_15.2_9.5 build/suse_15.2_9.6 build/suse_15.2_10 build/suse_15.2_11 build/suse_15.2_12 build/suse_15.2_13 +build/suse_15.2: build/suse_15.2_9.6 build/suse_15.2_10 build/suse_15.2_11 build/suse_15.2_12 build/suse_15.2_13 build/suse_15.2_14 @echo Rhel 15.1: done define build_suse diff --git a/packaging/pkg/Makefile.alt b/packaging/pkg/Makefile.alt index e3fbae26e..919d3f58c 100644 --- a/packaging/pkg/Makefile.alt +++ b/packaging/pkg/Makefile.alt @@ -4,71 +4,84 @@ build/alt_7_9.5: touch build/alt_7_9.5 build/alt_7_9.6: - $(call build_alt,alt,7,,9.6,9.6.21) + $(call build_alt,alt,7,,9.6,9.6.23) touch build/alt_7_9.6 build/alt_7_10: - $(call build_alt,alt,7,,10,10.17) + $(call build_alt,alt,7,,10,10.18) touch build/alt_7_10 build/alt_7_11: - $(call build_alt,alt,7,,11,11.11) + $(call build_alt,alt,7,,11,11.13) touch build/alt_7_11 build/alt_7_12: - $(call build_alt,alt,7,,12,12.6) + $(call build_alt,alt,7,,12,12.8) touch build/alt_7_12 build/alt_7_13: - $(call build_alt,alt,7,,13,13.2) + $(call build_alt,alt,7,,13,13.4) touch build/alt_7_13 +build/alt_7_14: + $(call build_alt,alt,7,,14,14.0) + touch build/alt_7_14 + # ALT 8 build/alt_8_9.5: $(call build_alt,alt,8,,9.5,9.5.25) touch build/alt_8_9.5 build/alt_8_9.6: - $(call build_alt,alt,8,,9.6,9.6.21) + $(call build_alt,alt,8,,9.6,9.6.23) touch build/alt_8_9.6 build/alt_8_10: - $(call build_alt,alt,8,,10,10.17) + $(call build_alt,alt,8,,10,10.18) touch build/alt_8_10 build/alt_8_11: - $(call build_alt,alt,8,,11,11.11) + $(call build_alt,alt,8,,11,11.13) touch build/alt_8_11 build/alt_8_12: - $(call build_alt,alt,8,,12,12.6) + $(call build_alt,alt,8,,12,12.8) touch build/alt_8_12 build/alt_8_13: - $(call build_alt,alt,8,,13,13.2) + $(call build_alt,alt,8,,13,13.4) touch build/alt_8_13 +build/alt_8_14: + $(call build_alt,alt,8,,14,14.0) + touch build/alt_8_14 + # ALT 9 build/alt_9_9.5: $(call build_alt,alt,9,,9.5,9.5.25) touch build/alt_9_9.5 build/alt_9_9.6: - $(call build_alt,alt,9,,9.6,9.6.21) + $(call build_alt,alt,9,,9.6,9.6.23) touch build/alt_9_9.6 build/alt_9_10: - $(call build_alt,alt,9,,10,10.17) + $(call build_alt,alt,9,,10,10.18) touch build/alt_9_10 build/alt_9_11: - $(call build_alt,alt,9,,11,11.11) + $(call build_alt,alt,9,,11,11.13) touch build/alt_9_11 build/alt_9_12: - $(call build_alt,alt,9,,12,12.6) + $(call build_alt,alt,9,,12,12.8) touch build/alt_9_12 build/alt_9_13: - $(call build_alt,alt,9,,13,13.2) + $(call build_alt,alt,9,,13,13.4) touch build/alt_9_13 + +build/alt_9_14: + $(call build_alt,alt,9,,14,14.0) + touch build/alt_9_14 + diff --git a/packaging/pkg/Makefile.centos b/packaging/pkg/Makefile.centos index 9353b2cde..9542a5202 100644 --- a/packaging/pkg/Makefile.centos +++ b/packaging/pkg/Makefile.centos @@ -4,46 +4,54 @@ build/centos_7_9.5: touch build/centos_7_9.5 build/centos_7_9.6: - $(call build_rpm,centos,7,,9.6,9.6.21) + $(call build_rpm,centos,7,,9.6,9.6.23) touch build/centos_7_9.6 build/centos_7_10: - $(call build_rpm,centos,7,,10,10.16) + $(call build_rpm,centos,7,,10,10.18) touch build/centos_7_10 build/centos_7_11: - $(call build_rpm,centos,7,,11,11.11) + $(call build_rpm,centos,7,,11,11.13) touch build/centos_7_11 build/centos_7_12: - $(call build_rpm,centos,7,,12,12.6) + $(call build_rpm,centos,7,,12,12.8) touch build/centos_7_12 build/centos_7_13: - $(call build_rpm,centos,7,,13,13.2) + $(call build_rpm,centos,7,,13,13.4) touch build/centos_7_13 +build/centos_7_14: + $(call build_rpm,centos,7,,14,14.0) + touch build/centos_7_14 + # CENTOS 8 build/centos_8_9.5: $(call build_rpm,centos,8,,9.5,9.5.25) touch build/centos_8_9.5 build/centos_8_9.6: - $(call build_rpm,centos,8,,9.6,9.6.21) + $(call build_rpm,centos,8,,9.6,9.6.23) touch build/centos_8_9.6 build/centos_8_10: - $(call build_rpm,centos,8,,10,10.16) + $(call build_rpm,centos,8,,10,10.18) touch build/centos_8_10 build/centos_8_11: - $(call build_rpm,centos,8,,11,11.11) + $(call build_rpm,centos,8,,11,11.13) touch build/centos_8_11 build/centos_8_12: - $(call build_rpm,centos,8,,12,12.6) + $(call build_rpm,centos,8,,12,12.8) touch build/centos_8_12 build/centos_8_13: - $(call build_rpm,centos,8,,13,13.2) + $(call build_rpm,centos,8,,13,13.4) touch build/centos_8_13 + +build/centos_8_14: + $(call build_rpm,centos,8,,14,14.0) + touch build/centos_8_14 diff --git a/packaging/pkg/Makefile.debian b/packaging/pkg/Makefile.debian index 9625a14e9..7c82a412b 100644 --- a/packaging/pkg/Makefile.debian +++ b/packaging/pkg/Makefile.debian @@ -4,96 +4,112 @@ build/debian_8_9.5: touch build/debian_8_9.5 build/debian_8_9.6: - $(call build_deb,debian,8,jessie,9.6,9.6.21) + $(call build_deb,debian,8,jessie,9.6,9.6.23) touch build/debian_8_9.6 build/debian_8_10: - $(call build_deb,debian,8,jessie,10,10.16) + $(call build_deb,debian,8,jessie,10,10.18) touch build/debian_8_10 build/debian_8_11: - $(call build_deb,debian,8,jessie,11,11.11) + $(call build_deb,debian,8,jessie,11,11.13) touch build/debian_8_11 build/debian_8_12: - $(call build_deb,debian,8,jessie,12,12.6) + $(call build_deb,debian,8,jessie,12,12.8) touch build/debian_8_12 build/debian_8_13: - $(call build_deb,debian,8,jessie,13,13.2) + $(call build_deb,debian,8,jessie,13,13.4) touch build/debian_8_13 +build/debian_8_14: + $(call build_deb,debian,8,jessie,14,14.0) + touch build/debian_8_14 + # DEBIAN 9 build/debian_9_9.5: $(call build_deb,debian,9,stretch,9.5,9.5.25) touch build/debian_9_9.5 build/debian_9_9.6: - $(call build_deb,debian,9,stretch,9.6,9.6.21) + $(call build_deb,debian,9,stretch,9.6,9.6.23) touch build/debian_9_9.6 build/debian_9_10: - $(call build_deb,debian,9,stretch,10,10.16) + $(call build_deb,debian,9,stretch,10,10.18) touch build/debian_9_10 build/debian_9_11: - $(call build_deb,debian,9,stretch,11,11.11) + $(call build_deb,debian,9,stretch,11,11.13) touch build/debian_9_11 build/debian_9_12: - $(call build_deb,debian,9,stretch,12,12.6) + $(call build_deb,debian,9,stretch,12,12.8) touch build/debian_9_12 build/debian_9_13: - $(call build_deb,debian,9,stretch,13,13.2) + $(call build_deb,debian,9,stretch,13,13.4) touch build/debian_9_13 +build/debian_9_14: + $(call build_deb,debian,9,stretch,14,14.0) + touch build/debian_9_14 + # DEBIAN 10 build/debian_10_9.5: $(call build_deb,debian,10,buster,9.5,9.5.25) touch build/debian_10_9.5 build/debian_10_9.6: - $(call build_deb,debian,10,buster,9.6,9.6.21) + $(call build_deb,debian,10,buster,9.6,9.6.23) touch build/debian_10_9.6 build/debian_10_10: - $(call build_deb,debian,10,buster,10,10.16) + $(call build_deb,debian,10,buster,10,10.18) touch build/debian_10_10 build/debian_10_11: - $(call build_deb,debian,10,buster,11,11.11) + $(call build_deb,debian,10,buster,11,11.13) touch build/debian_10_11 build/debian_10_12: - $(call build_deb,debian,10,buster,12,12.6) + $(call build_deb,debian,10,buster,12,12.8) touch build/debian_10_12 build/debian_10_13: - $(call build_deb,debian,10,buster,13,13.2) + $(call build_deb,debian,10,buster,13,13.4) touch build/debian_10_13 +build/debian_10_14: + $(call build_deb,debian,10,buster,14,14.0) + touch build/debian_10_14 + # DEBIAN 11 build/debian_11_9.5: $(call build_deb,debian,11,bullseye,9.5,9.5.25) touch build/debian_11_9.5 build/debian_11_9.6: - $(call build_deb,debian,11,bullseye,9.6,9.6.21) + $(call build_deb,debian,11,bullseye,9.6,9.6.23) touch build/debian_11_9.6 build/debian_11_10: - $(call build_deb,debian,11,bullseye,10,10.16) + $(call build_deb,debian,11,bullseye,10,10.18) touch build/debian_11_10 build/debian_11_11: - $(call build_deb,debian,11,bullseye,11,11.11) + $(call build_deb,debian,11,bullseye,11,11.13) touch build/debian_11_11 build/debian_11_12: - $(call build_deb,debian,11,bullseye,12,12.6) + $(call build_deb,debian,11,bullseye,12,12.8) touch build/debian_11_12 build/debian_11_13: - $(call build_deb,debian,11,bullseye,13,13.2) + $(call build_deb,debian,11,bullseye,13,13.4) touch build/debian_11_13 + +build/debian_11_14: + $(call build_deb,debian,11,bullseye,14,14.0) + touch build/debian_11_14 diff --git a/packaging/pkg/Makefile.oraclelinux b/packaging/pkg/Makefile.oraclelinux index f4eada23f..3dbdbd424 100644 --- a/packaging/pkg/Makefile.oraclelinux +++ b/packaging/pkg/Makefile.oraclelinux @@ -4,71 +4,84 @@ build/oraclelinux_6_9.5: touch build/oraclelinux_6_9.5 build/oraclelinux_6_9.6: - $(call build_rpm,oraclelinux,6,,9.6,9.6.21) + $(call build_rpm,oraclelinux,6,,9.6,9.6.23) touch build/oraclelinux_6_9.6 build/oraclelinux_6_10: - $(call build_rpm,oraclelinux,6,,10,10.16) + $(call build_rpm,oraclelinux,6,,10,10.18) touch build/oraclelinux_6_10 build/oraclelinux_6_11: - $(call build_rpm,oraclelinux,6,,11,11.11) + $(call build_rpm,oraclelinux,6,,11,11.13) touch build/oraclelinux_6_11 build/oraclelinux_6_12: - $(call build_rpm,oraclelinux,6,,12,12.6) + $(call build_rpm,oraclelinux,6,,12,12.8) touch build/oraclelinux_6_12 build/oraclelinux_6_13: - $(call build_rpm,oraclelinux,6,,13,13.2) + $(call build_rpm,oraclelinux,6,,13,13.4) touch build/oraclelinux_6_13 +build/oraclelinux_6_14: + $(call build_rpm,oraclelinux,6,,14,14.0) + touch build/oraclelinux_6_14 + # ORACLE LINUX 7 build/oraclelinux_7_9.5: $(call build_rpm,oraclelinux,7,,9.5,9.5.25) touch build/oraclelinux_7_9.5 build/oraclelinux_7_9.6: - $(call build_rpm,oraclelinux,7,,9.6,9.6.21) + $(call build_rpm,oraclelinux,7,,9.6,9.6.23) touch build/oraclelinux_7_9.6 build/oraclelinux_7_10: - $(call build_rpm,oraclelinux,7,,10,10.16) + $(call build_rpm,oraclelinux,7,,10,10.18) touch build/oraclelinux_7_10 build/oraclelinux_7_11: - $(call build_rpm,oraclelinux,7,,11,11.11) + $(call build_rpm,oraclelinux,7,,11,11.13) touch build/oraclelinux_7_11 build/oraclelinux_7_12: - $(call build_rpm,oraclelinux,7,,12,12.6) + $(call build_rpm,oraclelinux,7,,12,12.8) touch build/oraclelinux_7_12 build/oraclelinux_7_13: - $(call build_rpm,oraclelinux,7,,13,13.2) + $(call build_rpm,oraclelinux,7,,13,13.4) touch build/oraclelinux_7_13 +build/oraclelinux_7_14: + $(call build_rpm,oraclelinux,7,,14,14.0) + touch build/oraclelinux_7_14 + # ORACLE LINUX 8 build/oraclelinux_8_9.5: $(call build_rpm,oraclelinux,8,,9.5,9.5.25) touch build/oraclelinux_8_9.5 build/oraclelinux_8_9.6: - $(call build_rpm,oraclelinux,8,,9.6,9.6.21) + $(call build_rpm,oraclelinux,8,,9.6,9.6.23) touch build/oraclelinux_8_9.6 build/oraclelinux_8_10: - $(call build_rpm,oraclelinux,8,,10,10.16) + $(call build_rpm,oraclelinux,8,,10,10.18) touch build/oraclelinux_8_10 build/oraclelinux_8_11: - $(call build_rpm,oraclelinux,8,,11,11.11) + $(call build_rpm,oraclelinux,8,,11,11.13) touch build/oraclelinux_8_11 build/oraclelinux_8_12: - $(call build_rpm,oraclelinux,8,,12,12.6) + $(call build_rpm,oraclelinux,8,,12,12.8) touch build/oraclelinux_8_12 build/oraclelinux_8_13: - $(call build_rpm,oraclelinux,8,,13,13.2) + $(call build_rpm,oraclelinux,8,,13,13.4) touch build/oraclelinux_8_13 + +build/oraclelinux_8_14: + $(call build_rpm,oraclelinux,8,,14,14.0) + touch build/oraclelinux_8_14 + diff --git a/packaging/pkg/Makefile.rhel b/packaging/pkg/Makefile.rhel index f266966cf..b604a990d 100644 --- a/packaging/pkg/Makefile.rhel +++ b/packaging/pkg/Makefile.rhel @@ -4,46 +4,54 @@ build/rhel_7_9.5: touch build/rhel_7_9.5 build/rhel_7_9.6: - $(call build_rpm,rhel,7,7Server,9.6,9.6.21) + $(call build_rpm,rhel,7,7Server,9.6,9.6.23) touch build/rhel_7_9.6 build/rhel_7_10: - $(call build_rpm,rhel,7,7Server,10,10.16) + $(call build_rpm,rhel,7,7Server,10,10.18) touch build/rhel_7_10 build/rhel_7_11: - $(call build_rpm,rhel,7,7Server,11,11.11) + $(call build_rpm,rhel,7,7Server,11,11.13) touch build/rhel_7_11 build/rhel_7_12: - $(call build_rpm,rhel,7,7Server,12,12.6) + $(call build_rpm,rhel,7,7Server,12,12.8) touch build/rhel_7_12 build/rhel_7_13: - $(call build_rpm,rhel,7,7Server,13,13.2) + $(call build_rpm,rhel,7,7Server,13,13.4) touch build/rhel_7_13 +build/rhel_7_14: + $(call build_rpm,rhel,7,7Server,14,14.0) + touch build/rhel_7_14 + # RHEL 8 build/rhel_8_9.5: $(call build_rpm,rhel,8,8Server,9.5,9.5.25) touch build/rhel_8_9.5 build/rhel_8_9.6: - $(call build_rpm,rhel,8,8Server,9.6,9.6.21) + $(call build_rpm,rhel,8,8Server,9.6,9.6.23) touch build/rhel_8_9.6 build/rhel_8_10: - $(call build_rpm,rhel,8,8Server,10,10.16) + $(call build_rpm,rhel,8,8Server,10,10.18) touch build/rhel_8_10 build/rhel_8_11: - $(call build_rpm,rhel,8,8Server,11,11.11) + $(call build_rpm,rhel,8,8Server,11,11.13) touch build/rhel_8_11 build/rhel_8_12: - $(call build_rpm,rhel,8,8Server,12,12.6) + $(call build_rpm,rhel,8,8Server,12,12.8) touch build/rhel_8_12 build/rhel_8_13: - $(call build_rpm,rhel,8,8Server,13,13.2) + $(call build_rpm,rhel,8,8Server,13,13.4) touch build/rhel_8_13 + +build/rhel_8_14: + $(call build_rpm,rhel,8,8Server,14,14.0) + touch build/rhel_8_14 diff --git a/packaging/pkg/Makefile.suse b/packaging/pkg/Makefile.suse index a9f1eaa36..5af22c5d0 100644 --- a/packaging/pkg/Makefile.suse +++ b/packaging/pkg/Makefile.suse @@ -4,46 +4,54 @@ build/suse_15.1_9.5: touch build/suse_15.1_9.5 build/suse_15.1_9.6: - $(call build_suse,suse,15.1,,9.6,9.6.21) + $(call build_suse,suse,15.1,,9.6,9.6.23) touch build/suse_15.1_9.6 build/suse_15.1_10: - $(call build_suse,suse,15.1,,10,10.16) + $(call build_suse,suse,15.1,,10,10.18) touch build/suse_15.1_10 build/suse_15.1_11: - $(call build_suse,suse,15.1,,11,11.11) + $(call build_suse,suse,15.1,,11,11.13) touch build/suse_15.1_11 build/suse_15.1_12: - $(call build_suse,suse,15.1,,12,12.6) + $(call build_suse,suse,15.1,,12,12.8) touch build/suse_15.1_12 build/suse_15.1_13: - $(call build_suse,suse,15.1,,13,13.2) + $(call build_suse,suse,15.1,,13,13.4) touch build/suse_15.1_13 +build/suse_15.1_14: + $(call build_suse,suse,15.1,,14,14.0) + touch build/suse_15.1_14 + # Suse 15.2 build/suse_15.2_9.5: $(call build_suse,suse,15.2,,9.5,9.5.25) touch build/suse_15.2_9.5 build/suse_15.2_9.6: - $(call build_suse,suse,15.2,,9.6,9.6.21) + $(call build_suse,suse,15.2,,9.6,9.6.23) touch build/suse_15.2_9.6 build/suse_15.2_10: - $(call build_suse,suse,15.2,,10,10.16) + $(call build_suse,suse,15.2,,10,10.18) touch build/suse_15.2_10 build/suse_15.2_11: - $(call build_suse,suse,15.2,,11,11.11) + $(call build_suse,suse,15.2,,11,11.13) touch build/suse_15.2_11 build/suse_15.2_12: - $(call build_suse,suse,15.2,,12,12.6) + $(call build_suse,suse,15.2,,12,12.8) touch build/suse_15.2_12 build/suse_15.2_13: - $(call build_suse,suse,15.2,,13,13.2) + $(call build_suse,suse,15.2,,13,13.4) touch build/suse_15.2_13 + +build/suse_15.2_14: + $(call build_suse,suse,15.2,,14,14.0) + touch build/suse_15.2_14 diff --git a/packaging/pkg/Makefile.ubuntu b/packaging/pkg/Makefile.ubuntu index 3f76de516..88803c64f 100644 --- a/packaging/pkg/Makefile.ubuntu +++ b/packaging/pkg/Makefile.ubuntu @@ -4,96 +4,113 @@ build/ubuntu_20.04_9.5: touch build/ubuntu_20.04_9.5 build/ubuntu_20.04_9.6: - $(call build_deb,ubuntu,20.04,focal,9.6,9.6.21) + $(call build_deb,ubuntu,20.04,focal,9.6,9.6.23) touch build/ubuntu_20.04_9.6 build/ubuntu_20.04_10: - $(call build_deb,ubuntu,20.04,focal,10,10.16) + $(call build_deb,ubuntu,20.04,focal,10,10.18) touch build/ubuntu_20.04_10 build/ubuntu_20.04_11: - $(call build_deb,ubuntu,20.04,focal,11,11.11) + $(call build_deb,ubuntu,20.04,focal,11,11.13) touch build/ubuntu_20.04_11 build/ubuntu_20.04_12: - $(call build_deb,ubuntu,20.04,focal,12,12.6) + $(call build_deb,ubuntu,20.04,focal,12,12.8) touch build/ubuntu_20.04_12 build/ubuntu_20.04_13: - $(call build_deb,ubuntu,20.04,focal,13,13.2) + $(call build_deb,ubuntu,20.04,focal,13,13.4) touch build/ubuntu_20.04_13 +build/ubuntu_20.04_14: + $(call build_deb,ubuntu,20.04,focal,14,14.0) + touch build/ubuntu_20.04_14 + # UBUNTU 18.04 build/ubuntu_18.04_9.5: $(call build_deb,ubuntu,18.04,bionic,9.5,9.5.25) touch build/ubuntu_18.04_9.5 build/ubuntu_18.04_9.6: - $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.21) + $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.23) touch build/ubuntu_18.04_9.6 build/ubuntu_18.04_10: - $(call build_deb,ubuntu,18.04,bionic,10,10.16) + $(call build_deb,ubuntu,18.04,bionic,10,10.18) touch build/ubuntu_18.04_10 build/ubuntu_18.04_11: - $(call build_deb,ubuntu,18.04,bionic,11,11.11) + $(call build_deb,ubuntu,18.04,bionic,11,11.13) touch build/ubuntu_18.04_11 build/ubuntu_18.04_12: - $(call build_deb,ubuntu,18.04,bionic,12,12.6) + $(call build_deb,ubuntu,18.04,bionic,12,12.8) touch build/ubuntu_18.04_12 build/ubuntu_18.04_13: - $(call build_deb,ubuntu,18.04,bionic,13,13.2) + $(call build_deb,ubuntu,18.04,bionic,13,13.4) touch build/ubuntu_18.04_13 +build/ubuntu_18.04_14: + $(call build_deb,ubuntu,18.04,bionic,14,14.0) + touch build/ubuntu_18.04_14 + # UBUNTU 16.04 build/ubuntu_16.04_9.5: $(call build_deb,ubuntu,16.04,xenial,9.5,9.5.25) touch build/ubuntu_16.04_9.5 build/ubuntu_16.04_9.6: - $(call build_deb,ubuntu,16.04,xenial,9.6,9.6.21) + $(call build_deb,ubuntu,16.04,xenial,9.6,9.6.23) touch build/ubuntu_16.04_9.6 build/ubuntu_16.04_10: - $(call build_deb,ubuntu,16.04,xenial,10,10.16) + $(call build_deb,ubuntu,16.04,xenial,10,10.18) touch build/ubuntu_16.04_10 build/ubuntu_16.04_11: - $(call build_deb,ubuntu,16.04,xenial,11,11.11) + $(call build_deb,ubuntu,16.04,xenial,11,11.13) touch build/ubuntu_16.04_11 build/ubuntu_16.04_12: - $(call build_deb,ubuntu,16.04,xenial,12,12.6) + $(call build_deb,ubuntu,16.04,xenial,12,12.8) touch build/ubuntu_16.04_12 build/ubuntu_16.04_13: - $(call build_deb,ubuntu,16.04,xenial,13,13.2) + $(call build_deb,ubuntu,16.04,xenial,13,13.4) touch build/ubuntu_16.04_13 +build/ubuntu_16.04_14: + $(call build_deb,ubuntu,16.04,xenial,14,14.0) + touch build/ubuntu_16.04_14 + + # UBUNTU 14.04 build/ubuntu_14.04_9.5: $(call build_deb,ubuntu,14.04,trusty,9.5,9.5.25) touch build/ubuntu_14.04_9.5 build/ubuntu_14.04_9.6: - $(call build_deb,ubuntu,14.04,trusty,9.6,9.6.21) + $(call build_deb,ubuntu,14.04,trusty,9.6,9.6.23) touch build/ubuntu_14.04_9.6 build/ubuntu_14.04_10: - $(call build_deb,ubuntu,14.04,trusty,10,10.16) + $(call build_deb,ubuntu,14.04,trusty,10,10.18) touch build/ubuntu_14.04_10 build/ubuntu_14.04_11: - $(call build_deb,ubuntu,14.04,trusty,11,11.11) + $(call build_deb,ubuntu,14.04,trusty,11,11.13) touch build/ubuntu_14.04_11 build/ubuntu_14.04_12: - $(call build_deb,ubuntu,14.04,trusty,12,12.6) + $(call build_deb,ubuntu,14.04,trusty,12,12.8) touch build/ubuntu_14.04_12 build/ubuntu_14.04_13: - $(call build_deb,ubuntu,14.04,trusty,13,13.2) + $(call build_deb,ubuntu,14.04,trusty,13,13.4) touch build/ubuntu_14.04_13 + +build/ubuntu_14.04_14: + $(call build_deb,ubuntu,14.04,trusty,14,14.0) + touch build/ubuntu_14.04_14 diff --git a/packaging/pkg/scripts/deb.sh b/packaging/pkg/scripts/deb.sh index 2fe2018b6..6e134635a 100755 --- a/packaging/pkg/scripts/deb.sh +++ b/packaging/pkg/scripts/deb.sh @@ -11,20 +11,20 @@ set -o pipefail # fix https://github.com/moby/moby/issues/23137 ulimit -n 1024 -# THere is no std/ent packages for PG 9.5 +# There is no std/ent packages for PG 9.5 if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then exit 0 fi # PACKAGES NEEDED -apt-get update -y && apt-get install -y git wget bzip2 devscripts equivs +apt-get --allow-releaseinfo-change update -y && apt-get install -y git wget bzip2 devscripts equivs # Prepare export DEBIAN_FRONTEND=noninteractive echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections if [ ${CODENAME} == 'jessie' ]; then -printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list + printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list fi apt-get -qq update -y diff --git a/packaging/pkg/scripts/rpm.sh b/packaging/pkg/scripts/rpm.sh index fc95bf7dd..ffd681b75 100755 --- a/packaging/pkg/scripts/rpm.sh +++ b/packaging/pkg/scripts/rpm.sh @@ -23,6 +23,11 @@ if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then exit 0 fi +if [ -f /etc/centos-release ] ; then + sed -i 's|^baseurl=http://|baseurl=https://|g' /etc/yum.repos.d/*.repo + yum update -y +fi + # PACKAGES NEEDED yum install -y git wget bzip2 rpm-build @@ -67,11 +72,11 @@ else cd /root/rpmbuild/SOURCES/pgpro PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') - if [[ ${PBK_EDITION} == 'std' ]] ; then - git checkout "PGPRO${PGPRO_TOC}_1" - else - git checkout "PGPROEE${PGPRO_TOC}_1" - fi + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi rm -rf .git cd /root/rpmbuild/SOURCES/ @@ -110,7 +115,7 @@ else sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup-pgpro.spec if [ ${PG_VERSION} != '9.6' ]; then - sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup-pgpro.spec + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup-pgpro.spec fi sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo-forks.spec @@ -145,4 +150,4 @@ else # write artefacts to out directory rm -rf /app/out/* cp -arv /root/rpmbuild/RPMS /app/out -fi +fi From bd81f7fc12df7b0f45c5ce9105495c74f7da582f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Wed, 13 Oct 2021 21:18:03 +0300 Subject: [PATCH 1789/2107] =?UTF-8?q?[PGPRO-5673]=20add=20missing=20grants?= =?UTF-8?q?=20(caused=20by=20CVE-2018-1058=20fixes=20#415=20P=E2=80=A6=20(?= =?UTF-8?q?#441)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [PGPRO-5673] add missing grants (caused by CVE-2018-1058 fixes #415 PGPRO-5315) * tests.backup.BackupTest.test_missing_replication_permission_1: fix test for changed 14s output * tests.backup.BackupTest.test_missing_replication_permission: fix test for 9.5 * tests.checkdb.CheckdbTest.test_checkdb_with_least_privileges: remove grant for nonexistent (in 10) bt_index_check(regclass, bool) * tests.checkdb.CheckdbTest.test_checkdb_with_least_privileges: remove grant for nonexistent (in 9.5) pg_catalog.pg_control_system() * tests.checkdb.CheckdbTest.test_checkdb_with_least_privileges: remove grant for nonexistent (in amcheck_next) bt_index_check(regclass) * adapt tests/restore.py to Python-3.5 (used in travis tests) * skip issue_313 test --- doc/pgprobackup.xml | 4 ++++ tests/backup.py | 40 ++++++++++++++++++++++++---------------- tests/checkdb.py | 35 +++++++++++++++++++++++++++++++---- tests/ptrack.py | 6 ++++++ tests/restore.py | 18 +++++++++++++----- 5 files changed, 78 insertions(+), 25 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 740517313..6a634ea05 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -606,6 +606,7 @@ BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; @@ -624,6 +625,7 @@ BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; @@ -644,6 +646,7 @@ BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; @@ -5531,6 +5534,7 @@ BEGIN; CREATE ROLE backup WITH LOGIN REPLICATION; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; diff --git a/tests/backup.py b/tests/backup.py index 60e70cc28..a2a242534 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2020,10 +2020,12 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " @@ -2053,10 +2055,12 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -2091,8 +2095,10 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -3249,10 +3255,7 @@ def test_missing_replication_permission(self): if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - - node.safe_psql( - "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") sleep(2) @@ -3270,9 +3273,11 @@ def test_missing_replication_permission(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "FATAL: must be superuser or replication role to start walsender", + # 9.5: ERROR: must be superuser or replication role to run a backup + # >=9.6: FATAL: must be superuser or replication role to start walsender + self.assertRegex( e.message, + "ERROR: must be superuser or replication role to run a backup|FATAL: must be superuser or replication role to start walsender", "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -3330,7 +3335,8 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3353,7 +3359,8 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) # >= 10 else: node.safe_psql( @@ -3381,10 +3388,7 @@ def test_missing_replication_permission_1(self): if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - - node.safe_psql( - "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") replica.promote() @@ -3398,10 +3402,14 @@ def test_missing_replication_permission_1(self): self.assertIn( 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', output) - - self.assertIn( - 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender', - output) + + # Messages before 14 + # 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender' + # Messages for >=14 + # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb: (connection to server on socket "/tmp/.s.PGSQL.\d+" failed: ){0,1}FATAL: must be superuser or replication role to start walsender') # Clean after yourself self.del_test_dir(module_name, fname) diff --git a/tests/checkdb.py b/tests/checkdb.py index aecd4bde1..fcc40b2bf 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -562,15 +562,14 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' # amcheck-next function ) # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: @@ -588,6 +587,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' @@ -598,7 +598,33 @@ def test_checkdb_with_least_privileges(self): # 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' ) - # >= 10 + # PG 10 + elif self.get_version(node) > 100000 and self.get_version(node) < 110000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;' + ) + # >= 11 else: node.safe_psql( 'backupdb', @@ -614,6 +640,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' diff --git a/tests/ptrack.py b/tests/ptrack.py index aa0bbadc1..7a1090a81 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -402,10 +402,12 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " @@ -434,10 +436,12 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -470,8 +474,10 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " diff --git a/tests/restore.py b/tests/restore.py index d0353d05f..47419e5a9 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -4,7 +4,7 @@ import subprocess import sys from time import sleep -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import hashlib import shutil import json @@ -2140,7 +2140,8 @@ def test_restore_target_new_options(self): target_name = 'savepoint' - target_time = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z") + # in python-3.6+ it can be ...now()..astimezone()... + target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %z") with node.connect("postgres") as con: res = con.execute( "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") @@ -2503,7 +2504,7 @@ def test_partial_restore_exclude(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -2742,7 +2743,7 @@ def test_partial_restore_include(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -3222,10 +3223,12 @@ def test_missing_database_map(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " @@ -3255,10 +3258,12 @@ def test_missing_database_map(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -3292,8 +3297,10 @@ def test_missing_database_map(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -3868,7 +3875,8 @@ def test_concurrent_restore(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 + @unittest.skip("skip") def test_restore_issue_313(self): """ Check that partially restored PostgreSQL instance cannot be started From 76acd88e1555b0f9ab360ef5021bfa074a5f01d1 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 14:40:13 +0300 Subject: [PATCH 1790/2107] Stabilize tests.catchup.CatchupTest.test_tli_ptrack_catchup --- tests/catchup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/catchup.py b/tests/catchup.py index 45d999629..79ebdec9f 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -357,6 +357,11 @@ def test_tli_ptrack_catchup(self): self.set_replica(dst_pg, src_pg) # fake replication src_pg.slow_start(replica = True) src_pg.promote() + + src_pg.safe_psql("postgres", "CHECKPOINT") # force postgres to update tli in 'SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()' + src_tli = src_pg.safe_psql("postgres", "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()").decode('utf-8').rstrip() + self.assertEqual(src_tli, "2", "Postgres didn't update TLI after promote") + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") From de497aad52697ad3746bd2471153aefca1e86b87 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 15:11:08 +0300 Subject: [PATCH 1791/2107] Stabilize tests.ptrack.PtrackTest.test_horizon_lsn_ptrack --- tests/helpers/ptrack_helpers.py | 7 +++++++ tests/ptrack.py | 14 ++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 90cfb7be0..6db6fa04d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1602,6 +1602,13 @@ def get_version(self, node): return self.version_to_num( testgres.get_pg_config()['VERSION'].split(" ")[1]) + def get_ptrack_version(self, node): + version = node.safe_psql( + "postgres", + "SELECT extversion " + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() + return self.version_to_num(version) + def get_bin_path(self, binary): return testgres.get_bin_path(binary) diff --git a/tests/ptrack.py b/tests/ptrack.py index a20be54b8..3dfbea0a0 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4465,16 +4465,10 @@ def test_horizon_lsn_ptrack(self): "postgres", "CREATE EXTENSION ptrack") - # TODO: ptrack version must be 2.1 - ptrack_version = node.safe_psql( - "postgres", - "SELECT extversion " - "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() - - self.assertEqual( - ptrack_version, - "2.1", - "You need ptrack 2.1 for this test") + self.assertGreaterEqual( + self.get_ptrack_version(node), + self.version_to_num("2.1"), + "You need ptrack >=2.1 for this test") # set map_size to a minimal value self.set_auto_conf(node, {'ptrack.map_size': '1'}) From da5eb961c4e4c37ea82a4a018abd41050ba5ca49 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 16:06:10 +0300 Subject: [PATCH 1792/2107] Stabilize tests.backup.BackupTest.test_backup_modes_archive --- tests/backup.py | 8 ++++---- tests/helpers/ptrack_helpers.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 3548fa56b..d59445337 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -57,16 +57,16 @@ def test_backup_modes_archive(self): backup_dir, 'node', node, backup_type="page") show_backup_1 = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PAGE") + self.assertEqual(show_backup_1['status'], "OK") + self.assertEqual(show_backup_1['backup-mode'], "PAGE") # delta backup mode delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type="delta") show_backup_2 = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "DELTA") + self.assertEqual(show_backup_2['status'], "OK") + self.assertEqual(show_backup_2['backup-mode'], "DELTA") # Check parent backup self.assertEqual( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6db6fa04d..38977f108 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -980,6 +980,12 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(node.logs_dir) + ] + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( From 21abadfff731f95c720c9dd217d8eb0a8cd880ce Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 16:23:49 +0300 Subject: [PATCH 1793/2107] Fix broken in f26c95964 tests.config.ConfigTest.test_corrupt_backup_content --- src/catalog.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 9775968b8..a4af1d2a3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1144,7 +1144,9 @@ get_backup_filelist(pgBackup *backup, bool strict) { elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", backup_filelist_path, content_crc, backup->content_crc); - return NULL; + parray_free(files); + files = NULL; + } /* redundant sanity? */ From 80e88588c7c798b6bca8b284bd3d1c09a5828993 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 16:52:25 +0300 Subject: [PATCH 1794/2107] Fix broken in da5eb96 tests.validate.ValidateTest.test_basic_validate_nullified_heap_page_backup --- tests/helpers/ptrack_helpers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 38977f108..6db6fa04d 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -980,12 +980,6 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] - if self.verbose: - cmd_list += [ - '--log-level-file=VERBOSE', - '--log-directory={0}'.format(node.logs_dir) - ] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( From 01db7adecaf72c06f73c679133fdf1c682140b8d Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 17:27:57 +0300 Subject: [PATCH 1795/2107] test_ptrack_vacuum_full name was duplicated, rename one of them --- tests/ptrack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 3dfbea0a0..5282649ce 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3615,7 +3615,7 @@ def test_ptrack_vacuum_bits_visibility(self): # @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_vacuum_full(self): + def test_ptrack_vacuum_full_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, From c83a8d4ab97f6e57054c31ec5836d03c65522b1c Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 18:10:48 +0300 Subject: [PATCH 1796/2107] Stabilize tests.ptrack.PtrackTest.test_ptrack_vacuum_full_2 --- tests/ptrack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 5282649ce..71a1ddfd9 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3619,7 +3619,8 @@ def test_ptrack_vacuum_full_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, - ptrack_enable=True) + ptrack_enable=True, + pg_options={ 'wal_log_hints': 'on' }) backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') self.init_pb(backup_dir) From bd79fbbbbb2d0867e9ffe9b9fbf50da2e6d1ee3a Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 18:37:05 +0300 Subject: [PATCH 1797/2107] test_ptrack_vacuum_truncate name was duplicated, rename one of them --- tests/ptrack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index 71a1ddfd9..e482d8f41 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3794,7 +3794,7 @@ def test_ptrack_vacuum_full_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_vacuum_truncate(self): + def test_ptrack_vacuum_truncate_2(self): node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), set_replication=True, From 0545dd4a9257702944efd18b55a31fa8cfd940e7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 15 Oct 2021 12:53:07 +0300 Subject: [PATCH 1798/2107] Running tests.ptrack.PtrackTest.test_horizon_lsn_ptrack now depends on PGPROBACKUPBIN_OLD --- tests/ptrack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index e482d8f41..e8c291a79 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4441,11 +4441,14 @@ def test_corrupt_ptrack_map(self): # Clean after yourself self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") + # @unittest.skip("skip") def test_horizon_lsn_ptrack(self): """ https://github.com/postgrespro/pg_probackup/pull/386 """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") self.assertLessEqual( self.version_to_num(self.old_probackup_version), self.version_to_num('2.4.15'), From d9ba5d05ea0dba342b88ab1fdc139edc2fb8aafe Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 18:25:43 +0300 Subject: [PATCH 1799/2107] Change README for upcoming 2.5.2 release --- README.md | 70 +++++++++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b7e170cf5..95581ecfc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.5, 9.6, 10, 11, 12, 13, 14; +* PostgreSQL 9.6, 10, 11, 12, 13, 14; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -60,7 +60,7 @@ Documentation can be found at [github](https://postgrespro.github.io/pg_probacku * Stable version state can be found under the respective [release tag](https://github.com/postgrespro/pg_probackup/releases). * `master` branch contains minor fixes that are planned to the nearest minor release. -* Upcoming major release is developed in a release branch i.e. `release_2_5`. +* Upcoming major release is developed in a release branch i.e. `release_2_6`. For detailed release plans check [Milestones](https://github.com/postgrespro/pg_probackup/milestones) @@ -74,57 +74,57 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5} -sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}-dbg +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{13,12,11,10,9.6,9.5} +sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{13,12,11,10,9.6,9.5} -yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{13,12,11,10,9.6,9.5} -yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{13,12,11,10,9.6,9.5} -yum install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{13,12,11,10,9.6,9.5} +yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{13,12,11,10,9.6,9.5} -zypper install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} +zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{13,12,11,10,9.6,9.5} +zypper si pg_probackup-{14,13,12,11,10,9.6} #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise @@ -132,40 +132,40 @@ sudo apt-get install pg_probackup-{13,12,11,10,9.6,9.5}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6}-dbg #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From eb7eb165810fe5fab0ad208168043dcd697eebad Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 18:31:14 +0300 Subject: [PATCH 1800/2107] PTRACK now provides a separate patch for version 14 (see 3d6ccc6 ptrack commit), update travis testing configuration to use this standalone patch --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 873dd8f20..70a906e57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,8 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_VERSION=13 - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_VERSION=13 + - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_VERSION=master + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_VERSION=14 - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_VERSION=12 - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_VERSION=11 From e22cb930341d8fb83ce7d919e8bbb712ddb6ee85 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 23:08:04 +0300 Subject: [PATCH 1801/2107] Follow-up to 7ca590c6c, fix tests.backup.BackupTest.test_backup_modes_archive --- tests/backup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index d59445337..a68dd48d3 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -21,8 +21,7 @@ def test_backup_modes_archive(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 034c597cf8e4ebdd765495aeabaa9da5ffdb40d6 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 23:11:49 +0300 Subject: [PATCH 1802/2107] Follow-up to 7ca590c6c, fix tests.backup.BackupTest.test_incremental_backup_without_full --- tests/backup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index a68dd48d3..3c5f81c91 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -112,8 +112,7 @@ def test_incremental_backup_without_full(self): fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) + initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) From 807df12d3dc816927673eeedfcd3d0f846d0ac42 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 23:30:27 +0300 Subject: [PATCH 1803/2107] =?UTF-8?q?travis:=20more=20universal=20indicati?= =?UTF-8?q?on=20of=20the=20name=20of=20the=20ptra=D1=81k=20patch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 35 ++++++++++++++++++----------------- travis/Dockerfile.in | 2 +- travis/make_dockerfile.sh | 8 ++++---- travis/run_tests.sh | 4 ++-- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70a906e57..4e86b2e29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,30 +26,31 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_VERSION=master - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_VERSION=14 - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_VERSION=12 - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_VERSION=11 + - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=archive -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=backup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=catchup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=compression -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=delta -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=locking -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=merge -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=page -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=ptrack -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=replica -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=retention -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=restore +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=archive +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=compression +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=delta +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=locking +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=merge +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=page +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=ptrack +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=retention +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=restore jobs: allow_failures: - if: env(PG_BRANCH) = master + - if: env(PG_BRANCH) = 9.5 # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in index e6bbedb61..a67663d3b 100644 --- a/travis/Dockerfile.in +++ b/travis/Dockerfile.in @@ -10,7 +10,7 @@ RUN python3 -m pip install virtualenv # Environment ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} -ENV PTRACK_PATCH_PG_VERSION=${PTRACK_PATCH_PG_VERSION} +ENV PTRACK_PATCH_PG_BRANCH=${PTRACK_PATCH_PG_BRANCH} ENV PGPROBACKUP_GDB=${PGPROBACKUP_GDB} ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh index fc2742cdb..119125ced 100755 --- a/travis/make_dockerfile.sh +++ b/travis/make_dockerfile.sh @@ -14,8 +14,8 @@ if [ -z ${MODE+x} ]; then MODE=basic fi -if [ -z ${PTRACK_PATCH_PG_VERSION+x} ]; then - PTRACK_PATCH_PG_VERSION=off +if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then + PTRACK_PATCH_PG_BRANCH=off fi if [ -z ${PGPROBACKUP_GDB+x} ]; then @@ -25,13 +25,13 @@ fi echo PG_VERSION=${PG_VERSION} echo PG_BRANCH=${PG_BRANCH} echo MODE=${MODE} -echo PTRACK_PATCH_PG_VERSION=${PTRACK_PATCH_PG_VERSION} +echo PTRACK_PATCH_PG_BRANCH=${PTRACK_PATCH_PG_BRANCH} echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} sed \ -e 's/${PG_VERSION}/'${PG_VERSION}/g \ -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ -e 's/${MODE}/'${MODE}/g \ - -e 's/${PTRACK_PATCH_PG_VERSION}/'${PTRACK_PATCH_PG_VERSION}/g \ + -e 's/${PTRACK_PATCH_PG_BRANCH}/'${PTRACK_PATCH_PG_BRANCH}/g \ -e 's/${PGPROBACKUP_GDB}/'${PGPROBACKUP_GDB}/g \ Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 4a64fed80..44815407e 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -33,7 +33,7 @@ echo "############### Getting Postgres sources:" git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 # Clone ptrack -if [ "$PTRACK_PATCH_PG_VERSION" != "off" ]; then +if [ "$PTRACK_PATCH_PG_BRANCH" != "off" ]; then git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 export PG_PROBACKUP_PTRACK=on else @@ -45,7 +45,7 @@ fi echo "############### Compiling Postgres:" cd postgres # Go to postgres dir if [ "$PG_PROBACKUP_PTRACK" = "on" ]; then - git apply -3 ../ptrack/patches/REL_${PTRACK_PATCH_PG_VERSION}_STABLE-ptrack-core.diff + git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests make -s -j$(nproc) install From 9ba98ad2ef7437a5c68e3f78695d512fba9d86e4 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 16 Oct 2021 23:33:16 +0300 Subject: [PATCH 1804/2107] travis: make 9.5 test allowed to fail --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4e86b2e29..876289e82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master - - if: env(PG_BRANCH) = 9.5 + - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage From 0c3aff742991589e2fd6b439eb84dd07af129a4b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sun, 17 Oct 2021 00:07:05 +0300 Subject: [PATCH 1805/2107] Fix tests.pgpro2068.BugTest.test_minrecpoint_on_replica on 9.6 --- tests/pgpro2068.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index fc0cb50bd..a80d317d4 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -144,7 +144,7 @@ def test_minrecpoint_on_replica(self): DO $$ relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("SELECT min_recovery_end_lsn as lsn FROM pg_control_recovery()")[0]['lsn'] +current_xlog_lsn = plpy.execute("SELECT min_recovery_end_location as lsn FROM pg_control_recovery()")[0]['lsn'] plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) found_corruption = False for relation in relations: @@ -158,7 +158,7 @@ def test_minrecpoint_on_replica(self): found_corruption = True if found_corruption: plpy.error('Found Corruption') -$$ LANGUAGE plpythonu; +$$ LANGUAGE plpython3u; ''' else: script = ''' From fc752c89026e9f692b728b8d8c3454a25d14c8fa Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 18 Oct 2021 05:51:12 +0300 Subject: [PATCH 1806/2107] Stabilize tests.checkdb.CheckdbTest.test_checkdb_with_least_privileges: accounting differences in amcheck versions in various editions of PG-10 (bd81f7f follow-up) --- tests/checkdb.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index fcc40b2bf..044c057f6 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -621,9 +621,18 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup;' ) + if ProbackupTest.enterprise: + # amcheck-1.1 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') + else: + # amcheck-1.0 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') # >= 11 else: node.safe_psql( From 49caaa3b570bd620101e96c5833210d9c3598a6e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 18 Oct 2021 06:00:41 +0300 Subject: [PATCH 1807/2107] [skip travis] README.md update: fix info about supported versions of PGPRO with ptrack --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95581ecfc..86d192ecb 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL 11, 12, 13, 14 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 9.6, 10, 11, 12, 13 -* Postgres Pro Enterprise 9.6, 10, 11, 12, 13 +* Postgres Pro Standard 11, 12, 13 +* Postgres Pro Enterprise 11, 12, 13 ## Limitations From 3cd69fb9038b51f029676a72b8a8092d45216cfa Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 18 Oct 2021 17:20:35 +0300 Subject: [PATCH 1808/2107] relaxation of the requirement for calling the old pg_catalog.ptrack_version() function --- src/ptrack.c | 10 +++++++--- tests/helpers/ptrack_helpers.py | 9 ++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/ptrack.c b/src/ptrack.c index 3f395b286..ebcba1dd4 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -79,13 +79,17 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) return; } - res_db = pgut_execute(backup_conn, + /* + * it's ok not to have permission to call this old function in PGPRO-11 version (ok_error = true) + * see deprication notice https://postgrespro.com/docs/postgrespro/11/release-pro-11-9-1 + */ + res_db = pgut_execute_extended(backup_conn, "SELECT pg_catalog.ptrack_version()", - 0, NULL); + 0, NULL, true, true); if (PQntuples(res_db) == 0) { - /* TODO: Something went wrong, should we error out here? */ PQclear(res_db); + elog(WARNING, "Can't call pg_catalog.ptrack_version(), it is assumed that there is no ptrack extension installed."); return; } ptrack_version_str = PQgetvalue(res_db, 0, 0); diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6db6fa04d..5c0ce19bc 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -312,16 +312,11 @@ def __init__(self, *args, **kwargs): self.ptrack = False if 'PG_PROBACKUP_PTRACK' in self.test_env: if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': - self.ptrack = True + if self.pg_config_version >= self.version_to_num('11.0'): + self.ptrack = True os.environ["PGAPPNAME"] = "pg_probackup" - if self.ptrack: - self.assertGreaterEqual( - self.pg_config_version, - self.version_to_num('11.0'), - "ptrack testing require PostgreSQL >= 11") - @property def pg_config_version(self): return self.version_to_num( From a4308f07742a6b252ad56a81b1c13c41b220633e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 18 Oct 2021 17:40:43 +0300 Subject: [PATCH 1809/2107] Fix tests.backup.BackupTest.test_missing_replication_permission_1: make it compatible with PGPROBACKUP_SSH_REMOTE=ON on PG14 (follow bd81f7fc1) --- tests/backup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/backup.py b/tests/backup.py index b444c0fd9..b14f5fe98 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3234,9 +3234,11 @@ def test_missing_replication_permission_1(self): # 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender' # Messages for >=14 # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' + # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' self.assertRegex( output, - r'WARNING: could not connect to database backupdb: (connection to server on socket "/tmp/.s.PGSQL.\d+" failed: ){0,1}FATAL: must be superuser or replication role to start walsender') + r'WARNING: could not connect to database backupdb: (connection to server (on socket "/tmp/.s.PGSQL.\d+"|at "localhost" \(127.0.0.1\), port \d+) failed: ){0,1}' + 'FATAL: must be superuser or replication role to start walsender') # Clean after yourself self.del_test_dir(module_name, fname) From fd4b75ababb4152bfe53a8c1c2d9081304198915 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 18 Oct 2021 21:18:34 +0300 Subject: [PATCH 1810/2107] Adapt tests.replica.ReplicaTest.test_archive_replica_not_null_offset to pgpro enterprise edition --- tests/replica.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/replica.py b/tests/replica.py index fe33e9ac2..8fb89c222 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -980,15 +980,17 @@ def test_archive_replica_not_null_offset(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'LOG: Looking for LSN 0/4000060 in segment: 000000010000000000000004', + # vanilla -- 0/4000060 + # pgproee -- 0/4000078 + self.assertRegex( e.message, + r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Wait for LSN 0/4000060 in archived WAL segment', + self.assertRegex( e.message, + r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) From 73496c412496965bc18307cf7e93a6bf4fc1245a Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 14 Oct 2021 22:25:38 +0300 Subject: [PATCH 1811/2107] Fix broken tests.validate.ValidateTest.test_validate_instance_with_several_corrupt_backups_interrupt +fix small typo introduced in 02a3665 --- src/catalog.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 3ba17e9fd..54709f9c5 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -32,6 +32,8 @@ static int grab_excl_lock_file(const char *backup_dir, const char *backup_id, bo static int grab_shared_lock_file(pgBackup *backup); static int wait_shared_owners(pgBackup *backup); + +static void unlink_lock_atexit(bool fatal, void *userdata); static void unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive); static void release_excl_lock_file(const char *backup_dir); static void release_shared_lock_file(const char *backup_dir); @@ -83,8 +85,8 @@ timelineInfoFree(void *tliInfo) } /* Iterate over locked backups and unlock them */ -static void -unlink_lock_atexit(void) +void +unlink_lock_atexit(bool unused_fatal, void *unused_userdata) { int i; @@ -94,7 +96,7 @@ unlink_lock_atexit(void) for (i = 0; i < parray_num(locks); i++) { LockInfo *lock = (LockInfo *) parray_get(locks, i); - unlock_backup(lock->backup_dir, lock->backup_dir, lock->exclusive); + unlock_backup(lock->backup_dir, lock->backup_id, lock->exclusive); } parray_walk(locks, pg_free); @@ -267,7 +269,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) */ if (!backup_lock_exit_hook_registered) { - atexit(unlink_lock_atexit); + pgut_atexit_push(unlink_lock_atexit, NULL); backup_lock_exit_hook_registered = true; } From 66dd4b26e45696d8f0ffe07e20b19f23eede2453 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sun, 17 Oct 2021 19:37:04 +0300 Subject: [PATCH 1812/2107] [PGPRO-5705] remove snapfs (found in commits 9bf541b85, 8b8337047, 5c247d0ff) --- src/dir.c | 7 ------ tests/__init__.py | 3 +-- tests/snapfs.py | 61 ----------------------------------------------- 3 files changed, 1 insertion(+), 70 deletions(-) delete mode 100644 tests/snapfs.py diff --git a/src/dir.c b/src/dir.c index 00a4c4f82..eea7395c3 100644 --- a/src/dir.c +++ b/src/dir.c @@ -762,13 +762,6 @@ dir_check_file(pgFile *file, bool backup_logs) } else { - /* - * snapfs files: - * RELFILENODE.BLOCKNO.snapmap.SNAPID - * RELFILENODE.BLOCKNO.snap.SNAPID - */ - if (strstr(file->name, "snap") != NULL) - return true; len = strlen(file->name); /* reloid.cfm */ diff --git a/tests/__init__.py b/tests/__init__.py index 3a297c45e..732fdb734 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,7 +5,7 @@ backup, delete, delta, restore, validate, \ retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ - cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ + cfs_validate_backup, auth_test, time_stamp, logging, \ locking, remote, external, config, checkdb, set_backup, incr_restore, \ CVE_2018_1058 @@ -53,7 +53,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(retention)) suite.addTests(loader.loadTestsFromModule(set_backup)) suite.addTests(loader.loadTestsFromModule(show)) - suite.addTests(loader.loadTestsFromModule(snapfs)) suite.addTests(loader.loadTestsFromModule(time_stamp)) suite.addTests(loader.loadTestsFromModule(validate)) suite.addTests(loader.loadTestsFromModule(CVE_2018_1058)) diff --git a/tests/snapfs.py b/tests/snapfs.py deleted file mode 100644 index 991741952..000000000 --- a/tests/snapfs.py +++ /dev/null @@ -1,61 +0,0 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -module_name = 'snapfs' - - -class SnapFSTest(ProbackupTest, unittest.TestCase): - - # @unittest.expectedFailure - def test_snapfs_simple(self): - """standart backup modes with ARCHIVE WAL method""" - if not self.enterprise: - self.skipTest('This test must be run on enterprise') - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - 'postgres', - 'select pg_make_snapshot()') - - node.pgbench_init(scale=10) - - pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - 'postgres', - 'select pg_remove_snapshot(1)') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', - node, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) From 196a70bd3214749c3377b5ed10feb68914249d3e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 20 Oct 2021 20:37:44 +0300 Subject: [PATCH 1813/2107] Fixes for ptrack tests (test_ptrack_vacuum, test_ptrack_vacuum_bits_frozen, test_ptrack_vacuum_bits_visibility): this is workaround for spgist metadata update bug (PGPRO-5707) --- tests/helpers/ptrack_helpers.py | 62 +++++++++++++++++++++++---------- tests/ptrack.py | 10 ++++-- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 5c0ce19bc..1b54d3165 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1714,8 +1714,30 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): return directory_dict - def compare_pgdata(self, original_pgdata, restored_pgdata): - """ return dict with directory content. DO IT BEFORE RECOVERY""" + def get_known_bugs_comparision_exclusion_dict(self, node): + """ get dict of known datafiles difference, that can be used in compare_pgdata() """ + comparision_exclusion_dict = dict() + + # bug in spgist metapage update (PGPRO-5707) + spgist_filelist = node.safe_psql( + "postgres", + "SELECT pg_catalog.pg_relation_filepath(pg_class.oid) " + "FROM pg_am, pg_class " + "WHERE pg_am.amname = 'spgist' " + "AND pg_class.relam = pg_am.oid" + ).decode('utf-8').rstrip().splitlines() + for filename in spgist_filelist: + comparision_exclusion_dict[filename] = set([0]) + + return comparision_exclusion_dict + + + def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict()): + """ + return dict with directory content. DO IT BEFORE RECOVERY + exclusion_dict is used for exclude files (and it block_no) from comparision + it is a dict with relative filenames as keys and set of block numbers as values + """ fail = False error_message = 'Restored PGDATA is not equal to original!\n' @@ -1777,16 +1799,17 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): original_pgdata['files'][file]['md5'] != restored_pgdata['files'][file]['md5'] ): - fail = True - error_message += ( - '\nFile Checksumm mismatch.\n' - 'File_old: {0}\nChecksumm_old: {1}\n' - 'File_new: {2}\nChecksumm_new: {3}\n').format( - os.path.join(original_pgdata['pgdata'], file), - original_pgdata['files'][file]['md5'], - os.path.join(restored_pgdata['pgdata'], file), - restored_pgdata['files'][file]['md5'] - ) + if file not in exclusion_dict: + fail = True + error_message += ( + '\nFile Checksum mismatch.\n' + 'File_old: {0}\nChecksum_old: {1}\n' + 'File_new: {2}\nChecksum_new: {3}\n').format( + os.path.join(original_pgdata['pgdata'], file), + original_pgdata['files'][file]['md5'], + os.path.join(restored_pgdata['pgdata'], file), + restored_pgdata['files'][file]['md5'] + ) if original_pgdata['files'][file]['is_datafile']: for page in original_pgdata['files'][file]['md5_per_page']: @@ -1802,13 +1825,16 @@ def compare_pgdata(self, original_pgdata, restored_pgdata): ) continue - if original_pgdata['files'][file][ - 'md5_per_page'][page] != restored_pgdata[ - 'files'][file]['md5_per_page'][page]: + if not (file in exclusion_dict and page in exclusion_dict[file]): + if ( + original_pgdata['files'][file]['md5_per_page'][page] != + restored_pgdata['files'][file]['md5_per_page'][page] + ): + fail = True error_message += ( - '\n Page checksumm mismatch: {0}\n ' - ' PAGE Checksumm_old: {1}\n ' - ' PAGE Checksumm_new: {2}\n ' + '\n Page checksum mismatch: {0}\n ' + ' PAGE Checksum_old: {1}\n ' + ' PAGE Checksum_new: {2}\n ' ' File: {3}\n' ).format( page, diff --git a/tests/ptrack.py b/tests/ptrack.py index 93b9e3cce..a3109da48 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3210,6 +3210,8 @@ def test_ptrack_vacuum(self): idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) + node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') @@ -3253,7 +3255,7 @@ def test_ptrack_vacuum(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # Clean after yourself self.del_test_dir(module_name, self.fname) @@ -3403,6 +3405,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') self.backup_node( @@ -3438,7 +3441,7 @@ def test_ptrack_vacuum_bits_frozen(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # Clean after yourself self.del_test_dir(module_name, self.fname) @@ -3579,6 +3582,7 @@ def test_ptrack_vacuum_bits_visibility(self): i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') self.backup_node( @@ -3614,7 +3618,7 @@ def test_ptrack_vacuum_bits_visibility(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # Clean after yourself self.del_test_dir(module_name, self.fname) From 474a9561ee0273f13d7d3f9e8990ce5d1902dad7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 21 Oct 2021 13:52:23 +0300 Subject: [PATCH 1814/2107] [ci skip] README.md: fix astra linux version list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b66d8556..060883a28 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{13,12,11,10,9.6,9.5}{-dbg,} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm From f3341dab4a825197864a553ac1d7eda44a83f1b6 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 21 Oct 2021 16:03:46 +0300 Subject: [PATCH 1815/2107] [PGPRO-5750] fix windows compilation problem with PG-14 this is caused by upstream commit https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=bed90759fcbcd72d4d06969eebab81e47326f9a2 Reported by Victor Wagner and fixed by Victor Spirin --- src/utils/file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index f86e605cb..810b4b394 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1,8 +1,10 @@ #include #include -#include #include "pg_probackup.h" +/* sys/stat.h must be included after pg_probackup.h (see problems with compilation for windows described in PGPRO-5750) */ +#include + #include "file.h" #include "storage/checksum.h" From d57b5fd6edc9929f4a35379b0aabef16eeebed52 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 21 Oct 2021 17:01:14 +0300 Subject: [PATCH 1816/2107] Version 2.5.2 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index dfa7051a3..6a1feb014 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.1" +#define PROGRAM_VERSION "2.5.2" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 36e5d4c7a..e9d8c0955 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.1 \ No newline at end of file +pg_probackup 2.5.2 \ No newline at end of file From c2e4f00932444492524ce7770880c837eb882c6f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 22 Oct 2021 00:54:41 +0300 Subject: [PATCH 1817/2107] [ci skip] packaging: small fix in packaging instruction --- packaging/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Readme.md b/packaging/Readme.md index c6cbf16b5..749b9fc06 100644 --- a/packaging/Readme.md +++ b/packaging/Readme.md @@ -9,7 +9,7 @@ make pkg To build binaries for PostgresPro Standart or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/tarballs` directory: ``` -cd packaging/tarballs +cd packaging/pkg/tarballs git clone pgpro_repo pgpro tar -cjSf pgpro.tar.bz2 pgpro ``` From 36e6d0f95dde383383e7d8f8c374c2c4b2bee61a Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Fri, 22 Oct 2021 14:56:20 +0300 Subject: [PATCH 1818/2107] fix rhel dockerfiles for std|ent packaging --- packaging/Dockerfiles/Dockerfile-rhel_7 | 2 ++ packaging/Dockerfiles/Dockerfile-rhel_8 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packaging/Dockerfiles/Dockerfile-rhel_7 b/packaging/Dockerfiles/Dockerfile-rhel_7 index 322c44b59..f64819e13 100644 --- a/packaging/Dockerfiles/Dockerfile-rhel_7 +++ b/packaging/Dockerfiles/Dockerfile-rhel_7 @@ -5,3 +5,5 @@ RUN yum install -y tar wget yum-utils RUN yum install -y gcc make perl libicu-devel glibc-devel RUN yum install -y git RUN yum upgrade -y +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/bison-3.0.4-2.el7.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/flex-2.5.37-6.el7.x86_64.rpm diff --git a/packaging/Dockerfiles/Dockerfile-rhel_8 b/packaging/Dockerfiles/Dockerfile-rhel_8 index c8e1e225e..82385785b 100644 --- a/packaging/Dockerfiles/Dockerfile-rhel_8 +++ b/packaging/Dockerfiles/Dockerfile-rhel_8 @@ -3,3 +3,5 @@ RUN yum install -y tar wget rpm-build yum-utils RUN yum install -y gcc make perl libicu-devel glibc-devel RUN yum install -y git RUN yum upgrade -y +RUN yum install -y http://mirror.centos.org/centos/8/AppStream/x86_64/os/Packages/bison-3.0.4-10.el8.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/8/AppStream/x86_64/os/Packages/flex-2.6.1-9.el8.x86_64.rpm From 5b6ca624170e1b7955c293ce0173a812b6402d80 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Tue, 26 Oct 2021 00:06:23 +0300 Subject: [PATCH 1819/2107] [ci skip] packaging "tuning" --- packaging/Makefile.repo | 1 + packaging/Makefile.test | 14 ++++++++++++- .../rpmbuild/SPECS/pg_probackup-pgpro.spec | 2 +- .../SPECS/pg_probackup.alt.forks.spec | 2 +- packaging/test/Makefile.debian | 21 +++++++++++++++++++ packaging/test/scripts/deb.sh | 3 ++- packaging/test/scripts/rpm.sh | 1 + packaging/test/scripts/suse_forks.sh | 0 8 files changed, 40 insertions(+), 4 deletions(-) mode change 100644 => 100755 packaging/test/scripts/suse_forks.sh diff --git a/packaging/Makefile.repo b/packaging/Makefile.repo index 986c827e9..8186bfd80 100644 --- a/packaging/Makefile.repo +++ b/packaging/Makefile.repo @@ -113,6 +113,7 @@ build/repo_suse_15.2: repo_finish: # cd build/data/www/$(PBK_PKG_REPO)/ cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/rpm && sudo ln -nsf $(PBK_VERSION) latest + # following line only for vanilla cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/srpm && sudo ln -nsf $(PBK_VERSION) latest # sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/rpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/rpm/latest diff --git a/packaging/Makefile.test b/packaging/Makefile.test index fbb415c46..21e850ccd 100644 --- a/packaging/Makefile.test +++ b/packaging/Makefile.test @@ -13,7 +13,7 @@ build/test_all: build/test_debian build/test_ubuntu build/test_centos build/test @echo Package testing is done ### DEBIAN -build/test_debian: build/test_debian_9 build/test_debian_10 build/test_debian_11 +build/test_debian: build/test_debian_9 build/test_debian_10 #build/test_debian_11 @echo Debian: done build/test_debian_9: build/test_debian_9_9.6 build/test_debian_9_10 build/test_debian_9_11 build/test_debian_9_12 build/test_debian_9_13 @@ -58,9 +58,13 @@ build/test_centos: build/test_centos_7 build/test_centos_8 @echo Centos: done build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 build/test_centos_7_13 +# pgpro +#build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 @echo Centos 7: done build/test_centos_8: build/test_centos_8_9.6 build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 build/test_centos_8_13 +# pgpro +#build/test_centos_8: build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 @echo Centos 8: done # Oracle Linux @@ -68,9 +72,13 @@ build/test_oraclelinux: build/test_oraclelinux_7 build/test_oraclelinux_8 @echo Oraclelinux: done build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 build/test_oraclelinux_7_13 +# pgpro +#build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 @echo Oraclelinux 7: done build/test_oraclelinux_8: build/test_oraclelinux_8_9.6 build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 build/test_oraclelinux_8_13 +# pgpro +#build/test_oraclelinux_8: build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 @echo Oraclelinux 8: done # RHEL @@ -78,9 +86,13 @@ build/test_rhel: build/test_rhel_7 build/test_rhel_8 @echo Rhel: done build/test_rhel_7: build/test_rhel_7_9.5 build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 build/test_rhel_7_13 +# pgpro +#build/test_rhel_7: build/test_rhel_7_9.5 build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 @echo Rhel 7: done build/test_rhel_8: build/test_rhel_8_9.5 build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 build/test_rhel_8_13 +# pgpro +#build/test_rhel_8: build/test_rhel_8_9.5 build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 @echo Rhel 8: done define test_rpm diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec index d5811171d..8955b3fa7 100644 --- a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec @@ -42,7 +42,7 @@ mkdir %{_topdir}/SOURCES/pg_probackup-%{version} cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} %if "%{pgsql_major}" == "9.6" -./configure --enable-debug +./configure --enable-debug --without-readline %else ./configure --enable-debug --without-readline --prefix=%{prefix} %endif diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec index cbfd61a0f..cbb57e42a 100644 --- a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec @@ -43,7 +43,7 @@ cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} %if "%{pgsql_major}" == "9.6" ./configure --enable-debug %else -./configure --enable-debug --prefix=%{prefix} +./configure --enable-debug --disable-online-upgrade --prefix=%{prefix} %endif make -C 'src/common' make -C 'src/port' diff --git a/packaging/test/Makefile.debian b/packaging/test/Makefile.debian index f540f9205..084741069 100644 --- a/packaging/test/Makefile.debian +++ b/packaging/test/Makefile.debian @@ -39,3 +39,24 @@ build/test_debian_10_12: build/test_debian_10_13: $(call test_deb,debian,10,buster,13,13.2) touch build/test_debian_10_13 + +# DEBIAN 11 +build/test_debian_11_9.6: + $(call test_deb,debian,11,bullseye,9.6,9.6.21) + touch build/test_debian_11_9.6 + +build/test_debian_11_10: + $(call test_deb,debian,11,bullseye,10,10.16) + touch build/test_debian_11_10 + +build/test_debian_11_11: + $(call test_deb,debian,11,bullseye,11,11.11) + touch build/test_debian_11_11 + +build/test_debian_11_12: + $(call test_deb,debian,11,bullseye,12,12.6) + touch build/test_debian_11_12 + +build/test_debian_11_13: + $(call test_deb,debian,11,bullseye,13,13.2) + touch build/test_debian_11_13 diff --git a/packaging/test/scripts/deb.sh b/packaging/test/scripts/deb.sh index 76e3bb043..d7b957192 100755 --- a/packaging/test/scripts/deb.sh +++ b/packaging/test/scripts/deb.sh @@ -17,6 +17,7 @@ PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') export DEBIAN_FRONTEND=noninteractive echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +#apt-get -qq --allow-releaseinfo-change update apt-get -qq update apt-get -qq install -y wget nginx gnupg lsb-release #apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps @@ -45,7 +46,7 @@ nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) # install POSTGRESQL #if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + wget --no-check-certificate -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - apt-get update -y apt-get install -y postgresql-${PG_VERSION} #fi diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh index 3f24cc7e5..320d459f6 100755 --- a/packaging/test/scripts/rpm.sh +++ b/packaging/test/scripts/rpm.sh @@ -13,6 +13,7 @@ ulimit -n 1024 PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') +yum update -y # yum upgrade -y || echo 'some packages in docker failed to upgrade' # yum install -y sudo if [ ${DISTRIB} == 'rhel' ] && [ ${PG_TOG} == '13' ]; then # no packages for PG13 on PGDG diff --git a/packaging/test/scripts/suse_forks.sh b/packaging/test/scripts/suse_forks.sh old mode 100644 new mode 100755 From b87ca18bfc1094e356247ebf6329f2b2059987a7 Mon Sep 17 00:00:00 2001 From: dlepikhova <43872363+dlepikhova@users.noreply.github.com> Date: Mon, 22 Nov 2021 12:41:49 +0500 Subject: [PATCH 1820/2107] [Issue #265][PGPRO-5421] archive-push backward compatibility (#437) Restore the --wal-file-path option of the archive-push command (it was ignored since a196073) Co-authored-by: Mikhail A. Kulagin Co-authored-by: Elena Indrupskaya --- doc/pgprobackup.xml | 7 +- src/archive.c | 53 ++++--------- src/backup.c | 2 +- src/catchup.c | 6 +- src/help.c | 6 +- src/init.c | 2 +- src/pg_probackup.c | 98 +++++++++++++++++++++++- src/pg_probackup.h | 4 +- src/restore.c | 2 +- src/util.c | 6 +- src/utils/file.c | 27 +++++++ src/utils/file.h | 1 + src/utils/pgut.c | 16 ++++ src/utils/pgut.h | 1 + tests/archive.py | 127 ++++++++++++++++++++++++++++++++ tests/expected/option_help.out | 1 + tests/helpers/ptrack_helpers.py | 65 ++++++++-------- 17 files changed, 338 insertions(+), 86 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a347e7b43..76ec2cd76 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml backup_dir instance_name + wal_file_path wal_file_name option @@ -5367,7 +5368,9 @@ pg_probackup catchup -b catchup_mode Provides the path to the WAL file in archive_command and restore_command. Use the %p - variable as the value for this option for correct processing. + variable as the value for this option or explicitly specify the path to a file + outside of the data directory. If you skip this option, the path + specified in pg_probackup.conf will be used. @@ -5380,6 +5383,8 @@ pg_probackup catchup -b catchup_mode archive_command and restore_command. Use the %f variable as the value for this option for correct processing. + If the value of is a path + outside of the data directory, explicitly specify the filename. diff --git a/src/archive.c b/src/archive.c index 7bb8c1c03..0f32d9345 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2018-2019, Postgres Professional + * Portions Copyright (c) 2018-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -113,15 +113,13 @@ static parray *setup_push_filelist(const char *archive_status_dir, * Where archlog_path is $BACKUP_PATH/wal/instance_name */ void -do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path, +do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename) { uint64 i; - char current_dir[MAXPGPATH]; - char pg_xlog_dir[MAXPGPATH]; - char archive_status_dir[MAXPGPATH]; - uint64 system_id; + /* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */ + char archive_status_dir[MAXPGPATH] = ""; bool is_compress = false; /* arrays with meta info for multi threaded backup */ @@ -141,31 +139,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa parray *batch_files = NULL; int n_threads; - if (wal_file_name == NULL) - elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); - - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - - /* verify that archive-push --instance parameter is valid */ - system_id = get_system_identifier(current_dir, FIO_DB_HOST); - - if (instance->pgdata == NULL) - elog(ERROR, "Cannot read pg_probackup.conf for this instance"); - - if (system_id != instance->system_identifier) - elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instanceState->instance_name, instance->system_identifier, system_id); - - if (instance->compress_alg == PGLZ_COMPRESS) - elog(ERROR, "Cannot use pglz for WAL compression"); - - join_path_components(pg_xlog_dir, current_dir, XLOGDIR); - join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); - - /* Create 'archlog_path' directory. Do nothing if it already exists. */ - //fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST); + if (!no_ready_rename || batch_size > 1) + join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); #ifdef HAVE_LIBZ if (instance->compress_alg == ZLIB_COMPRESS) @@ -204,12 +179,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa { int rc; WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); + bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0; - rc = push_file(xlogfile, archive_status_dir, + rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir, pg_xlog_dir, instanceState->instance_wal_subdir_path, overwrite, no_sync, instance->archive_timeout, - no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false, + no_ready_rename || first_wal, is_compress && IsXLogFileName(xlogfile->name) ? true : false, instance->compress_level); if (rc == 0) @@ -233,7 +209,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa arg->first_filename = wal_file_name; arg->archive_dir = instanceState->instance_wal_subdir_path; arg->pg_xlog_dir = pg_xlog_dir; - arg->archive_status_dir = archive_status_dir; + arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL; arg->overwrite = overwrite; arg->compress = is_compress; arg->no_sync = no_sync; @@ -276,7 +252,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa /* Note, that we are leaking memory here, * because pushing into archive is a very - * time-sensetive operation, so we skip freeing stuff. + * time-sensitive operation, so we skip freeing stuff. */ push_done: @@ -356,9 +332,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, int compress_level) { int rc; - char wal_file_dummy[MAXPGPATH]; - - join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); elog(LOG, "pushing file \"%s\"", xlogfile->name); @@ -375,11 +348,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, #endif /* take '--no-ready-rename' flag into account */ - if (!no_ready_rename) + if (!no_ready_rename && archive_status_dir != NULL) { + char wal_file_dummy[MAXPGPATH]; char wal_file_ready[MAXPGPATH]; char wal_file_done[MAXPGPATH]; + join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready"); snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done"); diff --git a/src/backup.c b/src/backup.c index 1d08c3828..c575865c4 100644 --- a/src/backup.c +++ b/src/backup.c @@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata) uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST); + system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ diff --git a/src/catchup.c b/src/catchup.c index 5a0c8e45a..f9145a395 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons /* Get WAL segments size and system ID of source PG instance */ instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); - instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST); + instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false); current.start_time = time(NULL); strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); @@ -163,7 +163,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, uint64 source_conn_id, source_id, dest_id; source_conn_id = get_remote_system_identifier(source_conn); - source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */ + source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */ if (source_conn_id != source_id) elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", @@ -171,7 +171,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { - dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST); + dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false); if (source_conn_id != dest_id) elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", source_conn_id, dest_pgdata, dest_id); diff --git a/src/help.c b/src/help.c index 1515359e4..a6530fc0e 100644 --- a/src/help.c +++ b/src/help.c @@ -227,6 +227,7 @@ help_pg_probackup(void) printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); @@ -937,6 +938,7 @@ help_archive_push(void) { printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); @@ -951,6 +953,8 @@ help_archive_push(void) printf(_(" --instance=instance_name name of the instance to delete\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the file to copy into WAL archive\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path of the WAL archive\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --batch-size=NUM number of files to be copied\n")); printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n")); @@ -981,8 +985,8 @@ static void help_archive_get(void) { printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--no-validate-wal]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); diff --git a/src/init.c b/src/init.c index a4911cb5c..8773016b5 100644 --- a/src/init.c +++ b/src/init.c @@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST); + instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index d629d838d..49e226ace 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -35,7 +35,7 @@ * which includes info about pgdata directory and connection. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -151,6 +151,7 @@ static char *wal_file_path; static char *wal_file_name; static bool file_overwrite = false; static bool no_ready_rename = false; +static char archive_push_xlog_dir[MAXPGPATH] = ""; /* archive get options */ static char *prefetch_dir; @@ -788,7 +789,7 @@ main(int argc, char *argv[]) current.stream = stream_wal = true; if (instance_config.external_dir_str) elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); - // TODO проверить instance_config.conn_opt + // TODO check instance_config.conn_opt } /* sanity */ @@ -796,6 +797,97 @@ main(int argc, char *argv[]) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (backup_subcmd == ARCHIVE_PUSH_CMD) + { + /* Check archive-push parameters and construct archive_push_xlog_dir + * + * There are 4 cases: + * 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files + * (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push()) + * in this case we can use batches and threads + * 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata) + * in this case we can use this path, as well as batches and thread + * 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id + * in this case we can use this path, as well as batches and thread + * (replica for example, see test_archive_push_sanity) + * 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found + * disable optimizations and work with user specified path + */ + bool check_system_id = true; + uint64 system_id; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL) + elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); + + if (instance_config.pgdata == NULL) + elog(ERROR, "Cannot read pg_probackup.conf for this instance"); + + /* TODO may be remove in preference of checking inside compress_init()? */ + if (instance_config.compress_alg == PGLZ_COMPRESS) + elog(ERROR, "Cannot use pglz for WAL compression"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + if (wal_file_path == NULL) + { + /* 1st case */ + system_id = get_system_identifier(current_dir, FIO_DB_HOST, false); + join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR); + } + else + { + /* + * Usually we get something like + * wal_file_path = "pg_wal/0000000100000000000000A1" + * wal_file_name = "0000000100000000000000A1" + * instance_config.pgdata = "/pgdata/.../node/data" + * We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata + * and compare this directories. + * Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving) + */ + char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); + join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); + if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST)) + { + /* 2nd case */ + system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false); + /* archive_push_xlog_dir already have right value */ + } + else + { + if (strlen(stripped_wal_file_path) < MAXPGPATH) + strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH); + else + elog(ERROR, "Value specified to --wal_file_path is too long"); + + system_id = get_system_identifier(current_dir, FIO_DB_HOST, true); + /* 3rd case if control file present -- i.e. system_id != 0 */ + + if (system_id == 0) + { + /* 4th case */ + check_system_id = false; + + if (batch_size > 1 || num_threads > 1 || !no_ready_rename) + { + elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename"); + batch_size = 1; + num_threads = 1; + no_ready_rename = true; + } + } + } + pfree(stripped_wal_file_path); + } + + if (check_system_id && system_id != instance_config.system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, + wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id); + } + #if PG_VERSION_NUM >= 100000 if (temp_slot && perm_slot) elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option"); @@ -819,7 +911,7 @@ main(int argc, char *argv[]) switch (backup_subcmd) { case ARCHIVE_PUSH_CMD: - do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name, + do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name, batch_size, file_overwrite, no_sync, no_ready_rename); break; case ARCHIVE_GET_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6a1feb014..a51794d98 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -889,7 +889,7 @@ extern int do_init(CatalogState *catalogState); extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance); /* in archive.c */ -extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path, +extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename); extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, @@ -1153,7 +1153,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T extern TimeLineID get_current_timeline(PGconn *conn); extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path, fio_location location); +extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); diff --git a/src/restore.c b/src/restore.c index 005984aed..47e3b0344 100644 --- a/src/restore.c +++ b/src/restore.c @@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ elog(INFO, "Trying to read pg_control file in destination directory"); - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST); + system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); if (system_id_pgdata == instance_config.system_identifier) system_id_match = true; diff --git a/src/util.c b/src/util.c index f39b31d45..fb33fd046 100644 --- a/src/util.c +++ b/src/util.c @@ -247,15 +247,15 @@ get_checkpoint_location(PGconn *conn) } uint64 -get_system_identifier(const char *pgdata_path, fio_location location) +get_system_identifier(const char *pgdata_path, fio_location location, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); - if (buffer == NULL) + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location); + if (safe && buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); pg_free(buffer); diff --git a/src/utils/file.c b/src/utils/file.c index 810b4b394..7d1df554b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1141,6 +1141,33 @@ fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location lo } } +/* + * Compare, that filename1 and filename2 is the same file + * in windows compare only filenames + */ +bool +fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) +{ +#ifndef WIN32 + struct stat stat1, stat2; + + if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); + + if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); + + return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; +#else + char *abs_name1 = make_absolute_path(filename1); + char *abs_name2 = make_absolute_path(filename2); + bool result = strcmp(abs_name1, abs_name2) == 0; + free(abs_name2); + free(abs_name1); + return result; +#endif +} + /* * Read value of a symbolic link * this is a wrapper about readlink() syscall diff --git a/src/utils/file.h b/src/utils/file.h index edb5ea0f9..a554b4ab0 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -129,6 +129,7 @@ extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); extern int fio_access(char const* path, int mode, fio_location location); extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); +extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location); extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); extern DIR* fio_opendir(char const* path, fio_location location); extern struct dirent * fio_readdir(DIR *dirp); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 52599848d..2cf0ccbe7 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -977,6 +977,22 @@ pgut_strndup(const char *str, size_t n) return ret; } +/* + * Allocates new string, that contains part of filepath string minus trailing filename string + * If trailing filename string not found, returns copy of filepath. + * Result must be free by caller. + */ +char * +pgut_str_strip_trailing_filename(const char *filepath, const char *filename) +{ + size_t fp_len = strlen(filepath); + size_t fn_len = strlen(filename); + if (strncmp(filepath + fp_len - fn_len, filename, fn_len) == 0) + return pgut_strndup(filepath, fp_len - fn_len); + else + return pgut_strndup(filepath, fp_len); +} + FILE * pgut_fopen(const char *path, const char *mode, bool missing_ok) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index a1d7b5a93..fa0efe816 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -63,6 +63,7 @@ extern void *pgut_malloc0(size_t size); extern void *pgut_realloc(void *p, size_t size); extern char *pgut_strdup(const char *str); extern char *pgut_strndup(const char *str, size_t n); +extern char *pgut_str_strip_trailing_filename(const char *filepath, const char *filename); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) #define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) diff --git a/tests/archive.py b/tests/archive.py index 5157e8b89..22b9d8693 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -1828,6 +1828,133 @@ def test_archive_options_1(self): self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_undefined_wal_file_path(self): + """ + check that archive-push works correct with undefined + --wal-file-path + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + if os.name == 'posix': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node') + elif os.name == 'nt': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node').replace("\\","\\\\") + else: + self.assertTrue(False, 'Unexpected os family') + + self.set_auto_conf( + node, + {'archive_command': archive_command}) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_intermediate_archiving(self): + """ + check that archive-push works correct with --wal-file-path setting by user + """ + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + node_pg_options = {} + if node.major_version >= 13: + node_pg_options['wal_keep_size'] = '0MB' + else: + node_pg_options['wal_keep_segments'] = '0' + self.set_auto_conf(node, node_pg_options) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + wal_dir = os.path.join(self.tmp_path, module_name, fname, 'intermediate_dir') + shutil.rmtree(wal_dir, ignore_errors=True) + os.makedirs(wal_dir) + if os.name == 'posix': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) + elif os.name == 'nt': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) + else: + self.assertTrue(False, 'Unexpected os family') + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + wal_segment = '000000010000000000000001' + + self.run_pb(["archive-push", "-B", backup_dir, + "--instance=node", "-D", node.data_dir, + "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment]) + + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) + + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_waldir_outside_pgdata_archiving(self): + """ + check that archive-push works correct with symlinked waldir + """ + if self.pg_config_version < self.version_to_num('10.0'): + return unittest.skip( + 'Skipped because waldir outside pgdata is supported since PG 10') + + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + external_wal_dir = os.path.join(self.tmp_path, module_name, fname, 'ext_wal_dir') + shutil.rmtree(external_wal_dir, ignore_errors=True) + + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure def test_hexadecimal_timeline(self): diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 01384a893..dd3c4e865 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -144,6 +144,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup archive-push -B backup-path --instance=instance_name --wal-file-name=wal-file-name + [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] [--archive-timeout=timeout] [--no-ready-rename] [--no-sync] diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 1b54d3165..3b14b7170 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1296,7 +1296,8 @@ def get_recovery_conf(self, node): def set_archiving( self, backup_dir, instance, node, replica=False, overwrite=False, compress=True, old_binary=False, - log_level=False, archive_timeout=False): + log_level=False, archive_timeout=False, + custom_archive_command=None): # parse postgresql.auto.conf options = {} @@ -1306,45 +1307,47 @@ def set_archiving( else: options['archive_mode'] = 'on' - if os.name == 'posix': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) - - elif os.name == 'nt': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path.replace("\\","\\\\"), - backup_dir.replace("\\","\\\\"), instance) + if custom_archive_command is None: + if os.name == 'posix': + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path, backup_dir, instance) - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options['archive_command'] += '--remote-proto=ssh ' - options['archive_command'] += '--remote-host=localhost ' + elif os.name == 'nt': + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path.replace("\\","\\\\"), + backup_dir.replace("\\","\\\\"), instance) - if self.archive_compress and compress: - options['archive_command'] += '--compress ' + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options['archive_command'] += '--remote-proto=ssh ' + options['archive_command'] += '--remote-host=localhost ' - if overwrite: - options['archive_command'] += '--overwrite ' + if self.archive_compress and compress: + options['archive_command'] += '--compress ' - options['archive_command'] += '--log-level-console=VERBOSE ' - options['archive_command'] += '-j 5 ' - options['archive_command'] += '--batch-size 10 ' - options['archive_command'] += '--no-sync ' + if overwrite: + options['archive_command'] += '--overwrite ' - if archive_timeout: - options['archive_command'] += '--archive-timeout={0} '.format( - archive_timeout) + options['archive_command'] += '--log-level-console=VERBOSE ' + options['archive_command'] += '-j 5 ' + options['archive_command'] += '--batch-size 10 ' + options['archive_command'] += '--no-sync ' - if os.name == 'posix': - options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' + if archive_timeout: + options['archive_command'] += '--archive-timeout={0} '.format( + archive_timeout) - elif os.name == 'nt': - options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' + if os.name == 'posix': + options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' - if log_level: - options['archive_command'] += ' --log-level-console={0}'.format(log_level) - options['archive_command'] += ' --log-level-file={0} '.format(log_level) + elif os.name == 'nt': + options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' + if log_level: + options['archive_command'] += ' --log-level-console={0}'.format(log_level) + options['archive_command'] += ' --log-level-file={0} '.format(log_level) + else: # custom_archive_command is not None + options['archive_command'] = custom_archive_command self.set_auto_conf(node, options) From 758a32f09202adcc3312c37a51d3c8b9d55934d3 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 22 Nov 2021 13:16:48 +0300 Subject: [PATCH 1821/2107] Version 2.5.3 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a51794d98..b828343dc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.2" +#define PROGRAM_VERSION "2.5.3" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e9d8c0955..8b212ac1f 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.2 \ No newline at end of file +pg_probackup 2.5.3 \ No newline at end of file From 64ff0bbf7442b02c499c0b47607cc82a115e08b1 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Fri, 24 Dec 2021 13:18:11 +0300 Subject: [PATCH 1822/2107] [Issue #459][PGPRO-6034] Fix catchup (delta and ptrack) data corruption (#460) * [Issue #459][PGPRO-6034] Fix catchup (delta and ptrack) data corruption --- src/catchup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catchup.c b/src/catchup.c index f9145a395..78a1e5265 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -921,7 +921,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char fullpath[MAXPGPATH]; join_path_components(fullpath, dest_pgdata, file->rel_path); - fio_delete(file->mode, fullpath, FIO_DB_HOST); + fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); elog(VERBOSE, "Deleted file \"%s\"", fullpath); /* shrink dest pgdata list */ From ad932d8a2e26b9ca42d280a9e47ab5d957023599 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Fri, 24 Dec 2021 13:26:46 +0300 Subject: [PATCH 1823/2107] [PGPRO-6037] fix catchup timeline history checking (#462) * [PGPRO-6037] fix catchup timeline history checking --- src/catchup.c | 19 ++++++++++++++----- src/restore.c | 4 ++++ src/stream.c | 2 ++ tests/catchup.py | 32 ++++++++++++++++++++++++++------ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/catchup.c b/src/catchup.c index 78a1e5265..1b8f8084d 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -203,6 +203,8 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, /* fill dest_redo.lsn and dest_redo.tli */ get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + elog(VERBOSE, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", + current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); if (current.tli != 1) { @@ -285,11 +287,12 @@ catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli) { - PGresult *res; - PGconn *conn; - char *history; - char query[128]; - parray *result = NULL; + PGresult *res; + PGconn *conn; + char *history; + char query[128]; + parray *result = NULL; + TimeLineHistoryEntry *entry = NULL; snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); @@ -336,6 +339,12 @@ catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli) pg_free(history); PQclear(res); + /* append last timeline entry (as read_timeline_history() do) */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = InvalidXLogRecPtr; + parray_insert(result, 0, entry); + return result; } diff --git a/src/restore.c b/src/restore.c index 47e3b0344..d8d808a4e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1821,11 +1821,15 @@ satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn) { int i; + elog(VERBOSE, "satisfy_timeline() checking: tli = %X, lsn = %X/%X", + tli, (uint32) (lsn >> 32), (uint32) lsn); for (i = 0; i < parray_num(timelines); i++) { TimeLineHistoryEntry *timeline; timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); + elog(VERBOSE, "satisfy_timeline() check %i entry: timeline->tli = %X, timeline->end = %X/%X", + i, timeline->tli, (uint32) (timeline->end >> 32), (uint32) timeline->end); if (tli == timeline->tli && (XLogRecPtrIsInvalid(timeline->end) || lsn <= timeline->end)) diff --git a/src/stream.c b/src/stream.c index a53077391..1ee8dee37 100644 --- a/src/stream.c +++ b/src/stream.c @@ -615,6 +615,8 @@ parse_tli_history_buffer(char *history, TimeLineID tli) if (!result) result = parray_new(); parray_append(result, entry); + elog(VERBOSE, "parse_tli_history_buffer() found entry: tli = %X, end = %X/%X", + tli, switchpoint_hi, switchpoint_lo); /* we ignore the remainder of each line */ } diff --git a/tests/catchup.py b/tests/catchup.py index 79ebdec9f..8441deaaf 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -292,7 +292,7 @@ def test_tli_delta_catchup(self): src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - # do catchup + # do catchup (src_tli = 2, dst_tli = 1) self.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, @@ -310,15 +310,25 @@ def test_tli_delta_catchup(self): dst_options = {} dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() + self.set_replica(master = src_pg, replica = dst_pg) + dst_pg.slow_start(replica = True) # 2nd check: run verification query dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + dst_pg.stop() + + # do catchup (src_tli = 2, dst_tli = 2) + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + # Cleanup src_pg.stop() - dst_pg.stop() self.del_test_dir(module_name, self.fname) def test_tli_ptrack_catchup(self): @@ -365,7 +375,7 @@ def test_tli_ptrack_catchup(self): src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - # do catchup + # do catchup (src_tli = 2, dst_tli = 1) self.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, @@ -383,15 +393,25 @@ def test_tli_ptrack_catchup(self): dst_options = {} dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() + self.set_replica(master = src_pg, replica = dst_pg) + dst_pg.slow_start(replica = True) # 2nd check: run verification query dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + dst_pg.stop() + + # do catchup (src_tli = 2, dst_tli = 2) + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + # Cleanup src_pg.stop() - dst_pg.stop() self.del_test_dir(module_name, self.fname) ######################################### From f4c0ac3bf13a5896c1327d05377862c33951fb90 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 24 Dec 2021 13:49:09 +0300 Subject: [PATCH 1824/2107] Version 2.5.4 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b828343dc..b202b6152 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.3" +#define PROGRAM_VERSION "2.5.4" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 8b212ac1f..a69cee03d 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.3 \ No newline at end of file +pg_probackup 2.5.4 \ No newline at end of file From 58a5805b59dfcc6bf950a0bd3a502fda1a0d8ed8 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sat, 25 Dec 2021 18:07:00 +0300 Subject: [PATCH 1825/2107] [ci skip] packaging/Readme.md fix --- packaging/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Readme.md b/packaging/Readme.md index 749b9fc06..6fe38f277 100644 --- a/packaging/Readme.md +++ b/packaging/Readme.md @@ -7,7 +7,7 @@ export PBK_EDITION=std|ent make pkg ``` -To build binaries for PostgresPro Standart or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/tarballs` directory: +To build binaries for PostgresPro Standart or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/pkg/tarballs` directory: ``` cd packaging/pkg/tarballs git clone pgpro_repo pgpro From a454bd7d63e1329b2c46db6a71aa263ac7621cc6 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Sun, 26 Dec 2021 21:24:15 +0300 Subject: [PATCH 1826/2107] [ci skip] improve packaging: * adding testing of packages for PG-14 * updating postgres versions * adding reprepro config and rpmmacros to git * fixing forgotten rhel repo signing and package testing * adding alt-8 package testing * removing debian-8, ubuntu-14.04 and ubuntu-16.04 packaging * s/PGPRO Standart/PGPRO Standard/g --- packaging/Makefile.pkg | 24 +- packaging/Makefile.repo | 18 +- packaging/Makefile.test | 59 ++--- packaging/Readme.md | 6 +- packaging/pkg/Makefile.alt | 36 +-- packaging/pkg/Makefile.centos | 24 +- packaging/pkg/Makefile.debian | 65 ++--- packaging/pkg/Makefile.oraclelinux | 36 +-- packaging/pkg/Makefile.rhel | 24 +- packaging/pkg/Makefile.suse | 24 +- packaging/pkg/Makefile.ubuntu | 82 +----- packaging/pkg/scripts/alt.sh | 13 +- packaging/pkg/scripts/rpm.sh | 12 +- .../rpmbuild/SOURCES/pg_probackup-forks.repo | 2 +- .../SPECS/pg_probackup-repo-forks.spec | 2 +- packaging/repo/reprepro-conf/changelog.script | 246 ++++++++++++++++++ packaging/repo/reprepro-conf/distributions | 179 +++++++++++++ packaging/repo/rpm-conf/rpmmacros | 5 + packaging/repo/scripts/deb.sh | 4 +- packaging/repo/scripts/rpm.sh | 9 +- packaging/repo/scripts/suse.sh | 2 +- packaging/test/Makefile.alt | 40 ++- packaging/test/Makefile.centos | 28 +- packaging/test/Makefile.debian | 42 +-- packaging/test/Makefile.oraclelinux | 28 +- packaging/test/Makefile.rhel | 28 +- packaging/test/Makefile.suse | 28 +- packaging/test/Makefile.ubuntu | 49 ++-- packaging/test/scripts/deb.sh | 10 +- packaging/test/scripts/deb_forks.sh | 6 +- packaging/test/scripts/rpm.sh | 22 +- packaging/test/scripts/rpm_forks.sh | 7 +- 32 files changed, 771 insertions(+), 389 deletions(-) create mode 100755 packaging/repo/reprepro-conf/changelog.script create mode 100644 packaging/repo/reprepro-conf/distributions create mode 100644 packaging/repo/rpm-conf/rpmmacros diff --git a/packaging/Makefile.pkg b/packaging/Makefile.pkg index fc92ae408..e17243614 100644 --- a/packaging/Makefile.pkg +++ b/packaging/Makefile.pkg @@ -1,6 +1,6 @@ ifeq ($(PBK_EDITION),std) PBK_PKG_REPO = pg_probackup-forks - PBK_EDITION_FULL = Standart + PBK_EDITION_FULL = Standard PKG_NAME_SUFFIX = std- else ifeq ($(PBK_EDITION),ent) PBK_PKG_REPO = pg_probackup-forks @@ -42,12 +42,9 @@ build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt bu @echo Packaging is done ### DEBIAN -build/debian: build/debian_8 build/debian_9 build/debian_10 build/debian_11 +build/debian: build/debian_9 build/debian_10 build/debian_11 @echo Debian: done -build/debian_8: build/debian_8_9.6 build/debian_8_10 build/debian_8_11 build/debian_8_12 build/debian_8_13 build/debian_8_14 - @echo Debian 8: done - build/debian_9: build/debian_9_9.6 build/debian_9_10 build/debian_9_11 build/debian_9_12 build/debian_9_13 build/debian_9_14 @echo Debian 9: done @@ -58,15 +55,9 @@ build/debian_11: build/debian_11_9.6 build/debian_11_10 build/debian_11_11 build @echo Debian 11: done ### UBUNTU -build/ubuntu: build/ubuntu_14.04 build/ubuntu_16.04 build/ubuntu_18.04 build/ubuntu_20.04 +build/ubuntu: build/ubuntu_18.04 build/ubuntu_20.04 @echo Ubuntu: done -build/ubuntu_14.04: build/ubuntu_14.04_9.6 build/ubuntu_14.04_10 build/ubuntu_14.04_11 build/ubuntu_14.04_12 build/ubuntu_14.04_13 build/ubuntu_14.04_14 - @echo Ubuntu 14.04: done - -build/ubuntu_16.04: build/ubuntu_16.04_9.6 build/ubuntu_16.04_10 build/ubuntu_16.04_11 build/ubuntu_16.04_12 build/ubuntu_16.04_13 build/ubuntu_16.04_14 - @echo Ubuntu 16.04: done - build/ubuntu_18.04: build/ubuntu_18.04_9.6 build/ubuntu_18.04_10 build/ubuntu_18.04_11 build/ubuntu_18.04_12 build/ubuntu_18.04_13 build/ubuntu_18.04_14 @echo Ubuntu 18.04: done @@ -95,7 +86,8 @@ build/centos: build/centos_7 build/centos_8 #build/rpm_repo_package_centos build/centos_7: build/centos_7_9.6 build/centos_7_10 build/centos_7_11 build/centos_7_12 build/centos_7_13 build/centos_7_14 @echo Centos 7: done -build/centos_8: build/centos_8_9.6 build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 build/centos_8_14 +# pgpro-9.6@centos-8 doesn't exist +build/centos_8: build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 build/centos_8_14 #build/centos_8_9.6 @echo Centos 8: done # Oracle Linux @@ -108,7 +100,8 @@ build/oraclelinux_6: build/oraclelinux_6_9.6 build/oraclelinux_6_10 build/oracle build/oraclelinux_7: build/oraclelinux_7_9.6 build/oraclelinux_7_10 build/oraclelinux_7_11 build/oraclelinux_7_12 build/oraclelinux_7_13 build/oraclelinux_7_14 @echo Oraclelinux 7: done -build/oraclelinux_8: build/oraclelinux_8_9.6 build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 build/oraclelinux_8_14 +# pgpro-9.6@oraclelinux-8 doesn't exist +build/oraclelinux_8: build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 build/oraclelinux_8_14 #build/oraclelinux_8_9.6 @echo Oraclelinux 8: done # RHEL @@ -170,7 +163,8 @@ include packaging/pkg/Makefile.alt build/suse: build/suse_15.1 build/suse_15.2 @echo Suse: done -build/suse_15.1: build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 build/suse_15.1_14 +# there is no PG-14 in suse-15.1 repositories (test fails) +build/suse_15.1: build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 @echo Rhel 15.1: done build/suse_15.2: build/suse_15.2_9.6 build/suse_15.2_10 build/suse_15.2_11 build/suse_15.2_12 build/suse_15.2_13 build/suse_15.2_14 diff --git a/packaging/Makefile.repo b/packaging/Makefile.repo index 8186bfd80..10fb27137 100644 --- a/packaging/Makefile.repo +++ b/packaging/Makefile.repo @@ -1,15 +1,11 @@ #### REPO BUILD #### -repo: check_env repo/debian repo/ubuntu repo/centos repo/oraclelinux repo/alt repo/suse repo_finish #repo/rhel +repo: check_env repo/debian repo/ubuntu repo/centos repo/oraclelinux repo/rhel repo/alt repo/suse repo_finish @echo Build repo for all platform: done # Debian -repo/debian: build/repo_debian_8 build/repo_debian_9 build/repo_debian_10 build/repo_debian_11 +repo/debian: build/repo_debian_9 build/repo_debian_10 build/repo_debian_11 @echo Build repo for debian platforms: done -build/repo_debian_8: - $(call build_repo_deb,debian,8,jessie) - touch build/repo_debian_8 - build/repo_debian_9: $(call build_repo_deb,debian,9,stretch) touch build/repo_debian_9 @@ -23,17 +19,9 @@ build/repo_debian_11: touch build/repo_debian_11 # Ubuntu -repo/ubuntu: build/repo_ubuntu_14.04 build/repo_ubuntu_16.04 build/repo_ubuntu_18.04 build/repo_ubuntu_20.04 +repo/ubuntu: build/repo_ubuntu_18.04 build/repo_ubuntu_20.04 @echo Build repo for ubuntu platforms: done -build/repo_ubuntu_14.04: - $(call build_repo_deb,ubuntu,14.04,trusty) - touch build/repo_ubuntu_14.04 - -build/repo_ubuntu_16.04: - $(call build_repo_deb,ubuntu,16.04,xenial) - touch build/repo_ubuntu_16.04 - build/repo_ubuntu_18.04: $(call build_repo_deb,ubuntu,18.04,bionic) touch build/repo_ubuntu_18.04 diff --git a/packaging/Makefile.test b/packaging/Makefile.test index 21e850ccd..f5e004f01 100644 --- a/packaging/Makefile.test +++ b/packaging/Makefile.test @@ -9,33 +9,30 @@ endif test: build/test_all @echo Test for all platform: done -build/test_all: build/test_debian build/test_ubuntu build/test_centos build/test_oraclelinux build/test_alt build/test_suse # build/test_rhel +build/test_all: build/test_debian build/test_ubuntu build/test_centos build/test_oraclelinux build/test_alt build/test_suse #build/test_rhel @echo Package testing is done ### DEBIAN -build/test_debian: build/test_debian_9 build/test_debian_10 #build/test_debian_11 +build/test_debian: build/test_debian_9 build/test_debian_10 build/test_debian_11 @echo Debian: done -build/test_debian_9: build/test_debian_9_9.6 build/test_debian_9_10 build/test_debian_9_11 build/test_debian_9_12 build/test_debian_9_13 +build/test_debian_9: build/test_debian_9_9.6 build/test_debian_9_10 build/test_debian_9_11 build/test_debian_9_12 build/test_debian_9_13 build/test_debian_9_14 @echo Debian 9: done -build/test_debian_10: build/test_debian_10_9.6 build/test_debian_10_10 build/test_debian_10_11 build/test_debian_10_12 build/test_debian_10_13 +build/test_debian_10: build/test_debian_10_9.6 build/test_debian_10_10 build/test_debian_10_11 build/test_debian_10_12 build/test_debian_10_13 build/test_debian_10_14 @echo Debian 10: done -build/test_debian_11: build/test_debian_11_9.6 build/test_debian_11_10 build/test_debian_11_11 build/test_debian_11_12 build/test_debian_11_13 +build/test_debian_11: build/test_debian_11_9.6 build/test_debian_11_10 build/test_debian_11_11 build/test_debian_11_12 build/test_debian_11_13 build/test_debian_11_14 @echo Debian 11: done ### UBUNTU -build/test_ubuntu: build/test_ubuntu_16.04 build/test_ubuntu_18.04 build/test_ubuntu_20.04 +build/test_ubuntu: build/test_ubuntu_18.04 build/test_ubuntu_20.04 @echo Ubuntu: done -build/test_ubuntu_16.04: build/test_ubuntu_16.04_9.6 build/test_ubuntu_16.04_10 build/test_ubuntu_16.04_11 build/test_ubuntu_16.04_12 build/test_ubuntu_16.04_13 - @echo Ubuntu 16.04: done - -build/test_ubuntu_18.04: build/test_ubuntu_18.04_9.6 build/test_ubuntu_18.04_10 build/test_ubuntu_18.04_11 build/test_ubuntu_18.04_12 build/test_ubuntu_18.04_13 +build/test_ubuntu_18.04: build/test_ubuntu_18.04_9.6 build/test_ubuntu_18.04_10 build/test_ubuntu_18.04_11 build/test_ubuntu_18.04_12 build/test_ubuntu_18.04_13 build/test_ubuntu_18.04_14 @echo Ubuntu 18.04: done -build/test_ubuntu_20.04: build/test_ubuntu_20.04_9.6 build/test_ubuntu_20.04_10 build/test_ubuntu_20.04_11 build/test_ubuntu_20.04_12 build/test_ubuntu_20.04_13 +build/test_ubuntu_20.04: build/test_ubuntu_20.04_9.6 build/test_ubuntu_20.04_10 build/test_ubuntu_20.04_11 build/test_ubuntu_20.04_12 build/test_ubuntu_20.04_13 build/test_ubuntu_20.04_14 @echo Ubuntu 20.04: done define test_deb @@ -57,42 +54,32 @@ include packaging/test/Makefile.ubuntu build/test_centos: build/test_centos_7 build/test_centos_8 @echo Centos: done -build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 build/test_centos_7_13 -# pgpro -#build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 +build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 build/test_centos_7_13 #build/test_centos_7_14 @echo Centos 7: done -build/test_centos_8: build/test_centos_8_9.6 build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 build/test_centos_8_13 -# pgpro -#build/test_centos_8: build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 +# pgpro-9.6@centos-8 doesn't exist +build/test_centos_8: build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 build/test_centos_8_13 #build/test_centos_8_14 build/test_centos_8_9.6 @echo Centos 8: done # Oracle Linux build/test_oraclelinux: build/test_oraclelinux_7 build/test_oraclelinux_8 @echo Oraclelinux: done -build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 build/test_oraclelinux_7_13 -# pgpro -#build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 +build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 build/test_oraclelinux_7_13 #build/test_oraclelinux_7_14 @echo Oraclelinux 7: done -build/test_oraclelinux_8: build/test_oraclelinux_8_9.6 build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 build/test_oraclelinux_8_13 -# pgpro -#build/test_oraclelinux_8: build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 +# pgpro-9.6@oraclelinux-8 doesn't exist +build/test_oraclelinux_8: build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 build/test_oraclelinux_8_13 #build/test_oraclelinux_8_14 build/test_oraclelinux_8_9.6 @echo Oraclelinux 8: done # RHEL -build/test_rhel: build/test_rhel_7 build/test_rhel_8 +build/test_rhel: build/test_rhel_7 #build/test_rhel_8 @echo Rhel: done -build/test_rhel_7: build/test_rhel_7_9.5 build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 build/test_rhel_7_13 -# pgpro -#build/test_rhel_7: build/test_rhel_7_9.5 build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 +build/test_rhel_7: build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 build/test_rhel_7_13 #build/test_rhel_7_14 @echo Rhel 7: done -build/test_rhel_8: build/test_rhel_8_9.5 build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 build/test_rhel_8_13 -# pgpro -#build/test_rhel_8: build/test_rhel_8_9.5 build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 +build/test_rhel_8: build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 build/test_rhel_8_13 build/test_rhel_8_14 @echo Rhel 8: done define test_rpm @@ -112,10 +99,16 @@ include packaging/test/Makefile.rhel include packaging/test/Makefile.oraclelinux # Alt Linux -build/test_alt: build/test_alt_9 +build/test_alt: build/test_alt_8 build/test_alt_9 @echo Alt Linux: done -build/test_alt_9: build/test_alt_9_9.6 build/test_alt_9_10 build/test_alt_9_11 build/test_alt_9_12 build/test_alt_9_13 +# nginx@alt7 fall with 'nginx: [alert] sysctl(KERN_RTSIGMAX) failed (1: Operation not permitted)' +# within docker on modern host linux kernels (this nginx build require Linux between 2.2.19 and 2.6.17) + +build/test_alt_8: build/test_alt_8_9.6 build/test_alt_8_10 build/test_alt_8_11 build/test_alt_8_12 build/test_alt_8_13 build/test_alt_8_14 + @echo Alt Linux 8: done + +build/test_alt_9: build/test_alt_9_9.6 build/test_alt_9_10 build/test_alt_9_11 build/test_alt_9_12 build/test_alt_9_13 build/test_alt_9_14 @echo Alt Linux 9: done define test_alt @@ -139,7 +132,7 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2 build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13 @echo Rhel 15.1: done -build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 +build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14 @echo Rhel 15.1: done define test_suse diff --git a/packaging/Readme.md b/packaging/Readme.md index 6fe38f277..f4437d838 100644 --- a/packaging/Readme.md +++ b/packaging/Readme.md @@ -4,10 +4,10 @@ export PBK_VERSION=2.4.17 export PBK_HASH=57f871accce2604 export PBK_RELEASE=1 export PBK_EDITION=std|ent -make pkg +make --keep-going pkg ``` -To build binaries for PostgresPro Standart or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/pkg/tarballs` directory: +To build binaries for PostgresPro Standard or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/pkg/tarballs` directory: ``` cd packaging/pkg/tarballs git clone pgpro_repo pgpro @@ -19,3 +19,5 @@ Repo must be build using 1 thread (due to debian bullshit): ``` make repo -j1 ``` + + diff --git a/packaging/pkg/Makefile.alt b/packaging/pkg/Makefile.alt index 919d3f58c..28eabf53f 100644 --- a/packaging/pkg/Makefile.alt +++ b/packaging/pkg/Makefile.alt @@ -4,27 +4,27 @@ build/alt_7_9.5: touch build/alt_7_9.5 build/alt_7_9.6: - $(call build_alt,alt,7,,9.6,9.6.23) + $(call build_alt,alt,7,,9.6,9.6.24) touch build/alt_7_9.6 build/alt_7_10: - $(call build_alt,alt,7,,10,10.18) + $(call build_alt,alt,7,,10,10.19) touch build/alt_7_10 build/alt_7_11: - $(call build_alt,alt,7,,11,11.13) + $(call build_alt,alt,7,,11,11.14) touch build/alt_7_11 build/alt_7_12: - $(call build_alt,alt,7,,12,12.8) + $(call build_alt,alt,7,,12,12.9) touch build/alt_7_12 build/alt_7_13: - $(call build_alt,alt,7,,13,13.4) + $(call build_alt,alt,7,,13,13.5) touch build/alt_7_13 build/alt_7_14: - $(call build_alt,alt,7,,14,14.0) + $(call build_alt,alt,7,,14,14.1) touch build/alt_7_14 # ALT 8 @@ -33,27 +33,27 @@ build/alt_8_9.5: touch build/alt_8_9.5 build/alt_8_9.6: - $(call build_alt,alt,8,,9.6,9.6.23) + $(call build_alt,alt,8,,9.6,9.6.24) touch build/alt_8_9.6 build/alt_8_10: - $(call build_alt,alt,8,,10,10.18) + $(call build_alt,alt,8,,10,10.19) touch build/alt_8_10 build/alt_8_11: - $(call build_alt,alt,8,,11,11.13) + $(call build_alt,alt,8,,11,11.14) touch build/alt_8_11 build/alt_8_12: - $(call build_alt,alt,8,,12,12.8) + $(call build_alt,alt,8,,12,12.9) touch build/alt_8_12 build/alt_8_13: - $(call build_alt,alt,8,,13,13.4) + $(call build_alt,alt,8,,13,13.5) touch build/alt_8_13 build/alt_8_14: - $(call build_alt,alt,8,,14,14.0) + $(call build_alt,alt,8,,14,14.1) touch build/alt_8_14 # ALT 9 @@ -62,26 +62,26 @@ build/alt_9_9.5: touch build/alt_9_9.5 build/alt_9_9.6: - $(call build_alt,alt,9,,9.6,9.6.23) + $(call build_alt,alt,9,,9.6,9.6.24) touch build/alt_9_9.6 build/alt_9_10: - $(call build_alt,alt,9,,10,10.18) + $(call build_alt,alt,9,,10,10.19) touch build/alt_9_10 build/alt_9_11: - $(call build_alt,alt,9,,11,11.13) + $(call build_alt,alt,9,,11,11.14) touch build/alt_9_11 build/alt_9_12: - $(call build_alt,alt,9,,12,12.8) + $(call build_alt,alt,9,,12,12.9) touch build/alt_9_12 build/alt_9_13: - $(call build_alt,alt,9,,13,13.4) + $(call build_alt,alt,9,,13,13.5) touch build/alt_9_13 build/alt_9_14: - $(call build_alt,alt,9,,14,14.0) + $(call build_alt,alt,9,,14,14.1) touch build/alt_9_14 diff --git a/packaging/pkg/Makefile.centos b/packaging/pkg/Makefile.centos index 9542a5202..fb537d0a6 100644 --- a/packaging/pkg/Makefile.centos +++ b/packaging/pkg/Makefile.centos @@ -4,27 +4,27 @@ build/centos_7_9.5: touch build/centos_7_9.5 build/centos_7_9.6: - $(call build_rpm,centos,7,,9.6,9.6.23) + $(call build_rpm,centos,7,,9.6,9.6.24) touch build/centos_7_9.6 build/centos_7_10: - $(call build_rpm,centos,7,,10,10.18) + $(call build_rpm,centos,7,,10,10.19) touch build/centos_7_10 build/centos_7_11: - $(call build_rpm,centos,7,,11,11.13) + $(call build_rpm,centos,7,,11,11.14) touch build/centos_7_11 build/centos_7_12: - $(call build_rpm,centos,7,,12,12.8) + $(call build_rpm,centos,7,,12,12.9) touch build/centos_7_12 build/centos_7_13: - $(call build_rpm,centos,7,,13,13.4) + $(call build_rpm,centos,7,,13,13.5) touch build/centos_7_13 build/centos_7_14: - $(call build_rpm,centos,7,,14,14.0) + $(call build_rpm,centos,7,,14,14.1) touch build/centos_7_14 # CENTOS 8 @@ -33,25 +33,25 @@ build/centos_8_9.5: touch build/centos_8_9.5 build/centos_8_9.6: - $(call build_rpm,centos,8,,9.6,9.6.23) + $(call build_rpm,centos,8,,9.6,9.6.24) touch build/centos_8_9.6 build/centos_8_10: - $(call build_rpm,centos,8,,10,10.18) + $(call build_rpm,centos,8,,10,10.19) touch build/centos_8_10 build/centos_8_11: - $(call build_rpm,centos,8,,11,11.13) + $(call build_rpm,centos,8,,11,11.14) touch build/centos_8_11 build/centos_8_12: - $(call build_rpm,centos,8,,12,12.8) + $(call build_rpm,centos,8,,12,12.9) touch build/centos_8_12 build/centos_8_13: - $(call build_rpm,centos,8,,13,13.4) + $(call build_rpm,centos,8,,13,13.5) touch build/centos_8_13 build/centos_8_14: - $(call build_rpm,centos,8,,14,14.0) + $(call build_rpm,centos,8,,14,14.1) touch build/centos_8_14 diff --git a/packaging/pkg/Makefile.debian b/packaging/pkg/Makefile.debian index 7c82a412b..d9c885d3a 100644 --- a/packaging/pkg/Makefile.debian +++ b/packaging/pkg/Makefile.debian @@ -1,59 +1,30 @@ -# DEBIAN 8 -build/debian_8_9.5: - $(call build_deb,debian,8,jessie,9.5,9.5.25) - touch build/debian_8_9.5 - -build/debian_8_9.6: - $(call build_deb,debian,8,jessie,9.6,9.6.23) - touch build/debian_8_9.6 - -build/debian_8_10: - $(call build_deb,debian,8,jessie,10,10.18) - touch build/debian_8_10 - -build/debian_8_11: - $(call build_deb,debian,8,jessie,11,11.13) - touch build/debian_8_11 - -build/debian_8_12: - $(call build_deb,debian,8,jessie,12,12.8) - touch build/debian_8_12 - -build/debian_8_13: - $(call build_deb,debian,8,jessie,13,13.4) - touch build/debian_8_13 - -build/debian_8_14: - $(call build_deb,debian,8,jessie,14,14.0) - touch build/debian_8_14 - # DEBIAN 9 build/debian_9_9.5: $(call build_deb,debian,9,stretch,9.5,9.5.25) touch build/debian_9_9.5 build/debian_9_9.6: - $(call build_deb,debian,9,stretch,9.6,9.6.23) + $(call build_deb,debian,9,stretch,9.6,9.6.24) touch build/debian_9_9.6 build/debian_9_10: - $(call build_deb,debian,9,stretch,10,10.18) + $(call build_deb,debian,9,stretch,10,10.19) touch build/debian_9_10 build/debian_9_11: - $(call build_deb,debian,9,stretch,11,11.13) + $(call build_deb,debian,9,stretch,11,11.14) touch build/debian_9_11 build/debian_9_12: - $(call build_deb,debian,9,stretch,12,12.8) + $(call build_deb,debian,9,stretch,12,12.9) touch build/debian_9_12 build/debian_9_13: - $(call build_deb,debian,9,stretch,13,13.4) + $(call build_deb,debian,9,stretch,13,13.5) touch build/debian_9_13 build/debian_9_14: - $(call build_deb,debian,9,stretch,14,14.0) + $(call build_deb,debian,9,stretch,14,14.1) touch build/debian_9_14 # DEBIAN 10 @@ -62,27 +33,27 @@ build/debian_10_9.5: touch build/debian_10_9.5 build/debian_10_9.6: - $(call build_deb,debian,10,buster,9.6,9.6.23) + $(call build_deb,debian,10,buster,9.6,9.6.24) touch build/debian_10_9.6 build/debian_10_10: - $(call build_deb,debian,10,buster,10,10.18) + $(call build_deb,debian,10,buster,10,10.19) touch build/debian_10_10 build/debian_10_11: - $(call build_deb,debian,10,buster,11,11.13) + $(call build_deb,debian,10,buster,11,11.14) touch build/debian_10_11 build/debian_10_12: - $(call build_deb,debian,10,buster,12,12.8) + $(call build_deb,debian,10,buster,12,12.9) touch build/debian_10_12 build/debian_10_13: - $(call build_deb,debian,10,buster,13,13.4) + $(call build_deb,debian,10,buster,13,13.5) touch build/debian_10_13 build/debian_10_14: - $(call build_deb,debian,10,buster,14,14.0) + $(call build_deb,debian,10,buster,14,14.1) touch build/debian_10_14 # DEBIAN 11 @@ -91,25 +62,25 @@ build/debian_11_9.5: touch build/debian_11_9.5 build/debian_11_9.6: - $(call build_deb,debian,11,bullseye,9.6,9.6.23) + $(call build_deb,debian,11,bullseye,9.6,9.6.24) touch build/debian_11_9.6 build/debian_11_10: - $(call build_deb,debian,11,bullseye,10,10.18) + $(call build_deb,debian,11,bullseye,10,10.19) touch build/debian_11_10 build/debian_11_11: - $(call build_deb,debian,11,bullseye,11,11.13) + $(call build_deb,debian,11,bullseye,11,11.14) touch build/debian_11_11 build/debian_11_12: - $(call build_deb,debian,11,bullseye,12,12.8) + $(call build_deb,debian,11,bullseye,12,12.9) touch build/debian_11_12 build/debian_11_13: - $(call build_deb,debian,11,bullseye,13,13.4) + $(call build_deb,debian,11,bullseye,13,13.5) touch build/debian_11_13 build/debian_11_14: - $(call build_deb,debian,11,bullseye,14,14.0) + $(call build_deb,debian,11,bullseye,14,14.1) touch build/debian_11_14 diff --git a/packaging/pkg/Makefile.oraclelinux b/packaging/pkg/Makefile.oraclelinux index 3dbdbd424..127a578f1 100644 --- a/packaging/pkg/Makefile.oraclelinux +++ b/packaging/pkg/Makefile.oraclelinux @@ -4,27 +4,27 @@ build/oraclelinux_6_9.5: touch build/oraclelinux_6_9.5 build/oraclelinux_6_9.6: - $(call build_rpm,oraclelinux,6,,9.6,9.6.23) + $(call build_rpm,oraclelinux,6,,9.6,9.6.24) touch build/oraclelinux_6_9.6 build/oraclelinux_6_10: - $(call build_rpm,oraclelinux,6,,10,10.18) + $(call build_rpm,oraclelinux,6,,10,10.19) touch build/oraclelinux_6_10 build/oraclelinux_6_11: - $(call build_rpm,oraclelinux,6,,11,11.13) + $(call build_rpm,oraclelinux,6,,11,11.14) touch build/oraclelinux_6_11 build/oraclelinux_6_12: - $(call build_rpm,oraclelinux,6,,12,12.8) + $(call build_rpm,oraclelinux,6,,12,12.9) touch build/oraclelinux_6_12 build/oraclelinux_6_13: - $(call build_rpm,oraclelinux,6,,13,13.4) + $(call build_rpm,oraclelinux,6,,13,13.5) touch build/oraclelinux_6_13 build/oraclelinux_6_14: - $(call build_rpm,oraclelinux,6,,14,14.0) + $(call build_rpm,oraclelinux,6,,14,14.1) touch build/oraclelinux_6_14 # ORACLE LINUX 7 @@ -33,27 +33,27 @@ build/oraclelinux_7_9.5: touch build/oraclelinux_7_9.5 build/oraclelinux_7_9.6: - $(call build_rpm,oraclelinux,7,,9.6,9.6.23) + $(call build_rpm,oraclelinux,7,,9.6,9.6.24) touch build/oraclelinux_7_9.6 build/oraclelinux_7_10: - $(call build_rpm,oraclelinux,7,,10,10.18) + $(call build_rpm,oraclelinux,7,,10,10.19) touch build/oraclelinux_7_10 build/oraclelinux_7_11: - $(call build_rpm,oraclelinux,7,,11,11.13) + $(call build_rpm,oraclelinux,7,,11,11.14) touch build/oraclelinux_7_11 build/oraclelinux_7_12: - $(call build_rpm,oraclelinux,7,,12,12.8) + $(call build_rpm,oraclelinux,7,,12,12.9) touch build/oraclelinux_7_12 build/oraclelinux_7_13: - $(call build_rpm,oraclelinux,7,,13,13.4) + $(call build_rpm,oraclelinux,7,,13,13.5) touch build/oraclelinux_7_13 build/oraclelinux_7_14: - $(call build_rpm,oraclelinux,7,,14,14.0) + $(call build_rpm,oraclelinux,7,,14,14.1) touch build/oraclelinux_7_14 # ORACLE LINUX 8 @@ -62,26 +62,26 @@ build/oraclelinux_8_9.5: touch build/oraclelinux_8_9.5 build/oraclelinux_8_9.6: - $(call build_rpm,oraclelinux,8,,9.6,9.6.23) + $(call build_rpm,oraclelinux,8,,9.6,9.6.24) touch build/oraclelinux_8_9.6 build/oraclelinux_8_10: - $(call build_rpm,oraclelinux,8,,10,10.18) + $(call build_rpm,oraclelinux,8,,10,10.19) touch build/oraclelinux_8_10 build/oraclelinux_8_11: - $(call build_rpm,oraclelinux,8,,11,11.13) + $(call build_rpm,oraclelinux,8,,11,11.14) touch build/oraclelinux_8_11 build/oraclelinux_8_12: - $(call build_rpm,oraclelinux,8,,12,12.8) + $(call build_rpm,oraclelinux,8,,12,12.9) touch build/oraclelinux_8_12 build/oraclelinux_8_13: - $(call build_rpm,oraclelinux,8,,13,13.4) + $(call build_rpm,oraclelinux,8,,13,13.5) touch build/oraclelinux_8_13 build/oraclelinux_8_14: - $(call build_rpm,oraclelinux,8,,14,14.0) + $(call build_rpm,oraclelinux,8,,14,14.1) touch build/oraclelinux_8_14 diff --git a/packaging/pkg/Makefile.rhel b/packaging/pkg/Makefile.rhel index b604a990d..8c1b0687b 100644 --- a/packaging/pkg/Makefile.rhel +++ b/packaging/pkg/Makefile.rhel @@ -4,27 +4,27 @@ build/rhel_7_9.5: touch build/rhel_7_9.5 build/rhel_7_9.6: - $(call build_rpm,rhel,7,7Server,9.6,9.6.23) + $(call build_rpm,rhel,7,7Server,9.6,9.6.24) touch build/rhel_7_9.6 build/rhel_7_10: - $(call build_rpm,rhel,7,7Server,10,10.18) + $(call build_rpm,rhel,7,7Server,10,10.19) touch build/rhel_7_10 build/rhel_7_11: - $(call build_rpm,rhel,7,7Server,11,11.13) + $(call build_rpm,rhel,7,7Server,11,11.14) touch build/rhel_7_11 build/rhel_7_12: - $(call build_rpm,rhel,7,7Server,12,12.8) + $(call build_rpm,rhel,7,7Server,12,12.9) touch build/rhel_7_12 build/rhel_7_13: - $(call build_rpm,rhel,7,7Server,13,13.4) + $(call build_rpm,rhel,7,7Server,13,13.5) touch build/rhel_7_13 build/rhel_7_14: - $(call build_rpm,rhel,7,7Server,14,14.0) + $(call build_rpm,rhel,7,7Server,14,14.1) touch build/rhel_7_14 # RHEL 8 @@ -33,25 +33,25 @@ build/rhel_8_9.5: touch build/rhel_8_9.5 build/rhel_8_9.6: - $(call build_rpm,rhel,8,8Server,9.6,9.6.23) + $(call build_rpm,rhel,8,8Server,9.6,9.6.24) touch build/rhel_8_9.6 build/rhel_8_10: - $(call build_rpm,rhel,8,8Server,10,10.18) + $(call build_rpm,rhel,8,8Server,10,10.19) touch build/rhel_8_10 build/rhel_8_11: - $(call build_rpm,rhel,8,8Server,11,11.13) + $(call build_rpm,rhel,8,8Server,11,11.14) touch build/rhel_8_11 build/rhel_8_12: - $(call build_rpm,rhel,8,8Server,12,12.8) + $(call build_rpm,rhel,8,8Server,12,12.9) touch build/rhel_8_12 build/rhel_8_13: - $(call build_rpm,rhel,8,8Server,13,13.4) + $(call build_rpm,rhel,8,8Server,13,13.5) touch build/rhel_8_13 build/rhel_8_14: - $(call build_rpm,rhel,8,8Server,14,14.0) + $(call build_rpm,rhel,8,8Server,14,14.1) touch build/rhel_8_14 diff --git a/packaging/pkg/Makefile.suse b/packaging/pkg/Makefile.suse index 5af22c5d0..c71ebd389 100644 --- a/packaging/pkg/Makefile.suse +++ b/packaging/pkg/Makefile.suse @@ -4,27 +4,27 @@ build/suse_15.1_9.5: touch build/suse_15.1_9.5 build/suse_15.1_9.6: - $(call build_suse,suse,15.1,,9.6,9.6.23) + $(call build_suse,suse,15.1,,9.6,9.6.24) touch build/suse_15.1_9.6 build/suse_15.1_10: - $(call build_suse,suse,15.1,,10,10.18) + $(call build_suse,suse,15.1,,10,10.19) touch build/suse_15.1_10 build/suse_15.1_11: - $(call build_suse,suse,15.1,,11,11.13) + $(call build_suse,suse,15.1,,11,11.14) touch build/suse_15.1_11 build/suse_15.1_12: - $(call build_suse,suse,15.1,,12,12.8) + $(call build_suse,suse,15.1,,12,12.9) touch build/suse_15.1_12 build/suse_15.1_13: - $(call build_suse,suse,15.1,,13,13.4) + $(call build_suse,suse,15.1,,13,13.5) touch build/suse_15.1_13 build/suse_15.1_14: - $(call build_suse,suse,15.1,,14,14.0) + $(call build_suse,suse,15.1,,14,14.1) touch build/suse_15.1_14 # Suse 15.2 @@ -33,25 +33,25 @@ build/suse_15.2_9.5: touch build/suse_15.2_9.5 build/suse_15.2_9.6: - $(call build_suse,suse,15.2,,9.6,9.6.23) + $(call build_suse,suse,15.2,,9.6,9.6.24) touch build/suse_15.2_9.6 build/suse_15.2_10: - $(call build_suse,suse,15.2,,10,10.18) + $(call build_suse,suse,15.2,,10,10.19) touch build/suse_15.2_10 build/suse_15.2_11: - $(call build_suse,suse,15.2,,11,11.13) + $(call build_suse,suse,15.2,,11,11.14) touch build/suse_15.2_11 build/suse_15.2_12: - $(call build_suse,suse,15.2,,12,12.8) + $(call build_suse,suse,15.2,,12,12.9) touch build/suse_15.2_12 build/suse_15.2_13: - $(call build_suse,suse,15.2,,13,13.4) + $(call build_suse,suse,15.2,,13,13.5) touch build/suse_15.2_13 build/suse_15.2_14: - $(call build_suse,suse,15.2,,14,14.0) + $(call build_suse,suse,15.2,,14,14.1) touch build/suse_15.2_14 diff --git a/packaging/pkg/Makefile.ubuntu b/packaging/pkg/Makefile.ubuntu index 88803c64f..02acd6c67 100644 --- a/packaging/pkg/Makefile.ubuntu +++ b/packaging/pkg/Makefile.ubuntu @@ -4,27 +4,27 @@ build/ubuntu_20.04_9.5: touch build/ubuntu_20.04_9.5 build/ubuntu_20.04_9.6: - $(call build_deb,ubuntu,20.04,focal,9.6,9.6.23) + $(call build_deb,ubuntu,20.04,focal,9.6,9.6.24) touch build/ubuntu_20.04_9.6 build/ubuntu_20.04_10: - $(call build_deb,ubuntu,20.04,focal,10,10.18) + $(call build_deb,ubuntu,20.04,focal,10,10.19) touch build/ubuntu_20.04_10 build/ubuntu_20.04_11: - $(call build_deb,ubuntu,20.04,focal,11,11.13) + $(call build_deb,ubuntu,20.04,focal,11,11.14) touch build/ubuntu_20.04_11 build/ubuntu_20.04_12: - $(call build_deb,ubuntu,20.04,focal,12,12.8) + $(call build_deb,ubuntu,20.04,focal,12,12.9) touch build/ubuntu_20.04_12 build/ubuntu_20.04_13: - $(call build_deb,ubuntu,20.04,focal,13,13.4) + $(call build_deb,ubuntu,20.04,focal,13,13.5) touch build/ubuntu_20.04_13 build/ubuntu_20.04_14: - $(call build_deb,ubuntu,20.04,focal,14,14.0) + $(call build_deb,ubuntu,20.04,focal,14,14.1) touch build/ubuntu_20.04_14 # UBUNTU 18.04 @@ -33,84 +33,26 @@ build/ubuntu_18.04_9.5: touch build/ubuntu_18.04_9.5 build/ubuntu_18.04_9.6: - $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.23) + $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.24) touch build/ubuntu_18.04_9.6 build/ubuntu_18.04_10: - $(call build_deb,ubuntu,18.04,bionic,10,10.18) + $(call build_deb,ubuntu,18.04,bionic,10,10.19) touch build/ubuntu_18.04_10 build/ubuntu_18.04_11: - $(call build_deb,ubuntu,18.04,bionic,11,11.13) + $(call build_deb,ubuntu,18.04,bionic,11,11.14) touch build/ubuntu_18.04_11 build/ubuntu_18.04_12: - $(call build_deb,ubuntu,18.04,bionic,12,12.8) + $(call build_deb,ubuntu,18.04,bionic,12,12.9) touch build/ubuntu_18.04_12 build/ubuntu_18.04_13: - $(call build_deb,ubuntu,18.04,bionic,13,13.4) + $(call build_deb,ubuntu,18.04,bionic,13,13.5) touch build/ubuntu_18.04_13 build/ubuntu_18.04_14: - $(call build_deb,ubuntu,18.04,bionic,14,14.0) + $(call build_deb,ubuntu,18.04,bionic,14,14.1) touch build/ubuntu_18.04_14 -# UBUNTU 16.04 -build/ubuntu_16.04_9.5: - $(call build_deb,ubuntu,16.04,xenial,9.5,9.5.25) - touch build/ubuntu_16.04_9.5 - -build/ubuntu_16.04_9.6: - $(call build_deb,ubuntu,16.04,xenial,9.6,9.6.23) - touch build/ubuntu_16.04_9.6 - -build/ubuntu_16.04_10: - $(call build_deb,ubuntu,16.04,xenial,10,10.18) - touch build/ubuntu_16.04_10 - -build/ubuntu_16.04_11: - $(call build_deb,ubuntu,16.04,xenial,11,11.13) - touch build/ubuntu_16.04_11 - -build/ubuntu_16.04_12: - $(call build_deb,ubuntu,16.04,xenial,12,12.8) - touch build/ubuntu_16.04_12 - -build/ubuntu_16.04_13: - $(call build_deb,ubuntu,16.04,xenial,13,13.4) - touch build/ubuntu_16.04_13 - -build/ubuntu_16.04_14: - $(call build_deb,ubuntu,16.04,xenial,14,14.0) - touch build/ubuntu_16.04_14 - - -# UBUNTU 14.04 -build/ubuntu_14.04_9.5: - $(call build_deb,ubuntu,14.04,trusty,9.5,9.5.25) - touch build/ubuntu_14.04_9.5 - -build/ubuntu_14.04_9.6: - $(call build_deb,ubuntu,14.04,trusty,9.6,9.6.23) - touch build/ubuntu_14.04_9.6 - -build/ubuntu_14.04_10: - $(call build_deb,ubuntu,14.04,trusty,10,10.18) - touch build/ubuntu_14.04_10 - -build/ubuntu_14.04_11: - $(call build_deb,ubuntu,14.04,trusty,11,11.13) - touch build/ubuntu_14.04_11 - -build/ubuntu_14.04_12: - $(call build_deb,ubuntu,14.04,trusty,12,12.8) - touch build/ubuntu_14.04_12 - -build/ubuntu_14.04_13: - $(call build_deb,ubuntu,14.04,trusty,13,13.4) - touch build/ubuntu_14.04_13 - -build/ubuntu_14.04_14: - $(call build_deb,ubuntu,14.04,trusty,14,14.0) - touch build/ubuntu_14.04_14 diff --git a/packaging/pkg/scripts/alt.sh b/packaging/pkg/scripts/alt.sh index ae3c713fa..7c3971d6a 100755 --- a/packaging/pkg/scripts/alt.sh +++ b/packaging/pkg/scripts/alt.sh @@ -49,11 +49,11 @@ else cd /root/rpmbuild/SOURCES/pgpro PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') - if [[ ${PBK_EDITION} == 'std' ]] ; then - git checkout "PGPRO${PGPRO_TOC}_1" - else - git checkout "PGPROEE${PGPRO_TOC}_1" - fi + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi rm -rf .git cd /root/rpmbuild/SOURCES/ @@ -86,7 +86,7 @@ else sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.alt.forks.spec if [ ${PG_VERSION} != '9.6' ]; then - sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup.alt.forks.spec + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup.alt.forks.spec fi fi @@ -106,7 +106,6 @@ fi apt-get install -y flex libldap-devel libpam-devel libreadline-devel libssl-devel if [[ ${PBK_EDITION} == '' ]] ; then - # build pg_probackup rpmbuild -bs pg_probackup.alt.spec rpmbuild -ba pg_probackup.alt.spec #2>&1 | tee -ai /app/out/build.log diff --git a/packaging/pkg/scripts/rpm.sh b/packaging/pkg/scripts/rpm.sh index ffd681b75..d03915c20 100755 --- a/packaging/pkg/scripts/rpm.sh +++ b/packaging/pkg/scripts/rpm.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Copyright Notice: -# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ # Distributed under Apache License 2.0 # Распространяется по лицензии Apache 2.0 @@ -18,18 +18,16 @@ set -o pipefail # fix https://github.com/moby/moby/issues/23137 ulimit -n 1024 -# THere is no std/ent packages for PG 9.5 -if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then - exit 0 -fi - -if [ -f /etc/centos-release ] ; then +if [ ${DISTRIB} = 'centos' ] ; then sed -i 's|^baseurl=http://|baseurl=https://|g' /etc/yum.repos.d/*.repo yum update -y fi # PACKAGES NEEDED yum install -y git wget bzip2 rpm-build +if [ ${DISTRIB} = 'oraclelinux' -a ${DISTRIB_VERSION} = '8' -a -n ${PBK_EDITION} ] ; then + yum install -y bison flex +fi mkdir /root/build cd /root/build diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo index fcef58a9c..d26b058cd 100644 --- a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo @@ -1,5 +1,5 @@ [pg_probackup-forks] -name=PG_PROBACKUP @SHORT_CODENAME@ packages for PostgresPro Standart and Enterprise - $basearch +name=PG_PROBACKUP @SHORT_CODENAME@ packages for PostgresPro Standard and Enterprise - $basearch baseurl=https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/@DISTRIB@-$releasever-$basearch enabled=1 gpgcheck=1 diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec index fd4a99f2c..47adb250f 100644 --- a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec @@ -16,7 +16,7 @@ BuildArch: noarch %description This package contains yum configuration for @SHORT_CODENAME@, and also the GPG key -for pg_probackup RPMs for PostgresPro Standart and Enterprise. +for pg_probackup RPMs for PostgresPro Standard and Enterprise. %prep %setup -q -c -T diff --git a/packaging/repo/reprepro-conf/changelog.script b/packaging/repo/reprepro-conf/changelog.script new file mode 100755 index 000000000..4ff1f1787 --- /dev/null +++ b/packaging/repo/reprepro-conf/changelog.script @@ -0,0 +1,246 @@ +#!/bin/sh +# This is an example script that can be hooked into reprepro +# to either generate a hierachy like packages.debian.org/changelogs/ +# or to generate changelog files in the "third party sites" +# location apt-get changelogs looks if it is not found in +# Apt::Changelogs::Server. +# +# All you have to do is to: +# - copy it into you conf/ directory, +# - if you want "third party site" style changelogs, edit the +# CHANGELOGDIR variable below, +# and +# - add the following to any distribution in conf/distributions +# you want to have changelogs and copyright files extracted: +#Log: +# --type=dsc changelogs.example +# (note the space at the beginning of the second line). +# This will cause this script to extract changelogs for all +# newly added source packages. (To generate them for already +# existing packages, call "reprepro rerunnotifiers"). + +# DEPENDENCIES: dpkg >= 1.13.9 + +if test "x${REPREPRO_OUT_DIR:+set}" = xset ; then + # Note: due to cd, REPREPRO_*_DIR will no longer + # be usable. And only things relative to outdir will work... + cd "${REPREPRO_OUT_DIR}" || exit 1 +else + # this will also trigger if reprepro < 3.5.1 is used, + # in that case replace this with a manual cd to the + # correct directory... + cat "changelog.example needs to be run by reprepro!" >&2 + exit 1 +fi + +# CHANGELOGDIR set means generate full hierachy +# (clients need to set Apt::Changelogs::Server to use that) +#CHANGELOGDIR=changelogs + +# CHANGELOGDIR empty means generate changelog (and only changelog) files +# in the new "third party site" place apt-get changelog is using as fallback: +#CHANGELOGDIR= + +# Set to avoid using some predefined TMPDIR or even /tmp as +# tempdir: + +# TMPDIR=/var/cache/whateveryoucreated + +if test -z "$CHANGELOGDIR" ; then +addsource() { + DSCFILE="$1" + CANONDSCFILE="$(readlink --canonicalize "$DSCFILE")" + CHANGELOGFILE="${DSCFILE%.dsc}.changelog" + BASEDIR="$(dirname "$CHANGELOGFILE")" + if ! [ -f "$CHANGELOGFILE" ] ; then + EXTRACTDIR="$(mktemp -d)" + (cd -- "$EXTRACTDIR" && dpkg-source --no-copy -x "$CANONDSCFILE" > /dev/null) + install --mode=644 -- "$EXTRACTDIR"/*/debian/changelog "$CHANGELOGFILE" + chmod -R u+rwX -- "$EXTRACTDIR" + rm -r -- "$EXTRACTDIR" + fi + if [ -L "$BASEDIR"/current."$CODENAME" ] ; then + # should not be there, just to be sure + rm -f -- "$BASEDIR"/current."$CODENAME" + fi + # mark this as needed by this distribution + ln -s -- "$(basename "$CHANGELOGFILE")" "$BASEDIR/current.$CODENAME" + JUSTADDED="$CHANGELOGFILE" +} +delsource() { + DSCFILE="$1" + CHANGELOGFILE="${DSCFILE%.dsc}.changelog" + BASEDIR="$(dirname "$CHANGELOGFILE")" + BASENAME="$(basename "$CHANGELOGFILE")" + if [ "x$JUSTADDED" = "x$CHANGELOGFILE" ] ; then + exit 0 + fi +# echo "delete, basedir=$BASEDIR changelog=$CHANGELOGFILE, dscfile=$DSCFILE, " + if [ "x$(readlink "$BASEDIR/current.$CODENAME")" = "x$BASENAME" ] ; then + rm -- "$BASEDIR/current.$CODENAME" + fi + NEEDED=0 + for c in "$BASEDIR"/current.* ; do + if [ "x$(readlink -- "$c")" = "x$BASENAME" ] ; then + NEEDED=1 + fi + done + if [ "$NEEDED" -eq 0 -a -f "$CHANGELOGFILE" ] ; then + rm -r -- "$CHANGELOGFILE" + # to remove the directory if now empty + rmdir --ignore-fail-on-non-empty -- "$BASEDIR" + fi +} + +else # "$CHANGELOGDIR" set: + +addsource() { + DSCFILE="$1" + CANONDSCFILE="$(readlink --canonicalize "$DSCFILE")" + TARGETDIR="${CHANGELOGDIR}/${DSCFILE%.dsc}" + SUBDIR="$(basename $TARGETDIR)" + BASEDIR="$(dirname $TARGETDIR)" + if ! [ -d "$TARGETDIR" ] ; then + echo "extract $CANONDSCFILE information to $TARGETDIR" + mkdir -p -- "$TARGETDIR" + EXTRACTDIR="$(mktemp -d)" + (cd -- "$EXTRACTDIR" && dpkg-source --no-copy -x "$CANONDSCFILE" > /dev/null) + install --mode=644 -- "$EXTRACTDIR"/*/debian/copyright "$TARGETDIR/copyright" + install --mode=644 -- "$EXTRACTDIR"/*/debian/changelog "$TARGETDIR/changelog" + chmod -R u+rwX -- "$EXTRACTDIR" + rm -r -- "$EXTRACTDIR" + fi + if [ -L "$BASEDIR"/current."$CODENAME" ] ; then + # should not be there, just to be sure + rm -f -- "$BASEDIR"/current."$CODENAME" + fi + # mark this as needed by this distribution + ln -s -- "$SUBDIR" "$BASEDIR/current.$CODENAME" + JUSTADDED="$TARGETDIR" +} +delsource() { + DSCFILE="$1" + TARGETDIR="${CHANGELOGDIR}/${DSCFILE%.dsc}" + SUBDIR="$(basename $TARGETDIR)" + BASEDIR="$(dirname $TARGETDIR)" + if [ "x$JUSTADDED" = "x$TARGETDIR" ] ; then + exit 0 + fi +# echo "delete, basedir=$BASEDIR targetdir=$TARGETDIR, dscfile=$DSCFILE, " + if [ "x$(readlink "$BASEDIR/current.$CODENAME")" = "x$SUBDIR" ] ; then + rm -- "$BASEDIR/current.$CODENAME" + fi + NEEDED=0 + for c in "$BASEDIR"/current.* ; do + if [ "x$(readlink -- "$c")" = "x$SUBDIR" ] ; then + NEEDED=1 + fi + done + if [ "$NEEDED" -eq 0 -a -d "$TARGETDIR" ] ; then + rm -r -- "$TARGETDIR" + # to remove the directory if now empty + rmdir --ignore-fail-on-non-empty -- "$BASEDIR" + fi +} +fi # CHANGELOGDIR + +ACTION="$1" +CODENAME="$2" +PACKAGETYPE="$3" +if [ "x$PACKAGETYPE" != "xdsc" ] ; then +# the --type=dsc should cause this to never happen, but better safe than sorry. + exit 1 +fi +COMPONENT="$4" +ARCHITECTURE="$5" +if [ "x$ARCHITECTURE" != "xsource" ] ; then + exit 1 +fi +NAME="$6" +shift 6 +JUSTADDED="" +if [ "x$ACTION" = "xadd" -o "x$ACTION" = "xinfo" ] ; then + VERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + addsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +elif [ "x$ACTION" = "xremove" ] ; then + OLDVERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + delsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +elif [ "x$ACTION" = "xreplace" ] ; then + VERSION="$1" + shift + OLDVERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 -a "x$1" != "x--" ] ; do + case "$1" in + *.dsc) + addsource "$1" + ;; + esac + shift + done + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + delsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +fi + +exit 0 +# Copyright 2007,2008,2012 Bernhard R. Link +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA \ No newline at end of file diff --git a/packaging/repo/reprepro-conf/distributions b/packaging/repo/reprepro-conf/distributions new file mode 100644 index 000000000..7dce7e6d0 --- /dev/null +++ b/packaging/repo/reprepro-conf/distributions @@ -0,0 +1,179 @@ +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: squeeze +Architectures: amd64 i386 source +Components: main-squeeze +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: wheezy +Architectures: amd64 i386 source +Components: main-wheezy +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: jessie +Architectures: amd64 i386 source +Components: main-jessie +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: bullseye +Architectures: amd64 i386 source +Components: main-bullseye +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: wily +Architectures: amd64 i386 source +Components: main-wily +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: precise +Architectures: amd64 i386 source +Components: main-precise +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: vivid +Architectures: amd64 i386 source +Components: main-vivid +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: trusty +Architectures: amd64 i386 source +Components: main-trusty +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: lucid +Architectures: amd64 i386 source +Components: main-lucid +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: cosmic +Architectures: amd64 i386 source +Components: main-cosmic +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: xenial +Architectures: amd64 i386 source +Components: main-xenial +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: yakkety +Architectures: amd64 i386 source +Components: main-yakkety +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: zesty +Architectures: amd64 i386 source +Components: main-zesty +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: stretch +Architectures: amd64 i386 source +Components: main-stretch +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: buster +Architectures: amd64 i386 source +Components: main-buster +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: artful +Architectures: amd64 i386 source +Components: main-artful +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: bionic +Architectures: amd64 i386 source +Components: main-bionic +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: focal +Architectures: amd64 i386 source +Components: main-focal +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script diff --git a/packaging/repo/rpm-conf/rpmmacros b/packaging/repo/rpm-conf/rpmmacros new file mode 100644 index 000000000..e00a76d68 --- /dev/null +++ b/packaging/repo/rpm-conf/rpmmacros @@ -0,0 +1,5 @@ +%_signature gpg +%_gpg_path /root/.gnupg +%_gpg_name PostgreSQL Professional +%_gpgbin /usr/bin/gpg +%_gpg_check_password_cmd /bin/true diff --git a/packaging/repo/scripts/deb.sh b/packaging/repo/scripts/deb.sh index 6515e6b42..31416972d 100755 --- a/packaging/repo/scripts/deb.sh +++ b/packaging/repo/scripts/deb.sh @@ -20,7 +20,7 @@ cd $INPUT_DIR export DEB_DIR=$OUT_DIR/deb export KEYS_DIR=$OUT_DIR/keys -export CONF=/app/repo/${PBK_PKG_REPO}/conf +export CONF=/app/repo/reprepro-conf mkdir -p "$KEYS_DIR" cp -av /app/repo/${PBK_PKG_REPO}/gnupg /root/.gnupg @@ -29,7 +29,7 @@ echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt mkdir -p $DEB_DIR cd $DEB_DIR -cp -av $CONF ./ +cp -av $CONF ./conf # make remove-debpkg tool echo -n "#!" > remove-debpkg diff --git a/packaging/repo/scripts/rpm.sh b/packaging/repo/scripts/rpm.sh index d4e621c3e..524789cbf 100755 --- a/packaging/repo/scripts/rpm.sh +++ b/packaging/repo/scripts/rpm.sh @@ -20,24 +20,17 @@ export KEYS_DIR=$OUT_DIR/keys mkdir -p "$KEYS_DIR" rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP chmod 755 $KEYS_DIR -chmod +x /app/repo/$PBK_PKG_REPO/autosign.sh echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt cd $INPUT_DIR -cp -arv /app/repo/$PBK_PKG_REPO/rpmmacros /root/.rpmmacros +cp -arv /app/repo/rpm-conf/rpmmacros /root/.rpmmacros cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg chmod -R 0600 /root/.gnupg chown -R root:root /root/.gnupg for pkg in $(ls ${INPUT_DIR}); do for pkg_full_version in $(ls ${INPUT_DIR}/$pkg); do - - # THere is no std/ent packages for PG 9.5 - if [[ ${pkg} == 'pg_probackup-std-9.5' ]] || [[ ${pkg} == 'pg_probackup-ent-9.5' ]] ; then - continue; - fi - if [[ ${PBK_EDITION} == '' ]] ; then cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ $KEYS_DIR/pg_probackup-repo-$DISTRIB.noarch.rpm diff --git a/packaging/repo/scripts/suse.sh b/packaging/repo/scripts/suse.sh index 7253df700..c85e0ff10 100755 --- a/packaging/repo/scripts/suse.sh +++ b/packaging/repo/scripts/suse.sh @@ -33,7 +33,7 @@ rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt -cp -arv /app/repo/$PBK_PKG_REPO/rpmmacros /root/.rpmmacros +cp -arv /app/repo/rpm-conf/rpmmacros /root/.rpmmacros cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg chmod -R 0600 /root/.gnupg diff --git a/packaging/test/Makefile.alt b/packaging/test/Makefile.alt index 3c1899cb9..bd00cef7f 100644 --- a/packaging/test/Makefile.alt +++ b/packaging/test/Makefile.alt @@ -1,20 +1,50 @@ # ALT 9 build/test_alt_9_9.6: - $(call test_alt,alt,9,,9.6,9.6.21) + $(call test_alt,alt,9,,9.6,9.6.24) touch build/test_alt_9_9.6 build/test_alt_9_10: - $(call test_alt,alt,9,,10,10.17) + $(call test_alt,alt,9,,10,10.19) touch build/test_alt_9_10 build/test_alt_9_11: - $(call test_alt,alt,9,,11,11.11) + $(call test_alt,alt,9,,11,11.14) touch build/test_alt_9_11 build/test_alt_9_12: - $(call test_alt,alt,9,,12,12.6) + $(call test_alt,alt,9,,12,12.9) touch build/test_alt_9_12 build/test_alt_9_13: - $(call test_alt,alt,9,,13,13.2) + $(call test_alt,alt,9,,13,13.5) touch build/test_alt_9_13 + +build/test_alt_9_14: + $(call test_alt,alt,9,,14,14.1) + touch build/test_alt_9_14 + +# ALT 8 +build/test_alt_8_9.6: + $(call test_alt,alt,8,,9.6,9.6.24) + touch build/test_alt_8_9.6 + +build/test_alt_8_10: + $(call test_alt,alt,8,,10,10.19) + touch build/test_alt_8_10 + +build/test_alt_8_11: + $(call test_alt,alt,8,,11,11.14) + touch build/test_alt_8_11 + +build/test_alt_8_12: + $(call test_alt,alt,8,,12,12.9) + touch build/test_alt_8_12 + +build/test_alt_8_13: + $(call test_alt,alt,8,,13,13.5) + touch build/test_alt_8_13 + +build/test_alt_8_14: + $(call test_alt,alt,8,,14,14.1) + touch build/test_alt_8_14 + diff --git a/packaging/test/Makefile.centos b/packaging/test/Makefile.centos index e3787c612..9d30a324b 100644 --- a/packaging/test/Makefile.centos +++ b/packaging/test/Makefile.centos @@ -1,41 +1,49 @@ # CENTOS 7 build/test_centos_7_9.6: - $(call test_rpm,centos,7,,9.6,9.6.21) + $(call test_rpm,centos,7,,9.6,9.6.24) touch build/test_centos_7_9.6 build/test_centos_7_10: - $(call test_rpm,centos,7,,10,10.16) + $(call test_rpm,centos,7,,10,10.19) touch build/test_centos_7_10 build/test_centos_7_11: - $(call test_rpm,centos,7,,11,11.11) + $(call test_rpm,centos,7,,11,11.14) touch build/test_centos_7_11 build/test_centos_7_12: - $(call test_rpm,centos,7,,12,12.6) + $(call test_rpm,centos,7,,12,12.9) touch build/test_centos_7_12 build/test_centos_7_13: - $(call test_rpm,centos,7,,13,13.2) + $(call test_rpm,centos,7,,13,13.5) touch build/test_centos_7_13 +build/test_centos_7_14: + $(call test_rpm,centos,7,,14,14.1) + touch build/test_centos_7_14 + # CENTOS 8 build/test_centos_8_9.6: - $(call test_rpm,centos,8,,9.6,9.6.21) + $(call test_rpm,centos,8,,9.6,9.6.24) touch build/test_centos_8_9.6 build/test_centos_8_10: - $(call test_rpm,centos,8,,10,10.16) + $(call test_rpm,centos,8,,10,10.19) touch build/test_centos_8_10 build/test_centos_8_11: - $(call test_rpm,centos,8,,11,11.11) + $(call test_rpm,centos,8,,11,11.14) touch build/test_centos_8_11 build/test_centos_8_12: - $(call test_rpm,centos,8,,12,12.6) + $(call test_rpm,centos,8,,12,12.9) touch build/test_centos_8_12 build/test_centos_8_13: - $(call test_rpm,centos,8,,13,13.2) + $(call test_rpm,centos,8,,13,13.5) touch build/test_centos_8_13 + +build/test_centos_8_14: + $(call test_rpm,centos,8,,14,14.1) + touch build/test_centos_8_14 diff --git a/packaging/test/Makefile.debian b/packaging/test/Makefile.debian index 084741069..e4d904f62 100644 --- a/packaging/test/Makefile.debian +++ b/packaging/test/Makefile.debian @@ -1,62 +1,74 @@ # DEBIAN 9 build/test_debian_9_9.6: - $(call test_deb,debian,9,stretch,9.6,9.6.21) + $(call test_deb,debian,9,stretch,9.6,9.6.24) touch build/test_debian_9_9.6 build/test_debian_9_10: - $(call test_deb,debian,9,stretch,10,10.16) + $(call test_deb,debian,9,stretch,10,10.19) touch build/test_debian_9_10 build/test_debian_9_11: - $(call test_deb,debian,9,stretch,11,11.11) + $(call test_deb,debian,9,stretch,11,11.14) touch build/test_debian_9_11 build/test_debian_9_12: - $(call test_deb,debian,9,stretch,12,12.6) + $(call test_deb,debian,9,stretch,12,12.9) touch build/test_debian_9_12 build/test_debian_9_13: - $(call test_deb,debian,9,stretch,13,13.2) + $(call test_deb,debian,9,stretch,13,13.5) touch build/test_debian_9_13 +build/test_debian_9_14: + $(call test_deb,debian,9,stretch,14,14.1) + touch build/test_debian_9_14 + # DEBIAN 10 build/test_debian_10_9.6: - $(call test_deb,debian,10,buster,9.6,9.6.21) + $(call test_deb,debian,10,buster,9.6,9.6.24) touch build/test_debian_10_9.6 build/test_debian_10_10: - $(call test_deb,debian,10,buster,10,10.16) + $(call test_deb,debian,10,buster,10,10.19) touch build/test_debian_10_10 build/test_debian_10_11: - $(call test_deb,debian,10,buster,11,11.11) + $(call test_deb,debian,10,buster,11,11.14) touch build/test_debian_10_11 build/test_debian_10_12: - $(call test_deb,debian,10,buster,12,12.6) + $(call test_deb,debian,10,buster,12,12.9) touch build/test_debian_10_12 build/test_debian_10_13: - $(call test_deb,debian,10,buster,13,13.2) + $(call test_deb,debian,10,buster,13,13.5) touch build/test_debian_10_13 +build/test_debian_10_14: + $(call test_deb,debian,10,buster,14,14.1) + touch build/test_debian_10_14 + # DEBIAN 11 build/test_debian_11_9.6: - $(call test_deb,debian,11,bullseye,9.6,9.6.21) + $(call test_deb,debian,11,bullseye,9.6,9.6.24) touch build/test_debian_11_9.6 build/test_debian_11_10: - $(call test_deb,debian,11,bullseye,10,10.16) + $(call test_deb,debian,11,bullseye,10,10.19) touch build/test_debian_11_10 build/test_debian_11_11: - $(call test_deb,debian,11,bullseye,11,11.11) + $(call test_deb,debian,11,bullseye,11,11.14) touch build/test_debian_11_11 build/test_debian_11_12: - $(call test_deb,debian,11,bullseye,12,12.6) + $(call test_deb,debian,11,bullseye,12,12.9) touch build/test_debian_11_12 build/test_debian_11_13: - $(call test_deb,debian,11,bullseye,13,13.2) + $(call test_deb,debian,11,bullseye,13,13.5) touch build/test_debian_11_13 + +build/test_debian_11_14: + $(call test_deb,debian,11,bullseye,14,14.1) + touch build/test_debian_11_14 diff --git a/packaging/test/Makefile.oraclelinux b/packaging/test/Makefile.oraclelinux index fdf44de8b..0efe6574d 100644 --- a/packaging/test/Makefile.oraclelinux +++ b/packaging/test/Makefile.oraclelinux @@ -1,41 +1,49 @@ # ORACLE LINUX 7 build/test_oraclelinux_7_9.6: - $(call test_rpm,oraclelinux,7,,9.6,9.6.21) + $(call test_rpm,oraclelinux,7,,9.6,9.6.24) touch build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10: - $(call test_rpm,oraclelinux,7,,10,10.16) + $(call test_rpm,oraclelinux,7,,10,10.19) touch build/test_oraclelinux_7_10 build/test_oraclelinux_7_11: - $(call test_rpm,oraclelinux,7,,11,11.11) + $(call test_rpm,oraclelinux,7,,11,11.14) touch build/test_oraclelinux_7_11 build/test_oraclelinux_7_12: - $(call test_rpm,oraclelinux,7,,12,12.6) + $(call test_rpm,oraclelinux,7,,12,12.9) touch build/test_oraclelinux_7_12 build/test_oraclelinux_7_13: - $(call test_rpm,oraclelinux,7,,13,13.2) + $(call test_rpm,oraclelinux,7,,13,13.5) touch build/test_oraclelinux_7_13 +build/test_oraclelinux_7_14: + $(call test_rpm,oraclelinux,7,,14,14.1) + touch build/test_oraclelinux_7_14 + # ORACLE LINUX 8 build/test_oraclelinux_8_9.6: - $(call test_rpm,oraclelinux,8,,9.6,9.6.21) + $(call test_rpm,oraclelinux,8,,9.6,9.6.24) touch build/test_oraclelinux_8_9.6 build/test_oraclelinux_8_10: - $(call test_rpm,oraclelinux,8,,10,10.16) + $(call test_rpm,oraclelinux,8,,10,10.19) touch build/test_oraclelinux_8_10 build/test_oraclelinux_8_11: - $(call test_rpm,oraclelinux,8,,11,11.11) + $(call test_rpm,oraclelinux,8,,11,11.14) touch build/test_oraclelinux_8_11 build/test_oraclelinux_8_12: - $(call test_rpm,oraclelinux,8,,12,12.6) + $(call test_rpm,oraclelinux,8,,12,12.9) touch build/test_oraclelinux_8_12 build/test_oraclelinux_8_13: - $(call test_rpm,oraclelinux,8,,13,13.2) + $(call test_rpm,oraclelinux,8,,13,13.5) touch build/test_oraclelinux_8_13 + +build/test_oraclelinux_8_14: + $(call test_rpm,oraclelinux,8,,14,14.1) + touch build/test_oraclelinux_8_14 diff --git a/packaging/test/Makefile.rhel b/packaging/test/Makefile.rhel index 3169d11c9..3b26c8942 100644 --- a/packaging/test/Makefile.rhel +++ b/packaging/test/Makefile.rhel @@ -1,41 +1,49 @@ # RHEL 7 build/test_rhel_7_9.6: - $(call test_rpm,rhel,7,7Server,9.6,9.6.21) + $(call test_rpm,rhel,7,7Server,9.6,9.6.24) touch build/test_rhel_7_9.6 build/test_rhel_7_10: - $(call test_rpm,rhel,7,7Server,10,10.16) + $(call test_rpm,rhel,7,7Server,10,10.19) touch build/test_rhel_7_10 build/test_rhel_7_11: - $(call test_rpm,rhel,7,7Server,11,11.11) + $(call test_rpm,rhel,7,7Server,11,11.14) touch build/test_rhel_7_11 build/test_rhel_7_12: - $(call test_rpm,rhel,7,7Server,12,12.6) + $(call test_rpm,rhel,7,7Server,12,12.9) touch build/test_rhel_7_12 build/test_rhel_7_13: - $(call test_rpm,rhel,7,7Server,13,13.2) + $(call test_rpm,rhel,7,7Server,13,13.5) touch build/test_rhel_7_13 +build/test_rhel_7_14: + $(call test_rpm,rhel,7,7Server,14,14.1) + touch build/test_rhel_7_14 + # RHEL 8 build/test_rhel_8_9.6: - $(call test_rpm,rhel,8,8Server,9.6,9.6.21) + $(call test_rpm,rhel,8,8Server,9.6,9.6.24) touch build/test_rhel_8_9.6 build/test_rhel_8_10: - $(call test_rpm,rhel,8,8Server,10,10.16) + $(call test_rpm,rhel,8,8Server,10,10.19) touch build/test_rhel_8_10 build/test_rhel_8_11: - $(call test_rpm,rhel,8,8Server,11,11.11) + $(call test_rpm,rhel,8,8Server,11,11.14) touch build/test_rhel_8_11 build/test_rhel_8_12: - $(call test_rpm,rhel,8,8Server,12,12.6) + $(call test_rpm,rhel,8,8Server,12,12.9) touch build/test_rhel_8_12 build/test_rhel_8_13: - $(call test_rpm,rhel,8,8Server,13,13.2) + $(call test_rpm,rhel,8,8Server,13,13.5) touch build/test_rhel_8_13 + +build/test_rhel_8_14: + $(call test_rpm,rhel,8,8Server,14,14.1) + touch build/test_rhel_8_14 diff --git a/packaging/test/Makefile.suse b/packaging/test/Makefile.suse index 9257bdbfd..19e8d52d8 100644 --- a/packaging/test/Makefile.suse +++ b/packaging/test/Makefile.suse @@ -1,41 +1,49 @@ # Suse 15.1 build/test_suse_15.1_9.6: - $(call test_suse,suse,15.1,,9.6,9.6.21) + $(call test_suse,suse,15.1,,9.6,9.6.24) touch build/test_suse_15.1_9.6 build/test_suse_15.1_10: - $(call test_suse,suse,15.1,,10,10.16) + $(call test_suse,suse,15.1,,10,10.19) touch build/test_suse_15.1_10 build/test_suse_15.1_11: - $(call test_suse,suse,15.1,,11,11.11) + $(call test_suse,suse,15.1,,11,11.14) touch build/test_suse_15.1_11 build/test_suse_15.1_12: - $(call test_suse,suse,15.1,,12,12.6) + $(call test_suse,suse,15.1,,12,12.9) touch build/test_suse_15.1_12 build/test_suse_15.1_13: - $(call test_suse,suse,15.1,,13,13.2) + $(call test_suse,suse,15.1,,13,13.5) touch build/test_suse_15.1_13 +build/test_suse_15.1_14: + $(call test_suse,suse,15.1,,14,14.1) + touch build/test_suse_15.1_14 + # Suse 15.2 build/test_suse_15.2_9.6: - $(call test_suse,suse,15.2,,9.6,9.6.21) + $(call test_suse,suse,15.2,,9.6,9.6.24) touch build/test_suse_15.2_9.6 build/test_suse_15.2_10: - $(call test_suse,suse,15.2,,10,10.16) + $(call test_suse,suse,15.2,,10,10.19) touch build/test_suse_15.2_10 build/test_suse_15.2_11: - $(call test_suse,suse,15.2,,11,11.11) + $(call test_suse,suse,15.2,,11,11.14) touch build/test_suse_15.2_11 build/test_suse_15.2_12: - $(call test_suse,suse,15.2,,12,12.6) + $(call test_suse,suse,15.2,,12,12.9) touch build/test_suse_15.2_12 build/test_suse_15.2_13: - $(call test_suse,suse,15.2,,13,13.2) + $(call test_suse,suse,15.2,,13,13.5) touch build/test_suse_15.2_13 + +build/test_suse_15.2_14: + $(call test_suse,suse,15.2,,14,14.1) + touch build/test_suse_15.2_14 diff --git a/packaging/test/Makefile.ubuntu b/packaging/test/Makefile.ubuntu index 9e201a30b..86a257b91 100644 --- a/packaging/test/Makefile.ubuntu +++ b/packaging/test/Makefile.ubuntu @@ -1,62 +1,49 @@ -# UBUNTU 16.04 -build/test_ubuntu_16.04_9.6: - $(call test_deb,ubuntu,16.04,xenial,9.6,9.6.21) - touch build/test_ubuntu_16.04_9.6 - -build/test_ubuntu_16.04_10: - $(call test_deb,ubuntu,16.04,xenial,10,10.16) - touch build/test_ubuntu_16.04_10 - -build/test_ubuntu_16.04_11: - $(call test_deb,ubuntu,16.04,xenial,11,11.11) - touch build/test_ubuntu_16.04_11 - -build/test_ubuntu_16.04_12: - $(call test_deb,ubuntu,16.04,xenial,12,12.6) - touch build/test_ubuntu_16.04_12 - -build/test_ubuntu_16.04_13: - $(call test_deb,ubuntu,16.04,xenial,13,13.2) - touch build/test_ubuntu_16.04_13 - # UBUNTU 18.04 build/test_ubuntu_18.04_9.6: - $(call test_deb,ubuntu,18.04,bionic,9.6,9.6.21) + $(call test_deb,ubuntu,18.04,bionic,9.6,9.6.24) touch build/test_ubuntu_18.04_9.6 build/test_ubuntu_18.04_10: - $(call test_deb,ubuntu,18.04,bionic,10,10.16) + $(call test_deb,ubuntu,18.04,bionic,10,10.19) touch build/test_ubuntu_18.04_10 build/test_ubuntu_18.04_11: - $(call test_deb,ubuntu,18.04,bionic,11,11.11) + $(call test_deb,ubuntu,18.04,bionic,11,11.14) touch build/test_ubuntu_18.04_11 build/test_ubuntu_18.04_12: - $(call test_deb,ubuntu,18.04,bionic,12,12.6) + $(call test_deb,ubuntu,18.04,bionic,12,12.9) touch build/test_ubuntu_18.04_12 build/test_ubuntu_18.04_13: - $(call test_deb,ubuntu,18.04,bionic,13,13.2) + $(call test_deb,ubuntu,18.04,bionic,13,13.5) touch build/test_ubuntu_18.04_13 +build/test_ubuntu_18.04_14: + $(call test_deb,ubuntu,18.04,bionic,14,14.1) + touch build/test_ubuntu_18.04_14 + # UBUNTU 20.04 build/test_ubuntu_20.04_9.6: - $(call test_deb,ubuntu,20.04,focal,9.6,9.6.21) + $(call test_deb,ubuntu,20.04,focal,9.6,9.6.24) touch build/test_ubuntu_20.04_9.6 build/test_ubuntu_20.04_10: - $(call test_deb,ubuntu,20.04,focal,10,10.16) + $(call test_deb,ubuntu,20.04,focal,10,10.19) touch build/test_ubuntu_20.04_10 build/test_ubuntu_20.04_11: - $(call test_deb,ubuntu,20.04,focal,11,11.11) + $(call test_deb,ubuntu,20.04,focal,11,11.14) touch build/test_ubuntu_20.04_11 build/test_ubuntu_20.04_12: - $(call test_deb,ubuntu,20.04,focal,12,12.6) + $(call test_deb,ubuntu,20.04,focal,12,12.9) touch build/test_ubuntu_20.04_12 build/test_ubuntu_20.04_13: - $(call test_deb,ubuntu,20.04,focal,13,13.2) + $(call test_deb,ubuntu,20.04,focal,13,13.5) touch build/test_ubuntu_20.04_13 + +build/test_ubuntu_20.04_14: + $(call test_deb,ubuntu,20.04,focal,14,14.1) + touch build/test_ubuntu_20.04_14 diff --git a/packaging/test/scripts/deb.sh b/packaging/test/scripts/deb.sh index d7b957192..fca9a23d8 100755 --- a/packaging/test/scripts/deb.sh +++ b/packaging/test/scripts/deb.sh @@ -17,8 +17,14 @@ PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') export DEBIAN_FRONTEND=noninteractive echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -#apt-get -qq --allow-releaseinfo-change update -apt-get -qq update +if [ ${DISTRIB} = 'ubuntu' -a ${CODENAME} = 'xenial' ] ; then + apt-get -qq update +elif [ ${DISTRIB} = 'debian' -a ${CODENAME} = 'stretch' ] ; then + apt-get -qq update +else + apt-get -qq --allow-releaseinfo-change update +fi + apt-get -qq install -y wget nginx gnupg lsb-release #apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps diff --git a/packaging/test/scripts/deb_forks.sh b/packaging/test/scripts/deb_forks.sh index 5175f38db..e05695608 100755 --- a/packaging/test/scripts/deb_forks.sh +++ b/packaging/test/scripts/deb_forks.sh @@ -31,7 +31,11 @@ echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections #printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list #fi -apt-get -qq update +if [ ${DISTRIB} = 'debian' -a ${CODENAME} = 'stretch' ] ; then + apt-get -qq update +else + apt-get -qq --allow-releaseinfo-change update +fi apt-get -qq install -y wget nginx gnupg lsb-release apt-transport-https #apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh index 320d459f6..92804a7f4 100755 --- a/packaging/test/scripts/rpm.sh +++ b/packaging/test/scripts/rpm.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Copyright Notice: -# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ # Distributed under Apache License 2.0 # Распространяется по лицензии Apache 2.0 @@ -13,23 +13,19 @@ ulimit -n 1024 PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') -yum update -y +if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then + # update of rpm package is broken in rhel-7 (26/12/2022) + yum update -y +fi # yum upgrade -y || echo 'some packages in docker failed to upgrade' # yum install -y sudo -if [ ${DISTRIB} == 'rhel' ] && [ ${PG_TOG} == '13' ]; then # no packages for PG13 on PGDG - exit 0 -fi - -#if [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ] && [ ${PG_TOG} == '13' ]; then # no packages for PG13 on PGDG -# exit 0 -#fi -if [ ${DISTRIB_VERSION} == '6' ]; then +if [ ${DISTRIB_VERSION} = '6' ]; then yum install -y https://nginx.org/packages/rhel/6/x86_64/RPMS/nginx-1.8.1-1.el6.ngx.x86_64.rpm -elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then - yum install -y nginx -elif [ ${DISTRIB_VERSION} == '7' ]; then +elif [ ${DISTRIB_VERSION} = '7' ]; then yum install -y https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.8.1-1.el7.ngx.x86_64.rpm +elif [ ${DISTRIB_VERSION} = '8' -a \( ${DISTRIB} = 'rhel' -o ${DISTRIB} = 'oraclelinux' \) ]; then + yum install -y nginx else yum install epel-release -y yum install -y nginx diff --git a/packaging/test/scripts/rpm_forks.sh b/packaging/test/scripts/rpm_forks.sh index 8596f6656..0d72040ed 100755 --- a/packaging/test/scripts/rpm_forks.sh +++ b/packaging/test/scripts/rpm_forks.sh @@ -13,7 +13,12 @@ ulimit -n 1024 PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') -if [ ${PBK_PBK_EDITION} == 'ent' ]; then +if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then + # update of rpm package is broken in rhel-7 (26/12/2022) + yum update -y +fi + +if [ ${PBK_EDITION} == 'ent' ]; then exit 0 fi From 1790a990d2f98e17e891971b1ef957a85319dd1b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 11 Feb 2022 16:58:23 +0300 Subject: [PATCH 1827/2107] fix windows builds (add forgotten catchup.c into gen_probackup_project.pl) --- gen_probackup_project.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index abc779a40..c24db1228 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -155,6 +155,7 @@ sub build_pgprobackup 'archive.c', 'backup.c', 'catalog.c', + 'catchup.c', 'configure.c', 'data.c', 'delete.c', From 98f77d24fd9d30531e5788d8dea1b16e188ac9d2 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Sat, 12 Feb 2022 21:35:03 +0300 Subject: [PATCH 1828/2107] =?UTF-8?q?[PGPRO-5691]=20ptrack-2.3:=20move=20m?= =?UTF-8?q?mapped=20ptrack=20map=20into=20shared=20postgres=E2=80=A6=20(#4?= =?UTF-8?q?71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [PGPRO-5691] ptrack-2.3: move mmapped ptrack map into shared postgres memory In ptrack-2.3 ptrack.map.mmap will be removed and 'incorrect checksum' error will not be fatal (https://github.com/postgrespro/ptrack/pull/19) * added test_corrupt_ptrack_map test compatibility with both version 2.2 and version 2.3 of ptrack --- tests/ptrack.py | 63 ++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index a3109da48..5878f0700 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4314,6 +4314,8 @@ def test_corrupt_ptrack_map(self): "postgres", "CREATE EXTENSION ptrack") + ptrack_version = self.get_ptrack_version(node) + # Create table node.safe_psql( "postgres", @@ -4338,48 +4340,55 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) ptrack_map = os.path.join(node.data_dir, 'global', 'ptrack.map') - ptrack_map_mmap = os.path.join(node.data_dir, 'global', 'ptrack.map.mmap') - # Let`s do index corruption. ptrack.map, ptrack.map.mmap + # Let`s do index corruption. ptrack.map with open(ptrack_map, "rb+", 0) as f: f.seek(42) f.write(b"blablahblahs") f.flush() f.close - with open(ptrack_map_mmap, "rb+", 0) as f: - f.seek(42) - f.write(b"blablahblahs") - f.flush() - f.close - # os.remove(os.path.join(node.logs_dir, node.pg_log_name)) - try: + if self.verbose: + print('Ptrack version:', ptrack_version) + if ptrack_version >= self.version_to_num("2.3"): node.slow_start() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack.map is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except StartNodeException as e: + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertIn( - 'Cannot start node', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: ptrack read map: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() + node.stop(['-D', node.data_dir]) + else: + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack.map is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() - self.assertIn( - 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), - log_content) + self.assertIn( + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) self.set_auto_conf(node, {'ptrack.map_size': '0'}) - node.slow_start() try: From 6470693d2a42b980062a95490fba3c7607655c9b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 17 Feb 2022 01:24:08 +0300 Subject: [PATCH 1829/2107] [PGPRO-5612] Support for checkunique parameter of amcheck.bt_index_check() function (PR #456) Co-authored-by: Elena Indrupskaya --- doc/pgprobackup.xml | 35 ++++++-- src/checkdb.c | 124 +++++++++++++++++++------- src/help.c | 6 +- src/pg_probackup.c | 12 +++ src/pg_probackup.h | 1 + tests/checkdb.py | 155 ++++++++++++++++++++++++++++++++- tests/expected/option_help.out | 2 +- 7 files changed, 291 insertions(+), 44 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 76ec2cd76..76333b116 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4176,7 +4176,7 @@ pg_probackup restore -B backup_dir --instance backup_dir] [--instance instance_name] [-D data_dir] [--help] [-j num_threads] [--progress] -[--skip-block-validation] [--amcheck] [--heapallindexed] +[--skip-block-validation] [--amcheck [--checkunique] [--heapallindexed]] [connection_options] [logging_options] @@ -4195,17 +4195,24 @@ pg_probackup checkdb extension or the amcheck_next extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. + Additional options and + are effective depending on the version of amcheck installed. - + - Skip validation of data files. You can use this flag only - together with the flag, so that only logical - verification of indexes is performed. + Verifies unique constraints during logical verification of indexes. + You can use this flag only together with the flag when + the amcheck extension is + installed in the database. + + + This verification is only possible if it is supported by the version of the + amcheck extension you are using. @@ -4219,12 +4226,24 @@ pg_probackup checkdb flag. - This check is only possible if you are using the - amcheck extension of version 2.0 or higher, or - the amcheck_next extension of any version. + This check is only possible if it is supported by the version of the + amcheck extension you are using or + if the amcheck_next extension is used instead. + + + + + + + + + Skip validation of data files. You can use this flag only + together with the flag, so that only logical + verification of indexes is performed. + diff --git a/src/checkdb.c b/src/checkdb.c index e3f2df538..177fc3cc7 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -83,6 +83,7 @@ typedef struct pg_indexEntry char *name; char *namespace; bool heapallindexed_is_supported; + bool checkunique_is_supported; /* schema where amcheck extension is located */ char *amcheck_nspname; /* lock for synchronization of parallel threads */ @@ -351,10 +352,14 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, { PGresult *res; char *amcheck_nspname = NULL; + char *amcheck_extname = NULL; + char *amcheck_extversion = NULL; int i; bool heapallindexed_is_supported = false; + bool checkunique_is_supported = false; parray *index_list = NULL; + /* Check amcheck extension version */ res = pgut_execute(db_conn, "SELECT " "extname, nspname, extversion " "FROM pg_catalog.pg_namespace n " @@ -379,24 +384,68 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, return NULL; } + amcheck_extname = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); + strcpy(amcheck_extname, PQgetvalue(res, 0, 0)); amcheck_nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); strcpy(amcheck_nspname, PQgetvalue(res, 0, 1)); + amcheck_extversion = pgut_malloc(strlen(PQgetvalue(res, 0, 2)) + 1); + strcpy(amcheck_extversion, PQgetvalue(res, 0, 2)); + PQclear(res); /* heapallindexed_is_supported is database specific */ - if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && - strcmp(PQgetvalue(res, 0, 2), "1") != 0) + /* TODO this is wrong check, heapallindexed supported also in 1.1.1, 1.2 and 1.2.1... */ + if (strcmp(amcheck_extversion, "1.0") != 0 && + strcmp(amcheck_extversion, "1") != 0) heapallindexed_is_supported = true; elog(INFO, "Amchecking database '%s' using extension '%s' " "version %s from schema '%s'", - dbname, PQgetvalue(res, 0, 0), - PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + dbname, amcheck_extname, + amcheck_extversion, amcheck_nspname); if (!heapallindexed_is_supported && heapallindexed) elog(WARNING, "Extension '%s' version %s in schema '%s'" "do not support 'heapallindexed' option", - PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), - PQgetvalue(res, 0, 1)); + amcheck_extname, amcheck_extversion, + amcheck_nspname); + +#ifndef PGPRO_EE + /* + * Will support when the vanilla patch will commited https://commitfest.postgresql.org/32/2976/ + */ + checkunique_is_supported = false; +#else + /* + * Check bt_index_check function signature to determine support of checkunique parameter + * This can't be exactly checked by checking extension version, + * For example, 1.1.1 and 1.2.1 supports this parameter, but 1.2 doesn't (PGPROEE-12.4.1) + */ + res = pgut_execute(db_conn, "SELECT " + " oid " + "FROM pg_catalog.pg_proc " + "WHERE " + " pronamespace = $1::regnamespace " + "AND proname = 'bt_index_check' " + "AND 'checkunique' = ANY(proargnames) " + "AND (pg_catalog.string_to_array(proargtypes::text, ' ')::regtype[])[pg_catalog.array_position(proargnames, 'checkunique')] = 'bool'::regtype", + 1, (const char **) &amcheck_nspname); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(ERROR, "Cannot check 'checkunique' option is supported in bt_index_check function %s: %s", + dbname, PQerrorMessage(db_conn)); + } + + checkunique_is_supported = PQntuples(res) >= 1; + PQclear(res); +#endif + + if (!checkunique_is_supported && checkunique) + elog(WARNING, "Extension '%s' version %s in schema '%s' " + "do not support 'checkunique' parameter", + amcheck_extname, amcheck_extversion, + amcheck_nspname); /* * In order to avoid duplicates, select global indexes @@ -453,6 +502,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, strcpy(ind->namespace, namespace); /* enough buffer size guaranteed */ ind->heapallindexed_is_supported = heapallindexed_is_supported; + ind->checkunique_is_supported = checkunique_is_supported; ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); strcpy(ind->amcheck_nspname, amcheck_nspname); pg_atomic_clear_flag(&ind->lock); @@ -464,6 +514,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, } PQclear(res); + free(amcheck_extversion); + free(amcheck_nspname); + free(amcheck_extname); return index_list; } @@ -473,38 +526,46 @@ static bool amcheck_one_index(check_indexes_arg *arguments, pg_indexEntry *ind) { - PGresult *res; - char *params[2]; + PGresult *res; + char *params[3]; + static const char *queries[] = { + "SELECT %s.bt_index_check(index => $1)", + "SELECT %s.bt_index_check(index => $1, heapallindexed => $2)", + "SELECT %s.bt_index_check(index => $1, heapallindexed => $2, checkunique => $3)", + }; + int params_count; char *query = NULL; - params[0] = palloc(64); + if (interrupted) + elog(ERROR, "Interrupted"); +#define INDEXRELID 0 +#define HEAPALLINDEXED 1 +#define CHECKUNIQUE 2 /* first argument is index oid */ - sprintf(params[0], "%u", ind->indexrelid); + params[INDEXRELID] = palloc(64); + sprintf(params[INDEXRELID], "%u", ind->indexrelid); /* second argument is heapallindexed */ - params[1] = heapallindexed ? "true" : "false"; + params[HEAPALLINDEXED] = heapallindexed ? "true" : "false"; + /* third optional argument is checkunique */ + params[CHECKUNIQUE] = checkunique ? "true" : "false"; +#undef CHECKUNIQUE +#undef HEAPALLINDEXED - if (interrupted) - elog(ERROR, "Interrupted"); - - if (ind->heapallindexed_is_supported) - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); - sprintf(query, "SELECT %s.bt_index_check($1, $2)", ind->amcheck_nspname); + params_count = ind->checkunique_is_supported ? + 3 : + ( ind->heapallindexed_is_supported ? 2 : 1 ); - res = pgut_execute_parallel(arguments->conn_arg.conn, - arguments->conn_arg.cancel_conn, - query, 2, (const char **)params, true, true, true); - } - else - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); - sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); + /* + * Prepare query text with schema name + * +1 for \0 and -2 for %s + */ + query = palloc(strlen(ind->amcheck_nspname) + strlen(queries[params_count - 1]) + 1 - 2); + sprintf(query, queries[params_count - 1], ind->amcheck_nspname); - res = pgut_execute_parallel(arguments->conn_arg.conn, + res = pgut_execute_parallel(arguments->conn_arg.conn, arguments->conn_arg.cancel_conn, - query, 1, (const char **)params, true, true, true); - } + query, params_count, (const char **)params, true, true, true); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -512,7 +573,7 @@ amcheck_one_index(check_indexes_arg *arguments, arguments->thread_num, arguments->conn_opt.pgdatabase, ind->namespace, ind->name, PQresultErrorMessage(res)); - pfree(params[0]); + pfree(params[INDEXRELID]); pfree(query); PQclear(res); return false; @@ -522,7 +583,8 @@ amcheck_one_index(check_indexes_arg *arguments, arguments->thread_num, arguments->conn_opt.pgdatabase, ind->namespace, ind->name); - pfree(params[0]); + pfree(params[INDEXRELID]); +#undef INDEXRELID pfree(query); PQclear(res); return true; diff --git a/src/help.c b/src/help.c index a6530fc0e..a494ab209 100644 --- a/src/help.c +++ b/src/help.c @@ -190,7 +190,7 @@ help_pg_probackup(void) printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed]\n")); + printf(_(" [--heapallindexed] [--checkunique]\n")); printf(_(" [--help]\n")); printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); @@ -601,7 +601,7 @@ help_checkdb(void) printf(_("\n%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed]\n\n")); + printf(_(" [--heapallindexed] [--checkunique]\n\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -616,6 +616,8 @@ help_checkdb(void) printf(_(" using 'amcheck' or 'amcheck_next' extensions\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); printf(_(" can be used only with '--amcheck' option\n")); + printf(_(" --checkunique also check unique constraints\n")); + printf(_(" can be used only with '--amcheck' option\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 49e226ace..c5ed13175 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -126,6 +126,7 @@ static parray *exclude_relative_paths_list = NULL; /* checkdb options */ bool need_amcheck = false; bool heapallindexed = false; +bool checkunique = false; bool amcheck_parent = false; /* delete options */ @@ -240,6 +241,7 @@ static ConfigOption cmd_options[] = /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, + { 'b', 198, "checkunique", &checkunique, SOURCE_CMD_STRICT }, { 'b', 197, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, @@ -596,6 +598,16 @@ main(int argc, char *argv[]) instance_config.pgdata == NULL) elog(ERROR, "required parameter not specified: --instance"); + /* Check checkdb command options consistency */ + if (backup_subcmd == CHECKDB_CMD && + !need_amcheck) + { + if (heapallindexed) + elog(ERROR, "--heapallindexed can only be used with --amcheck option"); + if (checkunique) + elog(ERROR, "--checkunique can only be used with --amcheck option"); + } + /* Usually checkdb for file logging requires log_directory * to be specified explicitly, but if backup_dir and instance name are provided, * checkdb can use the usual default values or values from config diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b202b6152..783a14b1e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -829,6 +829,7 @@ extern ShowFormat show_format; /* checkdb options */ extern bool heapallindexed; +extern bool checkunique; extern bool skip_block_validation; /* current settings */ diff --git a/tests/checkdb.py b/tests/checkdb.py index 044c057f6..9b7adcd71 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -211,6 +211,7 @@ def test_checkdb_amcheck_only_sanity(self): # Clean after yourself gdb.kill() + node.stop() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -349,6 +350,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself + node.stop() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -445,6 +447,98 @@ def test_checkdb_block_validation_sanity(self): e.message) # Clean after yourself + node.stop() + self.del_test_dir(module_name, fname) + + def test_checkdb_checkunique(self): + """Test checkunique parameter of amcheck.bt_index_check function""" + fname = self.id().split('.')[3] + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + node.slow_start() + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + # Part of https://commitfest.postgresql.org/32/2976/ patch test + node.safe_psql( + "postgres", + "CREATE TABLE bttest_unique(a varchar(50), b varchar(1500), c bytea, d varchar(50)); " + "ALTER TABLE bttest_unique SET (autovacuum_enabled = false); " + "CREATE UNIQUE INDEX bttest_unique_idx ON bttest_unique(a,b); " + "UPDATE pg_catalog.pg_index SET indisunique = false " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "INSERT INTO bttest_unique " + " SELECT i::text::varchar, " + " array_to_string(array( " + " SELECT substr('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', ((random()*(36-1)+1)::integer), 1) " + " FROM generate_series(1,1300)),'')::varchar, " + " i::text::bytea, i::text::varchar " + " FROM generate_series(0,1) AS i, generate_series(0,30) AS x; " + "UPDATE pg_catalog.pg_index SET indisunique = true " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "DELETE FROM bttest_unique WHERE ctid::text='(0,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,3)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(9,3)';") + + # run without checkunique option (error will not detected) + output = self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + output) + self.assertIn( + 'All checked indexes are valid', + output) + + # run with checkunique option + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--checkunique', + '-d', 'postgres', '-p', str(node.port)]) + if (ProbackupTest.enterprise and + (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400)): + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of index corruption\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertRegex( + self.output, + r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") + except ProbackupException as e: + self.assertIn( + "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", + e.message) + + # Clean after yourself + node.stop() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -502,6 +596,7 @@ def test_checkdb_sigint_handling(self): # Clean after yourself gdb.kill() + node.stop() self.del_test_dir(module_name, fname) # @unittest.skip("skip") @@ -563,12 +658,15 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' # amcheck-next function ) # PG 9.6 @@ -588,6 +686,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' @@ -595,6 +694,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' # 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' ) @@ -615,13 +716,16 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' ) if ProbackupTest.enterprise: # amcheck-1.1 @@ -633,7 +737,45 @@ def test_checkdb_with_least_privileges(self): node.safe_psql( 'backupdb', 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') - # >= 11 + # >= 11 < 14 + elif self.get_version(node) > 110000 and self.get_version(node) < 140000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) + # checkunique parameter + if ProbackupTest.enterprise: + if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400): + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + # >= 14 else: node.safe_psql( 'backupdb', @@ -650,6 +792,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' @@ -657,9 +800,16 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' ) + # checkunique parameter + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") if ProbackupTest.enterprise: node.safe_psql( @@ -700,4 +850,5 @@ def test_checkdb_with_least_privileges(self): repr(e.message), self.cmd)) # Clean after yourself + node.stop() self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index dd3c4e865..a8b4a64b3 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -107,7 +107,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup checkdb [-B backup-path] [--instance=instance_name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] - [--heapallindexed] + [--heapallindexed] [--checkunique] [--help] pg_probackup show -B backup-path From d222659ee21700a95fc118eeaa941a2bb9b8f07d Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 17 Feb 2022 01:31:48 +0300 Subject: [PATCH 1830/2107] Version 2.5.5 --- src/pg_probackup.h | 4 ++-- tests/expected/option_version.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 783a14b1e..4cd65980c 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.4" +#define PROGRAM_VERSION "2.5.5" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index a69cee03d..29cd93f45 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.4 \ No newline at end of file +pg_probackup 2.5.5 \ No newline at end of file From 0834e54fc37bd841f11717e07291d59ba92e3333 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 17 Feb 2022 15:37:09 +0300 Subject: [PATCH 1831/2107] [PGPRO-6051] [DOC] [ci skip] before release last minute documentation changes --- doc/pgprobackup.xml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 76333b116..86063b843 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -977,6 +977,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup; @@ -4176,7 +4177,7 @@ pg_probackup restore -B backup_dir --instance backup_dir] [--instance instance_name] [-D data_dir] [--help] [-j num_threads] [--progress] -[--skip-block-validation] [--amcheck [--checkunique] [--heapallindexed]] +[--amcheck [--skip-block-validation] [--checkunique] [--heapallindexed]] [connection_options] [logging_options] @@ -4191,7 +4192,7 @@ pg_probackup checkdb Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking - data files. You must have the amcheck + data files. You must have the amcheck extension or the amcheck_next extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. @@ -4211,8 +4212,10 @@ pg_probackup checkdb installed in the database. - This verification is only possible if it is supported by the version of the - amcheck extension you are using. + The verification of unique constraints is only possible if in the version of the + amcheck extension you are using, the + bt_index_check function takes the + checkunique parameter. @@ -4226,9 +4229,10 @@ pg_probackup checkdb flag. - This check is only possible if it is supported by the version of the - amcheck extension you are using or - if the amcheck_next extension is used instead. + This check is only possible if in the version of the + amcheck/amcheck_next extension + you are using, the bt_index_check + function takes the heapallindexed parameter. From 22c808312f67a060cda3bb36e5a032784a5810f9 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Thu, 17 Mar 2022 11:33:18 +0300 Subject: [PATCH 1832/2107] [ci skip] [packaging] Fix CentOS-8 packaging, fix pgpro-std tests --- packaging/pkg/scripts/rpm.sh | 8 ++++++++ packaging/test/scripts/rpm.sh | 11 ++++++++++- packaging/test/scripts/rpm_forks.sh | 20 +++++++++++++++----- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packaging/pkg/scripts/rpm.sh b/packaging/pkg/scripts/rpm.sh index d03915c20..2fec4a700 100755 --- a/packaging/pkg/scripts/rpm.sh +++ b/packaging/pkg/scripts/rpm.sh @@ -20,7 +20,15 @@ ulimit -n 1024 if [ ${DISTRIB} = 'centos' ] ; then sed -i 's|^baseurl=http://|baseurl=https://|g' /etc/yum.repos.d/*.repo + if [ ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi yum update -y + if [ ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi fi # PACKAGES NEEDED diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh index 92804a7f4..3b6806993 100755 --- a/packaging/test/scripts/rpm.sh +++ b/packaging/test/scripts/rpm.sh @@ -15,7 +15,16 @@ PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then # update of rpm package is broken in rhel-7 (26/12/2022) - yum update -y + #yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi + yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi fi # yum upgrade -y || echo 'some packages in docker failed to upgrade' # yum install -y sudo diff --git a/packaging/test/scripts/rpm_forks.sh b/packaging/test/scripts/rpm_forks.sh index 0d72040ed..d57711697 100755 --- a/packaging/test/scripts/rpm_forks.sh +++ b/packaging/test/scripts/rpm_forks.sh @@ -15,7 +15,15 @@ PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then # update of rpm package is broken in rhel-7 (26/12/2022) + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi fi if [ ${PBK_EDITION} == 'ent' ]; then @@ -80,11 +88,13 @@ if [ $PBK_EDITION == 'std' ] ; then # install POSTGRESQL # rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-${DISTRIB_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm - if [[ ${PG_VERSION} == '11' ]] || [[ ${PG_VERSION} == '12' ]]; then - rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm - else - rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm - fi + #if [[ ${PG_VERSION} == '11' ]] || [[ ${PG_VERSION} == '12' ]]; then + # rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + #else + # rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + #fi + curl -o pgpro-repo-add.sh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/pgpro-repo-add.sh + sh pgpro-repo-add.sh if [[ ${PG_VERSION} == '9.6' ]]; then yum install -y postgrespro${PG_TOG}-server.x86_64 From 06994293a221366b969a6f83d5ae339444caf372 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Mon, 28 Mar 2022 06:30:39 +0300 Subject: [PATCH 1833/2107] [PGPRO-5387] Vanilla fixed idle replica archiving (#458) See https://www.postgresql.org/message-id/flat/20210901.121225.1339494423357751537.horikyota.ntt%40gmail.com#ba576416b65f28725488861280805e84 So we can revert two workarounds: * Revert "[PGPRO-5378] fix unstable tests.replica.ReplicaTest.test_replica_archive_page_backup" This reverts commit 90b9b5745e19909a6f5f28761def49ef6bfef0e4. * Revert ""fix" unstable backup.BackupTest.test_backup_with_less_privileges_role (disable tests in archive mode from replica)" This reverts commit 5dcd1ce2b817219180005b1b70a231798cd96ec5. --- tests/backup.py | 40 +++++++++++++++++++--------------------- tests/replica.py | 15 ++------------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index b14f5fe98..682409015 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2351,47 +2351,45 @@ def test_backup_with_less_privileges_role(self): replica.slow_start(replica=True) - # Archive backups from replica in this test are disabled, - # because WAL archiving on replica in idle DB in PostgreSQL is broken: - # replica will not archive the previous WAL until it receives new records in the next WAL file, - # this "lazy" archiving can be seen in src/backend/replication/walreceiver.c:XLogWalRcvWrite() - # (see !XLByteInSeg checking and XLogArchiveNotify() calling). - # # self.switch_wal_segment(node) - #self.backup_node( - # backup_dir, 'replica', replica, - # datname='backupdb', options=['-U', 'backup']) + # self.switch_wal_segment(node) + + self.backup_node( + backup_dir, 'replica', replica, + datname='backupdb', options=['-U', 'backup']) # stream full backup from replica self.backup_node( backup_dir, 'replica', replica, datname='backupdb', options=['--stream', '-U', 'backup']) +# self.switch_wal_segment(node) + # PAGE backup from replica - #self.switch_wal_segment(node) - #self.backup_node( - # backup_dir, 'replica', replica, backup_type='page', - # datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) self.backup_node( backup_dir, 'replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA backup from replica - #self.switch_wal_segment(node) - #self.backup_node( - # backup_dir, 'replica', replica, backup_type='delta', - # datname='backupdb', options=['-U', 'backup']) + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK backup from replica if self.ptrack: - #self.switch_wal_segment(node) - #self.backup_node( - # backup_dir, 'replica', replica, backup_type='ptrack', - # datname='backupdb', options=['-U', 'backup']) + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) diff --git a/tests/replica.py b/tests/replica.py index 8fb89c222..45eed3fb4 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -291,16 +291,6 @@ def test_replica_archive_page_backup(self): self.wait_until_replica_catch_with_master(master, replica) - master.pgbench_init(scale=5) - # Continuous making some changes on master, - # because WAL archiving on replica in idle DB in PostgreSQL is broken: - # replica will not archive the previous WAL until it receives new records in the next WAL file, - # this "lazy" archiving can be seen in src/backend/replication/walreceiver.c:XLogWalRcvWrite() - # (see !XLByteInSeg checking and XLogArchiveNotify() calling). - pgbench = master.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '3', '-c', '1', '--no-vacuum']) - backup_id = self.backup_node( backup_dir, 'replica', replica, options=[ @@ -309,9 +299,6 @@ def test_replica_archive_page_backup(self): '--master-db=postgres', '--master-port={0}'.format(master.port)]) - pgbench.wait() - pgbench.stdout.close() - self.validate_pb(backup_dir, 'replica') self.assertEqual( 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) @@ -334,6 +321,8 @@ def test_replica_archive_page_backup(self): # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal # to original data + master.pgbench_init(scale=5) + pgbench = master.pgbench( options=['-T', '30', '-c', '2', '--no-vacuum']) From e101bfda7abd8050a56952341f02fa0bfbb35230 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Tue, 12 Apr 2022 14:48:38 +0300 Subject: [PATCH 1834/2107] =?UTF-8?q?Fix=20incorrect=20PG=5FPROBACKUP=5FPT?= =?UTF-8?q?RACK=20definition=20in=20travis=20tests=20(led=20to=20the=20ina?= =?UTF-8?q?bility=20to=20run=20ptra=D1=81k=20tests)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 12 ++++++------ travis/make_dockerfile.sh | 2 +- travis/run_tests.sh | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 876289e82..663330918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,17 +34,17 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=archive +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=compression -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=delta -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=locking +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=merge -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=page +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=page # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=ptrack # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=off MODE=retention +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=retention # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=restore jobs: diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh index 119125ced..e780649d9 100755 --- a/travis/make_dockerfile.sh +++ b/travis/make_dockerfile.sh @@ -15,7 +15,7 @@ if [ -z ${MODE+x} ]; then fi if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then - PTRACK_PATCH_PG_BRANCH=off + PTRACK_PATCH_PG_BRANCH=OFF fi if [ -z ${PGPROBACKUP_GDB+x} ]; then diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 44815407e..52b05105b 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020, Postgres Professional +# Copyright (c) 2019-2022, Postgres Professional # set -xe @@ -33,18 +33,18 @@ echo "############### Getting Postgres sources:" git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 # Clone ptrack -if [ "$PTRACK_PATCH_PG_BRANCH" != "off" ]; then +if [ "$PTRACK_PATCH_PG_BRANCH" != "OFF" ]; then git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 - export PG_PROBACKUP_PTRACK=on + export PG_PROBACKUP_PTRACK=ON else - export PG_PROBACKUP_PTRACK=off + export PG_PROBACKUP_PTRACK=OFF fi # Compile and install Postgres echo "############### Compiling Postgres:" cd postgres # Go to postgres dir -if [ "$PG_PROBACKUP_PTRACK" = "on" ]; then +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests @@ -59,7 +59,7 @@ export PATH=$PGHOME/bin:$PATH export LD_LIBRARY_PATH=$PGHOME/lib export PG_CONFIG=$(which pg_config) -if [ "$PG_PROBACKUP_PTRACK" = "on" ]; then +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then echo "############### Compiling Ptrack:" make USE_PGXS=1 -C ../ptrack install fi From bdbc8265d45649e803e4dd9ad733250758e33e19 Mon Sep 17 00:00:00 2001 From: japinli Date: Tue, 19 Apr 2022 19:02:20 +0800 Subject: [PATCH 1835/2107] Fix comparison unsigned expression --- src/data.c | 2 +- src/delete.c | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/data.c b/src/data.c index f02e3fd14..ec42813a6 100644 --- a/src/data.c +++ b/src/data.c @@ -2321,7 +2321,7 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", to_fullpath, strerror(errno)); { - size_t pos = ftell(out); + long pos = ftell(out); if (pos < 0) elog(ERROR, "Cannot get position in destination file \"%s\": %s", diff --git a/src/delete.c b/src/delete.c index 6c70ff81e..b86ed43e6 100644 --- a/src/delete.c +++ b/src/delete.c @@ -36,7 +36,7 @@ do_delete(InstanceState *instanceState, time_t backup_id) parray *backup_list, *delete_list; pgBackup *target_backup = NULL; - size_t size_to_delete = 0; + int64 size_to_delete = 0; char size_to_delete_pretty[20]; /* Get complete list of backups */ @@ -682,12 +682,11 @@ do_retention_wal(InstanceState *instanceState, bool dry_run) * at least one backup and no file should be removed. * Unless wal-depth is enabled. */ - if ((tlinfo->closest_backup) && instance_config.wal_depth <= 0) + if ((tlinfo->closest_backup) && instance_config.wal_depth == 0) continue; /* WAL retention keeps this timeline from purge */ - if (instance_config.wal_depth >= 0 && tlinfo->anchor_tli > 0 && - tlinfo->anchor_tli != tlinfo->tli) + if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) continue; /* @@ -701,7 +700,7 @@ do_retention_wal(InstanceState *instanceState, bool dry_run) */ if (tlinfo->oldest_backup) { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) { delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); @@ -714,7 +713,7 @@ do_retention_wal(InstanceState *instanceState, bool dry_run) } else { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else @@ -942,7 +941,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name); /* save segment from purging */ - if (instance_config.wal_depth >= 0 && wal_file->keep) + if (wal_file->keep) { elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath); continue; @@ -1027,7 +1026,7 @@ do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, parray *backup_list, *delete_list; const char *pretty_status; int n_deleted = 0, n_found = 0; - size_t size_to_delete = 0; + int64 size_to_delete = 0; char size_to_delete_pretty[20]; pgBackup *backup; From 0ae30afe0aa24d970ffd1eb0ca3d3ee6ca32de3d Mon Sep 17 00:00:00 2001 From: japinli Date: Thu, 21 Apr 2022 20:25:28 +0800 Subject: [PATCH 1836/2107] Fix formattor for ftello --- src/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data.c b/src/data.c index ec42813a6..052e17486 100644 --- a/src/data.c +++ b/src/data.c @@ -2030,10 +2030,10 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, return false; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "Odd size page found at offset %lu of \"%s\"", + "Odd size page found at offset %ld of \"%s\"", ftello(in), fullpath); else - elog(ERROR, "Cannot read header at offset %lu of \"%s\": %s", + elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", ftello(in), fullpath, strerror(errno)); } From 94fd54ab6367fceb58d5f643d761ea48cf58ea05 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Fri, 29 Apr 2022 21:15:35 +0500 Subject: [PATCH 1837/2107] Fix test pgpro560.CheckSystemID.test_pgpro560_control_file_loss. File /global/pg_control doesn't removed permanently --- tests/pgpro560.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 53c7914a2..7e10fef6a 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -32,15 +32,16 @@ def test_pgpro560_control_file_loss(self): node.slow_start() file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') - os.remove(file) + # Not delete this file permanently + os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) try: self.backup_node(backup_dir, 'node', node, options=['--stream']) # we should die here because exception is what we expect to happen self.assertEqual( - 1, 0, - "Expecting Error because pg_control was deleted.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + 1, 0, + "Expecting Error because pg_control was deleted.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( 'ERROR: Could not open file' in e.message and @@ -49,6 +50,8 @@ def test_pgpro560_control_file_loss(self): repr(e.message), self.cmd)) # Clean after yourself + # Return this file to avoid Postger fail + os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) self.del_test_dir(module_name, fname) def test_pgpro560_systemid_mismatch(self): From 141e96a0e6cdaac8b1e41b871254bdb60005a368 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Mon, 23 May 2022 15:07:27 +0300 Subject: [PATCH 1838/2107] [DOC] [PBCKP-128] [skip travis] Describe catchup dry-run flag --- doc/pgprobackup.xml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 86063b843..cb615fb17 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3563,6 +3563,14 @@ pg_probackup catchup -b catchup_mode --source-pgdata= of threads with the option: pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --threads=num_threads + + + + Before cloning/synchronising a PostgreSQL instance, you can run the + catchup command with the flag + to estimate the size of data files to be transferred, but make no changes on disk: + +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --dry-run @@ -3576,7 +3584,7 @@ pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replic Another example shows how you can add a new remote standby server with the PostgreSQL data directory /replica-pgdata by running the catchup command in the FULL mode on four parallel threads: - + pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=FULL --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 @@ -4482,7 +4490,7 @@ pg_probackup archive-get -B backup_dir --instance catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir -[--help] [-j | --threads=num_threads] [--stream] +[--help] [-j | --threads=num_threads] [--stream] [--dry-run] [--temp-slot] [-P | --perm-slot] [-S | --slot=slot_name] [--exclude-path=PATHNAME] [-T OLDDIR=NEWDIR] @@ -4571,6 +4579,19 @@ pg_probackup catchup -b catchup_mode + + + + + Displays the total size of the files to be transferred by catchup. + This flag initiates a trial run of catchup, which does + not actually create, delete or move files on disk. WAL streaming is skipped with . + This flag also allows you to check that + all the options are correct and cloning/synchronising is ready to run. + + + + =path_prefix =path_prefix @@ -4591,17 +4612,6 @@ pg_probackup catchup -b catchup_mode - - - - - Copies the instance in STREAM WAL delivery mode, - including all the necessary WAL files by streaming them from - the instance server via replication protocol. - - - - From 4b2df86d6961937e062c54bb7fd5a4cdf96c1f58 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Mon, 23 May 2022 20:13:18 +0300 Subject: [PATCH 1839/2107] PBCKP-97: added localization of messages * PBCKP-97: Adding localization files Added localization of messages. Fixed some bugs. Added the --enable-nls tag for tests. Added a test to check the localization of messages. Co-authored-by: Vyacheslav Makarov --- .travis.yml | 1 + README.md | 14 + nls.mk | 6 + po/ru.po | 1880 +++++++++++++++++++++++++++++ src/help.c | 4 +- src/pg_probackup.c | 1 + tests/Readme.md | 2 +- tests/expected/option_help.out | 2 +- tests/expected/option_help_ru.out | 184 +++ tests/option.py | 11 + travis/run_tests.sh | 2 +- 11 files changed, 2102 insertions(+), 5 deletions(-) create mode 100644 nls.mk create mode 100644 po/ru.po create mode 100644 tests/expected/option_help_ru.out diff --git a/.travis.yml b/.travis.yml index 663330918..8e325c64f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,7 @@ env: # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=merge +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=option # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=page # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=ptrack # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica diff --git a/README.md b/README.md index 060883a28..5da8d199e 100644 --- a/README.md +++ b/README.md @@ -224,3 +224,17 @@ Postgres Professional, Moscow, Russia. ## Credits `pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. + + +### Localization files (*.po) + +Description of how to add new translation languages. +1. Add a flag --enable-nls in configure. +2. Build postgres. +3. Adding to nls.mk in folder pg_probackup required files in GETTEXT_FILES. +4. In folder pg_probackup do 'make update-po'. +5. As a result, the progname.pot file will be created. Copy the content and add it to the file with the desired language. +6. Adding to nls.mk in folder pg_probackup required language in AVAIL_LANGUAGES. + +For more information, follow the link below: +https://postgrespro.ru/docs/postgresql/12/nls-translator diff --git a/nls.mk b/nls.mk new file mode 100644 index 000000000..981c1c4fe --- /dev/null +++ b/nls.mk @@ -0,0 +1,6 @@ +# contrib/pg_probackup/nls.mk +CATALOG_NAME = pg_probackup +AVAIL_LANGUAGES = ru +GETTEXT_FILES = src/help.c +GETTEXT_TRIGGERS = $(FRONTEND_COMMON_GETTEXT_TRIGGERS) +GETTEXT_FLAGS = $(FRONTEND_COMMON_GETTEXT_FLAGS) diff --git a/po/ru.po b/po/ru.po new file mode 100644 index 000000000..1263675c2 --- /dev/null +++ b/po/ru.po @@ -0,0 +1,1880 @@ +# Russian message translation file for pg_probackup +# Copyright (C) 2022 PostgreSQL Global Development Group +# This file is distributed under the same license as the pg_probackup (PostgreSQL) package. +# Vyacheslav Makarov , 2022. +msgid "" +msgstr "" +"Project-Id-Version: pg_probackup (PostgreSQL)\n" +"Report-Msgid-Bugs-To: bugs@postgrespro.ru\n" +"POT-Creation-Date: 2022-04-08 11:33+0300\n" +"PO-Revision-Date: 2022-MO-DA HO:MI+ZONE\n" +"Last-Translator: Vyacheslav Makarov \n" +"Language-Team: Russian \n" +"Language: ru\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n" + +#: src/help.c:84 +#, c-format +msgid "" +"\n" +"%s - utility to manage backup/recovery of PostgreSQL database.\n" +msgstr "" +"\n" +"%s - утилита для управления резервным копированием/восстановлением базы данных PostgreSQL.\n" + +#: src/help.c:86 +#, c-format +msgid "" +"\n" +" %s help [COMMAND]\n" +msgstr "" + +#: src/help.c:88 +#, c-format +msgid "" +"\n" +" %s version\n" +msgstr "" + +#: src/help.c:90 +#, c-format +msgid "" +"\n" +" %s init -B backup-path\n" +msgstr "" + +#: src/help.c:92 +#, c-format +msgid "" +"\n" +" %s set-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:93 src/help.c:791 +#, c-format +msgid " [-D pgdata-path]\n" +msgstr "" + +#: src/help.c:94 src/help.c:130 src/help.c:218 +#, c-format +msgid " [--external-dirs=external-directories-paths]\n" +msgstr "" + +#: src/help.c:95 src/help.c:132 src/help.c:305 src/help.c:731 src/help.c:794 +#, c-format +msgid " [--log-level-console=log-level-console]\n" +msgstr "" + +#: src/help.c:96 src/help.c:133 src/help.c:306 src/help.c:732 src/help.c:795 +#, c-format +msgid " [--log-level-file=log-level-file]\n" +msgstr "" + +#: src/help.c:97 src/help.c:134 src/help.c:307 src/help.c:733 src/help.c:796 +#, c-format +msgid " [--log-filename=log-filename]\n" +msgstr "" + +#: src/help.c:98 src/help.c:135 src/help.c:308 src/help.c:734 src/help.c:797 +#, c-format +msgid " [--error-log-filename=error-log-filename]\n" +msgstr "" + +#: src/help.c:99 src/help.c:136 src/help.c:309 src/help.c:735 src/help.c:798 +#, c-format +msgid " [--log-directory=log-directory]\n" +msgstr "" + +#: src/help.c:100 src/help.c:137 src/help.c:310 src/help.c:736 src/help.c:799 +#, c-format +msgid " [--log-rotation-size=log-rotation-size]\n" +msgstr "" + +#: src/help.c:101 src/help.c:800 +#, c-format +msgid " [--log-rotation-age=log-rotation-age]\n" +msgstr "" + +#: src/help.c:102 src/help.c:140 src/help.c:203 src/help.c:313 src/help.c:674 +#: src/help.c:801 +#, c-format +msgid " [--retention-redundancy=retention-redundancy]\n" +msgstr "" + +#: src/help.c:103 src/help.c:141 src/help.c:204 src/help.c:314 src/help.c:675 +#: src/help.c:802 +#, c-format +msgid " [--retention-window=retention-window]\n" +msgstr "" + +#: src/help.c:104 src/help.c:142 src/help.c:205 src/help.c:315 src/help.c:676 +#: src/help.c:803 +#, c-format +msgid " [--wal-depth=wal-depth]\n" +msgstr "" + +#: src/help.c:105 src/help.c:144 src/help.c:235 src/help.c:317 src/help.c:804 +#: src/help.c:948 +#, c-format +msgid " [--compress-algorithm=compress-algorithm]\n" +msgstr "" + +#: src/help.c:106 src/help.c:145 src/help.c:236 src/help.c:318 src/help.c:805 +#: src/help.c:949 +#, c-format +msgid " [--compress-level=compress-level]\n" +msgstr "" + +#: src/help.c:107 src/help.c:232 src/help.c:806 src/help.c:945 +#, c-format +msgid " [--archive-timeout=timeout]\n" +msgstr "" + +#: src/help.c:108 src/help.c:147 src/help.c:259 src/help.c:320 src/help.c:807 +#: src/help.c:1045 +#, c-format +msgid " [-d dbname] [-h host] [-p port] [-U username]\n" +msgstr "" + +#: src/help.c:109 src/help.c:149 src/help.c:174 src/help.c:219 src/help.c:237 +#: src/help.c:247 src/help.c:261 src/help.c:322 src/help.c:449 src/help.c:808 +#: src/help.c:906 src/help.c:950 src/help.c:994 src/help.c:1047 +#, c-format +msgid " [--remote-proto] [--remote-host]\n" +msgstr "" + +#: src/help.c:110 src/help.c:150 src/help.c:175 src/help.c:220 src/help.c:238 +#: src/help.c:248 src/help.c:262 src/help.c:323 src/help.c:450 src/help.c:809 +#: src/help.c:907 src/help.c:951 src/help.c:995 src/help.c:1048 +#, c-format +msgid " [--remote-port] [--remote-path] [--remote-user]\n" +msgstr "" + +#: src/help.c:111 src/help.c:151 src/help.c:176 src/help.c:221 src/help.c:239 +#: src/help.c:249 src/help.c:263 src/help.c:324 src/help.c:451 src/help.c:1049 +#, c-format +msgid " [--ssh-options]\n" +msgstr "" + +#: src/help.c:112 +#, c-format +msgid " [--restore-command=cmdline] [--archive-host=destination]\n" +msgstr "" + +#: src/help.c:113 src/help.c:178 +#, c-format +msgid " [--archive-port=port] [--archive-user=username]\n" +msgstr "" + +#: src/help.c:114 src/help.c:119 src/help.c:123 src/help.c:153 src/help.c:179 +#: src/help.c:188 src/help.c:194 src/help.c:209 src/help.c:214 src/help.c:222 +#: src/help.c:226 src/help.c:240 src/help.c:250 src/help.c:264 +#, c-format +msgid " [--help]\n" +msgstr "" + +#: src/help.c:116 +#, c-format +msgid "" +"\n" +" %s set-backup -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:117 +#, c-format +msgid " -i backup-id [--ttl=interval] [--expire-time=timestamp]\n" +msgstr "" + +#: src/help.c:118 +#, c-format +msgid " [--note=text]\n" +msgstr "" + +#: src/help.c:121 +#, c-format +msgid "" +"\n" +" %s show-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:122 +#, c-format +msgid " [--format=format]\n" +msgstr "" + +#: src/help.c:125 +#, c-format +msgid "" +"\n" +" %s backup -B backup-path -b backup-mode --instance=instance_name\n" +msgstr "" + +#: src/help.c:126 src/help.c:299 +#, c-format +msgid " [-D pgdata-path] [-C]\n" +msgstr "" + +#: src/help.c:127 src/help.c:300 +#, c-format +msgid " [--stream [-S slot-name] [--temp-slot]]\n" +msgstr "" + +#: src/help.c:128 src/help.c:301 +#, c-format +msgid " [--backup-pg-log] [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:129 src/help.c:168 src/help.c:302 src/help.c:433 +#, c-format +msgid " [--no-validate] [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:131 src/help.c:304 +#, c-format +msgid " [--no-sync]\n" +msgstr "" + +#: src/help.c:138 src/help.c:311 +#, c-format +msgid " [--log-rotation-age=log-rotation-age] [--no-color]\n" +msgstr "" + +#: src/help.c:139 src/help.c:312 +#, c-format +msgid " [--delete-expired] [--delete-wal] [--merge-expired]\n" +msgstr "" + +#: src/help.c:143 src/help.c:316 +#, c-format +msgid " [--compress]\n" +msgstr "" + +#: src/help.c:146 src/help.c:319 +#, c-format +msgid " [--archive-timeout=archive-timeout]\n" +msgstr "" + +#: src/help.c:148 src/help.c:260 src/help.c:321 src/help.c:1046 +#, c-format +msgid " [-w --no-password] [-W --password]\n" +msgstr "" + +#: src/help.c:152 +#, c-format +msgid " [--ttl=interval] [--expire-time=timestamp] [--note=text]\n" +msgstr "" + +#: src/help.c:156 +#, c-format +msgid "" +"\n" +" %s restore -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:157 src/help.c:431 +#, c-format +msgid " [-D pgdata-path] [-i backup-id] [-j num-threads]\n" +msgstr "" + +#: src/help.c:158 src/help.c:183 src/help.c:439 src/help.c:552 +#, c-format +msgid " [--recovery-target-time=time|--recovery-target-xid=xid\n" +msgstr "" + +#: src/help.c:159 src/help.c:184 src/help.c:440 src/help.c:553 +#, c-format +msgid " |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n" +msgstr "" + +#: src/help.c:160 src/help.c:185 src/help.c:441 src/help.c:554 +#, c-format +msgid " [--recovery-target-timeline=timeline]\n" +msgstr "" + +#: src/help.c:161 src/help.c:442 +#, c-format +msgid " [--recovery-target=immediate|latest]\n" +msgstr "" + +#: src/help.c:162 src/help.c:186 src/help.c:443 src/help.c:555 +#, c-format +msgid " [--recovery-target-name=target-name]\n" +msgstr "" + +#: src/help.c:163 src/help.c:444 +#, c-format +msgid " [--recovery-target-action=pause|promote|shutdown]\n" +msgstr "" + +#: src/help.c:164 src/help.c:445 src/help.c:793 +#, c-format +msgid " [--restore-command=cmdline]\n" +msgstr "" + +#: src/help.c:165 +#, c-format +msgid " [-R | --restore-as-replica] [--force]\n" +msgstr "" + +#: src/help.c:166 src/help.c:447 +#, c-format +msgid " [--primary-conninfo=primary_conninfo]\n" +msgstr "" + +#: src/help.c:167 src/help.c:448 +#, c-format +msgid " [-S | --primary-slot-name=slotname]\n" +msgstr "" + +#: src/help.c:169 +#, c-format +msgid " [-T OLDDIR=NEWDIR] [--progress]\n" +msgstr "" + +#: src/help.c:170 src/help.c:435 +#, c-format +msgid " [--external-mapping=OLDDIR=NEWDIR]\n" +msgstr "" + +#: src/help.c:171 +#, c-format +msgid " [--skip-external-dirs] [--no-sync]\n" +msgstr "" + +#: src/help.c:172 src/help.c:437 +#, c-format +msgid " [-I | --incremental-mode=none|checksum|lsn]\n" +msgstr "" + +#: src/help.c:173 +#, c-format +msgid " [--db-include | --db-exclude]\n" +msgstr "" + +#: src/help.c:177 +#, c-format +msgid " [--archive-host=hostname]\n" +msgstr "" + +#: src/help.c:181 +#, c-format +msgid "" +"\n" +" %s validate -B backup-path [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:182 src/help.c:551 +#, c-format +msgid " [-i backup-id] [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:187 +#, c-format +msgid " [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:190 +#, c-format +msgid "" +"\n" +" %s checkdb [-B backup-path] [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:191 +#, c-format +msgid " [-D pgdata-path] [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:192 src/help.c:603 +#, c-format +msgid " [--amcheck] [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:193 +#, c-format +msgid " [--heapallindexed] [--checkunique]\n" +msgstr "" + +#: src/help.c:196 +#, c-format +msgid "" +"\n" +" %s show -B backup-path\n" +msgstr "" + +#: src/help.c:197 src/help.c:657 +#, c-format +msgid " [--instance=instance_name [-i backup-id]]\n" +msgstr "" + +#: src/help.c:198 +#, c-format +msgid " [--format=format] [--archive]\n" +msgstr "" + +#: src/help.c:199 +#, c-format +msgid " [--no-color] [--help]\n" +msgstr "" + +#: src/help.c:201 +#, c-format +msgid "" +"\n" +" %s delete -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:202 src/help.c:673 +#, c-format +msgid " [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:206 +#, c-format +msgid " [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n" +msgstr "" + +#: src/help.c:207 +#, c-format +msgid " [--delete-wal]\n" +msgstr "" + +#: src/help.c:208 +#, c-format +msgid " [--dry-run] [--no-validate] [--no-sync]\n" +msgstr "" + +#: src/help.c:211 +#, c-format +msgid "" +"\n" +" %s merge -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:212 +#, c-format +msgid " -i backup-id [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:213 src/help.c:730 +#, c-format +msgid " [--no-validate] [--no-sync]\n" +msgstr "" + +#: src/help.c:216 +#, c-format +msgid "" +"\n" +" %s add-instance -B backup-path -D pgdata-path\n" +msgstr "" + +#: src/help.c:217 src/help.c:225 src/help.c:904 +#, c-format +msgid " --instance=instance_name\n" +msgstr "" + +#: src/help.c:224 +#, c-format +msgid "" +"\n" +" %s del-instance -B backup-path\n" +msgstr "" + +#: src/help.c:228 +#, c-format +msgid "" +"\n" +" %s archive-push -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:229 src/help.c:244 src/help.c:942 src/help.c:990 +#, c-format +msgid " --wal-file-name=wal-file-name\n" +msgstr "" + +#: src/help.c:230 src/help.c:943 src/help.c:991 +#, c-format +msgid " [--wal-file-path=wal-file-path]\n" +msgstr "" + +#: src/help.c:231 src/help.c:245 src/help.c:944 src/help.c:992 +#, c-format +msgid " [-j num-threads] [--batch-size=batch_size]\n" +msgstr "" + +#: src/help.c:233 src/help.c:946 +#, c-format +msgid " [--no-ready-rename] [--no-sync]\n" +msgstr "" + +#: src/help.c:234 src/help.c:947 +#, c-format +msgid " [--overwrite] [--compress]\n" +msgstr "" + +#: src/help.c:242 +#, c-format +msgid "" +"\n" +" %s archive-get -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:243 +#, c-format +msgid " --wal-file-path=wal-file-path\n" +msgstr "" + +#: src/help.c:246 src/help.c:993 +#, c-format +msgid " [--no-validate-wal]\n" +msgstr "" + +#: src/help.c:252 +#, c-format +msgid "" +"\n" +" %s catchup -b catchup-mode\n" +msgstr "" + +#: src/help.c:253 src/help.c:1039 +#, c-format +msgid " --source-pgdata=path_to_pgdata_on_remote_server\n" +msgstr "" + +#: src/help.c:254 src/help.c:1040 +#, c-format +msgid " --destination-pgdata=path_to_local_dir\n" +msgstr "" + +#: src/help.c:255 +#, c-format +msgid " [--stream [-S slot-name] [--temp-slot | --perm-slot]]\n" +msgstr "" + +#: src/help.c:256 src/help.c:1042 +#, c-format +msgid " [-j num-threads]\n" +msgstr "" + +#: src/help.c:257 src/help.c:434 src/help.c:1043 +#, c-format +msgid " [-T OLDDIR=NEWDIR]\n" +msgstr "" + +#: src/help.c:258 src/help.c:1044 +#, c-format +msgid " [--exclude-path=path_prefix]\n" +msgstr "" + +#: src/help.c:270 +#, c-format +msgid "Read the website for details <%s>.\n" +msgstr "Подробнее читайте на сайте <%s>.\n" + +#: src/help.c:272 +#, c-format +msgid "Report bugs to <%s>.\n" +msgstr "Сообщайте об ошибках в <%s>.\n" + +#: src/help.c:279 +#, c-format +msgid "" +"\n" +"Unknown command. Try pg_probackup help\n" +"\n" +msgstr "" +"\n" +"Неизвестная команда. Попробуйте pg_probackup help\n" +"\n" + +#: src/help.c:285 +#, c-format +msgid "" +"\n" +"This command is intended for internal use\n" +"\n" +msgstr "" + +#: src/help.c:291 +#, c-format +msgid "" +"\n" +"%s init -B backup-path\n" +"\n" +msgstr "" + +#: src/help.c:292 +#, c-format +msgid "" +" -B, --backup-path=backup-path location of the backup storage area\n" +"\n" +msgstr "" + +#: src/help.c:298 +#, c-format +msgid "" +"\n" +"%s backup -B backup-path -b backup-mode --instance=instance_name\n" +msgstr "" + +#: src/help.c:303 src/help.c:792 +#, c-format +msgid " [-E external-directories-paths]\n" +msgstr "" + +#: src/help.c:325 +#, c-format +msgid "" +" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n" +"\n" +msgstr "" + +#: src/help.c:327 src/help.c:455 src/help.c:558 src/help.c:606 src/help.c:660 +#: src/help.c:679 src/help.c:739 src/help.c:812 src/help.c:895 src/help.c:910 +#: src/help.c:934 src/help.c:954 src/help.c:998 +#, c-format +msgid " -B, --backup-path=backup-path location of the backup storage area\n" +msgstr "" + +#: src/help.c:328 +#, c-format +msgid " -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n" +msgstr "" + +#: src/help.c:329 src/help.c:456 src/help.c:559 src/help.c:607 src/help.c:680 +#: src/help.c:740 src/help.c:813 src/help.c:896 +#, c-format +msgid " --instance=instance_name name of the instance\n" +msgstr "" + +#: src/help.c:330 src/help.c:458 src/help.c:608 src/help.c:814 src/help.c:911 +#, c-format +msgid " -D, --pgdata=pgdata-path location of the database storage area\n" +msgstr "" + +#: src/help.c:331 +#, c-format +msgid " -C, --smooth-checkpoint do smooth checkpoint before backup\n" +msgstr "" + +#: src/help.c:332 +#, c-format +msgid " --stream stream the transaction log and include it in the backup\n" +msgstr "" + +#: src/help.c:333 src/help.c:1054 +#, c-format +msgid " -S, --slot=SLOTNAME replication slot to use\n" +msgstr "" + +#: src/help.c:334 src/help.c:1055 +#, c-format +msgid " --temp-slot use temporary replication slot\n" +msgstr "" + +#: src/help.c:335 +#, c-format +msgid " --backup-pg-log backup of '%s' directory\n" +msgstr "" + +#: src/help.c:336 src/help.c:460 src/help.c:563 src/help.c:611 src/help.c:682 +#: src/help.c:743 src/help.c:960 src/help.c:1004 src/help.c:1058 +#, c-format +msgid " -j, --threads=NUM number of parallel threads\n" +msgstr "" + +#: src/help.c:337 src/help.c:462 src/help.c:562 src/help.c:610 src/help.c:683 +#: src/help.c:744 +#, c-format +msgid " --progress show progress\n" +msgstr "" + +#: src/help.c:338 +#, c-format +msgid " --no-validate disable validation after backup\n" +msgstr "" + +#: src/help.c:339 src/help.c:466 src/help.c:573 +#, c-format +msgid " --skip-block-validation set to validate only file-level checksum\n" +msgstr "" + +#: src/help.c:340 src/help.c:815 src/help.c:914 +#, c-format +msgid " -E --external-dirs=external-directories-paths\n" +msgstr "" + +#: src/help.c:341 src/help.c:816 src/help.c:915 +#, c-format +msgid " backup some directories not from pgdata \n" +msgstr "" + +#: src/help.c:342 src/help.c:817 src/help.c:916 +#, c-format +msgid " (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n" +msgstr "" + +#: src/help.c:343 +#, c-format +msgid " --no-sync do not sync backed up files to disk\n" +msgstr "" + +#: src/help.c:344 +#, c-format +msgid " --note=text add note to backup\n" +msgstr "" + +#: src/help.c:345 src/help.c:784 +#, c-format +msgid " (example: --note='backup before app update to v13.1')\n" +msgstr "" + +#: src/help.c:347 src/help.c:508 src/help.c:575 src/help.c:622 src/help.c:702 +#: src/help.c:748 src/help.c:820 +#, c-format +msgid "" +"\n" +" Logging options:\n" +msgstr "" + +#: src/help.c:348 src/help.c:509 src/help.c:576 src/help.c:623 src/help.c:703 +#: src/help.c:749 src/help.c:821 +#, c-format +msgid " --log-level-console=log-level-console\n" +msgstr "" + +#: src/help.c:349 src/help.c:510 src/help.c:577 src/help.c:624 src/help.c:704 +#: src/help.c:750 src/help.c:822 +#, c-format +msgid " level for console logging (default: info)\n" +msgstr "" + +#: src/help.c:350 src/help.c:353 src/help.c:511 src/help.c:514 src/help.c:578 +#: src/help.c:581 src/help.c:625 src/help.c:628 src/help.c:705 src/help.c:708 +#: src/help.c:751 src/help.c:754 src/help.c:823 src/help.c:826 +#, c-format +msgid " available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n" +msgstr "" + +#: src/help.c:351 src/help.c:512 src/help.c:579 src/help.c:626 src/help.c:706 +#: src/help.c:752 src/help.c:824 +#, c-format +msgid " --log-level-file=log-level-file\n" +msgstr "" + +#: src/help.c:352 src/help.c:513 src/help.c:580 src/help.c:627 src/help.c:707 +#: src/help.c:753 src/help.c:825 +#, c-format +msgid " level for file logging (default: off)\n" +msgstr "" + +#: src/help.c:354 src/help.c:515 src/help.c:582 src/help.c:629 src/help.c:709 +#: src/help.c:755 src/help.c:827 +#, c-format +msgid " --log-filename=log-filename\n" +msgstr "" + +#: src/help.c:355 src/help.c:516 src/help.c:583 src/help.c:630 src/help.c:710 +#: src/help.c:756 src/help.c:828 +#, c-format +msgid " filename for file logging (default: 'pg_probackup.log')\n" +msgstr "" + +#: src/help.c:356 src/help.c:517 src/help.c:584 src/help.c:711 src/help.c:757 +#: src/help.c:829 +#, c-format +msgid " support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n" +msgstr "" + +#: src/help.c:357 src/help.c:518 src/help.c:585 src/help.c:632 src/help.c:712 +#: src/help.c:758 src/help.c:830 +#, c-format +msgid " --error-log-filename=error-log-filename\n" +msgstr "" + +#: src/help.c:358 src/help.c:519 src/help.c:586 src/help.c:633 src/help.c:713 +#: src/help.c:759 src/help.c:831 +#, c-format +msgid " filename for error logging (default: none)\n" +msgstr "" + +#: src/help.c:359 src/help.c:520 src/help.c:587 src/help.c:634 src/help.c:714 +#: src/help.c:760 src/help.c:832 +#, c-format +msgid " --log-directory=log-directory\n" +msgstr "" + +#: src/help.c:360 src/help.c:521 src/help.c:588 src/help.c:635 src/help.c:715 +#: src/help.c:761 src/help.c:833 +#, c-format +msgid " directory for file logging (default: BACKUP_PATH/log)\n" +msgstr "" + +#: src/help.c:361 src/help.c:522 src/help.c:589 src/help.c:636 src/help.c:716 +#: src/help.c:762 src/help.c:834 +#, c-format +msgid " --log-rotation-size=log-rotation-size\n" +msgstr "" + +#: src/help.c:362 src/help.c:523 src/help.c:590 src/help.c:637 src/help.c:717 +#: src/help.c:763 src/help.c:835 +#, c-format +msgid " rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:363 src/help.c:524 src/help.c:591 src/help.c:638 src/help.c:718 +#: src/help.c:764 src/help.c:836 +#, c-format +msgid " available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n" +msgstr "" + +#: src/help.c:364 src/help.c:525 src/help.c:592 src/help.c:639 src/help.c:719 +#: src/help.c:765 src/help.c:837 +#, c-format +msgid " --log-rotation-age=log-rotation-age\n" +msgstr "" + +#: src/help.c:365 src/help.c:526 src/help.c:593 src/help.c:640 src/help.c:720 +#: src/help.c:766 src/help.c:838 +#, c-format +msgid " rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:366 src/help.c:527 src/help.c:594 src/help.c:641 src/help.c:721 +#: src/help.c:767 src/help.c:839 +#, c-format +msgid " available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n" +msgstr "" + +#: src/help.c:367 src/help.c:528 src/help.c:642 +#, c-format +msgid " --no-color disable the coloring of error and warning console messages\n" +msgstr "" + +#: src/help.c:369 src/help.c:687 src/help.c:841 +#, c-format +msgid "" +"\n" +" Retention options:\n" +msgstr "" + +#: src/help.c:370 src/help.c:688 +#, c-format +msgid " --delete-expired delete backups expired according to current\n" +msgstr "" + +#: src/help.c:371 src/help.c:373 +#, c-format +msgid " retention policy after successful backup completion\n" +msgstr "" + +#: src/help.c:372 src/help.c:690 +#, c-format +msgid " --merge-expired merge backups expired according to current\n" +msgstr "" + +#: src/help.c:374 src/help.c:692 +#, c-format +msgid " --delete-wal remove redundant files in WAL archive\n" +msgstr "" + +#: src/help.c:375 src/help.c:693 src/help.c:842 +#, c-format +msgid " --retention-redundancy=retention-redundancy\n" +msgstr "" + +#: src/help.c:376 src/help.c:694 src/help.c:843 +#, c-format +msgid " number of full backups to keep; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:377 src/help.c:695 src/help.c:844 +#, c-format +msgid " --retention-window=retention-window\n" +msgstr "" + +#: src/help.c:378 src/help.c:696 src/help.c:845 +#, c-format +msgid " number of days of recoverability; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:379 src/help.c:697 +#, c-format +msgid " --wal-depth=wal-depth number of latest valid backups per timeline that must\n" +msgstr "" + +#: src/help.c:380 src/help.c:698 +#, c-format +msgid " retain the ability to perform PITR; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:381 src/help.c:699 +#, c-format +msgid " --dry-run perform a trial run without any changes\n" +msgstr "" + +#: src/help.c:383 +#, c-format +msgid "" +"\n" +" Pinning options:\n" +msgstr "" + +#: src/help.c:384 src/help.c:778 +#, c-format +msgid " --ttl=interval pin backup for specified amount of time; 0 unpin\n" +msgstr "" + +#: src/help.c:385 src/help.c:779 +#, c-format +msgid " available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n" +msgstr "" + +#: src/help.c:386 src/help.c:780 +#, c-format +msgid " (example: --ttl=20d)\n" +msgstr "" + +#: src/help.c:387 src/help.c:781 +#, c-format +msgid " --expire-time=time pin backup until specified time stamp\n" +msgstr "" + +#: src/help.c:388 src/help.c:782 +#, c-format +msgid " (example: --expire-time='2024-01-01 00:00:00+03')\n" +msgstr "" + +#: src/help.c:390 src/help.c:849 src/help.c:967 +#, c-format +msgid "" +"\n" +" Compression options:\n" +msgstr "" + +#: src/help.c:391 src/help.c:850 src/help.c:968 +#, c-format +msgid " --compress alias for --compress-algorithm='zlib' and --compress-level=1\n" +msgstr "" + +#: src/help.c:392 src/help.c:851 src/help.c:969 +#, c-format +msgid " --compress-algorithm=compress-algorithm\n" +msgstr "" + +#: src/help.c:393 +#, c-format +msgid " available options: 'zlib', 'pglz', 'none' (default: none)\n" +msgstr "" + +#: src/help.c:394 src/help.c:853 src/help.c:971 +#, c-format +msgid " --compress-level=compress-level\n" +msgstr "" + +#: src/help.c:395 src/help.c:854 src/help.c:972 +#, c-format +msgid " level of compression [0-9] (default: 1)\n" +msgstr "" + +#: src/help.c:397 src/help.c:856 +#, c-format +msgid "" +"\n" +" Archive options:\n" +msgstr "" + +#: src/help.c:398 src/help.c:857 +#, c-format +msgid " --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n" +msgstr "" + +#: src/help.c:400 src/help.c:644 src/help.c:859 src/help.c:1066 +#, c-format +msgid "" +"\n" +" Connection options:\n" +msgstr "" + +#: src/help.c:401 src/help.c:645 src/help.c:860 src/help.c:1067 +#, c-format +msgid " -U, --pguser=USERNAME user name to connect as (default: current local user)\n" +msgstr "" + +#: src/help.c:402 src/help.c:646 src/help.c:861 src/help.c:1068 +#, c-format +msgid " -d, --pgdatabase=DBNAME database to connect (default: username)\n" +msgstr "" + +#: src/help.c:403 src/help.c:647 src/help.c:862 src/help.c:1069 +#, c-format +msgid " -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n" +msgstr "" + +#: src/help.c:404 src/help.c:648 src/help.c:863 src/help.c:1070 +#, c-format +msgid " -p, --pgport=PORT database server port (default: 5432)\n" +msgstr "" + +#: src/help.c:405 src/help.c:649 src/help.c:1071 +#, c-format +msgid " -w, --no-password never prompt for password\n" +msgstr "" + +#: src/help.c:406 +#, c-format +msgid " -W, --password force password prompt\n" +msgstr "" + +#: src/help.c:408 src/help.c:530 src/help.c:865 src/help.c:917 src/help.c:974 +#: src/help.c:1009 src/help.c:1074 +#, c-format +msgid "" +"\n" +" Remote options:\n" +msgstr "" + +#: src/help.c:409 src/help.c:531 src/help.c:866 src/help.c:918 src/help.c:975 +#: src/help.c:1010 src/help.c:1075 +#, c-format +msgid " --remote-proto=protocol remote protocol to use\n" +msgstr "" + +#: src/help.c:410 src/help.c:532 src/help.c:867 src/help.c:919 src/help.c:976 +#: src/help.c:1011 src/help.c:1076 +#, c-format +msgid " available options: 'ssh', 'none' (default: ssh)\n" +msgstr "" + +#: src/help.c:411 src/help.c:533 src/help.c:868 src/help.c:920 +#, c-format +msgid " --remote-host=destination remote host address or hostname\n" +msgstr "" + +#: src/help.c:412 src/help.c:534 src/help.c:869 src/help.c:921 src/help.c:978 +#: src/help.c:1013 src/help.c:1078 +#, c-format +msgid " --remote-port=port remote host port (default: 22)\n" +msgstr "" + +#: src/help.c:413 src/help.c:535 src/help.c:870 src/help.c:922 src/help.c:979 +#: src/help.c:1014 src/help.c:1079 +#, c-format +msgid " --remote-path=path path to directory with pg_probackup binary on remote host\n" +msgstr "" + +#: src/help.c:414 src/help.c:536 src/help.c:871 src/help.c:923 src/help.c:980 +#: src/help.c:1015 src/help.c:1080 +#, c-format +msgid " (default: current binary path)\n" +msgstr "" + +#: src/help.c:415 src/help.c:537 src/help.c:872 src/help.c:924 src/help.c:981 +#: src/help.c:1016 src/help.c:1081 +#, c-format +msgid " --remote-user=username user name for ssh connection (default: current user)\n" +msgstr "" + +#: src/help.c:416 src/help.c:538 src/help.c:873 src/help.c:925 src/help.c:982 +#: src/help.c:1017 src/help.c:1082 +#, c-format +msgid " --ssh-options=ssh_options additional ssh options (default: none)\n" +msgstr "" + +#: src/help.c:417 src/help.c:539 src/help.c:874 +#, c-format +msgid " (example: --ssh-options='-c cipher_spec -F configfile')\n" +msgstr "" + +#: src/help.c:419 src/help.c:881 +#, c-format +msgid "" +"\n" +" Replica options:\n" +msgstr "" + +#: src/help.c:420 src/help.c:882 +#, c-format +msgid " --master-user=user_name user name to connect to master (deprecated)\n" +msgstr "" + +#: src/help.c:421 src/help.c:883 +#, c-format +msgid " --master-db=db_name database to connect to master (deprecated)\n" +msgstr "" + +#: src/help.c:422 src/help.c:884 +#, c-format +msgid " --master-host=host_name database server host of master (deprecated)\n" +msgstr "" + +#: src/help.c:423 src/help.c:885 +#, c-format +msgid " --master-port=port database server port of master (deprecated)\n" +msgstr "" + +#: src/help.c:424 src/help.c:886 +#, c-format +msgid "" +" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n" +"\n" +msgstr "" + +#: src/help.c:430 +#, c-format +msgid "" +"\n" +"%s restore -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:432 +#, c-format +msgid " [--progress] [--force] [--no-sync]\n" +msgstr "" + +#: src/help.c:436 +#, c-format +msgid " [--skip-external-dirs]\n" +msgstr "" + +#: src/help.c:438 +#, c-format +msgid " [--db-include dbname | --db-exclude dbname]\n" +msgstr "" + +#: src/help.c:446 +#, c-format +msgid " [-R | --restore-as-replica]\n" +msgstr "" + +#: src/help.c:452 +#, c-format +msgid " [--archive-host=hostname] [--archive-port=port]\n" +msgstr "" + +#: src/help.c:453 +#, c-format +msgid "" +" [--archive-user=username]\n" +"\n" +msgstr "" + +#: src/help.c:459 +#, c-format +msgid " -i, --backup-id=backup-id backup to restore\n" +msgstr "" + +#: src/help.c:463 +#, c-format +msgid " --force ignore invalid status of the restored backup\n" +msgstr "" + +#: src/help.c:464 +#, c-format +msgid " --no-sync do not sync restored files to disk\n" +msgstr "" + +#: src/help.c:465 +#, c-format +msgid " --no-validate disable backup validation during restore\n" +msgstr "" + +#: src/help.c:468 src/help.c:1060 +#, c-format +msgid " -T, --tablespace-mapping=OLDDIR=NEWDIR\n" +msgstr "" + +#: src/help.c:469 src/help.c:1061 +#, c-format +msgid " relocate the tablespace from directory OLDDIR to NEWDIR\n" +msgstr "" + +#: src/help.c:470 +#, c-format +msgid " --external-mapping=OLDDIR=NEWDIR\n" +msgstr "" + +#: src/help.c:471 +#, c-format +msgid " relocate the external directory from OLDDIR to NEWDIR\n" +msgstr "" + +#: src/help.c:472 +#, c-format +msgid " --skip-external-dirs do not restore all external directories\n" +msgstr "" + +#: src/help.c:474 +#, c-format +msgid "" +"\n" +" Incremental restore options:\n" +msgstr "" + +#: src/help.c:475 +#, c-format +msgid " -I, --incremental-mode=none|checksum|lsn\n" +msgstr "" + +#: src/help.c:476 +#, c-format +msgid " reuse valid pages available in PGDATA if they have not changed\n" +msgstr "" + +#: src/help.c:477 +#, c-format +msgid " (default: none)\n" +msgstr "" + +#: src/help.c:479 +#, c-format +msgid "" +"\n" +" Partial restore options:\n" +msgstr "" + +#: src/help.c:480 +#, c-format +msgid " --db-include dbname restore only specified databases\n" +msgstr "" + +#: src/help.c:481 +#, c-format +msgid " --db-exclude dbname do not restore specified databases\n" +msgstr "" + +#: src/help.c:483 +#, c-format +msgid "" +"\n" +" Recovery options:\n" +msgstr "" + +#: src/help.c:484 src/help.c:564 +#, c-format +msgid " --recovery-target-time=time time stamp up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:485 src/help.c:565 +#, c-format +msgid " --recovery-target-xid=xid transaction ID up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:486 src/help.c:566 +#, c-format +msgid " --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:487 src/help.c:567 +#, c-format +msgid " --recovery-target-inclusive=boolean\n" +msgstr "" + +#: src/help.c:488 src/help.c:568 +#, c-format +msgid " whether we stop just after the recovery target\n" +msgstr "" + +#: src/help.c:489 src/help.c:569 +#, c-format +msgid " --recovery-target-timeline=timeline\n" +msgstr "" + +#: src/help.c:490 src/help.c:570 +#, c-format +msgid " recovering into a particular timeline\n" +msgstr "" + +#: src/help.c:491 +#, c-format +msgid " --recovery-target=immediate|latest\n" +msgstr "" + +#: src/help.c:492 +#, c-format +msgid " end recovery as soon as a consistent state is reached or as late as possible\n" +msgstr "" + +#: src/help.c:493 src/help.c:571 +#, c-format +msgid " --recovery-target-name=target-name\n" +msgstr "" + +#: src/help.c:494 src/help.c:572 +#, c-format +msgid " the named restore point to which recovery will proceed\n" +msgstr "" + +#: src/help.c:495 +#, c-format +msgid " --recovery-target-action=pause|promote|shutdown\n" +msgstr "" + +#: src/help.c:496 +#, c-format +msgid " action the server should take once the recovery target is reached\n" +msgstr "" + +#: src/help.c:497 +#, c-format +msgid " (default: pause)\n" +msgstr "" + +#: src/help.c:498 src/help.c:818 +#, c-format +msgid " --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n" +msgstr "" + +#: src/help.c:500 +#, c-format +msgid "" +"\n" +" Standby options:\n" +msgstr "" + +#: src/help.c:501 +#, c-format +msgid " -R, --restore-as-replica write a minimal recovery.conf in the output directory\n" +msgstr "" + +#: src/help.c:502 +#, c-format +msgid " to ease setting up a standby server\n" +msgstr "" + +#: src/help.c:503 +#, c-format +msgid " --primary-conninfo=primary_conninfo\n" +msgstr "" + +#: src/help.c:504 +#, c-format +msgid " connection string to be used for establishing connection\n" +msgstr "" + +#: src/help.c:505 +#, c-format +msgid " with the primary server\n" +msgstr "" + +#: src/help.c:506 +#, c-format +msgid " -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n" +msgstr "" + +#: src/help.c:541 src/help.c:876 +#, c-format +msgid "" +"\n" +" Remote WAL archive options:\n" +msgstr "" + +#: src/help.c:542 src/help.c:877 +#, c-format +msgid " --archive-host=destination address or hostname for ssh connection to archive host\n" +msgstr "" + +#: src/help.c:543 src/help.c:878 +#, c-format +msgid " --archive-port=port port for ssh connection to archive host (default: 22)\n" +msgstr "" + +#: src/help.c:544 +#, c-format +msgid "" +" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n" +"\n" +msgstr "" + +#: src/help.c:550 +#, c-format +msgid "" +"\n" +"%s validate -B backup-path [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:556 +#, c-format +msgid "" +" [--skip-block-validation]\n" +"\n" +msgstr "" + +#: src/help.c:560 +#, c-format +msgid " -i, --backup-id=backup-id backup to validate\n" +msgstr "" + +#: src/help.c:595 src/help.c:722 src/help.c:768 +#, c-format +msgid "" +" --no-color disable the coloring of error and warning console messages\n" +"\n" +msgstr "" + +#: src/help.c:601 +#, c-format +msgid "" +"\n" +"%s checkdb [-B backup-path] [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:602 +#, c-format +msgid " [-D pgdata-path] [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:604 +#, c-format +msgid "" +" [--heapallindexed] [--checkunique]\n" +"\n" +msgstr "" + +#: src/help.c:612 +#, c-format +msgid " --skip-block-validation skip file-level checking\n" +msgstr "" + +#: src/help.c:613 src/help.c:618 src/help.c:620 +#, c-format +msgid " can be used only with '--amcheck' option\n" +msgstr "" + +#: src/help.c:614 +#, c-format +msgid " --amcheck in addition to file-level block checking\n" +msgstr "" + +#: src/help.c:615 +#, c-format +msgid " check btree indexes via function 'bt_index_check()'\n" +msgstr "" + +#: src/help.c:616 +#, c-format +msgid " using 'amcheck' or 'amcheck_next' extensions\n" +msgstr "" + +#: src/help.c:617 +#, c-format +msgid " --heapallindexed also check that heap is indexed\n" +msgstr "" + +#: src/help.c:619 +#, c-format +msgid " --checkunique also check unique constraints\n" +msgstr "" + +#: src/help.c:631 +#, c-format +msgid " support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n" +msgstr "" + +#: src/help.c:650 src/help.c:1072 +#, c-format +msgid "" +" -W, --password force password prompt\n" +"\n" +msgstr "" + +#: src/help.c:656 +#, c-format +msgid "" +"\n" +"%s show -B backup-path\n" +msgstr "" + +#: src/help.c:658 +#, c-format +msgid "" +" [--format=format] [--archive]\n" +"\n" +msgstr "" + +#: src/help.c:661 +#, c-format +msgid " --instance=instance_name show info about specific instance\n" +msgstr "" + +#: src/help.c:662 +#, c-format +msgid " -i, --backup-id=backup-id show info about specific backups\n" +msgstr "" + +#: src/help.c:663 +#, c-format +msgid " --archive show WAL archive information\n" +msgstr "" + +#: src/help.c:664 +#, c-format +msgid " --format=format show format=PLAIN|JSON\n" +msgstr "" + +#: src/help.c:665 +#, c-format +msgid "" +" --no-color disable the coloring for plain format\n" +"\n" +msgstr "" + +#: src/help.c:671 +#, c-format +msgid "" +"\n" +"%s delete -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:672 +#, c-format +msgid " [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n" +msgstr "" + +#: src/help.c:677 +#, c-format +msgid "" +" [--no-validate] [--no-sync]\n" +"\n" +msgstr "" + +#: src/help.c:681 +#, c-format +msgid " -i, --backup-id=backup-id backup to delete\n" +msgstr "" + +#: src/help.c:684 src/help.c:745 +#, c-format +msgid " --no-validate disable validation during retention merge\n" +msgstr "" + +#: src/help.c:685 src/help.c:746 +#, c-format +msgid " --no-sync do not sync merged files to disk\n" +msgstr "" + +#: src/help.c:689 src/help.c:691 +#, c-format +msgid " retention policy\n" +msgstr "" + +#: src/help.c:700 +#, c-format +msgid " --status=backup_status delete all backups with specified status\n" +msgstr "" + +#: src/help.c:728 +#, c-format +msgid "" +"\n" +"%s merge -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:729 +#, c-format +msgid " -i backup-id [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:737 +#, c-format +msgid "" +" [--log-rotation-age=log-rotation-age]\n" +"\n" +msgstr "" + +#: src/help.c:741 +#, c-format +msgid " -i, --backup-id=backup-id backup to merge\n" +msgstr "" + +#: src/help.c:774 +#, c-format +msgid "" +"\n" +"%s set-backup -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:775 +#, c-format +msgid " -i backup-id\n" +msgstr "" + +#: src/help.c:776 +#, c-format +msgid "" +" [--ttl=interval] [--expire-time=time] [--note=text]\n" +"\n" +msgstr "" + +#: src/help.c:783 +#, c-format +msgid " --note=text add note to backup; 'none' to remove note\n" +msgstr "" + +#: src/help.c:790 +#, c-format +msgid "" +"\n" +"%s set-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:810 src/help.c:908 src/help.c:952 src/help.c:996 +#, c-format +msgid "" +" [--ssh-options]\n" +"\n" +msgstr "" + +#: src/help.c:846 +#, c-format +msgid " --wal-depth=wal-depth number of latest valid backups with ability to perform\n" +msgstr "" + +#: src/help.c:847 +#, c-format +msgid " the point in time recovery; disables; (default: 0)\n" +msgstr "" + +#: src/help.c:852 src/help.c:970 +#, c-format +msgid " available options: 'zlib','pglz','none' (default: 'none')\n" +msgstr "" + +#: src/help.c:879 +#, c-format +msgid " --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n" +msgstr "" + +#: src/help.c:892 +#, c-format +msgid "" +"\n" +"%s show-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:893 +#, c-format +msgid "" +" [--format=format]\n" +"\n" +msgstr "" + +#: src/help.c:897 +#, c-format +msgid "" +" --format=format show format=PLAIN|JSON\n" +"\n" +msgstr "" + +#: src/help.c:903 +#, c-format +msgid "" +"\n" +"%s add-instance -B backup-path -D pgdata-path\n" +msgstr "" + +#: src/help.c:905 +#, c-format +msgid " [-E external-directory-path]\n" +msgstr "" + +#: src/help.c:912 +#, c-format +msgid " --instance=instance_name name of the new instance\n" +msgstr "" + +#: src/help.c:926 src/help.c:983 src/help.c:1018 src/help.c:1083 +#, c-format +msgid "" +" (example: --ssh-options='-c cipher_spec -F configfile')\n" +"\n" +msgstr "" + +#: src/help.c:932 +#, c-format +msgid "" +"\n" +"%s del-instance -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:935 +#, c-format +msgid "" +" --instance=instance_name name of the instance to delete\n" +"\n" +msgstr "" + +#: src/help.c:941 +#, c-format +msgid "" +"\n" +"%s archive-push -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:955 src/help.c:999 +#, c-format +msgid " --instance=instance_name name of the instance to delete\n" +msgstr "" + +#: src/help.c:956 src/help.c:1002 +#, c-format +msgid " --wal-file-name=wal-file-name\n" +msgstr "" + +#: src/help.c:957 +#, c-format +msgid " name of the file to copy into WAL archive\n" +msgstr "" + +#: src/help.c:958 src/help.c:1000 +#, c-format +msgid " --wal-file-path=wal-file-path\n" +msgstr "" + +#: src/help.c:959 +#, c-format +msgid " relative destination path of the WAL archive\n" +msgstr "" + +#: src/help.c:961 +#, c-format +msgid " --batch-size=NUM number of files to be copied\n" +msgstr "" + +#: src/help.c:962 +#, c-format +msgid " --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n" +msgstr "" + +#: src/help.c:963 +#, c-format +msgid " --no-ready-rename do not rename '.ready' files in 'archive_status' directory\n" +msgstr "" + +#: src/help.c:964 +#, c-format +msgid " --no-sync do not sync WAL file to disk\n" +msgstr "" + +#: src/help.c:965 +#, c-format +msgid " --overwrite overwrite archived WAL file\n" +msgstr "" + +#: src/help.c:977 src/help.c:1012 src/help.c:1077 +#, c-format +msgid " --remote-host=hostname remote host address or hostname\n" +msgstr "" + +#: src/help.c:989 +#, c-format +msgid "" +"\n" +"%s archive-get -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:1001 +#, c-format +msgid " relative destination path name of the WAL file on the server\n" +msgstr "" + +#: src/help.c:1003 +#, c-format +msgid " name of the WAL file to retrieve from the archive\n" +msgstr "" + +#: src/help.c:1005 +#, c-format +msgid " --batch-size=NUM number of files to be prefetched\n" +msgstr "" + +#: src/help.c:1006 +#, c-format +msgid " --prefetch-dir=path location of the store area for prefetched WAL files\n" +msgstr "" + +#: src/help.c:1007 +#, c-format +msgid " --no-validate-wal skip validation of prefetched WAL file before using it\n" +msgstr "" + +#: src/help.c:1024 +#, c-format +msgid "" +"\n" +"%s help [command]\n" +msgstr "" + +#: src/help.c:1025 +#, c-format +msgid "" +"%s command --help\n" +"\n" +msgstr "" + +#: src/help.c:1031 +#, c-format +msgid "" +"\n" +"%s version\n" +msgstr "" + +#: src/help.c:1032 +#, c-format +msgid "" +"%s --version\n" +"\n" +msgstr "" + +#: src/help.c:1038 +#, c-format +msgid "" +"\n" +"%s catchup -b catchup-mode\n" +msgstr "" + +#: src/help.c:1041 +#, c-format +msgid " [--stream [-S slot-name]] [--temp-slot | --perm-slot]\n" +msgstr "" + +#: src/help.c:1050 +#, c-format +msgid "" +" [--help]\n" +"\n" +msgstr "" + +#: src/help.c:1052 +#, c-format +msgid " -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n" +msgstr "" + +#: src/help.c:1053 +#, c-format +msgid " --stream stream the transaction log (only supported mode)\n" +msgstr "" + +#: src/help.c:1056 +#, c-format +msgid " -P --perm-slot create permanent replication slot\n" +msgstr "" + +#: src/help.c:1062 +#, c-format +msgid " -x, --exclude-path=path_prefix files with path_prefix (relative to pgdata) will be\n" +msgstr "" + +#: src/help.c:1063 +#, c-format +msgid " excluded from catchup (can be used multiple times)\n" +msgstr "" + +#: src/help.c:1064 +#, c-format +msgid " Dangerous option! Use at your own risk!\n" +msgstr "" diff --git a/src/help.c b/src/help.c index a494ab209..8ebe734a3 100644 --- a/src/help.c +++ b/src/help.c @@ -267,9 +267,9 @@ help_pg_probackup(void) { printf("\n"); if (PROGRAM_URL) - printf("Read the website for details. <%s>\n", PROGRAM_URL); + printf(_("Read the website for details <%s>.\n"), PROGRAM_URL); if (PROGRAM_EMAIL) - printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + printf(_("Report bugs to <%s>.\n"), PROGRAM_EMAIL); } } diff --git a/src/pg_probackup.c b/src/pg_probackup.c index c5ed13175..b9b3af0b9 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -308,6 +308,7 @@ main(int argc, char *argv[]) init_config(&instance_config, instance_name); PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_probackup")); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); /* Get current time */ diff --git a/tests/Readme.md b/tests/Readme.md index 668552c94..ed1b22e03 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -31,7 +31,7 @@ Remote backup depends on key authentication to local machine via ssh as current export PGPROBACKUP_SSH_REMOTE=ON Run tests that are relied on advanced debugging features. For this mode, pg_probackup should be compiled without optimizations. For example: -CFLAGS="-O0" ./configure --prefix=/path/to/prefix --enable-debug --enable-cassert --enable-depend --enable-tap-tests +CFLAGS="-O0" ./configure --prefix=/path/to/prefix --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls export PGPROBACKUP_GDB=ON diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index a8b4a64b3..00b50d10c 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -180,5 +180,5 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--help] -Read the website for details. +Read the website for details . Report bugs to . diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out new file mode 100644 index 000000000..ee8da9a1c --- /dev/null +++ b/tests/expected/option_help_ru.out @@ -0,0 +1,184 @@ + +pg_probackup - утилита для управления резервным копированием/восстановлением базы данных PostgreSQL. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-path + + pg_probackup set-config -B backup-path --instance=instance_name + [-D pgdata-path] + [--external-dirs=external-directories-paths] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--archive-timeout=timeout] + [-d dbname] [-h host] [-p port] [-U username] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--restore-command=cmdline] [--archive-host=destination] + [--archive-port=port] [--archive-user=username] + [--help] + + pg_probackup set-backup -B backup-path --instance=instance_name + -i backup-id [--ttl=interval] [--expire-time=timestamp] + [--note=text] + [--help] + + pg_probackup show-config -B backup-path --instance=instance_name + [--format=format] + [--help] + + pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + [-D pgdata-path] [-C] + [--stream [-S slot-name] [--temp-slot]] + [--backup-pg-log] [-j num-threads] [--progress] + [--no-validate] [--skip-block-validation] + [--external-dirs=external-directories-paths] + [--no-sync] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] [--no-color] + [--delete-expired] [--delete-wal] [--merge-expired] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--archive-timeout=archive-timeout] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--help] + + pg_probackup restore -B backup-path --instance=instance_name + [-D pgdata-path] [-i backup-id] [-j num-threads] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] + [--recovery-target=immediate|latest] + [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-command=cmdline] + [-R | --restore-as-replica] [--force] + [--primary-conninfo=primary_conninfo] + [-S | --primary-slot-name=slotname] + [--no-validate] [--skip-block-validation] + [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] + [--skip-external-dirs] [--no-sync] + [-I | --incremental-mode=none|checksum|lsn] + [--db-include | --db-exclude] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--archive-host=hostname] + [--archive-port=port] [--archive-user=username] + [--help] + + pg_probackup validate -B backup-path [--instance=instance_name] + [-i backup-id] [--progress] [-j num-threads] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] + [--recovery-target-name=target-name] + [--skip-block-validation] + [--help] + + pg_probackup checkdb [-B backup-path] [--instance=instance_name] + [-D pgdata-path] [--progress] [-j num-threads] + [--amcheck] [--skip-block-validation] + [--heapallindexed] [--checkunique] + [--help] + + pg_probackup show -B backup-path + [--instance=instance_name [-i backup-id]] + [--format=format] [--archive] + [--no-color] [--help] + + pg_probackup delete -B backup-path --instance=instance_name + [-j num-threads] [--progress] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] + [--delete-wal] + [--dry-run] [--no-validate] [--no-sync] + [--help] + + pg_probackup merge -B backup-path --instance=instance_name + -i backup-id [--progress] [-j num-threads] + [--no-validate] [--no-sync] + [--help] + + pg_probackup add-instance -B backup-path -D pgdata-path + --instance=instance_name + [--external-dirs=external-directories-paths] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup del-instance -B backup-path + --instance=instance_name + [--help] + + pg_probackup archive-push -B backup-path --instance=instance_name + --wal-file-name=wal-file-name + [--wal-file-path=wal-file-path] + [-j num-threads] [--batch-size=batch_size] + [--archive-timeout=timeout] + [--no-ready-rename] [--no-sync] + [--overwrite] [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup archive-get -B backup-path --instance=instance_name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [-j num-threads] [--batch-size=batch_size] + [--no-validate-wal] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup catchup -b catchup-mode + --source-pgdata=path_to_pgdata_on_remote_server + --destination-pgdata=path_to_local_dir + [--stream [-S slot-name] [--temp-slot | --perm-slot]] + [-j num-threads] + [-T OLDDIR=NEWDIR] + [--exclude-path=path_prefix] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + +Подробнее читайте на сайте . +Сообщайте об ошибках в . diff --git a/tests/option.py b/tests/option.py index 023a0c2c6..b57d7ef43 100644 --- a/tests/option.py +++ b/tests/option.py @@ -1,6 +1,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import locale module_name = 'option' @@ -226,3 +227,13 @@ def test_options_5(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_help_6(self): + """help options""" + self.test_env['LC_ALL'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 52b05105b..a62ad4de7 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -47,7 +47,7 @@ cd postgres # Go to postgres dir if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi -CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests +CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls make -s -j$(nproc) install #make -s -j$(nproc) -C 'src/common' install #make -s -j$(nproc) -C 'src/port' install From 68b77a06bca055c24ef00dee1896409a0beb923b Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 25 May 2022 14:45:25 +0300 Subject: [PATCH 1840/2107] [PBCKP-150] Reading buffer is flushed each time we verify the checksum. (#487) The race condition is covered with a unit-test, the buffer is flushed now so each of 300 reads requests the data from the disc. --- .travis.yml | 1 + src/data.c | 2 ++ tests/Readme.md | 2 ++ tests/__init__.py | 8 ++++- tests/time_consuming.py | 76 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/time_consuming.py diff --git a/.travis.yml b/.travis.yml index 8e325c64f..26b2bc4e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ env: # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=retention # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=restore +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=time_consuming jobs: allow_failures: diff --git a/src/data.c b/src/data.c index 052e17486..e5a551127 100644 --- a/src/data.c +++ b/src/data.c @@ -349,6 +349,8 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, Assert(false); } } + /* avoid re-reading once buffered data, flushing on further attempts, see PBCKP-150 */ + fflush(in); } /* diff --git a/tests/Readme.md b/tests/Readme.md index ed1b22e03..500ed7c7a 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -41,6 +41,8 @@ Run suit of basic simple tests: Run ptrack tests: export PG_PROBACKUP_PTRACK=ON +Run long (time consuming) tests: + export PG_PROBACKUP_LONG=ON Usage: sudo echo 0 > /proc/sys/kernel/yama/ptrace_scope diff --git a/tests/__init__.py b/tests/__init__.py index 55d6ea9be..79537ad78 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,7 @@ compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ cfs_validate_backup, auth_test, time_stamp, logging, \ locking, remote, external, config, checkdb, set_backup, incr_restore, \ - catchup, CVE_2018_1058 + catchup, CVE_2018_1058, time_consuming def load_tests(loader, tests, pattern): @@ -21,6 +21,12 @@ def load_tests(loader, tests, pattern): if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': suite.addTests(loader.loadTestsFromModule(ptrack)) + # PG_PROBACKUP_LONG section for tests that are long + # by design e.g. they contain loops, sleeps and so on + if 'PG_PROBACKUP_LONG' in os.environ: + if os.environ['PG_PROBACKUP_LONG'] == 'ON': + suite.addTests(loader.loadTestsFromModule(time_consuming)) + # suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) diff --git a/tests/time_consuming.py b/tests/time_consuming.py new file mode 100644 index 000000000..396ab716e --- /dev/null +++ b/tests/time_consuming.py @@ -0,0 +1,76 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest +import subprocess +from time import sleep + +module_name = 'time_consuming' + +class TimeConsumingTests(ProbackupTest, unittest.TestCase): + def test_pbckp150(self): + """ + https://jira.postgrespro.ru/browse/PBCKP-150 + create a node filled with pgbench + create FULL backup followed by PTRACK backup + run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel + """ + # init node + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + node.append_conf('postgresql.conf', + """ + max_connections = 100 + wal_keep_size = 16000 + ptrack.map_size = 1 + shared_preload_libraries='ptrack' + log_statement = 'none' + fsync = off + log_checkpoints = on + autovacuum = off + """) + + # init probackup and add an instance + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # run the node and init ptrack + node.slow_start() + node.safe_psql("postgres", "CREATE EXTENSION ptrack") + # populate it with pgbench + node.pgbench_init(scale=5) + + # FULL backup followed by PTRACK backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + # run ordinary pgbench scenario to imitate some activity and another pgbench for vacuuming in parallel + nBenchDuration = 30 + pgbench = node.pgbench(options=['-c', '20', '-j', '8', '-T', str(nBenchDuration)]) + with open('/tmp/pbckp150vacuum.sql', 'w') as f: + f.write('VACUUM (FULL) pgbench_accounts, pgbench_tellers, pgbench_history; SELECT pg_sleep(1);\n') + pgbenchval = node.pgbench(options=['-c', '1', '-f', '/tmp/pbckp150vacuum.sql', '-T', str(nBenchDuration)]) + + # several PTRACK backups + for i in range(nBenchDuration): + print("[{}] backing up PTRACK diff...".format(i+1)) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) + sleep(0.1) + # if the activity pgbench has finished, stop backing up + if pgbench.poll() is not None: + break + + pgbench.kill() + pgbenchval.kill() + pgbench.wait() + pgbenchval.wait() + + backups = self.show_pb(backup_dir, 'node') + for b in backups: + self.assertEqual("OK", b['status']) + + # Clean after yourself + self.del_test_dir(module_name, fname) From 0b5b37e8930e75793b23e0829d2f57cc5a13a34d Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Thu, 26 May 2022 19:53:01 +0700 Subject: [PATCH 1841/2107] Add a workflow to build and test probackup on Windows (#484) * Add a workflow to build and test probackup on Windows * [PBCKP-149] fix test_basic_validate_nullified_heap_page_backup for windows Co-authored-by: Alexey Savchkov Co-authored-by: Mikhail A. Kulagin --- .github/workflows/build.yml | 94 +++++++++++++++++++++++++++++++++ gen_probackup_project.pl | 12 ++--- tests/helpers/ptrack_helpers.py | 6 +-- tests/validate.py | 5 +- 4 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..ab1a5888d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,94 @@ +name: Build Probackup + +on: + push: + branches: + - "**" + # Runs triggered by pull requests are disabled to prevent executing potentially unsafe code from public pull requests + # pull_request: + # branches: + # - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + + build-win2019: + + runs-on: + - windows-2019 + + env: + zlib_dir: C:\dep\zlib + + steps: + + - uses: actions/checkout@v2 + + - name: Install pacman packages + run: | + $env:PATH += ";C:\msys64\usr\bin" + pacman -S --noconfirm --needed bison flex + + - name: Make zlib + run: | + git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib.git + cd zlib + cmake -DCMAKE_INSTALL_PREFIX:PATH=C:\dep\zlib -G "Visual Studio 16 2019" . + cmake --build . --config Release --target ALL_BUILD + cmake --build . --config Release --target INSTALL + copy C:\dep\zlib\lib\zlibstatic.lib C:\dep\zlib\lib\zdll.lib + copy C:\dep\zlib\bin\zlib.dll C:\dep\zlib\lib + + - name: Get Postgres sources + run: git clone -b REL_14_STABLE https://github.com/postgres/postgres.git + + # Copy ptrack to contrib to build the ptrack extension + # Convert line breaks in the patch file to LF otherwise the patch doesn't apply + - name: Get Ptrack sources + run: | + git clone -b master --depth 1 https://github.com/postgrespro/ptrack.git + Copy-Item -Path ptrack -Destination postgres\contrib -Recurse + (Get-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Raw).Replace("`r`n","`n") | Set-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Force -NoNewline + cd postgres + git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff + + - name: Build Postgres + run: | + $env:PATH += ";C:\msys64\usr\bin" + cd postgres\src\tools\msvc + (Get-Content config_default.pl) -Replace "zlib *=>(.*?)(?=,? *#)", "zlib => '${{ env.zlib_dir }}'" | Set-Content config.pl + cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && .\build.bat" + + - name: Build Probackup + run: cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && perl .\gen_probackup_project.pl `"${{ github.workspace }}`"\postgres" + + - name: Install Postgres + run: | + cd postgres + src\tools\msvc\install.bat postgres_install + + - name: Install Testgres + run: | + git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git + cd testgres + python setup.py install + + # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder + - name: Test Probackup + run: | + icacls.exe "${{ github.workspace }}" /grant "${env:USERNAME}:(OI)(CI)F" + $env:PATH += ";${{ github.workspace }}\postgres\postgres_install\lib;${{ env.zlib_dir }}\lib" + $Env:LC_MESSAGES = "English" + $Env:PG_CONFIG = "${{ github.workspace }}\postgres\postgres_install\bin\pg_config.exe" + $Env:PGPROBACKUPBIN = "${{ github.workspace }}\postgres\Release\pg_probackup\pg_probackup.exe" + $Env:PG_PROBACKUP_PTRACK = "ON" + If (!$Env:MODE -Or $Env:MODE -Eq "basic") { + $Env:PG_PROBACKUP_TEST_BASIC = "ON" + python -m unittest -v tests + python -m unittest -v tests.init + } else { + python -m unittest -v tests.$Env:MODE + } + diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index c24db1228..8143b7d0d 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -13,11 +13,11 @@ BEGIN { $pgsrc = shift @ARGV; if($pgsrc eq "--help"){ - print STDERR "Usage $0 pg-source-dir \n"; - print STDERR "Like this: \n"; - print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; - print STDERR "May be need input this before: \n"; - print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + print STDERR "Usage $0 pg-source-dir\n"; + print STDERR "Like this:\n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro\n"; + print STDERR "May need to run this first:\n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat\" amd64\n"; exit 1; } } @@ -133,7 +133,7 @@ sub build_pgprobackup unless (-d 'src/tools/msvc' && -d 'src'); # my $vsVersion = DetermineVisualStudioVersion(); - my $vsVersion = '12.00'; + my $vsVersion = '16.00'; $solution = CreateSolution($vsVersion, $config); diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 3b14b7170..ffb87c5ec 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -89,11 +89,7 @@ def dir_files(base_dir): def is_enterprise(): # pg_config --help - if os.name == 'posix': - cmd = [os.environ['PG_CONFIG'], '--help'] - - elif os.name == 'nt': - cmd = [[os.environ['PG_CONFIG']], ['--help']] + cmd = [os.environ['PG_CONFIG'], '--help'] p = subprocess.Popen( cmd, diff --git a/tests/validate.py b/tests/validate.py index 0b04d92fe..e62826388 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -2,6 +2,7 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta +from pathlib import Path import subprocess from sys import exit import time @@ -58,7 +59,7 @@ def test_basic_validate_nullified_heap_page_backup(self): with open(log_file_path) as f: log_content = f.read() self.assertIn( - 'File: "{0}" blknum 1, empty page'.format(file), + 'File: "{0}" blknum 1, empty page'.format(Path(file).as_posix()), log_content, 'Failed to detect nullified block') @@ -4247,4 +4248,4 @@ def test_no_validate_tablespace_map(self): # 715 MAXALIGN(header.compressed_size), in); # 716 if (read_len != MAXALIGN(header.compressed_size)) # -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", -# 718 blknum, file->path, read_len, header.compressed_size); \ No newline at end of file +# 718 blknum, file->path, read_len, header.compressed_size); From 7be2e738a923bd65026cd7c95150d5a67d0ec228 Mon Sep 17 00:00:00 2001 From: avaness Date: Fri, 27 May 2022 18:56:38 +0300 Subject: [PATCH 1842/2107] PBCKP-145: added check of unlogged table is restored as empty table (#490) --- tests/exclude.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/exclude.py b/tests/exclude.py index b98a483d0..2c4925881 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -203,8 +203,10 @@ def test_exclude_unlogged_tables_1(self): # @unittest.skip("skip") def test_exclude_unlogged_tables_2(self): """ - make node, create unlogged, take FULL, check - that unlogged was not backed up + 1. make node, create unlogged, take FULL, DELTA, PAGE, + check that unlogged table files was not backed up + 2. restore FULL, DELTA, PAGE to empty db, + ensure unlogged table exist and is epmty """ fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -220,6 +222,8 @@ def test_exclude_unlogged_tables_2(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + backup_ids = [] + for backup_type in ['full', 'delta', 'page']: if backup_type == 'full': @@ -231,14 +235,16 @@ def test_exclude_unlogged_tables_2(self): 'postgres', 'insert into test select generate_series(0,20050000)::text') - rel_path = node.safe_psql( + rel_path = node.execute( 'postgres', - "select pg_relation_filepath('test')").decode('utf-8').rstrip() + "select pg_relation_filepath('test')")[0][0] backup_id = self.backup_node( backup_dir, 'node', node, backup_type=backup_type, options=['--stream']) + backup_ids.append(backup_id) + filelist = self.get_backup_filelist( backup_dir, 'node', backup_id) @@ -258,9 +264,25 @@ def test_exclude_unlogged_tables_2(self): rel_path + '.3', filelist, "Unlogged table was not excluded") + # ensure restoring retrieves back only empty unlogged table + for backup_id in backup_ids: + node.stop() + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + node.slow_start() + + self.assertEqual( + node.execute( + 'postgres', + 'select count(*) from test')[0][0], + 0) + # Clean after yourself self.del_test_dir(module_name, fname) + # @unittest.skip("skip") def test_exclude_log_dir(self): """ From 02aef65853aa04fc6611b82904d8dfbfe59fdecd Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Tue, 31 May 2022 11:21:59 +0500 Subject: [PATCH 1843/2107] Fix is_enterprise checking in ptrack_helpers.py --- tests/helpers/ptrack_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 3b14b7170..a4ec7c9cf 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -90,17 +90,17 @@ def dir_files(base_dir): def is_enterprise(): # pg_config --help if os.name == 'posix': - cmd = [os.environ['PG_CONFIG'], '--help'] + cmd = [os.environ['PG_CONFIG'], '--pgpro-edition'] elif os.name == 'nt': - cmd = [[os.environ['PG_CONFIG']], ['--help']] + cmd = [[os.environ['PG_CONFIG']], ['--pgpro-edition']] p = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if b'postgrespro.ru' in p.communicate()[0]: + if b'enterprise' in p.communicate()[0]: return True else: return False From 55a74902fde5c811c02b0ec2e64d8be89762e76a Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Tue, 31 May 2022 12:49:20 +0500 Subject: [PATCH 1844/2107] Fix test_checkdb_with_least_privileges. Add GRANT EXECUTE on function pgpro_edition for amcheck indexes --- tests/checkdb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/checkdb.py b/tests/checkdb.py index 9b7adcd71..e066c5777 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -726,6 +726,9 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) if ProbackupTest.enterprise: # amcheck-1.1 @@ -766,6 +769,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) # checkunique parameter if ProbackupTest.enterprise: @@ -804,6 +808,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) # checkunique parameter if ProbackupTest.enterprise: @@ -811,11 +816,6 @@ def test_checkdb_with_least_privileges(self): "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - # checkdb try: self.checkdb_node( From 5f2283c8deac88ea49ea6223a3aa72e2cf462eb5 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Fri, 27 May 2022 14:00:10 +0500 Subject: [PATCH 1845/2107] Add backup start time as parameter for do_backup --- src/backup.c | 4 ++-- src/catalog.c | 35 +++++++++++++++++++++++++++-------- src/pg_probackup.c | 5 ++++- src/pg_probackup.h | 4 ++-- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/backup.c b/src/backup.c index c575865c4..e8477da4c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -695,7 +695,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) */ int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs) + bool no_validate, bool no_sync, bool backup_logs, time_t start_time) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -710,7 +710,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, current.external_dir_str = instance_config.external_dir_str; /* Create backup directory and BACKUP_CONTROL_FILE */ - pgBackupCreateDir(¤t, instanceState->instance_backup_subdir_path); + pgBackupCreateDir(¤t, instanceState, start_time); if (!instance_config.pgdata) elog(ERROR, "required parameter not specified: PGDATA " diff --git a/src/catalog.c b/src/catalog.c index b4ed8c189..516ee0ff8 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -23,7 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); -static time_t create_backup_dir(pgBackup *backup, const char *backup_instance_path); +static void create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; static parray *locks = NULL; @@ -1420,10 +1420,12 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, */ void -pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) +pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time) { int i; parray *subdirs = parray_new(); + parray * backups; + pgBackup *target_backup; parray_append(subdirs, pg_strdup(DATABASE_DIR)); @@ -1444,7 +1446,26 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) free_dir_list(external_list); } - backup->backup_id = create_backup_dir(backup, backup_instance_path); + /* Get list of all backups*/ + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); + if (parray_num(backups) > 0) + { + target_backup = (pgBackup *) parray_get(backups, 0); + if (start_time > target_backup->backup_id) + { + backup->backup_id = start_time; + create_backup_dir(backup, instanceState->instance_backup_subdir_path); + } + else + { + elog(ERROR, "Cannot create directory for older backup"); + } + } + else + { + backup->backup_id = start_time; + create_backup_dir(backup, instanceState->instance_backup_subdir_path); + } if (backup->backup_id == 0) elog(ERROR, "Cannot create backup directory: %s", strerror(errno)); @@ -1471,7 +1492,7 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) * Create root directory for backup, * update pgBackup.root_dir if directory creation was a success */ -time_t +void create_backup_dir(pgBackup *backup, const char *backup_instance_path) { int attempts = 10; @@ -1480,9 +1501,8 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) { int rc; char path[MAXPGPATH]; - time_t backup_id = time(NULL); - join_path_components(path, backup_instance_path, base36enc(backup_id)); + join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); /* TODO: add wrapper for remote mode */ rc = dir_create_dir(path, DIR_PERMISSION, true); @@ -1490,7 +1510,7 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) if (rc == 0) { backup->root_dir = pgut_strdup(path); - return backup_id; + return; } else { @@ -1499,7 +1519,6 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) } } - return 0; } /* diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b9b3af0b9..8d45d6e7f 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -939,6 +939,9 @@ main(int argc, char *argv[]) return do_init(catalogState); case BACKUP_CMD: { + time_t start_time; + time(&start_time); + current.stream = stream_wal; /* sanity */ @@ -947,7 +950,7 @@ main(int argc, char *argv[]) "(-b, --backup-mode)"); return do_backup(instanceState, set_backup_params, - no_validate, no_sync, backup_logs); + no_validate, no_sync, backup_logs, start_time); } case CATCHUP_CMD: return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync, diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4cd65980c..e4159f4ab 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -840,7 +840,7 @@ extern char** commands_args; /* in backup.c */ extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs); + bool no_validate, bool no_sync, bool backup_logs, time_t start_time); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -981,7 +981,7 @@ extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path); +extern void pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); From 3c74ebf2f9a4eb22b9a9dfb955d3379e5c217f48 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Tue, 31 May 2022 18:03:31 +0500 Subject: [PATCH 1846/2107] Add --start-time option for backup --- src/pg_probackup.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 8d45d6e7f..15f2542b0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,6 +78,7 @@ pid_t my_pid = 0; __thread int my_thread_num = 1; bool progress = false; bool no_sync = false; +time_t start_time = 0; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; bool temp_slot = false; @@ -200,6 +201,7 @@ static ConfigOption cmd_options[] = { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, + { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -939,10 +941,9 @@ main(int argc, char *argv[]) return do_init(catalogState); case BACKUP_CMD: { - time_t start_time; - time(&start_time); - current.stream = stream_wal; + if (start_time == 0) + start_time = current_time; /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) From c81c54be4cac6f900e6b73df06788f349eecb3af Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Tue, 31 May 2022 18:35:54 +0500 Subject: [PATCH 1847/2107] Add --start-time option into help message --- src/help.c | 3 +++ tests/expected/option_help.out | 1 + 2 files changed, 4 insertions(+) diff --git a/src/help.c b/src/help.c index 8ebe734a3..7a1a1c580 100644 --- a/src/help.c +++ b/src/help.c @@ -150,6 +150,7 @@ help_pg_probackup(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n")); + printf(_(" [--start-time]\n")); printf(_(" [--help]\n")); @@ -323,6 +324,7 @@ help_backup(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); + printf(_(" [--start-time]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -343,6 +345,7 @@ help_backup(void) printf(_(" --no-sync do not sync backed up files to disk\n")); printf(_(" --note=text add note to backup\n")); printf(_(" (example: --note='backup before app update to v13.1')\n")); + printf(_(" --start-time set time of starting backup as a parameter for naming backup\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 00b50d10c..9026b99b3 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -68,6 +68,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--start-time] [--help] pg_probackup restore -B backup-path --instance=instance_name From 884e8b09f315a7ff9b53bea6f8395b44d0ed22f2 Mon Sep 17 00:00:00 2001 From: dlepikhova <43872363+dlepikhova@users.noreply.github.com> Date: Wed, 1 Jun 2022 12:49:09 +0500 Subject: [PATCH 1848/2107] [pbckp-128] dry-run option for catchup (#477) * Added dry-run option for catchup. Run catchup without affect on the files and WAL --- src/catchup.c | 84 +++++++++++------- src/help.c | 4 + tests/catchup.py | 154 +++++++++++++++++++++++++++++++++ tests/expected/option_help.out | 1 + travis/run_tests.sh | 9 ++ 5 files changed, 220 insertions(+), 32 deletions(-) diff --git a/src/catchup.c b/src/catchup.c index 1b8f8084d..3c522afb7 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -2,7 +2,7 @@ * * catchup.c: sync DB cluster * - * Copyright (c) 2021, Postgres Professional + * Copyright (c) 2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -507,16 +507,20 @@ catchup_multithreaded_copy(int num_threads, /* Run threads */ thread_interrupted = false; threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - for (i = 0; i < num_threads; i++) + if (!dry_run) { - elog(VERBOSE, "Start thread num: %i", i); - pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i])); + for (i = 0; i < num_threads; i++) + { + elog(VERBOSE, "Start thread num: %i", i); + pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i])); + } } /* Wait threads */ for (i = 0; i < num_threads; i++) { - pthread_join(threads[i], NULL); + if (!dry_run) + pthread_join(threads[i], NULL); all_threads_successful &= threads_args[i].completed; transfered_bytes_result += threads_args[i].transfered_bytes; } @@ -706,9 +710,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Start stream replication */ join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); - fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); - start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli, false); + if (!dry_run) + { + fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); + start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, + current.start_lsn, current.tli, false); + } + else + elog(INFO, "WAL streaming skipping with --dry-run option"); source_filelist = parray_new(); @@ -779,9 +788,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Build the page map from ptrack information */ make_pagemap_from_ptrack_2(source_filelist, source_conn, - source_node_info.ptrack_schema, - source_node_info.ptrack_version_num, - dest_redo.lsn); + source_node_info.ptrack_schema, + source_node_info.ptrack_version_num, + dest_redo.lsn); time(&end_time); elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", difftime(end_time, start_time)); @@ -820,9 +829,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char dirpath[MAXPGPATH]; join_path_components(dirpath, dest_pgdata, file->rel_path); - elog(VERBOSE, "Create directory '%s'", dirpath); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); + if (!dry_run) + fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); } else { @@ -853,15 +862,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", linked_path, to_path); - /* create tablespace directory */ - if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) - elog(ERROR, "Could not create tablespace directory \"%s\": %s", - linked_path, strerror(errno)); - - /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) - elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", - linked_path, to_path, strerror(errno)); + if (!dry_run) + { + /* create tablespace directory */ + if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) + elog(ERROR, "Could not create tablespace directory \"%s\": %s", + linked_path, strerror(errno)); + + /* create link to linked_path */ + if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) + elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", + linked_path, to_path, strerror(errno)); + } } } @@ -930,7 +942,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char fullpath[MAXPGPATH]; join_path_components(fullpath, dest_pgdata, file->rel_path); - fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); + if (!dry_run) + { + fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); + } elog(VERBOSE, "Deleted file \"%s\"", fullpath); /* shrink dest pgdata list */ @@ -961,7 +976,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, catchup_isok = transfered_datafiles_bytes != -1; /* at last copy control file */ - if (catchup_isok) + if (catchup_isok && !dry_run) { char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; @@ -972,7 +987,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, transfered_datafiles_bytes += source_pg_control_file->size; } - if (!catchup_isok) + if (!catchup_isok && !dry_run) { char pretty_time[20]; char pretty_transfered_data_bytes[20]; @@ -1010,14 +1025,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, pg_free(stop_backup_query_text); } - wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); + if (!dry_run) + wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); #if PG_VERSION_NUM >= 90600 /* Write backup_label */ Assert(stop_backup_result.backup_label_content != NULL); - pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", - stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, - NULL); + if (!dry_run) + { + pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, + NULL); + } free(stop_backup_result.backup_label_content); stop_backup_result.backup_label_content = NULL; stop_backup_result.backup_label_content_len = 0; @@ -1040,6 +1059,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, #endif /* wait for end of wal streaming and calculate wal size transfered */ + if (!dry_run) { parray *wal_files_list = NULL; wal_files_list = parray_new(); @@ -1091,17 +1111,17 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, } /* Sync all copied files unless '--no-sync' flag is used */ - if (sync_dest_files) + if (sync_dest_files && !dry_run) catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); else elog(WARNING, "Files are not synced to disk"); /* Cleanup */ - if (dest_filelist) + if (dest_filelist && !dry_run) { parray_walk(dest_filelist, pgFileFree); - parray_free(dest_filelist); } + parray_free(dest_filelist); parray_walk(source_filelist, pgFileFree); parray_free(source_filelist); pgFileFree(source_pg_control_file); diff --git a/src/help.c b/src/help.c index 8ebe734a3..b22fa912e 100644 --- a/src/help.c +++ b/src/help.c @@ -261,6 +261,7 @@ help_pg_probackup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--dry-run]\n")); printf(_(" [--help]\n")); if ((PROGRAM_URL || PROGRAM_EMAIL)) @@ -1047,6 +1048,7 @@ help_catchup(void) printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); + printf(_(" [--dry-run]\n")); printf(_(" [--help]\n\n")); printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n")); @@ -1081,4 +1083,6 @@ help_catchup(void) printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); + + printf(_(" --dry-run perform a trial run without any changes\n\n")); } diff --git a/tests/catchup.py b/tests/catchup.py index 8441deaaf..a83755c54 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -1455,3 +1455,157 @@ def test_config_exclusion(self): dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') self.del_test_dir(module_name, self.fname) + +######################################### +# --dry-run +######################################### + def test_dry_run_catchup_full(self): + """ + Test dry-run option for full catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do full catchup + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run'] + ) + + # compare data dirs before and after catchup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_dry_run_catchup_ptrack(self): + """ + Test dry-run option for catchup in incremental ptrack mode + """ + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do incremental catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run'] + ) + + # compare data dirs before and after cathup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + + def test_dry_run_catchup_delta(self): + """ + Test dry-run option for catchup in incremental delta mode + """ + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True, + initdb_params = ['--data-checksums'], + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', "--dry-run"] + ) + + # compare data dirs before and after cathup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + self.del_test_dir(module_name, self.fname) + diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 00b50d10c..8a1de1f67 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -178,6 +178,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] Read the website for details . diff --git a/travis/run_tests.sh b/travis/run_tests.sh index a62ad4de7..37614f970 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -100,11 +100,20 @@ source pyenv/bin/activate pip3 install testgres echo "############### Testing:" +echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA} +echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION} +echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD} +echo PGPROBACKUPBIN=${PGPROBACKUPBIN} +echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} +echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} +echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests python3 -m unittest -v tests.init else + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests.$MODE fi From 8bb0a618fb5608af098f12d04285e572054ad194 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Wed, 1 Jun 2022 10:59:19 +0300 Subject: [PATCH 1849/2107] Version 2.5.6 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 4cd65980c..2c4c61036 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.5" +#define PROGRAM_VERSION "2.5.6" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 29cd93f45..96f0f3446 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.5 \ No newline at end of file +pg_probackup 2.5.6 From 1b75b4ed62c6f2f8d2912bbd61efa9d7d3f27360 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Fri, 3 Jun 2022 13:45:50 +0500 Subject: [PATCH 1850/2107] Add tests for --start-time option on the one and few nodes --- tests/backup.py | 371 +++++++++++++++++++++++++++++++- tests/helpers/ptrack_helpers.py | 5 +- 2 files changed, 374 insertions(+), 2 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 682409015..58fb4238c 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1,6 +1,6 @@ import unittest import os -from time import sleep +from time import sleep, time from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import shutil from distutils.dir_util import copy_tree @@ -3400,3 +3400,372 @@ def test_pg_stop_backup_missing_permissions(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_start_time(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=['--stream', '--start-time', str(startTime)]) + + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=['--stream', '--start-time', str(startTime)]) + + if self.ptrack and node.major_version > 11: + node.safe_psql( + "postgres", + "create extension ptrack") + + # PTRACK backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type="ptrack", + options=['--stream', '--start-time', str(startTime)]) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_start_time_incorrect_time(self): + + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + startTime = int(time()) + #backup with correct start time + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--start-time', str(startTime)]) + #backups with incorrect start time + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=['--stream', '--start-time', str(startTime-10000)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=['--stream', '--start-time', str(startTime-10000)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=['--stream', '--start-time', str(startTime-10000)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + if self.ptrack and node.major_version > 11: + node.safe_psql( + "postgres", + "create extension ptrack") + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=['--stream', '--start-time', str(startTime-10000)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_start_time_few_nodes(self): + + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1') + self.init_pb(backup_dir1) + self.add_instance(backup_dir1, 'node1', node1) + self.set_archiving(backup_dir1, 'node1', node1) + node1.slow_start() + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2') + self.init_pb(backup_dir2) + self.add_instance(backup_dir2, 'node2', node2) + self.set_archiving(backup_dir2, 'node2', node2) + node2.slow_start() + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + + show_backup1 = self.show_pb(backup_dir1, 'node1')[0] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="delta", + options=['--stream', '--start-time', str(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type="delta", + options=['--stream', '--start-time', str(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[1] + show_backup2 = self.show_pb(backup_dir2, 'node2')[1] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="page", + options=['--stream', '--start-time', str(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type="page", + options=['--stream', '--start-time', str(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[2] + show_backup2 = self.show_pb(backup_dir2, 'node2')[2] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PTRACK backup + startTime = int(time()) + if self.ptrack and node1.major_version > 11: + node1.safe_psql( + "postgres", + "create extension ptrack") + self.backup_node( + backup_dir1, 'node1', node1, backup_type="ptrack", + options=['--stream', '--start-time', str(startTime)]) + + if self.ptrack and node2.major_version > 11: + node2.safe_psql( + "postgres", + "create extension ptrack") + self.backup_node( + backup_dir2, 'node2', node2, backup_type="ptrack", + options=['--stream', '--start-time', str(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[3] + show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # Clean after yourself + self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_start_time_few_nodes_incorrect_time(self): + + fname = self.id().split('.')[3] + node1 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node1'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1') + self.init_pb(backup_dir1) + self.add_instance(backup_dir1, 'node1', node1) + self.set_archiving(backup_dir1, 'node1', node1) + node1.slow_start() + + node2 = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node2'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2') + self.init_pb(backup_dir2) + self.add_instance(backup_dir2, 'node2', node2) + self.set_archiving(backup_dir2, 'node2', node2) + node2.slow_start() + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type="full", + options=['--stream', '--start-time', str(startTime-10000)]) + + show_backup1 = self.show_pb(backup_dir1, 'node1')[0] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertGreater(show_backup1['id'], show_backup2['id']) + + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="delta", + options=['--stream', '--start-time', str(startTime)]) + # make backup with start time definitelly earlier, than existing + try: + self.backup_node( + backup_dir2, 'node2', node2, backup_type="delta", + options=['--stream', '--start-time', str(10000)]) + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + show_backup1 = self.show_pb(backup_dir1, 'node1')[1] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertGreater(show_backup1['id'], show_backup2['id']) + + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="page", + options=['--stream', '--start-time', str(startTime)]) + # make backup with start time definitelly earlier, than existing + try: + self.backup_node( + backup_dir2, 'node2', node2, backup_type="page", + options=['--stream', '--start-time', str(10000)]) + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + show_backup1 = self.show_pb(backup_dir1, 'node1')[2] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertGreater(show_backup1['id'], show_backup2['id']) + + # PTRACK backup + startTime = int(time()) + if self.ptrack and node1.major_version > 11: + node1.safe_psql( + "postgres", + "create extension ptrack") + self.backup_node( + backup_dir1, 'node1', node1, backup_type="ptrack", + options=['--stream', '--start-time', str(startTime)]) + + if self.ptrack and node2.major_version > 11: + node2.safe_psql( + "postgres", + "create extension ptrack") + # make backup with start time definitelly earlier, than existing + try: + self.backup_node( + backup_dir2, 'node2', node2, backup_type="ptrack", + options=['--stream', '--start-time', str(10000)]) + self.assertEqual( + 1, 0, + "Expecting Error because start time for new backup must be newer " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + "ERROR: Cannot create directory for older backup", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir1, 'node1', node1, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type="full", + options=['--stream', '--start-time', str(startTime)]) + + show_backup1 = self.show_pb(backup_dir1, 'node1')[4] + show_backup2 = self.show_pb(backup_dir2, 'node2')[1] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # Clean after yourself + self.del_test_dir(module_name, fname) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index ffb87c5ec..5f1d40ab3 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -938,7 +938,7 @@ def backup_node( backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, old_binary=False, return_id=True, no_remote=False, - env=None + env=None, startTime=None ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -971,6 +971,9 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] + if startTime: + cmd_list += ['--start-time', startTime] + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( From 41855701c7033ff358de8b56822ce607dd3303c9 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 8 Jun 2022 00:38:51 +0300 Subject: [PATCH 1851/2107] [PBCKP-153] Added waldir option for location for the write-ahead log directory (-X, --waldir=WALDIR) --- src/dir.c | 30 +++++++++++++++++++++++++++++- src/help.c | 6 ++++++ src/merge.c | 2 +- src/pg_probackup.c | 17 +++++++++++++++++ src/pg_probackup.h | 5 ++++- src/restore.c | 2 +- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/dir.c b/src/dir.c index 4ebe0939b..ac794cee4 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1036,13 +1036,20 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) */ void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, - bool extract_tablespaces, bool incremental, fio_location location) + bool extract_tablespaces, bool incremental, fio_location location, + const char* waldir_path) { int i; parray *links = NULL; mode_t pg_tablespace_mode = DIR_PERMISSION; char to_path[MAXPGPATH]; + if (waldir_path && !dir_is_empty(waldir_path, location)) + { + elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path); + } + + /* get tablespace map */ if (extract_tablespaces) { @@ -1107,6 +1114,27 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba /* skip external directory content */ if (dir->external_dir_num != 0) continue; + /* Create WAL directory and symlink if waldir_path is setting */ + if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) { + /* get full path to PG_XLOG_DIR */ + + join_path_components(to_path, data_dir, PG_XLOG_DIR); + + elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + waldir_path, to_path); + + /* create tablespace directory from waldir_path*/ + fio_mkdir(waldir_path, pg_tablespace_mode, location); + + /* create link to linked_path */ + if (fio_symlink(waldir_path, to_path, incremental, location) < 0) + elog(ERROR, "Could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + continue; + + + } /* tablespace_map exists */ if (links) diff --git a/src/help.c b/src/help.c index b22fa912e..85894759e 100644 --- a/src/help.c +++ b/src/help.c @@ -169,6 +169,7 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--no-sync]\n")); + printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include | --db-exclude]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); @@ -435,6 +436,7 @@ help_restore(void) printf(_(" [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); @@ -472,6 +474,10 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + + printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n")); + + printf(_("\n Incremental restore options:\n")); printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); printf(_(" reuse valid pages available in PGDATA if they have not changed\n")); diff --git a/src/merge.c b/src/merge.c index ff39c2510..1ce92bb42 100644 --- a/src/merge.c +++ b/src/merge.c @@ -614,7 +614,7 @@ merge_chain(InstanceState *instanceState, /* Create directories */ create_data_directories(dest_backup->files, full_database_dir, - dest_backup->root_dir, false, false, FIO_BACKUP_HOST); + dest_backup->root_dir, false, false, FIO_BACKUP_HOST, NULL); /* External directories stuff */ if (dest_backup->external_dir_str) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b9b3af0b9..2c8100b83 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -122,6 +122,7 @@ static parray *datname_include_list = NULL; /* arrays for --exclude-path's */ static parray *exclude_absolute_paths_list = NULL; static parray *exclude_relative_paths_list = NULL; +static char* waldir_path = NULL; /* checkdb options */ bool need_amcheck = false; @@ -238,6 +239,7 @@ static ConfigOption cmd_options[] = { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, + { 's', 'X', "waldir", &waldir_path, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -754,6 +756,21 @@ main(int argc, char *argv[]) restore_params->partial_restore_type = INCLUDE; restore_params->partial_db_list = datname_include_list; } + + if (waldir_path) + { + /* clean up xlog directory name, check it's absolute */ + canonicalize_path(waldir_path); + if (!is_absolute_path(waldir_path)) + { + elog(ERROR, "WAL directory location must be an absolute path"); + } + if (strlen(waldir_path) > MAXPGPATH) + elog(ERROR, "Value specified to --waldir is too long"); + + } + restore_params->waldir = waldir_path; + } /* diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2c4c61036..13650be8b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -566,6 +566,8 @@ typedef struct pgRestoreParams /* options for partial restore */ PartialRestoreType partial_restore_type; parray *partial_db_list; + + char* waldir; } pgRestoreParams; /* Options needed for set-backup command */ @@ -1022,7 +1024,8 @@ extern void create_data_directories(parray *dest_files, const char *backup_dir, bool extract_tablespaces, bool incremental, - fio_location location); + fio_location location, + const char *waldir_path); extern void read_tablespace_map(parray *links, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); diff --git a/src/restore.c b/src/restore.c index d8d808a4e..fbf0c0398 100644 --- a/src/restore.c +++ b/src/restore.c @@ -801,7 +801,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, create_data_directories(dest_files, instance_config.pgdata, dest_backup->root_dir, backup_has_tblspc, params->incremental_mode != INCR_NONE, - FIO_DB_HOST); + FIO_DB_HOST, params->waldir); /* * Restore dest_backup external directories. From 48a2c835d1c12353e23e08b901beaf39695773f9 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 8 Jun 2022 17:40:49 +0300 Subject: [PATCH 1852/2107] [PBCKP-153] Added a test for the waldir option for the restore command --- tests/restore.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/restore.py b/tests/restore.py index bbdadeb23..668cff4fd 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3916,3 +3916,59 @@ def test_restore_issue_313(self): # Clean after yourself self.del_test_dir(module_name, fname) + + # @unittest.skip("skip") + def test_restore_with_waldir(self): + """recovery using tablespace-mapping option and page backup""" + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + with node.connect("postgres") as con: + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Full backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Create waldir + waldir_path = os.path.join(node.base_dir, "waldir") + os.makedirs(waldir_path) + + # Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-X", "%s" % (waldir_path)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + + # check pg_wal is symlink + if node.major_version >= 10: + wal_path=os.path.join(node.data_dir, "pg_wal") + else: + wal_path=os.path.join(node.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + + # Clean after yourself + self.del_test_dir(module_name, fname) From e72feb6813fa4862dbad12c657119c7bcfefe12b Mon Sep 17 00:00:00 2001 From: Sokolov Yura Date: Thu, 16 Jun 2022 09:26:02 +0300 Subject: [PATCH 1853/2107] rapid agent close + disable ssh control master. (#493) --- src/utils/file.c | 5 ++++- src/utils/remote.c | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/file.c b/src/utils/file.c index 7d1df554b..7103c8f1d 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -489,8 +489,10 @@ fio_disconnect(void) Assert(hdr.cop == FIO_DISCONNECTED); SYS_CHECK(close(fio_stdin)); SYS_CHECK(close(fio_stdout)); + SYS_CHECK(close(fio_stderr)); fio_stdin = 0; fio_stdout = 0; + fio_stderr = 0; wait_ssh(); } } @@ -3403,7 +3405,8 @@ fio_communicate(int in, int out) case FIO_DISCONNECT: hdr.cop = FIO_DISCONNECTED; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; + free(buf); + return; case FIO_GET_ASYNC_ERROR: fio_get_async_error_impl(out); break; diff --git a/src/utils/remote.c b/src/utils/remote.c index 2bfd24d1e..046ebd818 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -147,6 +147,9 @@ bool launch_agent(void) ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "Compression=no"; + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "ControlMaster=no"; + ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "LogLevel=error"; From acc8edcd62d399d972e9dab8df8ecd85dbeb0fa2 Mon Sep 17 00:00:00 2001 From: avaness Date: Thu, 16 Jun 2022 11:46:19 +0300 Subject: [PATCH 1854/2107] minor hotfix for OptionTest.test_help_6, OptionTest.test_version_2 and tests/Readme.md FAQ (#494) Co-authored-by: Ivan Lazarev --- tests/Readme.md | 31 ++++++++++++++++++++++++++++++- tests/expected/option_help_ru.out | 1 + tests/option.py | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/Readme.md b/tests/Readme.md index 500ed7c7a..f980b6aef 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,4 +1,4 @@ -[see wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) +****[see wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` Note: For now these tests work on Linux and "kinda" work on Windows @@ -50,3 +50,32 @@ Usage: export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` + +### Troubleshooting FAQ + +#### python test failures +1. Test failure reason like +``` +testgres.exceptions.QueryException ERROR: could not open extension control file "/home/avaness/postgres/postgres.build/share/extension/amcheck.control": No such file or directory +``` + +*Solution*: you have no `/contrib/` extensions installed + +```commandline +cd +make world install +``` + +2. Test failure + +``` +FAIL: test_help_6 (tests.option.OptionTest) +``` + +*Solution*: you didn't configure postgres build with `--enable-nls` + +```commandline +cd +make distclean + --enable-nls +``` diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index ee8da9a1c..68afb82f8 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -178,6 +178,7 @@ pg_probackup - утилита для управления резервным к [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] Подробнее читайте на сайте . diff --git a/tests/option.py b/tests/option.py index b57d7ef43..23aa97c84 100644 --- a/tests/option.py +++ b/tests/option.py @@ -24,7 +24,7 @@ def test_version_2(self): """help options""" with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: self.assertIn( - version_out.read().decode("utf-8"), + version_out.read().decode("utf-8").strip(), self.run_pb(["--version"]) ) From e11ca786b1a466aff773ebec5fae0b88692d140d Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 16 Jun 2022 12:02:27 +0300 Subject: [PATCH 1855/2107] [PBCKP-153] Changed expected/option_help.out and option_help_ru.out files for the tests.option.OptionTest.test_help_1 and help_6 --- tests/expected/option_help.out | 1 + tests/expected/option_help_ru.out | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 8a1de1f67..659164250 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -86,6 +86,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [--no-sync] + [-X WALDIR | --waldir=WALDIR] [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] [--remote-proto] [--remote-host] diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index ee8da9a1c..2e90eb297 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -86,6 +86,7 @@ pg_probackup - утилита для управления резервным к [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [--no-sync] + [-X WALDIR | --waldir=WALDIR] [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] [--remote-proto] [--remote-host] @@ -178,6 +179,7 @@ pg_probackup - утилита для управления резервным к [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] Подробнее читайте на сайте . From 55238d572fb4d463ed3336ffa083df5260bdd2ee Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Mon, 20 Jun 2022 13:44:42 +0300 Subject: [PATCH 1856/2107] [PBCKP-120] skip partitioned indexes for checkdb --amcheck Tags: pg_probackup --- src/checkdb.c | 10 +++++++--- tests/checkdb.py | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 177fc3cc7..1133a7b5d 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -473,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " - "(cls.reltablespace IN " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " + "AND (cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index 9b7adcd71..4608366be 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -36,6 +36,15 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( From 7e16642b663ccf66507b5fa7a270c7063db44633 Mon Sep 17 00:00:00 2001 From: avaness Date: Wed, 22 Jun 2022 12:54:20 +0300 Subject: [PATCH 1857/2107] [PBCKP-165] get_control_value() int64 buffer vulnerability fix (#496) * [PBCKP-165] get_control_value() int64 buffer vulnerability fix - added output buffer size limit check - splitted to get_get_control_value_str() & get_control_value_int64() api - included for windows build Co-authored-by: Ivan Lazarev --- src/catalog.c | 32 ++++++------ src/dir.c | 125 +++++++++++++++++++++++++-------------------- src/pg_probackup.h | 5 +- 3 files changed, 90 insertions(+), 72 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index b4ed8c189..9d817913e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1084,15 +1084,15 @@ get_backup_filelist(pgBackup *backup, bool strict) COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); - get_control_value(buf, "path", path, NULL, true); - get_control_value(buf, "size", NULL, &write_size, true); - get_control_value(buf, "mode", NULL, &mode, true); - get_control_value(buf, "is_datafile", NULL, &is_datafile, true); - get_control_value(buf, "is_cfs", NULL, &is_cfs, false); - get_control_value(buf, "crc", NULL, &crc, true); - get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); - get_control_value(buf, "dbOid", NULL, &dbOid, false); + get_control_value_str(buf, "path", path, sizeof(path),true); + get_control_value_int64(buf, "size", &write_size, true); + get_control_value_int64(buf, "mode", &mode, true); + get_control_value_int64(buf, "is_datafile", &is_datafile, true); + get_control_value_int64(buf, "is_cfs", &is_cfs, false); + get_control_value_int64(buf, "crc", &crc, true); + get_control_value_str(buf, "compress_alg", compress_alg_string, sizeof(compress_alg_string), false); + get_control_value_int64(buf, "external_dir_num", &external_dir_num, false); + get_control_value_int64(buf, "dbOid", &dbOid, false); file = pgFileInit(path); file->write_size = (int64) write_size; @@ -1107,28 +1107,28 @@ get_backup_filelist(pgBackup *backup, bool strict) /* * Optional fields */ - if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) + if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0]) { file->linked = pgut_strdup(linked); canonicalize_path(file->linked); } - if (get_control_value(buf, "segno", NULL, &segno, false)) + if (get_control_value_int64(buf, "segno", &segno, false)) file->segno = (int) segno; - if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) + if (get_control_value_int64(buf, "n_blocks", &n_blocks, false)) file->n_blocks = (int) n_blocks; - if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) + if (get_control_value_int64(buf, "n_headers", &n_headers, false)) file->n_headers = (int) n_headers; - if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) + if (get_control_value_int64(buf, "hdr_crc", &hdr_crc, false)) file->hdr_crc = (pg_crc32) hdr_crc; - if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) + if (get_control_value_int64(buf, "hdr_off", &hdr_off, false)) file->hdr_off = hdr_off; - if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false)) + if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) file->hdr_size = (int) hdr_size; parray_append(files, file); diff --git a/src/dir.c b/src/dir.c index 4ebe0939b..e76122ae7 100644 --- a/src/dir.c +++ b/src/dir.c @@ -8,6 +8,7 @@ *------------------------------------------------------------------------- */ +#include #include "pg_probackup.h" #include "utils/file.h" @@ -130,6 +131,9 @@ static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); static void cleanup_tablespace(const char *path); +static void control_string_bad_format(const char* str); + + /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; /* Extra directories mapping */ @@ -1467,7 +1471,7 @@ get_external_remap(char *current_dir) return current_dir; } -/* Parsing states for get_control_value() */ +/* Parsing states for get_control_value_str() */ #define CONTROL_WAIT_NAME 1 #define CONTROL_INNAME 2 #define CONTROL_WAIT_COLON 3 @@ -1481,26 +1485,62 @@ get_external_remap(char *current_dir) * The line has the following format: * {"name1":"value1", "name2":"value2"} * - * The value will be returned to "value_str" as string if it is not NULL. If it - * is NULL the value will be returned to "value_int64" as int64. + * The value will be returned in "value_int64" as int64. + * + * Returns true if the value was found in the line and parsed. + */ +bool +get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory) +{ + + char buf_int64[32]; + + assert(value_int64); + + /* Set default value */ + *value_int64 = 0; + + if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory)) + return false; + + if (!parse_int64(buf_int64, value_int64, 0)) + { + /* We assume that too big value is -1 */ + if (errno == ERANGE) + *value_int64 = BYTES_INVALID; + else + control_string_bad_format(str); + return false; + } + + return true; +} + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string. * * Returns true if the value was found in the line. */ + bool -get_control_value(const char *str, const char *name, - char *value_str, int64 *value_int64, bool is_mandatory) +get_control_value_str(const char *str, const char *name, + char *value_str, size_t value_str_size, bool is_mandatory) { int state = CONTROL_WAIT_NAME; char *name_ptr = (char *) name; char *buf = (char *) str; - char buf_int64[32], /* Buffer for "value_int64" */ - *buf_int64_ptr = buf_int64; + char *const value_str_start = value_str; - /* Set default values */ - if (value_str) - *value_str = '\0'; - else if (value_int64) - *value_int64 = 0; + assert(value_str); + assert(value_str_size > 0); + + /* Set default value */ + *value_str = '\0'; while (*buf) { @@ -1510,7 +1550,7 @@ get_control_value(const char *str, const char *name, if (*buf == '"') state = CONTROL_INNAME; else if (IsAlpha(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_INNAME: /* Found target field. Parse value. */ @@ -1529,57 +1569,32 @@ get_control_value(const char *str, const char *name, if (*buf == ':') state = CONTROL_WAIT_VALUE; else if (!IsSpace(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_WAIT_VALUE: if (*buf == '"') { state = CONTROL_INVALUE; - buf_int64_ptr = buf_int64; } else if (IsAlpha(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_INVALUE: /* Value was parsed, exit */ if (*buf == '"') { - if (value_str) - { - *value_str = '\0'; - } - else if (value_int64) - { - /* Length of buf_uint64 should not be greater than 31 */ - if (buf_int64_ptr - buf_int64 >= 32) - elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - - *buf_int64_ptr = '\0'; - if (!parse_int64(buf_int64, value_int64, 0)) - { - /* We assume that too big value is -1 */ - if (errno == ERANGE) - *value_int64 = BYTES_INVALID; - else - goto bad_format; - } - } - + *value_str = '\0'; return true; } else { - if (value_str) - { - *value_str = *buf; - value_str++; - } - else - { - *buf_int64_ptr = *buf; - buf_int64_ptr++; + /* verify if value_str not exceeds value_str_size limits */ + if (value_str - value_str_start >= value_str_size - 1) { + elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); } + *value_str = *buf; + value_str++; } break; case CONTROL_WAIT_NEXT_NAME: @@ -1596,18 +1611,20 @@ get_control_value(const char *str, const char *name, /* There is no close quotes */ if (state == CONTROL_INNAME || state == CONTROL_INVALUE) - goto bad_format; + control_string_bad_format(str); /* Did not find target field */ if (is_mandatory) elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", name, str, DATABASE_FILE_LIST); return false; +} -bad_format: - elog(ERROR, "%s file has invalid format in line %s", - DATABASE_FILE_LIST, str); - return false; /* Make compiler happy */ +static void +control_string_bad_format(const char* str) +{ + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); } /* @@ -1841,8 +1858,8 @@ read_database_map(pgBackup *backup) db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); - get_control_value(buf, "dbOid", NULL, &dbOid, true); - get_control_value(buf, "datname", datname, NULL, true); + get_control_value_int64(buf, "dbOid", &dbOid, true); + get_control_value_str(buf, "datname", datname, sizeof(datname), true); db_entry->dbOid = dbOid; db_entry->datname = pgut_strdup(datname); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2c4c61036..7eb62466f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1010,8 +1010,9 @@ extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); /* in dir.c */ -extern bool get_control_value(const char *str, const char *name, - char *value_str, int64 *value_int64, bool is_mandatory); +extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory); +extern bool get_control_value_str(const char *str, const char *name, + char *value_str, size_t value_str_size, bool is_mandatory); extern void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location); From 039e3c86786737264366b9a8bfcc675e10afeec4 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Wed, 22 Jun 2022 14:36:55 +0500 Subject: [PATCH 1858/2107] Add checking enable-nls option in configure For correct work test_help_6 we need skip this test if PostgreSQL configured without --enable-nls --- tests/helpers/ptrack_helpers.py | 14 ++++++++++++++ tests/option.py | 16 ++++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index ffb87c5ec..f2d316161 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -101,6 +101,19 @@ def is_enterprise(): else: return False +def enable_nls(): + cmd = [os.environ['PG_CONFIG'], '--configure'] + + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if b'enable-nls' in p.communicate()[0]: + return True + else: + return False + class ProbackupException(Exception): def __init__(self, message, cmd): @@ -147,6 +160,7 @@ def slow_start(self, replica=False): class ProbackupTest(object): # Class attributes enterprise = is_enterprise() + enable_nls = enable_nls() def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) diff --git a/tests/option.py b/tests/option.py index 23aa97c84..88e72ffd7 100644 --- a/tests/option.py +++ b/tests/option.py @@ -231,9 +231,13 @@ def test_options_5(self): # @unittest.skip("skip") def test_help_6(self): """help options""" - self.test_env['LC_ALL'] = 'ru_RU.utf-8' - with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: - self.assertEqual( - self.run_pb(["--help"]), - help_out.read().decode("utf-8") - ) + if ProbackupTest.enable_nls: + self.test_env['LC_ALL'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + else: + return unittest.skip( + 'You need configure PostgreSQL with --enabled-nls option for this test') From 61cd6209772c8ac8ec34a80444a074f66650a4bf Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Fri, 24 Jun 2022 11:36:56 +0300 Subject: [PATCH 1859/2107] [PBCKP-153] global variable waldir_path renamed to gl_waldir_path --- src/pg_probackup.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 2c8100b83..193cd9c39 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -122,7 +122,7 @@ static parray *datname_include_list = NULL; /* arrays for --exclude-path's */ static parray *exclude_absolute_paths_list = NULL; static parray *exclude_relative_paths_list = NULL; -static char* waldir_path = NULL; +static char* gl_waldir_path = NULL; /* checkdb options */ bool need_amcheck = false; @@ -239,7 +239,7 @@ static ConfigOption cmd_options[] = { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, - { 's', 'X', "waldir", &waldir_path, SOURCE_CMD_STRICT }, + { 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -757,19 +757,19 @@ main(int argc, char *argv[]) restore_params->partial_db_list = datname_include_list; } - if (waldir_path) + if (gl_waldir_path) { /* clean up xlog directory name, check it's absolute */ - canonicalize_path(waldir_path); - if (!is_absolute_path(waldir_path)) + canonicalize_path(gl_waldir_path); + if (!is_absolute_path(gl_waldir_path)) { elog(ERROR, "WAL directory location must be an absolute path"); } - if (strlen(waldir_path) > MAXPGPATH) + if (strlen(gl_waldir_path) > MAXPGPATH) elog(ERROR, "Value specified to --waldir is too long"); } - restore_params->waldir = waldir_path; + restore_params->waldir = gl_waldir_path; } From a3ac7d5e7a8d6ebeafda692ed1031eb2b34a8ab4 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Fri, 24 Jun 2022 17:12:26 +0500 Subject: [PATCH 1860/2107] Add grants for pgpro_edition --- tests/backup.py | 15 +++++++++------ tests/checkdb.py | 1 + tests/ptrack.py | 6 ++---- tests/restore.py | 5 ++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 682409015..b7fc4a924 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1915,6 +1915,7 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) # >= 10 else: @@ -1953,6 +1954,7 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) if self.ptrack: @@ -1966,9 +1968,6 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") node.safe_psql( "backupdb", @@ -3052,7 +3051,9 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" + ) # >= 10 else: node.safe_psql( @@ -3075,12 +3076,12 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") sleep(2) @@ -3185,6 +3186,7 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) # >= 10 else: @@ -3208,12 +3210,13 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" + ) if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") replica.promote() diff --git a/tests/checkdb.py b/tests/checkdb.py index e066c5777..fec2e792c 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -698,6 +698,7 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' # 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) # PG 10 elif self.get_version(node) > 100000 and self.get_version(node) < 110000: diff --git a/tests/ptrack.py b/tests/ptrack.py index 5878f0700..5ecc669bb 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -582,6 +582,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) # >= 10 else: @@ -618,6 +619,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' ) node.safe_psql( @@ -635,10 +637,6 @@ def test_ptrack_unprivileged(self): "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") diff --git a/tests/restore.py b/tests/restore.py index bbdadeb23..a9fe869e6 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3268,6 +3268,7 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) # >= 10 else: @@ -3305,6 +3306,7 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" ) if self.ptrack: @@ -3319,9 +3321,6 @@ def test_missing_database_map(self): "CREATE EXTENSION ptrack WITH SCHEMA ptrack") if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") node.safe_psql( "backupdb", From 55d3fa8979ec00eda90e36594a6976ae739d2876 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Wed, 29 Jun 2022 11:08:05 +0500 Subject: [PATCH 1861/2107] Rename enable_nls() function in ptrack_helpers.p is_nls_enabled() --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f2d316161..b5f1fe5b2 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -101,7 +101,7 @@ def is_enterprise(): else: return False -def enable_nls(): +def is_nls_enabled(): cmd = [os.environ['PG_CONFIG'], '--configure'] p = subprocess.Popen( @@ -160,7 +160,7 @@ def slow_start(self, replica=False): class ProbackupTest(object): # Class attributes enterprise = is_enterprise() - enable_nls = enable_nls() + enable_nls = is_nls_enabled() def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) From f544da1ecde143c57bda4205470267bed1a6056e Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Wed, 29 Jun 2022 22:17:31 +0500 Subject: [PATCH 1862/2107] Shorthand return-expression --- tests/helpers/ptrack_helpers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b5f1fe5b2..18fb3fc2e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -109,10 +109,7 @@ def is_nls_enabled(): stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if b'enable-nls' in p.communicate()[0]: - return True - else: - return False + return b'enable-nls' in p.communicate()[0] class ProbackupException(Exception): From 32aae17928d165be7a8a19b015b87f8b885bc5dd Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 30 Jun 2022 02:28:29 +0300 Subject: [PATCH 1863/2107] [PBCKP-220] minor updates for gdb checks, checking CI tests --- tests/archive.py | 2 ++ tests/delta.py | 7 ++----- tests/helpers/ptrack_helpers.py | 17 +++++++++++++++-- tests/pgpro2068.py | 2 ++ tests/ptrack.py | 2 ++ tests/replica.py | 30 ++++++++++-------------------- 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 22b9d8693..e01b7d37e 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -290,6 +290,8 @@ def test_pgpro434_4(self): Check pg_stop_backup_timeout, libpq-timeout requested. Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/delta.py b/tests/delta.py index f365b6f9b..82fb714f7 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -472,11 +472,8 @@ def test_delta_vacuum_full(self): make node, make full and delta stream backups, restore them and check data correctness """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index ffb87c5ec..b8449abe4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -180,8 +180,8 @@ def __init__(self, *args, **kwargs): self.test_env['LC_MESSAGES'] = 'C' self.test_env['LC_TIME'] = 'C' - self.gdb = 'PGPROBACKUP_GDB' in os.environ and \ - os.environ['PGPROBACKUP_GDB'] == 'ON' + self.gdb = 'PGPROBACKUP_GDB' in self.test_env and \ + self.test_env['PGPROBACKUP_GDB'] == 'ON' self.paranoia = 'PG_PROBACKUP_PARANOIA' in self.test_env and \ self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON' @@ -810,6 +810,7 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur if self.verbose: print(self.cmd) if gdb: + #TODO REVIEW XXX no self parameter return GDBobj([binary_path] + command, self.verbose) if asynchronous: return subprocess.Popen( @@ -1861,8 +1862,15 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict self.assertFalse(fail, error_message) def gdb_attach(self, pid): + #TODO REVIEW XXX no self parameter return GDBobj([str(pid)], self.verbose, attach=True) + def _check_gdb_flag_or_skip_test(self): + if not self.gdb: + self.skipTest( + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) class GdbException(Exception): def __init__(self, message=False): @@ -1877,6 +1885,11 @@ def __init__(self, cmd, verbose, attach=False): self.verbose = verbose self.output = '' + # Check gdb flag is set up + # if not self.gdb: + # raise GdbException("No `PGPROBACKUP_GDB=on` is set, " + # "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " + # "and be skipped") # Check gdb presense try: gdb_version, _ = subprocess.Popen( diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index a80d317d4..b76345b89 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -18,6 +18,8 @@ def test_minrecpoint_on_replica(self): """ https://jira.postgrespro.ru/browse/PGPRO-2068 """ + self._check_gdb_flag_or_skip_test() + if not self.gdb: self.skipTest( "Specify PGPROBACKUP_GDB and build without " diff --git a/tests/ptrack.py b/tests/ptrack.py index 5878f0700..08ea90f8d 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -824,6 +824,8 @@ def test_ptrack_uncommitted_xact(self): def test_ptrack_vacuum_full(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" + self._check_gdb_flag_or_skip_test() + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), diff --git a/tests/replica.py b/tests/replica.py index 45eed3fb4..ba7076fab 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -634,11 +634,8 @@ def test_replica_promote(self): def test_replica_stop_lsn_null_offset(self): """ """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -722,11 +719,8 @@ def test_replica_stop_lsn_null_offset(self): def test_replica_stop_lsn_null_offset_next_record(self): """ """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -828,6 +822,8 @@ def test_replica_stop_lsn_null_offset_next_record(self): def test_archive_replica_null_offset(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -998,11 +994,8 @@ def test_replica_toast(self): make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( @@ -1104,11 +1097,8 @@ def test_replica_toast(self): def test_start_stop_lsn_in_the_same_segno(self): """ """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( From 26939d67c444156bfea5b3701d34bd5495df0e83 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 1 Jul 2022 03:57:36 +0300 Subject: [PATCH 1864/2107] [PBCKP-220] removed inheritance GDBObj->ProbackupTest --- tests/helpers/ptrack_helpers.py | 26 ++++++++++++++------------ tests/replica.py | 4 ---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b8449abe4..e69d1213e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -810,8 +810,7 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur if self.verbose: print(self.cmd) if gdb: - #TODO REVIEW XXX no self parameter - return GDBobj([binary_path] + command, self.verbose) + return GDBobj([binary_path] + command, self) if asynchronous: return subprocess.Popen( [binary_path] + command, @@ -1862,8 +1861,7 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict self.assertFalse(fail, error_message) def gdb_attach(self, pid): - #TODO REVIEW XXX no self parameter - return GDBobj([str(pid)], self.verbose, attach=True) + return GDBobj([str(pid)], self, attach=True) def _check_gdb_flag_or_skip_test(self): if not self.gdb: @@ -1872,24 +1870,28 @@ def _check_gdb_flag_or_skip_test(self): "optimizations for run this test" ) + class GdbException(Exception): - def __init__(self, message=False): + def __init__(self, message="False"): self.message = message def __str__(self): return '\n ERROR: {0}\n'.format(repr(self.message)) -class GDBobj(ProbackupTest): - def __init__(self, cmd, verbose, attach=False): - self.verbose = verbose +#TODO REVIEW XXX no inheritance needed +# class GDBobj(ProbackupTest): +class GDBobj: + # TODO REVIEW XXX Type specification env:ProbackupTest is only for python3, is it ok? + def __init__(self, cmd, env: ProbackupTest, attach=False): + self.verbose = env.verbose self.output = '' # Check gdb flag is set up - # if not self.gdb: - # raise GdbException("No `PGPROBACKUP_GDB=on` is set, " - # "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " - # "and be skipped") + if not env.gdb: + raise GdbException("No `PGPROBACKUP_GDB=on` is set, " + "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " + "and be skipped") # Check gdb presense try: gdb_version, _ = subprocess.Popen( diff --git a/tests/replica.py b/tests/replica.py index ba7076fab..85034d501 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -719,7 +719,6 @@ def test_replica_stop_lsn_null_offset(self): def test_replica_stop_lsn_null_offset_next_record(self): """ """ - self._check_gdb_flag_or_skip_test() fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -743,7 +742,6 @@ def test_replica_stop_lsn_null_offset_next_record(self): # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) self.backup_node(backup_dir, 'master', master) @@ -1097,7 +1095,6 @@ def test_replica_toast(self): def test_start_stop_lsn_in_the_same_segno(self): """ """ - self._check_gdb_flag_or_skip_test() fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1121,7 +1118,6 @@ def test_start_stop_lsn_in_the_same_segno(self): # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) self.backup_node(backup_dir, 'master', master, options=['--stream']) From 9c6e3ce3f751162cf7ac5405d0cc4ff462324181 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 1 Jul 2022 13:52:20 +0300 Subject: [PATCH 1865/2107] [PBCKP-220] all gdb tests fixup --- .travis.yml | 1 + tests/archive.py | 2 ++ tests/backup.py | 18 ++++++++++++++++++ tests/checkdb.py | 2 ++ tests/helpers/ptrack_helpers.py | 3 --- tests/locking.py | 16 ++++++++++++++++ tests/logging.py | 4 ++++ tests/merge.py | 32 ++++++++++++++++++++++++++++++++ tests/replica.py | 1 + tests/restore.py | 4 ++++ tests/retention.py | 12 ++++++++++++ tests/validate.py | 2 ++ 12 files changed, 94 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26b2bc4e2..bac8a2c0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE PGPROBACKUP_GDB=ON PG_PROBACKUP_TEST_BASIC=OFF # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup diff --git a/tests/archive.py b/tests/archive.py index e01b7d37e..cd8d4404f 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -228,6 +228,8 @@ def test_pgpro434_3(self): Check pg_stop_backup_timeout, needed backup_timeout Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/backup.py b/tests/backup.py index 682409015..20ac480e0 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1095,6 +1095,8 @@ def test_tablespace_handling_2(self): # @unittest.skip("skip") def test_drop_rel_during_full_backup(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1244,6 +1246,8 @@ def test_drop_db_during_full_backup(self): # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1313,6 +1317,8 @@ def test_drop_rel_during_backup_delta(self): # @unittest.skip("skip") def test_drop_rel_during_backup_page(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1445,6 +1451,8 @@ def test_basic_temp_slot_for_stream_backup(self): # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1579,6 +1587,8 @@ def test_pg_11_adjusted_wal_segment_size(self): # @unittest.skip("skip") def test_sigint_handling(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1618,6 +1628,8 @@ def test_sigint_handling(self): # @unittest.skip("skip") def test_sigterm_handling(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1656,6 +1668,8 @@ def test_sigterm_handling(self): # @unittest.skip("skip") def test_sigquit_handling(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2906,6 +2920,8 @@ def test_incr_backup_filenode_map(self): # @unittest.skip("skip") def test_missing_wal_segment(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -3292,6 +3308,8 @@ def test_basic_backup_default_transaction_read_only(self): # @unittest.skip("skip") def test_backup_atexit(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/checkdb.py b/tests/checkdb.py index 9b7adcd71..68dec14b6 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -17,6 +17,8 @@ class CheckdbTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_checkdb_amcheck_only_sanity(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e69d1213e..e3036d9c4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1879,10 +1879,7 @@ def __str__(self): return '\n ERROR: {0}\n'.format(repr(self.message)) -#TODO REVIEW XXX no inheritance needed -# class GDBobj(ProbackupTest): class GDBobj: - # TODO REVIEW XXX Type specification env:ProbackupTest is only for python3, is it ok? def __init__(self, cmd, env: ProbackupTest, attach=False): self.verbose = env.verbose self.output = '' diff --git a/tests/locking.py b/tests/locking.py index ef7aa1f25..0fe954cae 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -17,6 +17,8 @@ def test_locking_running_validate_1(self): run validate, expect it to successfully executed, concurrent RUNNING backup with pid file and active process is legal """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -72,6 +74,8 @@ def test_locking_running_validate_2(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -142,6 +146,8 @@ def test_locking_running_validate_2_specific_id(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -240,6 +246,8 @@ def test_locking_running_3(self): RUNNING backup without pid file AND without active pid is legal, his status must be changed to ERROR """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -310,6 +318,8 @@ def test_locking_restore_locked(self): Expect restore to sucseed because read-only locks do not conflict """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -352,6 +362,8 @@ def test_concurrent_delete_and_restore(self): Expect restore to fail because validation of intermediate backup is impossible """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -443,6 +455,8 @@ def test_locking_concurren_restore_and_delete(self): and stop it in the middle, delete full backup. Expect it to fail. """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -585,6 +599,8 @@ def test_shared_lock(self): """ Make sure that shared lock leaves no files with pids """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/logging.py b/tests/logging.py index 47143cfb7..70ebcf6d1 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -12,6 +12,10 @@ class LogTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # PGPRO-2154 def test_log_rotation(self): + """ + """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/merge.py b/tests/merge.py index fe0927f49..5f092543c 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -975,6 +975,8 @@ def test_continue_failed_merge(self): """ Check that failed MERGE can be continued """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1051,6 +1053,8 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1148,6 +1152,8 @@ def test_continue_failed_merge_2(self): """ Check that failed MERGE on delete can be continued """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1219,6 +1225,8 @@ def test_continue_failed_merge_3(self): Check that failed MERGE cannot be continued if intermediate backup is missing. """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1409,6 +1417,8 @@ def test_crash_after_opening_backup_control_1(self): check that crashing after opening backup.control for writing will not result in losing backup metadata """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1461,6 +1471,8 @@ def test_crash_after_opening_backup_control_2(self): for writing will not result in losing metadata about backup files TODO: rewrite """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1552,6 +1564,8 @@ def test_losing_file_after_failed_merge(self): for writing will not result in losing metadata about backup files TODO: rewrite """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1639,6 +1653,8 @@ def test_losing_file_after_failed_merge(self): def test_failed_merge_after_delete(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1720,6 +1736,8 @@ def test_failed_merge_after_delete(self): def test_failed_merge_after_delete_1(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1796,6 +1814,8 @@ def test_failed_merge_after_delete_1(self): def test_failed_merge_after_delete_2(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -1858,6 +1878,8 @@ def test_failed_merge_after_delete_2(self): def test_failed_merge_after_delete_3(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2281,6 +2303,8 @@ def test_smart_merge(self): def test_idempotent_merge(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2580,6 +2604,8 @@ def test_merge_page_header_map_retry(self): page header map cannot be trusted when running retry """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2626,6 +2652,8 @@ def test_merge_page_header_map_retry(self): def test_missing_data_file(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2684,6 +2712,8 @@ def test_missing_data_file(self): def test_missing_non_data_file(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2741,6 +2771,8 @@ def test_missing_non_data_file(self): def test_merge_remote_mode(self): """ """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/replica.py b/tests/replica.py index 85034d501..0a75ea173 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -719,6 +719,7 @@ def test_replica_stop_lsn_null_offset(self): def test_replica_stop_lsn_null_offset_next_record(self): """ """ + self._check_gdb_flag_or_skip_test() fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/restore.py b/tests/restore.py index bbdadeb23..5a00bc23b 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -2379,6 +2379,8 @@ def test_pg_11_group_access(self): # @unittest.skip("skip") def test_restore_concurrent_drop_table(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -3797,6 +3799,8 @@ def test_truncate_postgresql_auto_conf(self): # @unittest.skip("skip") def test_concurrent_restore(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/retention.py b/tests/retention.py index 19204807b..b0399a239 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1499,6 +1499,8 @@ def test_window_error_backups_1(self): FULL -------window """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1546,6 +1548,8 @@ def test_window_error_backups_2(self): FULL -------window """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1588,6 +1592,8 @@ def test_window_error_backups_2(self): def test_retention_redundancy_overlapping_chains(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1636,6 +1642,8 @@ def test_retention_redundancy_overlapping_chains(self): def test_retention_redundancy_overlapping_chains_1(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), @@ -1744,6 +1752,8 @@ def test_failed_merge_redundancy_retention(self): """ Check that retention purge works correctly with MERGING backups """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( @@ -2536,6 +2546,8 @@ def test_concurrent_running_full_backup(self): """ https://github.com/postgrespro/pg_probackup/issues/328 """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/validate.py b/tests/validate.py index e62826388..41aa9ea23 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -3565,6 +3565,8 @@ def test_corrupt_pg_control_via_resetxlog(self): # @unittest.skip("skip") def test_validation_after_backup(self): """""" + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( From 125c9292a6ddd9372263894333b96ebdbb3ac767 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 4 Jul 2022 02:37:38 +0300 Subject: [PATCH 1866/2107] [PBCKP-220] ALL tests with PGPROBACKUP=ON on CI --- .travis.yml | 2 +- travis/run_tests.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bac8a2c0d..e6330c4f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE PGPROBACKUP_GDB=ON PG_PROBACKUP_TEST_BASIC=OFF + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=TMP # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 37614f970..c20c95dda 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -112,6 +112,9 @@ if [ "$MODE" = "basic" ]; then echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests python3 -m unittest -v tests.init +elif [ "$MODE" = "TMP" ]; then + echo MODE=TMP + PGPROBACKUP_GDB=ON python3 -m unittest -v tests else echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests.$MODE From 3e8a08edd5f9a20dd3d6f77914cd2b9c745a7980 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 4 Jul 2022 06:04:17 +0300 Subject: [PATCH 1867/2107] [PBCKP-220] removed FULL tests, PGPROBACKUP=ON and other flags added on CI --- .gitignore | 1 - .travis.yml | 3 ++- tests/Readme.md | 26 +++++++------------------- tests/checkdb.py | 7 ++----- tests/pgpro2068.py | 5 ----- tests/replica.py | 7 ++----- tests/validate.py | 7 ++----- travis/run_tests.sh | 13 +++++++------ 8 files changed, 22 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index c0b4de331..502473605 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,6 @@ /docker-compose.yml /Dockerfile /Dockerfile.in -/run_tests.sh /make_dockerfile.sh /backup_restore.sh diff --git a/.travis.yml b/.travis.yml index e6330c4f5..d5c9c68b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,8 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=TMP +# - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=FULL ENV_FLAGS=PGPROBACKUP_GDB=ON + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE ENV_FLAGS=PGPROBACKUP_GDB=ON # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup diff --git a/tests/Readme.md b/tests/Readme.md index f980b6aef..11c5272f9 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -51,31 +51,19 @@ Usage: python -m unittest [-v] tests[.specific_module][.class.test] ``` -### Troubleshooting FAQ +# Troubleshooting FAQ -#### python test failures -1. Test failure reason like +## Python tests failure +### 1. Could not open extension "..." ``` -testgres.exceptions.QueryException ERROR: could not open extension control file "/home/avaness/postgres/postgres.build/share/extension/amcheck.control": No such file or directory +testgres.exceptions.QueryException ERROR: could not open extension control file "/share/extension/amcheck.control": No such file or directory ``` -*Solution*: you have no `/contrib/` extensions installed +#### Solution: -```commandline -cd -make world install -``` - -2. Test failure - -``` -FAIL: test_help_6 (tests.option.OptionTest) -``` - -*Solution*: you didn't configure postgres build with `--enable-nls` +You have no `/contrib/...` extension installed, please do ```commandline cd -make distclean - --enable-nls +make install-world ``` diff --git a/tests/checkdb.py b/tests/checkdb.py index 68dec14b6..5b6dda250 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -546,11 +546,8 @@ def test_checkdb_checkunique(self): # @unittest.skip("skip") def test_checkdb_sigint_handling(self): """""" - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') node = self.make_simple_node( diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index b76345b89..3baa0ba0b 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -20,11 +20,6 @@ def test_minrecpoint_on_replica(self): """ self._check_gdb_flag_or_skip_test() - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/tests/replica.py b/tests/replica.py index 0a75ea173..acf655aac 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -539,11 +539,8 @@ def test_replica_promote(self): start backup from replica, during backup promote replica check that backup is failed """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') master = self.make_simple_node( diff --git a/tests/validate.py b/tests/validate.py index 41aa9ea23..22a03c3be 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -1089,11 +1089,8 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ check that interrupt during validation is handled correctly """ - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), diff --git a/travis/run_tests.sh b/travis/run_tests.sh index c20c95dda..5af619f97 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -107,17 +107,18 @@ echo PGPROBACKUPBIN=${PGPROBACKUPBIN} echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} +echo ADDITIONAL_ENV_FLAGS=${ENV_FLAGS} if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - python3 -m unittest -v tests - python3 -m unittest -v tests.init -elif [ "$MODE" = "TMP" ]; then - echo MODE=TMP - PGPROBACKUP_GDB=ON python3 -m unittest -v tests + ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests + ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests.init +#elif [ "$MODE" = "FULL" ]; then +# echo MODE=FULL +# ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests else echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - python3 -m unittest -v tests.$MODE + ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests.$MODE fi # Generate *.gcov files From 37244019508ae3396e521f2ed87bb8c2eca9108d Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 4 Jul 2022 14:37:36 +0300 Subject: [PATCH 1868/2107] [PBCKP-220] final junk cleanup --- .travis.yml | 2 -- travis/run_tests.sh | 10 +++------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index d5c9c68b7..26b2bc4e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,8 +34,6 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -# - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=FULL ENV_FLAGS=PGPROBACKUP_GDB=ON - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE ENV_FLAGS=PGPROBACKUP_GDB=ON # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 5af619f97..37614f970 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -107,18 +107,14 @@ echo PGPROBACKUPBIN=${PGPROBACKUPBIN} echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} -echo ADDITIONAL_ENV_FLAGS=${ENV_FLAGS} if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests - ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests.init -#elif [ "$MODE" = "FULL" ]; then -# echo MODE=FULL -# ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests + python3 -m unittest -v tests + python3 -m unittest -v tests.init else echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - ${ADDITIONAL_ENV_FLAGS} python3 -m unittest -v tests.$MODE + python3 -m unittest -v tests.$MODE fi # Generate *.gcov files From 94caeb1793feddc0fe13b5f94615efcd0ca776dd Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Wed, 29 Jun 2022 22:19:31 +0500 Subject: [PATCH 1869/2107] Shorthand return-expression --- tests/helpers/ptrack_helpers.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a4ec7c9cf..8da802193 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -100,11 +100,7 @@ def is_enterprise(): stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if b'enterprise' in p.communicate()[0]: - return True - else: - return False - + return b'postgrespro.ru' in p.communicate()[0] class ProbackupException(Exception): def __init__(self, message, cmd): From 3071cec4f647582eac8bd58ce5624f97e90745c4 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 11 Jul 2022 14:50:22 +0300 Subject: [PATCH 1870/2107] fix ArchiveTest.test_pgpro434_4 --- tests/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/archive.py b/tests/archive.py index cd8d4404f..52fb225e8 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -317,7 +317,7 @@ def test_pgpro434_4(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - self.set_auto_conf(node, {'archive_command': "'exit 1'"}) + self.set_auto_conf(node, {'archive_command': 'exit 1'}) node.reload() os.environ["PGAPPNAME"] = "foo" From 81c53ea0bbda3251ab527dc950327b86e69e7645 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 12 Jul 2022 15:04:21 +0300 Subject: [PATCH 1871/2107] [PBCKP-231] hotfix for python2 --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e3036d9c4..de7742749 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1880,7 +1880,7 @@ def __str__(self): class GDBobj: - def __init__(self, cmd, env: ProbackupTest, attach=False): + def __init__(self, cmd, env, attach=False): self.verbose = env.verbose self.output = '' From 65345f20a00cc29c4661298f60b7adfd51392360 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Mon, 1 Aug 2022 19:59:39 +0500 Subject: [PATCH 1872/2107] Revert "Add --start-time option into help message" This reverts commit c81c54be4cac6f900e6b73df06788f349eecb3af. --- src/help.c | 3 --- tests/expected/option_help.out | 1 - 2 files changed, 4 deletions(-) diff --git a/src/help.c b/src/help.c index 7a1a1c580..8ebe734a3 100644 --- a/src/help.c +++ b/src/help.c @@ -150,7 +150,6 @@ help_pg_probackup(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n")); - printf(_(" [--start-time]\n")); printf(_(" [--help]\n")); @@ -324,7 +323,6 @@ help_backup(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); - printf(_(" [--start-time]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); @@ -345,7 +343,6 @@ help_backup(void) printf(_(" --no-sync do not sync backed up files to disk\n")); printf(_(" --note=text add note to backup\n")); printf(_(" (example: --note='backup before app update to v13.1')\n")); - printf(_(" --start-time set time of starting backup as a parameter for naming backup\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 9026b99b3..00b50d10c 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -68,7 +68,6 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--ttl=interval] [--expire-time=timestamp] [--note=text] - [--start-time] [--help] pg_probackup restore -B backup-path --instance=instance_name From 8ce27c9713a1e63825c05a2c2ac7fe4481b69d34 Mon Sep 17 00:00:00 2001 From: "d.lepikhova" Date: Tue, 2 Aug 2022 18:13:11 +0500 Subject: [PATCH 1873/2107] Add warning about using --start-time option --- src/pg_probackup.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 15f2542b0..2738280c7 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -944,6 +944,11 @@ main(int argc, char *argv[]) current.stream = stream_wal; if (start_time == 0) start_time = current_time; + else + elog(WARNING, "Please do not use the --start-time option to start backup. " + "This is a service option required to work with other extensions. " + "We do not guarantee future support for this flag."); + /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) From 6d1d739400501dc3a770ee451def75c032a984c6 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 4 Aug 2022 16:15:40 +0300 Subject: [PATCH 1874/2107] [PBCKP-220] hotfix for PtrackTest --- tests/ptrack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 08ea90f8d..c101f9361 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -24,6 +24,8 @@ def test_drop_rel_during_backup_ptrack(self): """ drop relation during ptrack backup """ + self._check_gdb_flag_or_skip_test() + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), @@ -996,6 +998,8 @@ def test_ptrack_get_block(self): make node, make full and ptrack stream backups, restore them and check data correctness """ + self._check_gdb_flag_or_skip_test() + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), From 2e1950a7ff24b997d2b8c054060ca3a88c4973b5 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 4 Aug 2022 17:16:17 +0300 Subject: [PATCH 1875/2107] [PBCKP-129] change catchup logging levels (#473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [PBCKP-129] change catchup logging level verbosity: INFO – common information LOG – same as INFO + info about files VERBOSE – same as LOG + info about block and SQL queries --- src/archive.c | 16 ++++++++-------- src/backup.c | 35 +++++++++++++++++------------------ src/catchup.c | 30 +++++++++++++++--------------- src/data.c | 34 +++++++++++++++++----------------- src/dir.c | 16 ++++++++-------- src/merge.c | 25 ++++++++++++------------- src/restore.c | 29 ++++++++++++++--------------- src/util.c | 4 ++-- 8 files changed, 93 insertions(+), 96 deletions(-) diff --git a/src/archive.c b/src/archive.c index 0f32d9345..48114d955 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2018-2021, Postgres Professional + * Portions Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -361,7 +361,7 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, canonicalize_path(wal_file_ready); canonicalize_path(wal_file_done); /* It is ok to rename status file in archive_status directory */ - elog(VERBOSE, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); + elog(LOG, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); /* do not error out, if rename failed */ if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) @@ -505,7 +505,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d } part_opened: - elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); + elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); /* Check if possible to skip copying */ if (fileExists(to_fullpath, FIO_BACKUP_HOST)) { @@ -595,7 +595,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d to_fullpath_part, strerror(errno)); } - elog(VERBOSE, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); + elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -752,7 +752,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, } part_opened: - elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); + elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); /* Check if possible to skip copying, */ if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) @@ -844,7 +844,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, to_fullpath_gz_part, strerror(errno)); } - elog(VERBOSE, "Rename \"%s\" to \"%s\"", + elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_gz_part, to_fullpath_gz); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -1155,7 +1155,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false)) { fail_count = 0; - elog(INFO, "pg_probackup archive-get copied WAL file %s", wal_file_name); + elog(LOG, "pg_probackup archive-get copied WAL file %s", wal_file_name); n_fetched++; break; } @@ -1511,7 +1511,7 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ int exit_code = 0; - elog(VERBOSE, "Attempting to %s WAL file '%s'", + elog(LOG, "Attempting to %s WAL file '%s'", is_decompress ? "open compressed" : "open", from_path); /* open source file for read */ diff --git a/src/backup.c b/src/backup.c index c575865c4..f61266f5f 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3,7 +3,7 @@ * backup.c: backup DB cluster, archived WAL * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -116,7 +116,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, char pretty_time[20]; char pretty_bytes[20]; - elog(LOG, "Database backup start"); + elog(INFO, "Database backup start"); if(current.external_dir_str) { external_dirs = make_external_directory_list(current.external_dir_str, @@ -336,11 +336,11 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, /* Extract information about files in backup_list parsing their names:*/ parse_filelist_filenames(backup_files_list, instance_config.pgdata); - elog(LOG, "Current Start LSN: %X/%X, TLI: %X", + elog(INFO, "Current Start LSN: %X/%X, TLI: %X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), current.tli); if (current.backup_mode != BACKUP_MODE_FULL) - elog(LOG, "Parent Start LSN: %X/%X, TLI: %X", + elog(INFO, "Parent Start LSN: %X/%X, TLI: %X", (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), prev_backup->tli); @@ -412,7 +412,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, else join_path_components(dirpath, current.database_dir, file->rel_path); - elog(VERBOSE, "Create directory '%s'", dirpath); + elog(LOG, "Create directory '%s'", dirpath); fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } @@ -673,7 +673,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) nodeInfo->checksum_version = current.checksum_version; if (current.checksum_version) - elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + elog(INFO, "This PostgreSQL instance was initialized with data block checksums. " "Data block corruption will be detected"); else elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " @@ -1513,7 +1513,7 @@ wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBa stop_lsn_exists = true; } - elog(LOG, "stop_lsn: %X/%X", + elog(INFO, "stop_lsn: %X/%X", (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); /* @@ -1902,7 +1902,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb backup->recovery_xid = stop_backup_result.snapshot_xid; - elog(LOG, "Getting the Recovery Time from WAL"); + elog(INFO, "Getting the Recovery Time from WAL"); /* iterate over WAL from stop_backup lsn to start_backup lsn */ if (!read_recovery_info(xlog_path, backup->tli, @@ -1910,7 +1910,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb backup->start_lsn, backup->stop_lsn, &backup->recovery_time)) { - elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); + elog(INFO, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); backup->recovery_time = stop_backup_result.invocation_time; } @@ -1992,9 +1992,8 @@ backup_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "interrupted during backup"); - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_backup_files_list, file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->rel_path); /* Handle zero sized files */ if (file->size == 0) @@ -2064,11 +2063,11 @@ backup_files(void *arg) if (file->write_size == BYTES_INVALID) { - elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath); + elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath); continue; } - elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", from_fullpath, file->write_size); } @@ -2186,26 +2185,26 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) elog(ERROR, "Out of memory"); len = strlen("/pg_compression"); cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; - elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + elog(LOG, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); for (p = (int) i; p >= 0; p--) { prev_file = (pgFile *) parray_get(files, (size_t) p); - elog(VERBOSE, "Checking file in cfs tablespace %s", prev_file->rel_path); + elog(LOG, "Checking file in cfs tablespace %s", prev_file->rel_path); if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) { if (S_ISREG(prev_file->mode) && prev_file->is_datafile) { - elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", + elog(LOG, "Setting 'is_cfs' on file %s, name %s", prev_file->rel_path, prev_file->name); prev_file->is_cfs = true; } } else { - elog(VERBOSE, "Breaking on %s", prev_file->rel_path); + elog(LOG, "Breaking on %s", prev_file->rel_path); break; } } diff --git a/src/catchup.c b/src/catchup.c index 3c522afb7..79e3361a8 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -2,7 +2,7 @@ * * catchup.c: sync DB cluster * - * Copyright (c) 2022, Postgres Professional + * Copyright (c) 2021-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -203,7 +203,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, /* fill dest_redo.lsn and dest_redo.tli */ get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); - elog(VERBOSE, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", + elog(LOG, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); if (current.tli != 1) @@ -398,9 +398,8 @@ catchup_thread_runner(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during catchup"); - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_files, file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_files, file->rel_path); /* construct destination filepath */ Assert(file->external_dir_num == 0); @@ -447,12 +446,12 @@ catchup_thread_runner(void *arg) if (file->write_size == BYTES_INVALID) { - elog(VERBOSE, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); + elog(LOG, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); continue; } arguments->transfered_bytes += file->write_size; - elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", + elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", from_fullpath, file->write_size); } @@ -607,7 +606,7 @@ filter_filelist(parray *filelist, const char *pgdata, && parray_bsearch(exclude_relative_paths_list, file->rel_path, pgPrefixCompareString)!= NULL) ) { - elog(LOG, "%s file \"%s\" excluded with --exclude-path option", logging_string, full_path); + elog(INFO, "%s file \"%s\" excluded with --exclude-path option", logging_string, full_path); file->excluded = true; } } @@ -650,7 +649,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, if (exclude_relative_paths_list != NULL) parray_qsort(exclude_relative_paths_list, pgCompareString); - elog(LOG, "Database catchup start"); + elog(INFO, "Database catchup start"); if (current.backup_mode != BACKUP_MODE_FULL) { @@ -697,7 +696,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Call pg_start_backup function in PostgreSQL connect */ pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); - elog(LOG, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + elog(INFO, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); } /* Sanity: source cluster must be "in future" relatively to dest cluster */ @@ -772,11 +771,11 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, pretty_bytes); } - elog(LOG, "Start LSN (source): %X/%X, TLI: %X", + elog(INFO, "Start LSN (source): %X/%X, TLI: %X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), current.tli); if (current.backup_mode != BACKUP_MODE_FULL) - elog(LOG, "LSN in destination: %X/%X, TLI: %X", + elog(INFO, "LSN in destination: %X/%X, TLI: %X", (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn), dest_redo.tli); @@ -829,7 +828,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char dirpath[MAXPGPATH]; join_path_components(dirpath, dest_pgdata, file->rel_path); - elog(VERBOSE, "Create directory '%s'", dirpath); + + elog(LOG, "Create directory '%s'", dirpath); if (!dry_run) fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); } @@ -859,7 +859,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, join_path_components(to_path, dest_pgdata, file->rel_path); - elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + elog(INFO, "Create directory \"%s\" and symbolic link \"%s\"", linked_path, to_path); if (!dry_run) @@ -946,7 +946,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, { fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); } - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + elog(LOG, "Deleted file \"%s\"", fullpath); /* shrink dest pgdata list */ pgFileFree(file); diff --git a/src/data.c b/src/data.c index e5a551127..5c5fdf4f0 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -696,7 +696,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa use_pagemap = true; if (use_pagemap) - elog(VERBOSE, "Using pagemap for file \"%s\"", file->rel_path); + elog(LOG, "Using pagemap for file \"%s\"", file->rel_path); /* Remote mode */ if (fio_is_remote(FIO_DB_HOST)) @@ -795,7 +795,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, } /* - * If nonedata file exists in previous backup + * If non-data file exists in previous backup * and its mtime is less than parent backup start time ... */ if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) && (prev_file && file->exists_in_prev && @@ -1197,7 +1197,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers datapagemap_add(map, blknum); } - elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); + elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); return write_len; } @@ -1220,7 +1220,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during nonedata file restore"); + elog(ERROR, "Interrupted during non-data file restore"); read_len = fread(buf, 1, STDIO_BUFSIZE, in); @@ -1241,7 +1241,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, pg_free(buf); - elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); + elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } size_t @@ -1286,7 +1286,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, */ if (!tmp_file) { - elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", dest_file->rel_path, base36enc(tmp_backup->start_time)); continue; } @@ -1311,14 +1311,14 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* sanity */ if (!tmp_backup) - elog(ERROR, "Failed to locate a backup containing full copy of nonedata file \"%s\"", + elog(ERROR, "Failed to locate a backup containing full copy of non-data file \"%s\"", to_fullpath); if (!tmp_file) - elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", to_fullpath); + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of nonedata file has invalid size: %li. " + elog(ERROR, "Full copy of non-data file has invalid size: %li. " "Metadata corruption in backup %s in file: \"%s\"", tmp_file->write_size, base36enc(tmp_backup->start_time), to_fullpath); @@ -1331,7 +1331,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (file_crc == tmp_file->crc) { - elog(VERBOSE, "Already existing nonedata file \"%s\" has the same checksum, skip restore", + elog(LOG, "Already existing non-data file \"%s\" has the same checksum, skip restore", to_fullpath); return 0; } @@ -1359,7 +1359,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - /* disable stdio buffering for nonedata files */ + /* disable stdio buffering for non-data files */ setvbuf(in, NULL, _IONBF, BUFSIZ); /* do actual work */ @@ -1683,7 +1683,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, int n_hdr = -1; off_t cur_pos_in = 0; - elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); + elog(LOG, "Validate relation blocks for file \"%s\"", fullpath); /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); @@ -1742,7 +1742,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); else - elog(INFO, "Seek to %u", headers[n_hdr].pos); + elog(VERBOSE, "Seek to %u", headers[n_hdr].pos); cur_pos_in = headers[n_hdr].pos; } @@ -1766,7 +1766,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* backward compatibility kludge TODO: remove in 3.0 */ if (compressed_size == PageIsTruncated) { - elog(INFO, "Block %u of \"%s\" is truncated", + elog(VERBOSE, "Block %u of \"%s\" is truncated", blknum, fullpath); continue; } @@ -1837,10 +1837,10 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, switch (rc) { case PAGE_IS_NOT_FOUND: - elog(LOG, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); + elog(VERBOSE, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); break; case PAGE_IS_ZEROED: - elog(LOG, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); + elog(VERBOSE, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); break; case PAGE_HEADER_IS_INVALID: elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); diff --git a/src/dir.c b/src/dir.c index 0b2d18778..561586f87 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,7 +3,7 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -640,7 +640,7 @@ dir_check_file(pgFile *file, bool backup_logs) pgdata_exclude_files_non_exclusive[i]) == 0) { /* Skip */ - elog(VERBOSE, "Excluding file: %s", file->name); + elog(LOG, "Excluding file: %s", file->name); return CHECK_FALSE; } } @@ -649,7 +649,7 @@ dir_check_file(pgFile *file, bool backup_logs) if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) { /* Skip */ - elog(VERBOSE, "Excluding file: %s", file->name); + elog(LOG, "Excluding file: %s", file->name); return CHECK_FALSE; } } @@ -669,7 +669,7 @@ dir_check_file(pgFile *file, bool backup_logs) /* exclude by dirname */ if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) { - elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + elog(LOG, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; } } @@ -679,7 +679,7 @@ dir_check_file(pgFile *file, bool backup_logs) if (strcmp(file->rel_path, PG_LOG_DIR) == 0) { /* Skip */ - elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + elog(LOG, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; } } @@ -1166,7 +1166,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba join_path_components(to_path, data_dir, dir->rel_path); - elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + elog(LOG, "Create directory \"%s\" and symbolic link \"%s\"", linked_path, to_path); /* create tablespace directory */ @@ -1183,7 +1183,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba } /* This is not symlink, create directory */ - elog(VERBOSE, "Create directory \"%s\"", dir->rel_path); + elog(LOG, "Create directory \"%s\"", dir->rel_path); join_path_components(to_path, data_dir, dir->rel_path); @@ -1934,7 +1934,7 @@ cleanup_tablespace(const char *path) join_path_components(fullpath, path, file->rel_path); fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + elog(LOG, "Deleted file \"%s\"", fullpath); } parray_walk(files, pgFileFree); diff --git a/src/merge.c b/src/merge.c index 1ce92bb42..1ce49f9a2 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2,7 +2,7 @@ * * merge.c: merge FULL and incremental backups * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -171,7 +171,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool elog(ERROR, "Merge target is full backup and has multiple direct children, " "you must specify child backup id you want to merge with"); - elog(LOG, "Looking for closest incremental backup to merge with"); + elog(INFO, "Looking for closest incremental backup to merge with"); /* Look for closest child backup */ for (i = 0; i < parray_num(backups); i++) @@ -810,7 +810,7 @@ merge_chain(InstanceState *instanceState, join_path_components(full_file_path, full_database_dir, full_file->rel_path); pgFileDelete(full_file->mode, full_file_path); - elog(VERBOSE, "Deleted \"%s\"", full_file_path); + elog(LOG, "Deleted \"%s\"", full_file_path); } } @@ -956,9 +956,8 @@ merge_files(void *arg) if (S_ISDIR(dest_file->mode)) goto done; - if (progress) - elog(INFO, "Progress: (%d/%lu). Merging file \"%s\"", - i + 1, n_files, dest_file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%lu). Merging file \"%s\"", + i + 1, n_files, dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) tmp_file->segno = dest_file->segno; @@ -1063,7 +1062,7 @@ merge_files(void *arg) { BackupPageHeader2 *headers = NULL; - elog(VERBOSE, "The file didn`t changed since FULL backup, skip merge: \"%s\"", + elog(LOG, "The file didn`t changed since FULL backup, skip merge: \"%s\"", file->rel_path); tmp_file->crc = file->crc; @@ -1144,7 +1143,7 @@ remove_dir_with_files(const char *path) join_path_components(full_path, path, file->rel_path); pgFileDelete(file->mode, full_path); - elog(VERBOSE, "Deleted \"%s\"", full_path); + elog(LOG, "Deleted \"%s\"", full_path); } /* cleanup */ @@ -1193,7 +1192,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, char new_path[MAXPGPATH]; makeExternalDirPathByNum(old_path, externaldir_template, i + 1); makeExternalDirPathByNum(new_path, externaldir_template, from_num); - elog(VERBOSE, "Rename %s to %s", old_path, new_path); + elog(LOG, "Rename %s to %s", old_path, new_path); if (rename (old_path, new_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", old_path, new_path, strerror(errno)); @@ -1346,7 +1345,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, */ if (!from_file) { - elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", dest_file->rel_path, base36enc(from_backup->start_time)); continue; } @@ -1357,11 +1356,11 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, /* sanity */ if (!from_backup) - elog(ERROR, "Failed to found a backup containing full copy of nonedata file \"%s\"", + elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", dest_file->rel_path); if (!from_file) - elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", dest_file->rel_path); + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", dest_file->rel_path); /* set path to source file */ if (from_file->external_dir_num) @@ -1450,4 +1449,4 @@ is_forward_compatible(parray *parent_chain) } return true; -} \ No newline at end of file +} diff --git a/src/restore.c b/src/restore.c index fbf0c0398..c877290b1 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -843,7 +843,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, external_path = parray_get(external_dirs, file->external_dir_num - 1); join_path_components(dirpath, external_path, file->rel_path); - elog(VERBOSE, "Create external directory \"%s\"", dirpath); + elog(LOG, "Create external directory \"%s\"", dirpath); fio_mkdir(dirpath, file->mode, FIO_DB_HOST); } } @@ -923,7 +923,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(fullpath, pgdata_path, file->rel_path); fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + elog(LOG, "Deleted file \"%s\"", fullpath); /* shrink pgdata list */ pgFileFree(file); @@ -1131,9 +1131,8 @@ restore_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during restore"); - if (progress) - elog(INFO, "Progress: (%d/%lu). Restore file \"%s\"", - i + 1, n_files, dest_file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%lu). Restore file \"%s\"", + i + 1, n_files, dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) @@ -1149,7 +1148,7 @@ restore_files(void *arg) create_empty_file(FIO_BACKUP_HOST, arguments->to_root, FIO_DB_HOST, dest_file); - elog(VERBOSE, "Skip file due to partial restore: \"%s\"", + elog(LOG, "Skip file due to partial restore: \"%s\"", dest_file->rel_path); continue; } @@ -1159,7 +1158,7 @@ restore_files(void *arg) if ((dest_file->external_dir_num == 0) && strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) { - elog(VERBOSE, "Skip tablespace_map"); + elog(LOG, "Skip tablespace_map"); continue; } @@ -1167,7 +1166,7 @@ restore_files(void *arg) if ((dest_file->external_dir_num == 0) && strcmp(DATABASE_MAP, dest_file->rel_path) == 0) { - elog(VERBOSE, "Skip database_map"); + elog(LOG, "Skip database_map"); continue; } @@ -1239,9 +1238,9 @@ restore_files(void *arg) strerror(errno)); if (!dest_file->is_datafile || dest_file->is_cfs) - elog(VERBOSE, "Restoring nonedata file: \"%s\"", to_fullpath); + elog(LOG, "Restoring non-data file: \"%s\"", to_fullpath); else - elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); + elog(LOG, "Restoring data file: \"%s\"", to_fullpath); // If destination file is 0 sized, then just close it and go for the next if (dest_file->write_size == 0) @@ -1261,10 +1260,10 @@ restore_files(void *arg) } else { - /* disable stdio buffering for local destination nonedata file */ + /* disable stdio buffering for local destination non-data file */ if (!fio_is_remote_file(out)) setvbuf(out, NULL, _IONBF, BUFSIZ); - /* Destination file is nonedata file */ + /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, arguments->dest_backup, dest_file, out, to_fullpath, already_exists); @@ -1773,7 +1772,7 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict } if (fd && (ferror(fd))) - elog(ERROR, "Failed to read from file: \"%s\"", path); + elog(ERROR, "Failed to read from file: \"%s\"", path); if (fd) fclose(fd); @@ -2188,7 +2187,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, * data files content, because based on pg_control information we will * choose a backup suitable for lsn based incremental restore. */ - elog(INFO, "Trying to read pg_control file in destination directory"); + elog(LOG, "Trying to read pg_control file in destination directory"); system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); diff --git a/src/util.c b/src/util.c index fb33fd046..4d6c50a07 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -589,7 +589,7 @@ datapagemap_print_debug(datapagemap_t *map) iter = datapagemap_iterate(map); while (datapagemap_next(iter, &blocknum)) - elog(INFO, " block %u", blocknum); + elog(VERBOSE, " block %u", blocknum); pg_free(iter); } From c0c07ac2ff3c9a749c4ad5ec57a23a3d6e800cca Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 15 Jun 2022 22:55:59 +0300 Subject: [PATCH 1876/2107] [PBCKP-216] Setting C locale globally, env locale is only set while doing while printing big tables ...in order to impose dot-based floating point representation on logging and JSON-representation tags: pg_probackup --- src/pg_probackup.c | 6 +++++ src/pg_probackup.h | 2 ++ src/show.c | 53 +++++++++++++++++++++++++++++++++++++-- src/utils/configuration.c | 3 +++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b9b3af0b9..3ffc3bb9e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -311,6 +311,10 @@ main(int argc, char *argv[]) set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_probackup")); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); + // Setting C locale for numeric values in order to impose dot-based floating-point representation + memorize_environment_locale(); + setlocale(LC_NUMERIC, "C"); + /* Get current time */ current_time = time(NULL); @@ -1024,6 +1028,8 @@ main(int argc, char *argv[]) break; } + free_environment_locale(); + return 0; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 2c4c61036..77893eadc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -905,6 +905,8 @@ extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); /* in show.c */ extern int do_show(CatalogState *catalogState, InstanceState *instanceState, time_t requested_backup_id, bool show_archive); +extern void memorize_environment_locale(void); +extern void free_environment_locale(void); /* in delete.c */ extern void do_delete(InstanceState *instanceState, time_t backup_id); diff --git a/src/show.c b/src/show.c index 22c40cf43..db8a9e225 100644 --- a/src/show.c +++ b/src/show.c @@ -3,7 +3,7 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -12,6 +12,7 @@ #include #include +#include #include #include "utils/json.h" @@ -71,6 +72,43 @@ static PQExpBufferData show_buf; static bool first_instance = true; static int32 json_level = 0; +static const char* lc_env_locale; +typedef enum { + LOCALE_C, // Used for formatting output to unify the dot-based floating point representation + LOCALE_ENV // Default environment locale +} output_numeric_locale; + +#ifdef HAVE_USELOCALE +static locale_t env_locale, c_locale; +#endif +void memorize_environment_locale() { + lc_env_locale = (const char *)getenv("LC_NUMERIC"); + lc_env_locale = lc_env_locale != NULL ? lc_env_locale : "C"; +#ifdef HAVE_USELOCALE + env_locale = newlocale(LC_NUMERIC_MASK, lc_env_locale, (locale_t)0); + c_locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0); +#else +#ifdef HAVE__CONFIGTHREADLOCALE + _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); +#endif +#endif +} + +void free_environment_locale() { +#ifdef HAVE_USELOCALE + freelocale(env_locale); + freelocale(c_locale); +#endif +} + +static void set_output_numeric_locale(output_numeric_locale loc) { +#ifdef HAVE_USELOCALE + uselocale(loc == LOCALE_C ? c_locale : env_locale); +#else + setlocale(LC_NUMERIC, loc == LOCALE_C ? "C" : lc_env_locale); +#endif +} + /* * Entry point of pg_probackup SHOW subcommand. */ @@ -513,6 +551,9 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na ShowBackendRow *rows; TimeLineID parent_tli = 0; + // Since we've been printing a table, set LC_NUMERIC to its default environment value + set_output_numeric_locale(LOCALE_ENV); + for (i = 0; i < SHOW_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -726,6 +767,8 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na } pfree(rows); + // Restore the C locale + set_output_numeric_locale(LOCALE_C); } /* @@ -806,6 +849,9 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, uint32 widths_sum = 0; ShowArchiveRow *rows; + // Since we've been printing a table, set LC_NUMERIC to its default environment value + set_output_numeric_locale(LOCALE_ENV); + for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -973,6 +1019,8 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, } pfree(rows); + // Restore the C locale + set_output_numeric_locale(LOCALE_C); //TODO: free timelines } @@ -1045,8 +1093,9 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, appendPQExpBuffer(buf, "%lu", tlinfo->size); json_add_key(buf, "zratio", json_level); + if (tlinfo->size != 0) - zratio = ((float)xlog_seg_size*tlinfo->n_xlog_files) / tlinfo->size; + zratio = ((float) xlog_seg_size * tlinfo->n_xlog_files) / tlinfo->size; appendPQExpBuffer(buf, "%.2f", zratio); if (tlinfo->closest_backup != NULL) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 04bfbbe3b..7ab242aa3 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -18,6 +18,9 @@ #include "getopt_long.h" +#ifndef WIN32 +#include +#endif #include #define MAXPG_LSNCOMPONENT 8 From f88410e5d7d359a2355265a3fc35e67271877e17 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 4 Aug 2022 16:15:40 +0300 Subject: [PATCH 1877/2107] [PBCKP-220] hotfix for PtrackTest --- tests/ptrack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ptrack.py b/tests/ptrack.py index 9741c9561..d46ece119 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -24,6 +24,8 @@ def test_drop_rel_during_backup_ptrack(self): """ drop relation during ptrack backup """ + self._check_gdb_flag_or_skip_test() + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), @@ -994,6 +996,8 @@ def test_ptrack_get_block(self): make node, make full and ptrack stream backups, restore them and check data correctness """ + self._check_gdb_flag_or_skip_test() + backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'node'), From 1e6cf10c87ae6c59d9383b246f2b9f47a78771ab Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Thu, 4 Aug 2022 01:27:26 +0300 Subject: [PATCH 1878/2107] [PBCKP-169]: adding json format for logs. Added 2 flags. --log-format-console=plain|json --log-format-file=plain|json Option 'log-format-console' set only from terminal. --- src/configure.c | 124 ++++++++++----- src/help.c | 49 +++++- src/pg_probackup.c | 11 ++ src/utils/json.c | 18 +++ src/utils/json.h | 1 + src/utils/logger.c | 240 +++++++++++++++++++++++++----- src/utils/logger.h | 10 ++ tests/expected/option_help.out | 3 + tests/expected/option_help_ru.out | 3 + 9 files changed, 386 insertions(+), 73 deletions(-) diff --git a/src/configure.c b/src/configure.c index 9ffe2d7a7..6e8700de1 100644 --- a/src/configure.c +++ b/src/configure.c @@ -17,10 +17,14 @@ static void assign_log_level_console(ConfigOption *opt, const char *arg); static void assign_log_level_file(ConfigOption *opt, const char *arg); +static void assign_log_format_console(ConfigOption *opt, const char *arg); +static void assign_log_format_file(ConfigOption *opt, const char *arg); static void assign_compress_alg(ConfigOption *opt, const char *arg); static char *get_log_level_console(ConfigOption *opt); static char *get_log_level_file(ConfigOption *opt); +static char *get_log_format_console(ConfigOption *opt); +static char *get_log_format_file(ConfigOption *opt); static char *get_compress_alg(ConfigOption *opt); static void show_configure_start(void); @@ -154,90 +158,100 @@ ConfigOption instance_options[] = OPTION_LOG_GROUP, 0, get_log_level_file }, { - 's', 214, "log-filename", + 'f', 214, "log-format-console", + assign_log_format_console, SOURCE_CMD_STRICT, 0, + OPTION_LOG_GROUP, 0, get_log_format_console + }, + { + 'f', 215, "log-format-file", + assign_log_format_file, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, get_log_format_file + }, + { + 's', 216, "log-filename", &instance_config.logger.log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 215, "error-log-filename", + 's', 217, "error-log-filename", &instance_config.logger.error_log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 216, "log-directory", + 's', 218, "log-directory", &instance_config.logger.log_directory, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 217, "log-rotation-size", + 'U', 219, "log-rotation-size", &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 218, "log-rotation-age", + 'U', 220, "log-rotation-age", &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 219, "retention-redundancy", + 'u', 221, "retention-redundancy", &instance_config.retention_redundancy, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 220, "retention-window", + 'u', 222, "retention-window", &instance_config.retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 221, "wal-depth", + 'u', 223, "wal-depth", &instance_config.wal_depth, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 'f', 222, "compress-algorithm", + 'f', 224, "compress-algorithm", assign_compress_alg, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { - 'u', 223, "compress-level", + 'u', 225, "compress-level", &instance_config.compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 224, "remote-proto", + 's', 226, "remote-proto", &instance_config.remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-host", + 's', 227, "remote-host", &instance_config.remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-port", + 's', 228, "remote-port", &instance_config.remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-path", + 's', 229, "remote-path", &instance_config.remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "remote-user", + 's', 230, "remote-user", &instance_config.remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-options", + 's', 231, "ssh-options", &instance_config.remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 230, "ssh-config", + 's', 232, "ssh-config", &instance_config.remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, @@ -388,6 +402,8 @@ readInstanceConfigFile(InstanceState *instanceState) InstanceConfig *instance = pgut_new(InstanceConfig); char *log_level_console = NULL; char *log_level_file = NULL; + char *log_format_console = NULL; + char *log_format_file = NULL; char *compress_alg = NULL; int parsed_options; @@ -509,90 +525,100 @@ readInstanceConfigFile(InstanceState *instanceState) OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 214, "log-filename", + 's', 214, "log-format-console", + &log_format_console, SOURCE_CMD_STRICT, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 215, "log-format-file", + &log_format_file, SOURCE_CMD, 0, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 216, "log-filename", &instance->logger.log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 215, "error-log-filename", + 's', 217, "error-log-filename", &instance->logger.error_log_filename, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 216, "log-directory", + 's', 218, "log-directory", &instance->logger.log_directory, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 217, "log-rotation-size", + 'U', 219, "log-rotation-size", &instance->logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 218, "log-rotation-age", + 'U', 220, "log-rotation-age", &instance->logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 219, "retention-redundancy", + 'u', 221, "retention-redundancy", &instance->retention_redundancy, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 220, "retention-window", + 'u', 222, "retention-window", &instance->retention_window, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 221, "wal-depth", + 'u', 223, "wal-depth", &instance->wal_depth, SOURCE_CMD, 0, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 's', 222, "compress-algorithm", + 's', 224, "compress-algorithm", &compress_alg, SOURCE_CMD, 0, OPTION_LOG_GROUP, 0, option_get_value }, { - 'u', 223, "compress-level", + 'u', 225, "compress-level", &instance->compress_level, SOURCE_CMD, 0, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 224, "remote-proto", + 's', 226, "remote-proto", &instance->remote.proto, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-host", + 's', 227, "remote-host", &instance->remote.host, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-port", + 's', 228, "remote-port", &instance->remote.port, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-path", + 's', 229, "remote-path", &instance->remote.path, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "remote-user", + 's', 230, "remote-user", &instance->remote.user, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-options", + 's', 231, "ssh-options", &instance->remote.ssh_options, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 230, "ssh-config", + 's', 232, "ssh-config", &instance->remote.ssh_config, SOURCE_CMD, 0, OPTION_REMOTE_GROUP, 0, option_get_value }, @@ -625,6 +651,12 @@ readInstanceConfigFile(InstanceState *instanceState) if (log_level_file) instance->logger.log_level_file = parse_log_level(log_level_file); + if (log_format_console) + instance->logger.log_format_console = parse_log_format(log_format_console); + + if (log_format_file) + instance->logger.log_format_file = parse_log_format(log_format_file); + if (compress_alg) instance->compress_alg = parse_compress_alg(compress_alg); @@ -649,6 +681,18 @@ assign_log_level_file(ConfigOption *opt, const char *arg) instance_config.logger.log_level_file = parse_log_level(arg); } +static void +assign_log_format_console(ConfigOption *opt, const char *arg) +{ + instance_config.logger.log_format_console = parse_log_format(arg); +} + +static void +assign_log_format_file(ConfigOption *opt, const char *arg) +{ + instance_config.logger.log_format_file = parse_log_format(arg); +} + static void assign_compress_alg(ConfigOption *opt, const char *arg) { @@ -667,6 +711,18 @@ get_log_level_file(ConfigOption *opt) return pstrdup(deparse_log_level(instance_config.logger.log_level_file)); } +static char * +get_log_format_console(ConfigOption *opt) +{ + return pstrdup(deparse_log_format(instance_config.logger.log_format_console)); +} + +static char * +get_log_format_file(ConfigOption *opt) +{ + return pstrdup(deparse_log_format(instance_config.logger.log_format_file)); +} + static char * get_compress_alg(ConfigOption *opt) { diff --git a/src/help.c b/src/help.c index 85894759e..116a0711c 100644 --- a/src/help.c +++ b/src/help.c @@ -94,6 +94,7 @@ help_pg_probackup(void) printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -131,6 +132,8 @@ help_pg_probackup(void) printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -306,6 +309,8 @@ help_backup(void) printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -353,6 +358,12 @@ help_backup(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -476,7 +487,7 @@ help_restore(void) printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n")); - + printf(_("\n Incremental restore options:\n")); printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); @@ -519,6 +530,12 @@ help_restore(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -586,6 +603,12 @@ help_validate(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -633,6 +656,12 @@ help_checkdb(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); @@ -713,6 +742,12 @@ help_delete(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -737,6 +772,8 @@ help_merge(void) printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -759,6 +796,12 @@ help_merge(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -800,6 +843,7 @@ help_set_config(void) printf(_(" [--restore-command=cmdline]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -831,6 +875,9 @@ help_set_config(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index c5ff02d8a..a242d7f59 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -427,6 +427,17 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); + if (backup_subcmd == SET_CONFIG_CMD) + { + for (int i = 0; i < argc; i++) + { + if (strncmp("--log-format-console", argv[i], strlen("--log-format-console")) == 0) + { + elog(ERROR, "Option 'log-format-console' set only from terminal\n"); + } + } + } + pgut_init(); if (no_color) diff --git a/src/utils/json.c b/src/utils/json.c index 9f13a958f..2c8e0fe9b 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -144,3 +144,21 @@ json_add_escaped(PQExpBuffer buf, const char *str) } appendPQExpBufferChar(buf, '"'); } + +void +json_add_min(PQExpBuffer buf, JsonToken type) +{ + switch (type) + { + case JT_BEGIN_OBJECT: + appendPQExpBufferChar(buf, '{'); + add_comma = false; + break; + case JT_END_OBJECT: + appendPQExpBufferStr(buf, "}\n"); + add_comma = true; + break; + default: + break; + } +} diff --git a/src/utils/json.h b/src/utils/json.h index cc9f1168d..f80832e69 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -25,6 +25,7 @@ typedef enum } JsonToken; extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_min(PQExpBuffer buf, JsonToken type); extern void json_add_key(PQExpBuffer buf, const char *name, int32 level); extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, int32 level, bool escaped); diff --git a/src/utils/logger.c b/src/utils/logger.c index 70bd5dcc4..7ea41f74e 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -19,14 +19,19 @@ #include "utils/configuration.h" +#include "json.h" + /* Logger parameters */ LoggerConfig logger_config = { LOG_LEVEL_CONSOLE_DEFAULT, LOG_LEVEL_FILE_DEFAULT, LOG_FILENAME_DEFAULT, NULL, + NULL, LOG_ROTATION_SIZE_DEFAULT, - LOG_ROTATION_AGE_DEFAULT + LOG_ROTATION_AGE_DEFAULT, + LOG_FORMAT_CONSOLE_DEFAULT, + LOG_FORMAT_FILE_DEFAULT }; /* Implementation for logging.h */ @@ -227,6 +232,35 @@ write_elevel(FILE *stream, int elevel) } } +static void +write_elevel_for_json(PQExpBuffer buf, int elevel) +{ + switch (elevel) + { + case VERBOSE: + appendPQExpBufferStr(buf, "\"VERBOSE\""); + break; + case LOG: + appendPQExpBufferStr(buf, "\"LOG\""); + break; + case INFO: + appendPQExpBufferStr(buf, "\"INFO\""); + break; + case NOTICE: + appendPQExpBufferStr(buf, "\"NOTICE\""); + break; + case WARNING: + appendPQExpBufferStr(buf, "\"WARNING\""); + break; + case ERROR: + appendPQExpBufferStr(buf, "\"ERROR\""); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + /* * Exit with code if it is an error. * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup @@ -276,6 +310,12 @@ elog_internal(int elevel, bool file_only, const char *message) time_t log_time = (time_t) time(NULL); char strfbuf[128]; char str_pid[128]; + char str_pid_json[128]; + char str_thread_json[64]; + PQExpBufferData show_buf; + PQExpBuffer buf_json = &show_buf; + int8 format_console, + format_file; write_to_file = elevel >= logger_config.log_level_file && logger_config.log_directory @@ -283,6 +323,8 @@ elog_internal(int elevel, bool file_only, const char *message) write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && logger_config.log_directory && logger_config.log_directory[0] != '\0'; write_to_stderr = elevel >= logger_config.log_level_console && !file_only; + format_console = logger_config.log_format_console; + format_file = logger_config.log_format_file; if (remote_agent) { @@ -292,10 +334,27 @@ elog_internal(int elevel, bool file_only, const char *message) pthread_lock(&log_file_mutex); loggin_in_progress = true; - if (write_to_file || write_to_error_log || is_archive_cmd) + if (write_to_file || write_to_error_log || is_archive_cmd || + format_console == JSON) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); + if (format_file == JSON || format_console == JSON) + { + snprintf(str_pid_json, sizeof(str_pid_json), "%d", my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + + initPQExpBuffer(&show_buf); + json_add_min(buf_json, JT_BEGIN_OBJECT); + json_add_value(buf_json, "ts", strfbuf, 0, true); + json_add_value(buf_json, "pid", str_pid_json, 0, true); + json_add_key(buf_json, "level", 0); + write_elevel_for_json(buf_json, elevel); + json_add_value(buf_json, "msg", message, 0, true); + json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); + json_add_min(buf_json, JT_END_OBJECT); + } + snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); /* @@ -307,12 +366,18 @@ elog_internal(int elevel, bool file_only, const char *message) { if (log_file == NULL) open_logfile(&log_file, logger_config.log_filename ? logger_config.log_filename : LOG_FILENAME_DEFAULT); + if (format_file == JSON) + { + fputs(buf_json->data, log_file); + } + else + { + fprintf(log_file, "%s ", strfbuf); + fprintf(log_file, "%s ", str_pid); + write_elevel(log_file, elevel); - fprintf(log_file, "%s ", strfbuf); - fprintf(log_file, "%s ", str_pid); - write_elevel(log_file, elevel); - - fprintf(log_file, "%s\n", message); + fprintf(log_file, "%s\n", message); + } fflush(log_file); } @@ -326,11 +391,18 @@ elog_internal(int elevel, bool file_only, const char *message) if (error_log_file == NULL) open_logfile(&error_log_file, logger_config.error_log_filename); - fprintf(error_log_file, "%s ", strfbuf); - fprintf(error_log_file, "%s ", str_pid); - write_elevel(error_log_file, elevel); + if (format_file == JSON) + { + fputs(buf_json->data, error_log_file); + } + else + { + fprintf(error_log_file, "%s ", strfbuf); + fprintf(error_log_file, "%s ", str_pid); + write_elevel(error_log_file, elevel); - fprintf(error_log_file, "%s\n", message); + fprintf(error_log_file, "%s\n", message); + } fflush(error_log_file); } @@ -340,35 +412,47 @@ elog_internal(int elevel, bool file_only, const char *message) */ if (write_to_stderr) { - if (is_archive_cmd) + if (format_console == JSON) { - char str_thread[64]; - /* [Issue #213] fix pgbadger parsing */ - snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); - - fprintf(stderr, "%s ", strfbuf); - fprintf(stderr, "%s ", str_pid); - fprintf(stderr, "%s ", str_thread); + fprintf(stderr, "%s", buf_json->data); } - else if (show_color) + else { - /* color WARNING and ERROR messages */ - if (elevel == WARNING) - fprintf(stderr, "%s", TC_YELLOW_BOLD); - else if (elevel == ERROR) - fprintf(stderr, "%s", TC_RED_BOLD); - } + if (is_archive_cmd) + { + char str_thread[64]; + /* [Issue #213] fix pgbadger parsing */ + snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); - write_elevel(stderr, elevel); + fprintf(stderr, "%s ", strfbuf); + fprintf(stderr, "%s ", str_pid); + fprintf(stderr, "%s ", str_thread); + } + else if (show_color) + { + /* color WARNING and ERROR messages */ + if (elevel == WARNING) + fprintf(stderr, "%s", TC_YELLOW_BOLD); + else if (elevel == ERROR) + fprintf(stderr, "%s", TC_RED_BOLD); + } + + write_elevel(stderr, elevel); + + /* main payload */ + fprintf(stderr, "%s", message); - /* main payload */ - fprintf(stderr, "%s", message); + /* reset color to default */ + if (show_color && (elevel == WARNING || elevel == ERROR)) + fprintf(stderr, "%s", TC_RESET); - /* reset color to default */ - if (show_color && (elevel == WARNING || elevel == ERROR)) - fprintf(stderr, "%s", TC_RESET); + fprintf(stderr, "\n"); + } - fprintf(stderr, "\n"); + if (format_file == JSON || format_console == JSON) + { + termPQExpBuffer(buf_json); + } fflush(stderr); } @@ -386,7 +470,15 @@ elog_internal(int elevel, bool file_only, const char *message) static void elog_stderr(int elevel, const char *fmt, ...) { - va_list args; + va_list args; + PQExpBufferData show_buf; + PQExpBuffer buf_json = &show_buf; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + char str_pid[128]; + char str_thread_json[64]; + char *message; + int8 format_console; /* * Do not log message if severity level is less than log_level. @@ -397,11 +489,37 @@ elog_stderr(int elevel, const char *fmt, ...) va_start(args, fmt); - write_elevel(stderr, elevel); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); + format_console = logger_config.log_format_console; + if (format_console == JSON) + { + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + snprintf(str_pid, sizeof(str_pid), "%d", my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + + initPQExpBuffer(&show_buf); + json_add_min(buf_json, JT_BEGIN_OBJECT); + json_add_value(buf_json, "ts", strfbuf, 0, true); + json_add_value(buf_json, "pid", str_pid, 0, true); + json_add_key(buf_json, "level", 0); + write_elevel_for_json(buf_json, elevel); + message = get_log_message(fmt, args); + json_add_value(buf_json, "msg", message, 0, true); + json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); + json_add_min(buf_json, JT_END_OBJECT); + fputs(buf_json->data, stderr); + pfree(message); + termPQExpBuffer(buf_json); + } + else + { + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + } + + fflush(stderr); va_end(args); exit_if_necessary(elevel); @@ -570,6 +688,36 @@ parse_log_level(const char *level) return 0; } +int +parse_log_format(const char *format) +{ + const char *v = format; + size_t len; + + if (v == NULL) + { + elog(ERROR, "log-format got invalid value"); + return 0; + } + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-format is empty"); + + if (pg_strncasecmp("plain", v, len) == 0) + return PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + return JSON; + + /* Log format is invalid */ + elog(ERROR, "invalid log-format \"%s\"", format); + return 0; +} + /* * Converts integer representation of log level to string. */ @@ -599,6 +747,22 @@ deparse_log_level(int level) return NULL; } +const char * +deparse_log_format(int format) +{ + switch (format) + { + case PLAIN: + return "PLAIN"; + case JSON: + return "JSON"; + default: + elog(ERROR, "invalid log-format %d", format); + } + + return NULL; +} + /* * Construct logfile name using timestamp information. * diff --git a/src/utils/logger.h b/src/utils/logger.h index 6a7407e41..adc5061e0 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -21,6 +21,9 @@ #define ERROR 1 #define LOG_OFF 10 +#define PLAIN 0 +#define JSON 1 + typedef struct LoggerConfig { int log_level_console; @@ -32,6 +35,8 @@ typedef struct LoggerConfig uint64 log_rotation_size; /* Maximum lifetime of an individual log file in minutes */ uint64 log_rotation_age; + int8 log_format_console; + int8 log_format_file; } LoggerConfig; /* Logger parameters */ @@ -43,6 +48,9 @@ extern LoggerConfig logger_config; #define LOG_LEVEL_CONSOLE_DEFAULT INFO #define LOG_LEVEL_FILE_DEFAULT LOG_OFF +#define LOG_FORMAT_CONSOLE_DEFAULT PLAIN +#define LOG_FORMAT_FILE_DEFAULT PLAIN + #define LOG_FILENAME_DEFAULT "pg_probackup.log" #define LOG_DIRECTORY_DEFAULT "log" @@ -56,4 +64,6 @@ extern void init_console(void); extern int parse_log_level(const char *level); extern const char *deparse_log_level(int level); +extern int parse_log_format(const char *format); +extern const char *deparse_log_format(int format); #endif /* LOGGER_H */ diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 659164250..5948d0503 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -12,6 +12,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] @@ -49,6 +50,8 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--no-sync] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-console=log-format-console] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 2e90eb297..358c49428 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -12,6 +12,7 @@ pg_probackup - утилита для управления резервным к [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] @@ -49,6 +50,8 @@ pg_probackup - утилита для управления резервным к [--no-sync] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-console=log-format-console] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] From fffebdc0b38e70a0a0b20c1a0fa1e4b8ce71e820 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Wed, 17 Aug 2022 16:05:44 +0300 Subject: [PATCH 1879/2107] [DOC] [PGPRO-6635] Update pg_probackup documentation for version 2.5.7 [skip travis] --- doc/pgprobackup.xml | 68 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index cb615fb17..fc2a341e8 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -3993,6 +3993,7 @@ pg_probackup restore -B backup_dir --instance cmdline] [--primary-conninfo=primary_conninfo] [-S | --primary-slot-name=slot_name] +[-X wal_dir | --waldir=wal_dir] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -4160,6 +4161,17 @@ pg_probackup restore -B backup_dir --instance + + + + + + + Specifies the directory where WAL should be stored. + + + + @@ -5185,6 +5197,60 @@ pg_probackup catchup -b catchup_mode + + + + Defines the format of the console log. Only set from the command line. Note that you cannot + specify this option in the pg_probackup.conf configuration file through + the command and that the + command also treats this option specified in the configuration file as an error. + Possible values are: + + + + + plain — sets the plain-text format of the console log. + + + + + json — sets the JSON format of the console log. + + + + + + Default: plain + + + + + + + + + Defines the format of log files used. Possible values are: + + + + + plain — sets the plain-text format of log files. + + + + + json — sets the JSON format of log files. + + + + + + Default: plain + + + + + @@ -6056,6 +6122,8 @@ archive-timeout = 5min # Logging parameters log-level-console = INFO log-level-file = OFF +log-format-console = PLAIN +log-format-file = PLAIN log-filename = pg_probackup.log log-rotation-size = 0 log-rotation-age = 0 From 03ad0d092aa7d5b751de2d9edd15ea7cb8c8417f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 19 Aug 2022 11:44:58 +0300 Subject: [PATCH 1880/2107] Version 2.5.7 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e583d7745..8aad0a7cc 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.6" +#define PROGRAM_VERSION "2.5.7" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 96f0f3446..af186a98c 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.6 +pg_probackup 2.5.7 From a58c1831d6aef3514428b8981f6b478df18aca01 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 19 Aug 2022 15:50:17 +0300 Subject: [PATCH 1881/2107] [PGPRO-6635] Compatibility fix for older compilers --- src/pg_probackup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 618705945..5867bd490 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -433,7 +433,8 @@ main(int argc, char *argv[]) if (backup_subcmd == SET_CONFIG_CMD) { - for (int i = 0; i < argc; i++) + int i; + for (i = 0; i < argc; i++) { if (strncmp("--log-format-console", argv[i], strlen("--log-format-console")) == 0) { From e918bae0dc7c5b4f8b5bb6748a47485c526e803f Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 26 Aug 2022 20:00:58 +0300 Subject: [PATCH 1882/2107] Version 2.5.8 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8aad0a7cc..802cbb5c0 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -338,7 +338,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.7" +#define PROGRAM_VERSION "2.5.8" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index af186a98c..4de288907 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.7 +pg_probackup 2.5.8 From 3d7b9f0e807741a546f3d95eb48106e120a89bf7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Sun, 28 Aug 2022 00:39:33 +0300 Subject: [PATCH 1883/2107] =?UTF-8?q?[PBCKP-258]=20fix=20multiple=20permis?= =?UTF-8?q?sion=20tests=20(revert=20a3ac7d5e7a8d6ebeafd=E2=80=A6=20(#527)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 ++ tests/backup.py | 58 +++++++++++++++--------------------------------- tests/checkdb.py | 31 +++++++++++++------------- tests/restore.py | 14 +++++------- 4 files changed, 41 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26b2bc4e2..a7dae2ed1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ env: # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking @@ -52,6 +53,7 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master + - if: env(PG_BRANCH) = REL_15_STABLE - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) diff --git a/tests/backup.py b/tests/backup.py index 7d02f5b39..c5235120e 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1889,8 +1889,7 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -1928,9 +1927,7 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -1967,9 +1964,7 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if self.ptrack: node.safe_psql( @@ -1982,10 +1977,10 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") if ProbackupTest.enterprise: - node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup self.backup_node( @@ -2245,7 +2240,6 @@ def test_backup_with_less_privileges_role(self): if self.get_version(node) < 90600: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2256,14 +2250,11 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2275,14 +2266,11 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2294,9 +2282,7 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # enable STREAM backup node.safe_psql( @@ -3067,9 +3053,7 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3091,15 +3075,14 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") - + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + sleep(2) replica.promote() @@ -3177,8 +3160,7 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3201,9 +3183,7 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3225,15 +3205,13 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") replica.promote() diff --git a/tests/checkdb.py b/tests/checkdb.py index 2df946cf6..bcda0fb23 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -666,8 +666,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' # amcheck-next function - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function + # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -696,9 +696,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' # 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # PG 10 elif self.get_version(node) > 100000 and self.get_version(node) < 110000: node.safe_psql( @@ -726,10 +725,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') + if ProbackupTest.enterprise: # amcheck-1.1 node.safe_psql( @@ -768,9 +765,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # checkunique parameter if ProbackupTest.enterprise: if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 @@ -807,15 +803,20 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + # checkunique parameter if ProbackupTest.enterprise: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + if ProbackupTest.enterprise: + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + # checkdb try: self.checkdb_node( diff --git a/tests/restore.py b/tests/restore.py index 37f133573..b619078d5 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3230,8 +3230,7 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3269,9 +3268,7 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -3307,9 +3304,7 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") if self.ptrack: # TODO why backup works without these grants ? @@ -3326,7 +3321,8 @@ def test_missing_database_map(self): node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup without database_map backup_id = self.backup_node( From 7d3e7f864c0e373cc766df62ecef420dfef87bad Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 29 Aug 2022 05:04:05 +0300 Subject: [PATCH 1884/2107] [PBCKP-129] fix tests.replica.ReplicaTest.test_replica_stop_lsn_null_offset_next_record --- tests/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/replica.py b/tests/replica.py index acf655aac..24dbaa39e 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -806,7 +806,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content) self.assertIn( - 'LOG: stop_lsn: 0/4000000', + 'INFO: stop_lsn: 0/4000000', log_content) self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') From 95471acc75434834881a260f6e5299c1822bfcad Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Tue, 30 Aug 2022 12:42:47 +0300 Subject: [PATCH 1885/2107] [PBCKP-257] fix time_consuming.TimeConsumingTests.test_pbckp150 (#525) --- tests/time_consuming.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/time_consuming.py b/tests/time_consuming.py index 396ab716e..c778b9bc3 100644 --- a/tests/time_consuming.py +++ b/tests/time_consuming.py @@ -15,22 +15,28 @@ def test_pbckp150(self): run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel """ # init node + if self.pg_config_version < self.version_to_num('11.0'): + return unittest.skip('You need PostgreSQL >= 11 for this test') + if not self.ptrack: + return unittest.skip('Skipped because ptrack support is disabled') + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), set_replication=True, - initdb_params=['--data-checksums']) - node.append_conf('postgresql.conf', - """ - max_connections = 100 - wal_keep_size = 16000 - ptrack.map_size = 1 - shared_preload_libraries='ptrack' - log_statement = 'none' - fsync = off - log_checkpoints = on - autovacuum = off - """) + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={ + 'max_connections': 100, + 'log_statement': 'none', + 'log_checkpoints': 'on', + 'autovacuum': 'off', + 'ptrack.map_size': 1}) + + if node.major_version >= 13: + self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) + else: + self.set_auto_conf(node, {'wal_keep_segments': '1000'}) # init probackup and add an instance backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') From 1bb07627295ba44ed059d1d700db01d3a9799915 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 1 Sep 2022 14:38:17 +0300 Subject: [PATCH 1886/2107] =?UTF-8?q?[PBCKP-259]=20fix=20for=20'ERROR:=20C?= =?UTF-8?q?annot=20create=20directory=20for=20older=20backup'=E2=80=A6=20(?= =?UTF-8?q?#526)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [PBCKP-259] fix for 'ERROR: Cannot create directory for older backup', rewrite --start_time implementation * rewritten 5f2283c8deac88ea49ea6223a3aa72e2cf462eb5 * fixes for several tests * disabled tests.merge.MergeTest.test_merge_backup_from_future and tests.restore.RestoreTest.test_restore_backup_from_future as incorrect for now Co-authored-by: d.lepikhova --- src/backup.c | 60 +++++- src/catalog.c | 98 ++++----- src/pg_probackup.c | 8 +- src/pg_probackup.h | 7 +- tests/backup.py | 364 ++++++++------------------------ tests/helpers/ptrack_helpers.py | 25 ++- tests/merge.py | 6 +- tests/restore.py | 6 +- 8 files changed, 217 insertions(+), 357 deletions(-) diff --git a/src/backup.c b/src/backup.c index 9a451e72a..03ff7b72b 100644 --- a/src/backup.c +++ b/src/backup.c @@ -692,6 +692,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) /* * Entry point of pg_probackup BACKUP subcommand. + * + * if start_time == INVALID_BACKUP_ID then we can generate backup_id */ int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, @@ -699,8 +701,13 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; + time_t latest_backup_id = INVALID_BACKUP_ID; char pretty_bytes[20]; + if (!instance_config.pgdata) + elog(ERROR, "required parameter not specified: PGDATA " + "(-D, --pgdata)"); + /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); @@ -709,12 +716,55 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) current.external_dir_str = instance_config.external_dir_str; - /* Create backup directory and BACKUP_CONTROL_FILE */ - pgBackupCreateDir(¤t, instanceState, start_time); + /* Find latest backup_id */ + { + parray *backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - if (!instance_config.pgdata) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + if (parray_num(backup_list) > 0) + latest_backup_id = ((pgBackup *)parray_get(backup_list, 0))->backup_id; + + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } + + /* Try to pick backup_id and create backup directory with BACKUP_CONTROL_FILE */ + if (start_time != INVALID_BACKUP_ID) + { + /* If user already choosed backup_id for us, then try to use it. */ + if (start_time <= latest_backup_id) + /* don't care about freeing base36enc_dup memory, we exit anyway */ + elog(ERROR, "Can't assign backup_id from requested start_time (%s), " + "this time must be later that backup %s", + base36enc_dup(start_time), base36enc_dup(latest_backup_id)); + + current.backup_id = start_time; + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + } + else + { + /* We can generate our own unique backup_id + * Sometimes (when we try to backup twice in one second) + * backup_id will be duplicated -> try more times. + */ + int attempts = 10; + + if (time(NULL) < latest_backup_id) + elog(ERROR, "Can't assign backup_id, there is already a backup in future (%s)", + base36enc(latest_backup_id)); + + do + { + current.backup_id = time(NULL); + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + if (current.backup_id == INVALID_BACKUP_ID) + sleep(1); + } + while (current.backup_id == INVALID_BACKUP_ID && attempts-- > 0); + } + + /* If creation of backup dir was unsuccessful, there will be WARNINGS in logs already */ + if (current.backup_id == INVALID_BACKUP_ID) + elog(ERROR, "Can't create backup directory"); /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; diff --git a/src/catalog.c b/src/catalog.c index c118e954a..47513096c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -23,7 +23,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); -static void create_backup_dir(pgBackup *backup, const char *backup_instance_path); +static int create_backup_dir(pgBackup *backup, const char *backup_instance_path); static bool backup_lock_exit_hook_registered = false; static parray *locks = NULL; @@ -969,6 +969,7 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id } else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) { + /* TODO there is no such guarantees */ elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", base36enc(backup->start_time), backup_conf_path); } @@ -1411,22 +1412,34 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; } -/* Create backup directory in $BACKUP_PATH - * Note, that backup_id attribute is updated, - * so it is possible to get diffrent values in +/* + * Create backup directory in $BACKUP_PATH + * (with proposed backup->backup_id) + * and initialize this directory. + * If creation of directory fails, then + * backup_id will be cleared (set to INVALID_BACKUP_ID). + * It is possible to get diffrent values in * pgBackup.start_time and pgBackup.backup_id. * It may be ok or maybe not, so it's up to the caller * to fix it or let it be. */ void -pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time) +pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) { - int i; - parray *subdirs = parray_new(); - parray * backups; - pgBackup *target_backup; + int i; + char temp[MAXPGPATH]; + parray *subdirs; + /* Try to create backup directory at first */ + if (create_backup_dir(backup, backup_instance_path) != 0) + { + /* Clear backup_id as indication of error */ + backup->backup_id = INVALID_BACKUP_ID; + return; + } + + subdirs = parray_new(); parray_append(subdirs, pg_strdup(DATABASE_DIR)); /* Add external dirs containers */ @@ -1438,7 +1451,6 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t false); for (i = 0; i < parray_num(external_list); i++) { - char temp[MAXPGPATH]; /* Numeration of externaldirs starts with 1 */ makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); parray_append(subdirs, pg_strdup(temp)); @@ -1446,30 +1458,6 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t free_dir_list(external_list); } - /* Get list of all backups*/ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - if (parray_num(backups) > 0) - { - target_backup = (pgBackup *) parray_get(backups, 0); - if (start_time > target_backup->backup_id) - { - backup->backup_id = start_time; - create_backup_dir(backup, instanceState->instance_backup_subdir_path); - } - else - { - elog(ERROR, "Cannot create directory for older backup"); - } - } - else - { - backup->backup_id = start_time; - create_backup_dir(backup, instanceState->instance_backup_subdir_path); - } - - if (backup->backup_id == 0) - elog(ERROR, "Cannot create backup directory: %s", strerror(errno)); - backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); @@ -1479,10 +1467,8 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { - char path[MAXPGPATH]; - - join_path_components(path, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); + join_path_components(temp, backup->root_dir, parray_get(subdirs, i)); + fio_mkdir(temp, DIR_PERMISSION, FIO_BACKUP_HOST); } free_dir_list(subdirs); @@ -1491,34 +1477,26 @@ pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_t /* * Create root directory for backup, * update pgBackup.root_dir if directory creation was a success + * Return values (same as dir_create_dir()): + * 0 - ok + * -1 - error (warning message already emitted) */ -void +int create_backup_dir(pgBackup *backup, const char *backup_instance_path) { - int attempts = 10; + int rc; + char path[MAXPGPATH]; - while (attempts--) - { - int rc; - char path[MAXPGPATH]; - - join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); + join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); - /* TODO: add wrapper for remote mode */ - rc = dir_create_dir(path, DIR_PERMISSION, true); - - if (rc == 0) - { - backup->root_dir = pgut_strdup(path); - return; - } - else - { - elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); - sleep(1); - } - } + /* TODO: add wrapper for remote mode */ + rc = dir_create_dir(path, DIR_PERMISSION, true); + if (rc == 0) + backup->root_dir = pgut_strdup(path); + else + elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); + return rc; } /* diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 5867bd490..1f6b6313e 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -78,7 +78,7 @@ pid_t my_pid = 0; __thread int my_thread_num = 1; bool progress = false; bool no_sync = false; -time_t start_time = 0; +time_t start_time = INVALID_BACKUP_ID; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; bool temp_slot = false; @@ -202,7 +202,6 @@ static ConfigOption cmd_options[] = { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, - { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, @@ -217,6 +216,7 @@ static ConfigOption cmd_options[] = { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, + { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, /* catchup options */ { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, @@ -975,9 +975,7 @@ main(int argc, char *argv[]) case BACKUP_CMD: { current.stream = stream_wal; - if (start_time == 0) - start_time = current_time; - else + if (start_time != INVALID_BACKUP_ID) elog(WARNING, "Please do not use the --start-time option to start backup. " "This is a service option required to work with other extensions. " "We do not guarantee future support for this flag."); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 802cbb5c0..1885a191e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -450,7 +450,10 @@ struct pgBackup { BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ time_t backup_id; /* Identifier of the backup. - * Currently it's the same as start_time */ + * By default it's the same as start_time + * but can be increased if same backup_id + * already exists. It can be also set by + * start_time parameter */ BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ TimeLineID tli; /* timeline of start and stop backup lsns */ XLogRecPtr start_lsn; /* backup's starting transaction log location */ @@ -985,7 +988,7 @@ extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupCreateDir(pgBackup *backup, InstanceState *instanceState, time_t start_time); +extern void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); diff --git a/tests/backup.py b/tests/backup.py index c5235120e..0cba8fe79 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1,7 +1,7 @@ import unittest import os from time import sleep, time -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil from distutils.dir_util import copy_tree from testgres import ProcessType, QueryException @@ -313,7 +313,7 @@ def test_backup_detect_corruption(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -459,7 +459,7 @@ def test_backup_detect_invalid_block_header(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -600,7 +600,7 @@ def test_backup_detect_missing_permissions(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -3402,10 +3402,11 @@ def test_pg_stop_backup_missing_permissions(self): # @unittest.skip("skip") def test_start_time(self): - + """Test, that option --start-time allows to set backup_id and restore""" fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3418,138 +3419,81 @@ def test_start_time(self): # FULL backup startTime = int(time()) self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - - # DELTA backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) - - # PAGE backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) - - if self.ptrack and node.major_version > 11: - node.safe_psql( - "postgres", - "create extension ptrack") - - # PTRACK backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_start_time_incorrect_time(self): - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore FULL backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_full'), + backup_id=base36enc(startTime)) - startTime = int(time()) - #backup with correct start time - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--start-time', str(startTime)]) - #backups with incorrect start time + #FULL backup with incorrect start time try: + startTime = str(int(time()-100000)) self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=['--stream', '--start-time', str(startTime-10000)]) + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( + 'Expecting Error because start time for new backup must be newer ' + '\n Output: {0} \n CMD: {1}'.format( repr(self.output), self.cmd)) except ProbackupException as e: self.assertRegex( e.message, - "ERROR: Cannot create directory for older backup", + r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - try: - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore DELTA backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_delta'), + backup_id=base36enc(startTime)) - try: - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PAGE backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_page'), + backup_id=base36enc(startTime)) - if self.ptrack and node.major_version > 11: + # PTRACK backup + if self.ptrack: node.safe_psql( - "postgres", - "create extension ptrack") + 'postgres', + 'create extension ptrack') - try: - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=['--stream', '--start-time', str(startTime-10000)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PTRACK backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_ptrack'), + backup_id=base36enc(startTime)) # Clean after yourself self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_start_time_few_nodes(self): - + """Test, that we can synchronize backup_id's for different DBs""" fname = self.id().split('.')[3] node1 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node1'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3561,6 +3505,7 @@ def test_start_time_few_nodes(self): node2 = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node2'), + set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3571,200 +3516,61 @@ def test_start_time_few_nodes(self): node2.slow_start() # FULL backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - + backup_dir2, 'node2', node2, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[0] show_backup2 = self.show_pb(backup_dir2, 'node2')[0] self.assertEqual(show_backup1['id'], show_backup2['id']) # DELTA backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) + backup_dir2, 'node2', node2, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[1] show_backup2 = self.show_pb(backup_dir2, 'node2')[1] self.assertEqual(show_backup1['id'], show_backup2['id']) # PAGE backup - startTime = int(time()) + startTime = str(int(time())) self.backup_node( - backup_dir1, 'node1', node1, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) + backup_dir1, 'node1', node1, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir2, 'node2', node2, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) + backup_dir2, 'node2', node2, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) show_backup1 = self.show_pb(backup_dir1, 'node1')[2] show_backup2 = self.show_pb(backup_dir2, 'node2')[2] self.assertEqual(show_backup1['id'], show_backup2['id']) # PTRACK backup - startTime = int(time()) - if self.ptrack and node1.major_version > 11: + if self.ptrack: node1.safe_psql( - "postgres", - "create extension ptrack") - self.backup_node( - backup_dir1, 'node1', node1, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - if self.ptrack and node2.major_version > 11: + 'postgres', + 'create extension ptrack') node2.safe_psql( - "postgres", - "create extension ptrack") - self.backup_node( - backup_dir2, 'node2', node2, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[3] - show_backup2 = self.show_pb(backup_dir2, 'node2')[3] - self.assertEqual(show_backup1['id'], show_backup2['id']) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_start_time_few_nodes_incorrect_time(self): - - fname = self.id().split('.')[3] - node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1') - self.init_pb(backup_dir1) - self.add_instance(backup_dir1, 'node1', node1) - self.set_archiving(backup_dir1, 'node1', node1) - node1.slow_start() - - node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2') - self.init_pb(backup_dir2) - self.add_instance(backup_dir2, 'node2', node2) - self.set_archiving(backup_dir2, 'node2', node2) - node2.slow_start() - - # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime-10000)]) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[0] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) - - # DELTA backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="delta", - options=['--stream', '--start-time', str(startTime)]) - # make backup with start time definitelly earlier, than existing - try: - self.backup_node( - backup_dir2, 'node2', node2, backup_type="delta", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[1] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) + 'postgres', + 'create extension ptrack') - # PAGE backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="page", - options=['--stream', '--start-time', str(startTime)]) - # make backup with start time definitelly earlier, than existing - try: + startTime = str(int(time())) self.backup_node( - backup_dir2, 'node2', node2, backup_type="page", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[2] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertGreater(show_backup1['id'], show_backup2['id']) - - # PTRACK backup - startTime = int(time()) - if self.ptrack and node1.major_version > 11: - node1.safe_psql( - "postgres", - "create extension ptrack") + backup_dir1, 'node1', node1, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) self.backup_node( - backup_dir1, 'node1', node1, backup_type="ptrack", - options=['--stream', '--start-time', str(startTime)]) - - if self.ptrack and node2.major_version > 11: - node2.safe_psql( - "postgres", - "create extension ptrack") - # make backup with start time definitelly earlier, than existing - try: - self.backup_node( - backup_dir2, 'node2', node2, backup_type="ptrack", - options=['--stream', '--start-time', str(10000)]) - self.assertEqual( - 1, 0, - "Expecting Error because start time for new backup must be newer " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - "ERROR: Cannot create directory for older backup", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir1, 'node1', node1, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type="full", - options=['--stream', '--start-time', str(startTime)]) - - show_backup1 = self.show_pb(backup_dir1, 'node1')[4] - show_backup2 = self.show_pb(backup_dir2, 'node2')[1] - self.assertEqual(show_backup1['id'], show_backup2['id']) + backup_dir2, 'node2', node2, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[3] + show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + self.assertEqual(show_backup1['id'], show_backup2['id']) # Clean after yourself self.del_test_dir(module_name, fname) + diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 418ef4e17..59eb12aec 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -110,6 +110,26 @@ def is_nls_enabled(): return b'enable-nls' in p.communicate()[0] +def base36enc(number): + """Converts an integer to a base36 string.""" + alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + base36 = '' + sign = '' + + if number < 0: + sign = '-' + number = -number + + if 0 <= number < len(alphabet): + return sign + alphabet[number] + + while number != 0: + number, i = divmod(number, len(alphabet)) + base36 = alphabet[i] + base36 + + return sign + base36 + + class ProbackupException(Exception): def __init__(self, message, cmd): self.message = message @@ -947,7 +967,7 @@ def backup_node( backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, old_binary=False, return_id=True, no_remote=False, - env=None, startTime=None + env=None ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -980,9 +1000,6 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] - if startTime: - cmd_list += ['--start-time', startTime] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( diff --git a/tests/merge.py b/tests/merge.py index 5f092543c..4c374bdfb 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1965,7 +1965,11 @@ def test_failed_merge_after_delete_3(self): self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_merge_backup_from_future(self): """ take FULL backup, table PAGE backup from future, diff --git a/tests/restore.py b/tests/restore.py index b619078d5..ae1c7cbe0 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -1853,7 +1853,11 @@ def test_restore_chain_with_corrupted_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_restore_backup_from_future(self): """more complex test_restore_chain()""" fname = self.id().split('.')[3] From 24a1036037d7c7e05e4224935128309f2c9532c7 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 02:57:40 +0300 Subject: [PATCH 1887/2107] [PBCKP-258] fix tests.ptrack.PtrackTest.test_ptrack_unprivileged --- tests/ptrack.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index d46ece119..783d3b3e7 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -545,8 +545,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -583,9 +582,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # >= 10 else: node.safe_psql( @@ -620,9 +617,7 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup; ' - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") node.safe_psql( "backupdb", @@ -641,7 +636,8 @@ def test_ptrack_unprivileged(self): if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') self.backup_node( backup_dir, 'node', node, From b8c2076437514d31153488d774fcf2aa2ef8b6ab Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 03:04:18 +0300 Subject: [PATCH 1888/2107] [PBCKP-261] fix configure flags for tests.pgpro2068.BugTest.test_minrecpoint_on_replica --- travis/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 37614f970..1823b05de 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -47,7 +47,7 @@ cd postgres # Go to postgres dir if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi -CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls +CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls --with-python make -s -j$(nproc) install #make -s -j$(nproc) -C 'src/common' install #make -s -j$(nproc) -C 'src/port' install From a4a2abde5295ae359413dc3353b8e25e743006f9 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 5 Sep 2022 17:12:58 +0300 Subject: [PATCH 1889/2107] [PBCKP-178] fix rare 'buffer error' in tests.validate.ValidateTest.test_validate_corrupt_page_header_map --- tests/validate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index 22a03c3be..966ad81a8 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -4017,9 +4017,9 @@ def test_validate_corrupt_page_header_map(self): "Output: {0} \n CMD: {1}".format( self.output, self.cmd)) except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, + self.assertRegex( + e.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) From 25fc034509d22af508559b38fbab847ccf8db577 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 5 Sep 2022 19:38:14 +0300 Subject: [PATCH 1890/2107] [PBCKP-236] stable test failure, dirty version --- tests/compatibility.py | 72 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/compatibility.py b/tests/compatibility.py index e274c22be..262b940ef 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -10,6 +10,78 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_catchup_with_different_remote_major_pg(self): + "Decription in jira issue PBCKP-236" #TODO REVIEW XXX explain the test + self.verbose = True + self.remote = True + pg_config = os.environ['PG_CONFIG'] + pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.9.6/bin/' + pg_config_ee_9_6 = pg_path_ee_9_6 + 'pg_config' + probackup_path_ee_9_6 = pg_path_ee_9_6 + 'pg_probackup' + pg_path_ee_11 = '/home/avaness/postgres/postgres.build.11/bin/' + pg_config_ee_11 = pg_path_ee_11 + 'pg_config' + probackup_path_ee_11 = pg_path_ee_11 + 'pg_probackup' + + os.environ['PG_CONFIG'] = pg_config_ee_11 + self.probackup_path = probackup_path_ee_11 + # os.environ['PG_CONFIG'] = pg_config_ee_9_6 + # self.probackup_path = probackup_path_ee_9_6 + + # backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + src_pg = self.make_simple_node( + base_dir=os.path.join(module_name, self.fname, 'src'), + set_replication=True, + # initdb_params=['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # do full catchup + os.environ['PG_CONFIG'] = pg_config_ee_11 + self.probackup_path = probackup_path_ee_11 + + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + # dst_pg = self.make_simple_node( + # base_dir=os.path.join(module_name, self.fname, 'dst'), + # set_replication=True, + # # initdb_params=['--data-checksums'] + # ) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream']#, '--remote-path=' + pg_path_ee_9_6] + ) + + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") + + # do delta catchup + #TODO REVIEW XXX try to apply only one catchup (FULL) for test failure + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] + ) + + # Clean after yourself + self.del_test_dir(module_name, self.fname) + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): From 9e9509d8aab21565d95b44daeafcca6b7516597c Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" <16117281+kulaginm@users.noreply.github.com> Date: Thu, 8 Sep 2022 09:15:24 +0300 Subject: [PATCH 1891/2107] [PBCKP-263] fix for tests.archive.ArchiveTest.test_archive_get_batching_sanity (#532) --- src/archive.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/archive.c b/src/archive.c index 48114d955..1a19c3d84 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1263,6 +1263,7 @@ uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, arg->thread_num = i+1; arg->files = batch_files; + arg->n_fetched = 0; } /* Run threads */ From d4d78e18f34ee6d7bf04253a5da597165c8134b6 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 9 Sep 2022 22:09:41 +0300 Subject: [PATCH 1892/2107] [PBCKP-277] stabilize catchup.CatchupTest.test_config_exclusion --- .travis.yml | 24 +++++++++++++++--------- tests/catchup.py | 4 ++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7dae2ed1..52d6dba17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,17 +26,23 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE +# - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master +# - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE +# - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE +# - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE +# - PG_VERSION=10 PG_BRANCH=REL_10_STABLE +# - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE +# - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=10 PG_BRANCH=REL_10_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE MODE=catchup.CatchupTest.test_config_exclusion # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta diff --git a/tests/catchup.py b/tests/catchup.py index a83755c54..7ecd84697 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -1362,6 +1362,7 @@ def test_config_exclusion(self): dst_options = {} dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) + dst_pg._assign_master(src_pg) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1390,6 +1391,7 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') @@ -1419,6 +1421,7 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') @@ -1447,6 +1450,7 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)") src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_pg.catchup() # wait for replication dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') From 42241bd3ba8f2ee2d8d473637a3a3f143ba8eb3e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Fri, 9 Sep 2022 22:21:28 +0300 Subject: [PATCH 1893/2107] [PBCKP-277] fix .travis.yml typo --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52d6dba17..7cf50d0ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,10 @@ env: # - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE MODE=catchup.CatchupTest.test_config_exclusion + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=10 PG_BRANCH=REL_10_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE MODE=catchup.CatchupTest.test_config_exclusion - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE MODE=catchup.CatchupTest.test_config_exclusion From f78c63c8f56b8a7064f8799460156001dd1c6a76 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:14:18 +0300 Subject: [PATCH 1894/2107] [PBCKP-236] first-stage compatibility protocol impl with stubs --- src/pg_probackup.h | 8 ++++++-- src/utils/file.c | 20 +++++++++++++++++--- src/utils/file.h | 2 +- src/utils/remote.c | 34 +++++++++++++++++++++++++++++----- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1885a191e..e68afc571 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -341,8 +341,8 @@ typedef enum ShowFormat #define PROGRAM_VERSION "2.5.8" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20501 -#define AGENT_PROTOCOL_VERSION_STR "2.5.1" +#define AGENT_PROTOCOL_VERSION 20509 +#define AGENT_PROTOCOL_VERSION_STR "2.5.9" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" @@ -881,6 +881,10 @@ extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, IncrRestoreMode incremental_mode); +/* in remote.c */ +extern void check_remote_agent_compatibility(int agent_version, char *compatibility_str); +extern size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size); + /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); diff --git a/src/utils/file.c b/src/utils/file.c index 7103c8f1d..e3d1e5801 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -268,9 +268,10 @@ fio_write_all(int fd, void const* buf, size_t size) return offs; } +//TODO REVIEW XXX move to remote.c???? /* Get version of remote agent */ -int -fio_get_agent_version(void) +void +fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) { fio_header hdr; hdr.cop = FIO_AGENT_VERSION; @@ -278,8 +279,13 @@ fio_get_agent_version(void) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > payload_buf_size) + { + elog(ERROR, "Bad protocol, insufficient payload_buf_size=%u", payload_buf_size); + } - return hdr.arg; + *protocol = hdr.arg; + IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); } /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ @@ -3210,6 +3216,7 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ +//TODO REVIEW XXX move to remote.c? void fio_communicate(int in, int out) { @@ -3316,6 +3323,13 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? + { + size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); + IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); + //TODO REVIEW XXX make INFO to LOG or VERBOSE + elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); + } break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); diff --git a/src/utils/file.h b/src/utils/file.h index a554b4ab0..92c5f2eaa 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -91,7 +91,7 @@ extern fio_location MyLocation; extern void fio_redirect(int in, int out, int err); extern void fio_communicate(int in, int out); -extern int fio_get_agent_version(void); +extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); diff --git a/src/utils/remote.c b/src/utils/remote.c index 046ebd818..c7a1f9330 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -117,6 +117,9 @@ bool launch_agent(void) int infd[2]; int errfd[2]; int agent_version; + //TODO REVIEW XXX review buf_size + size_t payload_buf_size = 1024 * 8; + char payload_buf[payload_buf_size]; ssh_argc = 0; #ifdef WIN32 @@ -238,10 +241,31 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } - /* Make sure that remote agent has the same version - * TODO: we must also check PG version and fork edition + /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ - agent_version = fio_get_agent_version(); + fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); + check_remote_agent_compatibility(0, payload_buf); + + return true; +} + +//TODO REVIEW XXX review macro +#define STR(macro) #macro +size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) +{ + size_t payload_size = snprintf(compatibility_buf, buf_size, + "%s\n%s\n%s\n%s\n", + STR(PG_MAJORVERSION), PG_MAJORVERSION, + STR(PGPRO_EDN), PGPRO_EDN); + if (payload_size >= buf_size) + { + elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); + } + return payload_size + 1; +} + +void check_remote_agent_compatibility(int agent_version, char *compatibility_str) +{ if (agent_version != AGENT_PROTOCOL_VERSION) { char agent_version_str[1024]; @@ -254,6 +278,6 @@ bool launch_agent(void) "consider to upgrade pg_probackup binary", agent_version_str, AGENT_PROTOCOL_VERSION_STR); } - - return true; + assert(false); + elog(ERROR, " check_remote_agent_compatibility() not implemented"); } From 1dfa5b99c2c20a1a97d01b7f141d450a938a9a4f Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:37:46 +0300 Subject: [PATCH 1895/2107] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/file.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/file.c b/src/utils/file.c index e3d1e5801..6c7bdbbff 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -3323,6 +3323,7 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + assert(false); //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); From c3d3c026c2f8446b93e5e73150a3c55a153f4317 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:44:18 +0300 Subject: [PATCH 1896/2107] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/remote.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index c7a1f9330..6a6c3c12b 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -255,8 +255,8 @@ size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t bu { size_t payload_size = snprintf(compatibility_buf, buf_size, "%s\n%s\n%s\n%s\n", - STR(PG_MAJORVERSION), PG_MAJORVERSION, - STR(PGPRO_EDN), PGPRO_EDN); + STR(PG_MAJORVERSION), PG_MAJORVERSION); +// STR(PGPRO_EDN), PGPRO_EDN); if (payload_size >= buf_size) { elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); From 46b7079edd63b43c41c104830d7feef9d4536476 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 11 Sep 2022 04:46:15 +0300 Subject: [PATCH 1897/2107] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/remote.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 6a6c3c12b..e4963b62a 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -254,7 +254,8 @@ bool launch_agent(void) size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) { size_t payload_size = snprintf(compatibility_buf, buf_size, - "%s\n%s\n%s\n%s\n", +// "%s\n%s\n%s\n%s\n", + "%s\n%s\n", STR(PG_MAJORVERSION), PG_MAJORVERSION); // STR(PGPRO_EDN), PGPRO_EDN); if (payload_size >= buf_size) From f5fde7ef8e1ee479932df78086f7bed817c53902 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 12 Sep 2022 02:46:29 +0300 Subject: [PATCH 1898/2107] [PBCKP-236] draft, first-stage compatibility protocol impl with stubs --- src/utils/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 6c7bdbbff..65d0699c7 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,7 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - elog(ERROR, "Bad protocol, insufficient payload_buf_size=%u", payload_buf_size); + elog(ERROR, "Bad protocol, insufficient payload_buf_size=%zu", payload_buf_size); } *protocol = hdr.arg; @@ -3323,7 +3323,6 @@ fio_communicate(int in, int out) case FIO_AGENT_VERSION: hdr.arg = AGENT_PROTOCOL_VERSION; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - assert(false); //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); @@ -3331,6 +3330,7 @@ fio_communicate(int in, int out) //TODO REVIEW XXX make INFO to LOG or VERBOSE elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); } + assert(false); break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); From 6d3ad888cd00dbbdac734c316970ecec6f0e123c Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Tue, 21 Jun 2022 12:54:30 +0300 Subject: [PATCH 1899/2107] [PBCKP-125] changes in function call CreateWalDirectoryMethod for 15 version Tags: pg_probackup --- src/stream.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index 1ee8dee37..3b947af01 100644 --- a/src/stream.c +++ b/src/stream.c @@ -274,7 +274,13 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, + COMPRESSION_NONE, + 0, + false); +#elif PG_VERSION_NUM >= 100000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, // (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, From 53abc0b6e735944354b2c40483b9ac82bcc4c499 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Fri, 1 Jul 2022 15:46:53 +0300 Subject: [PATCH 1900/2107] [PGPRO-6938] pg_probackup has been ported to version 15 Has been tested on 15beta2 and 16 tags: pg_probackup --- src/backup.c | 34 +++--- src/parsexlog.c | 14 ++- src/stream.c | 2 +- src/utils/configuration.c | 1 + tests/archive.py | 13 ++- tests/auth_test.py | 105 +++++++++++++----- tests/backup.py | 186 +++++++++++++++++++++++++++----- tests/false_positive.py | 3 + tests/helpers/ptrack_helpers.py | 20 +++- tests/ptrack.py | 46 +++++++- tests/replica.py | 1 + tests/restore.py | 47 +++++++- tests/retention.py | 2 + 13 files changed, 379 insertions(+), 95 deletions(-) diff --git a/src/backup.c b/src/backup.c index 03ff7b72b..0fa8ee9fd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1056,20 +1056,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, uint32 lsn_lo; params[0] = label; - elog(INFO, "wait for pg_start_backup()"); + elog(INFO, "wait for pg_backup_start()"); /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - if (!exclusive_backup) - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2, false)", - 2, - params); - else - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2)", - 2, - params); + res = pgut_execute(conn, + "SELECT pg_catalog.pg_backup_start($1, $2)", + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it @@ -1618,7 +1612,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn", + " pg_catalog.pg_backup_stop() as lsn", stop_backup_on_master_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1626,7 +1620,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", + " FROM pg_catalog.pg_backup_stop(false)", stop_backup_on_master_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1634,7 +1628,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", + " FROM pg_catalog.pg_backup_stop()", /* * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. @@ -1646,7 +1640,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_wal_replay_lsn()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", + " FROM pg_catalog.pg_backup_stop(false)", stop_backup_on_replica_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1654,7 +1648,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_xlog_replay_location()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)"; + " FROM pg_catalog.pg_backup_stop()"; const char * const stop_backup_query = is_exclusive ? @@ -1682,7 +1676,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica */ sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); if (!sent) - elog(ERROR, "Failed to send pg_stop_backup query"); + elog(ERROR, "Failed to send pg_backup_stop query"); /* After we have sent pg_stop_backup, we don't need this callback anymore */ pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); @@ -1728,7 +1722,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (interrupted) { pgut_cancel(conn); - elog(ERROR, "interrupted during waiting for pg_stop_backup"); + elog(ERROR, "interrupted during waiting for pg_backup_stop"); } if (pg_stop_backup_timeout == 1) @@ -1741,7 +1735,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (pg_stop_backup_timeout > timeout) { pgut_cancel(conn); - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); + elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); } } else @@ -1753,7 +1747,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* Check successfull execution of pg_stop_backup() */ if (!query_result) - elog(ERROR, "pg_stop_backup() failed"); + elog(ERROR, "pg_backup_stop() failed"); else { switch (PQresultStatus(query_result)) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f1ca9c75..5cf760312 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -29,7 +29,10 @@ * RmgrNames is an array of resource manager names, to make error messages * a bit nicer. */ -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ + name, +#elif PG_VERSION_NUM >= 100000 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ name, #else @@ -1769,7 +1772,8 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, /* Is this a special record type that I recognize? */ - if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + if (rmid == RM_DBASE_ID + && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) { /* * New databases can be safely ignored. They would be completely @@ -1823,13 +1827,13 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, RmgrNames[rmid], info); } - for (block_id = 0; block_id <= record->max_block_id; block_id++) + for (block_id = 0; block_id <= record->record->max_block_id; block_id++) { RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; - if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) + if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) continue; /* We only care about the main fork; others are copied as is */ @@ -1946,4 +1950,4 @@ static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *r #else return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); #endif -} \ No newline at end of file +} diff --git a/src/stream.c b/src/stream.c index 3b947af01..7735f35fa 100644 --- a/src/stream.c +++ b/src/stream.c @@ -277,7 +277,7 @@ StreamLog(void *arg) #if PG_VERSION_NUM >= 150000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, - COMPRESSION_NONE, + PG_COMPRESSION_NONE, 0, false); #elif PG_VERSION_NUM >= 100000 diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 7ab242aa3..98c3b2994 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -22,6 +22,7 @@ #include #endif #include +#include #define MAXPG_LSNCOMPONENT 8 diff --git a/tests/archive.py b/tests/archive.py index 52fb225e8..81d013f6b 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -250,6 +250,7 @@ def test_pgpro434_3(self): "--log-level-file=LOG"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -314,6 +315,7 @@ def test_pgpro434_4(self): "--log-level-file=info"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -341,9 +343,14 @@ def test_pgpro434_4(self): with open(log_file, 'r') as f: log_content = f.read() - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: diff --git a/tests/auth_test.py b/tests/auth_test.py index 78af21be9..39786d7a9 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -51,16 +51,29 @@ def test_backup_via_unprivileged_user(self): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_start_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_backup_start", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_start_backup(text, boolean, boolean) TO backup;") + if self.get_version(node) < 150000: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_backup_start(text, boolean) TO backup;") if self.get_version(node) < 100000: node.safe_psql( @@ -97,17 +110,24 @@ def test_backup_via_unprivileged_user(self): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_stop_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_backup_stop", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) if self.get_version(node) < self.version_to_num('10.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - else: + elif self.get_vestion(node) < self.version_to_num('15.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION " @@ -116,6 +136,16 @@ def test_backup_via_unprivileged_user(self): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION " + "pg_backup_stop(boolean) TO backup") + # Do this for ptrack backups + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup") + self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -177,20 +207,37 @@ def setUpClass(cls): except StartNodeException: raise unittest.skip("Node hasn't started") - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + if cls.pb.get_version(cls.node) < 150000: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + else: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') @classmethod diff --git a/tests/backup.py b/tests/backup.py index 0cba8fe79..4f447c9bd 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1927,9 +1927,10 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -1964,7 +1965,46 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: node.safe_psql( @@ -2266,9 +2306,11 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -2282,7 +2324,28 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) # enable STREAM backup node.safe_psql( @@ -3054,8 +3117,8 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3075,7 +3138,31 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if ProbackupTest.enterprise: node.safe_psql( @@ -3183,9 +3270,10 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3205,7 +3293,31 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # > 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if ProbackupTest.enterprise: node.safe_psql( @@ -3331,7 +3443,7 @@ def test_backup_atexit(self): log_content) self.assertIn( - 'FROM pg_catalog.pg_stop_backup', + 'FROM pg_catalog.pg_backup_stop', log_content) self.assertIn( @@ -3369,10 +3481,15 @@ def test_pg_stop_backup_missing_permissions(self): node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') - else: + elif self.get_version(node) < 150000: node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') + else: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') + # Full backup in streaming mode try: @@ -3380,17 +3497,32 @@ def test_pg_stop_backup_missing_permissions(self): backup_dir, 'node', node, options=['--stream', '-U', 'backup']) # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_stop_backup " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + if self.get_version(node) < 150000: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_stop_backup " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_backup_stop " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - "ERROR: permission denied for function pg_stop_backup", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: permission denied for function pg_stop_backup", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: permission denied for function pg_backup_stop", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + self.assertIn( "query was: SELECT pg_catalog.txid_snapshot_xmax", e.message, diff --git a/tests/false_positive.py b/tests/false_positive.py index a101f8107..2ededdf12 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -198,6 +198,7 @@ def test_recovery_target_time_backup_victim(self): gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -257,6 +258,7 @@ def test_recovery_target_lsn_backup_victim(self): backup_dir, 'node', node, options=['--log-level-console=LOG'], gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -308,6 +310,7 @@ def test_streaming_timeout(self): backup_dir, 'node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 59eb12aec..d800f0d3e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -476,8 +476,8 @@ def simple_bootstrap(self, node, role) -> None: 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'postgres', 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' @@ -492,6 +492,22 @@ def simple_bootstrap(self, node, role) -> None: 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) + # >= 15 + else: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( diff --git a/tests/ptrack.py b/tests/ptrack.py index 783d3b3e7..a01405d6a 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -582,9 +582,10 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -617,7 +618,44 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) node.safe_psql( "backupdb", diff --git a/tests/replica.py b/tests/replica.py index 24dbaa39e..ea69e2d01 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -775,6 +775,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): '--stream'], gdb=True) + # Attention! this breakpoint is set to a probackup internal function, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() diff --git a/tests/restore.py b/tests/restore.py index ae1c7cbe0..49538bd1f 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -3272,9 +3272,10 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 - else: + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -3308,7 +3309,45 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: # TODO why backup works without these grants ? diff --git a/tests/retention.py b/tests/retention.py index b0399a239..122ab28ad 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1519,6 +1519,7 @@ def test_window_error_backups_1(self): gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -1568,6 +1569,7 @@ def test_window_error_backups_2(self): gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb._execute('signal SIGKILL') From 0a1a075b12a466b17be6f1f9fcc8895e5a7e247f Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 23 Aug 2022 20:40:42 +0300 Subject: [PATCH 1901/2107] [PGPRO-6938] macro conditions fixed tags: pg_probackup --- .travis.yml | 1 + README.md | 76 ++++++++++++++++++++++----------------------- doc/pgprobackup.xml | 23 +++++++++++++- src/backup.c | 74 ++++++++++++++++++++++++++++++++++--------- src/parsexlog.c | 12 +++++++ src/pg_probackup.h | 6 ++++ 6 files changed, 139 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7dae2ed1..f113d05c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE diff --git a/README.md b/README.md index 5da8d199e..433978473 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14; +* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 11, 12, 13, 14 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13 -* Postgres Pro Enterprise 11, 12, 13 +* vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 11, 12, 13, 14 +* Postgres Pro Enterprise 11, 12, 13, 14 ## Limitations @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get source pg-probackup-{15,14,13,12,11,10,9.6} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} +yumdownloader --source pg_probackup-{15,14,13,12,11,10,9.6} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} -zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10,9.6} +zypper install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{14,13,12,11,10,9.6} +zypper si pg_probackup-{15,14,13,12,11,10,9.6} #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index fc2a341e8..6babf00f7 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -653,7 +653,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; - For PostgreSQL 10 or higher: + For PostgreSQL 10: BEGIN; @@ -672,6 +672,27 @@ GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; + + + For PostgreSQL 15 or higher: + + +BEGIN; +CREATE ROLE backup WITH LOGIN; +GRANT USAGE ON SCHEMA pg_catalog TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; +COMMIT; In the diff --git a/src/backup.c b/src/backup.c index 0fa8ee9fd..31289978d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1056,14 +1056,22 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, uint32 lsn_lo; params[0] = label; +#if PG_VERSION_NUM >= 150000 elog(INFO, "wait for pg_backup_start()"); +#else + elog(INFO, "wait for pg_start_backup()"); +#endif /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; res = pgut_execute(conn, - "SELECT pg_catalog.pg_backup_start($1, $2)", - 2, - params); +#if PG_VERSION_NUM >= 150000 + "SELECT pg_catalog.pg_backup_start($1, $2)", +#else + "SELECT pg_catalog.pg_start_backup($1, $2, false)", +#endif + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it @@ -1612,7 +1620,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," " current_timestamp(0)::timestamptz," - " pg_catalog.pg_backup_stop() as lsn", + " pg_catalog.pg_stop_backup() as lsn", stop_backup_on_master_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1620,7 +1628,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)", + " FROM pg_catalog.pg_stop_backup(false, false)", stop_backup_on_master_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1628,7 +1636,15 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " lsn," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop()", + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_master_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)", /* * In case of backup from replica >= 9.6 we do not trust minRecPoint * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. @@ -1640,7 +1656,7 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_wal_replay_lsn()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)", + " FROM pg_catalog.pg_stop_backup(false, false)", stop_backup_on_replica_before10_query[] = "SELECT" " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," @@ -1648,19 +1664,33 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica " pg_catalog.pg_last_xlog_replay_location()," " labelfile," " spcmapfile" - " FROM pg_catalog.pg_backup_stop()"; + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_replica_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_wal_replay_lsn()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)"; const char * const stop_backup_query = is_exclusive ? stop_exlusive_backup_query : - server_version >= 100000 ? + server_version >= 150000 ? (is_started_on_replica ? - stop_backup_on_replica_query : - stop_backup_on_master_query + stop_backup_on_replica_after15_query : + stop_backup_on_master_after15_query ) : - (is_started_on_replica ? - stop_backup_on_replica_before10_query : - stop_backup_on_master_before10_query + (server_version >= 100000 ? + (is_started_on_replica ? + stop_backup_on_replica_query : + stop_backup_on_master_query + ) : + (is_started_on_replica ? + stop_backup_on_replica_before10_query : + stop_backup_on_master_before10_query + ) ); bool sent = false; @@ -1676,7 +1706,11 @@ pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica */ sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); if (!sent) +#if PG_VERSION_NUM >= 150000 elog(ERROR, "Failed to send pg_backup_stop query"); +#else + elog(ERROR, "Failed to send pg_stop_backup query"); +#endif /* After we have sent pg_stop_backup, we don't need this callback anymore */ pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); @@ -1722,7 +1756,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (interrupted) { pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 elog(ERROR, "interrupted during waiting for pg_backup_stop"); +#else + elog(ERROR, "interrupted during waiting for pg_stop_backup"); +#endif } if (pg_stop_backup_timeout == 1) @@ -1735,7 +1773,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, if (pg_stop_backup_timeout > timeout) { pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); +#else + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); +#endif } } else @@ -1747,7 +1789,11 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* Check successfull execution of pg_stop_backup() */ if (!query_result) +#if PG_VERSION_NUM >= 150000 elog(ERROR, "pg_backup_stop() failed"); +#else + elog(ERROR, "pg_stop_backup() failed"); +#endif else { switch (PQresultStatus(query_result)) diff --git a/src/parsexlog.c b/src/parsexlog.c index 5cf760312..f12aae904 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1773,7 +1773,11 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, /* Is this a special record type that I recognize? */ if (rmid == RM_DBASE_ID +#if PG_VERSION_NUM >= 150000 && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) +#else + && rminfo == XLOG_DBASE_CREATE) +#endif { /* * New databases can be safely ignored. They would be completely @@ -1827,13 +1831,21 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, RmgrNames[rmid], info); } +#if PG_VERSION_NUM >= 150000 for (block_id = 0; block_id <= record->record->max_block_id; block_id++) +#else + for (block_id = 0; block_id <= record->max_block_id; block_id++) +#endif { RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; +#if PG_VERSION_NUM >= 150000 if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) +#else + if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) +#endif continue; /* We only care about the main fork; others are copied as is */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1885a191e..533b05d58 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -50,6 +50,12 @@ #include #endif +#if PG_VERSION_NUM >= 150000 +// _() is explicitly undefined in libpq-int.h +// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0 +#define _(s) gettext(s) +#endif + /* Wrap the code that we're going to delete after refactoring in this define*/ #define REFACTORE_ME From 497751c0b63b8cfdf825beeec26f1d66a902be6e Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 15 Sep 2022 05:47:36 +0300 Subject: [PATCH 1902/2107] [PBCKP-236] draft, tests.CompatibilityTest.test_catchup_with_different_remote_major_pg fixes --- tests/compatibility.py | 50 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 262b940ef..0562b441c 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,29 +14,30 @@ def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): - "Decription in jira issue PBCKP-236" #TODO REVIEW XXX explain the test + """ + Decription in jira issue PBCKP-236 + This test requires builds both PGPROEE11 and PGPROEE9_6 + + prerequisites: + - git tag for PBCKP 2.5.1 + - master probackup build should be inside PGPROEE11 + - agent probackup build is inside PGPROEE9_6 + + calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem + + please correct path for agent's pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + """ + self.verbose = True self.remote = True - pg_config = os.environ['PG_CONFIG'] - pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.9.6/bin/' - pg_config_ee_9_6 = pg_path_ee_9_6 + 'pg_config' - probackup_path_ee_9_6 = pg_path_ee_9_6 + 'pg_probackup' - pg_path_ee_11 = '/home/avaness/postgres/postgres.build.11/bin/' - pg_config_ee_11 = pg_path_ee_11 + 'pg_config' - probackup_path_ee_11 = pg_path_ee_11 + 'pg_probackup' - - os.environ['PG_CONFIG'] = pg_config_ee_11 - self.probackup_path = probackup_path_ee_11 - # os.environ['PG_CONFIG'] = pg_config_ee_9_6 - # self.probackup_path = probackup_path_ee_9_6 - - # backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + # please use your own local path + pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), set_replication=True, - # initdb_params=['--data-checksums'] ) src_pg.slow_start() src_pg.safe_psql( @@ -44,20 +45,12 @@ def test_catchup_with_different_remote_major_pg(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # do full catchup - os.environ['PG_CONFIG'] = pg_config_ee_11 - self.probackup_path = probackup_path_ee_11 - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) - # dst_pg = self.make_simple_node( - # base_dir=os.path.join(module_name, self.fname, 'dst'), - # set_replication=True, - # # initdb_params=['--data-checksums'] - # ) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream']#, '--remote-path=' + pg_path_ee_9_6] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] ) dst_options = {} @@ -70,12 +63,13 @@ def test_catchup_with_different_remote_major_pg(self): "postgres", "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") - # do delta catchup - #TODO REVIEW XXX try to apply only one catchup (FULL) for test failure + # do delta catchup with remote pg_probackup agent with another postgres major version + # this DELTA backup should fail without PBCKP-236 patch. self.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, + # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] ) From 044c0376c849608cb79a0e1d53863a619c4799c5 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 15 Sep 2022 15:45:47 +0300 Subject: [PATCH 1903/2107] [PBCKP-125] changes in function call CreateWalDirectoryMethod for 15 version --- src/stream.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stream.c b/src/stream.c index 7735f35fa..f7bbeae5a 100644 --- a/src/stream.c +++ b/src/stream.c @@ -274,18 +274,20 @@ StreamLog(void *arg) ctl.synchronous = false; ctl.mark_done = false; +#if PG_VERSION_NUM >= 100000 #if PG_VERSION_NUM >= 150000 ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, PG_COMPRESSION_NONE, 0, false); -#elif PG_VERSION_NUM >= 100000 +#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */ ctl.walmethod = CreateWalDirectoryMethod( stream_arg->basedir, // (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, 0, false); +#endif /* PG_VERSION_NUM >= 150000 */ ctl.replication_slot = replication_slot; ctl.stop_socket = PGINVALID_SOCKET; ctl.do_sync = false; /* We sync all files at the end of backup */ From eefd88768a8cb2b6350d69ca47cc28a5e8beb160 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 01:06:52 +0300 Subject: [PATCH 1904/2107] [PBCKP-236] draft, solution without couple of unapplied shortenings --- src/pg_probackup.h | 5 ++- src/utils/file.c | 16 ++++--- src/utils/remote.c | 103 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 99 insertions(+), 25 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index e68afc571..13dfe1989 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -882,8 +882,9 @@ extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgda IncrRestoreMode incremental_mode); /* in remote.c */ -extern void check_remote_agent_compatibility(int agent_version, char *compatibility_str); -extern size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size); +extern void check_remote_agent_compatibility(int agent_version, + char *compatibility_str, size_t compatibility_str_max_size); +extern size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size); /* in merge.c */ extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); diff --git a/src/utils/file.c b/src/utils/file.c index 65d0699c7..b0dc39ae9 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,8 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - elog(ERROR, "Bad protocol, insufficient payload_buf_size=%zu", payload_buf_size); + //TODO REVIEW XXX %zu is C99 but not ANSI S standard, should we cast to unsigned long? + elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } *protocol = hdr.arg; @@ -3321,17 +3322,18 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_AGENT_VERSION: - hdr.arg = AGENT_PROTOCOL_VERSION; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - //TODO REVIEW XXX is it allowed by ANSI C to declare new scope inside??? { - size_t payload_size = prepare_remote_agent_compatibility_str(buf, buf_size); + size_t payload_size = prepare_compatibility_str(buf, buf_size); + + hdr.arg = AGENT_PROTOCOL_VERSION; + hdr.size = payload_size; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); //TODO REVIEW XXX make INFO to LOG or VERBOSE elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); + break; } - assert(false); - break; case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); diff --git a/src/utils/remote.c b/src/utils/remote.c index e4963b62a..af3e460c0 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -118,7 +118,7 @@ bool launch_agent(void) int errfd[2]; int agent_version; //TODO REVIEW XXX review buf_size - size_t payload_buf_size = 1024 * 8; + int payload_buf_size = 1024 * 8; char payload_buf[payload_buf_size]; ssh_argc = 0; @@ -244,29 +244,83 @@ bool launch_agent(void) /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); - check_remote_agent_compatibility(0, payload_buf); + check_remote_agent_compatibility(agent_version, payload_buf, payload_buf_size); return true; } -//TODO REVIEW XXX review macro -#define STR(macro) #macro -size_t prepare_remote_agent_compatibility_str(char* compatibility_buf, size_t buf_size) +#define COMPATIBILITY_VAL(macro) #macro, macro +#define COMPATIBILITY_STR(macro) #macro +#define COMPATIBILITY_VAL_STR(macro) #macro, COMPATIBILITY_STR(macro) + +#define COMPATIBILITY_VAL_SEPARATOR "=" +#define COMPATIBILITY_LINE_SEPARATOR "\n" + +static char* compatibility_params[] = { + COMPATIBILITY_VAL(PG_MAJORVERSION), + //TODO remove? + //TODO doesn't work macro name check for ints!!!! + COMPATIBILITY_VAL_STR(SIZEOF_VOID_P), + //TODO REVIEW XXX can use edition.h/extract_pgpro_edition() +#ifdef PGPRO_EDN + //TODO add vanilla + //TODO make "1c" -> "vanilla" + COMPATIBILITY_VAL(PGPRO_EDN), +#endif +}; + +/* + * Compose compatibility string to be sent by pg_probackup agent + * through ssh and to be verified by pg_probackup peer. + * Compatibility string contains postgres essential vars as strings + * in format "var_name" + COMPATIBILITY_VAL_SEPARATOR + "var_value" + COMPATIBILITY_LINE_SEPARATOR + */ +size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - size_t payload_size = snprintf(compatibility_buf, buf_size, -// "%s\n%s\n%s\n%s\n", - "%s\n%s\n", - STR(PG_MAJORVERSION), PG_MAJORVERSION); -// STR(PGPRO_EDN), PGPRO_EDN); - if (payload_size >= buf_size) + char tmp_buf[1024]; + int size_inc = 0; + size_t result_size = 1; + size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; + + *compatibility_buf = '\0'; + Assert(compatibility_params_array_size % 2 == 0); + + //TODO !!!! + for (int i = 0; i < compatibility_params_array_size; i+=2) { - elog(ERROR, "TODO REVIEW XXX too bad message buffer exhaust"); + size_inc = snprintf(compatibility_buf + size_inc, compatibility_buf_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i], compatibility_params[i+1]); + +// size_inc = snprintf(tmp_buf, sizeof tmp_buf, +// "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, +// compatibility_params[i], compatibility_params[i+1]); + if (size_inc >= sizeof tmp_buf) + { + //TODO make Assert + elog(ERROR, "Compatibility params from agent doesn't fit to %zu chars, %s=%s", + sizeof tmp_buf - 1, compatibility_params[i], compatibility_params[i+1] ); + } + + result_size += size_inc; + if (result_size > compatibility_buf_size) + { + //TODO make Assert + elog(ERROR, "Can't fit compatibility string size %zu to buffer size %zu:\n%s\n%s", + result_size, compatibility_buf_size, compatibility_buf, tmp_buf); + } + strcat(compatibility_buf, tmp_buf); } - return payload_size + 1; + return result_size; } -void check_remote_agent_compatibility(int agent_version, char *compatibility_str) +/* + * Check incoming remote agent's compatibility params for equality to local ones. + */ +void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) { + elog(LOG, "Agent version=%d", agent_version); + if (agent_version != AGENT_PROTOCOL_VERSION) { char agent_version_str[1024]; @@ -279,6 +333,23 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str "consider to upgrade pg_probackup binary", agent_version_str, AGENT_PROTOCOL_VERSION_STR); } - assert(false); - elog(ERROR, " check_remote_agent_compatibility() not implemented"); + + if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) + { + elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); + } + + elog(LOG, "Agent compatibility params: '%s'", compatibility_str); + + /* checking compatibility params */ + { + char *buf[compatibility_str_max_size]; + + prepare_compatibility_str(buf, sizeof buf); + if(!strcmp(compatibility_str, buf)) + { + elog(ERROR, "Incompatible agent params, expected %s", buf); + } + } + } From 0604cce21de69a3e3b0ea6a6634a6dac53ae39ea Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 01:55:25 +0300 Subject: [PATCH 1905/2107] [PBCKP-236] draft, solution without macros cleanup --- src/utils/file.c | 4 +--- src/utils/remote.c | 48 +++++++++++++--------------------------------- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index b0dc39ae9..fa0983947 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -281,7 +281,7 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - //TODO REVIEW XXX %zu is C99 but not ANSI S standard, should we cast to unsigned long? + //TODO REVIEW XXX %zu is C99 but not ANSI S compatible, should we use ints, %zu is also applied at data.c:501 data.c:1638?? elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } @@ -3330,8 +3330,6 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); - //TODO REVIEW XXX make INFO to LOG or VERBOSE - elog(INFO, "TODO REVIEW XXX sent agent compatibility\n %s", buf); break; } case FIO_STAT: /* Get information about file with specified path */ diff --git a/src/utils/remote.c b/src/utils/remote.c index af3e460c0..2babe79ae 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -277,41 +277,20 @@ static char* compatibility_params[] = { */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - char tmp_buf[1024]; - int size_inc = 0; - size_t result_size = 1; + size_t result_size = 0; size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; *compatibility_buf = '\0'; Assert(compatibility_params_array_size % 2 == 0); - //TODO !!!! for (int i = 0; i < compatibility_params_array_size; i+=2) { - size_inc = snprintf(compatibility_buf + size_inc, compatibility_buf_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i], compatibility_params[i+1]); - -// size_inc = snprintf(tmp_buf, sizeof tmp_buf, -// "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, -// compatibility_params[i], compatibility_params[i+1]); - if (size_inc >= sizeof tmp_buf) - { - //TODO make Assert - elog(ERROR, "Compatibility params from agent doesn't fit to %zu chars, %s=%s", - sizeof tmp_buf - 1, compatibility_params[i], compatibility_params[i+1] ); - } - - result_size += size_inc; - if (result_size > compatibility_buf_size) - { - //TODO make Assert - elog(ERROR, "Can't fit compatibility string size %zu to buffer size %zu:\n%s\n%s", - result_size, compatibility_buf_size, compatibility_buf, tmp_buf); - } - strcat(compatibility_buf, tmp_buf); + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i], compatibility_params[i+1]); + Assert(result_size < compatibility_buf_size); } - return result_size; + return result_size + 1; } /* @@ -319,7 +298,7 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b */ void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) { - elog(LOG, "Agent version=%d", agent_version); + elog(LOG, "Agent version=%d\n", agent_version); if (agent_version != AGENT_PROTOCOL_VERSION) { @@ -331,25 +310,24 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " "consider to upgrade pg_probackup binary", - agent_version_str, AGENT_PROTOCOL_VERSION_STR); + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } + /* checking compatibility params */ if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) { elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); } - elog(LOG, "Agent compatibility params: '%s'", compatibility_str); + elog(LOG, "Agent compatibility params:\n%s", compatibility_str); - /* checking compatibility params */ { - char *buf[compatibility_str_max_size]; + char buf[compatibility_str_max_size]; prepare_compatibility_str(buf, sizeof buf); - if(!strcmp(compatibility_str, buf)) + if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible agent params, expected %s", buf); + elog(ERROR, "Incompatible remote agent params, expected:\n%s", buf); } } - } From f61be78e783a5948750e5fc77f31eeab54a82ae2 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 04:39:18 +0300 Subject: [PATCH 1906/2107] [PBCKP-236] working solution cleaned up --- src/utils/remote.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 2babe79ae..79456d9fb 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,26 +249,13 @@ bool launch_agent(void) return true; } -#define COMPATIBILITY_VAL(macro) #macro, macro -#define COMPATIBILITY_STR(macro) #macro -#define COMPATIBILITY_VAL_STR(macro) #macro, COMPATIBILITY_STR(macro) +#define COMPATIBILITY_VAL_STR(macro) #macro, macro +#define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) +#define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) #define COMPATIBILITY_VAL_SEPARATOR "=" #define COMPATIBILITY_LINE_SEPARATOR "\n" -static char* compatibility_params[] = { - COMPATIBILITY_VAL(PG_MAJORVERSION), - //TODO remove? - //TODO doesn't work macro name check for ints!!!! - COMPATIBILITY_VAL_STR(SIZEOF_VOID_P), - //TODO REVIEW XXX can use edition.h/extract_pgpro_edition() -#ifdef PGPRO_EDN - //TODO add vanilla - //TODO make "1c" -> "vanilla" - COMPATIBILITY_VAL(PGPRO_EDN), -#endif -}; - /* * Compose compatibility string to be sent by pg_probackup agent * through ssh and to be verified by pg_probackup peer. @@ -277,6 +264,18 @@ static char* compatibility_params[] = { */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { + char compatibility_val_int_macro_helper_buf[32]; + char* compatibility_params[] = { + COMPATIBILITY_VAL_STR(PG_MAJORVERSION), +#ifdef PGPRO_EDN + //TODO REVIEW can use edition.h/extract_pgpro_edition() + COMPATIBILITY_VAL_STR(PGPRO_EDN), +#endif + //TODO REVIEW remove? no difference between 32/64 in global/pg_control. + COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, + compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), + }; + size_t result_size = 0; size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; From b2091cd2c277d28ec8bc4f1c7980b1c1798076ea Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 05:51:11 +0300 Subject: [PATCH 1907/2107] [PBCKP-236] [skip] removed unnecessary TODOs --- src/utils/file.c | 2 -- src/utils/remote.c | 16 ++++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index fa0983947..5a2aa61a8 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -268,7 +268,6 @@ fio_write_all(int fd, void const* buf, size_t size) return offs; } -//TODO REVIEW XXX move to remote.c???? /* Get version of remote agent */ void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) @@ -3217,7 +3216,6 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ -//TODO REVIEW XXX move to remote.c? void fio_communicate(int in, int out) { diff --git a/src/utils/remote.c b/src/utils/remote.c index 79456d9fb..97c8f3d4a 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -117,9 +117,6 @@ bool launch_agent(void) int infd[2]; int errfd[2]; int agent_version; - //TODO REVIEW XXX review buf_size - int payload_buf_size = 1024 * 8; - char payload_buf[payload_buf_size]; ssh_argc = 0; #ifdef WIN32 @@ -241,10 +238,13 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } - /* Make sure that remote agent has the same version, fork and other features to be binary compatible - */ - fio_get_agent_version(&agent_version, payload_buf, payload_buf_size); - check_remote_agent_compatibility(agent_version, payload_buf, payload_buf_size); + + /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ + { + char payload_buf[1024]; + fio_get_agent_version(&agent_version, payload_buf, sizeof payload_buf); + check_remote_agent_compatibility(agent_version, payload_buf, sizeof payload_buf); + } return true; } @@ -268,7 +268,7 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b char* compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), #ifdef PGPRO_EDN - //TODO REVIEW can use edition.h/extract_pgpro_edition() + //TODO REVIEW can use edition.h/extract_pgpro_edition() or similar COMPATIBILITY_VAL_STR(PGPRO_EDN), #endif //TODO REVIEW remove? no difference between 32/64 in global/pg_control. From 56598848c260c1cae5a6fb9d739f840c105668ed Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 16 Sep 2022 06:43:16 +0300 Subject: [PATCH 1908/2107] [PBCKP-236] ANSI C fix --- src/utils/remote.c | 2 +- tests/compatibility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 97c8f3d4a..a5294c705 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -321,7 +321,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(LOG, "Agent compatibility params:\n%s", compatibility_str); { - char buf[compatibility_str_max_size]; + char buf[1024]; prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) diff --git a/tests/compatibility.py b/tests/compatibility.py index 0562b441c..b0a9b0e50 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From 35df5060d5e1a20bdfa45dbaa5aab429f0fd4aa6 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 26 Sep 2022 06:00:03 +0300 Subject: [PATCH 1909/2107] [PBCKP-236] 1c+certified editions check --- src/utils/file.c | 1 - src/utils/remote.c | 58 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 5a2aa61a8..242810b5e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -280,7 +280,6 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.size > payload_buf_size) { - //TODO REVIEW XXX %zu is C99 but not ANSI S compatible, should we use ints, %zu is also applied at data.c:501 data.c:1638?? elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); } diff --git a/src/utils/remote.c b/src/utils/remote.c index a5294c705..786b4bfb1 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,6 +249,57 @@ bool launch_agent(void) return true; } +/* PGPRO 10-13 check to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ +static bool check_certified() +{ +#ifdef PGPRO_VERSION_STR + return strstr(PGPRO_VERSION_STR, "(certified)") || + strstr(PGPRO_VERSION_STR, ("(standard certified)")); +#endif + return false; +} + +//TODO REVIEW review coding standard https://jira.postgrespro.ru/browse/PBCKP-251 with @funny_falcon, newlines, braces etc +static char* extract_pg_edition_str() +{ + static char *vanilla = "vanilla"; + static char *std = "standard"; + static char *ent = "enterprise"; + static char *std_cert = "standard-certified"; + static char *ent_cert = "enterprise-certified"; + +#ifdef PGPRO_EDITION + if (strcasecmp(PGPRO_EDITION, "1C") == 0) + return vanilla; + + /* these "certified" checks are applicable to PGPRO from 9.6 up to 12 versions. + * 13+ certified versions are compatible to non-certified ones */ + if (PG_VERSION_NUM < 100000) + { + if (strcmp(PGPRO_EDITION, "standard-certified") == 0) + return std_cert; + else if (strcmp(PGPRO_EDITION, "enterprise-certified")) + return ent_cert; + else + Assert("Bad #define PGPRO_EDITION value" == 0); + } + + if (check_certified()) + { + if (strcmp(PGPRO_EDITION, "standard")) + return std_cert; + else if (strcmp(PGPRO_EDITION, "enterprise") == 0) + return ent_cert; + else + Assert("Bad #define PGPRO_EDITION value" == 0); + } + + return PGPRO_EDITION; +#else + return vanilla; +#endif +} + #define COMPATIBILITY_VAL_STR(macro) #macro, macro #define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) #define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) @@ -267,11 +318,8 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b char compatibility_val_int_macro_helper_buf[32]; char* compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), -#ifdef PGPRO_EDN - //TODO REVIEW can use edition.h/extract_pgpro_edition() or similar - COMPATIBILITY_VAL_STR(PGPRO_EDN), -#endif - //TODO REVIEW remove? no difference between 32/64 in global/pg_control. + "edition", extract_pg_edition_str(), + /* 32/64 bits compatibility */ COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), }; From 0dc826fc118ce5988b984b9ee7912a054920d941 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 26 Sep 2022 10:59:37 +0300 Subject: [PATCH 1910/2107] Fix packaging/Makefile.test typo [ci skip] --- packaging/Makefile.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/Makefile.test b/packaging/Makefile.test index f5e004f01..11c63619a 100644 --- a/packaging/Makefile.test +++ b/packaging/Makefile.test @@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2 @echo Suse: done build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13 - @echo Rhel 15.1: done + @echo Suse 15.1: done build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14 - @echo Rhel 15.1: done + @echo Suse 15.2: done define test_suse docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ From 4d907c958d2cd74b6e85eaf426ed93b18e9f6c39 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 26 Sep 2022 11:01:21 +0300 Subject: [PATCH 1911/2107] Fix test packagin script for 9.6 [ci skip] --- packaging/test/scripts/rpm.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh index 3b6806993..87d430ef8 100755 --- a/packaging/test/scripts/rpm.sh +++ b/packaging/test/scripts/rpm.sh @@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then dnf -qy module disable postgresql fi +# PGDG doesn't support install of PG-9.6 from repo package anymore +if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then + # ugly hack: use repo settings from PG10 + sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo +fi + yum install -y postgresql${PG_TOG}-server.x86_64 export PGDATA=/var/lib/pgsql/${PG_VERSION}/data From b3351b50d664b9a639537d4aea694d971c9cd7d5 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 27 Sep 2022 06:00:57 +0300 Subject: [PATCH 1912/2107] [PBCKP-236] final update --- src/utils/remote.c | 32 ++++++++++++-------------------- tests/compatibility.py | 8 ++++---- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 786b4bfb1..7d86be4c1 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -249,19 +249,18 @@ bool launch_agent(void) return true; } -/* PGPRO 10-13 check to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ +#ifdef PGPRO_EDITION +/* PGPRO 10-13 checks to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ static bool check_certified() { -#ifdef PGPRO_VERSION_STR return strstr(PGPRO_VERSION_STR, "(certified)") || strstr(PGPRO_VERSION_STR, ("(standard certified)")); -#endif - return false; } +#endif -//TODO REVIEW review coding standard https://jira.postgrespro.ru/browse/PBCKP-251 with @funny_falcon, newlines, braces etc static char* extract_pg_edition_str() { + static char *_1C = "1C"; static char *vanilla = "vanilla"; static char *std = "standard"; static char *ent = "enterprise"; @@ -269,26 +268,19 @@ static char* extract_pg_edition_str() static char *ent_cert = "enterprise-certified"; #ifdef PGPRO_EDITION - if (strcasecmp(PGPRO_EDITION, "1C") == 0) + if (strcmp(PGPRO_EDITION, _1C) == 0) return vanilla; - /* these "certified" checks are applicable to PGPRO from 9.6 up to 12 versions. - * 13+ certified versions are compatible to non-certified ones */ if (PG_VERSION_NUM < 100000) - { - if (strcmp(PGPRO_EDITION, "standard-certified") == 0) - return std_cert; - else if (strcmp(PGPRO_EDITION, "enterprise-certified")) - return ent_cert; - else - Assert("Bad #define PGPRO_EDITION value" == 0); - } + return PGPRO_EDITION; - if (check_certified()) + /* these "certified" checks are applicable to PGPRO from 10 up to 12 versions. + * 13+ certified versions are compatible to non-certified ones */ + if (PG_VERSION_NUM < 130000 && check_certified()) { - if (strcmp(PGPRO_EDITION, "standard")) + if (strcmp(PGPRO_EDITION, std) == 0) return std_cert; - else if (strcmp(PGPRO_EDITION, "enterprise") == 0) + else if (strcmp(PGPRO_EDITION, ent) == 0) return ent_cert; else Assert("Bad #define PGPRO_EDITION value" == 0); @@ -374,7 +366,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible remote agent params, expected:\n%s", buf); + elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s ", buf, compatibility_str); } } } diff --git a/tests/compatibility.py b/tests/compatibility.py index b0a9b0e50..04af1478f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 @@ -27,13 +27,13 @@ def test_catchup_with_different_remote_major_pg(self): calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem - please correct path for agent's pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + please correct path for agent's pg_path_remote_version = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' """ self.verbose = True self.remote = True # please use your own local path - pg_path_ee_9_6 = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + pg_path_remote_version = '/home/avaness/postgres/postgres.build.clean/bin' src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), @@ -70,7 +70,7 @@ def test_catchup_with_different_remote_major_pg(self): source_pgdata = src_pg.data_dir, destination_node = dst_pg, # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_ee_9_6] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_remote_version] ) # Clean after yourself From 6e671232b8791ccb8bf6090c9dcf38081fc9c2dd Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 28 Sep 2022 04:32:47 +0300 Subject: [PATCH 1913/2107] [PBCKP-236] final update after review --- src/utils/remote.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 7d86be4c1..0f254d147 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -292,12 +292,8 @@ static char* extract_pg_edition_str() #endif } -#define COMPATIBILITY_VAL_STR(macro) #macro, macro -#define COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) (snprintf(helper_buf, buf_size, "%d", macro), helper_buf) -#define COMPATIBILITY_VAL_INT(macro, helper_buf, buf_size) #macro, COMPATIBILITY_VAL_INT_HELPER(macro, helper_buf, buf_size) - -#define COMPATIBILITY_VAL_SEPARATOR "=" -#define COMPATIBILITY_LINE_SEPARATOR "\n" +#define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } +#define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } /* * Compose compatibility string to be sent by pg_probackup agent @@ -307,13 +303,10 @@ static char* extract_pg_edition_str() */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - char compatibility_val_int_macro_helper_buf[32]; - char* compatibility_params[] = { + struct { const char* name; const char* strval; int intval; } compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), - "edition", extract_pg_edition_str(), - /* 32/64 bits compatibility */ - COMPATIBILITY_VAL_INT(SIZEOF_VOID_P, - compatibility_val_int_macro_helper_buf, sizeof compatibility_val_int_macro_helper_buf), + { "edition", extract_pg_edition_str(), 0 }, + COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), }; size_t result_size = 0; @@ -324,9 +317,16 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b for (int i = 0; i < compatibility_params_array_size; i+=2) { - result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i], compatibility_params[i+1]); + if (compatibility_params[i].strval != NULL) + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s=%s/n", + compatibility_params[i].name, + compatibility_params[i].strval); + else + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s=%d/n", + compatibility_params[i].name, + compatibility_params[i].intval); Assert(result_size < compatibility_buf_size); } return result_size + 1; @@ -349,7 +349,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " "consider to upgrade pg_probackup binary", - agent_version_str, AGENT_PROTOCOL_VERSION_STR); + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } /* checking compatibility params */ From 1ce38ed70cccabcdea5dbda7f1030424ceeeef03 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 28 Sep 2022 04:52:11 +0300 Subject: [PATCH 1914/2107] [PBCKP-236] final update after review --- src/utils/remote.c | 7 ++----- tests/compatibility.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 0f254d147..f3608e566 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -310,12 +310,9 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b }; size_t result_size = 0; - size_t compatibility_params_array_size = sizeof compatibility_params / sizeof compatibility_params[0];; - *compatibility_buf = '\0'; - Assert(compatibility_params_array_size % 2 == 0); - for (int i = 0; i < compatibility_params_array_size; i+=2) + for (int i = 0; i < sizeof compatibility_params; i+=2) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, @@ -366,7 +363,7 @@ void check_remote_agent_compatibility(int agent_version, char *compatibility_str prepare_compatibility_str(buf, sizeof buf); if(strcmp(compatibility_str, buf)) { - elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s ", buf, compatibility_str); + elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s", buf, compatibility_str); } } } diff --git a/tests/compatibility.py b/tests/compatibility.py index 04af1478f..4e5e27f0e 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From c52659791b91012b13a2a8ebec1367dddc60187b Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 29 Sep 2022 03:01:57 +0300 Subject: [PATCH 1915/2107] [PBCKP-236] assert fix --- src/utils/remote.c | 17 +++++++++++++---- tests/compatibility.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index f3608e566..91468b54c 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -295,6 +295,9 @@ static char* extract_pg_edition_str() #define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } #define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } +#define COMPATIBILITY_VAL_SEPARATOR "=" +#define COMPATIBILITY_LINE_SEPARATOR "\n" + /* * Compose compatibility string to be sent by pg_probackup agent * through ssh and to be verified by pg_probackup peer. @@ -303,7 +306,13 @@ static char* extract_pg_edition_str() */ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) { - struct { const char* name; const char* strval; int intval; } compatibility_params[] = { + typedef struct compatibility_param_tag { + const char* name; + const char* strval; + int intval; + } compatibility_param; + + compatibility_param compatibility_params[] = { COMPATIBILITY_VAL_STR(PG_MAJORVERSION), { "edition", extract_pg_edition_str(), 0 }, COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), @@ -312,16 +321,16 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b size_t result_size = 0; *compatibility_buf = '\0'; - for (int i = 0; i < sizeof compatibility_params; i+=2) + for (int i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s=%s/n", + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, compatibility_params[i].name, compatibility_params[i].strval); else result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s=%d/n", + "%s" COMPATIBILITY_VAL_SEPARATOR "%d" COMPATIBILITY_LINE_SEPARATOR, compatibility_params[i].name, compatibility_params[i].intval); Assert(result_size < compatibility_buf_size); diff --git a/tests/compatibility.py b/tests/compatibility.py index 4e5e27f0e..04af1478f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -14,7 +14,7 @@ def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - # @unittest.skip("skip") + @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 From 03d55d079b836d285716a3df67a213fa1674a50a Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 29 Sep 2022 05:03:51 +0300 Subject: [PATCH 1916/2107] [PBCKP-236] fix excessive warnings for vanilla --- src/utils/remote.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 91468b54c..9feb44a9c 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -260,14 +260,14 @@ static bool check_certified() static char* extract_pg_edition_str() { - static char *_1C = "1C"; static char *vanilla = "vanilla"; +#ifdef PGPRO_EDITION + static char *_1C = "1C"; static char *std = "standard"; static char *ent = "enterprise"; static char *std_cert = "standard-certified"; static char *ent_cert = "enterprise-certified"; -#ifdef PGPRO_EDITION if (strcmp(PGPRO_EDITION, _1C) == 0) return vanilla; From 23d5ee4abfb57506fe3f1400b4e9c635a42e6bf6 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 11 Oct 2022 12:20:41 +0300 Subject: [PATCH 1917/2107] [PBCKP-235] review fixes tags: pg_probackup --- .travis.yml | 1 - README.md | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index f113d05c4..8a67e77b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,6 @@ env: jobs: allow_failures: - if: env(PG_BRANCH) = master - - if: env(PG_BRANCH) = REL_15_STABLE - if: env(PG_BRANCH) = REL9_5_STABLE # - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) diff --git a/README.md b/README.md index 433978473..d1ccd9866 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14 -* Postgres Pro Enterprise 11, 12, 13, 14 +* Postgres Pro Standard 11, 12, 13, 14, 15 +* Postgres Pro Enterprise 11, 12, 13, 14, 15 ## Limitations @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From d808a16640be611e363b66900b18b2b6f8a52747 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 11 Oct 2022 16:09:57 +0300 Subject: [PATCH 1918/2107] [PBCKP-236] removed excessive brackets --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 9feb44a9c..addd73dc8 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -254,7 +254,7 @@ bool launch_agent(void) static bool check_certified() { return strstr(PGPRO_VERSION_STR, "(certified)") || - strstr(PGPRO_VERSION_STR, ("(standard certified)")); + strstr(PGPRO_VERSION_STR, "(standard certified)"); } #endif From 96ad6e2eb9d0a9a6ef166d61d2bffa4098354cd7 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 12 Oct 2022 17:11:58 +0300 Subject: [PATCH 1919/2107] version macro increment, Release 2.5.9 --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 533b05d58..27deeee9b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -344,7 +344,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.8" +#define PROGRAM_VERSION "2.5.9" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20501 From 80efb85029a254c0a885931058d08c260c3c35d6 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 30 Sep 2022 11:06:16 +0300 Subject: [PATCH 1920/2107] [PBCKP-146] Small fix for remote_agent --- src/pg_probackup.c | 3 ++- src/pg_probackup.h | 2 +- src/utils/configuration.c | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 1f6b6313e..849685278 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -88,7 +88,7 @@ bool perm_slot = false; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -char *remote_agent; +bool remote_agent = false; static char *backup_note = NULL; /* catchup options */ static char *catchup_source_pgdata = NULL; @@ -361,6 +361,7 @@ main(int argc, char *argv[]) elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " "is launched as an agent for pg_probackup binary with version '%s'", PROGRAM_VERSION, argv[2]); + remote_agent = true; fio_communicate(STDIN_FILENO, STDOUT_FILENO); return 0; case HELP_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 533b05d58..495fbdcad 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -798,7 +798,7 @@ extern bool perm_slot; extern bool smooth_checkpoint; /* remote probackup options */ -extern char* remote_agent; +extern bool remote_agent; extern bool exclusive_backup; diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 98c3b2994..93f29c488 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -531,7 +531,6 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], opt = option_find(c, options); if (opt - && !remote_agent && opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) elog(ERROR, "Option %s cannot be specified in command line", opt->lname); From 4730857b7946e0fb136e66acb27de44ca08e4977 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 30 Sep 2022 12:22:14 +0300 Subject: [PATCH 1921/2107] [PBCKP-146] - fio_get_crc32 - add "missing_ok" parameter --- src/archive.c | 9 ++++----- src/data.c | 9 ++++++--- src/utils/file.c | 22 +++++++++++++++------- src/utils/file.h | 3 ++- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/archive.c b/src/archive.c index 1a19c3d84..2ae86bd6a 100644 --- a/src/archive.c +++ b/src/archive.c @@ -512,8 +512,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d pg_crc32 crc32_src; pg_crc32 crc32_dst; - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false); if (crc32_src == crc32_dst) { @@ -760,9 +760,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, pg_crc32 crc32_src; pg_crc32 crc32_dst; - /* TODO: what if one of them goes missing? */ - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false); if (crc32_src == crc32_dst) { diff --git a/src/data.c b/src/data.c index 5c5fdf4f0..753f247f7 100644 --- a/src/data.c +++ b/src/data.c @@ -801,8 +801,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, (prev_file && file->exists_in_prev && file->mtime <= parent_backup_time)) { - - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + /* + * file could be deleted under our feets. + * But then backup_non_data_file_internal will handle it safely + */ + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) @@ -1327,7 +1330,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (already_exists) { /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); if (file_crc == tmp_file->crc) { diff --git a/src/utils/file.c b/src/utils/file.c index 7103c8f1d..727b48c60 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1355,9 +1355,15 @@ fio_sync(char const* path, fio_location location) } } +enum { + GET_CRC32_DECOMPRESS = 1, + GET_CRC32_MISSING_OK = 2 +}; + /* Get crc32 of file */ pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, bool decompress) +fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok) { if (fio_is_remote(location)) { @@ -1370,7 +1376,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress) hdr.arg = 0; if (decompress) - hdr.arg = 1; + hdr.arg = GET_CRC32_DECOMPRESS; + if (missing_ok) + hdr.arg |= GET_CRC32_MISSING_OK; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); @@ -1381,9 +1389,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress) else { if (decompress) - return pgFileGetCRCgz(file_path, true, true); + return pgFileGetCRCgz(file_path, true, missing_ok); else - return pgFileGetCRC(file_path, true, true); + return pgFileGetCRC(file_path, true, missing_ok); } } @@ -3380,10 +3388,10 @@ fio_communicate(int in, int out) break; case FIO_GET_CRC32: /* calculate crc32 for a file */ - if (hdr.arg == 1) - crc = pgFileGetCRCgz(buf, true, true); + if ((hdr.arg & GET_CRC32_DECOMPRESS)) + crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); else - crc = pgFileGetCRC(buf, true, true); + crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: diff --git a/src/utils/file.h b/src/utils/file.h index a554b4ab0..ec478b451 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -120,7 +120,8 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); +extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); From e16c62e8fd705d65d2cc1a7fd3beb0ea40e31277 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 5 Oct 2022 12:16:10 +0300 Subject: [PATCH 1922/2107] [PBCKP-146] prettify forkname handling. --- src/catalog.c | 5 ++- src/dir.c | 87 ++++++++++++++++++++++------------------------ src/pg_probackup.h | 2 ++ 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 47513096c..03099d1a2 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1132,6 +1132,9 @@ get_backup_filelist(pgBackup *backup, bool strict) if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) file->hdr_size = (int) hdr_size; + if (file->external_dir_num == 0) + set_forkname(file); + parray_append(files, file); } @@ -2488,7 +2491,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char control_path[MAXPGPATH]; char control_path_temp[MAXPGPATH]; size_t i = 0; - #define BUFFERSZ 1024*1024 + #define BUFFERSZ (1024*1024) char *buf; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; diff --git a/src/dir.c b/src/dir.c index 561586f87..00e918d0f 100644 --- a/src/dir.c +++ b/src/dir.c @@ -758,57 +758,22 @@ dir_check_file(pgFile *file, bool backup_logs) return CHECK_FALSE; else if (isdigit(file->name[0])) { - char *fork_name; - int len; - char suffix[MAXPGPATH]; + set_forkname(file); - fork_name = strstr(file->name, "_"); - if (fork_name) - { - /* Auxiliary fork of the relfile */ - if (strcmp(fork_name, "_vm") == 0) - file->forkName = vm; - - else if (strcmp(fork_name, "_fsm") == 0) - file->forkName = fsm; - - else if (strcmp(fork_name, "_cfm") == 0) - file->forkName = cfm; - - else if (strcmp(fork_name, "_ptrack") == 0) - file->forkName = ptrack; - - else if (strcmp(fork_name, "_init") == 0) - file->forkName = init; - - // extract relOid for certain forks - if (file->forkName == vm || - file->forkName == fsm || - file->forkName == init || - file->forkName == cfm) - { - // sanity - if (sscanf(file->name, "%u_*", &(file->relOid)) != 1) - file->relOid = 0; - } + if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */ + return CHECK_FALSE; + else if (file->forkName != none) + return CHECK_TRUE; - /* Do not backup ptrack files */ - if (file->forkName == ptrack) - return CHECK_FALSE; - } - else + /* Set is_datafile flag */ { + char suffix[MAXFNAMELEN]; - len = strlen(file->name); - /* reloid.cfm */ - if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) - return CHECK_TRUE; - + /* check if file is datafile */ sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), &(file->segno), suffix); - if (sscanf_res == 0) - elog(ERROR, "Cannot parse file name \"%s\"", file->name); - else if (sscanf_res == 1 || sscanf_res == 2) + Assert(sscanf_res > 0); /* since first char is digit */ + if (sscanf_res == 1 || sscanf_res == 2) file->is_datafile = true; } } @@ -1954,3 +1919,35 @@ pfilearray_clear_locks(parray *file_list) pg_atomic_clear_flag(&file->lock); } } + +/* Set forkName if possible */ +void +set_forkname(pgFile *file) +{ + int name_len = strlen(file->name); + + /* Auxiliary fork of the relfile */ + if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0) + file->forkName = vm; + + else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0) + file->forkName = fsm; + + else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0) + file->forkName = cfm; + + else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0) + file->forkName = init; + + else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0) + file->forkName = ptrack; + + // extract relOid for certain forks + + if ((file->forkName == vm || + file->forkName == fsm || + file->forkName == init || + file->forkName == cfm) && + (sscanf(file->name, "%u*", &(file->relOid)) != 1)) + file->relOid = 0; +} diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 495fbdcad..bc9f9b8a8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -215,6 +215,7 @@ typedef enum CompressAlg typedef enum ForkName { + none, vm, fsm, cfm, @@ -1091,6 +1092,7 @@ extern int pgCompareString(const void *str1, const void *str2); extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); +extern void set_forkname(pgFile *file); /* in data.c */ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, From 8f504fc95cbace9297da363ac098126b4e75c6c8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 10 Oct 2022 17:07:41 +0300 Subject: [PATCH 1923/2107] [PBCKP-146] stabilize couple of tests. --- tests/cfs_backup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index d820360fe..436db31e7 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -995,6 +995,11 @@ def test_delete_random_cfm_file_from_tablespace_dir(self): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_cmf = find_by_extensions( [self.get_tblspace_path(self.node, tblspace_name)], ['.cfm']) @@ -1044,6 +1049,11 @@ def test_delete_random_data_file_from_tablespace_dir(self): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_data_files = find_by_pattern( [self.get_tblspace_path(self.node, tblspace_name)], '^.*/\d+$') From 51a141c4b0b07899c001f75c5c172492cbefa076 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 11 Oct 2022 18:59:48 +0300 Subject: [PATCH 1924/2107] [PBCKP-146] fix cfs test python3 compatibility --- tests/cfs_restore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 07cf891aa..611afc49e 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -103,6 +103,7 @@ def test_restore_empty_tablespace_from_fullbackup(self): "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) ) + tblspace = str(tblspace) self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not restored or it restored without compressions" From 3e17c8c8daa21ba8d79fecc614c591c429dda7d1 Mon Sep 17 00:00:00 2001 From: "Andrew A. Bille" Date: Mon, 19 Sep 2022 12:25:23 +0700 Subject: [PATCH 1925/2107] Fix remembered check gdb flag in test with GDB --- tests/locking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/locking.py b/tests/locking.py index 0fe954cae..4042a1462 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -419,6 +419,8 @@ def test_locking_concurrent_validate_and_backup(self): and stop it in the middle, take page backup. Expect PAGE backup to be successfully executed """ + self._check_gdb_flag_or_skip_test() + fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(module_name, fname, 'node'), From b568f2254a9f714760abe75c436a6476723463b8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 12 Oct 2022 12:17:48 +0300 Subject: [PATCH 1926/2107] [PBCKP-146] truncate cfm files - store cfm files truncated to non-zero head with coarse granularity (64-4096 bytes) - and calculate crc for truncated file - during restoration calculate crc for cfm as if it was truncated --- src/archive.c | 8 +- src/catalog.c | 9 + src/data.c | 124 ++----- src/dir.c | 133 +------- src/merge.c | 2 +- src/pg_probackup.h | 13 +- src/utils/file.c | 519 +++++++++++++++++++++++++++++- src/utils/file.h | 4 +- tests/expected/option_version.out | 2 +- 9 files changed, 564 insertions(+), 250 deletions(-) diff --git a/src/archive.c b/src/archive.c index 2ae86bd6a..734602cac 100644 --- a/src/archive.c +++ b/src/archive.c @@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg); + rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg); if (rc == FILE_MISSING) #endif /* ... failing that, use uncompressed */ - rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg); /* When not in prefetch mode, try to use partial file */ if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) @@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* '.gz.partial' goes first ... */ snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg); + rc = fio_send_file_gz(from_partial, out, &errmsg); if (rc == FILE_MISSING) #endif { /* ... failing that, use '.partial' */ snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file(from_partial, out, false, NULL, &errmsg); } if (rc == SEND_OK) diff --git a/src/catalog.c b/src/catalog.c index 03099d1a2..9668427bb 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict) char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; int64 write_size, + full_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, @@ -1087,6 +1088,8 @@ get_backup_filelist(pgBackup *backup, bool strict) get_control_value_str(buf, "path", path, sizeof(path),true); get_control_value_int64(buf, "size", &write_size, true); + if (!get_control_value_int64(buf, "full_size", &full_size, false)) + full_size = write_size; get_control_value_int64(buf, "mode", &mode, true); get_control_value_int64(buf, "is_datafile", &is_datafile, true); get_control_value_int64(buf, "is_cfs", &is_cfs, false); @@ -1097,6 +1100,7 @@ get_backup_filelist(pgBackup *backup, bool strict) file = pgFileInit(path); file->write_size = (int64) write_size; + file->uncompressed_size = full_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; @@ -2561,6 +2565,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, file->external_dir_num, file->dbOid); + if (file->uncompressed_size != 0 && + file->uncompressed_size != file->write_size) + len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"", + file->uncompressed_size); + if (file->is_datafile) len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); diff --git a/src/data.c b/src/data.c index 753f247f7..a020c6efc 100644 --- a/src/data.c +++ b/src/data.c @@ -799,6 +799,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, * and its mtime is less than parent backup start time ... */ if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) && (prev_file && file->exists_in_prev && + file->size == prev_file->size && file->mtime <= parent_backup_time)) { /* @@ -1330,7 +1331,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (already_exists) { /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); + pg_crc32 file_crc; + if (tmp_file->forkName == cfm && + tmp_file->uncompressed_size > tmp_file->write_size) + file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST); + else + file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); if (file_crc == tmp_file->crc) { @@ -1387,10 +1393,12 @@ backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok) { - FILE *in = NULL; FILE *out = NULL; - ssize_t read_len = 0; - char *buf = NULL; + char *errmsg = NULL; + int rc; + bool cut_zero_tail; + + cut_zero_tail = file->forkName == cfm; INIT_FILE_CRC32(true, file->crc); @@ -1412,107 +1420,43 @@ backup_non_data_file_internal(const char *from_fullpath, /* backup remote file */ if (fio_is_remote(FIO_DB_HOST)) - { - char *errmsg = NULL; - int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg); + rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg); + else + rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg); - /* handle errors */ - if (rc == FILE_MISSING) - { - /* maybe deleted, it's not error in case of backup */ - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); - else if (rc != SEND_OK) + /* handle errors */ + if (rc == FILE_MISSING) + { + /* maybe deleted, it's not error in case of backup */ + if (missing_ok) { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; } - - pg_free(errmsg); + else + elog(ERROR, "File \"%s\" is not found", from_fullpath); } - /* backup local file */ - else + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); + else if (rc != SEND_OK) { - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* maybe deleted, it's not error in case of backup */ - if (errno == ENOENT) - { - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, - strerror(errno)); - } - - /* disable stdio buffering for local input/output files to avoid triple buffering */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); - - /* allocate 64kB buffer */ - buf = pgut_malloc(CHUNK_SIZE); - - /* copy content and calc CRC */ - for (;;) - { - read_len = fread(buf, 1, CHUNK_SIZE, in); - - if (ferror(in)) - elog(ERROR, "Cannot read from file \"%s\": %s", - from_fullpath, strerror(errno)); - - if (read_len > 0) - { - if (fwrite(buf, 1, read_len, out) != read_len) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath, - strerror(errno)); - - /* update CRC */ - COMP_FILE_CRC32(true, file->crc, buf, read_len); - file->read_size += read_len; - } - - if (feof(in)) - break; - } + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); } - file->write_size = (int64) file->read_size; + pg_free(errmsg); /* ????? */ - if (file->write_size > 0) - file->uncompressed_size = file->write_size; + file->uncompressed_size = file->read_size; cleanup: /* finish CRC calculation and store into pgFile */ FIN_FILE_CRC32(true, file->crc); - if (in && fclose(in)) - elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); - if (out && fclose(out)) elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); - - pg_free(buf); } /* diff --git a/src/dir.c b/src/dir.c index 00e918d0f..73d6db09b 100644 --- a/src/dir.c +++ b/src/dir.c @@ -262,137 +262,6 @@ pgFileDelete(mode_t mode, const char *full_path) } } -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) -{ - FILE *fp; - pg_crc32 crc = 0; - char *buf; - size_t len = 0; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fread(buf, 1, STDIO_BUFSIZE, fp); - - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - - if (feof(fp)) - break; - } - - FIN_FILE_CRC32(use_crc32c, crc); - fclose(fp); - pg_free(buf); - - return crc; -} - -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) -{ - gzFile fp; - pg_crc32 crc = 0; - int len = 0; - int err; - char *buf; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = gzopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = gzread(fp, buf, STDIO_BUFSIZE); - - if (len <= 0) - { - /* we either run into eof or error */ - if (gzeof(fp)) - break; - else - { - const char *err_str = NULL; - - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); - } - } - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - } - - FIN_FILE_CRC32(use_crc32c, crc); - gzclose(fp); - pg_free(buf); - - return crc; -} - void pgFileFree(void *file) { @@ -1812,7 +1681,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ FIO_BACKUP_HOST); file->crc = pgFileGetCRC(database_map_path, true, false); file->write_size = file->size; - file->uncompressed_size = file->read_size; + file->uncompressed_size = file->size; parray_append(backup_files_list, file); } diff --git a/src/merge.c b/src/merge.c index 1ce49f9a2..79498f48c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1078,7 +1078,7 @@ merge_files(void *arg) tmp_file->hdr_crc = file->hdr_crc; } else - tmp_file->uncompressed_size = tmp_file->write_size; + tmp_file->uncompressed_size = tmp_file->uncompressed_size; /* Copy header metadata from old map into a new one */ tmp_file->n_headers = file->n_headers; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc9f9b8a8..d1d912045 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -345,11 +345,11 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.8" +#define PROGRAM_VERSION "2.5.9" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20501 -#define AGENT_PROTOCOL_VERSION_STR "2.5.1" +#define AGENT_PROTOCOL_VERSION 20509 +#define AGENT_PROTOCOL_VERSION_STR "2.5.9" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" @@ -1077,6 +1077,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c); extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); @@ -1240,9 +1241,11 @@ extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pg XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber *err_blknum, char **errormsg); /* return codes for fio_send_pages */ -extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); -extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, +extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg); +extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, pgFile *file, char **errormsg); +extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, + pgFile *file, char **errormsg); extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); diff --git a/src/utils/file.c b/src/utils/file.c index 727b48c60..8e3701af6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -18,6 +18,10 @@ static __thread int fio_stdin = 0; static __thread int fio_stderr = 0; static char *async_errormsg = NULL; +#define PAGE_ZEROSEARCH_COARSE_GRANULARITY 4096 +#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64 +static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0}; + fio_location MyLocation; typedef struct @@ -1357,14 +1361,20 @@ fio_sync(char const* path, fio_location location) enum { GET_CRC32_DECOMPRESS = 1, - GET_CRC32_MISSING_OK = 2 + GET_CRC32_MISSING_OK = 2, + GET_CRC32_TRUNCATED = 4 }; /* Get crc32 of file */ -pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, - bool decompress, bool missing_ok) +static pg_crc32 +fio_get_crc32_ex(const char *file_path, fio_location location, + bool decompress, bool missing_ok, bool truncated) { + if (decompress && truncated) + elog(ERROR, "Could not calculate CRC for compressed truncated file"); + if (missing_ok && truncated) + elog(ERROR, "CRC calculation for missing truncated file is forbidden"); + if (fio_is_remote(location)) { fio_header hdr; @@ -1379,6 +1389,8 @@ fio_get_crc32(const char *file_path, fio_location location, hdr.arg = GET_CRC32_DECOMPRESS; if (missing_ok) hdr.arg |= GET_CRC32_MISSING_OK; + if (truncated) + hdr.arg |= GET_CRC32_TRUNCATED; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); @@ -1390,11 +1402,26 @@ fio_get_crc32(const char *file_path, fio_location location, { if (decompress) return pgFileGetCRCgz(file_path, true, missing_ok); + else if (truncated) + return pgFileGetCRCTruncated(file_path, true); else return pgFileGetCRC(file_path, true, missing_ok); } } +pg_crc32 +fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok) +{ + return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false); +} + +pg_crc32 +fio_get_crc32_truncated(const char *file_path, fio_location location) +{ + return fio_get_crc32_ex(file_path, location, false, false, true); +} + /* Remove file */ int fio_unlink(char const* path, fio_location location) @@ -2455,7 +2482,7 @@ fio_send_pages_impl(int out, char* buf) * REMOTE_ERROR (-6) */ int -fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg) +fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; @@ -2604,6 +2631,105 @@ fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, return exit_code; } +typedef struct send_file_state { + bool calc_crc; + uint32_t crc; + int64_t read_size; + int64_t write_size; +} send_file_state; + +/* find page border of all-zero tail */ +static size_t +find_zero_tail(char *buf, size_t len) +{ + size_t i, l; + size_t granul = sizeof(zerobuf); + + if (len == 0) + return 0; + + /* fast check for last bytes */ + i = (len-1) & ~(PAGE_ZEROSEARCH_FINE_GRANULARITY-1); + l = len - i; + if (memcmp(buf + i, zerobuf, i) != 0) + return len; + + /* coarse search for zero tail */ + i = (len-1) & ~(granul-1); + l = len - i; + for (;;) + { + if (memcmp(buf+i, zerobuf, l) != 0) + { + i += l; + break; + } + if (i == 0) + break; + i -= granul; + l = granul; + } + + len = i; + /* search zero tail with finer granularity */ + for (granul = sizeof(zerobuf)/2; + len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY; + granul /= 2) + { + if (granul > l) + continue; + i = (len-1) & ~(granul-1); + l = len - i; + if (memcmp(buf+i, zerobuf, l) == 0) + len = i; + } + + return len; +} + +static void +fio_send_file_crc(send_file_state* st, char *buf, size_t len) +{ + int64_t write_size; + + if (!st->calc_crc) + return; + + write_size = st->write_size; + while (st->read_size > write_size) + { + size_t crc_len = Min(st->read_size - write_size, sizeof(zerobuf)); + COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len); + write_size += crc_len; + } + + if (len > 0) + COMP_FILE_CRC32(true, st->crc, buf, len); +} + +static bool +fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len) +{ + if (len == 0) + return true; + + if (st->read_size > st->write_size && + fseeko(out, st->read_size, SEEK_SET) != 0) + { + return false; + } + + if (fwrite(buf, 1, len, out) != len) + { + return false; + } + + st->read_size += len; + st->write_size = st->read_size; + + return true; +} + /* Receive chunks of data and write them to destination file. * Return codes: * SEND_OK (0) @@ -2616,13 +2742,22 @@ fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, * If pgFile is not NULL then we must calculate crc and read_size for it. */ int -fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, +fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, pgFile *file, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; size_t path_len = strlen(from_fullpath) + 1; char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */ + send_file_state st = {false, 0, 0, 0}; + + memset(&hdr, 0, sizeof(hdr)); + + if (file) + { + st.calc_crc = true; + st.crc = file->crc; + } hdr.cop = FIO_SEND_FILE; hdr.size = path_len; @@ -2640,6 +2775,37 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, if (hdr.cop == FIO_SEND_FILE_EOF) { + if (st.write_size < st.read_size) + { + if (!cut_zero_tail) + { + /* + * We still need to calc crc for zero tail. + */ + fio_send_file_crc(&st, NULL, 0); + + /* + * Let's write single zero byte to the end of file to restore + * logical size. + * Well, it would be better to use ftruncate here actually, + * but then we need to change interface. + */ + st.read_size -= 1; + buf[0] = 0; + if (!fio_send_file_write(out, &st, buf, 1)) + { + exit_code = WRITE_FAILED; + break; + } + } + } + + if (file) + { + file->crc = st.crc; + file->read_size = st.read_size; + file->write_size = st.write_size; + } break; } else if (hdr.cop == FIO_ERROR) @@ -2660,17 +2826,23 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); /* We have received a chunk of data data, lets write it out */ - if (fwrite(buf, 1, hdr.size, out) != hdr.size) + fio_send_file_crc(&st, buf, hdr.size); + if (!fio_send_file_write(out, &st, buf, hdr.size)) { exit_code = WRITE_FAILED; break; } + } + else if (hdr.cop == FIO_PAGE_ZERO) + { + Assert(hdr.size == 0); + Assert(hdr.arg <= CHUNK_SIZE); - if (file) - { - file->read_size += hdr.size; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - } + /* + * We have received a chunk of zero data, lets just think we + * wrote it. + */ + st.read_size += hdr.arg; } else { @@ -2686,6 +2858,117 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, return exit_code; } +int +fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, + pgFile *file, char **errormsg) +{ + FILE* in; + char* buf; + size_t read_len, non_zero_len; + int exit_code = SEND_OK; + send_file_state st = {false, 0, 0, 0}; + + if (file) + { + st.calc_crc = true; + st.crc = file->crc; + } + + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* maybe deleted, it's not error in case of backup */ + if (errno == ENOENT) + return FILE_MISSING; + + + *errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath, + strerror(errno)); + return OPEN_FAILED; + } + + /* disable stdio buffering for local input/output files to avoid triple buffering */ + setvbuf(in, NULL, _IONBF, BUFSIZ); + setvbuf(out, NULL, _IONBF, BUFSIZ); + + /* allocate 64kB buffer */ + buf = pgut_malloc(CHUNK_SIZE); + + /* copy content and calc CRC */ + for (;;) + { + read_len = fread(buf, 1, CHUNK_SIZE, in); + + if (ferror(in)) + { + *errormsg = psprintf("Cannot read from file \"%s\": %s", + from_fullpath, strerror(errno)); + exit_code = READ_FAILED; + goto cleanup; + } + + if (read_len > 0) + { + non_zero_len = find_zero_tail(buf, read_len); + if (non_zero_len > 0) + { + fio_send_file_crc(&st, buf, non_zero_len); + if (!fio_send_file_write(out, &st, buf, non_zero_len)) + { + exit_code = WRITE_FAILED; + goto cleanup; + } + } + if (non_zero_len < read_len) + { + /* Just pretend we wrote it. */ + st.read_size += read_len - non_zero_len; + } + } + + if (feof(in)) + break; + } + + if (st.write_size < st.read_size) + { + /* + * We still need to calc crc for zero tail. + */ + fio_send_file_crc(&st, NULL, 0); + + if (!cut_zero_tail) + { + /* + * Let's write single zero byte to the end of file to restore + * logical size. + * Well, it would be better to use ftruncate here actually, + * but then we need to change interface. + */ + st.read_size -= 1; + buf[0] = 0; + if (!fio_send_file_write(out, &st, buf, 1)) + { + exit_code = WRITE_FAILED; + goto cleanup; + } + } + } + + if (file) + { + file->crc = st.crc; + file->read_size = st.read_size; + file->write_size = st.write_size; + } + +cleanup: + free(buf); + fclose(in); + return exit_code; +} + /* Send file content * On error we return FIO_ERROR message with following codes * FIO_ERROR: @@ -2746,6 +3029,7 @@ fio_send_file_impl(int out, char const* path) for (;;) { read_len = fread(buf, 1, CHUNK_SIZE, fp); + memset(&hdr, 0, sizeof(hdr)); /* report error */ if (ferror(fp)) @@ -2766,10 +3050,22 @@ fio_send_file_impl(int out, char const* path) if (read_len > 0) { /* send chunk */ - hdr.cop = FIO_PAGE; - hdr.size = read_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, read_len), read_len); + size_t non_zero_len = find_zero_tail(buf, read_len); + if (non_zero_len > 0) + { + hdr.cop = FIO_PAGE; + hdr.size = non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len); + } + + if (non_zero_len < read_len) + { + hdr.cop = FIO_PAGE_ZERO; + hdr.size = 0; + hdr.arg = read_len - non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } } if (feof(fp)) @@ -2788,6 +3084,193 @@ fio_send_file_impl(int out, char const* path) return; } +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + */ +pg_crc32 +pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) +{ + FILE *fp; + pg_crc32 crc = 0; + char *buf; + size_t len = 0; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + + if (feof(fp)) + break; + } + + FIN_FILE_CRC32(use_crc32c, crc); + fclose(fp); + pg_free(buf); + + return crc; +} + +/* + * Read the local file to compute CRC for it extened to real_size. + */ +pg_crc32 +pgFileGetCRCTruncated(const char *file_path, bool use_crc32c) +{ + FILE *fp; + char *buf; + size_t len = 0; + size_t non_zero_len; + send_file_state st = {true, 0, 0, 0}; + + INIT_FILE_CRC32(use_crc32c, st.crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(CHUNK_SIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + non_zero_len = find_zero_tail(buf, len); + if (non_zero_len) + { + fio_send_file_crc(&st, buf, non_zero_len); + st.write_size += st.read_size + non_zero_len; + } + st.read_size += len; + + if (feof(fp)) + break; + } + + FIN_FILE_CRC32(use_crc32c, st.crc); + fclose(fp); + pg_free(buf); + + return st.crc; +} + +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + */ +pg_crc32 +pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +{ + gzFile fp; + pg_crc32 crc = 0; + int len = 0; + int err; + char *buf; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = gzopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = gzread(fp, buf, STDIO_BUFSIZE); + + if (len <= 0) + { + /* we either run into eof or error */ + if (gzeof(fp)) + break; + else + { + const char *err_str = NULL; + + err_str = gzerror(fp, &err); + elog(ERROR, "Cannot read from compressed file %s", err_str); + } + } + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + } + + FIN_FILE_CRC32(use_crc32c, crc); + gzclose(fp); + pg_free(buf); + + return crc; +} + /* Compile the array of files located on remote machine in directory root */ static void fio_list_dir_internal(parray *files, const char *root, bool exclude, @@ -3387,9 +3870,13 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_GET_CRC32: + Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 || + (hdr.arg & GET_CRC32_TRUNCATED) == GET_CRC32_TRUNCATED); /* calculate crc32 for a file */ if ((hdr.arg & GET_CRC32_DECOMPRESS)) crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); + else if ((hdr.arg & GET_CRC32_TRUNCATED)) + crc = pgFileGetCRCTruncated(buf, true); else crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); diff --git a/src/utils/file.h b/src/utils/file.h index ec478b451..890babf55 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -56,7 +56,8 @@ typedef enum FIO_CHECK_POSTMASTER, FIO_GET_ASYNC_ERROR, FIO_WRITE_ASYNC, - FIO_READLINK + FIO_READLINK, + FIO_PAGE_ZERO } fio_operations; typedef enum @@ -122,6 +123,7 @@ extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress, bool missing_ok); +extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 4de288907..7c9fcbfe0 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.8 +pg_probackup 2.5.9 From 202f2ade7f7ae7b1fdf1f7f8b413b2950d040fa4 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 16:52:38 +0300 Subject: [PATCH 1927/2107] find_zero_tail: fix last bytes check --- src/utils/file.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index 8e3701af6..b45fea0e7 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2649,9 +2649,9 @@ find_zero_tail(char *buf, size_t len) return 0; /* fast check for last bytes */ - i = (len-1) & ~(PAGE_ZEROSEARCH_FINE_GRANULARITY-1); - l = len - i; - if (memcmp(buf + i, zerobuf, i) != 0) + l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY); + i = len - l; + if (memcmp(buf + i, zerobuf, l) != 0) return len; /* coarse search for zero tail */ From 2eaeb942c68a6b149f6e9b04a9f094c57f3e8b96 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 17:15:59 +0300 Subject: [PATCH 1928/2107] [PBCKP-146] add test for cfm size --- tests/cfs_backup.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 436db31e7..861c9f1ea 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -171,12 +171,16 @@ def test_fullbackup_after_create_table(self): "ERROR: File pg_compression not found in {0}".format( os.path.join(self.backup_dir, 'node', backup_id)) ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) + cfms = find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) # @unittest.expectedFailure # @unittest.skip("skip") From 5f71f7710fc4df684f53b99c575182b00f2c8a99 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 17:28:07 +0300 Subject: [PATCH 1929/2107] fix fio_send_file_local --- src/utils/file.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index b45fea0e7..7f88a3ad0 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2933,13 +2933,13 @@ fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, if (st.write_size < st.read_size) { - /* - * We still need to calc crc for zero tail. - */ - fio_send_file_crc(&st, NULL, 0); - if (!cut_zero_tail) { + /* + * We still need to calc crc for zero tail. + */ + fio_send_file_crc(&st, NULL, 0); + /* * Let's write single zero byte to the end of file to restore * logical size. From 302db1c49f49ff61662b2f201e0e4342b77de539 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 20:35:11 +0300 Subject: [PATCH 1930/2107] [PGPRO-146] store at least cfs header size bytes. --- src/utils/file.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/utils/file.c b/src/utils/file.c index 7f88a3ad0..b4ba30594 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2911,6 +2911,17 @@ fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, if (read_len > 0) { non_zero_len = find_zero_tail(buf, read_len); + /* + * It is dirty trick to silence warnings in CFS GC process: + * backup at least cfs header size bytes. + */ + if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + st.read_size + read_len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + st.read_size + read_len); + non_zero_len -= st.read_size; + } if (non_zero_len > 0) { fio_send_file_crc(&st, buf, non_zero_len); From a20eb7bddb65578e5cfb2dff1ad73c67dd55ece2 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 21:57:14 +0300 Subject: [PATCH 1931/2107] [PGPRO-146] pgdata_content: checksum for truncated cfm --- tests/helpers/ptrack_helpers.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d800f0d3e..abb715b7e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1709,8 +1709,18 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_relpath = os.path.relpath(file_fullpath, pgdata) directory_dict['files'][file_relpath] = {'is_datafile': False} with open(file_fullpath, 'rb') as f: - directory_dict['files'][file_relpath]['md5'] = hashlib.md5(f.read()).hexdigest() - f.close() + content = f.read() + # truncate cfm's content's zero tail + if file_relpath.endswith('.cfm'): + zero64 = b"\x00"*64 + l = len(content) + while l > 64: + s = (l - 1) & ~63 + if content[s:l] != zero64[:l-s]: + break + l = s + content = content[:l] + directory_dict['files'][file_relpath]['md5'] = hashlib.md5(content).hexdigest() # directory_dict['files'][file_relpath]['md5'] = hashlib.md5( # f = open(file_fullpath, 'rb').read()).hexdigest() From f2537982503d988cf960b8c5b5b37fa39b602d8e Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 24 Oct 2022 21:57:55 +0300 Subject: [PATCH 1932/2107] [PGPRO-146] cfs_catchup test Test full catchup and delta catchup. --- tests/cfs_catchup.py | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/cfs_catchup.py diff --git a/tests/cfs_catchup.py b/tests/cfs_catchup.py new file mode 100644 index 000000000..068311035 --- /dev/null +++ b/tests/cfs_catchup.py @@ -0,0 +1,107 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +module_name = 'cfs_catchup' +tblspace_name = 'cfs_tblspace' + + +class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): + def setUp(self): + self.fname = self.id().split('.')[3] + + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_full_catchup_with_tablespace(self): + """ + Test tablespace transfers + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') + self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # do full catchup with tablespace mapping + dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # make changes in master tablespace + src_pg.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # and now delta backup + dst_pg.stop() + + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + + # 3rd check: run verification query + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + src_pg.stop() + dst_pg.stop() + self.del_test_dir(module_name, self.fname) From 26f9992b2ab484a0bdd605d9669b2c86a07c28ed Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Mon, 24 Oct 2022 18:28:30 +0300 Subject: [PATCH 1933/2107] [PBCKP-236] added PGPROBACKUP_MANUAL testing and PGPROBACKUP_SSH_AGENT_PATH flags. --- tests/compatibility.py | 53 +++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/compatibility.py b/tests/compatibility.py index 04af1478f..6c2bc9204 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -8,32 +8,48 @@ module_name = 'compatibility' +def check_manual_tests_enabled(): + return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' + + +def check_ssh_agent_path_exists(): + return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ + + class CompatibilityTest(ProbackupTest, unittest.TestCase): def setUp(self): self.fname = self.id().split('.')[3] # @unittest.expectedFailure - @unittest.skip("skip") + @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') + @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') + # @unittest.skip("skip") def test_catchup_with_different_remote_major_pg(self): """ Decription in jira issue PBCKP-236 - This test requires builds both PGPROEE11 and PGPROEE9_6 + This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6 + + Prerequisites: + - pg_probackup git tag for PBCKP 2.5.1 + - master pg_probackup build should be made for PGPROEE11 + - agent pg_probackup build should be made for PGPROEE9_6 - prerequisites: - - git tag for PBCKP 2.5.1 - - master probackup build should be inside PGPROEE11 - - agent probackup build is inside PGPROEE9_6 + Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes + the PBCKP-236 problem - calling probackup PGPROEE9_6 agent from PGPROEE11 probackup master for DELTA backup causes the PBCKP-236 problem + Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH= + for the test - please correct path for agent's pg_path_remote_version = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + without pg_probackup executable """ self.verbose = True self.remote = True - # please use your own local path - pg_path_remote_version = '/home/avaness/postgres/postgres.build.clean/bin' + # please use your own local path like + # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' + pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] src_pg = self.make_simple_node( base_dir=os.path.join(module_name, self.fname, 'src'), @@ -47,14 +63,13 @@ def test_catchup_with_different_remote_major_pg(self): # do full catchup dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] ) - dst_options = {} - dst_options['port'] = str(dst_pg.port) + dst_options = {'port': str(dst_pg.port)} self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() dst_pg.stop() @@ -66,11 +81,11 @@ def test_catchup_with_different_remote_major_pg(self): # do delta catchup with remote pg_probackup agent with another postgres major version # this DELTA backup should fail without PBCKP-236 patch. self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pg_path_remote_version] + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] ) # Clean after yourself From 97355f1562041136c952add40b5ccafc76eaff58 Mon Sep 17 00:00:00 2001 From: dlepikhova <43872363+dlepikhova@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:34:36 +0500 Subject: [PATCH 1934/2107] release_2_5_9-pbckp-227 (#533) [PBCKP-227]: Fix some potential problems in pg_probackup code * Fix fwrite parameters in fio_open_stream * Remove unused get_system_dbstate function * Set actual sourse for fields in instance_options to SOURCE_DEFAULT * Remove get_system_dbstate declaration from header file --- src/configure.c | 132 ++++++++++++++++++++++----------------------- src/pg_probackup.h | 1 - src/util.c | 16 ------ src/utils/file.c | 2 +- 4 files changed, 67 insertions(+), 84 deletions(-) diff --git a/src/configure.c b/src/configure.c index 6e8700de1..f7befb0c5 100644 --- a/src/configure.c +++ b/src/configure.c @@ -53,7 +53,7 @@ ConfigOption instance_options[] = /* Instance options */ { 's', 'D', "pgdata", - &instance_config.pgdata, SOURCE_CMD, 0, + &instance_config.pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -70,49 +70,49 @@ ConfigOption instance_options[] = #endif { 's', 'E', "external-dirs", - &instance_config.external_dir_str, SOURCE_CMD, 0, + &instance_config.external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance_config.conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance_config.conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance_config.conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance_config.conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance_config.master_conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance_config.master_conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance_config.master_conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -128,17 +128,17 @@ ConfigOption instance_options[] = }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { @@ -149,37 +149,37 @@ ConfigOption instance_options[] = /* Logging options */ { 'f', 212, "log-level-console", - assign_log_level_console, SOURCE_CMD, 0, + assign_log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_console }, { 'f', 213, "log-level-file", - assign_log_level_file, SOURCE_CMD, 0, + assign_log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_file }, { 'f', 214, "log-format-console", - assign_log_format_console, SOURCE_CMD_STRICT, 0, + assign_log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_format_console }, { 'f', 215, "log-format-file", - assign_log_format_file, SOURCE_CMD, 0, + assign_log_format_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_format_file }, { 's', 216, "log-filename", - &instance_config.logger.log_filename, SOURCE_CMD, 0, + &instance_config.logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 217, "error-log-filename", - &instance_config.logger.error_log_filename, SOURCE_CMD, 0, + &instance_config.logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 218, "log-directory", - &instance_config.logger.log_directory, SOURCE_CMD, 0, + &instance_config.logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { @@ -195,64 +195,64 @@ ConfigOption instance_options[] = /* Retention options */ { 'u', 221, "retention-redundancy", - &instance_config.retention_redundancy, SOURCE_CMD, 0, + &instance_config.retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 222, "retention-window", - &instance_config.retention_window, SOURCE_CMD, 0, + &instance_config.retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 223, "wal-depth", - &instance_config.wal_depth, SOURCE_CMD, 0, + &instance_config.wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { 'f', 224, "compress-algorithm", - assign_compress_alg, SOURCE_CMD, 0, + assign_compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { 'u', 225, "compress-level", - &instance_config.compress_level, SOURCE_CMD, 0, + &instance_config.compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { 's', 226, "remote-proto", - &instance_config.remote.proto, SOURCE_CMD, 0, + &instance_config.remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 227, "remote-host", - &instance_config.remote.host, SOURCE_CMD, 0, + &instance_config.remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 228, "remote-port", - &instance_config.remote.port, SOURCE_CMD, 0, + &instance_config.remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 229, "remote-path", - &instance_config.remote.path, SOURCE_CMD, 0, + &instance_config.remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 230, "remote-user", - &instance_config.remote.user, SOURCE_CMD, 0, + &instance_config.remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 231, "ssh-options", - &instance_config.remote.ssh_options, SOURCE_CMD, 0, + &instance_config.remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 232, "ssh-config", - &instance_config.remote.ssh_config, SOURCE_CMD, 0, + &instance_config.remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } @@ -412,7 +412,7 @@ readInstanceConfigFile(InstanceState *instanceState) /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -429,49 +429,49 @@ readInstanceConfigFile(InstanceState *instanceState) #endif { 's', 'E', "external-dirs", - &instance->external_dir_str, SOURCE_CMD, 0, + &instance->external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance->conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance->conn_opt.pghost, SOURCE_CMD, 0, + &instance->conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance->conn_opt.pgport, SOURCE_CMD, 0, + &instance->conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance->conn_opt.pguser, SOURCE_CMD, 0, + &instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance->master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance->master_conn_opt.pghost, SOURCE_CMD, 0, + &instance->master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance->master_conn_opt.pgport, SOURCE_CMD, 0, + &instance->master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance->master_conn_opt.pguser, SOURCE_CMD, 0, + &instance->master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -487,66 +487,66 @@ readInstanceConfigFile(InstanceState *instanceState) }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 211, "restore-command", - &instance->restore_command, SOURCE_CMD, 0, + &instance->restore_command, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Logging options */ { 's', 212, "log-level-console", - &log_level_console, SOURCE_CMD, 0, + &log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 213, "log-level-file", - &log_level_file, SOURCE_CMD, 0, + &log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 214, "log-format-console", - &log_format_console, SOURCE_CMD_STRICT, 0, + &log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 215, "log-format-file", - &log_format_file, SOURCE_CMD, 0, + &log_format_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 216, "log-filename", - &instance->logger.log_filename, SOURCE_CMD, 0, + &instance->logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 217, "error-log-filename", - &instance->logger.error_log_filename, SOURCE_CMD, 0, + &instance->logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 218, "log-directory", - &instance->logger.log_directory, SOURCE_CMD, 0, + &instance->logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { @@ -562,64 +562,64 @@ readInstanceConfigFile(InstanceState *instanceState) /* Retention options */ { 'u', 221, "retention-redundancy", - &instance->retention_redundancy, SOURCE_CMD, 0, + &instance->retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 222, "retention-window", - &instance->retention_window, SOURCE_CMD, 0, + &instance->retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { 'u', 223, "wal-depth", - &instance->wal_depth, SOURCE_CMD, 0, + &instance->wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { 's', 224, "compress-algorithm", - &compress_alg, SOURCE_CMD, 0, + &compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 'u', 225, "compress-level", - &instance->compress_level, SOURCE_CMD, 0, + &instance->compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { 's', 226, "remote-proto", - &instance->remote.proto, SOURCE_CMD, 0, + &instance->remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 227, "remote-host", - &instance->remote.host, SOURCE_CMD, 0, + &instance->remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 228, "remote-port", - &instance->remote.port, SOURCE_CMD, 0, + &instance->remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 229, "remote-path", - &instance->remote.path, SOURCE_CMD, 0, + &instance->remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 230, "remote-user", - &instance->remote.user, SOURCE_CMD, 0, + &instance->remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 231, "ssh-options", - &instance->remote.ssh_options, SOURCE_CMD, 0, + &instance->remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 's', 232, "ssh-config", - &instance->remote.ssh_config, SOURCE_CMD, 0, + &instance->remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bc9f9b8a8..ffb74da1a 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1175,7 +1175,6 @@ extern uint64 get_system_identifier(const char *pgdata_path, fio_location locati extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern DBState get_system_dbstate(const char *pgdata_path, fio_location location); extern uint32 get_xlog_seg_size(const char *pgdata_path); extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, diff --git a/src/util.c b/src/util.c index 4d6c50a07..d19877f06 100644 --- a/src/util.c +++ b/src/util.c @@ -349,22 +349,6 @@ get_pgcontrol_checksum(const char *pgdata_path) return ControlFile.crc; } -DBState -get_system_dbstate(const char *pgdata_path, fio_location location) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); - if (buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.state; -} - void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) { diff --git a/src/utils/file.c b/src/utils/file.c index 727b48c60..e32696f15 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -305,7 +305,7 @@ fio_open_stream(char const* path, fio_location location) IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); #ifdef WIN32 f = tmpfile(); - IO_CHECK(fwrite(f, 1, hdr.size, fio_stdin_buffer), hdr.size); + IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size); SYS_CHECK(fseek(f, 0, SEEK_SET)); #else f = fmemopen(fio_stdin_buffer, hdr.size, "r"); From 4ac1b536a5f6be93901ad2f7fef216d80d868229 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Fri, 14 Oct 2022 14:01:41 +0300 Subject: [PATCH 1935/2107] [PBCKP-235] README.md now contains correct package versions tags: pg_probackup --- .travis.yml | 5 ++-- README.md | 72 ++++++++++++++++++++++++++--------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d0d786c4..bd3c8a09a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,8 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=16 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE @@ -54,7 +55,7 @@ jobs: allow_failures: - if: env(PG_BRANCH) = master - if: env(PG_BRANCH) = REL9_5_STABLE -# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) +# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) # Only run CI for master branch commits to limit our travis usage #branches: diff --git a/README.md b/README.md index d1ccd9866..bae1171cb 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp `PTRACK` backup support provided via following options: * vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14, 15 -* Postgres Pro Enterprise 11, 12, 13, 14, 15 +* Postgres Pro Standard 11, 12, 13, 14 +* Postgres Pro Enterprise 11, 12, 13, 14 ## Limitations @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{15,14,13,12,11,10,9.6} +sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{14,13,12,11,10,9.6} +yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{15,14,13,12,11,10,9.6} +yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10,9.6} -zypper install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} +zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{15,14,13,12,11,10,9.6} +zypper si pg_probackup-{14,13,12,11,10,9.6} #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise @@ -137,8 +137,8 @@ sudo apt-get install pg_probackup-{15,14,13,12,11,10,9.6}-debuginfo #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{15,14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' @@ -148,35 +148,35 @@ sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 7 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{15,14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} +sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From 1a48b6c5959f75ceba18223bb7e597bce6035763 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 28 Oct 2022 19:40:15 +0300 Subject: [PATCH 1936/2107] [PBCKP-236] added remote shh agent path to log output --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index addd73dc8..8562c85e3 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -229,7 +229,7 @@ bool launch_agent(void) return false; } else { #endif - elog(LOG, "Start SSH client process, pid %d", child_pid); + elog(LOG, "Start SSH client process, pid %d, cmd \"%s\"", child_pid, cmd); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); SYS_CHECK(close(errfd[1])); From 79009c652cd7231edf9609837a669f7d20ae67f2 Mon Sep 17 00:00:00 2001 From: MetalDream666 <61190185+MetalDream666@users.noreply.github.com> Date: Sat, 29 Oct 2022 01:17:13 +0300 Subject: [PATCH 1937/2107] Revert "[PBCKP-120] skip partitioned indexes for checkdb --amcheck" --- src/checkdb.c | 10 +++------- tests/checkdb.py | 9 --------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 1133a7b5d..177fc3cc7 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,9 +461,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " + "WHERE am.amname='btree' AND cls.relpersistence != 't' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -475,10 +473,8 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " - "AND (cls.reltablespace IN " + "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " + "(cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index 07b55c6db..bcda0fb23 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -38,15 +38,6 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") - - node.safe_psql( - "postgres", - "create table idxpart (a int) " - "partition by range (a)") - - node.safe_psql( - "postgres", - "create index on idxpart(a)") try: node.safe_psql( From 0b8cf419c344499d0540636a19ab944dcc167527 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Mon, 20 Jun 2022 13:44:42 +0300 Subject: [PATCH 1938/2107] [PBCKP-120] skip partitioned indexes for checkdb --amcheck Tags: pg_probackup --- src/checkdb.c | 10 +++++++--- tests/checkdb.py | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/checkdb.c b/src/checkdb.c index 177fc3cc7..1133a7b5d 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -461,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -473,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " - "(cls.reltablespace IN " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " + "AND (cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", diff --git a/tests/checkdb.py b/tests/checkdb.py index bcda0fb23..07b55c6db 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -38,6 +38,15 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( From 0b474d261686f4554d5b853803444b3379b650ce Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 1 Nov 2022 11:53:17 +0300 Subject: [PATCH 1939/2107] [PBCKP-236] hotfix for C89 --- src/utils/remote.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 8562c85e3..9068c9406 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -321,7 +321,8 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b size_t result_size = 0; *compatibility_buf = '\0'; - for (int i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) + int i; + for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, From e36924a0fd5c42a0538421411a92377672bbf3c3 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 1 Nov 2022 12:53:46 +0300 Subject: [PATCH 1940/2107] [PBCKP-146] review fixes --- src/catalog.c | 10 ++++++---- src/data.c | 5 +++-- tests/cfs_backup.py | 2 ++ tests/cfs_catchup.py | 20 ++++++++++++++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 9668427bb..561ab876e 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1069,7 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict) char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; int64 write_size, - full_size, + uncompressed_size, mode, /* bit length of mode_t depends on platforms */ is_datafile, is_cfs, @@ -1088,8 +1088,6 @@ get_backup_filelist(pgBackup *backup, bool strict) get_control_value_str(buf, "path", path, sizeof(path),true); get_control_value_int64(buf, "size", &write_size, true); - if (!get_control_value_int64(buf, "full_size", &full_size, false)) - full_size = write_size; get_control_value_int64(buf, "mode", &mode, true); get_control_value_int64(buf, "is_datafile", &is_datafile, true); get_control_value_int64(buf, "is_cfs", &is_cfs, false); @@ -1100,7 +1098,6 @@ get_backup_filelist(pgBackup *backup, bool strict) file = pgFileInit(path); file->write_size = (int64) write_size; - file->uncompressed_size = full_size; file->mode = (mode_t) mode; file->is_datafile = is_datafile ? true : false; file->is_cfs = is_cfs ? true : false; @@ -1136,6 +1133,11 @@ get_backup_filelist(pgBackup *backup, bool strict) if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) file->hdr_size = (int) hdr_size; + if (get_control_value_int64(buf, "full_size", &uncompressed_size, false)) + file->uncompressed_size = uncompressed_size; + else + file->uncompressed_size = write_size; + if (file->external_dir_num == 0) set_forkname(file); diff --git a/src/data.c b/src/data.c index a020c6efc..2a8806cde 100644 --- a/src/data.c +++ b/src/data.c @@ -1447,11 +1447,12 @@ backup_non_data_file_internal(const char *from_fullpath, elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); } - pg_free(errmsg); /* ????? */ - file->uncompressed_size = file->read_size; cleanup: + if (errmsg != NULL) + pg_free(errmsg); + /* finish CRC calculation and store into pgFile */ FIN_FILE_CRC32(true, file->crc); diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 861c9f1ea..306c2396c 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -171,6 +171,8 @@ def test_fullbackup_after_create_table(self): "ERROR: File pg_compression not found in {0}".format( os.path.join(self.backup_dir, 'node', backup_id)) ) + + # check cfm size cfms = find_by_extensions( [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], ['.cfm']) diff --git a/tests/cfs_catchup.py b/tests/cfs_catchup.py index 068311035..2cbb46729 100644 --- a/tests/cfs_catchup.py +++ b/tests/cfs_catchup.py @@ -56,6 +56,16 @@ def test_full_catchup_with_tablespace(self): self.pgdata_content(dst_pg.data_dir) ) + # check cfm size + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + # make changes in master tablespace src_pg.safe_psql( "postgres", @@ -89,6 +99,16 @@ def test_full_catchup_with_tablespace(self): ] ) + # check cfm size again + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + # run&recover catchup'ed instance dst_options = {} dst_options['port'] = str(dst_pg.port) From 64b84d0ca64a95db6154185aafc1291a6aa142df Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 1 Nov 2022 14:41:02 +0300 Subject: [PATCH 1941/2107] [PBCKP-236] hotfix-2 for C89 compatibility --- src/utils/remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/remote.c b/src/utils/remote.c index 9068c9406..7ef8d3239 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -319,9 +319,9 @@ size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_b }; size_t result_size = 0; + int i; *compatibility_buf = '\0'; - int i; for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) { if (compatibility_params[i].strval != NULL) From feacabd8ab15129743fdd9ef287dfccd793fdfe3 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 2 Nov 2022 14:25:12 +0300 Subject: [PATCH 1942/2107] [PBCKP-308] Changed check_server_version function for postgresql version for 1c. --- src/backup.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 31289978d..c73ee56c7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -946,10 +946,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) */ #ifdef PGPRO_VERSION if (!res) + { /* It seems we connected to PostgreSQL (not Postgres Pro) */ - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with PostgreSQL %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + if(strcmp(PGPRO_EDITION, "1C") != 0) + { + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } + /* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C + * Check the major version + */ + if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } else { if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && From eaf3b14c22ec4cae50e5546539404232f56a9d7a Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 02:37:29 +0300 Subject: [PATCH 1943/2107] fix set_forkname Fork detection were broken before set_forkname extraction, and its bug were copied into. Lets reimplement it to be like `parse_filename_for_nonetemp_relation` in PostgreSQL code. --- src/catalog.c | 11 ++++++ src/dir.c | 94 +++++++++++++++++++++++++++++----------------- src/pg_probackup.h | 2 +- 3 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 561ab876e..80cdacdc5 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1139,7 +1139,18 @@ get_backup_filelist(pgBackup *backup, bool strict) file->uncompressed_size = write_size; if (file->external_dir_num == 0) + { + bool is_datafile = file->is_datafile; set_forkname(file); + if (is_datafile != file->is_datafile) + { + elog(WARNING, "File '%s' was stored as datafile, but looks like it is not", + file->rel_path); + /* Lets fail in tests */ + Assert(file->is_datafile == file->is_datafile); + file->is_datafile = is_datafile; + } + } parray_append(files, file); } diff --git a/src/dir.c b/src/dir.c index 73d6db09b..b55f25e18 100644 --- a/src/dir.c +++ b/src/dir.c @@ -631,20 +631,6 @@ dir_check_file(pgFile *file, bool backup_logs) if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */ return CHECK_FALSE; - else if (file->forkName != none) - return CHECK_TRUE; - - /* Set is_datafile flag */ - { - char suffix[MAXFNAMELEN]; - - /* check if file is datafile */ - sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), - &(file->segno), suffix); - Assert(sscanf_res > 0); /* since first char is digit */ - if (sscanf_res == 1 || sscanf_res == 2) - file->is_datafile = true; - } } } @@ -1789,34 +1775,74 @@ pfilearray_clear_locks(parray *file_list) } } +static inline bool +is_forkname(char *name, size_t *pos, const char *forkname) +{ + size_t fnlen = strlen(forkname); + if (strncmp(name + *pos, forkname, fnlen) != 0) + return false; + *pos += fnlen; + return true; +} + +#define OIDCHARS 10 + /* Set forkName if possible */ -void +bool set_forkname(pgFile *file) { - int name_len = strlen(file->name); - - /* Auxiliary fork of the relfile */ - if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0) - file->forkName = vm; + size_t i = 0; + uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */ - else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0) - file->forkName = fsm; + /* pretend it is not relation file */ + file->relOid = 0; + file->forkName = none; + file->is_datafile = false; - else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0) - file->forkName = cfm; + for (i = 0; isdigit(file->name[i]); i++) + { + if (i == 0 && file->name[i] == '0') + return false; + oid = oid * 10 + file->name[i] - '0'; + } + if (i == 0 || i > OIDCHARS || oid > UINT32_MAX) + return false; - else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0) + /* usual fork name */ + /* /^\d+_(vm|fsm|init|ptrack)$/ */ + if (is_forkname(file->name, &i, "_vm")) + file->forkName = vm; + else if (is_forkname(file->name, &i, "_fsm")) + file->forkName = fsm; + else if (is_forkname(file->name, &i, "_init")) file->forkName = init; - - else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0) + else if (is_forkname(file->name, &i, "_ptrack")) file->forkName = ptrack; - // extract relOid for certain forks + /* segment number */ + /* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */ + if (file->name[i] == '.' && isdigit(file->name[i+1])) + { + for (i++; isdigit(file->name[i]); i++) + ; + } + + /* CFS "fork name" */ + if (file->forkName == none && + is_forkname(file->name, &i, ".cfm")) + { + /* /^\d+(\.\d+)?.cfm$/ */ + file->forkName = cfm; + } + + /* If there are excess characters, it is not relation file */ + if (file->name[i] != 0) + { + file->forkName = none; + return false; + } - if ((file->forkName == vm || - file->forkName == fsm || - file->forkName == init || - file->forkName == cfm) && - (sscanf(file->name, "%u*", &(file->relOid)) != 1)) - file->relOid = 0; + file->relOid = oid; + file->is_datafile = file->forkName == none; + return true; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 82504bb9a..f2201ebdd 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1098,7 +1098,7 @@ extern int pgCompareString(const void *str1, const void *str2); extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); extern void pfilearray_clear_locks(parray *file_list); -extern void set_forkname(pgFile *file); +extern bool set_forkname(pgFile *file); /* in data.c */ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, From a8ee334c3fbf1dda7c6c5ad58a8845672666a643 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 02:37:58 +0300 Subject: [PATCH 1944/2107] [PBCKP-235] fix one test for <15.0 --- tests/backup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/backup.py b/tests/backup.py index 4f447c9bd..6028a3ff6 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -3441,10 +3441,15 @@ def test_backup_atexit(self): self.assertIn( 'WARNING: backup in progress, stop backup', log_content) - - self.assertIn( - 'FROM pg_catalog.pg_backup_stop', - log_content) + + if self.get_version(node) < 150000: + self.assertIn( + 'FROM pg_catalog.pg_stop_backup', + log_content) + else: + self.assertIn( + 'FROM pg_catalog.pg_backup_stop', + log_content) self.assertIn( 'setting its status to ERROR', From 85708251bb623ed2ba55e2158adcb5c85838393a Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 03:21:27 +0300 Subject: [PATCH 1945/2107] fix for forkname detection in get_backup_filelist --- src/catalog.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 80cdacdc5..60bf4184d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1138,14 +1138,18 @@ get_backup_filelist(pgBackup *backup, bool strict) else file->uncompressed_size = write_size; - if (file->external_dir_num == 0) + if (file->external_dir_num == 0 && S_ISREG(file->mode)) { bool is_datafile = file->is_datafile; set_forkname(file); if (is_datafile != file->is_datafile) { - elog(WARNING, "File '%s' was stored as datafile, but looks like it is not", - file->rel_path); + if (is_datafile) + elog(WARNING, "File '%s' was stored as datafile, but looks like it is not", + file->rel_path); + else + elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is", + file->rel_path); /* Lets fail in tests */ Assert(file->is_datafile == file->is_datafile); file->is_datafile = is_datafile; From 03f210b2becaa1577e950a8b49258a008a3644d0 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 18 Oct 2022 12:51:57 +0300 Subject: [PATCH 1946/2107] [PBCKP-304] cfs tests moved back to build --- tests/__init__.py | 6 +++--- tests/cfs_backup.py | 12 +++++++----- tests/cfs_restore.py | 9 ++++++--- tests/helpers/ptrack_helpers.py | 25 +++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 79537ad78..c02788e29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,9 +35,9 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(compatibility)) suite.addTests(loader.loadTestsFromModule(checkdb)) suite.addTests(loader.loadTestsFromModule(config)) -# suite.addTests(loader.loadTestsFromModule(cfs_backup)) -# suite.addTests(loader.loadTestsFromModule(cfs_restore)) -# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) + suite.addTests(loader.loadTestsFromModule(cfs_backup)) + suite.addTests(loader.loadTestsFromModule(cfs_restore)) + suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) suite.addTests(loader.loadTestsFromModule(compression)) suite.addTests(loader.loadTestsFromModule(delete)) suite.addTests(loader.loadTestsFromModule(delta)) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 436db31e7..8e625e534 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -4,7 +4,7 @@ import shutil from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, is_test_result_ok module_name = 'cfs_backup' tblspace_name = 'cfs_tblspace' @@ -1159,10 +1159,12 @@ def test_broken_file_pg_compression_into_tablespace_dir(self): ) # # --- End ---# -# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') -# def tearDown(self): -# self.node.cleanup() -# self.del_test_dir(module_name, self.fname) + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def tearDown(self): + module_name = self.id().split('.')[1] + fname = self.id().split('.')[3] + if is_test_result_ok(self): + self.del_test_dir(module_name, fname) #class CfsBackupEncTest(CfsBackupNoEncTest): diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 611afc49e..0b1bb886f 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -13,7 +13,7 @@ import shutil from .helpers.cfs_helpers import find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, is_test_result_ok module_name = 'cfs_restore' @@ -60,9 +60,12 @@ def setUp(self): def add_data_in_cluster(self): pass + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def tearDown(self): - self.node.cleanup() - self.del_test_dir(module_name, self.fname) + module_name = self.id().split('.')[1] + fname = self.id().split('.')[3] + if is_test_result_ok(self): + self.del_test_dir(module_name, fname) class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index d800f0d3e..e19cde7d0 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,6 +1,7 @@ # you need os for unittest to work import os import gc +import unittest from sys import exit, argv, version_info import subprocess import shutil @@ -172,6 +173,30 @@ def slow_start(self, replica=False): sleep(0.5) + +def is_test_result_ok(test_case): + # sources of solution: + # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: + # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 + # + # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 + + if hasattr(test_case, '_outcome'): # Python 3.4+ + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ + result = test_case._outcome.result + else: # Python 2.7, 3.0-3.3 + result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + + ok = all(test != test_case for test, text in result.errors + result.failures) + + return ok + + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() From fc8b89079b83ce221ac7327a4e74e91fb1033ffa Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 25 Oct 2022 12:24:22 +0300 Subject: [PATCH 1947/2107] [PBCKP-304] fix cfs_restore test --- tests/cfs_restore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 0b1bb886f..cadbff8a1 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -105,8 +105,7 @@ def test_restore_empty_tablespace_from_fullbackup(self): tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ) - tblspace = str(tblspace) + ).decode("UTF-8") self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not restored or it restored without compressions" From bc945994def2b41034f8a29f2470b98e6c81d1f7 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 25 Oct 2022 12:27:01 +0300 Subject: [PATCH 1948/2107] [PBCKP-304] auto tests cleanup added to ptrack_helper.py --- tests/cfs_backup.py | 14 +++----------- tests/cfs_restore.py | 21 +++++---------------- tests/helpers/ptrack_helpers.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 8e625e534..4509b7e7b 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -4,9 +4,8 @@ import shutil from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, is_test_result_ok +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_backup' tblspace_name = 'cfs_tblspace' @@ -14,11 +13,10 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): # --- Begin --- # @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def setUp(self): - self.fname = self.id().split('.')[3] self.backup_dir = os.path.join( - self.tmp_path, module_name, self.fname, 'backup') + self.tmp_path, self.module_name, self.fname, 'backup') self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, self.fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1159,12 +1157,6 @@ def test_broken_file_pg_compression_into_tablespace_dir(self): ) # # --- End ---# - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def tearDown(self): - module_name = self.id().split('.')[1] - fname = self.id().split('.')[3] - if is_test_result_ok(self): - self.del_test_dir(module_name, fname) #class CfsBackupEncTest(CfsBackupNoEncTest): diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index cadbff8a1..660cef9c6 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -13,10 +13,7 @@ import shutil from .helpers.cfs_helpers import find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, is_test_result_ok - - -module_name = 'cfs_restore' +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException tblspace_name = 'cfs_tblspace' tblspace_name_new = 'cfs_tblspace_new' @@ -24,11 +21,10 @@ class CfsRestoreBase(ProbackupTest, unittest.TestCase): def setUp(self): - self.fname = self.id().split('.')[3] - self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, self.fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -60,13 +56,6 @@ def setUp(self): def add_data_in_cluster(self): pass - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def tearDown(self): - module_name = self.id().split('.')[1] - fname = self.id().split('.')[3] - if is_test_result_ok(self): - self.del_test_dir(module_name, fname) - class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): # @unittest.expectedFailure @@ -214,7 +203,7 @@ def test_restore_from_fullbackup_to_new_location(self): self.node.cleanup() shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) node_new.cleanup() try: @@ -257,7 +246,7 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): self.node.cleanup() shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) node_new.cleanup() try: diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e19cde7d0..bd5ea01fd 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -204,6 +204,11 @@ class ProbackupTest(object): def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) + + if isinstance(self, unittest.TestCase): + self.module_name = self.id().split('.')[1] + self.fname = self.id().split('.')[3] + if '-v' in argv or '--verbose' in argv: self.verbose = True else: @@ -367,6 +372,13 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" + def tearDown(self): + if isinstance(self, unittest.TestCase): + module_name = self.id().split('.')[1] + fname = self.id().split('.')[3] + if is_test_result_ok(self): + self.del_test_dir(module_name, fname) + @property def pg_config_version(self): return self.version_to_num( From 693bffe08ded725de24569a8fa20b6e16b547d6e Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 11:47:08 +0300 Subject: [PATCH 1949/2107] [PBCKP-146] fix cfm truncated crc calculation in delta/page backup - On backup we should compare truncated crc with previous version. - copying remote file didn't honor "don't truncate first 64 bytes" rule. - crc calculation didn't honoer "don't truncate first 64 bytes" rule. --- src/data.c | 7 +++++-- src/pg_probackup.h | 2 +- src/utils/file.c | 49 +++++++++++++++++++++++++++++++++++++--------- src/utils/file.h | 3 ++- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/data.c b/src/data.c index 2a8806cde..08727d41c 100644 --- a/src/data.c +++ b/src/data.c @@ -806,7 +806,10 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, * file could be deleted under our feets. * But then backup_non_data_file_internal will handle it safely */ - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); + if (file->forkName != cfm) + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); + else + file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) @@ -1334,7 +1337,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, pg_crc32 file_crc; if (tmp_file->forkName == cfm && tmp_file->uncompressed_size > tmp_file->write_size) - file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST); + file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false); else file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index f2201ebdd..6aeba189e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1082,7 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c); +extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok); extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); diff --git a/src/utils/file.c b/src/utils/file.c index 627fbbad7..c4ed9c721 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1377,8 +1377,6 @@ fio_get_crc32_ex(const char *file_path, fio_location location, { if (decompress && truncated) elog(ERROR, "Could not calculate CRC for compressed truncated file"); - if (missing_ok && truncated) - elog(ERROR, "CRC calculation for missing truncated file is forbidden"); if (fio_is_remote(location)) { @@ -1408,7 +1406,7 @@ fio_get_crc32_ex(const char *file_path, fio_location location, if (decompress) return pgFileGetCRCgz(file_path, true, missing_ok); else if (truncated) - return pgFileGetCRCTruncated(file_path, true); + return pgFileGetCRCTruncated(file_path, true, missing_ok); else return pgFileGetCRC(file_path, true, missing_ok); } @@ -1422,9 +1420,10 @@ fio_get_crc32(const char *file_path, fio_location location, } pg_crc32 -fio_get_crc32_truncated(const char *file_path, fio_location location) +fio_get_crc32_truncated(const char *file_path, fio_location location, + bool missing_ok) { - return fio_get_crc32_ex(file_path, location, false, false, true); + return fio_get_crc32_ex(file_path, location, false, missing_ok, true); } /* Remove file */ @@ -3003,6 +3002,7 @@ fio_send_file_impl(int out, char const* path) fio_header hdr; char *buf = pgut_malloc(CHUNK_SIZE); size_t read_len = 0; + int64_t read_size = 0; char *errormsg = NULL; /* open source file for read */ @@ -3066,7 +3066,19 @@ fio_send_file_impl(int out, char const* path) if (read_len > 0) { /* send chunk */ - size_t non_zero_len = find_zero_tail(buf, read_len); + int64_t non_zero_len = find_zero_tail(buf, read_len); + /* + * It is dirty trick to silence warnings in CFS GC process: + * backup at least cfs header size bytes. + */ + if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + read_size + read_len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + read_size + read_len); + non_zero_len -= read_size; + } + if (non_zero_len > 0) { hdr.cop = FIO_PAGE; @@ -3082,6 +3094,8 @@ fio_send_file_impl(int out, char const* path) hdr.arg = read_len - non_zero_len; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } + + read_size += read_len; } if (feof(fp)) @@ -3166,7 +3180,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) * Read the local file to compute CRC for it extened to real_size. */ pg_crc32 -pgFileGetCRCTruncated(const char *file_path, bool use_crc32c) +pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok) { FILE *fp; char *buf; @@ -3180,6 +3194,15 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c) fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, st.crc); + return st.crc; + } + } + elog(ERROR, "Cannot open file \"%s\": %s", file_path, strerror(errno)); } @@ -3200,6 +3223,14 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c) elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); non_zero_len = find_zero_tail(buf, len); + /* same trick as in fio_send_file */ + if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + st.read_size + len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + st.read_size + len); + non_zero_len -= st.read_size; + } if (non_zero_len) { fio_send_file_crc(&st, buf, non_zero_len); @@ -3894,12 +3925,12 @@ fio_communicate(int in, int out) break; case FIO_GET_CRC32: Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 || - (hdr.arg & GET_CRC32_TRUNCATED) == GET_CRC32_TRUNCATED); + (hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED); /* calculate crc32 for a file */ if ((hdr.arg & GET_CRC32_DECOMPRESS)) crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); else if ((hdr.arg & GET_CRC32_TRUNCATED)) - crc = pgFileGetCRCTruncated(buf, true); + crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); else crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); diff --git a/src/utils/file.h b/src/utils/file.h index 621a4bf9f..01e5a24f4 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -123,7 +123,8 @@ extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress, bool missing_ok); -extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location); +extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location, + bool missing_ok); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); From 7cadc3378c11250c393dfe7415a0e28b8fc32ccf Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 11:48:46 +0300 Subject: [PATCH 1950/2107] [PBCKP-146] fix filesize filling file->size were not filled while reading backup filelist. That lead to excess non-data file backups. --- src/catalog.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/catalog.c b/src/catalog.c index 60bf4184d..488d7349f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1137,6 +1137,8 @@ get_backup_filelist(pgBackup *backup, bool strict) file->uncompressed_size = uncompressed_size; else file->uncompressed_size = write_size; + if (!file->is_datafile || file->is_cfs) + file->size = file->uncompressed_size; if (file->external_dir_num == 0 && S_ISREG(file->mode)) { From b17669c96920ac8056454d61595412caca12cd57 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 3 Nov 2022 11:49:33 +0300 Subject: [PATCH 1951/2107] [PBCKP-146] add test for "unchanged cfm is not backuped" --- tests/cfs_backup.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/cfs_backup.py b/tests/cfs_backup.py index 306c2396c..fe2af20e3 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup.py @@ -419,6 +419,69 @@ def test_fullbackup_empty_tablespace_page_after_create_table(self): "ERROR: .cfm files not found in backup dir" ) + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_page_doesnt_store_unchanged_cfm(self): + """ + Case: Test page backup doesn't store cfm file if table were not modified + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files is found in backup dir" + ) + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') From 9f3f530ec7999b47e49763217f6bb62c6cfd5d5c Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Sun, 23 Oct 2022 05:30:13 +0300 Subject: [PATCH 1952/2107] [PBCKP-304] extracted all cleanup routines to ptrack.py.TearDown() --- tests/CVE_2018_1058.py | 24 +-- tests/archive.py | 280 ++++++++----------------- tests/auth_test.py | 11 +- tests/backup.py | 434 +++++++++++---------------------------- tests/catchup.py | 138 +++++-------- tests/checkdb.py | 39 ++-- tests/compatibility.py | 158 +++++--------- tests/compression.py | 51 ++--- tests/config.py | 14 +- tests/delete.py | 102 +++------ tests/delta.py | 145 ++++--------- tests/exclude.py | 54 ++--- tests/external.py | 286 ++++++++------------------ tests/false_positive.py | 66 ++---- tests/incr_restore.py | 305 ++++++++------------------- tests/init.py | 35 +--- tests/locking.py | 72 ++----- tests/logging.py | 56 ++--- tests/merge.py | 287 ++++++++------------------ tests/option.py | 20 +- tests/page.py | 155 ++++---------- tests/pgpro2068.py | 13 +- tests/pgpro560.py | 20 +- tests/pgpro589.py | 11 +- tests/ptrack.py | 430 +++++++++++++------------------------- tests/remote.py | 11 +- tests/replica.py | 220 +++++++------------- tests/restore.py | 444 ++++++++++++---------------------------- tests/retention.py | 212 ++++++------------- tests/set_backup.py | 65 ++---- tests/show.py | 83 ++------ tests/time_consuming.py | 9 +- tests/time_stamp.py | 41 +--- tests/validate.py | 334 ++++++++---------------------- 34 files changed, 1321 insertions(+), 3304 deletions(-) diff --git a/tests/CVE_2018_1058.py b/tests/CVE_2018_1058.py index 3da41f116..cfd55cc60 100644 --- a/tests/CVE_2018_1058.py +++ b/tests/CVE_2018_1058.py @@ -2,17 +2,14 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'CVE-2018-1058' - class CVE_2018_1058(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_basic_default_search_path(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True) self.init_pb(backup_dir) @@ -31,16 +28,12 @@ def test_basic_default_search_path(self): self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_backup_modified_search_path(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True) self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) @@ -77,15 +70,11 @@ def test_basic_backup_modified_search_path(self): self.assertFalse( 'pg_probackup vulnerable!' in log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_checkdb_modified_search_path(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) node.slow_start() @@ -138,6 +127,3 @@ def test_basic_checkdb_modified_search_path(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/archive.py b/tests/archive.py index 81d013f6b..f40cf3c5d 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -10,19 +10,15 @@ from distutils.dir_util import copy_tree -module_name = 'archive' - - class ArchiveTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_pgpro434_1(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -64,8 +60,6 @@ def test_pgpro434_1(self): self.assertEqual( result, node.safe_psql("postgres", "SELECT * FROM t_heap"), 'data after restore not equal to original data') - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -74,10 +68,9 @@ def test_pgpro434_2(self): Check that timelines are correct. WAITING PGPRO-1053 for --immediate """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -85,7 +78,7 @@ def test_pgpro434_2(self): ) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') @@ -219,9 +212,6 @@ def test_pgpro434_2(self): "SELECT * FROM t_heap"), 'data after restore not equal to original data') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro434_3(self): """ @@ -230,10 +220,9 @@ def test_pgpro434_3(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -284,9 +273,6 @@ def test_pgpro434_3(self): log_content, 'PostgreSQL crashed because of a failed assert') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro434_4(self): """ @@ -295,10 +281,9 @@ def test_pgpro434_4(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -361,16 +346,12 @@ def test_pgpro434_4(self): log_content, 'PostgreSQL crashed because of a failed assert') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_file_exists(self): """Archive-push if file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -454,16 +435,12 @@ def test_archive_push_file_exists(self): print(log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_file_exists_overwrite(self): """Archive-push if file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'checkpoint_timeout': '30s'}) @@ -530,16 +507,12 @@ def test_archive_push_file_exists_overwrite(self): 'WAL file already exists in archive with ' 'different checksum, overwriting', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): """Archive-push if stale '.part' file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -613,16 +586,12 @@ def test_archive_push_partial_file_exists(self): 'Reusing stale temp WAL file', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_part_file_exists_not_stale(self): """Archive-push if .part file exists and it is not stale""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -693,9 +662,6 @@ def test_archive_push_part_file_exists_not_stale(self): # 'is not stale', # log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): @@ -704,10 +670,9 @@ def test_replica_archive(self): turn it into replica, set replica with archiving, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -716,7 +681,7 @@ def test_replica_archive(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -726,7 +691,7 @@ def test_replica_archive(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.psql( @@ -775,7 +740,7 @@ def test_replica_archive(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -824,9 +789,6 @@ def test_replica_archive(self): after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_master_and_replica_parallel_archiving(self): @@ -836,10 +798,9 @@ def test_master_and_replica_parallel_archiving(self): set replica with archiving, make archive backup from replica, make archive backup from master """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -847,12 +808,12 @@ def test_master_and_replica_parallel_archiving(self): ) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -916,9 +877,6 @@ def test_master_and_replica_parallel_archiving(self): self.assertEqual( 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_basic_master_and_replica_concurrent_archiving(self): @@ -931,10 +889,9 @@ def test_basic_master_and_replica_concurrent_archiving(self): if self.pg_config_version < self.version_to_num('9.6.0'): return unittest.skip('You need PostgreSQL >= 9.6 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -942,12 +899,12 @@ def test_basic_master_and_replica_concurrent_archiving(self): 'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -1020,10 +977,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.backup_node(backup_dir, 'master', master) self.backup_node(backup_dir, 'master', replica) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.expectedFailure # @unittest.skip("skip") def test_concurrent_archiving(self): @@ -1037,10 +990,9 @@ def test_concurrent_archiving(self): if self.pg_config_version < self.version_to_num('11.0'): return unittest.skip('You need PostgreSQL >= 11 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) @@ -1056,7 +1008,7 @@ def test_concurrent_archiving(self): # Settings for Replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -1067,7 +1019,7 @@ def test_concurrent_archiving(self): # create cascade replicas replica1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica1')) + base_dir=os.path.join(self.module_name, self.fname, 'replica1')) replica1.cleanup() # Settings for casaced replica @@ -1103,17 +1055,13 @@ def test_concurrent_archiving(self): log_content = f.read() self.assertNotIn('different checksum', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog(self): """Test backup with pg_receivexlog wal delivary method""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1177,16 +1125,14 @@ def test_archive_pg_receivexlog(self): # Clean after yourself pg_receivexlog.kill() - self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog_compression_pg10(self): """Test backup with pg_receivewal compressed wal delivary method""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1245,7 +1191,6 @@ def test_archive_pg_receivexlog_compression_pg10(self): # Clean after yourself pg_receivexlog.kill() - self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") @@ -1266,10 +1211,9 @@ def test_archive_catalog(self): ARCHIVE master: t1 -Z1--Z2--- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1277,7 +1221,7 @@ def test_archive_catalog(self): 'checkpoint_timeout': '30s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1307,7 +1251,7 @@ def test_archive_catalog(self): backup_dir, 'master', master, backup_type='page') replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) @@ -1576,8 +1520,6 @@ def test_archive_catalog(self): self.assertEqual(timeline_2['parent-tli'], 1) self.assertEqual(timeline_1['parent-tli'], 0) - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_catalog_1(self): @@ -1588,10 +1530,9 @@ def test_archive_catalog_1(self): self.skipTest('You need to enable ARCHIVE_COMPRESSION ' 'for this test to run') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1631,8 +1572,6 @@ def test_archive_catalog_1(self): '000000010000000000000001') self.assertEqual(timeline['status'], 'OK') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_catalog_2(self): @@ -1643,10 +1582,9 @@ def test_archive_catalog_2(self): self.skipTest('You need to enable ARCHIVE_COMPRESSION ' 'for this test to run') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1688,8 +1626,6 @@ def test_archive_catalog_2(self): '000000010000000000000002') self.assertEqual(timeline['status'], 'OK') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_options(self): @@ -1700,10 +1636,9 @@ def test_archive_options(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1766,8 +1701,6 @@ def test_archive_options(self): 'postgres', 'select 1') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_options_1(self): @@ -1775,10 +1708,9 @@ def test_archive_options_1(self): check that '--archive-host', '--archive-user', '--archiver-port' and '--restore-command' are working as expected with set-config """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1837,8 +1769,6 @@ def test_archive_options_1(self): self.probackup_path, backup_dir, 'node', self.user), recovery_content) - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_undefined_wal_file_path(self): @@ -1846,10 +1776,9 @@ def test_undefined_wal_file_path(self): check that archive-push works correct with undefined --wal-file-path """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1879,19 +1808,16 @@ def test_undefined_wal_file_path(self): # check self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_intermediate_archiving(self): """ check that archive-push works correct with --wal-file-path setting by user """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) node_pg_options = {} @@ -1904,7 +1830,7 @@ def test_intermediate_archiving(self): self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - wal_dir = os.path.join(self.tmp_path, module_name, fname, 'intermediate_dir') + wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'intermediate_dir') shutil.rmtree(wal_dir, ignore_errors=True) os.makedirs(wal_dir) if os.name == 'posix': @@ -1929,8 +1855,6 @@ def test_intermediate_archiving(self): self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_waldir_outside_pgdata_archiving(self): @@ -1941,13 +1865,12 @@ def test_waldir_outside_pgdata_archiving(self): return unittest.skip( 'Skipped because waldir outside pgdata is supported since PG 10') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - external_wal_dir = os.path.join(self.tmp_path, module_name, fname, 'ext_wal_dir') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') shutil.rmtree(external_wal_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) self.init_pb(backup_dir) @@ -1964,18 +1887,15 @@ def test_waldir_outside_pgdata_archiving(self): # check self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_hexadecimal_timeline(self): """ Check that timelines are correct. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2023,9 +1943,6 @@ def test_hexadecimal_timeline(self): '0000000D000000000000001C', tli13['max-segno']) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_archiving_and_slots(self): @@ -2033,10 +1950,9 @@ def test_archiving_and_slots(self): Check that archiving don`t break slot guarantee. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -2087,15 +2003,11 @@ def test_archiving_and_slots(self): exit(1) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_push_sanity(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -2125,14 +2037,14 @@ def test_archive_push_sanity(self): self.assertNotIn('WARNING', postgres_log_content) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( backup_dir, 'node', replica, data_dir=replica.data_dir, options=['-R']) - #self.set_archiving(backup_dir, 'replica', replica, replica=True) + # self.set_archiving(backup_dir, 'replica', replica, replica=True) self.set_auto_conf(replica, {'port': replica.port}) self.set_auto_conf(replica, {'archive_mode': 'always'}) self.set_auto_conf(replica, {'hot_standby': 'on'}) @@ -2160,22 +2072,18 @@ def test_archive_push_sanity(self): self.assertNotIn('WARNING', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog_partial_handling(self): """check that archive-get delivers .partial and .gz.partial files""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -2234,7 +2142,7 @@ def test_archive_pg_receivexlog_partial_handling(self): pg_receivexlog.kill() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2255,16 +2163,12 @@ def test_archive_pg_receivexlog_partial_handling(self): self.assertEqual(result, result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2367,24 +2271,20 @@ def test_multi_timeline_recovery_prefetching(self): 'WAL segment 000000010000000000000006, prefetch state: 5/10', postgres_log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_get_batching_sanity(self): """ Make sure that batching works. .gz file is corrupted and uncompressed is not, check that both corruption detected and uncompressed file is used. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -2399,7 +2299,7 @@ def test_archive_get_batching_sanity(self): node.pgbench_init(scale=50) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( @@ -2440,18 +2340,14 @@ def test_archive_get_batching_sanity(self): self.assertIn('prefetch state: 9/10', postgres_log_content) self.assertIn('prefetch state: 8/10', postgres_log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_get_prefetch_corruption(self): """ Make sure that WAL corruption is detected. And --prefetch-dir is honored. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2466,7 +2362,7 @@ def test_archive_get_prefetch_corruption(self): node.pgbench_init(scale=50) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( @@ -2576,19 +2472,15 @@ def test_archive_get_prefetch_corruption(self): 'LOG: restored log file "{0}" from archive'.format(filename), postgres_log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_show_partial_files_handling(self): """ check that files with '.part', '.part.gz', '.partial' and '.partial.gz' siffixes are handled correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2707,19 +2599,15 @@ def test_archive_show_partial_files_handling(self): 'WARNING', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_empty_history_file(self): """ https://github.com/postgrespro/pg_probackup/issues/326 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2800,8 +2688,6 @@ def test_archive_empty_history_file(self): 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), log_content) - self.del_test_dir(module_name, fname) - # TODO test with multiple not archived segments. # TODO corrupted file in archive. diff --git a/tests/auth_test.py b/tests/auth_test.py index 39786d7a9..4b0c4a5b2 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -30,13 +30,12 @@ def test_backup_via_unprivileged_user(self): run a backups without EXECUTE rights on certain functions """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -146,7 +145,6 @@ def test_backup_via_unprivileged_user(self): "postgres", "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup") - self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -176,14 +174,12 @@ def test_backup_via_unprivileged_user(self): # backup_dir, 'node', node, # backup_type='ptrack', options=['-U', 'backup']) - # Clean after yourself - self.del_test_dir(module_name, fname) - class AuthTest(unittest.TestCase): pb = None node = None + # TODO move to object scope, replace module_name @classmethod def setUpClass(cls): @@ -240,6 +236,7 @@ def setUpClass(cls): cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + # TODO move to object scope, replace module_name @classmethod def tearDownClass(cls): cls.node.cleanup() diff --git a/tests/backup.py b/tests/backup.py index 4f447c9bd..b7bb1b8b4 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -8,9 +8,6 @@ import subprocess -module_name = 'backup' - - class BackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -18,12 +15,11 @@ class BackupTest(ProbackupTest, unittest.TestCase): # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -80,18 +76,14 @@ def test_backup_modes_archive(self): backup_dir, 'node', backup_id=show_backup_2['id'])["parent-backup-id"]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -103,18 +95,14 @@ def test_smooth_checkpoint(self): self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") node.stop() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incremental_backup_without_full(self): """page backup without validated full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -139,18 +127,14 @@ def test_incremental_backup_without_full(self): self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -200,19 +184,15 @@ def test_incremental_backup_corrupt_full(self): self.assertEqual( self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_threads_stream(self): """delta multi thread backup mode and stream""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -227,21 +207,17 @@ def test_delta_threads_stream(self): backup_type="delta", options=["-j", "4", "--stream"]) self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -292,21 +268,16 @@ def test_page_detect_corruption(self): 'ERROR', "Backup Status should be ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_backup_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -439,20 +410,16 @@ def test_backup_detect_corruption(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_detect_invalid_block_header(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -580,20 +547,16 @@ def test_backup_detect_invalid_block_header(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_detect_missing_permissions(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -721,22 +684,18 @@ def test_backup_detect_missing_permissions(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_truncate_misaligned(self): """ make node, truncate file to size not even to BLCKSIZE, take backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -772,19 +731,15 @@ def test_backup_truncate_misaligned(self): self.assertIn("WARNING: File", output) self.assertIn("invalid file size", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_tablespace_in_pgdata_pgpro_1376(self): """PGPRO-1376 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -866,9 +821,6 @@ def test_tablespace_in_pgdata_pgpro_1376(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_tablespace_handling(self): """ @@ -877,13 +829,12 @@ def test_basic_tablespace_handling(self): check that restore with tablespace mapping will end with success """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -925,7 +876,7 @@ def test_basic_tablespace_handling(self): tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -979,22 +930,18 @@ def test_basic_tablespace_handling(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_tablespace_handling_1(self): """ make node with tablespace A, take full backup, check that restore with tablespace mapping of tablespace B will end with error """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1014,7 +961,7 @@ def test_tablespace_handling_1(self): options=["-j", "4", "--stream"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1037,22 +984,18 @@ def test_tablespace_handling_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_tablespace_handling_2(self): """ make node without tablespaces, take full backup, check that restore with tablespace mapping will end with error """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1066,7 +1009,7 @@ def test_tablespace_handling_2(self): options=["-j", "4", "--stream"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1089,18 +1032,14 @@ def test_tablespace_handling_2(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_full_backup(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1172,16 +1111,12 @@ def test_drop_rel_during_full_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_drop_db_during_full_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1240,18 +1175,14 @@ def test_drop_db_during_full_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1311,18 +1242,14 @@ def test_drop_rel_during_backup_delta(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_backup_page(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1379,16 +1306,12 @@ def test_drop_rel_during_backup_page(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_persistent_slot_for_stream_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1413,16 +1336,12 @@ def test_persistent_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--slot=slot_1']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_temp_slot_for_stream_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'max_wal_size': '40MB'}) @@ -1445,18 +1364,14 @@ def test_basic_temp_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--slot=slot_1', '--temp-slot']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1493,19 +1408,15 @@ def test_backup_concurrent_drop_table(self): self.assertEqual(show_backup['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): """""" if self.pg_config_version < self.version_to_num('11.0'): return unittest.skip('You need PostgreSQL >= 11 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', @@ -1581,18 +1492,14 @@ def test_pg_11_adjusted_wal_segment_size(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigint_handling(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1622,18 +1529,14 @@ def test_sigint_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigterm_handling(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1662,18 +1565,14 @@ def test_sigterm_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigquit_handling(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1701,16 +1600,12 @@ def test_sigquit_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_table(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1736,19 +1631,15 @@ def test_drop_table(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_missing_file_permissions(self): """""" if os.name == 'nt': return unittest.skip('Skipped because it is POSIX only test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1783,19 +1674,15 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 700) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): """""" if os.name == 'nt': return unittest.skip('Skipped because it is POSIX only test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1826,16 +1713,12 @@ def test_basic_missing_dir_permissions(self): os.rmdir(full_path) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -2055,9 +1938,6 @@ def test_backup_with_least_privileges_role(self): backup_dir, 'node', node, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing(self): """ @@ -2066,10 +1946,9 @@ def test_parent_choosing(self): PAGE1 <- CORRUPT FULL """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2119,9 +1998,6 @@ def test_parent_choosing(self): backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], full_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing_1(self): """ @@ -2130,10 +2006,9 @@ def test_parent_choosing_1(self): PAGE1 <- (missing) FULL """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2179,9 +2054,6 @@ def test_parent_choosing_1(self): backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], full_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing_2(self): """ @@ -2190,10 +2062,9 @@ def test_parent_choosing_2(self): PAGE1 <- OK FULL <- (missing) """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2239,19 +2110,15 @@ def test_parent_choosing_2(self): backup_dir, 'node')[2]['status'], 'ERROR') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_with_less_privileges_role(self): """ check permissions correctness from documentation: https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -2386,12 +2253,12 @@ def test_backup_with_less_privileges_role(self): datname='backupdb', options=['--stream', '-U', 'backup']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return # Restore as replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -2456,18 +2323,14 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_issue_132(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2494,18 +2357,14 @@ def test_issue_132(self): exit(1) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_issue_132_1(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2654,17 +2513,13 @@ def test_issue_132_1(self): 'INFO: Restore of backup {0} completed.'.format(delta_id), output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_note_sanity(self): """ test that adding note to backup works as expected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2692,18 +2547,14 @@ def test_note_sanity(self): 'note', backup_meta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_backup_made_by_newer_version(self): """incremental backup with parent made by newer version""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2745,20 +2596,16 @@ def test_parent_backup_made_by_newer_version(self): self.assertEqual( self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_issue_289(self): """ https://github.com/postgrespro/pg_probackup/issues/289 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2790,20 +2637,16 @@ def test_issue_289(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_issue_290(self): """ https://github.com/postgrespro/pg_probackup/issues/290 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2839,18 +2682,14 @@ def test_issue_290(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_issue_203(self): """ https://github.com/postgrespro/pg_probackup/issues/203 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2870,7 +2709,7 @@ def test_issue_203(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', @@ -2879,18 +2718,14 @@ def test_issue_203(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_issue_231(self): """ https://github.com/postgrespro/pg_probackup/issues/231 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2911,17 +2746,13 @@ def test_issue_231(self): # it is a bit racy self.assertIn("WARNING: Cannot create directory", out) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incr_backup_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2930,7 +2761,7 @@ def test_incr_backup_filenode_map(self): node.slow_start() node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), initdb_params=['--data-checksums']) node1.cleanup() @@ -2962,18 +2793,14 @@ def test_incr_backup_filenode_map(self): 'postgres', 'select 1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_missing_wal_segment(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -3039,16 +2866,12 @@ def test_missing_wal_segment(self): # TODO: check the same for PAGE backup - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_missing_replication_permission(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3063,7 +2886,7 @@ def test_missing_replication_permission(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -3193,16 +3016,12 @@ def test_missing_replication_permission(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_missing_replication_permission_1(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3217,7 +3036,7 @@ def test_missing_replication_permission_1(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -3347,16 +3166,12 @@ def test_missing_replication_permission_1(self): r'WARNING: could not connect to database backupdb: (connection to server (on socket "/tmp/.s.PGSQL.\d+"|at "localhost" \(127.0.0.1\), port \d+) failed: ){0,1}' 'FATAL: must be superuser or replication role to start walsender') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_backup_default_transaction_read_only(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'default_transaction_read_only': 'on'}) @@ -3395,18 +3210,14 @@ def test_basic_backup_default_transaction_read_only(self): # PAGE backup self.backup_node(backup_dir, 'node', node, backup_type='page') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_atexit(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3450,16 +3261,12 @@ def test_backup_atexit(self): 'setting its status to ERROR', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pg_stop_backup_missing_permissions(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3529,20 +3336,16 @@ def test_pg_stop_backup_missing_permissions(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_start_time(self): """Test, that option --start-time allows to set backup_id and restore""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3556,7 +3359,7 @@ def test_start_time(self): # restore FULL backup by backup_id calculated from start-time self.restore_node( backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_full'), + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_full'), backup_id=base36enc(startTime)) #FULL backup with incorrect start time @@ -3586,7 +3389,7 @@ def test_start_time(self): # restore DELTA backup by backup_id calculated from start-time self.restore_node( backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_delta'), + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_delta'), backup_id=base36enc(startTime)) # PAGE backup @@ -3597,7 +3400,7 @@ def test_start_time(self): # restore PAGE backup by backup_id calculated from start-time self.restore_node( backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_page'), + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_page'), backup_id=base36enc(startTime)) # PTRACK backup @@ -3613,35 +3416,31 @@ def test_start_time(self): # restore PTRACK backup by backup_id calculated from start-time self.restore_node( backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, module_name, fname, 'node_restored_ptrack'), + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_ptrack'), backup_id=base36enc(startTime)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_start_time_few_nodes(self): """Test, that we can synchronize backup_id's for different DBs""" - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir1 = os.path.join(self.tmp_path, module_name, fname, 'backup1') + backup_dir1 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup1') self.init_pb(backup_dir1) self.add_instance(backup_dir1, 'node1', node1) self.set_archiving(backup_dir1, 'node1', node1) node1.slow_start() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), + base_dir=os.path.join(self.module_name, self.fname, 'node2'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir2 = os.path.join(self.tmp_path, module_name, fname, 'backup2') + backup_dir2 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup2') self.init_pb(backup_dir2) self.add_instance(backup_dir2, 'node2', node2) self.set_archiving(backup_dir2, 'node2', node2) @@ -3703,6 +3502,3 @@ def test_start_time_few_nodes(self): show_backup2 = self.show_pb(backup_dir2, 'node2')[3] self.assertEqual(show_backup1['id'], show_backup2['id']) - # Clean after yourself - self.del_test_dir(module_name, fname) - diff --git a/tests/catchup.py b/tests/catchup.py index 7ecd84697..12622207a 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -4,11 +4,7 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'catchup' - class CatchupTest(ProbackupTest, unittest.TestCase): - def setUp(self): - self.fname = self.id().split('.')[3] ######################################### # Basic tests @@ -19,7 +15,7 @@ def test_basic_full_catchup(self): """ # preparation src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() @@ -29,7 +25,7 @@ def test_basic_full_catchup(self): src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") # do full catchup - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -57,7 +53,7 @@ def test_basic_full_catchup(self): # Cleanup dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(module_name, self.fname) + self.del_test_dir(self.module_name, self.fname) def test_full_catchup_with_tablespace(self): """ @@ -65,7 +61,7 @@ def test_full_catchup_with_tablespace(self): """ # preparation src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() @@ -77,7 +73,7 @@ def test_full_catchup_with_tablespace(self): src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') self.catchup_node( backup_mode = 'FULL', @@ -115,7 +111,6 @@ def test_full_catchup_with_tablespace(self): # Cleanup dst_pg.stop() - self.del_test_dir(module_name, self.fname) def test_basic_delta_catchup(self): """ @@ -123,7 +118,7 @@ def test_basic_delta_catchup(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -133,7 +128,7 @@ def test_basic_delta_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -183,7 +178,6 @@ def test_basic_delta_catchup(self): # Cleanup dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(module_name, self.fname) def test_basic_ptrack_catchup(self): """ @@ -194,7 +188,7 @@ def test_basic_ptrack_catchup(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, initdb_params = ['--data-checksums'] @@ -206,7 +200,7 @@ def test_basic_ptrack_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -256,7 +250,6 @@ def test_basic_ptrack_catchup(self): # Cleanup dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(module_name, self.fname) def test_tli_delta_catchup(self): """ @@ -264,14 +257,14 @@ def test_tli_delta_catchup(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -329,7 +322,6 @@ def test_tli_delta_catchup(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_tli_ptrack_catchup(self): """ @@ -340,7 +332,7 @@ def test_tli_ptrack_catchup(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, initdb_params = ['--data-checksums'] @@ -349,7 +341,7 @@ def test_tli_ptrack_catchup(self): src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -412,7 +404,6 @@ def test_tli_ptrack_catchup(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) ######################################### # Test various corner conditions @@ -423,7 +414,7 @@ def test_table_drop_with_delta(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -433,7 +424,7 @@ def test_table_drop_with_delta(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -468,7 +459,6 @@ def test_table_drop_with_delta(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_table_drop_with_ptrack(self): """ @@ -479,7 +469,7 @@ def test_table_drop_with_ptrack(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, initdb_params = ['--data-checksums'] @@ -491,7 +481,7 @@ def test_table_drop_with_ptrack(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -526,7 +516,6 @@ def test_table_drop_with_ptrack(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_tablefile_truncation_with_delta(self): """ @@ -534,7 +523,7 @@ def test_tablefile_truncation_with_delta(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -549,7 +538,7 @@ def test_tablefile_truncation_with_delta(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -583,7 +572,6 @@ def test_tablefile_truncation_with_delta(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_tablefile_truncation_with_ptrack(self): """ @@ -594,7 +582,7 @@ def test_tablefile_truncation_with_ptrack(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, initdb_params = ['--data-checksums'] @@ -611,7 +599,7 @@ def test_tablefile_truncation_with_ptrack(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -645,7 +633,6 @@ def test_tablefile_truncation_with_ptrack(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) ######################################### # Test reaction on user errors @@ -657,7 +644,7 @@ def test_local_tablespace_without_mapping(self): if self.remote: return unittest.skip('Skipped because this test tests local catchup error handling') - src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'src')) + src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'src')) src_pg.slow_start() tblspace_path = self.get_tblspace_path(src_pg, 'tblspace') @@ -669,7 +656,7 @@ def test_local_tablespace_without_mapping(self): "postgres", "CREATE TABLE ultimate_question TABLESPACE tblspace AS SELECT 42 AS answer") - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) try: self.catchup_node( backup_mode = 'FULL', @@ -691,7 +678,6 @@ def test_local_tablespace_without_mapping(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_running_dest_postmaster(self): """ @@ -699,14 +685,14 @@ def test_running_dest_postmaster(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -738,7 +724,6 @@ def test_running_dest_postmaster(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_same_db_id(self): """ @@ -747,12 +732,12 @@ def test_same_db_id(self): # preparation: # source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() # destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -765,9 +750,9 @@ def test_same_db_id(self): dst_pg.slow_start() dst_pg.stop() # fake destination - fake_dst_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'fake_dst')) + fake_dst_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_dst')) # fake source - fake_src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'fake_src')) + fake_src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_src')) # try delta catchup (src (with correct src conn), fake_dst) try: @@ -803,7 +788,6 @@ def test_same_db_id(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_tli_destination_mismatch(self): """ @@ -811,14 +795,14 @@ def test_tli_destination_mismatch(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -860,7 +844,6 @@ def test_tli_destination_mismatch(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_tli_source_mismatch(self): """ @@ -868,14 +851,14 @@ def test_tli_source_mismatch(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: fake source (promouted copy) - fake_src_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'fake_src')) + fake_src_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'fake_src')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -899,7 +882,7 @@ def test_tli_source_mismatch(self): fake_src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 'trash' AS garbage") # preparation 3: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -940,7 +923,6 @@ def test_tli_source_mismatch(self): # Cleanup src_pg.stop() fake_src_pg.stop() - self.del_test_dir(module_name, self.fname) ######################################### # Test unclean destination @@ -951,7 +933,7 @@ def test_unclean_delta_catchup(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -961,7 +943,7 @@ def test_unclean_delta_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1028,7 +1010,6 @@ def test_unclean_delta_catchup(self): # Cleanup dst_pg.stop() - self.del_test_dir(module_name, self.fname) def test_unclean_ptrack_catchup(self): """ @@ -1039,7 +1020,7 @@ def test_unclean_ptrack_catchup(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, pg_options = { 'wal_log_hints': 'on' } @@ -1051,7 +1032,7 @@ def test_unclean_ptrack_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1118,7 +1099,6 @@ def test_unclean_ptrack_catchup(self): # Cleanup dst_pg.stop() - self.del_test_dir(module_name, self.fname) ######################################### # Test replication slot logic @@ -1139,13 +1119,13 @@ def test_catchup_with_replication_slot(self): """ # preparation src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() # 1a. --slot option - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_1a')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1a')) try: self.catchup_node( backup_mode = 'FULL', @@ -1165,7 +1145,7 @@ def test_catchup_with_replication_slot(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 1b. --slot option - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_1b')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1b')) src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_1b')") self.catchup_node( backup_mode = 'FULL', @@ -1178,7 +1158,7 @@ def test_catchup_with_replication_slot(self): ) # 2a. --slot --perm-slot - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_2a')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2a')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1191,7 +1171,7 @@ def test_catchup_with_replication_slot(self): ) # 2b. and 4. --slot --perm-slot - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_2b')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2b')) src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_2b')") try: self.catchup_node( @@ -1213,7 +1193,7 @@ def test_catchup_with_replication_slot(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) # 3. --perm-slot --slot - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_3')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_3')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1233,7 +1213,7 @@ def test_catchup_with_replication_slot(self): # 5. --perm-slot --temp-slot (PG>=10) if self.get_version(src_pg) >= self.version_to_num('10.0'): - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst_5')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_5')) try: self.catchup_node( backup_mode = 'FULL', @@ -1254,7 +1234,6 @@ def test_catchup_with_replication_slot(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(module_name, self.fname) ######################################### # --exclude-path @@ -1265,7 +1244,7 @@ def test_catchup_with_exclude_path(self): """ # preparation src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() @@ -1282,7 +1261,7 @@ def test_catchup_with_exclude_path(self): f.flush() f.close - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1333,7 +1312,6 @@ def test_catchup_with_exclude_path(self): #self.assertEqual(1, 0, 'Stop test') src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_config_exclusion(self): """ @@ -1341,7 +1319,7 @@ def test_config_exclusion(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -1351,7 +1329,7 @@ def test_config_exclusion(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1458,7 +1436,6 @@ def test_config_exclusion(self): src_pg.stop() dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(module_name, self.fname) ######################################### # --dry-run @@ -1469,13 +1446,13 @@ def test_dry_run_catchup_full(self): """ # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) src_pg.pgbench_init(scale = 10) pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) @@ -1500,7 +1477,6 @@ def test_dry_run_catchup_full(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) def test_dry_run_catchup_ptrack(self): """ @@ -1511,7 +1487,7 @@ def test_dry_run_catchup_ptrack(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, ptrack_enable = True, initdb_params = ['--data-checksums'] @@ -1524,7 +1500,7 @@ def test_dry_run_catchup_ptrack(self): pgbench.wait() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1557,7 +1533,7 @@ def test_dry_run_catchup_ptrack(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) + self.del_test_dir(self.module_name, self.fname) def test_dry_run_catchup_delta(self): """ @@ -1566,7 +1542,7 @@ def test_dry_run_catchup_delta(self): # preparation 1: source src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True, initdb_params = ['--data-checksums'], pg_options = { 'wal_log_hints': 'on' } @@ -1578,7 +1554,7 @@ def test_dry_run_catchup_delta(self): pgbench.wait() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, @@ -1611,5 +1587,3 @@ def test_dry_run_catchup_delta(self): # Cleanup src_pg.stop() - self.del_test_dir(module_name, self.fname) - diff --git a/tests/checkdb.py b/tests/checkdb.py index 07b55c6db..2caf4fcb2 100644 --- a/tests/checkdb.py +++ b/tests/checkdb.py @@ -9,9 +9,6 @@ import time -module_name = 'checkdb' - - class CheckdbTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -19,10 +16,9 @@ def test_checkdb_amcheck_only_sanity(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) @@ -223,15 +219,13 @@ def test_checkdb_amcheck_only_sanity(self): # Clean after yourself gdb.kill() node.stop() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_basic_checkdb_amcheck_only_sanity(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) @@ -362,18 +356,16 @@ def test_basic_checkdb_amcheck_only_sanity(self): # Clean after yourself node.stop() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): """make node, corrupt some pages, check that checkdb failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -459,14 +451,12 @@ def test_checkdb_block_validation_sanity(self): # Clean after yourself node.stop() - self.del_test_dir(module_name, fname) def test_checkdb_checkunique(self): """Test checkunique parameter of amcheck.bt_index_check function""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) node.slow_start() @@ -550,17 +540,15 @@ def test_checkdb_checkunique(self): # Clean after yourself node.stop() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_checkdb_sigint_handling(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -605,15 +593,13 @@ def test_checkdb_sigint_handling(self): # Clean after yourself gdb.kill() node.stop() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_checkdb_with_least_privileges(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -861,4 +847,3 @@ def test_checkdb_with_least_privileges(self): # Clean after yourself node.stop() - self.del_test_dir(module_name, fname) diff --git a/tests/compatibility.py b/tests/compatibility.py index 6c2bc9204..8a7812c57 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -5,8 +5,6 @@ from sys import exit import shutil -module_name = 'compatibility' - def check_manual_tests_enabled(): return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' @@ -52,7 +50,7 @@ def test_catchup_with_different_remote_major_pg(self): pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] src_pg = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'src'), + base_dir=os.path.join(self.module_name, self.fname, 'src'), set_replication=True, ) src_pg.slow_start() @@ -61,7 +59,7 @@ def test_catchup_with_different_remote_major_pg(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # do full catchup - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode='FULL', source_pgdata=src_pg.data_dir, @@ -89,16 +87,15 @@ def test_catchup_with_different_remote_major_pg(self): ) # Clean after yourself - self.del_test_dir(module_name, self.fname) + self.del_test_dir(self.module_name, self.fname) # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -126,7 +123,7 @@ def test_backward_compatibility_page(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -223,17 +220,13 @@ def test_backward_compatibility_page(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_delta(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -261,7 +254,7 @@ def test_backward_compatibility_delta(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -357,9 +350,6 @@ def test_backward_compatibility_delta(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_ptrack(self): @@ -368,10 +358,9 @@ def test_backward_compatibility_ptrack(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -404,7 +393,7 @@ def test_backward_compatibility_ptrack(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -471,17 +460,13 @@ def test_backward_compatibility_ptrack(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_compression(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -504,7 +489,7 @@ def test_backward_compatibility_compression(self): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -630,9 +615,6 @@ def test_backward_compatibility_compression(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge(self): @@ -640,10 +622,9 @@ def test_backward_compatibility_merge(self): Create node, take FULL and PAGE backups with old binary, merge them with new binary """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -674,7 +655,7 @@ def test_backward_compatibility_merge(self): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -685,9 +666,6 @@ def test_backward_compatibility_merge(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_1(self): @@ -696,10 +674,9 @@ def test_backward_compatibility_merge_1(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -749,7 +726,7 @@ def test_backward_compatibility_merge_1(self): # restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -757,9 +734,6 @@ def test_backward_compatibility_merge_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_2(self): @@ -768,10 +742,9 @@ def test_backward_compatibility_merge_2(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -788,7 +761,7 @@ def test_backward_compatibility_merge_2(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node(backup_dir, 'node', node, old_binary=True) @@ -879,9 +852,6 @@ def test_backward_compatibility_merge_2(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_3(self): @@ -890,10 +860,9 @@ def test_backward_compatibility_merge_3(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -910,7 +879,7 @@ def test_backward_compatibility_merge_3(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node( @@ -1002,9 +971,6 @@ def test_backward_compatibility_merge_3(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_4(self): @@ -1016,10 +982,9 @@ def test_backward_compatibility_merge_4(self): self.assertTrue( False, 'You need pg_probackup old_binary =< 2.4.0 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1036,7 +1001,7 @@ def test_backward_compatibility_merge_4(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node( @@ -1079,9 +1044,6 @@ def test_backward_compatibility_merge_4(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_5(self): @@ -1098,10 +1060,9 @@ def test_backward_compatibility_merge_5(self): self.version_to_num(self.old_probackup_version), self.version_to_num(self.probackup_version)) - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1151,7 +1112,7 @@ def test_backward_compatibility_merge_5(self): # restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1159,9 +1120,6 @@ def test_backward_compatibility_merge_5(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate(self): """ @@ -1173,10 +1131,9 @@ def test_page_vacuum_truncate(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1224,7 +1181,7 @@ def test_page_vacuum_truncate(self): pgdata3 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1263,9 +1220,6 @@ def test_page_vacuum_truncate(self): node_restored.slow_start() node_restored.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate_compression(self): """ @@ -1277,10 +1231,9 @@ def test_page_vacuum_truncate_compression(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1330,7 +1283,7 @@ def test_page_vacuum_truncate_compression(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1342,9 +1295,6 @@ def test_page_vacuum_truncate_compression(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate_compressed_1(self): """ @@ -1356,10 +1306,9 @@ def test_page_vacuum_truncate_compressed_1(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1411,7 +1360,7 @@ def test_page_vacuum_truncate_compressed_1(self): pgdata3 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1450,9 +1399,6 @@ def test_page_vacuum_truncate_compressed_1(self): node_restored.slow_start() node_restored.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_hidden_files(self): """ @@ -1461,10 +1407,9 @@ def test_hidden_files(self): with old binary, then try to delete backup with new binary """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1479,21 +1424,17 @@ def test_hidden_files(self): self.delete_pb(backup_dir, 'node', backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + # @unittest.skip("skip") def test_compatibility_tablespace(self): """ https://github.com/postgrespro/pg_probackup/issues/348 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1519,7 +1460,7 @@ def test_compatibility_tablespace(self): tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1560,6 +1501,3 @@ def test_compatibility_tablespace(self): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/compression.py b/tests/compression.py index c10a59489..94f2dffff 100644 --- a/tests/compression.py +++ b/tests/compression.py @@ -5,9 +5,6 @@ import subprocess -module_name = 'compression' - - class CompressionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -18,10 +15,9 @@ def test_basic_compression_stream_zlib(self): check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -116,19 +112,15 @@ def test_basic_compression_stream_zlib(self): delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") self.assertEqual(delta_result, delta_result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_archive_zlib(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -219,19 +211,15 @@ def test_compression_archive_zlib(self): self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_stream_pglz(self): """ make archive node, make full and page stream backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -324,19 +312,15 @@ def test_compression_stream_pglz(self): self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_archive_pglz(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -429,19 +413,15 @@ def test_compression_archive_pglz(self): self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_wrong_algorithm(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -467,9 +447,6 @@ def test_compression_wrong_algorithm(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incompressible_pages(self): """ @@ -477,10 +454,9 @@ def test_incompressible_pages(self): take backup with compression, make sure that page was not compressed, restore backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -517,6 +493,3 @@ def test_incompressible_pages(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/config.py b/tests/config.py index b41382204..b1a0f9295 100644 --- a/tests/config.py +++ b/tests/config.py @@ -5,19 +5,16 @@ from sys import exit from shutil import copyfile -module_name = 'config' - class ConfigTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_remove_instance_config(self): - """remove pg_probackup.conf""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """remove pg_probackup.conself.f""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -57,10 +54,9 @@ def test_remove_instance_config(self): # @unittest.skip("skip") def test_corrupt_backup_content(self): """corrupt backup_content.control""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) diff --git a/tests/delete.py b/tests/delete.py index 345a70284..6b30cc712 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -2,10 +2,6 @@ import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess -from sys import exit - - -module_name = 'delete' class DeleteTest(ProbackupTest, unittest.TestCase): @@ -14,12 +10,11 @@ class DeleteTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_delete_full_backups(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -51,19 +46,15 @@ def test_delete_full_backups(self): self.assertEqual(show_backups[0]['id'], id_1) self.assertEqual(show_backups[1]['id'], id_3) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_del_instance_archive(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -83,19 +74,15 @@ def test_del_instance_archive(self): # Delete instance self.del_instance(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_archive_mix_compress_and_non_compressed_segments(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving( @@ -142,18 +129,14 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): '--retention-redundancy=3', '--delete-expired']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_increment_page(self): """delete increment and all after him""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -182,22 +165,18 @@ def test_delete_increment_page(self): self.assertEqual(show_backups[1]['backup-mode'], "FULL") self.assertEqual(show_backups[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_increment_ptrack(self): """delete increment and all after him""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -230,9 +209,6 @@ def test_delete_increment_ptrack(self): self.assertEqual(show_backups[1]['backup-mode'], "FULL") self.assertEqual(show_backups[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_orphaned_wal_segments(self): """ @@ -240,12 +216,11 @@ def test_delete_orphaned_wal_segments(self): delete second backup without --wal option, then delete orphaned wals via --wal option """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -302,9 +277,6 @@ def test_delete_orphaned_wal_segments(self): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] self.assertEqual (0, len(wals), "Number of wals should be equal to 0") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_wal_between_multiple_timelines(self): """ @@ -315,12 +287,11 @@ def test_delete_wal_between_multiple_timelines(self): [A1, B1) are deleted and backups B1 and A2 keep their WAL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -332,7 +303,7 @@ def test_delete_wal_between_multiple_timelines(self): node.pgbench_init(scale=3) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2) @@ -356,22 +327,18 @@ def test_delete_wal_between_multiple_timelines(self): self.validate_pb(backup_dir) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_backup_with_empty_control_file(self): """ take backup, truncate its control file, try to delete it via 'delete' command """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -397,18 +364,14 @@ def test_delete_backup_with_empty_control_file(self): self.delete_pb(backup_dir, 'node', backup_id=backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -521,9 +484,6 @@ def test_delete_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_multiple_descendants(self): """ @@ -536,12 +496,11 @@ def test_delete_multiple_descendants(self): FULLb | FULLa should be deleted """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -693,9 +652,6 @@ def test_delete_multiple_descendants(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_multiple_descendants_dry_run(self): """ @@ -706,12 +662,11 @@ def test_delete_multiple_descendants_dry_run(self): | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -798,17 +753,13 @@ def test_delete_multiple_descendants_dry_run(self): self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delete_error_backups(self): """delete increment and all after him""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -869,6 +820,3 @@ def test_delete_error_backups(self): self.assertEqual(show_backups[1]['status'], "OK") self.assertEqual(show_backups[2]['status'], "OK") self.assertEqual(show_backups[3]['status'], "OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta.py index 82fb714f7..386403151 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -8,9 +8,6 @@ from threading import Thread -module_name = 'delta' - - class DeltaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -21,15 +18,14 @@ def test_basic_delta_vacuum_truncate(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -77,9 +73,6 @@ def test_basic_delta_vacuum_truncate(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): """ @@ -88,15 +81,14 @@ def test_delta_vacuum_truncate_1(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -161,9 +153,6 @@ def test_delta_vacuum_truncate_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_vacuum_truncate_2(self): """ @@ -172,15 +161,14 @@ def test_delta_vacuum_truncate_2(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -223,19 +211,15 @@ def test_delta_vacuum_truncate_2(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_stream(self): """ make archive node, take full and delta stream backups, restore them and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -306,9 +290,6 @@ def test_delta_stream(self): self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_archive(self): """ @@ -316,10 +297,9 @@ def test_delta_archive(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -381,19 +361,15 @@ def test_delta_archive(self): self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_multiple_segments(self): """ Make node, create table with multiple segments, write some data to it, check delta and data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -434,7 +410,7 @@ def test_delta_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -463,9 +439,6 @@ def test_delta_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_vacuum_full(self): """ @@ -474,15 +447,14 @@ def test_delta_vacuum_full(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.init_pb(backup_dir) @@ -542,19 +514,15 @@ def test_delta_vacuum_full(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_create_db(self): """ Make node, take full backup, create database db1, take delta backup, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -596,7 +564,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -667,19 +635,15 @@ def test_create_db(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exists_in_previous_backup(self): """ Make node, take full backup, create table, take page backup, take delta backup, check that file is no fully copied to delta backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -750,7 +714,7 @@ def test_exists_in_previous_backup(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -773,19 +737,15 @@ def test_exists_in_previous_backup(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_table_set_tablespace_delta(self): """ Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -826,7 +786,7 @@ def test_alter_table_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -858,9 +818,6 @@ def test_alter_table_set_tablespace_delta(self): self.assertEqual(result, result_new, 'lost some data after restore') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_database_set_tablespace_delta(self): """ @@ -868,10 +825,9 @@ def test_alter_database_set_tablespace_delta(self): take delta backup, alter database tablespace location, take delta backup restore last delta backup. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ) @@ -919,7 +875,7 @@ def test_alter_database_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -947,19 +903,15 @@ def test_alter_database_set_tablespace_delta(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_delete(self): """ Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -1005,7 +957,7 @@ def test_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -1029,20 +981,16 @@ def test_delta_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_nullified_heap_page_backup(self): """ make node, take full backup, nullify some heap block, take delta backup, restore, physically compare pgdata`s """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1093,7 +1041,7 @@ def test_delta_nullified_heap_page_backup(self): # Restore DELTA backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1103,21 +1051,17 @@ def test_delta_nullified_heap_page_backup(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_backup_from_past(self): """ make node, take FULL stream backup, take DELTA stream backup, restore FULL backup, try to take second DELTA stream backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1158,22 +1102,18 @@ def test_delta_backup_from_past(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_delta_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1259,6 +1199,3 @@ def test_delta_pg_resetxlog(self): # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude.py index 2c4925881..cb3530cd5 100644 --- a/tests/exclude.py +++ b/tests/exclude.py @@ -3,19 +3,15 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'exclude' - - class ExcludeTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_exclude_temp_files(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -53,9 +49,6 @@ def test_exclude_temp_files(self): # TODO check temporary tablespaces - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_exclude_temp_tables(self): @@ -63,10 +56,9 @@ def test_exclude_temp_tables(self): make node without archiving, create temp table, take full backup, check that temp table not present in backup catalogue """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -139,9 +131,6 @@ def test_exclude_temp_tables(self): "Found temp table file in backup catalogue.\n " "Filepath: {0}".format(file)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exclude_unlogged_tables_1(self): """ @@ -149,10 +138,9 @@ def test_exclude_unlogged_tables_1(self): alter table to unlogged, take delta backup, restore delta backup, check that PGDATA`s are physically the same """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -186,7 +174,7 @@ def test_exclude_unlogged_tables_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -197,9 +185,6 @@ def test_exclude_unlogged_tables_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exclude_unlogged_tables_2(self): """ @@ -208,10 +193,9 @@ def test_exclude_unlogged_tables_2(self): 2. restore FULL, DELTA, PAGE to empty db, ensure unlogged table exist and is epmty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -279,19 +263,14 @@ def test_exclude_unlogged_tables_2(self): 'select count(*) from test')[0][0], 0) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_exclude_log_dir(self): """ check that by default 'log' and 'pg_log' directories are not backed up """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -321,18 +300,14 @@ def test_exclude_log_dir(self): self.assertTrue(os.path.exists(path)) self.assertFalse(os.path.exists(log_file)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exclude_log_dir_1(self): """ check that "--backup-pg-log" works correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -361,6 +336,3 @@ def test_exclude_log_dir_1(self): log_file = os.path.join(path, 'postgresql.log') self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(log_file)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/external.py b/tests/external.py index 530e7fb26..27928a43c 100644 --- a/tests/external.py +++ b/tests/external.py @@ -6,8 +6,6 @@ import shutil -module_name = 'external' - # TODO: add some ptrack tests class ExternalTest(ProbackupTest, unittest.TestCase): @@ -19,15 +17,14 @@ def test_basic_external(self): with external directory, restore backup, check that external directory was successfully copied """ - fname = self.id().split('.')[3] - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory @@ -91,9 +88,6 @@ def test_basic_external(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_external_none(self): @@ -103,13 +97,12 @@ def test_external_none(self): restore delta backup, check that external directory was not copied """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory @@ -153,9 +146,6 @@ def test_external_none(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_external_dirs_overlapping(self): @@ -164,13 +154,12 @@ def test_external_dirs_overlapping(self): take backup with two external directories pointing to the same directory, backup should fail """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') @@ -207,9 +196,6 @@ def test_external_dirs_overlapping(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_external_dir_mapping(self): """ @@ -218,13 +204,12 @@ def test_external_dir_mapping(self): check that restore with external-dir mapping will end with success """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -247,7 +232,7 @@ def test_external_dir_mapping(self): data_dir=external_dir2, options=["-j", "4"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') @@ -300,20 +285,16 @@ def test_external_dir_mapping(self): node_restored.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_backup_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -361,9 +342,6 @@ def test_backup_multiple_external(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility(self): @@ -376,10 +354,9 @@ def test_external_backward_compatibility(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -447,7 +424,7 @@ def test_external_backward_compatibility(self): # RESTORE chain with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -466,9 +443,6 @@ def test_external_backward_compatibility(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility_merge_1(self): @@ -480,10 +454,9 @@ def test_external_backward_compatibility_merge_1(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -542,7 +515,7 @@ def test_external_backward_compatibility_merge_1(self): # Restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -561,9 +534,6 @@ def test_external_backward_compatibility_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility_merge_2(self): @@ -575,10 +545,9 @@ def test_external_backward_compatibility_merge_2(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -666,7 +635,7 @@ def test_external_backward_compatibility_merge_2(self): # Restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -689,9 +658,6 @@ def test_external_backward_compatibility_merge_2(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge(self): @@ -699,10 +665,9 @@ def test_external_merge(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -779,17 +744,13 @@ def test_external_merge(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_skip_external_dirs(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -877,17 +838,13 @@ def test_external_merge_skip_external_dirs(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_1(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -957,17 +914,13 @@ def test_external_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_3(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1050,17 +1003,13 @@ def test_external_merge_3(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_2(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1144,17 +1093,13 @@ def test_external_merge_2(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_external_changed_data(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1242,17 +1187,13 @@ def test_restore_external_changed_data(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_external_changed_data_1(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1349,17 +1290,13 @@ def test_restore_external_changed_data_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_merge_external_changed_data(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1452,19 +1389,15 @@ def test_merge_external_changed_data(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_skip_external(self): """ Check that --skip-external-dirs works correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1523,9 +1456,6 @@ def test_restore_skip_external(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_is_symlink(self): @@ -1537,12 +1467,11 @@ def test_external_dir_is_symlink(self): if os.name == 'nt': return unittest.skip('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1557,7 +1486,7 @@ def test_external_dir_is_symlink(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1581,7 +1510,7 @@ def test_external_dir_is_symlink(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1606,9 +1535,6 @@ def test_external_dir_is_symlink(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_contain_symlink_on_dir(self): @@ -1620,12 +1546,11 @@ def test_external_dir_contain_symlink_on_dir(self): if os.name == 'nt': return unittest.skip('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1641,7 +1566,7 @@ def test_external_dir_contain_symlink_on_dir(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1666,7 +1591,7 @@ def test_external_dir_contain_symlink_on_dir(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1691,9 +1616,6 @@ def test_external_dir_contain_symlink_on_dir(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_contain_symlink_on_file(self): @@ -1705,12 +1627,11 @@ def test_external_dir_contain_symlink_on_file(self): if os.name == 'nt': return unittest.skip('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1726,7 +1647,7 @@ def test_external_dir_contain_symlink_on_file(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1753,7 +1674,7 @@ def test_external_dir_contain_symlink_on_file(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1778,9 +1699,6 @@ def test_external_dir_contain_symlink_on_file(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_is_tablespace(self): @@ -1788,12 +1706,11 @@ def test_external_dir_is_tablespace(self): Check that backup fails with error if external directory points to tablespace """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1828,21 +1745,17 @@ def test_external_dir_is_tablespace(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_not_empty(self): """ Check that backup fails with error if external directory point to not empty tablespace and if remapped directory also isn`t empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1908,9 +1821,6 @@ def test_restore_external_dir_not_empty(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_is_missing(self): """ take FULL backup with not empty external directory @@ -1918,12 +1828,11 @@ def test_restore_external_dir_is_missing(self): take DELTA backup with external directory, which should fail """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1990,9 +1899,6 @@ def test_restore_external_dir_is_missing(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_is_missing(self): """ take FULL backup with not empty external directory @@ -2003,12 +1909,11 @@ def test_merge_external_dir_is_missing(self): merge it into FULL, restore and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2078,9 +1983,6 @@ def test_merge_external_dir_is_missing(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_is_empty(self): """ take FULL backup with not empty external directory @@ -2089,12 +1991,11 @@ def test_restore_external_dir_is_empty(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2142,9 +2043,6 @@ def test_restore_external_dir_is_empty(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_is_empty(self): """ take FULL backup with not empty external directory @@ -2153,12 +2051,11 @@ def test_merge_external_dir_is_empty(self): merge backups and restore FULL, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2209,9 +2106,6 @@ def test_merge_external_dir_is_empty(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_string_order(self): """ take FULL backup with not empty external directory @@ -2220,12 +2114,11 @@ def test_restore_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2289,9 +2182,6 @@ def test_restore_external_dir_string_order(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_string_order(self): """ take FULL backup with not empty external directory @@ -2300,12 +2190,11 @@ def test_merge_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2372,9 +2261,6 @@ def test_merge_external_dir_string_order(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smart_restore_externals(self): """ @@ -2383,13 +2269,12 @@ def test_smart_restore_externals(self): make sure that files from externals are not copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2451,9 +2336,6 @@ def test_smart_restore_externals(self): for file in filelist_diff: self.assertNotIn(file, logfile_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_external_validation(self): """ @@ -2462,13 +2344,12 @@ def test_external_validation(self): corrupt external file in backup, run validate which should fail """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2522,6 +2403,3 @@ def test_external_validation(self): 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], 'Backup STATUS should be "CORRUPT"') - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/false_positive.py b/tests/false_positive.py index 2ededdf12..8e2e74cc0 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -1,13 +1,12 @@ import unittest import os +from asyncio import sleep + from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess -module_name = 'false_positive' - - class FalsePositive(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -16,13 +15,12 @@ def test_validate_wal_lost_segment(self): """ Loose segment located between backups. ExpectedFailure. This is BUG """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -47,19 +45,15 @@ def test_validate_wal_lost_segment(self): backup_dir, 'node')) ######## - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.expectedFailure # Need to force validation of ancestor-chain def test_incremental_backup_corrupt_full_1(self): """page-level backup with corrupted full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -104,9 +98,6 @@ def test_incremental_backup_corrupt_full_1(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_pg_10_waldir(self): @@ -116,18 +107,18 @@ def test_pg_10_waldir(self): if self.pg_config_version < self.version_to_num('10.0'): return unittest.skip('You need PostgreSQL >= 10 for this test') - fname = self.id().split('.')[3] wal_dir = os.path.join( - os.path.join(self.tmp_path, module_name, fname), 'wal_dir') + os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') + import shutil shutil.rmtree(wal_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', '--waldir={0}'.format(wal_dir)]) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -140,7 +131,7 @@ def test_pg_10_waldir(self): # restore backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -154,9 +145,6 @@ def test_pg_10_waldir(self): os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), 'pg_wal should be symlink') - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_time_backup_victim(self): @@ -165,10 +153,9 @@ def test_recovery_target_time_backup_victim(self): probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -216,9 +203,6 @@ def test_recovery_target_time_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_lsn_backup_victim(self): @@ -227,10 +211,9 @@ def test_recovery_target_lsn_backup_victim(self): probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -280,9 +263,6 @@ def test_recovery_target_lsn_backup_victim(self): backup_dir, 'node', options=['--recovery-target-lsn={0}'.format(target_lsn)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_streaming_timeout(self): @@ -291,10 +271,9 @@ def test_streaming_timeout(self): message because our WAL streaming engine is "borrowed" from pg_receivexlog """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -331,20 +310,16 @@ def test_streaming_timeout(self): 'ERROR: Problem in receivexlog', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_validate_all_empty_catalog(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) try: @@ -360,6 +335,3 @@ def test_validate_all_empty_catalog(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/incr_restore.py b/tests/incr_restore.py index cb684a23a..55d59fa99 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -12,20 +12,16 @@ from testgres import QueryException -module_name = 'incr_restore' - - class IncrRestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_basic_incr_restore(self): """incremental restore in CHECKSUM mode""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -76,18 +72,14 @@ def test_basic_incr_restore(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_incr_restore_into_missing_directory(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -124,19 +116,15 @@ def test_basic_incr_restore_into_missing_directory(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_checksum_corruption_detection(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -181,20 +169,16 @@ def test_checksum_corruption_detection(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -224,19 +208,15 @@ def test_incr_restore_with_tablespace(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_1(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -282,22 +262,18 @@ def test_incr_restore_with_tablespace_1(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_2(self): """ If "--tablespace-mapping" option is used with incremental restore, then new directory must be empty. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -305,7 +281,7 @@ def test_incr_restore_with_tablespace_2(self): self.backup_node(backup_dir, 'node', node, options=['--stream']) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) # fill node1 with data out = self.restore_node( @@ -355,20 +331,16 @@ def test_incr_restore_with_tablespace_2(self): pgdata_restored = self.pgdata_content(node_1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_3(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -396,21 +368,17 @@ def test_incr_restore_with_tablespace_3(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_4(self): """ Check that system ID mismatch is detected, """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -427,7 +395,7 @@ def test_incr_restore_with_tablespace_4(self): # recreate node node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) node.slow_start() @@ -469,9 +437,6 @@ def test_incr_restore_with_tablespace_4(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure @unittest.skip("skip") def test_incr_restore_with_tablespace_5(self): @@ -481,13 +446,12 @@ def test_incr_restore_with_tablespace_5(self): with some old content, that belongs to an instance with different system id. """ - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) node1.slow_start() @@ -503,7 +467,7 @@ def test_incr_restore_with_tablespace_5(self): # recreate node node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), + base_dir=os.path.join(self.module_name, self.fname, 'node2'), set_replication=True, initdb_params=['--data-checksums']) node2.slow_start() @@ -530,21 +494,17 @@ def test_incr_restore_with_tablespace_5(self): pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_6(self): """ Empty pgdata, not empty tablespace """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -591,22 +551,18 @@ def test_incr_restore_with_tablespace_6(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_7(self): """ Restore backup without tablespace into PGDATA with tablespace. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -647,19 +603,15 @@ def test_incr_restore_with_tablespace_7(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_incr_restore_sanity(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -690,7 +642,7 @@ def test_basic_incr_restore_sanity(self): repr(e.message), self.cmd)) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) try: self.restore_node( @@ -714,9 +666,6 @@ def test_basic_incr_restore_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_checksum_restore(self): """ @@ -725,13 +674,12 @@ def test_incr_checksum_restore(self): X - is instance, we want to return it to C state. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -758,7 +706,7 @@ def test_incr_checksum_restore(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -803,9 +751,6 @@ def test_incr_checksum_restore(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_lsn_restore(self): @@ -815,13 +760,12 @@ def test_incr_lsn_restore(self): X - is instance, we want to return it to C state. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -848,7 +792,7 @@ def test_incr_lsn_restore(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -892,9 +836,6 @@ def test_incr_lsn_restore(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_lsn_sanity(self): """ @@ -904,13 +845,12 @@ def test_incr_lsn_sanity(self): X - is instance, we want to return it to state B. fail is expected behaviour in case of lsn restore. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -920,7 +860,7 @@ def test_incr_lsn_sanity(self): node.pgbench_init(scale=10) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -961,9 +901,6 @@ def test_incr_lsn_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_checksum_sanity(self): """ @@ -972,12 +909,11 @@ def test_incr_checksum_sanity(self): X - is instance, we want to return it to state B. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -987,7 +923,7 @@ def test_incr_checksum_sanity(self): node.pgbench_init(scale=20) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -1019,22 +955,17 @@ def test_incr_checksum_sanity(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_incr_checksum_corruption_detection(self): """ check that corrupted page got detected and replaced """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), # initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1078,21 +1009,17 @@ def test_incr_checksum_corruption_detection(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_lsn_corruption_detection(self): """ check that corrupted page got detected and replaced """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1136,20 +1063,16 @@ def test_incr_lsn_corruption_detection(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1207,20 +1130,16 @@ def test_incr_restore_multiple_external(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_restore_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1278,22 +1197,18 @@ def test_incr_lsn_restore_multiple_external(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_restore_backward(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1386,23 +1301,19 @@ def test_incr_lsn_restore_backward(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_checksum_restore_backward(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1479,21 +1390,20 @@ def test_incr_checksum_restore_backward(self): self.compare_pgdata(delta_pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) # @unittest.skip("skip") def test_make_replica_via_incr_checksum_restore(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1503,7 +1413,7 @@ def test_make_replica_via_incr_checksum_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.pgbench_init(scale=20) @@ -1551,22 +1461,18 @@ def test_make_replica_via_incr_checksum_restore(self): pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_make_replica_via_incr_lsn_restore(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1576,7 +1482,7 @@ def test_make_replica_via_incr_lsn_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.pgbench_init(scale=20) @@ -1624,20 +1530,16 @@ def test_make_replica_via_incr_lsn_restore(self): pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_checksum_long_xact(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1694,9 +1596,6 @@ def test_incr_checksum_long_xact(self): 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure # This test will pass with Enterprise @@ -1705,12 +1604,11 @@ def test_incr_checksum_long_xact(self): def test_incr_lsn_long_xact_1(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1774,24 +1672,20 @@ def test_incr_lsn_long_xact_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_long_xact_2(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'full_page_writes': 'off', 'wal_log_hints': 'off'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1861,21 +1755,17 @@ def test_incr_lsn_long_xact_2(self): 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_zero_size_file_checksum(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1934,21 +1824,17 @@ def test_incr_restore_zero_size_file_checksum(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_zero_size_file_lsn(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2013,15 +1899,11 @@ def test_incr_restore_zero_size_file_lsn(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incremental_partial_restore_exclude_checksum(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2060,11 +1942,11 @@ def test_incremental_partial_restore_exclude_checksum(self): # restore FULL backup into second node2 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() # restore some data into node2 @@ -2118,15 +2000,11 @@ def test_incremental_partial_restore_exclude_checksum(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incremental_partial_restore_exclude_lsn(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2167,11 +2045,11 @@ def test_incremental_partial_restore_exclude_lsn(self): # restore FULL backup into second node2 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() # restore some data into node2 @@ -2228,15 +2106,11 @@ def test_incremental_partial_restore_exclude_lsn(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incremental_partial_restore_exclude_tablespace_checksum(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2282,13 +2156,13 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): # node1 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node1_tablespace = self.get_tblspace_path(node1, 'somedata') # node2 node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() node2_tablespace = self.get_tblspace_path(node2, 'somedata') @@ -2372,17 +2246,13 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incremental_pg_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2391,7 +2261,7 @@ def test_incremental_pg_filenode_map(self): node.slow_start() node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), initdb_params=['--data-checksums']) node1.cleanup() @@ -2432,7 +2302,4 @@ def test_incremental_pg_filenode_map(self): 'postgres', 'select 1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn diff --git a/tests/init.py b/tests/init.py index f5715d249..94b076fef 100644 --- a/tests/init.py +++ b/tests/init.py @@ -4,18 +4,14 @@ import shutil -module_name = 'init' - - class InitTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_success(self): """Success normal init""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.assertEqual( dir_files(backup_dir), @@ -64,15 +60,11 @@ def test_success(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_already_exist(self): """Failure with backup catalog already existed""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) try: self.show_pb(backup_dir, 'node') @@ -84,15 +76,11 @@ def test_already_exist(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_abs_path(self): """failure with backup catalog should be given as absolute path""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) try: self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( @@ -103,18 +91,14 @@ def test_abs_path(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_add_instance_idempotence(self): """ https://github.com/postgrespro/pg_probackup/issues/219 """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -152,6 +136,3 @@ def test_add_instance_idempotence(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/locking.py b/tests/locking.py index 4042a1462..8531d7de5 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -4,9 +4,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'locking' - - class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -19,12 +16,11 @@ def test_locking_running_validate_1(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -64,7 +60,6 @@ def test_locking_running_validate_1(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_running_validate_2(self): """ @@ -76,12 +71,11 @@ def test_locking_running_validate_2(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -135,7 +129,6 @@ def test_locking_running_validate_2(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_running_validate_2_specific_id(self): """ @@ -148,12 +141,11 @@ def test_locking_running_validate_2_specific_id(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -236,7 +228,6 @@ def test_locking_running_validate_2_specific_id(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_running_3(self): """ @@ -248,12 +239,11 @@ def test_locking_running_3(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -308,7 +298,6 @@ def test_locking_running_3(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_restore_locked(self): """ @@ -320,12 +309,11 @@ def test_locking_restore_locked(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -352,7 +340,6 @@ def test_locking_restore_locked(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_concurrent_delete_and_restore(self): """ @@ -364,12 +351,11 @@ def test_concurrent_delete_and_restore(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -411,7 +397,6 @@ def test_concurrent_delete_and_restore(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_concurrent_validate_and_backup(self): """ @@ -421,12 +406,11 @@ def test_locking_concurrent_validate_and_backup(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -449,7 +433,6 @@ def test_locking_concurrent_validate_and_backup(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_locking_concurren_restore_and_delete(self): """ @@ -459,12 +442,11 @@ def test_locking_concurren_restore_and_delete(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -495,17 +477,15 @@ def test_locking_concurren_restore_and_delete(self): # Clean after yourself gdb.kill() - self.del_test_dir(module_name, fname) def test_backup_directory_name(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -549,18 +529,16 @@ def test_backup_directory_name(self): self.show_pb(backup_dir, 'node', page_id_2)) # Clean after yourself - self.del_test_dir(module_name, fname) def test_empty_lock_file(self): """ https://github.com/postgrespro/pg_probackup/issues/308 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -594,21 +572,17 @@ def test_empty_lock_file(self): # p1.wait() # p2.wait() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_shared_lock(self): """ Make sure that shared lock leaves no files with pids """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.name, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -653,5 +627,3 @@ def test_shared_lock(self): os.path.exists(lockfile_shr), "File should not exist: {0}".format(lockfile_shr)) - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/logging.py b/tests/logging.py index 70ebcf6d1..c5cdfa344 100644 --- a/tests/logging.py +++ b/tests/logging.py @@ -3,9 +3,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import datetime -module_name = 'logging' - - class LogTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -16,13 +13,12 @@ def test_log_rotation(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -43,17 +39,13 @@ def test_log_rotation(self): gdb.run_until_break() gdb.continue_execution_until_exit() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_log_filename_strftime(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -76,17 +68,13 @@ def test_log_filename_strftime(self): self.assertTrue(os.path.isfile(path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_truncate_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -151,17 +139,13 @@ def test_truncate_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_unlink_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -223,17 +207,13 @@ def test_unlink_rotation_file(self): os.stat(log_file_path).st_size, log_file_size) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_garbage_in_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -261,9 +241,6 @@ def test_garbage_in_rotation_file(self): # mangle .rotation file with open(rotation_file_path, "w+b", 0) as f: f.write(b"blah") - f.flush() - f.close - output = self.backup_node( backup_dir, 'node', node, options=[ @@ -302,24 +279,20 @@ def test_garbage_in_rotation_file(self): os.stat(log_file_path).st_size, log_file_size) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_issue_274(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -370,6 +343,3 @@ def test_issue_274(self): log_content = f.read() self.assertIn('INFO: command:', log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/merge.py b/tests/merge.py index 4c374bdfb..fa0da7b2b 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -9,21 +9,17 @@ import time import subprocess -module_name = "merge" - - class MergeTest(ProbackupTest, unittest.TestCase): def test_basic_merge_full_page(self): """ Test MERGE command, it merges FULL backup with target PAGE backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=["--data-checksums"]) self.init_pb(backup_dir) @@ -100,19 +96,15 @@ def test_basic_merge_full_page(self): count2 = node.execute("postgres", "select count(*) from test") self.assertEqual(count1, count2) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_compressed_backups(self): """ Test MERGE command with compressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=["--data-checksums"]) self.init_pb(backup_dir) @@ -163,18 +155,16 @@ def test_merge_compressed_backups(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_backups_1(self): """ Test MERGE command with compressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"]) self.init_pb(backup_dir) @@ -234,18 +224,16 @@ def test_merge_compressed_backups_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], ) @@ -306,18 +294,16 @@ def test_merge_compressed_and_uncompressed_backups(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups_1(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], ) @@ -380,18 +366,16 @@ def test_merge_compressed_and_uncompressed_backups_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups_2(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], ) @@ -450,11 +434,6 @@ def test_merge_compressed_and_uncompressed_backups_2(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - node.cleanup() - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_merge_tablespaces(self): """ @@ -463,10 +442,9 @@ def test_merge_tablespaces(self): tablespace, take page backup, merge it and restore """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ) @@ -538,10 +516,9 @@ def test_merge_tablespaces_1(self): drop first tablespace and take delta backup, merge it and restore """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ) @@ -607,9 +584,6 @@ def test_merge_tablespaces_1(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_page_truncate(self): """ make node, create table, take full backup, @@ -617,17 +591,16 @@ def test_merge_page_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -696,9 +669,6 @@ def test_merge_page_truncate(self): self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_delta_truncate(self): """ make node, create table, take full backup, @@ -706,17 +676,16 @@ def test_merge_delta_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -786,7 +755,7 @@ def test_merge_delta_truncate(self): self.assertEqual(result1, result2) # Clean after yourself - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) def test_merge_ptrack_truncate(self): """ @@ -798,10 +767,9 @@ def test_merge_ptrack_truncate(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], ptrack_enable=True) @@ -850,7 +818,7 @@ def test_merge_ptrack_truncate(self): self.validate_pb(backup_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -881,9 +849,6 @@ def test_merge_ptrack_truncate(self): self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_merge_delta_delete(self): """ @@ -891,10 +856,9 @@ def test_merge_delta_delete(self): alter tablespace location, take delta backup, merge full and delta, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -943,7 +907,7 @@ def test_merge_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -967,9 +931,6 @@ def test_merge_delta_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_continue_failed_merge(self): """ @@ -977,11 +938,10 @@ def test_continue_failed_merge(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join( - module_name, fname, 'node'), + self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1045,9 +1005,6 @@ def test_continue_failed_merge(self): node.cleanup() self.restore_node(backup_dir, 'node', node) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_continue_failed_merge_with_corrupted_delta_backup(self): """ @@ -1055,10 +1012,9 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1145,19 +1101,15 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_continue_failed_merge_2(self): """ Check that failed MERGE on delete can be continued """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1217,8 +1169,6 @@ def test_continue_failed_merge_2(self): # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) def test_continue_failed_merge_3(self): """ @@ -1227,10 +1177,9 @@ def test_continue_failed_merge_3(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1315,17 +1264,13 @@ def test_continue_failed_merge_3(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_different_compression_algo(self): """ Check that backups with different compression algorithms can be merged """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1368,17 +1313,14 @@ def test_merge_different_compression_algo(self): self.merge_backup(backup_dir, "node", backup_id) - self.del_test_dir(module_name, fname) - def test_merge_different_wal_modes(self): """ Check that backups with different wal modes can be merged correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1410,8 +1352,6 @@ def test_merge_different_wal_modes(self): self.assertEqual( 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) - self.del_test_dir(module_name, fname) - def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control @@ -1419,10 +1359,9 @@ def test_crash_after_opening_backup_control_1(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1462,7 +1401,7 @@ def test_crash_after_opening_backup_control_1(self): self.assertEqual( 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) # @unittest.skip("skip") def test_crash_after_opening_backup_control_2(self): @@ -1473,10 +1412,9 @@ def test_crash_after_opening_backup_control_2(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1555,8 +1493,6 @@ def test_crash_after_opening_backup_control_2(self): self.compare_pgdata(pgdata, pgdata_restored) - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_losing_file_after_failed_merge(self): """ @@ -1566,10 +1502,9 @@ def test_losing_file_after_failed_merge(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1648,17 +1583,14 @@ def test_losing_file_after_failed_merge(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1731,17 +1663,14 @@ def test_failed_merge_after_delete(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_1(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1809,17 +1738,14 @@ def test_failed_merge_after_delete_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_2(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1873,17 +1799,14 @@ def test_failed_merge_after_delete_2(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_3(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1963,8 +1886,6 @@ def test_failed_merge_after_delete_3(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - # Skipped, because backups from the future are invalid. # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" # now (PBCKP-259). We can conduct such a test again when we @@ -1975,13 +1896,12 @@ def test_merge_backup_from_future(self): take FULL backup, table PAGE backup from future, try to merge page with FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2021,7 +1941,7 @@ def test_merge_backup_from_future(self): 'SELECT * from pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2052,9 +1972,6 @@ def test_merge_backup_from_future(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_merge_multiple_descendants(self): """ @@ -2067,12 +1984,11 @@ def test_merge_multiple_descendants(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2241,9 +2157,6 @@ def test_merge_multiple_descendants(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smart_merge(self): """ @@ -2253,13 +2166,12 @@ def test_smart_merge(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2301,18 +2213,14 @@ def test_smart_merge(self): with open(logfile, 'r') as f: logfile_content = f.read() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_idempotent_merge(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2378,18 +2286,15 @@ def test_idempotent_merge(self): self.assertEqual( page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) - self.del_test_dir(module_name, fname) - def test_merge_correct_inheritance(self): """ Make sure that backup metainformation fields 'note' and 'expire-time' are correctly inherited during merge """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2432,18 +2337,15 @@ def test_merge_correct_inheritance(self): page_meta['expire-time'], self.show_pb(backup_dir, 'node', page_id)['expire-time']) - self.del_test_dir(module_name, fname) - def test_merge_correct_inheritance_1(self): """ Make sure that backup metainformation fields 'note' and 'expire-time' are correctly inherited during merge """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2481,8 +2383,6 @@ def test_merge_correct_inheritance_1(self): 'expire-time', self.show_pb(backup_dir, 'node', page_id)) - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multi_timeline_merge(self): @@ -2497,10 +2397,9 @@ def test_multi_timeline_merge(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2568,7 +2467,7 @@ def test_multi_timeline_merge(self): "postgres", "select * from pgbench_accounts") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -2598,9 +2497,6 @@ def test_multi_timeline_merge(self): '--amcheck', '-d', 'postgres', '-p', str(node_restored.port)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_merge_page_header_map_retry(self): @@ -2610,10 +2506,9 @@ def test_merge_page_header_map_retry(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2649,19 +2544,15 @@ def test_merge_page_header_map_retry(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_missing_data_file(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2710,18 +2601,15 @@ def test_missing_data_file(self): 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), logfile_content) - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_missing_non_data_file(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2769,7 +2657,7 @@ def test_missing_non_data_file(self): self.assertEqual( 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) # @unittest.skip("skip") def test_merge_remote_mode(self): @@ -2777,10 +2665,9 @@ def test_merge_remote_mode(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2827,16 +2714,13 @@ def test_merge_remote_mode(self): self.assertEqual( 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - self.del_test_dir(module_name, fname) - def test_merge_pg_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2845,7 +2729,7 @@ def test_merge_pg_filenode_map(self): node.slow_start() node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), initdb_params=['--data-checksums']) node1.cleanup() @@ -2878,8 +2762,5 @@ def test_merge_pg_filenode_map(self): 'postgres', 'select 1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # 1. Need new test with corrupted FULL backup # 2. different compression levels diff --git a/tests/option.py b/tests/option.py index 88e72ffd7..a8b964cb0 100644 --- a/tests/option.py +++ b/tests/option.py @@ -4,9 +4,6 @@ import locale -module_name = 'option' - - class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -41,14 +38,12 @@ def test_without_backup_path_3(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # @unittest.skip("skip") def test_options_4(self): """check options test""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -112,16 +107,12 @@ def test_options_4(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_options_5(self): """check options test""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) output = self.init_pb(backup_dir) self.assertIn( @@ -225,9 +216,6 @@ def test_options_5(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_help_6(self): """help options""" diff --git a/tests/page.py b/tests/page.py index c1cba6b40..b9398ec7a 100644 --- a/tests/page.py +++ b/tests/page.py @@ -7,9 +7,6 @@ import gzip import shutil -module_name = 'page' - - class PageTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -20,17 +17,16 @@ def test_basic_page_vacuum_truncate(self): take page backup, take second page backup, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -98,9 +94,6 @@ def test_basic_page_vacuum_truncate(self): self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate_1(self): """ @@ -109,10 +102,9 @@ def test_page_vacuum_truncate_1(self): take page backup, insert some data, take second page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -159,7 +151,7 @@ def test_page_vacuum_truncate_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -171,9 +163,6 @@ def test_page_vacuum_truncate_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_stream(self): """ @@ -181,10 +170,9 @@ def test_page_stream(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -258,9 +246,6 @@ def test_page_stream(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_archive(self): """ @@ -268,10 +253,9 @@ def test_page_archive(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -352,19 +336,15 @@ def test_page_archive(self): self.assertEqual(page_result, page_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_multiple_segments(self): """ Make node, create table with multiple segments, write some data to it, check page and data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -399,7 +379,7 @@ def test_page_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -427,9 +407,6 @@ def test_page_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_delete(self): """ @@ -437,10 +414,9 @@ def test_page_delete(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -477,7 +453,7 @@ def test_page_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -499,9 +475,6 @@ def test_page_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_delete_1(self): """ @@ -509,10 +482,9 @@ def test_page_delete_1(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -554,7 +526,7 @@ def test_page_delete_1(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -577,26 +549,22 @@ def test_page_delete_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_parallel_pagemap(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={ "hot_standby": "on" } ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -652,18 +620,16 @@ def test_parallel_pagemap(self): # Clean after yourself node.cleanup() node_restored.cleanup() - self.del_test_dir(module_name, fname) def test_parallel_pagemap_1(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={} ) @@ -704,7 +670,6 @@ def test_parallel_pagemap_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_page_backup_with_lost_wal_segment(self): @@ -715,12 +680,11 @@ def test_page_backup_with_lost_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -785,9 +749,6 @@ def test_page_backup_with_lost_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_backup_with_corrupted_wal_segment(self): """ @@ -797,12 +758,11 @@ def test_page_backup_with_corrupted_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -896,9 +856,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_backup_with_alien_wal_segment(self): """ @@ -910,18 +867,17 @@ def test_page_backup_with_alien_wal_segment(self): expecting error because of alien wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) alien_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'alien_node'), + base_dir=os.path.join(self.module_name, self.fname, 'alien_node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1017,20 +973,16 @@ def test_page_backup_with_alien_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_multithread_page_backup_with_toast(self): """ make node, create toast, do multithread PAGE backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1050,9 +1002,6 @@ def test_multithread_page_backup_with_toast(self): backup_dir, 'node', node, backup_type='page', options=["-j", "4"]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_create_db(self): """ @@ -1060,10 +1009,9 @@ def test_page_create_db(self): restore database and check it presense """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1101,7 +1049,7 @@ def test_page_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1162,9 +1110,6 @@ def test_page_create_db(self): repr(e.message), self.cmd) ) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multi_timeline_page(self): @@ -1179,10 +1124,9 @@ def test_multi_timeline_page(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1251,7 +1195,7 @@ def test_multi_timeline_page(self): "postgres", "select * from pgbench_accounts") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1303,9 +1247,6 @@ def test_multi_timeline_page(self): backup_list[4]['id']) self.assertEqual(backup_list[5]['current-tli'], 7) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multitimeline_page_1(self): @@ -1317,10 +1258,9 @@ def test_multitimeline_page_1(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) @@ -1373,7 +1313,7 @@ def test_multitimeline_page_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1384,22 +1324,18 @@ def test_multitimeline_page_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_page_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1486,6 +1422,3 @@ def test_page_pg_resetxlog(self): # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 3baa0ba0b..434ce2800 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -9,9 +9,6 @@ from testgres import ProcessType -module_name = '2068' - - class BugTest(ProbackupTest, unittest.TestCase): def test_minrecpoint_on_replica(self): @@ -20,9 +17,8 @@ def test_minrecpoint_on_replica(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -33,7 +29,7 @@ def test_minrecpoint_on_replica(self): 'bgwriter_lru_multiplier': '4.0', 'max_wal_size': '256MB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -45,7 +41,7 @@ def test_minrecpoint_on_replica(self): # start replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica, options=['-R']) @@ -190,6 +186,3 @@ def test_minrecpoint_on_replica(self): # do basebackup # do pg_probackup, expect error - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560.py index 7e10fef6a..eeab59960 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -6,9 +6,6 @@ from time import sleep -module_name = 'pgpro560' - - class CheckSystemID(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -20,13 +17,12 @@ def test_pgpro560_control_file_loss(self): make backup check that backup failed """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -52,7 +48,7 @@ def test_pgpro560_control_file_loss(self): # Clean after yourself # Return this file to avoid Postger fail os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) def test_pgpro560_systemid_mismatch(self): """ @@ -61,21 +57,20 @@ def test_pgpro560_systemid_mismatch(self): feed to backup PGDATA from node1 and PGPORT from node2 check that backup failed """ - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) node1.slow_start() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), + base_dir=os.path.join(self.module_name, self.fname, 'node2'), set_replication=True, initdb_params=['--data-checksums']) node2.slow_start() - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) @@ -128,6 +123,3 @@ def test_pgpro560_systemid_mismatch(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589.py index d6381a8b5..8ce8e1f56 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589.py @@ -5,9 +5,6 @@ import subprocess -module_name = 'pgpro589' - - class ArchiveCheck(ProbackupTest, unittest.TestCase): def test_pgpro589(self): @@ -17,12 +14,11 @@ def test_pgpro589(self): check that backup status equal to ERROR check that no files where copied to backup catalogue """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -74,6 +70,3 @@ def test_pgpro589(self): "\n Start LSN was not found in archive but datafiles where " "copied to backup catalogue.\n For example: {0}\n " "It is not optimal".format(file)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack.py index a01405d6a..2b8d2d49e 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -10,14 +10,10 @@ from threading import Thread -module_name = 'ptrack' - - class PtrackTest(ProbackupTest, unittest.TestCase): def setUp(self): if self.pg_config_version < self.version_to_num('11.0'): return unittest.skip('You need PostgreSQL >= 11 for this test') - self.fname = self.id().split('.')[3] # @unittest.skip("skip") def test_drop_rel_during_backup_ptrack(self): @@ -26,9 +22,9 @@ def test_drop_rel_during_backup_ptrack(self): """ self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -85,18 +81,15 @@ def test_drop_rel_during_backup_ptrack(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_without_full(self): """ptrack backup without validated full backup""" node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -125,18 +118,15 @@ def test_ptrack_without_full(self): self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -156,9 +146,6 @@ def test_ptrack_threads(self): backup_type="ptrack", options=["-j", "4"]) self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_stop_pg(self): """ @@ -166,9 +153,9 @@ def test_ptrack_stop_pg(self): restart node, check that ptrack backup can be taken """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -193,18 +180,15 @@ def test_ptrack_stop_pg(self): backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_multi_timeline_backup(self): """ t2 /------P2 t1 ------F---*-----P1 """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -269,9 +253,6 @@ def test_ptrack_multi_timeline_backup(self): self.assertEqual('0', balance) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_multi_timeline_backup_1(self): """ @@ -282,9 +263,9 @@ def test_ptrack_multi_timeline_backup_1(self): t2 /------P2 t1 ---F--------* """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -343,17 +324,14 @@ def test_ptrack_multi_timeline_backup_1(self): self.assertEqual('0', balance) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_eat_my_data(self): """ PGPRO-4051 """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -372,7 +350,7 @@ def test_ptrack_eat_my_data(self): self.backup_node(backup_dir, 'node', node) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) @@ -422,16 +400,13 @@ def test_ptrack_eat_my_data(self): 'SELECT * FROM pgbench_accounts'), 'Data loss') - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_simple(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -469,7 +444,7 @@ def test_ptrack_simple(self): result = node.safe_psql("postgres", "SELECT * FROM t_heap") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -491,15 +466,12 @@ def test_ptrack_simple(self): result, node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_unprivileged(self): """""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -690,9 +662,9 @@ def test_ptrack_unprivileged(self): # @unittest.expectedFailure def test_ptrack_enable(self): """make ptrack without full backup, should result in error""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -728,9 +700,6 @@ def test_ptrack_enable(self): ' CMD: {1}'.format(repr(e.message), self.cmd) ) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_disable(self): @@ -739,9 +708,9 @@ def test_ptrack_disable(self): enable ptrack, restart postgresql, take ptrack backup which should fail """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -793,15 +762,12 @@ def test_ptrack_disable(self): ) ) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_uncommitted_xact(self): """make ptrack backup while there is uncommitted open transaction""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -831,7 +797,7 @@ def test_ptrack_uncommitted_xact(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -851,18 +817,15 @@ def test_ptrack_uncommitted_xact(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_vacuum_full(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -915,7 +878,7 @@ def test_ptrack_vacuum_full(self): process.join() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -938,18 +901,15 @@ def test_ptrack_vacuum_full(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_vacuum_truncate(self): """make node, create table, take full backup, delete last 3 pages, vacuum relation, take ptrack backup, take second ptrack backup, restore last ptrack backup and check data correctness""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -996,7 +956,7 @@ def test_ptrack_vacuum_truncate(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -1021,9 +981,6 @@ def test_ptrack_vacuum_truncate(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_get_block(self): """ @@ -1032,9 +989,9 @@ def test_ptrack_get_block(self): """ self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -1090,16 +1047,13 @@ def test_ptrack_get_block(self): result, node.safe_psql("postgres", "SELECT * FROM t_heap")) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_stream(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1178,16 +1132,13 @@ def test_ptrack_stream(self): ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(ptrack_result, ptrack_result_new) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_archive(self): """make archive node, make full and ptrack backups, check data correctness in restored instance""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1288,9 +1239,6 @@ def test_ptrack_archive(self): node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_ptrack_pgpro417(self): """ @@ -1298,9 +1246,9 @@ def test_ptrack_pgpro417(self): delete ptrack backup. Try to take ptrack backup, which should fail. Actual only for PTRACK 1.x """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1366,9 +1314,6 @@ def test_ptrack_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_page_pgpro417(self): """ @@ -1376,9 +1321,9 @@ def test_page_pgpro417(self): delete page backup. Try to take ptrack backup, which should fail. Actual only for PTRACK 1.x """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1432,9 +1377,6 @@ def test_page_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_full_pgpro417(self): """ @@ -1442,9 +1384,9 @@ def test_full_pgpro417(self): Try to take ptrack backup, which should fail. Relevant only for PTRACK 1.x """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1504,18 +1446,15 @@ def test_full_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_create_db(self): """ Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1558,7 +1497,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1619,9 +1558,6 @@ def test_create_db(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_create_db_on_replica(self): """ @@ -1630,9 +1566,9 @@ def test_create_db_on_replica(self): create database db1, take ptrack backup from replica, restore database and check it presense """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1654,7 +1590,7 @@ def test_create_db_on_replica(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node( @@ -1707,7 +1643,7 @@ def test_create_db_on_replica(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1720,16 +1656,13 @@ def test_create_db_on_replica(self): node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_alter_table_set_tablespace_ptrack(self): """Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1776,7 +1709,7 @@ def test_alter_table_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1810,17 +1743,14 @@ def test_alter_table_set_tablespace_ptrack(self): # # self.assertEqual(result, result_new, 'lost some data after restore') - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_alter_database_set_tablespace_ptrack(self): """Make node, create tablespace with database," " take full backup, alter tablespace location," " take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1857,7 +1787,7 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1878,18 +1808,15 @@ def test_alter_database_set_tablespace_ptrack(self): node_restored.port = node.port node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_drop_tablespace(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -1972,18 +1899,15 @@ def test_drop_tablespace(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -2029,7 +1953,7 @@ def test_ptrack_alter_tablespace(self): # Restore ptrack backup restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') @@ -2087,18 +2011,15 @@ def test_ptrack_alter_tablespace(self): "postgres", "select * from t_heap") self.assertEqual(result, result_new) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_multiple_segments(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -2163,7 +2084,7 @@ def test_ptrack_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -2196,9 +2117,6 @@ def test_ptrack_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_atexit_fail(self): """ @@ -2206,14 +2124,14 @@ def test_atexit_fail(self): Relevant only for PTRACK 1.x """ node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'max_connections': '15'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2248,9 +2166,6 @@ def test_atexit_fail(self): "select * from pg_is_in_backup()").rstrip(), "f") - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): @@ -2259,12 +2174,12 @@ def test_ptrack_clean(self): Relevant only for PTRACK 1.x """ node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2356,9 +2271,6 @@ def test_ptrack_clean(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_ptrack_clean_replica(self): """ @@ -2367,14 +2279,14 @@ def test_ptrack_clean_replica(self): Relevant only for PTRACK 1.x """ master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2382,7 +2294,7 @@ def test_ptrack_clean_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2494,18 +2406,16 @@ def test_ptrack_clean_replica(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - # Clean after yourself - self.del_test_dir(module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_on_btree(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2557,18 +2467,15 @@ def test_ptrack_cluster_on_btree(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_cluster_on_gist(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2628,18 +2535,15 @@ def test_ptrack_cluster_on_gist(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_cluster_on_btree_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2652,7 +2556,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2716,7 +2620,7 @@ def test_ptrack_cluster_on_btree_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node) @@ -2724,17 +2628,14 @@ def test_ptrack_cluster_on_btree_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_cluster_on_gist_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2747,7 +2648,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2817,7 +2718,7 @@ def test_ptrack_cluster_on_gist_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node) @@ -2826,20 +2727,17 @@ def test_ptrack_cluster_on_gist_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2877,7 +2775,7 @@ def test_ptrack_empty(self): node.safe_psql('postgres', 'checkpoint') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') @@ -2902,9 +2800,6 @@ def test_ptrack_empty(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_empty_replica(self): @@ -2913,12 +2808,12 @@ def test_ptrack_empty_replica(self): and check that PTRACK on replica is clean """ master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2931,7 +2826,7 @@ def test_ptrack_empty_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2987,7 +2882,7 @@ def test_ptrack_empty_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2998,19 +2893,16 @@ def test_ptrack_empty_replica(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_truncate(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3079,13 +2971,10 @@ def test_ptrack_truncate(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_basic_ptrack_truncate_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -3094,7 +2983,7 @@ def test_basic_ptrack_truncate_replica(self): 'archive_timeout': '10s', 'checkpoint_timeout': '5min'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3107,7 +2996,7 @@ def test_basic_ptrack_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3189,7 +3078,7 @@ def test_basic_ptrack_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) @@ -3208,18 +3097,18 @@ def test_basic_ptrack_truncate_replica(self): 'select 1') # Clean after yourself - self.del_test_dir(module_name, self.fname) + self.del_test_dir(self.module_name, self.fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3295,20 +3184,17 @@ def test_ptrack_vacuum(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_vacuum_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3321,7 +3207,7 @@ def test_ptrack_vacuum_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3394,7 +3280,7 @@ def test_ptrack_vacuum_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) @@ -3402,19 +3288,16 @@ def test_ptrack_vacuum_replica(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3481,18 +3364,15 @@ def test_ptrack_vacuum_bits_frozen(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_ptrack_vacuum_bits_frozen_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3505,7 +3385,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3579,19 +3459,16 @@ def test_ptrack_vacuum_bits_frozen_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3658,19 +3535,16 @@ def test_ptrack_vacuum_bits_visibility(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_2(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, pg_options={ 'wal_log_hints': 'on' }) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3736,19 +3610,16 @@ def test_ptrack_vacuum_full_2(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3760,7 +3631,7 @@ def test_ptrack_vacuum_full_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3837,19 +3708,16 @@ def test_ptrack_vacuum_full_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_2(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3905,7 +3773,7 @@ def test_ptrack_vacuum_truncate_2(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3913,19 +3781,16 @@ def test_ptrack_vacuum_truncate_2(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -3938,7 +3803,7 @@ def test_ptrack_vacuum_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -4010,7 +3875,7 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'replica', node_restored) @@ -4018,9 +3883,6 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - @unittest.skip("skip") def test_ptrack_recovery(self): """ @@ -4028,12 +3890,12 @@ def test_ptrack_recovery(self): Actual only for PTRACK 1.x """ node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4079,14 +3941,11 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery_1(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -4094,7 +3953,7 @@ def test_ptrack_recovery_1(self): 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4155,7 +4014,7 @@ def test_ptrack_recovery_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -4164,19 +4023,16 @@ def test_ptrack_recovery_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_zero_changes(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4208,14 +4064,11 @@ def test_ptrack_zero_changes(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -4223,7 +4076,7 @@ def test_ptrack_pg_resetxlog(self): 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4331,19 +4184,16 @@ def test_ptrack_pg_resetxlog(self): # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_ptrack_map(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -4495,9 +4345,6 @@ def test_corrupt_ptrack_map(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, self.fname) - # @unittest.skip("skip") def test_horizon_lsn_ptrack(self): """ @@ -4511,9 +4358,9 @@ def test_horizon_lsn_ptrack(self): self.version_to_num('2.4.15'), 'You need pg_probackup old_binary =< 2.4.15 for this test') - backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, self.fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -4560,6 +4407,3 @@ def test_horizon_lsn_ptrack(self): # make sure that backup size is exactly the same self.assertEqual(delta_bytes, ptrack_bytes) - - # Clean after yourself - self.del_test_dir(module_name, self.fname) diff --git a/tests/remote.py b/tests/remote.py index 4d46447f0..2d36d7346 100644 --- a/tests/remote.py +++ b/tests/remote.py @@ -5,21 +5,17 @@ from .helpers.cfs_helpers import find_by_name -module_name = 'remote' - - class RemoteTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_remote_sanity(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -45,6 +41,3 @@ def test_remote_sanity(self): # e.message, # "\n Unexpected Error Message: {0}\n CMD: {1}".format( # repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica.py index ea69e2d01..3fb68633f 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -9,8 +9,6 @@ from time import sleep -module_name = 'replica' - class ReplicaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -21,15 +19,14 @@ def test_replica_switchover(self): over the course of several switchovers https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -41,7 +38,7 @@ def test_replica_switchover(self): # take full backup and restore it self.backup_node(backup_dir, 'node1', node1, options=['--stream']) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() # create replica @@ -92,9 +89,6 @@ def test_replica_switchover(self): # https://github.com/postgrespro/pg_probackup/issues/251 self.validate_pb(backup_dir) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_replica_stream_ptrack_backup(self): @@ -109,10 +103,9 @@ def test_replica_stream_ptrack_backup(self): return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -138,7 +131,7 @@ def test_replica_stream_ptrack_backup(self): # take full backup and restore it self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) @@ -172,7 +165,7 @@ def test_replica_stream_ptrack_backup(self): # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -219,19 +212,15 @@ def test_replica_stream_ptrack_backup(self): after = node.safe_psql("postgres", "SELECT * FROM t_heap") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_archive_page_backup(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -240,7 +229,7 @@ def test_replica_archive_page_backup(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -250,7 +239,7 @@ def test_replica_archive_page_backup(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -305,7 +294,7 @@ def test_replica_archive_page_backup(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -363,26 +352,22 @@ def test_replica_archive_page_backup(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_make_replica_via_restore(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -392,7 +377,7 @@ def test_basic_make_replica_via_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -421,9 +406,6 @@ def test_basic_make_replica_via_restore(self): backup_dir, 'replica', replica, options=['--archive-timeout=30s', '--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_take_backup_from_delayed_replica(self): """ @@ -431,16 +413,15 @@ def test_take_backup_from_delayed_replica(self): restore full backup as delayed replica, launch pgbench, take FULL, PAGE and DELTA backups from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -450,7 +431,7 @@ def test_take_backup_from_delayed_replica(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -530,9 +511,6 @@ def test_take_backup_from_delayed_replica(self): pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote(self): """ @@ -541,10 +519,9 @@ def test_replica_promote(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -553,7 +530,7 @@ def test_replica_promote(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -563,7 +540,7 @@ def test_replica_promote(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -624,19 +601,15 @@ def test_replica_promote(self): 'setting its status to ERROR'.format(backup_id), log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_stop_lsn_null_offset(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -644,7 +617,7 @@ def test_replica_stop_lsn_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -661,7 +634,7 @@ def test_replica_stop_lsn_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -710,7 +683,6 @@ def test_replica_stop_lsn_null_offset(self): # Clean after yourself gdb_checkpointer.kill() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_replica_stop_lsn_null_offset_next_record(self): @@ -718,10 +690,9 @@ def test_replica_stop_lsn_null_offset_next_record(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -729,7 +700,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -745,7 +716,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -812,19 +783,15 @@ def test_replica_stop_lsn_null_offset_next_record(self): self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_replica_null_offset(self): """ """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -832,7 +799,7 @@ def test_archive_replica_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -845,7 +812,7 @@ def test_archive_replica_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -898,17 +865,13 @@ def test_archive_replica_null_offset(self): print(output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_replica_not_null_offset(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -916,7 +879,7 @@ def test_archive_replica_not_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -929,7 +892,7 @@ def test_archive_replica_not_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -982,9 +945,6 @@ def test_archive_replica_not_null_offset(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_toast(self): """ @@ -993,10 +953,9 @@ def test_replica_toast(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1005,7 +964,7 @@ def test_replica_toast(self): 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1022,7 +981,7 @@ def test_replica_toast(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1088,17 +1047,15 @@ def test_replica_toast(self): # Clean after yourself gdb_checkpointer.kill() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_start_stop_lsn_in_the_same_segno(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1107,7 +1064,7 @@ def test_start_stop_lsn_in_the_same_segno(self): 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1122,7 +1079,7 @@ def test_start_stop_lsn_in_the_same_segno(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1167,17 +1124,13 @@ def test_start_stop_lsn_in_the_same_segno(self): '--stream'], return_id=False) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_replica_promote_1(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1185,7 +1138,7 @@ def test_replica_promote_1(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1199,7 +1152,7 @@ def test_replica_promote_1(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1242,17 +1195,13 @@ def test_replica_promote_1(self): os.path.exists(wal_file_partial), "File {0} disappeared".format(wal_file_partial)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_2(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) @@ -1267,7 +1216,7 @@ def test_replica_promote_2(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1291,9 +1240,6 @@ def test_replica_promote_2(self): backup_dir, 'master', replica, data_dir=replica.data_dir, backup_type='page') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_archive_delta(self): """ @@ -1301,10 +1247,9 @@ def test_replica_promote_archive_delta(self): t2 /-------> t1 --F---D1--D2-- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1312,7 +1257,7 @@ def test_replica_promote_archive_delta(self): 'archive_timeout': '30s'}) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1328,7 +1273,7 @@ def test_replica_promote_archive_delta(self): # Create replica node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2, node2.data_dir) @@ -1416,9 +1361,6 @@ def test_replica_promote_archive_delta(self): pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_archive_page(self): """ @@ -1426,10 +1368,9 @@ def test_replica_promote_archive_page(self): t2 /-------> t1 --F---P1--P2-- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1437,7 +1378,7 @@ def test_replica_promote_archive_page(self): 'archive_timeout': '30s'}) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1453,7 +1394,7 @@ def test_replica_promote_archive_page(self): # Create replica node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2, node2.data_dir) @@ -1544,22 +1485,18 @@ def test_replica_promote_archive_page(self): pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1572,7 +1509,7 @@ def test_parent_choosing(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1614,17 +1551,13 @@ def test_parent_choosing(self): backup_dir, 'replica', replica, backup_type='delta', options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_instance_from_the_past(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1661,17 +1594,13 @@ def test_instance_from_the_past(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_via_basebackup(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'hot_standby': 'on'}) @@ -1721,7 +1650,7 @@ def test_replica_via_basebackup(self): node.slow_start() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() pg_basebackup_path = self.get_bin_path('pg_basebackup') @@ -1735,9 +1664,6 @@ def test_replica_via_basebackup(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start(replica=True) - # Clean after yourself - self.del_test_dir(module_name, fname) - # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) # archiving from promoted delayed replica diff --git a/tests/restore.py b/tests/restore.py index 49538bd1f..52db63c1c 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -13,21 +13,17 @@ from stat import S_ISDIR -module_name = 'restore' - - class RestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_restore_full_to_latest(self): """recovery to latest from full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -67,18 +63,14 @@ def test_restore_full_to_latest(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -113,18 +105,14 @@ def test_restore_full_page_to_latest(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_specific_timeline(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -179,19 +167,15 @@ def test_restore_to_specific_timeline(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_time(self): """recovery to target time""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'TimeZone': 'GMT'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -229,18 +213,14 @@ def test_restore_to_time(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): """recovery to target xid""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -289,18 +269,14 @@ def test_restore_to_xid_inclusive(self): self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_xid_not_inclusive(self): """recovery with target inclusive false""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -350,22 +326,18 @@ def test_restore_to_xid_not_inclusive(self): self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_lsn_inclusive(self): """recovery to target lsn""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -421,22 +393,18 @@ def test_restore_to_lsn_inclusive(self): self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_lsn_not_inclusive(self): """recovery to target lsn""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -493,22 +461,18 @@ def test_restore_to_lsn_not_inclusive(self): self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -547,22 +511,18 @@ def test_restore_full_ptrack_archive(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -608,23 +568,19 @@ def test_restore_ptrack(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -663,9 +619,6 @@ def test_restore_full_ptrack_stream(self): after = node.execute("postgres", "SELECT * FROM pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): """ @@ -675,14 +628,13 @@ def test_restore_full_ptrack_under_load(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -732,9 +684,6 @@ def test_restore_full_ptrack_under_load(self): "postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): """ @@ -744,14 +693,13 @@ def test_restore_full_under_load_ptrack(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -803,18 +751,14 @@ def test_restore_full_under_load_ptrack(self): "postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -940,18 +884,14 @@ def test_restore_with_tablespace_mapping_1(self): result = node.execute("postgres", "SELECT id FROM test OFFSET 1") self.assertEqual(result[0][0], 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1017,18 +957,14 @@ def test_restore_with_tablespace_mapping_2(self): count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_missing_or_corrupted_tablespace_map(self): """restore backup with missing or corrupted tablespace_map""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1051,7 +987,7 @@ def test_restore_with_missing_or_corrupted_tablespace_map(self): pgdata = self.pgdata_content(node.data_dir) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() olddir = self.get_tblspace_path(node, 'tblspace') @@ -1147,22 +1083,18 @@ def test_restore_with_missing_or_corrupted_tablespace_map(self): pgdata_restored = self.pgdata_content(node2.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_node_backup_stream_restore_to_recovery_time(self): """ make node with archiving, make stream backup, make PITR to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1195,9 +1127,6 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_stream_restore_to_recovery_time(self): @@ -1205,13 +1134,12 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): make node with archiving, make stream backup, make PITR to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1242,9 +1170,6 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_stream_pitr(self): @@ -1253,13 +1178,12 @@ def test_archive_node_backup_stream_pitr(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1290,9 +1214,6 @@ def test_archive_node_backup_stream_pitr(self): result = node.psql("postgres", 'select * from t_heap') self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_archive_pitr_2(self): @@ -1301,12 +1222,11 @@ def test_archive_node_backup_archive_pitr_2(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1320,7 +1240,7 @@ def test_archive_node_backup_archive_pitr_2(self): node.stop() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() recovery_time = self.show_pb( @@ -1348,9 +1268,6 @@ def test_archive_node_backup_archive_pitr_2(self): result = node_restored.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_restore_to_restore_point(self): @@ -1359,12 +1276,11 @@ def test_archive_restore_to_restore_point(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1401,18 +1317,14 @@ def test_archive_restore_to_restore_point(self): self.assertEqual(result, result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1463,7 +1375,7 @@ def test_zags_block_corrupt(self): "insert into tbl select i from generate_series(0,100) as i") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), initdb_params=['--data-checksums']) node_restored.cleanup() @@ -1480,14 +1392,13 @@ def test_zags_block_corrupt(self): @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt_1(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={ 'full_page_writes': 'on'} ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1538,7 +1449,7 @@ def test_zags_block_corrupt_1(self): self.switch_wal_segment(node) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), initdb_params=['--data-checksums']) pgdata = self.pgdata_content(node.data_dir) @@ -1588,13 +1499,12 @@ def test_restore_chain(self): ERROR delta backups, take valid delta backup, restore must be successfull """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1670,19 +1580,15 @@ def test_restore_chain(self): self.restore_node(backup_dir, 'node', node) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_chain_with_corrupted_backup(self): """more complex test_restore_chain()""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1850,9 +1756,6 @@ def test_restore_chain_with_corrupted_backup(self): node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # Skipped, because backups from the future are invalid. # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" # now (PBCKP-259). We can conduct such a test again when we @@ -1860,13 +1763,12 @@ def test_restore_chain_with_corrupted_backup(self): @unittest.skip("skip") def test_restore_backup_from_future(self): """more complex test_restore_chain()""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1909,22 +1811,18 @@ def test_restore_backup_from_future(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_immediate_stream(self): """ correct handling of immediate recovery target for STREAM backups """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1966,22 +1864,18 @@ def test_restore_target_immediate_stream(self): os.path.isfile(recovery_conf), "File {0} do not exists".format(recovery_conf)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_immediate_archive(self): """ correct handling of immediate recovery target for ARCHIVE backups """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2026,22 +1920,18 @@ def test_restore_target_immediate_archive(self): with open(recovery_conf, 'r') as f: self.assertIn("recovery_target = 'immediate'", f.read()) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_latest_archive(self): """ make sure that recovery_target 'latest' is default recovery target """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2092,22 +1982,18 @@ def test_restore_target_latest_archive(self): self.assertEqual(content_1, content_2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_new_options(self): """ check that new --recovery-target-* options are working correctly """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2272,9 +2158,6 @@ def test_restore_target_new_options(self): node.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smart_restore(self): """ @@ -2284,13 +2167,12 @@ def test_smart_restore(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2336,9 +2218,6 @@ def test_smart_restore(self): for file in filelist_diff: self.assertNotIn(file, logfile_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pg_11_group_access(self): """ @@ -2347,15 +2226,14 @@ def test_pg_11_group_access(self): if self.pg_config_version < self.version_to_num('11.0'): return unittest.skip('You need PostgreSQL >= 11 for this test') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', '--allow-group-access']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2367,7 +2245,7 @@ def test_pg_11_group_access(self): # restore backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2377,18 +2255,14 @@ def test_pg_11_group_access(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_concurrent_drop_table(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2434,16 +2308,12 @@ def test_restore_concurrent_drop_table(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_lost_non_data_file(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2481,15 +2351,11 @@ def test_lost_non_data_file(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_exclude(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2520,7 +2386,7 @@ def test_partial_restore_exclude(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() try: @@ -2557,7 +2423,7 @@ def test_partial_restore_exclude(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() self.restore_node( @@ -2596,15 +2462,11 @@ def test_partial_restore_exclude(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_exclude_tablespace(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2651,7 +2513,7 @@ def test_partial_restore_exclude_tablespace(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') @@ -2677,7 +2539,7 @@ def test_partial_restore_exclude_tablespace(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') @@ -2719,16 +2581,12 @@ def test_partial_restore_exclude_tablespace(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_include(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2759,7 +2617,7 @@ def test_partial_restore_include(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() try: @@ -2798,7 +2656,7 @@ def test_partial_restore_include(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() self.restore_node( @@ -2845,9 +2703,6 @@ def test_partial_restore_include(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_backward_compatibility_1(self): """ old binary should be of version < 2.2.0 @@ -2855,10 +2710,9 @@ def test_partial_restore_backward_compatibility_1(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2881,7 +2735,7 @@ def test_partial_restore_backward_compatibility_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -2941,7 +2795,7 @@ def test_partial_restore_backward_compatibility_1(self): # get new node node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() self.restore_node( @@ -2961,10 +2815,9 @@ def test_partial_restore_backward_compatibility_merge(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2987,7 +2840,7 @@ def test_partial_restore_backward_compatibility_merge(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -3047,7 +2900,7 @@ def test_partial_restore_backward_compatibility_merge(self): # get new node node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() # merge @@ -3065,10 +2918,9 @@ def test_partial_restore_backward_compatibility_merge(self): def test_empty_and_mangled_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3096,7 +2948,7 @@ def test_empty_and_mangled_database_map(self): f.close() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -3178,10 +3030,9 @@ def test_empty_and_mangled_database_map(self): def test_missing_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) @@ -3375,7 +3226,7 @@ def test_missing_database_map(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() # backup has missing database_map and that is legal @@ -3419,9 +3270,6 @@ def test_missing_database_map(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_stream_restore_command_option(self): """ @@ -3436,14 +3284,13 @@ def test_stream_restore_command_option(self): as replica, check that PostgreSQL recovery uses restore_command to obtain WAL from archive. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'max_wal_size': '32MB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3500,20 +3347,16 @@ def test_stream_restore_command_option(self): self.assertEqual('2', timeline_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_primary_conninfo(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3526,7 +3369,7 @@ def test_restore_primary_conninfo(self): #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' @@ -3553,20 +3396,16 @@ def test_restore_primary_conninfo(self): self.assertIn(str_conninfo, recovery_conf_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_primary_slot_info(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3577,7 +3416,7 @@ def test_restore_primary_slot_info(self): node.pgbench_init(scale=1) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() node.safe_psql( @@ -3598,17 +3437,13 @@ def test_restore_primary_slot_info(self): replica.slow_start(replica=True) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_issue_249(self): """ https://github.com/postgrespro/pg_probackup/issues/249 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3643,7 +3478,7 @@ def test_issue_249(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() self.restore_node( @@ -3667,9 +3502,6 @@ def test_issue_249(self): except QueryException as e: self.assertIn('FATAL', e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_pg_12_probackup_recovery_conf_compatibility(self): """ https://github.com/postgrespro/pg_probackup/issues/249 @@ -3685,10 +3517,9 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3741,9 +3572,6 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): node.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_drop_postgresql_auto_conf(self): """ https://github.com/postgrespro/pg_probackup/issues/249 @@ -3754,10 +3582,9 @@ def test_drop_postgresql_auto_conf(self): if self.pg_config_version < self.version_to_num('12.0'): return unittest.skip('You need PostgreSQL >= 12 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3786,9 +3613,6 @@ def test_drop_postgresql_auto_conf(self): self.assertTrue(os.path.exists(auto_path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_truncate_postgresql_auto_conf(self): """ https://github.com/postgrespro/pg_probackup/issues/249 @@ -3799,10 +3623,9 @@ def test_truncate_postgresql_auto_conf(self): if self.pg_config_version < self.version_to_num('12.0'): return unittest.skip('You need PostgreSQL >= 12 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3831,18 +3654,14 @@ def test_truncate_postgresql_auto_conf(self): self.assertTrue(os.path.exists(auto_path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_concurrent_restore(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3869,7 +3688,7 @@ def test_concurrent_restore(self): pgdata1 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node.cleanup() node_restored.cleanup() @@ -3892,19 +3711,15 @@ def test_concurrent_restore(self): self.compare_pgdata(pgdata1, pgdata2) self.compare_pgdata(pgdata2, pgdata3) - # Clean after yourself - self.del_test_dir(module_name, fname) - # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 @unittest.skip("skip") def test_restore_issue_313(self): """ Check that partially restored PostgreSQL instance cannot be started """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3925,7 +3740,7 @@ def test_restore_issue_313(self): count += 1 node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3956,18 +3771,14 @@ def test_restore_issue_313(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_waldir(self): """recovery using tablespace-mapping option and page backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -4011,6 +3822,3 @@ def test_restore_with_waldir(self): wal_path=os.path.join(node.data_dir, "pg_xlog") self.assertEqual(os.path.islink(wal_path), True) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/retention.py b/tests/retention.py index 122ab28ad..5043366f4 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -6,21 +6,17 @@ from distutils.dir_util import copy_tree -module_name = 'retention' - - class RetentionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -71,18 +67,14 @@ def test_retention_redundancy_1(self): self.assertTrue(wal_name >= min_wal) self.assertTrue(wal_name <= max_wal) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -124,18 +116,14 @@ def test_retention_window_2(self): self.delete_expired(backup_dir, 'node', options=['--expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -171,18 +159,14 @@ def test_retention_window_3(self): # count wal files in ARCHIVE - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_4(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -232,18 +216,14 @@ def test_retention_window_4(self): n_wals = len(os.listdir(wals_dir)) self.assertTrue(n_wals == 0) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -356,18 +336,14 @@ def test_window_expire_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_redundancy_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -466,18 +442,14 @@ def test_redundancy_expire_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -603,9 +575,6 @@ def test_window_merge_interleaved_incremental_chains(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains_1(self): """ @@ -616,12 +585,11 @@ def test_window_merge_interleaved_incremental_chains_1(self): FULLb FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -745,9 +713,6 @@ def test_window_merge_interleaved_incremental_chains_1(self): pgdata_restored_b3 = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata_b3, pgdata_restored_b3) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants(self): """ @@ -761,12 +726,11 @@ def test_basic_window_merge_multiple_descendants(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1005,9 +969,6 @@ def test_basic_window_merge_multiple_descendants(self): self.show_pb(backup_dir, 'node')[0]['backup-mode'], 'FULL') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): """ @@ -1021,12 +982,11 @@ def test_basic_window_merge_multiple_descendants_1(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1271,9 +1231,6 @@ def test_basic_window_merge_multiple_descendants_1(self): '--retention-window=1', '--delete-expired', '--log-level-console=log']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_chains(self): """ @@ -1286,12 +1243,11 @@ def test_window_chains(self): PAGE FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1357,9 +1313,6 @@ def test_window_chains(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_chains_1(self): """ @@ -1372,12 +1325,11 @@ def test_window_chains_1(self): PAGE FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1451,9 +1403,6 @@ def test_window_chains_1(self): "Purging finished", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_window_error_backups(self): """ @@ -1466,12 +1415,11 @@ def test_window_error_backups(self): FULL -------redundancy """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1488,9 +1436,6 @@ def test_window_error_backups(self): # Change FULLb backup status to ERROR # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_error_backups_1(self): """ @@ -1501,12 +1446,11 @@ def test_window_error_backups_1(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1538,9 +1482,6 @@ def test_window_error_backups_1(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_error_backups_2(self): """ @@ -1551,12 +1492,11 @@ def test_window_error_backups_2(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1589,23 +1529,19 @@ def test_window_error_backups_2(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) - # Clean after yourself - # self.del_test_dir(module_name, fname) - def test_retention_redundancy_overlapping_chains(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1639,23 +1575,19 @@ def test_retention_redundancy_overlapping_chains(self): self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_retention_redundancy_overlapping_chains_1(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) + self.del_test_dir(self.module_name, self.fname) return unittest.skip('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1689,19 +1621,15 @@ def test_retention_redundancy_overlapping_chains_1(self): self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_wal_purge_victim(self): """ https://github.com/postgrespro/pg_probackup/issues/103 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1746,9 +1674,6 @@ def test_wal_purge_victim(self): "WARNING: Backup {0} has missing parent 0".format(page_id), e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_failed_merge_redundancy_retention(self): """ @@ -1756,11 +1681,10 @@ def test_failed_merge_redundancy_retention(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join( - module_name, fname, 'node'), + self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1844,9 +1768,6 @@ def test_failed_merge_redundancy_retention(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_wal_depth_1(self): """ |-------------B5----------> WAL timeline3 @@ -1855,10 +1776,9 @@ def test_wal_depth_1(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1901,7 +1821,7 @@ def test_wal_depth_1(self): # Timeline 2 node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -1960,8 +1880,6 @@ def test_wal_depth_1(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_wal_purge(self): """ -------------------------------------> tli5 @@ -1982,10 +1900,9 @@ def test_wal_purge(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2022,7 +1939,7 @@ def test_wal_purge(self): # TLI 2 node_tli2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) node_tli2.cleanup() output = self.restore_node( @@ -2056,7 +1973,7 @@ def test_wal_purge(self): # TLI3 node_tli3 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli3')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) node_tli3.cleanup() # Note, that successful validation here is a happy coincidence @@ -2077,7 +1994,7 @@ def test_wal_purge(self): # TLI4 node_tli4 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli4')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) node_tli4.cleanup() self.restore_node( @@ -2099,7 +2016,7 @@ def test_wal_purge(self): # TLI5 node_tli5 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli5')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) node_tli5.cleanup() self.restore_node( @@ -2182,8 +2099,6 @@ def test_wal_purge(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_wal_depth_2(self): """ -------------------------------------> tli5 @@ -2205,10 +2120,9 @@ def test_wal_depth_2(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2243,7 +2157,7 @@ def test_wal_depth_2(self): # TLI 2 node_tli2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) node_tli2.cleanup() output = self.restore_node( @@ -2277,7 +2191,7 @@ def test_wal_depth_2(self): # TLI3 node_tli3 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli3')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) node_tli3.cleanup() # Note, that successful validation here is a happy coincidence @@ -2298,7 +2212,7 @@ def test_wal_depth_2(self): # TLI4 node_tli4 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli4')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) node_tli4.cleanup() self.restore_node( @@ -2320,7 +2234,7 @@ def test_wal_depth_2(self): # TLI5 node_tli5 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli5')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) node_tli5.cleanup() self.restore_node( @@ -2439,8 +2353,6 @@ def test_wal_depth_2(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_basic_wal_depth(self): """ B1---B1----B3-----B4----B5------> tli1 @@ -2450,10 +2362,9 @@ def test_basic_wal_depth(self): wal-depth=1 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2542,18 +2453,15 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_concurrent_running_full_backup(self): """ https://github.com/postgrespro/pg_probackup/issues/328 """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2621,5 +2529,3 @@ def test_concurrent_running_full_backup(self): self.assertEqual( len(self.show_pb(backup_dir, 'node')), 6) - - self.del_test_dir(module_name, fname) diff --git a/tests/set_backup.py b/tests/set_backup.py index 02ce007bf..e789d174a 100644 --- a/tests/set_backup.py +++ b/tests/set_backup.py @@ -5,8 +5,6 @@ from sys import exit from datetime import datetime, timedelta -module_name = 'set_backup' - class SetBackupTest(ProbackupTest, unittest.TestCase): @@ -14,10 +12,9 @@ class SetBackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_set_backup_sanity(self): """general sanity for set-backup command""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -120,19 +117,15 @@ def test_set_backup_sanity(self): # parse string to datetime object #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_pinning(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -174,18 +167,14 @@ def test_retention_redundancy_pinning(self): '{1} is guarded by retention'.format(full_id, page_id), log) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_pinning(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -237,9 +226,6 @@ def test_retention_window_pinning(self): '{1} is guarded by retention'.format(backup_id_1, page1), out) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_wal_retention_and_pinning(self): """ @@ -251,13 +237,12 @@ def test_wal_retention_and_pinning(self): B1 B2---P---B3---> """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -317,9 +302,6 @@ def test_wal_retention_and_pinning(self): '000000010000000000000004') self.assertEqual(timeline['status'], 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_wal_retention_and_pinning_1(self): """ @@ -331,12 +313,11 @@ def test_wal_retention_and_pinning_1(self): P---B1---> """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -383,19 +364,15 @@ def test_wal_retention_and_pinning_1(self): self.validate_pb(backup_dir) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_add_note_newlines(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -418,19 +395,15 @@ def test_add_note_newlines(self): backup_meta = self.show_pb(backup_dir, 'node', backup_id) self.assertNotIn('note', backup_meta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_add_big_note(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -472,19 +445,16 @@ def test_add_big_note(self): backup_meta = self.show_pb(backup_dir, 'node', backup_id) self.assertEqual(backup_meta['note'], note) - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_add_big_note_1(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -504,6 +474,3 @@ def test_add_big_note_1(self): print(backup_meta) self.assertEqual(backup_meta['note'], note) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/show.py b/tests/show.py index 5a46e5ef7..c4b96499d 100644 --- a/tests/show.py +++ b/tests/show.py @@ -3,19 +3,15 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'show' - - class ShowTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_show_1(self): """Status DONE and OK""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -31,17 +27,13 @@ def test_show_1(self): ) self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_show_json(self): """Status DONE and OK""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -58,16 +50,12 @@ def test_show_json(self): self.backup_node(backup_dir, 'node', node) self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -102,16 +90,12 @@ def test_corrupt_2(self): ) self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_no_control_file(self): """backup.control doesn't exist""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -137,16 +121,12 @@ def test_no_control_file(self): 'doesn\'t exist', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_empty_control_file(self): """backup.control is empty""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -173,17 +153,13 @@ def test_empty_control_file(self): 'is empty', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_control_file(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -205,9 +181,6 @@ def test_corrupt_control_file(self): 'WARNING: Invalid option "statuss" in file', self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness(self): @@ -215,10 +188,9 @@ def test_corrupt_correctness(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -299,9 +271,6 @@ def test_corrupt_correctness(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness_1(self): @@ -309,10 +278,9 @@ def test_corrupt_correctness_1(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -397,9 +365,6 @@ def test_corrupt_correctness_1(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness_2(self): @@ -407,10 +372,9 @@ def test_corrupt_correctness_2(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -512,17 +476,13 @@ def test_corrupt_correctness_2(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_color_with_no_terminal(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={'autovacuum': 'off'}) @@ -547,6 +507,3 @@ def test_color_with_no_terminal(self): '[0m', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/time_consuming.py b/tests/time_consuming.py index c778b9bc3..8270298cc 100644 --- a/tests/time_consuming.py +++ b/tests/time_consuming.py @@ -4,7 +4,6 @@ import subprocess from time import sleep -module_name = 'time_consuming' class TimeConsumingTests(ProbackupTest, unittest.TestCase): def test_pbckp150(self): @@ -20,9 +19,8 @@ def test_pbckp150(self): if not self.ptrack: return unittest.skip('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -39,7 +37,7 @@ def test_pbckp150(self): self.set_auto_conf(node, {'wal_keep_segments': '1000'}) # init probackup and add an instance - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -77,6 +75,3 @@ def test_pbckp150(self): backups = self.show_pb(backup_dir, 'node') for b in backups: self.assertEqual("OK", b['status']) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/time_stamp.py b/tests/time_stamp.py index c49d183da..170c62cd4 100644 --- a/tests/time_stamp.py +++ b/tests/time_stamp.py @@ -5,22 +5,19 @@ from time import sleep -module_name = 'time_stamp' - class TimeStamp(ProbackupTest, unittest.TestCase): def test_start_time_format(self): """Test backup ID changing after start-time editing in backup.control. We should convert local time in UTC format""" # Create simple node - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -58,19 +55,16 @@ def test_start_time_format(self): self.assertNotIn("backup ID in control file", output) node.stop() - # Clean after yourself - self.del_test_dir(module_name, fname) def test_server_date_style(self): """Issue #112""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={"datestyle": "GERMAN, DMY"}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.start() @@ -78,18 +72,14 @@ def test_server_date_style(self): self.backup_node( backup_dir, 'node', node, options=['--stream', '-j 2']) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_handling_of_TZ_env_variable(self): """Issue #284""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.start() @@ -104,17 +94,13 @@ def test_handling_of_TZ_env_variable(self): self.assertNotIn("backup ID in control file", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_dst_timezone_handling(self): """for manual testing""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -180,16 +166,12 @@ def test_dst_timezone_handling(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_dst_timezone_handling_backward_compatibilty(self): """for manual testing""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -252,6 +234,3 @@ def test_dst_timezone_handling_backward_compatibilty(self): ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/validate.py b/tests/validate.py index 966ad81a8..5c3e31fe3 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -9,9 +9,6 @@ import hashlib -module_name = 'validate' - - class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -20,12 +17,11 @@ def test_basic_validate_nullified_heap_page_backup(self): """ make node with nullified heap block """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -71,9 +67,6 @@ def test_basic_validate_nullified_heap_page_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_validate_wal_unreal_values(self): @@ -81,12 +74,11 @@ def test_validate_wal_unreal_values(self): make node with archiving, make archive backup validate to both real and unreal values """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -213,9 +205,6 @@ def test_validate_wal_unreal_values(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_validate_corrupted_intermediate_backup(self): """ @@ -224,12 +213,11 @@ def test_basic_validate_corrupted_intermediate_backup(self): run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -298,9 +286,6 @@ def test_basic_validate_corrupted_intermediate_backup(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): """ @@ -309,12 +294,11 @@ def test_validate_corrupted_intermediate_backups(self): expect FULL and PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -419,9 +403,6 @@ def test_validate_corrupted_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_error_intermediate_backups(self): """ @@ -431,12 +412,11 @@ def test_validate_specific_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -507,9 +487,6 @@ def test_validate_specific_error_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_error_intermediate_backups(self): """ @@ -519,12 +496,11 @@ def test_validate_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -591,9 +567,6 @@ def test_validate_error_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups_1(self): """ @@ -602,12 +575,11 @@ def test_validate_corrupted_intermediate_backups_1(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -788,9 +760,6 @@ def test_validate_corrupted_intermediate_backups_1(self): 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_target_corrupted_intermediate_backups(self): """ @@ -799,12 +768,11 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -980,9 +948,6 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_several_corrupt_backups(self): """ @@ -991,12 +956,11 @@ def test_validate_instance_with_several_corrupt_backups(self): expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1081,9 +1045,6 @@ def test_validate_instance_with_several_corrupt_backups(self): 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ @@ -1091,12 +1052,11 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1180,9 +1140,6 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): self.assertNotIn( 'Interrupted while locking backup', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_corrupted_page(self): """ @@ -1190,12 +1147,11 @@ def test_validate_instance_with_corrupted_page(self): corrupt file in PAGE1 backup and run validate on instance, expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1327,20 +1283,16 @@ def test_validate_instance_with_corrupted_page(self): 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_corrupted_full_and_try_restore(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, try to restore backup with --no-validation option""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1423,19 +1375,15 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_corrupted_full(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1517,18 +1465,14 @@ def test_validate_instance_with_corrupted_full(self): self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_1(self): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1579,17 +1523,13 @@ def test_validate_corrupt_wal_1(self): self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1645,9 +1585,6 @@ def test_validate_corrupt_wal_2(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): """make archive node, make archive full backup, @@ -1655,12 +1592,11 @@ def test_validate_wal_lost_segment_1(self): run validate, expecting error because of missing wal segment make sure that backup status is 'CORRUPT' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1722,21 +1658,17 @@ def test_validate_wal_lost_segment_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): """ make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1815,22 +1747,18 @@ def test_validate_corrupt_wal_between_backups(self): self.show_pb(backup_dir, 'node')[1]['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro702_688(self): """ make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1856,22 +1784,18 @@ def test_pgpro702_688(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro688(self): """ make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1885,9 +1809,6 @@ def test_pgpro688(self): backup_dir, 'node', options=["--time={0}".format(recovery_time), "-j", "4"]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_pgpro561(self): @@ -1895,13 +1816,12 @@ def test_pgpro561(self): make node with archiving, make stream backup, restore it to node1, check that archiving is not successful on node1 """ - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) self.set_archiving(backup_dir, 'node1', node1) @@ -1911,7 +1831,7 @@ def test_pgpro561(self): backup_dir, 'node1', node1, options=["--stream"]) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() node1.psql( @@ -1993,9 +1913,6 @@ def test_pgpro561(self): self.assertFalse( 'pg_probackup archive-push completed successfully' in log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full(self): """ @@ -2006,15 +1923,14 @@ def test_validate_corrupted_full(self): remove corruption and run valudate again, check that second full backup and his page backups are OK """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2114,9 +2030,6 @@ def test_validate_corrupted_full(self): self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_1(self): """ @@ -2130,13 +2043,12 @@ def test_validate_corrupted_full_1(self): second page should be CORRUPT third page should be ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2216,9 +2128,6 @@ def test_validate_corrupted_full_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_2(self): """ @@ -2241,13 +2150,12 @@ def test_validate_corrupted_full_2(self): remove corruption from PAGE2_2 and run validate on PAGE2_4 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2578,9 +2486,6 @@ def test_validate_corrupted_full_2(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_missing(self): """ @@ -2593,13 +2498,12 @@ def test_validate_corrupted_full_missing(self): second full backup and his firts page backups are OK, third page should be ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2811,18 +2715,14 @@ def test_validate_corrupted_full_missing(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_file_size_corruption_no_validate(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), # initdb_params=['--data-checksums'], ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2873,9 +2773,6 @@ def test_file_size_corruption_no_validate(self): "ERROR: Backup files restoring failed" in e.message, repr(e.message)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup(self): """ @@ -2894,11 +2791,11 @@ def test_validate_specific_backup_with_missing_backup(self): """ fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3016,9 +2913,6 @@ def test_validate_specific_backup_with_missing_backup(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup_1(self): """ @@ -3035,13 +2929,12 @@ def test_validate_specific_backup_with_missing_backup_1(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3137,9 +3030,6 @@ def test_validate_specific_backup_with_missing_backup_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_with_missing_backup_1(self): """ @@ -3156,13 +3046,12 @@ def test_validate_with_missing_backup_1(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3326,9 +3215,6 @@ def test_validate_with_missing_backup_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_with_missing_backup_2(self): """ @@ -3345,13 +3231,12 @@ def test_validate_with_missing_backup_2(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3486,19 +3371,15 @@ def test_validate_with_missing_backup_2(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_corrupt_pg_control_via_resetxlog(self): """ PGPRO-2096 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3556,18 +3437,14 @@ def test_corrupt_pg_control_via_resetxlog(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validation_after_backup(self): """""" self._check_gdb_flag_or_skip_test() - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3596,19 +3473,15 @@ def test_validation_after_backup(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_corrupt_tablespace_map(self): """ Check that corruption in tablespace_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3651,9 +3524,6 @@ def test_validate_corrupt_tablespace_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - #TODO fix the test @unittest.expectedFailure # @unittest.skip("skip") @@ -3661,10 +3531,9 @@ def test_validate_target_lsn(self): """ Check validation to specific LSN """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3683,7 +3552,7 @@ def test_validate_target_lsn(self): "from generate_series(0,10000) i") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3709,17 +3578,13 @@ def test_validate_target_lsn(self): '--recovery-target-timeline=2', '--recovery-target-lsn={0}'.format(target_lsn)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_empty_and_mangled_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3783,16 +3648,12 @@ def test_partial_validate_empty_and_mangled_database_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_exclude(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3855,17 +3716,13 @@ def test_partial_validate_exclude(self): self.assertIn( "VERBOSE: Skip file validation due to partial restore", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_include(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3917,18 +3774,14 @@ def test_partial_validate_include(self): self.assertNotIn( "VERBOSE: Skip file validation due to partial restore", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_not_validate_diffenent_pg_version(self): """Do not validate backup, if binary is compiled with different PG version""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3971,19 +3824,15 @@ def test_not_validate_diffenent_pg_version(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_corrupt_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -4045,19 +3894,15 @@ def test_validate_corrupt_page_header_map(self): self.assertIn("WARNING: Some backups are not valid", e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_truncated_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -4108,19 +3953,15 @@ def test_validate_truncated_page_header_map(self): self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) self.assertIn("WARNING: Some backups are not valid", e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_missing_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -4168,19 +4009,15 @@ def test_validate_missing_page_header_map(self): self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) self.assertIn("WARNING: Some backups are not valid", e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_no_validate_tablespace_map(self): """ Check that --no-validate is propagated to tablespace_map """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -4233,9 +4070,6 @@ def test_no_validate_tablespace_map(self): tblspace_new, "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # validate empty backup list # page from future during validate # page from future during backup From 39e06f576bc1b3f73e5243afeb668befb412ef54 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 9 Nov 2022 01:56:11 +0300 Subject: [PATCH 1953/2107] [PBCKP-304] test dirs cleanup fixed - added logic for nodes GC and dirs cleanup --- tests/helpers/ptrack_helpers.py | 34 +++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bd5ea01fd..11966dd22 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -205,6 +205,8 @@ class ProbackupTest(object): def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) + self.nodes_to_cleanup = [] + if isinstance(self, unittest.TestCase): self.module_name = self.id().split('.')[1] self.fname = self.id().split('.')[3] @@ -373,11 +375,23 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" def tearDown(self): - if isinstance(self, unittest.TestCase): - module_name = self.id().split('.')[1] - fname = self.id().split('.')[3] - if is_test_result_ok(self): + if is_test_result_ok(self): + for node in self.nodes_to_cleanup: + node.cleanup() + # we do clear refs to nodes to gather them by gc inside self.del_test_dir() + self.nodes_to_cleanup.clear() + + if isinstance(self, unittest.TestCase): + module_name = self.id().split('.')[1] + fname = self.id().split('.')[3] self.del_test_dir(module_name, fname) + else: + for node in self.nodes_to_cleanup: + # TODO VERIFY do we want to remain failed test's db data for further investigations? + # TODO VERIFY or just to leave logs only without node/data? + # node._try_shutdown(max_attempts=1) + node.cleanup() + self.nodes_to_cleanup.clear() @property def pg_config_version(self): @@ -475,6 +489,9 @@ def make_simple_node( if node.major_version >= 13: self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) + + self.nodes_to_cleanup.append(node) + return node def simple_bootstrap(self, node, role) -> None: @@ -1689,6 +1706,15 @@ def get_bin_path(self, binary): return testgres.get_bin_path(binary) def clean_all(self): + # pre gc.collect() all dropped nodes + for o in gc.get_referrers(testgres.PostgresNode): + if o.__class__ is testgres.PostgresNode: + # removing node from slow_start enclosure + # after this the node is collectable by gc + o.slow_start = None + gc.collect() + + # only when there are unhandled nodes left we do the cleanup for them for o in gc.get_referrers(testgres.PostgresNode): if o.__class__ is testgres.PostgresNode: o.cleanup() From a2c1e6d6b4adade6d386d603f2652b60f996fb93 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 9 Nov 2022 02:50:36 +0300 Subject: [PATCH 1954/2107] [PBCKP-304] removed direct calls to ProbackupTest.del_test_dir() from tests --- tests/archive.py | 7 ------- tests/backup.py | 1 - tests/catchup.py | 2 -- tests/compatibility.py | 3 --- tests/incr_restore.py | 5 ----- tests/merge.py | 7 ------- tests/pgpro560.py | 2 -- tests/ptrack.py | 3 --- tests/replica.py | 15 --------------- tests/restore.py | 2 -- tests/retention.py | 2 -- 11 files changed, 49 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index f40cf3c5d..acdf39eca 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -78,7 +78,6 @@ def test_pgpro434_2(self): ) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') @@ -681,7 +680,6 @@ def test_replica_archive(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -808,7 +806,6 @@ def test_master_and_replica_parallel_archiving(self): ) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -899,7 +896,6 @@ def test_basic_master_and_replica_concurrent_archiving(self): 'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1221,7 +1217,6 @@ def test_archive_catalog(self): 'checkpoint_timeout': '30s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -2083,7 +2078,6 @@ def test_archive_pg_receivexlog_partial_handling(self): initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -2284,7 +2278,6 @@ def test_archive_get_batching_sanity(self): initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') diff --git a/tests/backup.py b/tests/backup.py index b7bb1b8b4..5f8a9038c 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -2253,7 +2253,6 @@ def test_backup_with_less_privileges_role(self): datname='backupdb', options=['--stream', '-U', 'backup']) if self.get_version(node) < 90600: - self.del_test_dir(self.module_name, self.fname) return # Restore as replica diff --git a/tests/catchup.py b/tests/catchup.py index 12622207a..d0dd11e81 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -53,7 +53,6 @@ def test_basic_full_catchup(self): # Cleanup dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') - self.del_test_dir(self.module_name, self.fname) def test_full_catchup_with_tablespace(self): """ @@ -1533,7 +1532,6 @@ def test_dry_run_catchup_ptrack(self): # Cleanup src_pg.stop() - self.del_test_dir(self.module_name, self.fname) def test_dry_run_catchup_delta(self): """ diff --git a/tests/compatibility.py b/tests/compatibility.py index 8a7812c57..a244ce687 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -86,9 +86,6 @@ def test_catchup_with_different_remote_major_pg(self): options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] ) - # Clean after yourself - self.del_test_dir(self.module_name, self.fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): diff --git a/tests/incr_restore.py b/tests/incr_restore.py index 55d59fa99..08a92b5b0 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1389,9 +1389,6 @@ def test_incr_checksum_restore_backward(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(self.module_name, self.fname) - # @unittest.skip("skip") def test_make_replica_via_incr_checksum_restore(self): """ @@ -1403,7 +1400,6 @@ def test_make_replica_via_incr_checksum_restore(self): initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1472,7 +1468,6 @@ def test_make_replica_via_incr_lsn_restore(self): initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') diff --git a/tests/merge.py b/tests/merge.py index fa0da7b2b..a323bbba2 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -754,9 +754,6 @@ def test_merge_delta_truncate(self): self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(self.module_name, self.fname) - def test_merge_ptrack_truncate(self): """ make node, create table, take full backup, @@ -1401,8 +1398,6 @@ def test_crash_after_opening_backup_control_1(self): self.assertEqual( 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - self.del_test_dir(self.module_name, self.fname) - # @unittest.skip("skip") def test_crash_after_opening_backup_control_2(self): """ @@ -2657,8 +2652,6 @@ def test_missing_non_data_file(self): self.assertEqual( 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - self.del_test_dir(self.module_name, self.fname) - # @unittest.skip("skip") def test_merge_remote_mode(self): """ diff --git a/tests/pgpro560.py b/tests/pgpro560.py index eeab59960..b665fd200 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560.py @@ -45,10 +45,8 @@ def test_pgpro560_control_file_loss(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself # Return this file to avoid Postger fail os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) - self.del_test_dir(self.module_name, self.fname) def test_pgpro560_systemid_mismatch(self): """ diff --git a/tests/ptrack.py b/tests/ptrack.py index 2b8d2d49e..74db2e554 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -3096,9 +3096,6 @@ def test_basic_ptrack_truncate_replica(self): 'postgres', 'select 1') - # Clean after yourself - self.del_test_dir(self.module_name, self.fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): diff --git a/tests/replica.py b/tests/replica.py index 3fb68633f..c74385c5c 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -26,7 +26,6 @@ def test_replica_switchover(self): initdb_params=['--data-checksums']) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -229,7 +228,6 @@ def test_replica_archive_page_backup(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -367,7 +365,6 @@ def test_basic_make_replica_via_restore(self): 'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -421,7 +418,6 @@ def test_take_backup_from_delayed_replica(self): pg_options={'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -530,7 +526,6 @@ def test_replica_promote(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -617,7 +612,6 @@ def test_replica_stop_lsn_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -700,7 +694,6 @@ def test_replica_stop_lsn_null_offset_next_record(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -799,7 +792,6 @@ def test_archive_replica_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -879,7 +871,6 @@ def test_archive_replica_not_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -964,7 +955,6 @@ def test_replica_toast(self): 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1064,7 +1054,6 @@ def test_start_stop_lsn_in_the_same_segno(self): 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1138,7 +1127,6 @@ def test_replica_promote_1(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1257,7 +1245,6 @@ def test_replica_promote_archive_delta(self): 'archive_timeout': '30s'}) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1378,7 +1365,6 @@ def test_replica_promote_archive_page(self): 'archive_timeout': '30s'}) if self.get_version(node1) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') @@ -1496,7 +1482,6 @@ def test_parent_choosing(self): initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(self.module_name, self.fname) return unittest.skip( 'Skipped because backup from replica is not supported in PG 9.5') diff --git a/tests/restore.py b/tests/restore.py index 52db63c1c..5cd389e8b 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -334,7 +334,6 @@ def test_restore_to_lsn_inclusive(self): initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(self.module_name, self.fname) return backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') @@ -401,7 +400,6 @@ def test_restore_to_lsn_not_inclusive(self): initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(self.module_name, self.fname) return backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') diff --git a/tests/retention.py b/tests/retention.py index 5043366f4..f9969c31a 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1538,7 +1538,6 @@ def test_retention_redundancy_overlapping_chains(self): initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(self.module_name, self.fname) return unittest.skip('Skipped because ptrack support is disabled') backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') @@ -1584,7 +1583,6 @@ def test_retention_redundancy_overlapping_chains_1(self): initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(self.module_name, self.fname) return unittest.skip('Skipped because ptrack support is disabled') backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') From cda016c9552d48fd809c99fff4221d0779e87f2b Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 9 Nov 2022 03:44:47 +0300 Subject: [PATCH 1955/2107] [PBCKP-304] removed ProbackupTest.clean_all() GC magic, simplified ProbackupTest.tearDown() --- tests/helpers/ptrack_helpers.py | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 11966dd22..2729b7a4f 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -378,20 +378,15 @@ def tearDown(self): if is_test_result_ok(self): for node in self.nodes_to_cleanup: node.cleanup() - # we do clear refs to nodes to gather them by gc inside self.del_test_dir() - self.nodes_to_cleanup.clear() + self.del_test_dir(self.module_name, self.fname) - if isinstance(self, unittest.TestCase): - module_name = self.id().split('.')[1] - fname = self.id().split('.')[3] - self.del_test_dir(module_name, fname) else: for node in self.nodes_to_cleanup: - # TODO VERIFY do we want to remain failed test's db data for further investigations? - # TODO VERIFY or just to leave logs only without node/data? - # node._try_shutdown(max_attempts=1) - node.cleanup() - self.nodes_to_cleanup.clear() + # TODO make decorator with proper stop() vs cleanup() + node._try_shutdown(max_attempts=1) + # node.cleanup() + + self.nodes_to_cleanup.clear() @property def pg_config_version(self): @@ -1705,25 +1700,9 @@ def get_ptrack_version(self, node): def get_bin_path(self, binary): return testgres.get_bin_path(binary) - def clean_all(self): - # pre gc.collect() all dropped nodes - for o in gc.get_referrers(testgres.PostgresNode): - if o.__class__ is testgres.PostgresNode: - # removing node from slow_start enclosure - # after this the node is collectable by gc - o.slow_start = None - gc.collect() - - # only when there are unhandled nodes left we do the cleanup for them - for o in gc.get_referrers(testgres.PostgresNode): - if o.__class__ is testgres.PostgresNode: - o.cleanup() - def del_test_dir(self, module_name, fname): """ Del testdir and optimistically try to del module dir""" - self.clean_all() - shutil.rmtree( os.path.join( self.tmp_path, From 04e05b151dd26878ba64b9650be6db3b7ebf5a7e Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 9 Nov 2022 12:17:10 +0300 Subject: [PATCH 1956/2107] Fix auth_test.py --- tests/__init__.py | 2 +- tests/auth_test.py | 189 ++++++++++++++++++++------------------------- 2 files changed, 86 insertions(+), 105 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index c02788e29..40d5faf65 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -27,7 +27,7 @@ def load_tests(loader, tests, pattern): if os.environ['PG_PROBACKUP_LONG'] == 'ON': suite.addTests(loader.loadTestsFromModule(time_consuming)) -# suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(auth_test)) suite.addTests(loader.loadTestsFromModule(archive)) suite.addTests(loader.loadTestsFromModule(backup)) suite.addTests(loader.loadTestsFromModule(catchup)) diff --git a/tests/auth_test.py b/tests/auth_test.py index 4b0c4a5b2..7e0b6fcfb 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -126,24 +126,16 @@ def test_backup_via_unprivileged_user(self): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - elif self.get_vestion(node) < self.version_to_num('15.0'): + elif self.get_version(node) < self.version_to_num('15.0'): node.safe_psql( "postgres", - "GRANT EXECUTE ON FUNCTION " - "pg_stop_backup(boolean, boolean) TO backup") - # Do this for ptrack backups - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup;") else: node.safe_psql( "postgres", - "GRANT EXECUTE ON FUNCTION " - "pg_backup_stop(boolean) TO backup") - # Do this for ptrack backups - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup") + "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup;") self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -193,7 +185,10 @@ def setUpClass(cls): set_replication=True, initdb_params=['--data-checksums', '--auth-host=md5'] ) - modify_pg_hba(cls.node) + + cls.username = cls.pb.get_username() + + cls.modify_pg_hba(cls.node) cls.pb.init_pb(cls.backup_dir) cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) @@ -203,7 +198,7 @@ def setUpClass(cls): except StartNodeException: raise unittest.skip("Node hasn't started") - if cls.pb.get_version(cls.node) < 150000: + if cls.pb.get_version(cls.node) < 100000: cls.node.safe_psql( "postgres", "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " @@ -218,6 +213,21 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + elif cls.pb.get_version(cls.node) < 150000: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") else: cls.node.safe_psql( "postgres", @@ -229,7 +239,7 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") @@ -244,12 +254,13 @@ def tearDownClass(cls): @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") def setUp(self): - self.cmd = ['backup', + self.pb_cmd = ['backup', '-B', self.backup_dir, '--instance', self.node.name, '-h', '127.0.0.1', '-p', str(self.node.port), '-U', 'backup', + '-d', 'postgres', '-b', 'FULL' ] @@ -269,44 +280,31 @@ def test_empty_password(self): """ Test case: PGPB_AUTH03 - zero password length """ try: self.assertIn("ERROR: no password supplied", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) - ) + self.run_pb_with_auth('\0\r\n')) except (TIMEOUT, ExceptionPexpect) as e: self.fail(e.value) def test_wrong_password(self): """ Test case: PGPB_AUTH04 - incorrect password """ - try: - self.assertIn("password authentication failed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.assertIn("password authentication failed", + self.run_pb_with_auth('wrong_password\r\n')) def test_right_password(self): """ Test case: PGPB_AUTH01 - correct password """ - try: - self.assertIn("completed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.assertIn("completed", + self.run_pb_with_auth('password\r\n')) def test_right_password_and_wrong_pgpass(self): """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - create_pgpass(self.pgpass_file, line) - try: - self.assertIn("completed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.create_pgpass(self.pgpass_file, line) + self.assertIn("completed", + self.run_pb_with_auth('password\r\n', add_args=["-W"])) def test_ctrl_c_event(self): """ Test case: PGPB_AUTH02 - send interrupt signal """ try: - run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + self.run_pb_with_auth(kill=True) except TIMEOUT: self.fail("Error: CTRL+C event ignored") @@ -314,91 +312,74 @@ def test_pgpassfile_env(self): """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - create_pgpass(path, line) + self.create_pgpass(path, line) self.pb.test_env["PGPASSFILE"] = path - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpass(self): """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - create_pgpass(self.pgpass_file, line) - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.create_pgpass(self.pgpass_file, line) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpassword(self): """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ self.pb.test_env["PGPASSWORD"] = "password" - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpassword_and_wrong_pgpass(self): """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - create_pgpass(self.pgpass_file, line) + self.create_pgpass(self.pgpass_file, line) self.pb.test_env["PGPASSWORD"] = "password" - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) - + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) -def run_pb_with_auth(cmd, password=None, kill=False): - try: - with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + def run_pb_with_auth(self, password=None, add_args = [], kill=False): + with spawn(self.pb.probackup_path, self.pb_cmd + add_args, encoding='utf-8', timeout=10) as probackup: result = probackup.expect(u"Password for user .*:", 5) if kill: probackup.kill(signal.SIGINT) elif result == 0: probackup.sendline(password) probackup.expect(EOF) - return probackup.before + return str(probackup.before) else: raise ExceptionPexpect("Other pexpect errors.") - except TIMEOUT: - raise TIMEOUT("Timeout error.") - except ExceptionPexpect: - raise ExceptionPexpect("Pexpect error.") - - -def modify_pg_hba(node): - """ - Description: - Add trust authentication for user postgres. Need for add new role and set grant. - :param node: - :return None: - """ - hba_conf = os.path.join(node.data_dir, "pg_hba.conf") - with open(hba_conf, 'r+') as fio: - data = fio.read() - fio.seek(0) - fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) - - -def create_pgpass(path, line): - with open(path, 'w') as passfile: - # host:port:db:username:password - passfile.write(line) - os.chmod(path, 0o600) + + + @classmethod + def modify_pg_hba(cls, node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data)) + + + def create_pgpass(self, path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) From d7cc00b3585515c975d05a471e49d733924da588 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 9 Nov 2022 23:02:29 +0700 Subject: [PATCH 1957/2107] Check pg_probackup binary in PATH --- tests/helpers/ptrack_helpers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index abb715b7e..1b90ac3ff 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -242,10 +242,7 @@ def __init__(self, *args, **kwargs): self.user = self.get_username() self.probackup_path = None if 'PGPROBACKUPBIN' in self.test_env: - if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and - os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) - ): + if shutil.which(self.test_env["PGPROBACKUPBIN"]): self.probackup_path = self.test_env["PGPROBACKUPBIN"] else: if self.verbose: From 4869a564d0017a6ba58c293e9273f1413f68469c Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 11 Nov 2022 22:34:46 +0300 Subject: [PATCH 1958/2107] [PBCKP-336] Fix segno calculation. It were lost during fork-name detection fix at eaf3b14c22 . And since there were no basic test for this, it were not detected. --- src/dir.c | 13 ++++++++++++- tests/backup.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index b55f25e18..6609b9f19 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1786,6 +1786,8 @@ is_forkname(char *name, size_t *pos, const char *forkname) } #define OIDCHARS 10 +#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1) +#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */ /* Set forkName if possible */ bool @@ -1793,6 +1795,7 @@ set_forkname(pgFile *file) { size_t i = 0; uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */ + uint64_t segno = 0; /* pretend it is not relation file */ file->relOid = 0; @@ -1823,8 +1826,15 @@ set_forkname(pgFile *file) /* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */ if (file->name[i] == '.' && isdigit(file->name[i+1])) { + size_t start = i+1; for (i++; isdigit(file->name[i]); i++) - ; + { + if (i == start && file->name[i] == '0') + return false; + segno = segno * 10 + file->name[i] - '0'; + } + if (i - start > SEGNOCHARS || segno > MAXSEGNO) + return false; } /* CFS "fork name" */ @@ -1843,6 +1853,7 @@ set_forkname(pgFile *file) } file->relOid = oid; + file->segno = segno; file->is_datafile = file->forkName == none; return true; } diff --git a/tests/backup.py b/tests/backup.py index 6028a3ff6..ae7852da9 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -13,6 +13,38 @@ class BackupTest(ProbackupTest, unittest.TestCase): + def test_basic_full_backup(self): + """ + Just test full backup with at least two segments + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-707 From 4505def249841374797fc85679c633f7b8d1893f Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 15 Nov 2022 11:13:06 +0300 Subject: [PATCH 1959/2107] properly skip ptrack tests. --- tests/ptrack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ptrack.py b/tests/ptrack.py index a01405d6a..c02aba17c 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -16,7 +16,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): def setUp(self): if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') self.fname = self.id().split('.')[3] # @unittest.skip("skip") From b7bd831b8d9c45b7b555bd2d2033125d033987f0 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 15 Nov 2022 11:23:49 +0300 Subject: [PATCH 1960/2107] correctly test skips --- tests/archive.py | 22 +++++++++++----------- tests/backup.py | 8 ++++---- tests/catchup.py | 14 +++++++------- tests/compatibility.py | 2 +- tests/delete.py | 2 +- tests/external.py | 6 +++--- tests/false_positive.py | 2 +- tests/helpers/__init__.py | 9 ++++++++- tests/incr_restore.py | 4 ++-- tests/merge.py | 2 +- tests/option.py | 2 +- tests/ptrack.py | 2 +- tests/replica.py | 34 +++++++++++++++++----------------- tests/restore.py | 18 +++++++++--------- tests/retention.py | 4 ++-- tests/time_consuming.py | 4 ++-- 16 files changed, 71 insertions(+), 64 deletions(-) diff --git a/tests/archive.py b/tests/archive.py index 81d013f6b..a65b85dba 100644 --- a/tests/archive.py +++ b/tests/archive.py @@ -86,7 +86,7 @@ def test_pgpro434_2(self): if self.get_version(node) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') self.init_pb(backup_dir) @@ -717,7 +717,7 @@ def test_replica_archive(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -848,7 +848,7 @@ def test_master_and_replica_parallel_archiving(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') replica = self.make_simple_node( @@ -929,7 +929,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): make sure that archiving on both node is working. """ if self.pg_config_version < self.version_to_num('9.6.0'): - return unittest.skip('You need PostgreSQL >= 9.6 for this test') + self.skipTest('You need PostgreSQL >= 9.6 for this test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -943,7 +943,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') replica = self.make_simple_node( @@ -1035,7 +1035,7 @@ def test_concurrent_archiving(self): """ if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1196,7 +1196,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): self.add_instance(backup_dir, 'node', node) node.slow_start() if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') + self.skipTest('You need PostgreSQL >= 10 for this test') else: pg_receivexlog_path = self.get_bin_path('pg_receivewal') @@ -1278,7 +1278,7 @@ def test_archive_catalog(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1938,7 +1938,7 @@ def test_waldir_outside_pgdata_archiving(self): check that archive-push works correct with symlinked waldir """ if self.pg_config_version < self.version_to_num('10.0'): - return unittest.skip( + self.skipTest( 'Skipped because waldir outside pgdata is supported since PG 10') fname = self.id().split('.')[3] @@ -2176,7 +2176,7 @@ def test_archive_pg_receivexlog_partial_handling(self): if self.get_version(node) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -2385,7 +2385,7 @@ def test_archive_get_batching_sanity(self): if self.get_version(node) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) diff --git a/tests/backup.py b/tests/backup.py index 6028a3ff6..2b099613a 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -1428,7 +1428,7 @@ def test_basic_temp_slot_for_stream_backup(self): pg_options={'max_wal_size': '40MB'}) if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') + self.skipTest('You need PostgreSQL >= 10 for this test') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1500,7 +1500,7 @@ def test_backup_concurrent_drop_table(self): def test_pg_11_adjusted_wal_segment_size(self): """""" if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1743,7 +1743,7 @@ def test_drop_table(self): def test_basic_missing_file_permissions(self): """""" if os.name == 'nt': - return unittest.skip('Skipped because it is POSIX only test') + self.skipTest('Skipped because it is POSIX only test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1790,7 +1790,7 @@ def test_basic_missing_file_permissions(self): def test_basic_missing_dir_permissions(self): """""" if os.name == 'nt': - return unittest.skip('Skipped because it is POSIX only test') + self.skipTest('Skipped because it is POSIX only test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/catchup.py b/tests/catchup.py index 7ecd84697..baef9d29f 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -190,7 +190,7 @@ def test_basic_ptrack_catchup(self): Test ptrack catchup """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( @@ -336,7 +336,7 @@ def test_tli_ptrack_catchup(self): Test that we correctly follow timeline change with ptrack catchup """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( @@ -475,7 +475,7 @@ def test_table_drop_with_ptrack(self): Test that dropped table in source will be dropped in ptrack catchup'ed instance too """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( @@ -590,7 +590,7 @@ def test_tablefile_truncation_with_ptrack(self): Test that truncated table in source will be truncated in ptrack catchup'ed instance too """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( @@ -655,7 +655,7 @@ def test_local_tablespace_without_mapping(self): Test that we detect absence of needed --tablespace-mapping option """ if self.remote: - return unittest.skip('Skipped because this test tests local catchup error handling') + self.skipTest('Skipped because this test tests local catchup error handling') src_pg = self.make_simple_node(base_dir = os.path.join(module_name, self.fname, 'src')) src_pg.slow_start() @@ -1035,7 +1035,7 @@ def test_unclean_ptrack_catchup(self): Test that we correctly recover uncleanly shutdowned destination """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( @@ -1507,7 +1507,7 @@ def test_dry_run_catchup_ptrack(self): Test dry-run option for catchup in incremental ptrack mode """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source src_pg = self.make_simple_node( diff --git a/tests/compatibility.py b/tests/compatibility.py index 6c2bc9204..3b913aba5 100644 --- a/tests/compatibility.py +++ b/tests/compatibility.py @@ -366,7 +366,7 @@ def test_backward_compatibility_ptrack(self): """Description in jira issue PGPRO-434""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/delete.py b/tests/delete.py index 345a70284..55d08f23b 100644 --- a/tests/delete.py +++ b/tests/delete.py @@ -189,7 +189,7 @@ def test_delete_increment_page(self): def test_delete_increment_ptrack(self): """delete increment and all after him""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( diff --git a/tests/external.py b/tests/external.py index 530e7fb26..a4e3d58f4 100644 --- a/tests/external.py +++ b/tests/external.py @@ -1535,7 +1535,7 @@ def test_external_dir_is_symlink(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1618,7 +1618,7 @@ def test_external_dir_contain_symlink_on_dir(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -1703,7 +1703,7 @@ def test_external_dir_contain_symlink_on_file(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/false_positive.py b/tests/false_positive.py index 2ededdf12..6ffc4db10 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -114,7 +114,7 @@ def test_pg_10_waldir(self): test group access for PG >= 11 """ if self.pg_config_version < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') + self.skipTest('You need PostgreSQL >= 10 for this test') fname = self.id().split('.')[3] wal_dir = os.path.join( diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index ac64c4230..4ae3ef8c4 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,2 +1,9 @@ __all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] -#from . import * \ No newline at end of file + +import unittest + +# python 2.7 compatibility +if not hasattr(unittest.TestCase, "skipTest"): + def skipTest(self, reason): + raise unittest.SkipTest(reason) + unittest.TestCase.skipTest = skipTest \ No newline at end of file diff --git a/tests/incr_restore.py b/tests/incr_restore.py index cb684a23a..5bd3711a1 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore.py @@ -1494,7 +1494,7 @@ def test_make_replica_via_incr_checksum_restore(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1567,7 +1567,7 @@ def test_make_replica_via_incr_lsn_restore(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) diff --git a/tests/merge.py b/tests/merge.py index 4c374bdfb..566de549b 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -796,7 +796,7 @@ def test_merge_ptrack_truncate(self): restore last page backup and check data correctness """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/option.py b/tests/option.py index 88e72ffd7..61c60a1dd 100644 --- a/tests/option.py +++ b/tests/option.py @@ -239,5 +239,5 @@ def test_help_6(self): help_out.read().decode("utf-8") ) else: - return unittest.skip( + self.skipTest( 'You need configure PostgreSQL with --enabled-nls option for this test') diff --git a/tests/ptrack.py b/tests/ptrack.py index a01405d6a..c02aba17c 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -16,7 +16,7 @@ class PtrackTest(ProbackupTest, unittest.TestCase): def setUp(self): if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') self.fname = self.id().split('.')[3] # @unittest.skip("skip") diff --git a/tests/replica.py b/tests/replica.py index ea69e2d01..d5c24fbc1 100644 --- a/tests/replica.py +++ b/tests/replica.py @@ -30,7 +30,7 @@ def test_replica_switchover(self): if self.get_version(node1) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -103,10 +103,10 @@ def test_replica_stream_ptrack_backup(self): take full stream backup from replica """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') if self.pg_config_version > self.version_to_num('9.6.0'): - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') fname = self.id().split('.')[3] @@ -241,7 +241,7 @@ def test_replica_archive_page_backup(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -383,7 +383,7 @@ def test_basic_make_replica_via_restore(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -441,7 +441,7 @@ def test_take_backup_from_delayed_replica(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -554,7 +554,7 @@ def test_replica_promote(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -645,7 +645,7 @@ def test_replica_stop_lsn_null_offset(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -730,7 +730,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -833,7 +833,7 @@ def test_archive_replica_null_offset(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -917,7 +917,7 @@ def test_archive_replica_not_null_offset(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1006,7 +1006,7 @@ def test_replica_toast(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1108,7 +1108,7 @@ def test_start_stop_lsn_in_the_same_segno(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1186,7 +1186,7 @@ def test_replica_promote_1(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1313,7 +1313,7 @@ def test_replica_promote_archive_delta(self): if self.get_version(node1) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1438,7 +1438,7 @@ def test_replica_promote_archive_page(self): if self.get_version(node1) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1560,7 +1560,7 @@ def test_parent_choosing(self): if self.get_version(master) < self.version_to_num('9.6.0'): self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) diff --git a/tests/restore.py b/tests/restore.py index 49538bd1f..e99f1158e 100644 --- a/tests/restore.py +++ b/tests/restore.py @@ -500,7 +500,7 @@ def test_restore_to_lsn_not_inclusive(self): def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -554,7 +554,7 @@ def test_restore_full_ptrack_archive(self): def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -615,7 +615,7 @@ def test_restore_ptrack(self): def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -673,7 +673,7 @@ def test_restore_full_ptrack_under_load(self): with loads when ptrack backup do """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -742,7 +742,7 @@ def test_restore_full_under_load_ptrack(self): with loads when full backup do """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -2345,7 +2345,7 @@ def test_pg_11_group_access(self): test group access for PG >= 11 """ if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') fname = self.id().split('.')[3] node = self.make_simple_node( @@ -3680,7 +3680,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') + self.skipTest('You need PostgreSQL >= 12 for this test') if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') @@ -3752,7 +3752,7 @@ def test_drop_postgresql_auto_conf(self): """ if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') + self.skipTest('You need PostgreSQL >= 12 for this test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') @@ -3797,7 +3797,7 @@ def test_truncate_postgresql_auto_conf(self): """ if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') + self.skipTest('You need PostgreSQL >= 12 for this test') fname = self.id().split('.')[3] backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') diff --git a/tests/retention.py b/tests/retention.py index 122ab28ad..4f95ad5e2 100644 --- a/tests/retention.py +++ b/tests/retention.py @@ -1603,7 +1603,7 @@ def test_retention_redundancy_overlapping_chains(self): if self.get_version(node) < 90600: self.del_test_dir(module_name, fname) - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) @@ -1653,7 +1653,7 @@ def test_retention_redundancy_overlapping_chains_1(self): if self.get_version(node) < 90600: self.del_test_dir(module_name, fname) - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') self.init_pb(backup_dir) diff --git a/tests/time_consuming.py b/tests/time_consuming.py index c778b9bc3..8908dfd34 100644 --- a/tests/time_consuming.py +++ b/tests/time_consuming.py @@ -16,9 +16,9 @@ def test_pbckp150(self): """ # init node if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') fname = self.id().split('.')[3] node = self.make_simple_node( From 348283b3074e286e779e221d2cb044e616898bcd Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 15 Nov 2022 13:43:09 +0300 Subject: [PATCH 1961/2107] update version to 2.5.10 --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6aeba189e..44b33d16f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -345,7 +345,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.9" +#define PROGRAM_VERSION "2.5.10" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20509 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 7c9fcbfe0..8abfe7fdd 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.9 +pg_probackup 2.5.10 From 9924ab0142f9a3192104eff143fd87613e1648dd Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 15 Nov 2022 17:45:28 +0300 Subject: [PATCH 1962/2107] [PBCKP-304] extended testgres.PosgresNode to filter excessive close() and its log entries calls --- tests/helpers/ptrack_helpers.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 9e2a10c24..bd65f7962 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -140,7 +140,7 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) -def slow_start(self, replica=False): +def _slow_start(self, replica=False): # wait for https://github.com/postgrespro/testgres/pull/50 # self.start() @@ -174,7 +174,7 @@ def slow_start(self, replica=False): sleep(0.5) -def is_test_result_ok(test_case): +def _is_test_result_ok(test_case): # sources of solution: # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 @@ -197,6 +197,28 @@ def is_test_result_ok(test_case): return ok +class PostgresNodeExtended(testgres.PostgresNode): + + def __init__(self, base_dir=None, *args, **kwargs): + super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs) + self.is_started = False + + def slow_start(self, replica=False): + _slow_start(self, replica=replica) + + def start(self, *args, **kwargs): + if not self.is_started: + super(PostgresNodeExtended, self).start(*args, **kwargs) + self.is_started = True + return self + + def stop(self, *args, **kwargs): + if self.is_started: + result = super(PostgresNodeExtended, self).stop(*args, **kwargs) + self.is_started = False + return result + + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() @@ -375,7 +397,7 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" def tearDown(self): - if is_test_result_ok(self): + if _is_test_result_ok(self): for node in self.nodes_to_cleanup: node.cleanup() self.del_test_dir(self.module_name, self.fname) @@ -418,10 +440,10 @@ def make_empty_node( shutil.rmtree(real_base_dir, ignore_errors=True) os.makedirs(real_base_dir) - node = testgres.get_new_node('test', base_dir=real_base_dir) - # bound method slow_start() to 'node' class instance - node.slow_start = slow_start.__get__(node) + node = PostgresNodeExtended(base_dir=real_base_dir) node.should_rm_dirs = True + self.nodes_to_cleanup.append(node) + return node def make_simple_node( @@ -485,8 +507,6 @@ def make_simple_node( self.set_auto_conf( node, {}, 'postgresql.conf', ['wal_keep_segments']) - self.nodes_to_cleanup.append(node) - return node def simple_bootstrap(self, node, role) -> None: From e3189e425db7917dbdde862787085b64afb30183 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Tue, 15 Nov 2022 17:56:40 +0300 Subject: [PATCH 1963/2107] [PBCKP-304] missed self.module_name & self.fname fixes --- tests/cfs_catchup.py | 14 ++------------ tests/cfs_validate_backup.py | 1 - tests/delta.py | 2 +- tests/locking.py | 2 +- tests/page.py | 2 +- tests/ptrack.py | 2 +- tests/validate.py | 3 +-- 7 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/cfs_catchup.py b/tests/cfs_catchup.py index 2cbb46729..43c3f18f1 100644 --- a/tests/cfs_catchup.py +++ b/tests/cfs_catchup.py @@ -6,13 +6,8 @@ from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_catchup' -tblspace_name = 'cfs_tblspace' - class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): - def setUp(self): - self.fname = self.id().split('.')[3] @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_full_catchup_with_tablespace(self): @@ -21,7 +16,7 @@ def test_full_catchup_with_tablespace(self): """ # preparation src_pg = self.make_simple_node( - base_dir = os.path.join(module_name, self.fname, 'src'), + base_dir = os.path.join(self.module_name, self.fname, 'src'), set_replication = True ) src_pg.slow_start() @@ -36,7 +31,7 @@ def test_full_catchup_with_tablespace(self): "CHECKPOINT") # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst')) + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') self.catchup_node( backup_mode = 'FULL', @@ -120,8 +115,3 @@ def test_full_catchup_with_tablespace(self): src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - - # Cleanup - src_pg.stop() - dst_pg.stop() - self.del_test_dir(module_name, self.fname) diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup.py index eea6f0e21..343020dfc 100644 --- a/tests/cfs_validate_backup.py +++ b/tests/cfs_validate_backup.py @@ -5,7 +5,6 @@ from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_validate_backup' tblspace_name = 'cfs_tblspace' diff --git a/tests/delta.py b/tests/delta.py index 386403151..23583fd93 100644 --- a/tests/delta.py +++ b/tests/delta.py @@ -1191,7 +1191,7 @@ def test_delta_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( diff --git a/tests/locking.py b/tests/locking.py index 8531d7de5..5367c2610 100644 --- a/tests/locking.py +++ b/tests/locking.py @@ -579,7 +579,7 @@ def test_shared_lock(self): self._check_gdb_flag_or_skip_test() node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.name, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') diff --git a/tests/page.py b/tests/page.py index b9398ec7a..e77e5c827 100644 --- a/tests/page.py +++ b/tests/page.py @@ -1414,7 +1414,7 @@ def test_page_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( diff --git a/tests/ptrack.py b/tests/ptrack.py index 74db2e554..b5eb95baf 100644 --- a/tests/ptrack.py +++ b/tests/ptrack.py @@ -4172,7 +4172,7 @@ def test_ptrack_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, self.fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( diff --git a/tests/validate.py b/tests/validate.py index 5c3e31fe3..98a0fd13f 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -2789,13 +2789,12 @@ def test_validate_specific_backup_with_missing_backup(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, self.module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) From af4fb2e8c6bf0c5f18fa862b7eb09e2013752ae0 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 16 Nov 2022 02:00:05 +0300 Subject: [PATCH 1964/2107] [PBCKP-304] moved _is_result_is_ok() into ProbackupTest class scope --- tests/helpers/ptrack_helpers.py | 50 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bd65f7962..be5d1fcb4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -174,29 +174,6 @@ def _slow_start(self, replica=False): sleep(0.5) -def _is_test_result_ok(test_case): - # sources of solution: - # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: - # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 - # - # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 - - if hasattr(test_case, '_outcome'): # Python 3.4+ - if hasattr(test_case._outcome, 'errors'): - # Python 3.4 - 3.10 (These two methods have no side effects) - result = test_case.defaultTestResult() # These two methods have no side effects - test_case._feedErrorsToResult(result, test_case._outcome.errors) - else: - # Python 3.11+ - result = test_case._outcome.result - else: # Python 2.7, 3.0-3.3 - result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) - - ok = all(test != test_case for test, text in result.errors + result.failures) - - return ok - - class PostgresNodeExtended(testgres.PostgresNode): def __init__(self, base_dir=None, *args, **kwargs): @@ -396,8 +373,33 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" + def __is_test_result_ok(test_case): + # sources of solution: + # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: + # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 + # + # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 + + if not isinstance(test_case, unittest.TestCase): + raise AssertionError("test_case is not instance of unittest.TestCase") + + if hasattr(test_case, '_outcome'): # Python 3.4+ + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ + result = test_case._outcome.result + else: # Python 2.7, 3.0-3.3 + result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + + ok = all(test != test_case for test, text in result.errors + result.failures) + + return ok + def tearDown(self): - if _is_test_result_ok(self): + if self.__is_test_result_ok(): for node in self.nodes_to_cleanup: node.cleanup() self.del_test_dir(self.module_name, self.fname) From 8c670f0f9b3e1358976e52e67518d18f305bd973 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 16 Nov 2022 13:43:20 +0300 Subject: [PATCH 1965/2107] move test_basic_full_backup => test_full_backup run it in separate travis action + add same test with stream replication --- .travis.yml | 3 ++- tests/backup.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd3c8a09a..8315f7842 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,8 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup_stream # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb diff --git a/tests/backup.py b/tests/backup.py index ae7852da9..085476f6d 100644 --- a/tests/backup.py +++ b/tests/backup.py @@ -13,7 +13,7 @@ class BackupTest(ProbackupTest, unittest.TestCase): - def test_basic_full_backup(self): + def test_full_backup(self): """ Just test full backup with at least two segments """ @@ -45,6 +45,38 @@ def test_basic_full_backup(self): # Clean after yourself self.del_test_dir(module_name, fname) + def test_full_backup_stream(self): + """ + Just test full backup with at least two segments in stream mode + """ + fname = self.id().split('.')[3] + node = self.make_simple_node( + base_dir=os.path.join(module_name, fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, + options=["--stream"]) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + # Clean after yourself + self.del_test_dir(module_name, fname) + # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-707 From 4c823b39303696e32d620b5854018b1ff282d576 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 16 Nov 2022 16:03:52 +0300 Subject: [PATCH 1966/2107] ... --- tests/helpers/ptrack_helpers.py | 71 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index bd4f20946..e35f57bce 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -139,41 +139,6 @@ def __init__(self, message, cmd): def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) - -def _slow_start(self, replica=False): - - # wait for https://github.com/postgrespro/testgres/pull/50 -# self.start() -# self.poll_query_until( -# "postgres", -# "SELECT not pg_is_in_recovery()", -# suppress={testgres.NodeConnection}) - if replica: - query = 'SELECT pg_is_in_recovery()' - else: - query = 'SELECT not pg_is_in_recovery()' - - self.start() - while True: - try: - output = self.safe_psql('template1', query).decode("utf-8").rstrip() - - if output == 't': - break - - except testgres.QueryException as e: - if 'database system is starting up' in e.message: - pass - elif 'FATAL: the database system is not accepting connections' in e.message: - pass - elif replica and 'Hot standby mode is disabled' in e.message: - raise e - else: - raise e - - sleep(0.5) - - class PostgresNodeExtended(testgres.PostgresNode): def __init__(self, base_dir=None, *args, **kwargs): @@ -181,7 +146,37 @@ def __init__(self, base_dir=None, *args, **kwargs): self.is_started = False def slow_start(self, replica=False): - _slow_start(self, replica=replica) + + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.start() + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # suppress={testgres.NodeConnection}) + if replica: + query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' + + self.start() + while True: + try: + output = self.safe_psql('template1', query).decode("utf-8").rstrip() + + if output == 't': + break + + except testgres.QueryException as e: + if 'database system is starting up' in e.message: + pass + elif 'FATAL: the database system is not accepting connections' in e.message: + pass + elif replica and 'Hot standby mode is disabled' in e.message: + raise e + else: + raise e + + sleep(0.5) def start(self, *args, **kwargs): if not self.is_started: @@ -370,7 +365,7 @@ def __init__(self, *args, **kwargs): os.environ["PGAPPNAME"] = "pg_probackup" - def __is_test_result_ok(test_case): + def is_test_result_ok(test_case): # sources of solution: # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 @@ -396,7 +391,7 @@ def __is_test_result_ok(test_case): return ok def tearDown(self): - if self.__is_test_result_ok(): + if self.is_test_result_ok(): for node in self.nodes_to_cleanup: node.cleanup() self.del_test_dir(self.module_name, self.fname) From 97f3e70edb3a8adbfce75a86593e1a453668a198 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 17 Nov 2022 12:55:15 +0300 Subject: [PATCH 1967/2107] asyncio.sleep -> time.sleep --- tests/false_positive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/false_positive.py b/tests/false_positive.py index 09dc323a8..fbb785c60 100644 --- a/tests/false_positive.py +++ b/tests/false_positive.py @@ -1,6 +1,6 @@ import unittest import os -from asyncio import sleep +from time import sleep from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta From 8f20e7eb5899c96063b39eca6cd14e0570b8c1db Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 17 Nov 2022 18:50:17 +0300 Subject: [PATCH 1968/2107] Fixed link in README.md file [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bae1171cb..c5b01ced2 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ For detailed release plans check [Milestones](https://github.com/postgrespro/pg_ ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.4.15). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/latest). ### Linux Installation #### pg_probackup for vanilla PostgreSQL From eed28135b91e790956eab9021fe65bb48989e8e3 Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Wed, 23 Nov 2022 13:47:26 +0300 Subject: [PATCH 1969/2107] hotfix: cfs_restore.py decorated to postgres enterprise only --- tests/cfs_restore.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/cfs_restore.py b/tests/cfs_restore.py index 660cef9c6..6b69b4ffe 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore.py @@ -20,6 +20,7 @@ class CfsRestoreBase(ProbackupTest, unittest.TestCase): + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def setUp(self): self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') @@ -60,6 +61,7 @@ def add_data_in_cluster(self): class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_empty_tablespace_from_fullbackup(self): """ Case: Restore empty tablespace from valid full backup. @@ -118,6 +120,7 @@ def add_data_in_cluster(self): # --- Restore from full backup ---# # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location(self): """ Case: Restore instance from valid full backup to old location. @@ -157,6 +160,7 @@ def test_restore_from_fullbackup_to_old_location(self): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_3_jobs(self): """ Case: Restore instance from valid full backup to old location. @@ -195,6 +199,7 @@ def test_restore_from_fullbackup_to_old_location_3_jobs(self): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_new_location(self): """ Case: Restore instance from valid full backup to new location. @@ -238,6 +243,7 @@ def test_restore_from_fullbackup_to_new_location(self): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_new_location_5_jobs(self): """ Case: Restore instance from valid full backup to new location. @@ -281,6 +287,7 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): self.node.stop() self.node.cleanup() @@ -327,6 +334,7 @@ def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): self.node.stop() self.node.cleanup() From 22e6c408fe99d05874f49d6a8093157420dcd1ac Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 23 Nov 2022 16:51:31 +0300 Subject: [PATCH 1970/2107] base36enc: abuse C99 compound literals lifetime. C99 introduced compound literals (`(char[14]){0}` - literal of array). Compound literals have same lifetime as block local variable, ie till the end of block. There for it is save to initiate several of them in same block and assume they are all live. This way we may rewrite base36enc into macros which uses compound literal instead of static variable to extend its result lifetime. --- src/backup.c | 2 +- src/catalog.c | 11 ++--------- src/delete.c | 18 ++++++------------ src/merge.c | 29 ++++++++--------------------- src/pg_probackup.h | 6 ++++-- src/restore.c | 15 +++++++-------- src/util.c | 33 +++++++++------------------------ src/utils/pgut.h | 6 ++++++ src/validate.c | 25 ++++++++++++------------- 9 files changed, 55 insertions(+), 90 deletions(-) diff --git a/src/backup.c b/src/backup.c index c73ee56c7..639dda63d 100644 --- a/src/backup.c +++ b/src/backup.c @@ -735,7 +735,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, /* don't care about freeing base36enc_dup memory, we exit anyway */ elog(ERROR, "Can't assign backup_id from requested start_time (%s), " "this time must be later that backup %s", - base36enc_dup(start_time), base36enc_dup(latest_backup_id)); + base36enc(start_time), base36enc(latest_backup_id)); current.backup_id = start_time; pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); diff --git a/src/catalog.c b/src/catalog.c index 488d7349f..7ad62b5d1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1219,7 +1219,6 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current int i; pgBackup *full_backup = NULL; pgBackup *tmp_backup = NULL; - char *invalid_backup_id; /* backup_list is sorted in order of descending ID */ for (i = 0; i < parray_num(backup_list); i++) @@ -1255,20 +1254,14 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current { /* broken chain */ case ChainIsBroken: - invalid_backup_id = base36enc_dup(tmp_backup->parent_backup); - elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", - base36enc(backup->start_time), invalid_backup_id); - pg_free(invalid_backup_id); + base36enc(backup->start_time), base36enc(tmp_backup->parent_backup)); continue; /* chain is intact, but at least one parent is invalid */ case ChainIsInvalid: - invalid_backup_id = base36enc_dup(tmp_backup->start_time); - elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", - base36enc(backup->start_time), invalid_backup_id); - pg_free(invalid_backup_id); + base36enc(backup->start_time), base36enc(tmp_backup->start_time)); continue; /* chain is ok */ diff --git a/src/delete.c b/src/delete.c index b86ed43e6..a1ab81331 100644 --- a/src/delete.c +++ b/src/delete.c @@ -451,7 +451,6 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, /* Merging happens here */ for (i = 0; i < parray_num(to_keep_list); i++) { - char *keep_backup_id = NULL; pgBackup *full_backup = NULL; parray *merge_list = NULL; @@ -488,10 +487,9 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, * backups from purge_list. */ - keep_backup_id = base36enc_dup(keep_backup->start_time); elog(INFO, "Merge incremental chain between full backup %s and backup %s", - base36enc(full_backup->start_time), keep_backup_id); - pg_free(keep_backup_id); + base36enc(full_backup->start_time), + base36enc(keep_backup->start_time)); merge_list = parray_new(); @@ -599,8 +597,6 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) */ for (i = 0; i < parray_num(to_keep_list); i++) { - char *keeped_backup_id; - pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); /* item could have been nullified in merge */ @@ -611,10 +607,9 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) if (keep_backup->backup_mode == BACKUP_MODE_FULL) continue; - keeped_backup_id = base36enc_dup(keep_backup->start_time); - elog(LOG, "Check if backup %s is parent of backup %s", - base36enc(delete_backup->start_time), keeped_backup_id); + base36enc(delete_backup->start_time), + base36enc(keep_backup->start_time)); if (is_parent(delete_backup->start_time, keep_backup, true)) { @@ -622,13 +617,12 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) /* We must not delete this backup, evict it from purge list */ elog(LOG, "Retain backup %s because his " "descendant %s is guarded by retention", - base36enc(delete_backup->start_time), keeped_backup_id); + base36enc(delete_backup->start_time), + base36enc(keep_backup->start_time)); purge = false; - pg_free(keeped_backup_id); break; } - pg_free(keeped_backup_id); } /* Retain backup */ diff --git a/src/merge.c b/src/merge.c index 79498f48c..ac1e97e71 100644 --- a/src/merge.c +++ b/src/merge.c @@ -223,11 +223,9 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool } if (!dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(ERROR, "Full backup %s has unfinished merge with missing backup %s", - tmp_backup_id, + base36enc(full_backup->start_time), base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); } } else if (full_backup->status == BACKUP_STATUS_MERGED) @@ -253,11 +251,9 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool } if (!dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(WARNING, "Full backup %s has unfinished merge with missing backup %s", - tmp_backup_id, + base36enc(full_backup->start_time), base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); } } else @@ -344,10 +340,9 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool full_backup->status == BACKUP_STATUS_MERGED) && dest_backup->start_time != full_backup->merge_dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(ERROR, "Full backup %s has unfinished merge with backup %s", - tmp_backup_id, base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); + base36enc(full_backup->start_time), + base36enc(full_backup->merge_dest_backup)); } } @@ -441,7 +436,6 @@ merge_chain(InstanceState *instanceState, bool no_validate, bool no_sync) { int i; - char *dest_backup_id; char full_external_prefix[MAXPGPATH]; char full_database_dir[MAXPGPATH]; parray *full_externals = NULL, @@ -487,17 +481,11 @@ merge_chain(InstanceState *instanceState, if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && full_backup->merge_dest_backup != dest_backup->start_time) { - char *merge_dest_backup_current = base36enc_dup(dest_backup->start_time); - char *merge_dest_backup = base36enc_dup(full_backup->merge_dest_backup); - elog(ERROR, "Cannot run merge for %s, because full backup %s has " "unfinished merge with backup %s", - merge_dest_backup_current, + base36enc(dest_backup->start_time), base36enc(full_backup->start_time), - merge_dest_backup); - - pg_free(merge_dest_backup_current); - pg_free(merge_dest_backup); + base36enc(full_backup->merge_dest_backup)); } /* @@ -880,9 +868,9 @@ merge_chain(InstanceState *instanceState, /* * Merging finished, now we can safely update ID of the FULL backup */ - dest_backup_id = base36enc_dup(full_backup->merge_dest_backup); elog(INFO, "Rename merged full backup %s to %s", - base36enc(full_backup->start_time), dest_backup_id); + base36enc(full_backup->start_time), + base36enc(full_backup->merge_dest_backup)); full_backup->status = BACKUP_STATUS_OK; full_backup->start_time = full_backup->merge_dest_backup; @@ -891,7 +879,6 @@ merge_chain(InstanceState *instanceState, /* Critical section end */ /* Cleanup */ - pg_free(dest_backup_id); if (threads) { pfree(threads_args); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 44b33d16f..8a12cc488 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1192,8 +1192,10 @@ extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); const char *status2str_color(BackupStatus status); extern BackupStatus str2status(const char *status); -extern const char *base36enc(long unsigned int value); -extern char *base36enc_dup(long unsigned int value); +#define base36bufsize 14 +extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]); +/* Abuse C99 Compound Literal's lifetime */ +#define base36enc(value) (base36enc_to((value), (char[base36bufsize]){0})) extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); diff --git a/src/restore.c b/src/restore.c index c877290b1..6bd2ad3b4 100644 --- a/src/restore.c +++ b/src/restore.c @@ -76,11 +76,11 @@ static void set_orphan_status(parray *backups, pgBackup *parent_backup) { /* chain is intact, but at least one parent is invalid */ - char *parent_backup_id; + const char *parent_backup_id; int j; /* parent_backup_id is a human-readable backup ID */ - parent_backup_id = base36enc_dup(parent_backup->start_time); + parent_backup_id = base36enc(parent_backup->start_time); for (j = 0; j < parray_num(backups); j++) { @@ -108,7 +108,6 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) } } } - pg_free(parent_backup_id); } /* @@ -348,11 +347,11 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg /* chain is broken, determine missing backup ID * and orphinize all his descendants */ - char *missing_backup_id; + const char *missing_backup_id; time_t missing_backup_start_time; missing_backup_start_time = tmp_backup->parent_backup; - missing_backup_id = base36enc_dup(tmp_backup->parent_backup); + missing_backup_id = base36enc(tmp_backup->parent_backup); for (j = 0; j < parray_num(backups); j++) { @@ -363,22 +362,22 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg */ if (is_parent(missing_backup_start_time, backup, false)) { + const char *backup_id = base36enc(backup->start_time); if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(backup->start_time), missing_backup_id); + backup_id, missing_backup_id); } else { elog(WARNING, "Backup %s has missing parent %s", - base36enc(backup->start_time), missing_backup_id); + backup_id, missing_backup_id); } } } - pg_free(missing_backup_id); /* No point in doing futher */ elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time)); } diff --git a/src/util.c b/src/util.c index d19877f06..420795d72 100644 --- a/src/util.c +++ b/src/util.c @@ -32,38 +32,23 @@ static const char *statusName[] = }; const char * -base36enc(long unsigned int value) +base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]) { const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ - static char buffer[14]; - unsigned int offset = sizeof(buffer); + char buffer[14]; + char *p; - buffer[--offset] = '\0'; + p = &buffer[sizeof(buffer)-1]; + *p = '\0'; do { - buffer[--offset] = base36[value % 36]; + *(--p) = base36[value % 36]; } while (value /= 36); - return &buffer[offset]; -} - -/* - * Same as base36enc(), but the result must be released by the user. - */ -char * -base36enc_dup(long unsigned int value) -{ - const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ - char buffer[14]; - unsigned int offset = sizeof(buffer); - - buffer[--offset] = '\0'; - do { - buffer[--offset] = base36[value % 36]; - } while (value /= 36); + /* I know, it doesn't look safe */ + strncpy(buf, p, base36bufsize); - return strdup(&buffer[offset]); + return buf; } long unsigned int diff --git a/src/utils/pgut.h b/src/utils/pgut.h index fa0efe816..116ee41c0 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -108,4 +108,10 @@ extern int sleep(unsigned int seconds); extern int usleep(unsigned int usec); #endif +#ifdef _MSC_VER +#define ARG_SIZE_HINT +#else +#define ARG_SIZE_HINT static +#endif + #endif /* PGUT_H */ diff --git a/src/validate.c b/src/validate.c index 4044ac158..139beabd6 100644 --- a/src/validate.c +++ b/src/validate.c @@ -503,10 +503,12 @@ do_validate_instance(InstanceState *instanceState) /* chain is broken */ if (result == ChainIsBroken) { - char *parent_backup_id; + const char *parent_backup_id; + const char *current_backup_id; /* determine missing backup ID */ - parent_backup_id = base36enc_dup(tmp_backup->parent_backup); + parent_backup_id = base36enc(tmp_backup->parent_backup); + current_backup_id = base36enc(current_backup->parent_backup); corrupted_backup_found = true; /* orphanize current_backup */ @@ -515,15 +517,13 @@ do_validate_instance(InstanceState *instanceState) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(current_backup->start_time), - parent_backup_id); + current_backup_id, parent_backup_id); } else { elog(WARNING, "Backup %s has missing parent %s", - base36enc(current_backup->start_time), parent_backup_id); + current_backup_id, parent_backup_id); } - pg_free(parent_backup_id); continue; } /* chain is whole, but at least one parent is invalid */ @@ -532,23 +532,23 @@ do_validate_instance(InstanceState *instanceState) /* Oldest corrupt backup has a chance for revalidation */ if (current_backup->start_time != tmp_backup->start_time) { - char *backup_id = base36enc_dup(tmp_backup->start_time); + const char *tmp_backup_id = base36enc(tmp_backup->start_time); + const char *cur_backup_id = base36enc(current_backup->start_time); /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(current_backup->start_time), backup_id, + cur_backup_id, tmp_backup_id, status2str(tmp_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(current_backup->start_time), backup_id, + cur_backup_id, tmp_backup_id, status2str(tmp_backup->status)); } - pg_free(backup_id); continue; } base_full_backup = find_parent_full_backup(current_backup); @@ -589,7 +589,7 @@ do_validate_instance(InstanceState *instanceState) */ if (current_backup->status != BACKUP_STATUS_OK) { - char *current_backup_id; + const char *current_backup_id; /* This is ridiculous but legal. * PAGE_b2 <- OK * PAGE_a2 <- OK @@ -599,7 +599,7 @@ do_validate_instance(InstanceState *instanceState) */ corrupted_backup_found = true; - current_backup_id = base36enc_dup(current_backup->start_time); + current_backup_id = base36enc(current_backup->start_time); for (j = i - 1; j >= 0; j--) { @@ -619,7 +619,6 @@ do_validate_instance(InstanceState *instanceState) } } } - free(current_backup_id); } /* For every OK backup we try to revalidate all his ORPHAN descendants. */ From 20e12edc807ce59ec4b01234136f91eba9282b8c Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 23 Nov 2022 16:51:48 +0300 Subject: [PATCH 1971/2107] fix one of backup_id output. --- src/backup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup.c b/src/backup.c index 639dda63d..4daec866e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -604,7 +604,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, "It may indicate that we are trying to backup PostgreSQL instance from the past.", (uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn), (uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn), - base36enc(prev_backup->stop_lsn)); + base36enc(prev_backup->start_time)); /* clean external directories list */ if (external_dirs) From 71f8ccf4cd3df57a2d5e8c11f0a30d3415578d91 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 23 Nov 2022 17:13:38 +0300 Subject: [PATCH 1972/2107] replace base36enc(backup->start_time) with helper backup_id_of(backup) Lazily store encoded backup_id in backup itself. --- src/backup.c | 8 ++--- src/catalog.c | 56 +++++++++++++++++------------------ src/data.c | 4 +-- src/delete.c | 42 +++++++++++++------------- src/dir.c | 4 +-- src/merge.c | 48 +++++++++++++++--------------- src/parsexlog.c | 14 ++++----- src/pg_probackup.h | 9 ++++-- src/restore.c | 74 ++++++++++++++++++++++------------------------ src/show.c | 6 ++-- src/util.c | 13 ++++++-- src/validate.c | 44 +++++++++++++-------------- 12 files changed, 163 insertions(+), 159 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4daec866e..5097f46ec 100644 --- a/src/backup.c +++ b/src/backup.c @@ -190,9 +190,9 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); + PROGRAM_VERSION, backup_id_of(prev_backup), prev_backup->program_version); - elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); + elog(INFO, "Parent backup: %s", backup_id_of(prev_backup)); /* Files of previous backup needed by DELTA backup */ prev_backup_filelist = get_backup_filelist(prev_backup, true); @@ -233,7 +233,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, "It may indicate that we are trying to backup PostgreSQL instance from the past.", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), - base36enc(prev_backup->start_time)); + backup_id_of(prev_backup)); /* Update running backup meta with START LSN */ write_backup(¤t, true); @@ -604,7 +604,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, "It may indicate that we are trying to backup PostgreSQL instance from the past.", (uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn), (uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn), - base36enc(prev_backup->start_time)); + backup_id_of(prev_backup)); /* clean external directories list */ if (external_dirs) diff --git a/src/catalog.c b/src/catalog.c index 7ad62b5d1..d19e4a27d 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -153,7 +153,7 @@ write_backup_status(pgBackup *backup, BackupStatus status, /* lock backup in exclusive mode */ if (!lock_backup(tmp, strict, true)) - elog(ERROR, "Cannot lock backup %s directory", base36enc(backup->start_time)); + elog(ERROR, "Cannot lock backup %s directory", backup_id_of(backup)); write_backup(tmp, strict); @@ -193,7 +193,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); - rc = grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict); + rc = grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict); if (rc == LOCK_FAIL_TIMEOUT) return false; @@ -258,7 +258,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) * freed some space on filesystem, thanks to unlinking of BACKUP_RO_LOCK_FILE. * If somebody concurrently acquired exclusive lock file first, then we should give up. */ - if (grab_excl_lock_file(backup->root_dir, base36enc(backup->start_time), strict) == LOCK_FAIL_TIMEOUT) + if (grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict) == LOCK_FAIL_TIMEOUT) return false; return true; @@ -521,7 +521,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) } // elog(LOG, "Acquired exclusive lock for backup %s after %ds", -// base36enc(backup->start_time), +// backup_id_of(backup), // LOCK_TIMEOUT - ntries + LOCK_STALE_TIMEOUT - empty_tries); return LOCK_OK; @@ -561,7 +561,7 @@ wait_shared_owners(pgBackup *backup) { if (interrupted) elog(ERROR, "Interrupted while locking backup %s", - base36enc(backup->start_time)); + backup_id_of(backup)); if (encoded_pid == my_pid) break; @@ -573,10 +573,10 @@ wait_shared_owners(pgBackup *backup) if ((ntries % LOG_FREQ) == 0) { elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", - encoded_pid, base36enc(backup->start_time)); + encoded_pid, backup_id_of(backup)); elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, - base36enc(backup->start_time)); + backup_id_of(backup)); } sleep(1); @@ -604,7 +604,7 @@ wait_shared_owners(pgBackup *backup) if (ntries <= 0) { elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", - base36enc(backup->start_time), encoded_pid); + backup_id_of(backup), encoded_pid); return 1; } @@ -963,15 +963,15 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id if (!backup) { - backup = pgut_new(pgBackup); + backup = pgut_new0(pgBackup); pgBackupInit(backup); backup->start_time = base36dec(data_ent->d_name); } - else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) + else if (strcmp(backup_id_of(backup), data_ent->d_name) != 0) { /* TODO there is no such guarantees */ elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", - base36enc(backup->start_time), backup_conf_path); + backup_id_of(backup), backup_conf_path); } backup->root_dir = pgut_strdup(data_path); @@ -1010,7 +1010,7 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id { pgBackup *curr = parray_get(backups, i); pgBackup **ancestor; - pgBackup key; + pgBackup key = {0}; if (curr->backup_mode == BACKUP_MODE_FULL) continue; @@ -1180,7 +1180,7 @@ get_backup_filelist(pgBackup *backup, bool strict) /* redundant sanity? */ if (!files) - elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time)); + elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", backup_id_of(backup)); return files; } @@ -1206,7 +1206,7 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool str pgBackup *backup = (pgBackup *) parray_get(backup_list, i); if (!lock_backup(backup, strict, exclusive)) elog(ERROR, "Cannot lock backup %s directory", - base36enc(backup->start_time)); + backup_id_of(backup)); } } @@ -1239,7 +1239,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current return NULL; elog(LOG, "Latest valid FULL backup: %s", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* FULL backup is found, lets find his latest child */ for (i = 0; i < parray_num(backup_list); i++) @@ -1255,13 +1255,13 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current /* broken chain */ case ChainIsBroken: elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", - base36enc(backup->start_time), base36enc(tmp_backup->parent_backup)); + backup_id_of(backup), base36enc(tmp_backup->parent_backup)); continue; /* chain is intact, but at least one parent is invalid */ case ChainIsInvalid: elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", - base36enc(backup->start_time), base36enc(tmp_backup->start_time)); + backup_id_of(backup), backup_id_of(tmp_backup)); continue; /* chain is ok */ @@ -1280,7 +1280,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current else { elog(WARNING, "Backup %s has status: %s. Cannot be a parent.", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); } } @@ -1366,7 +1366,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; else elog(LOG, "Latest valid full backup: %s, tli: %i", - base36enc(ancestor_backup->start_time), ancestor_backup->tli); + backup_id_of(ancestor_backup), ancestor_backup->tli); /* At this point we found suitable full backup, * now we must find his latest child, suitable to be @@ -1871,7 +1871,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) { elog(LOG, "Pinned backup %s is ignored for the " "purpose of WAL retention", - base36enc(backup->start_time)); + backup_id_of(backup)); continue; } @@ -2057,7 +2057,7 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) elog(LOG, "Archive backup %s to stay consistent " "protect from purge WAL interval " "between %s and %s on timeline %i", - base36enc(backup->start_time), + backup_id_of(backup), begin_segno_str, end_segno_str, backup->tli); if (tlinfo->keep_segments == NULL) @@ -2266,7 +2266,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) if (target_backup->expire_time == 0) { elog(WARNING, "Backup %s is not pinned, nothing to unpin", - base36enc(target_backup->start_time)); + backup_id_of(target_backup)); return; } target_backup->expire_time = 0; @@ -2286,11 +2286,11 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) char expire_timestamp[100]; time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time, false); - elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), + elog(INFO, "Backup %s is pinned until '%s'", backup_id_of(target_backup), expire_timestamp); } else - elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time)); + elog(INFO, "Backup %s is unpinned", backup_id_of(target_backup)); return; } @@ -2310,7 +2310,7 @@ add_note(pgBackup *target_backup, char *note) { target_backup->note = NULL; elog(INFO, "Removing note from backup %s", - base36enc(target_backup->start_time)); + backup_id_of(target_backup)); } else { @@ -2325,7 +2325,7 @@ add_note(pgBackup *target_backup, char *note) target_backup->note = note_string; elog(INFO, "Adding note to backup %s: '%s'", - base36enc(target_backup->start_time), target_backup->note); + backup_id_of(target_backup), target_backup->note); } /* Update backup.control */ @@ -2644,7 +2644,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, static pgBackup * readBackupControlFile(const char *path) { - pgBackup *backup = pgut_new(pgBackup); + pgBackup *backup = pgut_new0(pgBackup); char *backup_mode = NULL; char *start_lsn = NULL; char *stop_lsn = NULL; @@ -3047,7 +3047,7 @@ find_parent_full_backup(pgBackup *current_backup) base36enc(base_full_backup->parent_backup)); else elog(WARNING, "Failed to find parent FULL backup for %s", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); return NULL; } diff --git a/src/data.c b/src/data.c index 08727d41c..490faf9b6 100644 --- a/src/data.c +++ b/src/data.c @@ -1294,7 +1294,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (!tmp_file) { elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", - dest_file->rel_path, base36enc(tmp_backup->start_time)); + dest_file->rel_path, backup_id_of(tmp_backup)); continue; } @@ -1327,7 +1327,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (tmp_file->write_size <= 0) elog(ERROR, "Full copy of non-data file has invalid size: %li. " "Metadata corruption in backup %s in file: \"%s\"", - tmp_file->write_size, base36enc(tmp_backup->start_time), + tmp_file->write_size, backup_id_of(tmp_backup), to_fullpath); /* incremental restore */ diff --git a/src/delete.c b/src/delete.c index a1ab81331..3f299d78b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -71,7 +71,7 @@ do_delete(InstanceState *instanceState, time_t backup_id) parray_append(delete_list, backup); elog(LOG, "Backup %s %s be deleted", - base36enc(backup->start_time), dry_run? "can":"will"); + backup_id_of(backup), dry_run? "can":"will"); size_to_delete += backup->data_bytes; if (backup->stream) @@ -84,7 +84,7 @@ do_delete(InstanceState *instanceState, time_t backup_id) { pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); elog(INFO, "Resident data size to free by delete of backup %s : %s", - base36enc(target_backup->start_time), size_to_delete_pretty); + backup_id_of(target_backup), size_to_delete_pretty); } if (!dry_run) @@ -321,12 +321,12 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time, false); elog(LOG, "Backup %s is pinned until '%s', retain", - base36enc(backup->start_time), expire_timestamp); + backup_id_of(backup), expire_timestamp); continue; } /* Add backup to purge_list */ - elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time)); + elog(VERBOSE, "Mark backup %s for purge.", backup_id_of(backup)); parray_append(to_purge_list, backup); continue; } @@ -406,7 +406,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* TODO: add ancestor(chain full backup) ID */ elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", - base36enc(backup->start_time), + backup_id_of(backup), pgBackupGetBackupMode(backup, false), status2str(backup->status), cur_full_backup_num, @@ -460,7 +460,7 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, if (!keep_backup) continue; - elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time)); + elog(INFO, "Consider backup %s for merge", backup_id_of(keep_backup)); /* Got valid incremental backup, find its FULL ancestor */ full_backup = find_parent_full_backup(keep_backup); @@ -468,7 +468,7 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, /* Failed to find parent */ if (!full_backup) { - elog(WARNING, "Failed to find FULL parent for %s", base36enc(keep_backup->start_time)); + elog(WARNING, "Failed to find FULL parent for %s", backup_id_of(keep_backup)); continue; } @@ -478,7 +478,7 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, pgBackupCompareIdDesc)) { elog(WARNING, "Skip backup %s for merging, " - "because his FULL parent is not marked for purge", base36enc(keep_backup->start_time)); + "because his FULL parent is not marked for purge", backup_id_of(keep_backup)); continue; } @@ -488,8 +488,8 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, */ elog(INFO, "Merge incremental chain between full backup %s and backup %s", - base36enc(full_backup->start_time), - base36enc(keep_backup->start_time)); + backup_id_of(full_backup), + backup_id_of(keep_backup)); merge_list = parray_new(); @@ -531,7 +531,7 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, // if (is_prolific(backup_list, full_backup)) // { // elog(WARNING, "Backup %s has multiple valid descendants. " -// "Automatic merge is not possible.", base36enc(full_backup->start_time)); +// "Automatic merge is not possible.", backup_id_of(full_backup)); // } /* Merge list example: @@ -557,7 +557,7 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, if (!no_validate) pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time)); + elog(ERROR, "Merging of backup %s failed", backup_id_of(full_backup)); /* Cleanup */ parray_free(merge_list); @@ -589,7 +589,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j); elog(LOG, "Consider backup %s for purge", - base36enc(delete_backup->start_time)); + backup_id_of(delete_backup)); /* Evaluate marked for delete backup against every backup in keep list. * If marked for delete backup is recognized as parent of one of those, @@ -608,8 +608,8 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) continue; elog(LOG, "Check if backup %s is parent of backup %s", - base36enc(delete_backup->start_time), - base36enc(keep_backup->start_time)); + backup_id_of(delete_backup), + backup_id_of(keep_backup)); if (is_parent(delete_backup->start_time, keep_backup, true)) { @@ -617,8 +617,8 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) /* We must not delete this backup, evict it from purge list */ elog(LOG, "Retain backup %s because his " "descendant %s is guarded by retention", - base36enc(delete_backup->start_time), - base36enc(keep_backup->start_time)); + backup_id_of(delete_backup), + backup_id_of(keep_backup)); purge = false; break; @@ -634,7 +634,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) { /* If the backup still is used, do not interrupt and go to the next */ elog(WARNING, "Cannot lock backup %s directory, skip purging", - base36enc(delete_backup->start_time)); + backup_id_of(delete_backup)); continue; } @@ -737,7 +737,7 @@ delete_backup_files(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) { elog(WARNING, "Backup %s already deleted", - base36enc(backup->start_time)); + backup_id_of(backup)); return; } @@ -747,7 +747,7 @@ delete_backup_files(pgBackup *backup) time2iso(timestamp, lengthof(timestamp), backup->start_time, false); elog(INFO, "Delete: %s %s", - base36enc(backup->start_time), timestamp); + backup_id_of(backup), timestamp); /* * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which @@ -1076,7 +1076,7 @@ do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, backup = (pgBackup *)parray_get(delete_list, i); elog(INFO, "Backup %s with status %s %s be deleted", - base36enc(backup->start_time), status2str(backup->status), dry_run ? "can" : "will"); + backup_id_of(backup), status2str(backup->status), dry_run ? "can" : "will"); size_to_delete += backup->data_bytes; if (backup->stream) diff --git a/src/dir.c b/src/dir.c index 6609b9f19..182f0a51d 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1101,7 +1101,7 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg bool tblspaces_are_empty = true; elog(LOG, "Checking tablespace directories of backup %s", - base36enc(backup->start_time)); + backup_id_of(backup)); /* validate tablespace map, * if there are no tablespaces, then there is nothing left to do @@ -1250,7 +1250,7 @@ check_external_dir_mapping(pgBackup *backup, bool incremental) int i; elog(LOG, "check external directories of backup %s", - base36enc(backup->start_time)); + backup_id_of(backup)); if (!backup->external_dir_str) { diff --git a/src/merge.c b/src/merge.c index ac1e97e71..a0d3fee1f 100644 --- a/src/merge.c +++ b/src/merge.c @@ -105,7 +105,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool backup->status != BACKUP_STATUS_MERGED && backup->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); dest_backup = backup; break; @@ -154,12 +154,12 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool full_backup = dest_backup; dest_backup = NULL; elog(INFO, "Merge target backup %s is full backup", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* sanity */ if (full_backup->status == BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), status2str(full_backup->status)); /* Case #1 */ @@ -194,7 +194,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool if (dest_backup == NULL) elog(ERROR, "Failed to find merge candidate, " "backup %s has no valid children", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); } /* Case #2 */ @@ -224,7 +224,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool if (!dest_backup) { elog(ERROR, "Full backup %s has unfinished merge with missing backup %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); } } @@ -252,13 +252,13 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool if (!dest_backup) { elog(WARNING, "Full backup %s has unfinished merge with missing backup %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); } } else elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), status2str(full_backup->status)); } else @@ -296,7 +296,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool if (dest_backup->status == BACKUP_STATUS_MERGING || dest_backup->status == BACKUP_STATUS_DELETING) elog(WARNING, "Rerun unfinished merge for backup %s", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); /* First we should try to find parent FULL backup */ full_backup = find_parent_full_backup(dest_backup); @@ -310,7 +310,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool */ if (dest_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Failed to find parent full backup for %s", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); /* Find FULL backup that has unfinished merge with dest backup */ for (i = 0; i < parray_num(backups); i++) @@ -327,7 +327,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool if (!full_backup) elog(ERROR, "Failed to find full backup that has unfinished merge" "with backup %s, cannot rerun merge", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); if (full_backup->status == BACKUP_STATUS_MERGED) elog(WARNING, "Incremental chain is broken, try to recover unfinished merge"); @@ -341,7 +341,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool dest_backup->start_time != full_backup->merge_dest_backup) { elog(ERROR, "Full backup %s has unfinished merge with backup %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); } @@ -357,7 +357,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool * having status MERGED */ if (dest_backup == NULL && full_backup->status != BACKUP_STATUS_MERGED) elog(ERROR, "Cannot run merge for full backup %s", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* sanity */ if (full_backup->status != BACKUP_STATUS_OK && @@ -366,7 +366,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool full_backup->status != BACKUP_STATUS_MERGED && full_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), status2str(full_backup->status)); + backup_id_of(full_backup), status2str(full_backup->status)); /* Form merge list */ dest_backup_tmp = dest_backup; @@ -384,7 +384,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool dest_backup_tmp->status != BACKUP_STATUS_MERGED && dest_backup_tmp->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup_tmp->start_time), + backup_id_of(dest_backup_tmp), status2str(dest_backup_tmp->status)); if (dest_backup_tmp->backup_mode == BACKUP_MODE_FULL) @@ -472,10 +472,10 @@ merge_chain(InstanceState *instanceState, full_backup->status == BACKUP_STATUS_MERGED) { is_retry = true; - elog(INFO, "Retry failed merge of backup %s with parent chain", base36enc(dest_backup->start_time)); + elog(INFO, "Retry failed merge of backup %s with parent chain", backup_id_of(dest_backup)); } else - elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); + elog(INFO, "Merging backup %s with parent chain", backup_id_of(dest_backup)); /* sanity */ if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && @@ -483,8 +483,8 @@ merge_chain(InstanceState *instanceState, { elog(ERROR, "Cannot run merge for %s, because full backup %s has " "unfinished merge with backup %s", - base36enc(dest_backup->start_time), - base36enc(full_backup->start_time), + backup_id_of(dest_backup), + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); } @@ -506,7 +506,7 @@ merge_chain(InstanceState *instanceState, elog(ERROR, "Backup %s has been produced by pg_probackup version %s, " "but current program version is %s. Forward compatibility " "is not supported.", - base36enc(backup->start_time), + backup_id_of(backup), backup->program_version, PROGRAM_VERSION); } @@ -549,7 +549,7 @@ merge_chain(InstanceState *instanceState, if (!no_validate) { elog(INFO, "Validate parent chain for backup %s", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); for (i = parray_num(parent_chain) - 1; i >= 0; i--) { @@ -566,7 +566,7 @@ merge_chain(InstanceState *instanceState, if (backup->status != BACKUP_STATUS_OK) elog(ERROR, "Backup %s has status %s, merge is aborted", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); } } @@ -869,7 +869,7 @@ merge_chain(InstanceState *instanceState, * Merging finished, now we can safely update ID of the FULL backup */ elog(INFO, "Rename merged full backup %s to %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); full_backup->status = BACKUP_STATUS_OK; @@ -1333,7 +1333,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, if (!from_file) { elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", - dest_file->rel_path, base36enc(from_backup->start_time)); + dest_file->rel_path, backup_id_of(from_backup)); continue; } @@ -1429,7 +1429,7 @@ is_forward_compatible(parray *parent_chain) elog(WARNING, "In-place merge is disabled because of storage format incompatibility. " "Backup %s storage format version: %s, " "current storage format version: %s", - base36enc(oldest_ver_backup->start_time), + backup_id_of(oldest_ver_backup), oldest_ver_backup->program_version, STORAGE_FORMAT_VERSION); return false; diff --git a/src/parsexlog.c b/src/parsexlog.c index f12aae904..3a791dc5d 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -392,7 +392,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", - base36enc(backup->start_time), + backup_id_of(backup), (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), (uint32) (backup->stop_lsn >> 32), @@ -410,24 +410,20 @@ validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, uint32 wal_seg_size) { - const char *backup_id; XLogRecTarget last_rec; char last_timestamp[100], target_timestamp[100]; bool all_wal = false; - /* We need free() this later */ - backup_id = base36enc(backup->start_time); - if (!XRecOffIsValid(backup->start_lsn)) elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), - backup_id); + backup_id_of(backup)); if (!XRecOffIsValid(backup->stop_lsn)) elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), - backup_id); + backup_id_of(backup)); /* * Check that the backup has all wal files needed @@ -450,7 +446,7 @@ validate_wal(pgBackup *backup, const char *archivedir, if (backup->status == BACKUP_STATUS_CORRUPT) { - elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id_of(backup)); return; } /* @@ -461,7 +457,7 @@ validate_wal(pgBackup *backup, const char *archivedir, !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ - elog(INFO, "Backup %s WAL segments are valid", backup_id); + elog(INFO, "Backup %s WAL segments are valid", backup_id_of(backup)); return; } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8a12cc488..5ef6cbe31 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -135,6 +135,9 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +/* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ +#define base36bufsize 14 + /* Text Coloring macro */ #define TC_LEN 11 #define TC_RED "\033[0;31m" @@ -151,7 +154,6 @@ extern const char *PROGRAM_EMAIL; #define TC_CYAN_BOLD "\033[1;36m" #define TC_RESET "\033[0m" - typedef struct RedoParams { TimeLineID tli; @@ -532,6 +534,8 @@ struct pgBackup /* map used for access to page headers */ HeaderMap hdr_map; + + char backup_id_encoded[base36bufsize]; }; /* Recovery target for restore and validate subcommands */ @@ -882,6 +886,8 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions( extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); +extern const char* backup_id_of(pgBackup *backup); + extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); @@ -1192,7 +1198,6 @@ extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); const char *status2str_color(BackupStatus status); extern BackupStatus str2status(const char *status); -#define base36bufsize 14 extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]); /* Abuse C99 Compound Literal's lifetime */ #define base36enc(value) (base36enc_to((value), (char[base36bufsize]){0})) diff --git a/src/restore.c b/src/restore.c index 6bd2ad3b4..6c0e1881f 100644 --- a/src/restore.c +++ b/src/restore.c @@ -76,12 +76,8 @@ static void set_orphan_status(parray *backups, pgBackup *parent_backup) { /* chain is intact, but at least one parent is invalid */ - const char *parent_backup_id; int j; - /* parent_backup_id is a human-readable backup ID */ - parent_backup_id = base36enc(parent_backup->start_time); - for (j = 0; j < parray_num(backups); j++) { @@ -96,14 +92,15 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - parent_backup_id, + backup_id_of(backup), + backup_id_of(parent_backup), status2str(parent_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(backup->start_time), parent_backup_id, + backup_id_of(backup), + backup_id_of(parent_backup), status2str(parent_backup->status)); } } @@ -242,7 +239,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg current_backup->status != BACKUP_STATUS_DONE)) { elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); continue; } @@ -272,10 +269,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg current_backup->status == BACKUP_STATUS_RUNNING) && (!params->no_validate || params->force)) elog(WARNING, "Backup %s has status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); else elog(ERROR, "Backup %s has status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); } if (rt->target_tli) @@ -362,24 +359,23 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg */ if (is_parent(missing_backup_start_time, backup, false)) { - const char *backup_id = base36enc(backup->start_time); if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - backup_id, missing_backup_id); + backup_id_of(backup), missing_backup_id); } else { elog(WARNING, "Backup %s has missing parent %s", - backup_id, missing_backup_id); + backup_id_of(backup), missing_backup_id); } } } /* No point in doing futher */ - elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time)); + elog(ERROR, "%s of backup %s failed.", action, backup_id_of(dest_backup)); } else if (result == ChainIsInvalid) { @@ -390,7 +386,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg /* sanity */ if (!tmp_backup) elog(ERROR, "Parent full backup for the given backup %s was not found", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); } /* We have found full backup */ @@ -510,7 +506,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (redo.tli == tmp_backup->tli) { elog(INFO, "Backup %s is chosen as shiftpoint, its Stop LSN will be used as shift LSN", - base36enc(tmp_backup->start_time)); + backup_id_of(tmp_backup)); shift_lsn = tmp_backup->stop_lsn; break; @@ -534,7 +530,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg else elog(INFO, "Backup %s cannot be a shiftpoint, " "because its tli %i is not in history of redo timeline %i", - base36enc(tmp_backup->start_time), tmp_backup->tli, redo.tli); + backup_id_of(tmp_backup), tmp_backup->tli, redo.tli); } tmp_backup = tmp_backup->parent_backup_link; @@ -543,13 +539,13 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (XLogRecPtrIsInvalid(shift_lsn)) elog(ERROR, "Cannot perform incremental restore of backup chain %s in 'lsn' mode, " "because destination directory redo point %X/%X on tli %i is out of reach", - base36enc(dest_backup->start_time), + backup_id_of(dest_backup), (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); else elog(INFO, "Destination directory redo point %X/%X on tli %i is " "within reach of backup %s with Stop LSN %X/%X on tli %i", (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, - base36enc(tmp_backup->start_time), + backup_id_of(tmp_backup), (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, tmp_backup->tli); @@ -563,7 +559,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (!params->is_restore || !params->no_validate) { if (dest_backup->backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + elog(INFO, "Validating parents for backup %s", backup_id_of(dest_backup)); /* * Validate backups from base_full_backup to dest_backup. @@ -576,7 +572,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (!lock_backup(tmp_backup, true, false)) { elog(ERROR, "Cannot lock backup %s directory", - base36enc(tmp_backup->start_time)); + backup_id_of(tmp_backup)); } /* validate datafiles only */ @@ -623,27 +619,27 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg dest_backup->status == BACKUP_STATUS_DONE) { if (params->no_validate) - elog(WARNING, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is used without validation.", backup_id_of(dest_backup)); else - elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + elog(INFO, "Backup %s is valid.", backup_id_of(dest_backup)); } else if (dest_backup->status == BACKUP_STATUS_CORRUPT) { if (params->force) - elog(WARNING, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is corrupt.", backup_id_of(dest_backup)); else - elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + elog(ERROR, "Backup %s is corrupt.", backup_id_of(dest_backup)); } else if (dest_backup->status == BACKUP_STATUS_ORPHAN) { if (params->force) - elog(WARNING, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is orphan.", backup_id_of(dest_backup)); else - elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + elog(ERROR, "Backup %s is orphan.", backup_id_of(dest_backup)); } else elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup->start_time), status2str(dest_backup->status)); + backup_id_of(dest_backup), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required */ @@ -667,7 +663,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (rt->lsn_string && parse_server_version(dest_backup->server_version) < 100000) elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", - base36enc(dest_backup->start_time), + backup_id_of(dest_backup), dest_backup->server_version); restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, @@ -682,7 +678,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg fio_disconnect(); elog(INFO, "%s of backup %s completed.", - action, base36enc(dest_backup->start_time)); + action, backup_id_of(dest_backup)); /* cleanup */ parray_walk(backups, pgBackupFree); @@ -733,17 +729,17 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); if (!lock_backup(backup, true, false)) - elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); + elog(ERROR, "Cannot lock backup %s", backup_id_of(backup)); if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) { if (params->force) elog(WARNING, "Backup %s is not valid, restore is forced", - base36enc(backup->start_time)); + backup_id_of(backup)); else elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); + backup_id_of(backup)); } /* confirm block size compatibility */ @@ -1620,7 +1616,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, strerror(errno)); fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", - base36enc(backup->start_time), current_time_str); + backup_id_of(backup), current_time_str); if (params->recovery_settings_mode == PITR_REQUESTED) print_recovery_settings(instanceState, fp, backup, params, rt); @@ -2030,7 +2026,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, if (!database_map_file) elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", - base36enc(backup->start_time)); + backup_id_of(backup)); join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); @@ -2048,7 +2044,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* partial restore requested but database_map is missing */ if (!database_map) elog(ERROR, "Backup %s has empty or mangled database_map, partial restore is impossible.", - base36enc(backup->start_time)); + backup_id_of(backup)); /* * So we have a list of datnames and a database_map for it. @@ -2078,7 +2074,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* If specified datname is not found in database_map, error out */ if (!found_match) elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, base36enc(backup->start_time)); + datname, backup_id_of(backup)); } /* At this moment only databases to exclude are left in the map */ @@ -2116,14 +2112,14 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* If specified datname is not found in database_map, error out */ if (!found_match) elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, base36enc(backup->start_time)); + datname, backup_id_of(backup)); } } /* extra sanity: ensure that list is not empty */ if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", - base36enc(backup->start_time)); + backup_id_of(backup)); /* clean backup filelist */ if (files) diff --git a/src/show.c b/src/show.c index db8a9e225..2e06582ed 100644 --- a/src/show.c +++ b/src/show.c @@ -353,7 +353,7 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "id", base36enc(backup->start_time), json_level, + json_add_value(buf, "id", backup_id_of(backup), json_level, true); if (backup->parent_backup != 0) @@ -583,7 +583,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na /* ID */ snprintf(row->backup_id, lengthof(row->backup_id), "%s", - base36enc(backup->start_time)); + backup_id_of(backup)); widths[cur] = Max(widths[cur], strlen(row->backup_id)); cur++; @@ -1100,7 +1100,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, if (tlinfo->closest_backup != NULL) snprintf(tmp_buf, lengthof(tmp_buf), "%s", - base36enc(tlinfo->closest_backup->start_time)); + backup_id_of(tlinfo->closest_backup)); else snprintf(tmp_buf, lengthof(tmp_buf), "%s", ""); diff --git a/src/util.c b/src/util.c index 420795d72..d3cfcb37e 100644 --- a/src/util.c +++ b/src/util.c @@ -35,8 +35,7 @@ const char * base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]) { const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ - char buffer[14]; + char buffer[base36bufsize]; char *p; p = &buffer[sizeof(buffer)-1]; @@ -562,3 +561,13 @@ datapagemap_print_debug(datapagemap_t *map) pg_free(iter); } + +const char* +backup_id_of(pgBackup *backup) +{ + if (backup->backup_id_encoded[0] == '\x00') + { + base36enc_to(backup->start_time, backup->backup_id_encoded); + } + return backup->backup_id_encoded; +} diff --git a/src/validate.c b/src/validate.c index 139beabd6..7a9140bbc 100644 --- a/src/validate.c +++ b/src/validate.c @@ -63,18 +63,18 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + PROGRAM_VERSION, backup_id_of(backup), backup->program_version); /* Check backup server version */ if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " "compiled with server version %s", - base36enc(backup->start_time), backup->server_version, PG_MAJORVERSION); + backup_id_of(backup), backup->server_version, PG_MAJORVERSION); if (backup->status == BACKUP_STATUS_RUNNING) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); write_backup_status(backup, BACKUP_STATUS_ERROR, true); corrupted_backup_found = true; return; @@ -88,7 +88,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status != BACKUP_STATUS_CORRUPT) { elog(WARNING, "Backup %s has status %s. Skip validation.", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); corrupted_backup_found = true; return; } @@ -98,28 +98,28 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status == BACKUP_STATUS_MERGING) { elog(WARNING, "Full backup %s has status %s, skip validation", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); return; } if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE || backup->status == BACKUP_STATUS_MERGING) - elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + elog(INFO, "Validating backup %s", backup_id_of(backup)); else - elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + elog(INFO, "Revalidating backup %s", backup_id_of(backup)); if (backup->backup_mode != BACKUP_MODE_FULL && backup->backup_mode != BACKUP_MODE_DIFF_PAGE && backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && backup->backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + elog(WARNING, "Invalid backup_mode of backup %s", backup_id_of(backup)); join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); files = get_backup_filelist(backup, false); if (!files) { - elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time)); + elog(WARNING, "Backup %s file list is corrupted", backup_id_of(backup)); backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); return; @@ -189,9 +189,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) BACKUP_STATUS_OK, true); if (corrupted) - elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + elog(WARNING, "Backup %s data files are corrupted", backup_id_of(backup)); else - elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); + elog(INFO, "Backup %s data files are valid", backup_id_of(backup)); /* Issue #132 kludge */ if (!corrupted && @@ -208,7 +208,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) elog(WARNING, "Backup %s is a victim of metadata corruption. " "Additional information can be found here: " "https://github.com/postgrespro/pg_probackup/issues/132", - base36enc(backup->start_time)); + backup_id_of(backup)); backup->status = BACKUP_STATUS_CORRUPT; write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); } @@ -532,21 +532,21 @@ do_validate_instance(InstanceState *instanceState) /* Oldest corrupt backup has a chance for revalidation */ if (current_backup->start_time != tmp_backup->start_time) { - const char *tmp_backup_id = base36enc(tmp_backup->start_time); - const char *cur_backup_id = base36enc(current_backup->start_time); /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - cur_backup_id, tmp_backup_id, + backup_id_of(current_backup), + backup_id_of(tmp_backup), status2str(tmp_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - cur_backup_id, tmp_backup_id, + backup_id_of(current_backup), + backup_id_of(tmp_backup), status2str(tmp_backup->status)); } continue; @@ -556,7 +556,7 @@ do_validate_instance(InstanceState *instanceState) /* sanity */ if (!base_full_backup) elog(ERROR, "Parent full backup for the given backup %s was not found", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); } /* chain is whole, all parents are valid at first glance, * current backup validation can proceed @@ -571,7 +571,7 @@ do_validate_instance(InstanceState *instanceState) if (!lock_backup(current_backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); skipped_due_to_lock = true; continue; } @@ -589,7 +589,6 @@ do_validate_instance(InstanceState *instanceState) */ if (current_backup->status != BACKUP_STATUS_OK) { - const char *current_backup_id; /* This is ridiculous but legal. * PAGE_b2 <- OK * PAGE_a2 <- OK @@ -599,7 +598,6 @@ do_validate_instance(InstanceState *instanceState) */ corrupted_backup_found = true; - current_backup_id = base36enc(current_backup->start_time); for (j = i - 1; j >= 0; j--) { @@ -613,8 +611,8 @@ do_validate_instance(InstanceState *instanceState) write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - current_backup_id, + backup_id_of(backup), + backup_id_of(current_backup), status2str(current_backup->status)); } } @@ -665,7 +663,7 @@ do_validate_instance(InstanceState *instanceState) if (!lock_backup(backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(backup->start_time)); + backup_id_of(backup)); skipped_due_to_lock = true; continue; } From 44fef8894ec54cf29200167eb75c580d9e4547e3 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 23 Nov 2022 17:50:51 +0300 Subject: [PATCH 1973/2107] a bit more backup_id_of --- src/backup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 5097f46ec..86c92618a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -887,13 +887,13 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, pretty_size(current.data_bytes + current.wal_bytes, pretty_bytes, lengthof(pretty_bytes)); else pretty_size(current.data_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Backup %s resident size: %s", base36enc(current.start_time), pretty_bytes); + elog(INFO, "Backup %s resident size: %s", backup_id_of(¤t), pretty_bytes); if (current.status == BACKUP_STATUS_OK || current.status == BACKUP_STATUS_DONE) - elog(INFO, "Backup %s completed", base36enc(current.start_time)); + elog(INFO, "Backup %s completed", backup_id_of(¤t)); else - elog(ERROR, "Backup %s failed", base36enc(current.start_time)); + elog(ERROR, "Backup %s failed", backup_id_of(¤t)); /* * After successful backup completion remove backups @@ -2034,7 +2034,7 @@ backup_cleanup(bool fatal, void *userdata) if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) { elog(WARNING, "Backup %s is running, setting its status to ERROR", - base36enc(current.start_time)); + backup_id_of(¤t)); current.end_time = time(NULL); current.status = BACKUP_STATUS_ERROR; write_backup(¤t, true); From 87dd3f20217114fc720159ab9e9789bb54b3c724 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 23 Nov 2022 17:59:25 +0300 Subject: [PATCH 1974/2107] base36enc(backup->backup_id) -> backup_id_of(backup) Here we assume backup_id == start_time. It is really so at the moment, but could change in future. Well, it almost always same. Sometime backup_id is set while start_time is not set yet (backup creation). And we had to fix places where start_time were changed without change of backup_id. --- src/backup.c | 7 ++++--- src/catalog.c | 15 ++++++++++----- src/dir.c | 2 +- src/merge.c | 2 ++ src/pg_probackup.h | 1 + src/util.c | 12 +++++++++++- src/validate.c | 4 ++-- 7 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/backup.c b/src/backup.c index 86c92618a..35fc98092 100644 --- a/src/backup.c +++ b/src/backup.c @@ -768,6 +768,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ current.start_time = current.backup_id; strlcpy(current.program_version, PROGRAM_VERSION, @@ -778,13 +779,13 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instanceState->instance_name, base36enc(current.backup_id), pgBackupGetBackupMode(¤t, false), + PROGRAM_VERSION, instanceState->instance_name, backup_id_of(¤t), pgBackupGetBackupMode(¤t, false), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); if (!lock_backup(¤t, true, true)) elog(ERROR, "Cannot lock backup %s directory", - base36enc(current.backup_id)); + backup_id_of(¤t)); write_backup(¤t, true); /* set the error processing function for the backup process */ @@ -799,7 +800,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); if (current.from_replica) - elog(INFO, "Backup %s is going to be taken from standby", base36enc(current.backup_id)); + elog(INFO, "Backup %s is going to be taken from standby", backup_id_of(¤t)); /* TODO, print PostgreSQL full version */ //elog(INFO, "PostgreSQL version: %s", nodeInfo.server_version_str); diff --git a/src/catalog.c b/src/catalog.c index d19e4a27d..92a2d84b7 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -275,7 +275,7 @@ lock_backup(pgBackup *backup, bool strict, bool exclusive) /* save lock metadata for later unlocking */ lock = pgut_malloc(sizeof(LockInfo)); - snprintf(lock->backup_id, 10, "%s", base36enc(backup->backup_id)); + snprintf(lock->backup_id, 10, "%s", backup_id_of(backup)); snprintf(lock->backup_dir, MAXPGPATH, "%s", backup->root_dir); lock->exclusive = exclusive; @@ -966,6 +966,9 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id backup = pgut_new0(pgBackup); pgBackupInit(backup); backup->start_time = base36dec(data_ent->d_name); + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); + backup->backup_id = backup->start_time; } else if (strcmp(backup_id_of(backup), data_ent->d_name) != 0) { @@ -983,7 +986,6 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id init_header_map(backup); /* TODO: save encoded backup id */ - backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID && requested_backup_id != backup->start_time) { @@ -1454,7 +1456,7 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) if (create_backup_dir(backup, backup_instance_path) != 0) { /* Clear backup_id as indication of error */ - backup->backup_id = INVALID_BACKUP_ID; + reset_backup_id(backup); return; } @@ -1506,7 +1508,7 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) int rc; char path[MAXPGPATH]; - join_path_components(path, backup_instance_path, base36enc(backup->backup_id)); + join_path_components(path, backup_instance_path, backup_id_of(backup)); /* TODO: add wrapper for remote mode */ rc = dir_create_dir(path, DIR_PERMISSION, true); @@ -2252,7 +2254,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) /* sanity, backup must have positive recovery-time */ if (target_backup->recovery_time <= 0) elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'", - base36enc(target_backup->backup_id)); + backup_id_of(target_backup)); /* Pin comes from ttl */ if (set_backup_params->ttl > 0) @@ -2714,6 +2716,9 @@ readBackupControlFile(const char *path) pgBackupFree(backup); return NULL; } + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); + backup->backup_id = backup->start_time; if (backup_mode) { diff --git a/src/dir.c b/src/dir.c index 182f0a51d..0a55c0f67 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1115,7 +1115,7 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg */ if (tablespace_dirs.head != NULL) elog(ERROR, "Backup %s has no tablespaceses, nothing to remap " - "via \"--tablespace-mapping\" option", base36enc(backup->backup_id)); + "via \"--tablespace-mapping\" option", backup_id_of(backup)); return NoTblspc; } diff --git a/src/merge.c b/src/merge.c index a0d3fee1f..0017c9e9c 100644 --- a/src/merge.c +++ b/src/merge.c @@ -874,6 +874,8 @@ merge_chain(InstanceState *instanceState, full_backup->status = BACKUP_STATUS_OK; full_backup->start_time = full_backup->merge_dest_backup; + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + full_backup->backup_id = full_backup->start_time; full_backup->merge_dest_backup = INVALID_BACKUP_ID; write_backup(full_backup, true); /* Critical section end */ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5ef6cbe31..843fb3522 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -887,6 +887,7 @@ extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); extern const char* backup_id_of(pgBackup *backup); +extern void reset_backup_id(pgBackup *backup); extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); diff --git a/src/util.c b/src/util.c index d3cfcb37e..e371d2c6d 100644 --- a/src/util.c +++ b/src/util.c @@ -565,9 +565,19 @@ datapagemap_print_debug(datapagemap_t *map) const char* backup_id_of(pgBackup *backup) { + /* Change this Assert when backup_id will not be bound to start_time */ + Assert(backup->backup_id == backup->start_time || backup->start_time == 0); + if (backup->backup_id_encoded[0] == '\x00') { - base36enc_to(backup->start_time, backup->backup_id_encoded); + base36enc_to(backup->backup_id, backup->backup_id_encoded); } return backup->backup_id_encoded; } + +void +reset_backup_id(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + memset(backup->backup_id_encoded, 0, sizeof(backup->backup_id_encoded)); +} diff --git a/src/validate.c b/src/validate.c index 7a9140bbc..b89b67b84 100644 --- a/src/validate.c +++ b/src/validate.c @@ -734,7 +734,7 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) if (!fileExists(map_path, FIO_BACKUP_HOST)) elog(ERROR, "Tablespace map is missing: \"%s\", " "probably backup %s is corrupt, validate it", - map_path, base36enc(backup->backup_id)); + map_path, backup_id_of(backup)); /* check tablespace map checksumms */ if (!no_validate) @@ -744,7 +744,7 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) if ((*tablespace_map)->crc != crc) elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " "probably backup %s is corrupt, validate it", - map_path, crc, (*tablespace_map)->crc, base36enc(backup->backup_id)); + map_path, crc, (*tablespace_map)->crc, backup_id_of(backup)); } pgFileFree(dummy); From 2b8a1532350af09eed827f3024f8f4a30baf04fb Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Thu, 24 Nov 2022 10:23:59 +0100 Subject: [PATCH 1975/2107] PBCKP-306 add '_test' to tests files --- tests/CVE_2018_1058_test.py | 129 + tests/archive_test.py | 2707 ++++++++++++++++++ tests/backup_test.py | 3564 +++++++++++++++++++++++ tests/cfs_backup_test.py | 1235 ++++++++ tests/cfs_catchup_test.py | 117 + tests/cfs_restore_test.py | 450 +++ tests/cfs_validate_backup_test.py | 24 + tests/checkdb_test.py | 849 ++++++ tests/compatibility_test.py | 1500 ++++++++++ tests/compression_test.py | 495 ++++ tests/config_test.py | 113 + tests/delete_test.py | 822 ++++++ tests/delta_test.py | 1201 ++++++++ tests/exclude_test.py | 338 +++ tests/external_test.py | 2405 ++++++++++++++++ tests/false_positive_test.py | 337 +++ tests/incr_restore_test.py | 2300 +++++++++++++++ tests/init_test.py | 138 + tests/locking_test.py | 629 ++++ tests/logging_test.py | 345 +++ tests/merge_test.py | 2759 ++++++++++++++++++ tests/option_test.py | 231 ++ tests/page_test.py | 1424 ++++++++++ tests/pgpro2068_test.py | 188 ++ tests/pgpro560_test.py | 123 + tests/pgpro589_test.py | 72 + tests/ptrack_test.py | 4407 +++++++++++++++++++++++++++++ tests/remote_test.py | 43 + tests/replica_test.py | 1654 +++++++++++ tests/restore_test.py | 3822 +++++++++++++++++++++++++ tests/retention_test.py | 2529 +++++++++++++++++ tests/set_backup_test.py | 476 ++++ tests/show_test.py | 509 ++++ tests/time_consuming_test.py | 77 + tests/time_stamp_test.py | 236 ++ tests/validate_test.py | 4083 ++++++++++++++++++++++++++ 36 files changed, 42331 insertions(+) create mode 100644 tests/CVE_2018_1058_test.py create mode 100644 tests/archive_test.py create mode 100644 tests/backup_test.py create mode 100644 tests/cfs_backup_test.py create mode 100644 tests/cfs_catchup_test.py create mode 100644 tests/cfs_restore_test.py create mode 100644 tests/cfs_validate_backup_test.py create mode 100644 tests/checkdb_test.py create mode 100644 tests/compatibility_test.py create mode 100644 tests/compression_test.py create mode 100644 tests/config_test.py create mode 100644 tests/delete_test.py create mode 100644 tests/delta_test.py create mode 100644 tests/exclude_test.py create mode 100644 tests/external_test.py create mode 100644 tests/false_positive_test.py create mode 100644 tests/incr_restore_test.py create mode 100644 tests/init_test.py create mode 100644 tests/locking_test.py create mode 100644 tests/logging_test.py create mode 100644 tests/merge_test.py create mode 100644 tests/option_test.py create mode 100644 tests/page_test.py create mode 100644 tests/pgpro2068_test.py create mode 100644 tests/pgpro560_test.py create mode 100644 tests/pgpro589_test.py create mode 100644 tests/ptrack_test.py create mode 100644 tests/remote_test.py create mode 100644 tests/replica_test.py create mode 100644 tests/restore_test.py create mode 100644 tests/retention_test.py create mode 100644 tests/set_backup_test.py create mode 100644 tests/show_test.py create mode 100644 tests/time_consuming_test.py create mode 100644 tests/time_stamp_test.py create mode 100644 tests/validate_test.py diff --git a/tests/CVE_2018_1058_test.py b/tests/CVE_2018_1058_test.py new file mode 100644 index 000000000..cfd55cc60 --- /dev/null +++ b/tests/CVE_2018_1058_test.py @@ -0,0 +1,129 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +class CVE_2018_1058(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_default_search_path(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pgpro_edition() " + "RETURNS text " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + # @unittest.skip("skip") + def test_basic_backup_modified_search_path(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_control_checkpoint(OUT timeline_id integer, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_proc(OUT proname name, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertFalse( + 'pg_probackup vulnerable!' in log_content) + + # @unittest.skip("skip") + def test_basic_checkdb_modified_search_path(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_database(OUT datname name, OUT oid oid, OUT dattablespace oid) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_database AS SELECT * FROM public.pg_database()") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_extension(OUT extname name, OUT extnamespace oid, OUT extversion text) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE FUNCTION public.pg_namespace(OUT oid oid, OUT nspname name) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_extension AS SELECT * FROM public.pg_extension();" + "CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();" + ) + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + self.assertEqual( + 1, 0, + "Expecting Error because amcheck{,_next} not installed\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) diff --git a/tests/archive_test.py b/tests/archive_test.py new file mode 100644 index 000000000..5e59dd268 --- /dev/null +++ b/tests/archive_test.py @@ -0,0 +1,2707 @@ +import os +import shutil +import gzip +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException +from datetime import datetime, timedelta +import subprocess +from sys import exit +from time import sleep +from distutils.dir_util import copy_tree + + +class ArchiveTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_pgpro434_1(self): + """Description in jira issue PGPRO-434""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector from " + "generate_series(0,100) i") + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node) + node.slow_start() + + # Recreate backup catalog + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Make backup + self.backup_node(backup_dir, 'node', node) + node.cleanup() + + # Restore Database + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro434_2(self): + """ + Check that timelines are correct. + WAITING PGPRO-1053 for --immediate + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FIRST TIMELINE + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + backup_id = self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "insert into t_heap select 100501 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + # SECOND TIMELIN + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 " + "from t_heap where id = 100501)")[0][0], + 'data after restore not equal to original data') + + node.safe_psql( + "postgres", + "insert into t_heap select 2 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(100,200) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select 100502 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # THIRD TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print( + node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + node.safe_psql( + "postgres", + "insert into t_heap select 3 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(200,300) i") + + backup_id = self.backup_node(backup_dir, 'node', node) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.safe_psql( + "postgres", + "insert into t_heap select 100503 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + # FOURTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fourth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # FIFTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Fifth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + # SIXTH TIMELINE + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--immediate', '--recovery-target-action=promote']) + node.slow_start() + + if self.verbose: + print('Sixth timeline') + print(node.safe_psql( + "postgres", + "select redo_wal_file from pg_control_checkpoint()")) + + self.assertFalse( + node.execute( + "postgres", + "select exists(select 1 from t_heap where id > 100500)")[0][0], + 'data after restore not equal to original data') + + self.assertEqual( + result, + node.safe_psql( + "postgres", + "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # @unittest.skip("skip") + def test_pgpro434_3(self): + """ + Check pg_stop_backup_timeout, needed backup_timeout + Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + gdb = self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=LOG"], + gdb=True) + + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + self.set_auto_conf(node, {'archive_command': 'exit 1'}) + node.reload() + + gdb.continue_execution_until_exit() + + sleep(1) + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + + # in PG =< 9.6 pg_stop_backup always wait + if self.get_version(node) < 100000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: WAL segment 000000010000000000000003 could not be archived in 60 seconds", + log_content) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # @unittest.skip("skip") + def test_pgpro434_4(self): + """ + Check pg_stop_backup_timeout, libpq-timeout requested. + Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + gdb = self.backup_node( + backup_dir, 'node', node, + options=[ + "--archive-timeout=60", + "--log-level-file=info"], + gdb=True) + + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + self.set_auto_conf(node, {'archive_command': 'exit 1'}) + node.reload() + + os.environ["PGAPPNAME"] = "foo" + + pid = node.safe_psql( + "postgres", + "SELECT pid " + "FROM pg_stat_activity " + "WHERE application_name = 'pg_probackup'").decode('utf-8').rstrip() + + os.environ["PGAPPNAME"] = "pg_probackup" + + postgres_gdb = self.gdb_attach(pid) + postgres_gdb.set_breakpoint('do_pg_stop_backup') + postgres_gdb.continue_execution_until_running() + + gdb.continue_execution_until_exit() + # gdb._execute('detach') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + log_content) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertNotIn( + 'FailedAssertion', + log_content, + 'PostgreSQL crashed because of a failed assert') + + # @unittest.skip("skip") + def test_archive_push_file_exists(self): + """Archive-push if file exists""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = '000000010000000000000001.gz' + file = os.path.join(wals_dir, filename) + else: + filename = '000000010000000000000001' + file = os.path.join(wals_dir, filename) + + with open(file, 'a+b') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + self.switch_wal_segment(node) + sleep(1) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertIn( + 'LOG: archive command failed with exit code 1', + log_content) + + self.assertIn( + 'DETAIL: The failed archive command was:', + log_content) + + self.assertIn( + 'pg_probackup archive-push WAL file', + log_content) + + self.assertIn( + 'WAL file already exists in archive with different checksum', + log_content) + + self.assertNotIn( + 'pg_probackup archive-push completed successfully', log_content) + + if self.get_version(node) < 100000: + wal_src = os.path.join( + node.data_dir, 'pg_xlog', '000000010000000000000001') + else: + wal_src = os.path.join( + node.data_dir, 'pg_wal', '000000010000000000000001') + + if self.archive_compress: + with open(wal_src, 'rb') as f_in, gzip.open( + file, 'wb', compresslevel=1) as f_out: + shutil.copyfileobj(f_in, f_out) + else: + shutil.copyfile(wal_src, file) + + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'pg_probackup archive-push completed successfully', + log_content) + + # btw check that console coloring codes are not slipped into log file + self.assertNotIn('[0m', log_content) + + print(log_content) + + # @unittest.skip("skip") + def test_archive_push_file_exists_overwrite(self): + """Archive-push if file exists""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = '000000010000000000000001.gz' + file = os.path.join(wals_dir, filename) + else: + filename = '000000010000000000000001' + file = os.path.join(wals_dir, filename) + + with open(file, 'a+b') as f: + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close() + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100500) i") + log_file = os.path.join(node.logs_dir, 'postgresql.log') + + self.switch_wal_segment(node) + sleep(1) + + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'LOG: archive command failed with exit code 1', log_content) + self.assertIn( + 'DETAIL: The failed archive command was:', log_content) + self.assertIn( + 'pg_probackup archive-push WAL file', log_content) + self.assertNotIn( + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log_content) + self.assertIn( + 'WAL file already exists in archive with ' + 'different checksum', log_content) + + self.assertNotIn( + 'pg_probackup archive-push completed successfully', log_content) + + self.set_archiving(backup_dir, 'node', node, overwrite=True) + node.reload() + self.switch_wal_segment(node) + sleep(5) + + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'pg_probackup archive-push completed successfully' in log_content, + 'Expecting messages about successfull execution archive_command') + + self.assertIn( + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log_content) + + # @unittest.skip("skip") + def test_archive_push_partial_file_exists(self): + """Archive-push if stale '.part' file exists""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving( + backup_dir, 'node', node, + log_level='verbose', archive_timeout=60) + + node.slow_start() + + # this backup is needed only for validation to xid + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t1(a int)") + + xid = node.safe_psql( + "postgres", + "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() + + if self.get_version(node) < 100000: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() + else: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + + filename_orig = filename_orig.decode('utf-8') + + # form up path to next .part WAL segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = filename_orig + '.gz' + '.part' + file = os.path.join(wals_dir, filename) + else: + filename = filename_orig + '.part' + file = os.path.join(wals_dir, filename) + + # emulate stale .part file + with open(file, 'a+b') as f: + f.write(b"blahblah") + f.flush() + f.close() + + self.switch_wal_segment(node) + sleep(70) + + # check that segment is archived + if self.archive_compress: + filename_orig = filename_orig + '.gz' + + file = os.path.join(wals_dir, filename_orig) + self.assertTrue(os.path.isfile(file)) + + # successful validate means that archive-push reused stale wal segment + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-xid={0}'.format(xid)]) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'Reusing stale temp WAL file', + log_content) + + # @unittest.skip("skip") + def test_archive_push_part_file_exists_not_stale(self): + """Archive-push if .part file exists and it is not stale""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, archive_timeout=60) + + node.slow_start() + + node.safe_psql( + "postgres", + "create table t1()") + self.switch_wal_segment(node) + + node.safe_psql( + "postgres", + "create table t2()") + + if self.get_version(node) < 100000: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() + else: + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + + filename_orig = filename_orig.decode('utf-8') + + # form up path to next .part WAL segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + if self.archive_compress: + filename = filename_orig + '.gz' + '.part' + file = os.path.join(wals_dir, filename) + else: + filename = filename_orig + '.part' + file = os.path.join(wals_dir, filename) + + with open(file, 'a+b') as f: + f.write(b"blahblah") + f.flush() + f.close() + + self.switch_wal_segment(node) + sleep(30) + + with open(file, 'a+b') as f: + f.write(b"blahblahblahblah") + f.flush() + f.close() + + sleep(40) + + # check that segment is NOT archived + if self.archive_compress: + filename_orig = filename_orig + '.gz' + + file = os.path.join(wals_dir, filename_orig) + + self.assertFalse(os.path.isfile(file)) + + # log_file = os.path.join(node.logs_dir, 'postgresql.log') + # with open(log_file, 'r') as f: + # log_content = f.read() + # self.assertIn( + # 'is not stale', + # log_content) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_replica_archive(self): + """ + make node without archiving, take stream backup and + turn it into replica, set replica with archiving, + make archive backup from replica + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s', + 'max_wal_size': '32MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica, synchronous=True) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,80680) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + self.wait_until_replica_catch_with_master(master, replica) + + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=60', + '--master-db=postgres', + '--master-host=localhost', + '--master-port={0}'.format(master.port), + '--stream']) + + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_master_and_replica_parallel_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, make archive backup from replica, + make archive backup from master + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s'} + ) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node(backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica) + # ADD INSTANCE REPLICA + self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'replica', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0, 60000) i") + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_basic_master_and_replica_concurrent_archiving(self): + """ + make node 'master 'with archiving, + take archive backup and turn it into replica, + set replica with archiving, + make sure that archiving on both node is working. + """ + if self.pg_config_version < self.version_to_num('9.6.0'): + self.skipTest('You need PostgreSQL >= 9.6 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'archive_timeout': '10s'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.init_pb(backup_dir) + # ADD INSTANCE 'MASTER' + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + master.pgbench_init(scale=5) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'master', master) + # GET LOGICAL CONTENT FROM MASTER + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + # GET PHYSICAL CONTENT FROM MASTER + pgdata_master = self.pgdata_content(master.data_dir) + + # Settings for Replica + self.restore_node( + backup_dir, 'master', replica) + # CHECK PHYSICAL CORRECTNESS on REPLICA + pgdata_replica = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata_master, pgdata_replica) + + self.set_replica(master, replica, synchronous=False) + # ADD INSTANCE REPLICA + # self.add_instance(backup_dir, 'replica', replica) + # SET ARCHIVING FOR REPLICA + self.set_archiving(backup_dir, 'master', replica, replica=True) + replica.slow_start(replica=True) + + # CHECK LOGICAL CORRECTNESS on REPLICA + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # TAKE FULL ARCHIVE BACKUP FROM REPLICA + backup_id = self.backup_node(backup_dir, 'master', replica) + + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + backup_id = self.backup_node(backup_dir, 'master', master) + self.validate_pb(backup_dir, 'master') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + + master.pgbench_init(scale=10) + + sleep(10) + + replica.promote() + + master.pgbench_init(scale=10) + replica.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'master', master) + self.backup_node(backup_dir, 'master', replica) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_concurrent_archiving(self): + """ + Concurrent archiving from master, replica and cascade replica + https://github.com/postgrespro/pg_probackup/issues/327 + + For PG >= 11 it is expected to pass this test + """ + + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + master.pgbench_init(scale=10) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'node', master) + + # Settings for Replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + replica.slow_start(replica=True) + + # create cascade replicas + replica1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica1')) + replica1.cleanup() + + # Settings for casaced replica + self.restore_node(backup_dir, 'node', replica1) + self.set_replica(replica, replica1, synchronous=False) + self.set_auto_conf(replica1, {'port': replica1.port}) + replica1.slow_start(replica=True) + + # Take full backup from master + self.backup_node(backup_dir, 'node', master) + + pgbench = master.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '30', '-c', '1']) + + # Take several incremental backups from master + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + pgbench.wait() + pgbench.stdout.close() + + with open(os.path.join(master.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog(self): + """Test backup with pg_receivexlog wal delivary method""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node') + ], asynchronous=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, + node.safe_psql( + "postgres", "SELECT * FROM t_heap" + ), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_compression_pg10(self): + """Test backup with pg_receivewal compressed wal delivary method""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + if self.get_version(node) < self.version_to_num('10.0'): + self.skipTest('You need PostgreSQL >= 10 for this test') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') + ], asynchronous=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'node', node, + backup_type='page' + ) + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.validate_pb(backup_dir) + + # Check data correctness + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'data after restore not equal to original data') + + # Clean after yourself + pg_receivexlog.kill() + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog(self): + """ + ARCHIVE replica: + + t6 |----------------------- + t5 | |------- + | | + t4 | |-------------- + | | + t3 | |--B1--|/|--B2-|/|-B3--- + | | + t2 |--A1--------A2--- + t1 ---------Y1--Y2-- + + ARCHIVE master: + t1 -Z1--Z2--- + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + + master.slow_start() + + # FULL + master.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + self.backup_node(backup_dir, 'master', master) + + # PAGE + master.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + + self.backup_node( + backup_dir, 'master', master, backup_type='page') + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + # FULL backup replica + Y1 = self.backup_node( + backup_dir, 'replica', replica, + options=['--stream', '--archive-timeout=60s']) + + master.pgbench_init(scale=5) + + # PAGE backup replica + Y2 = self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream', '--archive-timeout=60s']) + + # create timeline t2 + replica.promote() + + # FULL backup replica + A1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=5) + + replica.safe_psql( + 'postgres', + "CREATE TABLE t1 (a text)") + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # DELTA backup replica + A2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='delta') + + # create timeline t3 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + replica.slow_start() + + B1 = self.backup_node( + backup_dir, 'replica', replica) + + replica.pgbench_init(scale=2) + + B2 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + B3 = self.backup_node( + backup_dir, 'replica', replica, backup_type='page') + + replica.pgbench_init(scale=2) + + # create timeline t4 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't2 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + target_xid = None + with replica.connect("postgres") as con: + res = con.execute( + "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't3 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,10) i') + + # create timeline t5 + replica.cleanup() + self.restore_node( + backup_dir, 'replica', replica, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=4', + '--recovery-target-action=promote']) + + replica.slow_start() + + replica.safe_psql( + 'postgres', + 'CREATE TABLE ' + 't4 as select i, ' + 'repeat(md5(i::text),5006056) as fat_attr ' + 'from generate_series(0,6) i') + + # create timeline t6 + replica.cleanup() + + self.restore_node( + backup_dir, 'replica', replica, backup_id=A1, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + replica.slow_start() + + replica.pgbench_init(scale=2) + + sleep(5) + + show = self.show_archive(backup_dir, as_text=True) + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + if instance['instance'] == 'master': + master_timelines = instance['timelines'] + + # check that all timelines are ok + for timeline in replica_timelines: + self.assertTrue(timeline['status'], 'OK') + + # check that all timelines are ok + for timeline in master_timelines: + self.assertTrue(timeline['status'], 'OK') + + # create holes in t3 + wals_dir = os.path.join(backup_dir, 'wal', 'replica') + wals = [ + f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) + and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') + ] + wals.sort() + + # check that t3 is ok + self.show_archive(backup_dir) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') + if self.archive_compress: + file = file + '.gz' + os.remove(file) + + # check that t3 is not OK + show = self.show_archive(backup_dir) + + show = self.show_archive(backup_dir) + + for instance in show: + if instance['instance'] == 'replica': + replica_timelines = instance['timelines'] + + # sanity + for timeline in replica_timelines: + if timeline['tli'] == 1: + timeline_1 = timeline + continue + + if timeline['tli'] == 2: + timeline_2 = timeline + continue + + if timeline['tli'] == 3: + timeline_3 = timeline + continue + + if timeline['tli'] == 4: + timeline_4 = timeline + continue + + if timeline['tli'] == 5: + timeline_5 = timeline + continue + + if timeline['tli'] == 6: + timeline_6 = timeline + continue + + self.assertEqual(timeline_6['status'], "OK") + self.assertEqual(timeline_5['status'], "OK") + self.assertEqual(timeline_4['status'], "OK") + self.assertEqual(timeline_3['status'], "DEGRADED") + self.assertEqual(timeline_2['status'], "OK") + self.assertEqual(timeline_1['status'], "OK") + + self.assertEqual(len(timeline_3['lost-segments']), 2) + self.assertEqual( + timeline_3['lost-segments'][0]['begin-segno'], + '000000030000000000000012') + self.assertEqual( + timeline_3['lost-segments'][0]['end-segno'], + '000000030000000000000013') + self.assertEqual( + timeline_3['lost-segments'][1]['begin-segno'], + '000000030000000000000017') + self.assertEqual( + timeline_3['lost-segments'][1]['end-segno'], + '000000030000000000000017') + + self.assertEqual(len(timeline_6['backups']), 0) + self.assertEqual(len(timeline_5['backups']), 0) + self.assertEqual(len(timeline_4['backups']), 0) + self.assertEqual(len(timeline_3['backups']), 3) + self.assertEqual(len(timeline_2['backups']), 2) + self.assertEqual(len(timeline_1['backups']), 2) + + # check closest backup correctness + self.assertEqual(timeline_6['closest-backup-id'], A1) + self.assertEqual(timeline_5['closest-backup-id'], B2) + self.assertEqual(timeline_4['closest-backup-id'], B2) + self.assertEqual(timeline_3['closest-backup-id'], A1) + self.assertEqual(timeline_2['closest-backup-id'], Y2) + + # check parent tli correctness + self.assertEqual(timeline_6['parent-tli'], 2) + self.assertEqual(timeline_5['parent-tli'], 4) + self.assertEqual(timeline_4['parent-tli'], 3) + self.assertEqual(timeline_3['parent-tli'], 2) + self.assertEqual(timeline_2['parent-tli'], 1) + self.assertEqual(timeline_1['parent-tli'], 0) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog_1(self): + """ + double segment - compressed and not + """ + if not self.archive_compress: + self.skipTest('You need to enable ARCHIVE_COMPRESSION ' + 'for this test to run') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + original_file = os.path.join(wals_dir, '000000010000000000000001.gz') + tmp_file = os.path.join(wals_dir, '000000010000000000000001') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + os.rename( + os.path.join(wals_dir, '000000010000000000000001'), + os.path.join(wals_dir, '000000010000000000000002')) + + show = self.show_archive(backup_dir) + + for instance in show: + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000001') + self.assertEqual(timeline['status'], 'OK') + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_catalog_2(self): + """ + double segment - compressed and not + """ + if not self.archive_compress: + self.skipTest('You need to enable ARCHIVE_COMPRESSION ' + 'for this test to run') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + original_file = os.path.join(wals_dir, '000000010000000000000001.gz') + tmp_file = os.path.join(wals_dir, '000000010000000000000001') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + os.rename( + os.path.join(wals_dir, '000000010000000000000001'), + os.path.join(wals_dir, '000000010000000000000002')) + + os.remove(original_file) + + show = self.show_archive(backup_dir) + + for instance in show: + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000002') + self.assertEqual(timeline['status'], 'OK') + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_options(self): + """ + check that '--archive-host', '--archive-user', '--archiver-port' + and '--restore-command' are working as expected. + """ + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + node.cleanup() + + wal_dir = os.path.join(backup_dir, 'wal', 'node') + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command="cp {0}/%f %p"'.format(wal_dir), + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user) + ]) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), + recovery_content) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user)]) + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " + "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " + "--remote-port=22 --remote-user={3}'".format( + self.probackup_path, backup_dir, 'node', self.user), + recovery_content) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_options_1(self): + """ + check that '--archive-host', '--archive-user', '--archiver-port' + and '--restore-command' are working as expected with set-config + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + node.cleanup() + + wal_dir = os.path.join(backup_dir, 'wal', 'node') + self.set_config( + backup_dir, 'node', + options=[ + '--restore-command="cp {0}/%f %p"'.format(wal_dir), + '--archive-host=localhost', + '--archive-port=22', + '--archive-user={0}'.format(self.user)]) + self.restore_node(backup_dir, 'node', node) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), + recovery_content) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command=none', + '--archive-host=localhost1', + '--archive-port=23', + '--archive-user={0}'.format(self.user) + ]) + + with open(recovery_conf, 'r') as f: + recovery_content = f.read() + + self.assertIn( + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " + "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " + "--remote-port=23 --remote-user={3}'".format( + self.probackup_path, backup_dir, 'node', self.user), + recovery_content) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_undefined_wal_file_path(self): + """ + check that archive-push works correct with undefined + --wal-file-path + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + if os.name == 'posix': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node') + elif os.name == 'nt': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node').replace("\\","\\\\") + else: + self.assertTrue(False, 'Unexpected os family') + + self.set_auto_conf( + node, + {'archive_command': archive_command}) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_intermediate_archiving(self): + """ + check that archive-push works correct with --wal-file-path setting by user + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + node_pg_options = {} + if node.major_version >= 13: + node_pg_options['wal_keep_size'] = '0MB' + else: + node_pg_options['wal_keep_segments'] = '0' + self.set_auto_conf(node, node_pg_options) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'intermediate_dir') + shutil.rmtree(wal_dir, ignore_errors=True) + os.makedirs(wal_dir) + if os.name == 'posix': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) + elif os.name == 'nt': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) + else: + self.assertTrue(False, 'Unexpected os family') + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + wal_segment = '000000010000000000000001' + + self.run_pb(["archive-push", "-B", backup_dir, + "--instance=node", "-D", node.data_dir, + "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment]) + + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_waldir_outside_pgdata_archiving(self): + """ + check that archive-push works correct with symlinked waldir + """ + if self.pg_config_version < self.version_to_num('10.0'): + self.skipTest( + 'Skipped because waldir outside pgdata is supported since PG 10') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') + shutil.rmtree(external_wal_dir, ignore_errors=True) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_hexadecimal_timeline(self): + """ + Check that timelines are correct. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, log_level='verbose') + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=2) + + # create timelines + for i in range(1, 13): + # print(i) + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=['--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + node.pgbench_init(scale=2) + + sleep(5) + + show = self.show_archive(backup_dir) + + timelines = show[0]['timelines'] + + print(timelines[0]) + + tli13 = timelines[0] + + self.assertEqual( + 13, + tli13['tli']) + + self.assertEqual( + 12, + tli13['parent-tli']) + + self.assertEqual( + backup_id, + tli13['closest-backup-id']) + + self.assertEqual( + '0000000D000000000000001C', + tli13['max-segno']) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_archiving_and_slots(self): + """ + Check that archiving don`t break slot + guarantee. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'max_wal_size': '64MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, log_level='verbose') + node.slow_start() + + if self.get_version(node) < 100000: + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " + # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" + + self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '--create-slot', '--slot', 'archive_slot', '--if-not-exists' + ]) + + node.pgbench_init(scale=10) + + pg_receivexlog = self.run_binary( + [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node'), + '--no-loop', '--slot', 'archive_slot', + '-Z', '1' + ], asynchronous=True) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + sleep(2) + + pg_receivexlog.kill() + + backup_id = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + exit(1) + + def test_archive_push_sanity(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_mode': 'on', + 'archive_command': 'exit 1'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=50) + node.stop() + + self.set_archiving(backup_dir, 'node', node) + os.remove(os.path.join(node.logs_dir, 'postgresql.log')) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + # print(postgres_log_content) + # make sure that .backup file is not compressed + self.assertNotIn('.backup.gz', postgres_log_content) + self.assertNotIn('WARNING', postgres_log_content) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, + data_dir=replica.data_dir, options=['-R']) + + # self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + self.set_auto_conf(replica, {'archive_mode': 'always'}) + self.set_auto_conf(replica, {'hot_standby': 'on'}) + replica.slow_start(replica=True) + + self.wait_until_replica_catch_with_master(node, replica) + + node.pgbench_init(scale=5) + + replica.promote() + replica.pgbench_init(scale=10) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + replica_log_content = f.read() + + # make sure that .partial file is not compressed + self.assertNotIn('.partial.gz', replica_log_content) + # make sure that .history file is not compressed + self.assertNotIn('.history.gz', replica_log_content) + self.assertNotIn('WARNING', replica_log_content) + + output = self.show_archive( + backup_dir, 'node', as_json=False, as_text=True, + options=['--log-level-console=INFO']) + + self.assertNotIn('WARNING', output) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_pg_receivexlog_partial_handling(self): + """check that archive-get delivers .partial and .gz.partial files""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + if self.get_version(node) < 100000: + app_name = 'pg_receivexlog' + pg_receivexlog_path = self.get_bin_path('pg_receivexlog') + else: + app_name = 'pg_receivewal' + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + + cmdline = [ + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node')] + + if self.archive_compress and node.major_version >= 10: + cmdline += ['-Z', '1'] + + env = self.test_env + env["PGAPPNAME"] = app_name + pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env=env) + + if pg_receivexlog.returncode: + self.assertFalse( + True, + 'Failed to start pg_receivexlog: {0}'.format( + pg_receivexlog.communicate()[1])) + + self.set_auto_conf(node, {'synchronous_standby_names': app_name}) + self.set_auto_conf(node, {'synchronous_commit': 'on'}) + node.reload() + + # FULL + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000000) i") + + # PAGE + self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--stream']) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(1000000,2000000) i") + + pg_receivexlog.kill() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, node_restored.data_dir, + options=['--recovery-target=latest', '--recovery-target-action=promote']) + self.set_auto_conf(node_restored, {'port': node_restored.port}) + self.set_auto_conf(node_restored, {'hot_standby': 'off'}) + + node_restored.slow_start() + + result = node.safe_psql( + "postgres", + "select sum(id) from t_heap").decode('utf-8').rstrip() + + result_new = node_restored.safe_psql( + "postgres", + "select sum(id) from t_heap").decode('utf-8').rstrip() + + self.assertEqual(result, result_new) + + @unittest.skip("skip") + def test_multi_timeline_recovery_prefetching(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=50) + + target_xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + node.pgbench_init(scale=20) + + node.stop() + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-action=promote']) + + node.slow_start() + + node.pgbench_init(scale=20) + + target_xid = node.safe_psql( + 'postgres', + 'select txid_current()').rstrip() + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ +# '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', +# '--recovery-target-action=promote', + '--no-validate']) + node.slow_start() + + node.pgbench_init(scale=20) + result = node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + node.stop() + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ +# '--recovery-target-xid=100500', + '--recovery-target-timeline=3', +# '--recovery-target-action=promote', + '--no-validate']) + os.remove(os.path.join(node.logs_dir, 'postgresql.log')) + + restore_command = self.get_restore_command(backup_dir, 'node', node) + restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' + + if node.major_version >= 12: + node.append_conf( + 'postgresql.auto.conf', "restore_command = '{0}'".format(restore_command)) + else: + node.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + node.slow_start() + + result_new = node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + + self.assertEqual(result, result_new) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + # check that requesting of non-existing segment do not + # throwns aways prefetch + self.assertIn( + 'pg_probackup archive-get failed to ' + 'deliver WAL file: 000000030000000000000006', + postgres_log_content) + + self.assertIn( + 'pg_probackup archive-get failed to ' + 'deliver WAL file: 000000020000000000000006', + postgres_log_content) + + self.assertIn( + 'pg_probackup archive-get used prefetched ' + 'WAL segment 000000010000000000000006, prefetch state: 5/10', + postgres_log_content) + + def test_archive_get_batching_sanity(self): + """ + Make sure that batching works. + .gz file is corrupted and uncompressed is not, check that both + corruption detected and uncompressed file is used. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=50) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, replica.data_dir) + self.set_replica(node, replica, log_shipping=True) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + else: + replica.append_conf('recovery.conf', "restore_command = 'exit 1'") + + replica.slow_start(replica=True) + + # at this point replica is consistent + restore_command = self.get_restore_command(backup_dir, 'node', replica) + + restore_command += ' -j 2 --batch-size=10' + + # print(restore_command) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + replica.restart() + + sleep(5) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'pg_probackup archive-get completed successfully, fetched: 10/10', + postgres_log_content) + self.assertIn('used prefetched WAL segment', postgres_log_content) + self.assertIn('prefetch state: 9/10', postgres_log_content) + self.assertIn('prefetch state: 8/10', postgres_log_content) + + def test_archive_get_prefetch_corruption(self): + """ + Make sure that WAL corruption is detected. + And --prefetch-dir is honored. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=50) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node( + backup_dir, 'node', replica, replica.data_dir) + self.set_replica(node, replica, log_shipping=True) + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + else: + replica.append_conf('recovery.conf', "restore_command = 'exit 1'") + + replica.slow_start(replica=True) + + # at this point replica is consistent + restore_command = self.get_restore_command(backup_dir, 'node', replica) + + restore_command += ' -j5 --batch-size=10 --log-level-console=VERBOSE' + #restore_command += ' --batch-size=2 --log-level-console=VERBOSE' + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + replica.restart() + + sleep(5) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'pg_probackup archive-get completed successfully, fetched: 10/10', + postgres_log_content) + self.assertIn('used prefetched WAL segment', postgres_log_content) + self.assertIn('prefetch state: 9/10', postgres_log_content) + self.assertIn('prefetch state: 8/10', postgres_log_content) + + replica.stop() + + # generate WAL, copy it into prefetch directory, then corrupt + # some segment + node.pgbench_init(scale=20) + sleep(20) + + # now copy WAL files into prefetch directory and corrupt some of them + archive_dir = os.path.join(backup_dir, 'wal', 'node') + files = os.listdir(archive_dir) + files.sort() + + for filename in [files[-4], files[-3], files[-2], files[-1]]: + src_file = os.path.join(archive_dir, filename) + + if node.major_version >= 10: + wal_dir = 'pg_wal' + else: + wal_dir = 'pg_xlog' + + if filename.endswith('.gz'): + dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename[:-3]) + with gzip.open(src_file, 'rb') as f_in, open(dst_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + else: + dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) + shutil.copyfile(src_file, dst_file) + + # print(dst_file) + + # corrupt file + if files[-2].endswith('.gz'): + filename = files[-2][:-3] + else: + filename = files[-2] + + prefetched_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) + + with open(prefetched_file, "rb+", 0) as f: + f.seek(8192*2) + f.write(b"SURIKEN") + f.flush() + f.close + + # enable restore_command + restore_command = self.get_restore_command(backup_dir, 'node', replica) + restore_command += ' --batch-size=2 --log-level-console=VERBOSE' + + if node.major_version >= 12: + self.set_auto_conf(replica, {'restore_command': restore_command}) + else: + replica.append_conf( + 'recovery.conf', "restore_command = '{0}'".format(restore_command)) + + os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) + replica.slow_start(replica=True) + + sleep(60) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + postgres_log_content = f.read() + + self.assertIn( + 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), + postgres_log_content) + + self.assertIn( + 'LOG: restored log file "{0}" from archive'.format(filename), + postgres_log_content) + + # @unittest.skip("skip") + def test_archive_show_partial_files_handling(self): + """ + check that files with '.part', '.part.gz', '.partial' and '.partial.gz' + siffixes are handled correctly + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node, compress=False) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + wals_dir = os.path.join(backup_dir, 'wal', 'node') + + # .part file + node.safe_psql( + "postgres", + "create table t1()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + filename = filename.decode('utf-8') + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.part'.format(filename))) + + # .gz.part file + node.safe_psql( + "postgres", + "create table t2()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + filename = filename.decode('utf-8') + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.gz.part'.format(filename))) + + # .partial file + node.safe_psql( + "postgres", + "create table t3()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + filename = filename.decode('utf-8') + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.partial'.format(filename))) + + # .gz.partial file + node.safe_psql( + "postgres", + "create table t4()") + + if self.get_version(node) < 100000: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() + else: + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + + filename = filename.decode('utf-8') + + self.switch_wal_segment(node) + + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}.gz.partial'.format(filename))) + + self.show_archive(backup_dir, 'node', options=['--log-level-file=VERBOSE']) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log'), 'r') as f: + log_content = f.read() + + self.assertNotIn( + 'WARNING', + log_content) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_empty_history_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/326 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.pgbench_init(scale=5) + + # FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote']) + + # Node in timeline 2 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + # Node in timeline 3 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + # Node in timeline 4 + node.slow_start() + node.pgbench_init(scale=5) + + # Truncate history files + for tli in range(2, 5): + file = os.path.join( + backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) + with open(file, "w+") as f: + f.truncate() + + timelines = self.show_archive(backup_dir, 'node', options=['--log-level-file=INFO']) + + # check that all timelines has zero switchpoint + for timeline in timelines: + self.assertEqual(timeline['switchpoint'], '0/0') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + wal_dir = os.path.join(backup_dir, 'wal', 'node') + + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), + log_content) + +# TODO test with multiple not archived segments. +# TODO corrupted file in archive. + +# important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. +# so write WAL validation code accordingly + +# change wal-seg-size +# +# +#t3 ---------------- +# / +#t2 ---------------- +# / +#t1 -A-------- +# +# + + +#t3 ---------------- +# / +#t2 ---------------- +# / +#t1 -A-------- +# diff --git a/tests/backup_test.py b/tests/backup_test.py new file mode 100644 index 000000000..db7ccf5a0 --- /dev/null +++ b/tests/backup_test.py @@ -0,0 +1,3564 @@ +import unittest +import os +from time import sleep, time +from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException +import shutil +from distutils.dir_util import copy_tree +from testgres import ProcessType, QueryException +import subprocess + + +class BackupTest(ProbackupTest, unittest.TestCase): + + def test_full_backup(self): + """ + Just test full backup with at least two segments + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + def test_full_backup_stream(self): + """ + Just test full backup with at least two segments in stream mode + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, + options=["--stream"]) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-707 + def test_backup_modes_archive(self): + """standart backup modes with ARCHIVE WAL method""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_backup_id = self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # postmaster.pid and postmaster.opts shouldn't be copied + excluded = True + db_dir = os.path.join( + backup_dir, "backups", 'node', full_backup_id, "database") + + for f in os.listdir(db_dir): + if ( + os.path.isfile(os.path.join(db_dir, f)) and + ( + f == "postmaster.pid" or + f == "postmaster.opts" + ) + ): + excluded = False + self.assertEqual(excluded, True) + + # page backup mode + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_backup_1 = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup_1['status'], "OK") + self.assertEqual(show_backup_1['backup-mode'], "PAGE") + + # delta backup mode + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + show_backup_2 = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup_2['status'], "OK") + self.assertEqual(show_backup_2['backup-mode'], "DELTA") + + # Check parent backup + self.assertEqual( + full_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup_1['id'])["parent-backup-id"]) + + self.assertEqual( + page_backup_id, + self.show_pb( + backup_dir, 'node', + backup_id=show_backup_2['id'])["parent-backup-id"]) + + # @unittest.skip("skip") + def test_smooth_checkpoint(self): + """full backup with smooth checkpoint""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + options=["-C"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + node.stop() + + # @unittest.skip("skip") + def test_incremental_backup_without_full(self): + """page backup without validated full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # @unittest.skip("skip") + def test_incremental_backup_corrupt_full(self): + """page-level backup with corrupted full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of validation of corrupted backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message and + "WARNING: Backup file" in e.message and "is not found" in e.message and + "WARNING: Backup {0} data files are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # @unittest.skip("skip") + def test_delta_threads_stream(self): + """delta multi thread backup mode and stream""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + + # @unittest.skip("skip") + def test_page_detect_corruption(self): + """make node, corrupt some page, check that backup failed""" + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) + self.assertEqual( + 1, 0, + "Expecting Error because data file is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Corruption detected in file "{0}", ' + 'block 1: page verification failed, calculated checksum'.format(path), + e.message) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], + 'ERROR', + "Backup Status should be ERROR") + + # @unittest.skip("skip") + def test_backup_detect_corruption(self): + """make node, corrupt some page, check that backup failed""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.ptrack: + node.safe_psql( + "postgres", + "create extension ptrack") + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + + node.stop() + + heap_fullpath = os.path.join(node.data_dir, heap_path) + + with open(heap_fullpath, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_backup_detect_invalid_block_header(self): + """make node, corrupt some page, check that backup failed""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.ptrack: + node.safe_psql( + "postgres", + "create extension ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + + node.stop() + + heap_fullpath = os.path.join(node.data_dir, heap_path) + with open(heap_fullpath, "rb+", 0) as f: + f.seek(8193) + f.write(b"blahblahblahblah") + f.flush() + f.close + + node.slow_start() + +# self.backup_node( +# backup_dir, 'node', node, +# backup_type="full", options=["-j", "4", "--stream"]) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_backup_detect_missing_permissions(self): + """make node, corrupt some page, check that backup failed""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + if self.ptrack: + node.safe_psql( + "postgres", + "create extension ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "select count(*) from t_heap") + + node.safe_psql( + "postgres", + "update t_heap set id = id + 10000") + + node.stop() + + heap_fullpath = os.path.join(node.data_dir, heap_path) + with open(heap_fullpath, "rb+", 0) as f: + f.seek(8193) + f.write(b"blahblahblahblah") + f.flush() + f.close + + node.slow_start() + +# self.backup_node( +# backup_dir, 'node', node, +# backup_type="full", options=["-j", "4", "--stream"]) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="page", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + if self.ptrack: + try: + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4", "--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of block corruption" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page header invalid, pd_lower'.format(heap_fullpath), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_backup_truncate_misaligned(self): + """ + make node, truncate file to size not even to BLCKSIZE, + take backup + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100000) i") + + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + output = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"], return_id=False) + + self.assertIn("WARNING: File", output) + self.assertIn("invalid file size", output) + + # @unittest.skip("skip") + def test_tablespace_in_pgdata_pgpro_1376(self): + """PGPRO-1376 """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=( + os.path.join( + node.data_dir, 'somedirectory', '100500')) + ) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=(os.path.join(node.data_dir)) + ) + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace tblspace1 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + backup_id_1 = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node.safe_psql( + "postgres", + "drop table t_heap2") + node.safe_psql( + "postgres", + "drop tablespace tblspace2") + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content(node.data_dir) + + relfilenode = node.safe_psql( + "postgres", + "select 't_heap1'::regclass::oid" + ).decode('utf-8').rstrip() + + list = [] + for root, dirs, files in os.walk(os.path.join( + backup_dir, 'backups', 'node', backup_id_1)): + for file in files: + if file == relfilenode: + path = os.path.join(root, file) + list = list + [path] + + # We expect that relfilenode can be encountered only once + if len(list) > 1: + message = "" + for string in list: + message = message + string + "\n" + self.assertEqual( + 1, 0, + "Following file copied twice by backup:\n {0}".format( + message) + ) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_basic_tablespace_handling(self): + """ + make node, take full backup, check that restore with + tablespace mapping will end with error, take page backup, + check that restore with tablespace mapping will end with + success + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') + + self.create_tblspace_in_node( + node, 'some_lame_tablespace') + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=tblspace1_old_path) + + self.create_tblspace_in_node( + node, 'tblspace2', + tblspc_path=tblspace2_old_path) + + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace some_lame_tablespace " + "as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + node.safe_psql( + "postgres", + "create table t_heap2 tablespace tblspace2 as select 1 as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + tblspace1_new_path = self.get_tblspace_path(node, 'tblspace1_new') + tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node.safe_psql( + "postgres", + "drop table t_heap_lame") + + node.safe_psql( + "postgres", + "drop tablespace some_lame_tablespace") + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"]) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_tablespace_handling_1(self): + """ + make node with tablespace A, take full backup, check that restore with + tablespace mapping of tablespace B will end with error + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') + + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + self.create_tblspace_in_node( + node, 'tblspace1', + tblspc_path=tblspace1_old_path) + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --tablespace-mapping option' in e.message and + 'have an entry in tablespace_map file' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_tablespace_handling_2(self): + """ + make node without tablespaces, take full backup, check that restore with + tablespace mapping will end with error + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_drop_rel_during_full_backup(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 512): + node.safe_psql( + "postgres", + "create table t_heap_{0} as select i" + " as id from generate_series(0,100) i".format(i)) + + node.safe_psql( + "postgres", + "VACUUM") + + node.pgbench_init(scale=10) + + relative_path_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() + + relative_path_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() + + absolute_path_1 = os.path.join(node.data_dir, relative_path_1) + absolute_path_2 = os.path.join(node.data_dir, relative_path_2) + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=LOG', '--log-level-console=LOG', '--progress'], + gdb=True) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + for i in range(1, 512): + node.safe_psql( + "postgres", + "drop table t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "CHECKPOINT") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + # log_content = f.read() + # self.assertTrue( + # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + @unittest.skip("skip") + def test_drop_db_during_full_backup(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 2): + node.safe_psql( + "postgres", + "create database t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "VACUUM") + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=[ + '--stream', '--log-level-file=LOG', + '--log-level-console=LOG', '--progress']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + for i in range(1, 2): + node.safe_psql( + "postgres", + "drop database t_heap_{0}".format(i)) + + node.safe_psql( + "postgres", + "CHECKPOINT") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + # log_content = f.read() + # self.assertTrue( + # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, + # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_drop_rel_during_backup_delta(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DELTA backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + gdb=True, options=['--log-level-file=LOG']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + node.safe_psql( + "postgres", + "DROP TABLE t_heap") + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_drop_rel_during_backup_page(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "insert into t_heap select i" + " as id from generate_series(101,102) i") + + # PAGE backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', + gdb=True, options=['--log-level-file=LOG']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + gdb.kill() + + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + self.assertNotIn(relative_path, filelist) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_persistent_slot_for_stream_backup(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '40MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "SELECT pg_create_physical_replication_slot('slot_1')") + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1']) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1']) + + # @unittest.skip("skip") + def test_basic_temp_slot_for_stream_backup(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'max_wal_size': '40MB'}) + + if self.get_version(node) < self.version_to_num('10.0'): + self.skipTest('You need PostgreSQL >= 10 for this test') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--temp-slot']) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--slot=slot_1', '--temp-slot']) + + # @unittest.skip("skip") + def test_backup_concurrent_drop_table(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress'], + gdb=True) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + node.safe_psql( + 'postgres', + 'DROP TABLE pgbench_accounts') + + # do checkpoint to guarantee filenode removal + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + gdb.kill() + + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + + # @unittest.skip("skip") + def test_pg_11_adjusted_wal_segment_size(self): + """""" + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--wal-segsize=64'], + pg_options={ + 'min_wal_size': '128MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # FULL STREAM backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # PAGE STREAM backup + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # DELTA STREAM backup + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # FULL ARCHIVE backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # PAGE ARCHIVE backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '5', '-c', '2']) + pgbench.wait() + + # DELTA ARCHIVE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + pgdata = self.pgdata_content(node.data_dir) + + # delete + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--expired', + '--delete-wal', + '--retention-redundancy=1']) + + # validate + self.validate_pb(backup_dir) + + # merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # restore + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_sigint_handling(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=LOG']) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + gdb.kill() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # @unittest.skip("skip") + def test_sigterm_handling(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=LOG']) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGTERM') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # @unittest.skip("skip") + def test_sigquit_handling(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, options=['--stream']) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGQUIT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # @unittest.skip("skip") + def test_drop_table(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + connect_1 = node.connect("postgres") + connect_1.execute( + "create table t_heap as select i" + " as id from generate_series(0,100) i") + connect_1.commit() + + connect_2 = node.connect("postgres") + connect_2.execute("SELECT * FROM t_heap") + connect_2.commit() + + # DROP table + connect_2.execute("DROP TABLE t_heap") + connect_2.commit() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # @unittest.skip("skip") + def test_basic_missing_file_permissions(self): + """""" + if os.name == 'nt': + self.skipTest('Skipped because it is POSIX only test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pg_class')").decode('utf-8').rstrip() + + full_path = os.path.join(node.data_dir, relative_path) + + os.chmod(full_path, 000) + + try: + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Cannot open file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + os.chmod(full_path, 700) + + # @unittest.skip("skip") + def test_basic_missing_dir_permissions(self): + """""" + if os.name == 'nt': + self.skipTest('Skipped because it is POSIX only test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + full_path = os.path.join(node.data_dir, 'pg_twophase') + + os.chmod(full_path, 000) + + try: + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Cannot open directory', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + os.rmdir(full_path) + + # @unittest.skip("skip") + def test_backup_with_least_privileges_role(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + if self.ptrack: + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if self.ptrack: + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', '-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['-U', 'backup']) + + # PAGE + self.backup_node( + backup_dir, 'node', node, backup_type='page', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='page', datname='backupdb', + options=['--stream', '-U', 'backup']) + + # DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # PTRACK + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # @unittest.skip("skip") + def test_parent_choosing(self): + """ + PAGE3 <- RUNNING(parent should be FULL) + PAGE2 <- OK + PAGE1 <- CORRUPT + FULL + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGE1 to ERROR + self.change_backup_status(backup_dir, 'node', page1_id, 'ERROR') + + # PAGE3 + page3_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_file_content = f.read() + + self.assertIn( + "WARNING: Backup {0} has invalid parent: {1}. " + "Cannot be a parent".format(page2_id, page1_id), + log_file_content) + + self.assertIn( + "WARNING: Backup {0} has status: ERROR. " + "Cannot be a parent".format(page1_id), + log_file_content) + + self.assertIn( + "Parent backup: {0}".format(full_id), + log_file_content) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + full_id) + + # @unittest.skip("skip") + def test_parent_choosing_1(self): + """ + PAGE3 <- RUNNING(parent should be FULL) + PAGE2 <- OK + PAGE1 <- (missing) + FULL + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Delete PAGE1 + shutil.rmtree( + os.path.join(backup_dir, 'backups', 'node', page1_id)) + + # PAGE3 + page3_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_file_content = f.read() + + self.assertIn( + "WARNING: Backup {0} has missing parent: {1}. " + "Cannot be a parent".format(page2_id, page1_id), + log_file_content) + + self.assertIn( + "Parent backup: {0}".format(full_id), + log_file_content) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + full_id) + + # @unittest.skip("skip") + def test_parent_choosing_2(self): + """ + PAGE3 <- RUNNING(backup should fail) + PAGE2 <- OK + PAGE1 <- OK + FULL <- (missing) + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + page1_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + page2_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Delete FULL + shutil.rmtree( + os.path.join(backup_dir, 'backups', 'node', full_id)) + + # PAGE3 + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--log-level-file=LOG']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Valid full backup on current timeline 1 is not found' in e.message and + 'ERROR: Create new full backup before an incremental one' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb( + backup_dir, 'node')[2]['status'], + 'ERROR') + + # @unittest.skip("skip") + def test_backup_with_less_privileges_role(self): + """ + check permissions correctness from documentation: + https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'archive_mode': 'always', + 'checkpoint_timeout': '60s', + 'wal_level': 'logical'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + if self.ptrack: + node.safe_psql( + 'backupdb', + 'CREATE EXTENSION ptrack') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) + + # enable STREAM backup + node.safe_psql( + 'backupdb', + 'ALTER ROLE backup WITH REPLICATION;') + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', '-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['-U', 'backup']) + + # PAGE + self.backup_node( + backup_dir, 'node', node, backup_type='page', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='page', datname='backupdb', + options=['--stream', '-U', 'backup']) + + # DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # PTRACK + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) + + if self.get_version(node) < 90600: + return + + # Restore as replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'node', replica) + self.set_replica(node, replica) + self.add_instance(backup_dir, 'replica', replica) + self.set_config( + backup_dir, 'replica', + options=['--archive-timeout=120s', '--log-level-console=LOG']) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_auto_conf(replica, {'hot_standby': 'on'}) + + # freeze bgwriter to get rid of RUNNING XACTS records + # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] + # gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + copy_tree( + os.path.join(backup_dir, 'wal', 'node'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + # self.switch_wal_segment(node) + # self.switch_wal_segment(node) + + self.backup_node( + backup_dir, 'replica', replica, + datname='backupdb', options=['-U', 'backup']) + + # stream full backup from replica + self.backup_node( + backup_dir, 'replica', replica, + datname='backupdb', options=['--stream', '-U', 'backup']) + +# self.switch_wal_segment(node) + + # PAGE backup from replica + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + + self.backup_node( + backup_dir, 'replica', replica, backup_type='page', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # DELTA backup from replica + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'replica', replica, backup_type='delta', + datname='backupdb', options=['--stream', '-U', 'backup']) + + # PTRACK backup from replica + if self.ptrack: + self.switch_wal_segment(node) + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + datname='backupdb', options=['-U', 'backup']) + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + datname='backupdb', options=['--stream', '-U', 'backup']) + + @unittest.skip("skip") + def test_issue_132(self): + """ + https://github.com/postgrespro/pg_probackup/issues/132 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + for i in range(50000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + exit(1) + + @unittest.skip("skip") + def test_issue_132_1(self): + """ + https://github.com/postgrespro/pg_probackup/issues/132 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + for i in range(30000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() + + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream'], old_binary=True) + + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream'], old_binary=True) + + node.cleanup() + + # make sure that new binary can detect corruption + try: + self.validate_pb(backup_dir, 'node', backup_id=full_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir, 'node', backup_id=delta_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "ORPHAN"') + + # check that revalidation is working correctly + try: + self.restore_node( + backup_dir, 'node', node, backup_id=delta_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because FULL backup is CORRUPT" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "ORPHAN"') + + # check that '--no-validate' do not allow to restore ORPHAN backup +# try: +# self.restore_node( +# backup_dir, 'node', node, backup_id=delta_id, +# options=['--no-validate']) +# # we should die here because exception is what we expect to happen +# self.assertEqual( +# 1, 0, +# "Expecting Error because FULL backup is CORRUPT" +# "\n Output: {0} \n CMD: {1}".format( +# repr(self.output), self.cmd)) +# except ProbackupException as e: +# self.assertIn( +# 'Insert data', +# e.message, +# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( +# repr(e.message), self.cmd)) + + node.cleanup() + + output = self.restore_node( + backup_dir, 'node', node, backup_id=full_id, options=['--force']) + + self.assertIn( + 'WARNING: Backup {0} has status: CORRUPT'.format(full_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is corrupt.'.format(full_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), + output) + + self.assertIn( + 'INFO: Restore of backup {0} completed.'.format(full_id), + output) + + node.cleanup() + + output = self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, options=['--force']) + + self.assertIn( + 'WARNING: Backup {0} is orphan.'.format(delta_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), + output) + + self.assertIn( + 'WARNING: Backup {0} is not valid, restore is forced'.format(delta_id), + output) + + self.assertIn( + 'INFO: Restore of backup {0} completed.'.format(delta_id), + output) + + def test_note_sanity(self): + """ + test that adding note to backup works as expected + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=LOG', '--note=test_note']) + + show_backups = self.show_pb(backup_dir, 'node') + + print(self.show_pb(backup_dir, as_text=True, as_json=True)) + + self.assertEqual(show_backups[0]['note'], "test_note") + + self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + + self.assertNotIn( + 'note', + backup_meta) + + # @unittest.skip("skip") + def test_parent_backup_made_by_newer_version(self): + """incremental backup with parent made by newer version""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + control_file = os.path.join( + backup_dir, "backups", "node", backup_id, + "backup.control") + + version = self.probackup_version + fake_new_version = str(int(version.split('.')[0]) + 1) + '.0.0' + + with open(control_file, 'r') as f: + data = f.read(); + + data = data.replace(version, fake_new_version) + + with open(control_file, 'w') as f: + f.write(data); + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "if parent made by newer version.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + + # @unittest.skip("skip") + def test_issue_289(self): + """ + https://github.com/postgrespro/pg_probackup/issues/289 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "ERROR: Create new full backup before an incremental one", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + + # @unittest.skip("skip") + def test_issue_290(self): + """ + https://github.com/postgrespro/pg_probackup/issues/290 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + os.rmdir( + os.path.join(backup_dir, "wal", "node")) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WAL archive directory is not accessible", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + + @unittest.skip("skip") + def test_issue_203(self): + """ + https://github.com/postgrespro/pg_probackup/issues/203 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + for i in range(1000000): + conn.execute( + "CREATE TABLE t_{0} as select 1".format(i)) + conn.commit() + + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j2']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', + node_restored, data_dir=node_restored.data_dir) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_issue_231(self): + """ + https://github.com/postgrespro/pg_probackup/issues/231 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + datadir = os.path.join(node.data_dir, '123') + + try: + self.backup_node( + backup_dir, 'node', node, data_dir='{0}'.format(datadir)) + except: + pass + + out = self.backup_node(backup_dir, 'node', node, options=['--stream'], return_id=False) + + # it is a bit racy + self.assertIn("WARNING: Cannot create directory", out) + + def test_incr_backup_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # incremental restore into node1 + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # @unittest.skip("skip") + def test_missing_wal_segment(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # get segments in pg_wal, sort then and remove all but the latest + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + + if node.major_version >= 10: + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + else: + pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog') + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', '--log-level-file=INFO'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('start_WAL_streaming') + gdb.run_until_break() + + # generate some more data + node.pgbench_init(scale=3) + + # remove redundant WAL segments in pg_wal + files = os.listdir(pg_wal_dir) + files.sort(reverse=True) + + # leave first two files in list + del files[:2] + for filename in files: + os.remove(os.path.join(pg_wal_dir, filename)) + + gdb.continue_execution_until_exit() + + self.assertIn( + 'unexpected termination of replication stream: ERROR: requested WAL segment', + gdb.output) + + self.assertIn( + 'has already been removed', + gdb.output) + + self.assertIn( + 'ERROR: Interrupted during waiting for WAL streaming', + gdb.output) + + self.assertIn( + 'WARNING: backup in progress, stop backup', + gdb.output) + + # TODO: check the same for PAGE backup + + # @unittest.skip("skip") + def test_missing_replication_permission(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) +# self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + sleep(2) + replica.promote() + + # Delta backup + try: + self.backup_node( + backup_dir, 'node', replica, backup_type='delta', + data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + # 9.5: ERROR: must be superuser or replication role to run a backup + # >=9.6: FATAL: must be superuser or replication role to start walsender + self.assertRegex( + e.message, + "ERROR: must be superuser or replication role to run a backup|FATAL: must be superuser or replication role to start walsender", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_missing_replication_permission_1(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # > 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + replica.promote() + + # PAGE + output = self.backup_node( + backup_dir, 'node', replica, backup_type='page', + data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'], + return_id=False) + + self.assertIn( + 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', + output) + + # Messages before 14 + # 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender' + # Messages for >=14 + # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' + # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb: (connection to server (on socket "/tmp/.s.PGSQL.\d+"|at "localhost" \(127.0.0.1\), port \d+) failed: ){0,1}' + 'FATAL: must be superuser or replication role to start walsender') + + # @unittest.skip("skip") + def test_basic_backup_default_transaction_read_only(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'default_transaction_read_only': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + node.safe_psql( + 'postgres', + 'create temp table t1()') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertIn( + "cannot execute CREATE TABLE in a read-only transaction", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream']) + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # @unittest.skip("skip") + def test_backup_atexit(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=VERBOSE'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + sleep(1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node')[0]['status'], 'ERROR') + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + #print(log_content) + self.assertIn( + 'WARNING: backup in progress, stop backup', + log_content) + + if self.get_version(node) < 150000: + self.assertIn( + 'FROM pg_catalog.pg_stop_backup', + log_content) + else: + self.assertIn( + 'FROM pg_catalog.pg_backup_stop', + log_content) + + self.assertIn( + 'setting its status to ERROR', + log_content) + + # @unittest.skip("skip") + def test_pg_stop_backup_missing_permissions(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + self.simple_bootstrap(node, 'backup') + + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') + elif self.get_version(node) < 150000: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') + else: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') + + + # Full backup in streaming mode + try: + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + if self.get_version(node) < 150000: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_stop_backup " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_backup_stop " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: permission denied for function pg_stop_backup", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: permission denied for function pg_backup_stop", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "query was: SELECT pg_catalog.txid_snapshot_xmax", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_start_time(self): + """Test, that option --start-time allows to set backup_id and restore""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore FULL backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_full'), + backup_id=base36enc(startTime)) + + #FULL backup with incorrect start time + try: + startTime = str(int(time()-100000)) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + 'Expecting Error because start time for new backup must be newer ' + '\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore DELTA backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_delta'), + backup_id=base36enc(startTime)) + + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PAGE backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_page'), + backup_id=base36enc(startTime)) + + # PTRACK backup + if self.ptrack: + node.safe_psql( + 'postgres', + 'create extension ptrack') + + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PTRACK backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_ptrack'), + backup_id=base36enc(startTime)) + + # @unittest.skip("skip") + def test_start_time_few_nodes(self): + """Test, that we can synchronize backup_id's for different DBs""" + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir1 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup1') + self.init_pb(backup_dir1) + self.add_instance(backup_dir1, 'node1', node1) + self.set_archiving(backup_dir1, 'node1', node1) + node1.slow_start() + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir2 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup2') + self.init_pb(backup_dir2) + self.add_instance(backup_dir2, 'node2', node2) + self.set_archiving(backup_dir2, 'node2', node2) + node2.slow_start() + + # FULL backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[0] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # DELTA backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[1] + show_backup2 = self.show_pb(backup_dir2, 'node2')[1] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PAGE backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[2] + show_backup2 = self.show_pb(backup_dir2, 'node2')[2] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PTRACK backup + if self.ptrack: + node1.safe_psql( + 'postgres', + 'create extension ptrack') + node2.safe_psql( + 'postgres', + 'create extension ptrack') + + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[3] + show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + self.assertEqual(show_backup1['id'], show_backup2['id']) + diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py new file mode 100644 index 000000000..28ef275df --- /dev/null +++ b/tests/cfs_backup_test.py @@ -0,0 +1,1235 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +tblspace_name = 'cfs_tblspace' + + +class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): + # --- Begin --- # + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.backup_dir = os.path.join( + self.tmp_path, self.module_name, self.fname, 'backup') + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'cfs_encryption': 'off', + 'max_wal_senders': '2', + 'shared_buffers': '200MB' + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.slow_start() + + self.node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( + tblspace_name)) + + self.assertIn( + tblspace_name, str(tblspace), + "ERROR: The tablespace not created " + "or it create without compressions") + + self.assertIn( + "compression=true", str(tblspace), + "ERROR: The tablespace not created " + "or it create without compressions") + + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + + # --- Section: Full --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace(self): + """Case: Check fullbackup empty compressed tablespace""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_stream(self): + """Case: Check fullbackup empty compressed tablespace with options stream""" + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table(self): + """Case: Make full backup after created table in the tablespace""" + if not self.enterprise: + return + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "\n ERROR: {0}\n CMD: {1}".format( + repr(e.message), + repr(self.cmd) + ) + ) + return False + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in {0}".format( + os.path.join(self.backup_dir, 'node', backup_id)) + ) + + # check cfm size + cfms = find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + # PGPRO-1018 invalid file size + def test_fullbackup_after_create_table_stream(self): + """ + Case: Make full backup after created table in the tablespace with option --stream + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Full backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # --- Section: Incremental from empty tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_page_doesnt_store_unchanged_cfm(self): + """ + Case: Test page backup doesn't store cfm file if table were not modified + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files is found in backup dir" + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace. + Make page backup after create table + """ + + try: + self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + backup_id = None + try: + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['_ptrack']), + "ERROR: _ptrack files was found in backup dir" + ) + + # --- Section: Incremental from fill tablespace --- # + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='ptrack') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,25) i".format('t2', tblspace_name) + ) + + backup_id_ptrack = None + try: + backup_id_ptrack = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='ptrack', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_ptrack = self.show_pb( + self.backup_dir, 'node', backup_id_ptrack) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_ptrack["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_ptrack["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup size should not be greater than full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap', tblspace_name) + ) + + full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap') + ) + + page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + self.restore_node( + self.backup_dir, 'node', self.node, backup_id=backup_id_full, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + + self.node.slow_start() + self.assertEqual( + full_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + self.node.cleanup() + shutil.rmtree( + self.get_tblspace_path(self.node, tblspace_name), + ignore_errors=True) + self.restore_node( + self.backup_dir, 'node', self.node, backup_id=backup_id_page, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + + self.node.slow_start() + self.assertEqual( + page_result, + self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_multiple_segments_in_multiple_tablespaces(self): + """ + Case: Make full backup before created table in the tablespace. + Make ptrack backup after create table. + Check: incremental backup will not greater as full + """ + tblspace_name_1 = 'tblspace_name_1' + tblspace_name_2 = 'tblspace_name_2' + + self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) + self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_1', tblspace_name_1)) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format( + 't_heap_2', tblspace_name_2)) + + full_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + full_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_1') + ) + + self.node.safe_psql( + "postgres", + "INSERT INTO {0} " + "SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format( + 't_heap_2') + ) + + page_result_1 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_1") + page_result_2 = self.node.safe_psql( + "postgres", "SELECT * FROM t_heap_2") + + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # CHECK FULL BACKUP + self.node.stop() + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_full, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() + + self.assertEqual( + full_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + full_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # CHECK PAGE BACKUP + self.node.stop() + + self.restore_node( + self.backup_dir, 'node', self.node, + backup_id=backup_id_page, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() + + self.assertEqual( + page_result_1, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + 'Lost data after restore') + self.assertEqual( + page_result_2, + self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + 'Lost data after restore') + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_fullbackup_after_create_table_page_after_create_table_stream(self): + """ + Case: Make full backup before created table in the tablespace(--stream). + Make ptrack backup after create table(--stream). + Check: incremental backup will not greater as full + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,1005000) i".format('t1', tblspace_name) + ) + + backup_id_full = None + try: + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='full', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,10) i".format('t2', tblspace_name) + ) + + backup_id_page = None + try: + backup_id_page = self.backup_node( + self.backup_dir, 'node', self.node, + backup_type='page', options=['--stream']) + except ProbackupException as e: + self.fail( + "ERROR: Incremental backup failed.\n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + show_backup_full = self.show_pb( + self.backup_dir, 'node', backup_id_full) + show_backup_page = self.show_pb( + self.backup_dir, 'node', backup_id_page) + self.assertGreater( + show_backup_full["data-bytes"], + show_backup_page["data-bytes"], + "ERROR: Size of incremental backup greater than full. \n " + "INFO: {0} >{1}".format( + show_backup_page["data-bytes"], + show_backup_full["data-bytes"] + ) + ) + + # --- Make backup with not valid data(broken .cfm) --- # + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_cfm_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + os.remove(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_file_pg_compression_from_tablespace_dir(self): + os.remove( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0]) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_delete_random_data_file_from_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + os.remove(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_cfm_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_cmf = find_by_extensions( + [self.get_tblspace_path(self.node, tblspace_name)], + ['.cfm']) + self.assertTrue( + list_cmf, + "ERROR: .cfm-files not found into tablespace dir" + ) + + corrupt_file(random.choice(list_cmf)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_random_data_file_into_tablespace_dir(self): + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + list_data_files = find_by_pattern( + [self.get_tblspace_path(self.node, tblspace_name)], + '^.*/\d+$') + self.assertTrue( + list_data_files, + "ERROR: Files of data not found into tablespace dir" + ) + + corrupt_file(random.choice(list_data_files)) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + + @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_broken_file_pg_compression_into_tablespace_dir(self): + + corrupted_file = find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression'])[0] + + self.assertTrue( + corrupt_file(corrupted_file), + "ERROR: File is not corrupted or it missing" + ) + + self.assertRaises( + ProbackupException, + self.backup_node, + self.backup_dir, + 'node', + self.node, + backup_type='full' + ) + +# # --- End ---# + + +#class CfsBackupEncTest(CfsBackupNoEncTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py new file mode 100644 index 000000000..43c3f18f1 --- /dev/null +++ b/tests/cfs_catchup_test.py @@ -0,0 +1,117 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): + + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_full_catchup_with_tablespace(self): + """ + Test tablespace transfers + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') + self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # do full catchup with tablespace mapping + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # check cfm size + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + + # make changes in master tablespace + src_pg.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # and now delta backup + dst_pg.stop() + + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # check cfm size again + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + + # 3rd check: run verification query + src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py new file mode 100644 index 000000000..6b69b4ffe --- /dev/null +++ b/tests/cfs_restore_test.py @@ -0,0 +1,450 @@ +""" +restore + Syntax: + + pg_probackup restore -B backupdir --instance instance_name + [-D datadir] + [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] + [-j num_threads] [--progress] [-q] [-v] + +""" +import os +import unittest +import shutil + +from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +tblspace_name = 'cfs_tblspace' +tblspace_name_new = 'cfs_tblspace_new' + + +class CfsRestoreBase(ProbackupTest, unittest.TestCase): + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def setUp(self): + self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ +# 'ptrack_enable': 'on', + 'cfs_encryption': 'off', + } + ) + + self.init_pb(self.backup_dir) + self.add_instance(self.backup_dir, 'node', self.node) + self.set_archiving(self.backup_dir, 'node', self.node) + + self.node.slow_start() + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) + + self.add_data_in_cluster() + + self.backup_id = None + try: + self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') + except ProbackupException as e: + self.fail( + "ERROR: Full backup failed \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + def add_data_in_cluster(self): + pass + + +class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_empty_tablespace_from_fullbackup(self): + """ + Case: Restore empty tablespace from valid full backup. + """ + self.node.stop(["-m", "immediate"]) + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), + "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." + ) + + try: + self.node.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + tblspace = self.node.safe_psql( + "postgres", + "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) + ).decode("UTF-8") + self.assertTrue( + tblspace_name in tblspace and "compression=true" in tblspace, + "ERROR: The tablespace not restored or it restored without compressions" + ) + + +class CfsRestoreNoencTest(CfsRestoreBase): + def add_data_in_cluster(self): + self.node.safe_psql( + "postgres", + 'CREATE TABLE {0} TABLESPACE {1} \ + AS SELECT i AS id, MD5(i::text) AS text, \ + MD5(repeat(i::text,10))::tsvector AS tsvector \ + FROM generate_series(0,1e5) i'.format('t1', tblspace_name) + ) + self.table_t1 = self.node.safe_psql( + "postgres", + "SELECT * FROM t1" + ) + + # --- Restore from full backup ---# + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_old_location(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in tablespace dir" + ) + try: + self.node.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_old_location_3_jobs(self): + """ + Case: Restore instance from valid full backup to old location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + try: + self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + self.node.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_new_location(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) + node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id) + self.set_auto_conf(node_new, {'port': node_new.port}) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node_new.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_new_location_5_jobs(self): + """ + Case: Restore instance from valid full backup to new location. + """ + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) + node_new.cleanup() + + try: + self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) + self.set_auto_conf(node_new, {'port': node_new.port}) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), + "ERROR: File pg_compression not found in backup dir" + ) + try: + node_new.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + node_new.cleanup() + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): + self.node.stop() + self.node.cleanup() + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) + + os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) + + try: + self.restore_node( + self.backup_dir, + 'node', self.node, + backup_id=self.backup_id, + options=["-j", "3", "-T", "{0}={1}".format( + self.get_tblspace_path(self.node, tblspace_name), + self.get_tblspace_path(self.node, tblspace_name_new) + ) + ] + ) + except ProbackupException as e: + self.fail( + "ERROR: Restore from full backup failed. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + self.assertTrue( + find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), + "ERROR: File pg_compression not found in new tablespace location" + ) + try: + self.node.slow_start() + except ProbackupException as e: + self.fail( + "ERROR: Instance not started after restore. \n {0} \n {1}".format( + repr(self.cmd), + repr(e.message) + ) + ) + + self.assertEqual( + repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), + repr(self.table_t1) + ) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_ptrack_new_jobs(self): + pass + +# --------------------------------------------------------- # + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page(self): + """ + Case: Restore from backup to old location + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_jobs(self): + """ + Case: Restore from backup to old location, four jobs + """ + pass + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_restore_from_page_new_jobs(self): + """ + Case: Restore from backup to new location, four jobs + """ + pass + + +#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() +# +# +#class CfsRestoreEncTest(CfsRestoreNoencTest): +# # --- Begin --- # +# def setUp(self): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup_test.py b/tests/cfs_validate_backup_test.py new file mode 100644 index 000000000..343020dfc --- /dev/null +++ b/tests/cfs_validate_backup_test.py @@ -0,0 +1,24 @@ +import os +import unittest +import random + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +tblspace_name = 'cfs_tblspace' + + +class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): + def setUp(self): + pass + + def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): + pass + + def tearDown(self): + pass + + +#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): +# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" +# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py new file mode 100644 index 000000000..2caf4fcb2 --- /dev/null +++ b/tests/checkdb_test.py @@ -0,0 +1,849 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException +import shutil +import sys +import time + + +class CheckdbTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_checkdb_amcheck_only_sanity(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + node.safe_psql( + "postgres", + "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + node.safe_psql( + "postgres", + "create index on idxpart(a)") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + # simple sanity + try: + self.checkdb_node( + options=['--skip-block-validation']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because --amcheck option is missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Option '--skip-block-validation' must be " + "used with '--amcheck' option", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # simple sanity + output = self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + output) + self.assertIn( + 'All checked indexes are valid', + output) + + # logging to file sanity + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because log_directory missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Cannot save checkdb logs to a file. " + "You must specify --log-directory option when " + "running checkdb with --log-level-file option enabled", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # If backup_dir provided, then instance name must be + # provided too + try: + self.checkdb_node( + backup_dir, + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because log_directory missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: required parameter not specified: --instance", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # checkdb can use default or set in config values, + # if backup_dir and instance name are provided + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '-d', 'postgres', '-p', str(node.port)]) + + # check that file present and full of messages + os.path.isfile(log_file_path) + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + os.unlink(log_file_path) + + # log-level-file and log-directory are provided + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) + + # check that file present and full of messages + os.path.isfile(log_file_path) + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + os.unlink(log_file_path) + + gdb = self.checkdb_node( + gdb=True, + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) + + gdb.set_breakpoint('amcheck_one_index') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "drop table t_heap") + + gdb.remove_all_breakpoints() + + gdb.continue_execution_until_exit() + + # check that message about missing index is present + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + 'ERROR: checkdb --amcheck finished with failure', + log_file_content) + self.assertIn( + "WARNING: Thread [1]. Amcheck failed in database 'postgres' " + "for index: 'public.t_heap_id_idx':", + log_file_content) + self.assertIn( + 'ERROR: could not open relation with OID', + log_file_content) + + # Clean after yourself + gdb.kill() + node.stop() + + # @unittest.skip("skip") + def test_basic_checkdb_amcheck_only_sanity(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # create two databases + node.safe_psql("postgres", "create database db1") + try: + node.safe_psql( + "db1", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "db1", + "create extension amcheck_next") + + node.safe_psql("postgres", "create database db2") + try: + node.safe_psql( + "db2", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "db2", + "create extension amcheck_next") + + # init pgbench in two databases and corrupt both indexes + node.pgbench_init(scale=5, dbname='db1') + node.pgbench_init(scale=5, dbname='db2') + + node.safe_psql( + "db2", + "alter index pgbench_accounts_pkey rename to some_index") + + index_path_1 = os.path.join( + node.data_dir, + node.safe_psql( + "db1", + "select pg_relation_filepath('pgbench_accounts_pkey')").decode('utf-8').rstrip()) + + index_path_2 = os.path.join( + node.data_dir, + node.safe_psql( + "db2", + "select pg_relation_filepath('some_index')").decode('utf-8').rstrip()) + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because some db was not amchecked" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Some databases were not amchecked", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + node.stop() + + # Let`s do index corruption + with open(index_path_1, "rb+", 0) as f: + f.seek(42000) + f.write(b"blablahblahs") + f.flush() + f.close + + with open(index_path_2, "rb+", 0) as f: + f.seek(42000) + f.write(b"blablahblahs") + f.flush() + f.close + + node.slow_start() + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--log-level-file=verbose', + '--log-directory={0}'.format( + os.path.join(backup_dir, 'log')), + '-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because some db was not amchecked" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: checkdb --amcheck finished with failure", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # corruption of both indexes in db1 and db2 must be detected + # also the that amcheck is not installed in 'postgres' + # should be logged + with open(log_file_path) as f: + log_file_content = f.read() + self.assertIn( + "WARNING: Thread [1]. Amcheck failed in database 'db1' " + "for index: 'public.pgbench_accounts_pkey':", + log_file_content) + + self.assertIn( + "WARNING: Thread [1]. Amcheck failed in database 'db2' " + "for index: 'public.some_index':", + log_file_content) + + self.assertIn( + "ERROR: checkdb --amcheck finished with failure", + log_file_content) + + # Clean after yourself + node.stop() + + # @unittest.skip("skip") + def test_checkdb_block_validation_sanity(self): + """make node, corrupt some pages, check that checkdb failed""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + # sanity + try: + self.checkdb_node() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pgdata must be specified\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: required parameter not specified: PGDATA (-D, --pgdata)", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.checkdb_node( + data_dir=node.data_dir, + options=['-d', 'postgres', '-p', str(node.port)]) + + self.checkdb_node( + backup_dir, 'node', + options=['-d', 'postgres', '-p', str(node.port)]) + + heap_full_path = os.path.join(node.data_dir, heap_path) + + with open(heap_full_path, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + with open(heap_full_path, "rb+", 0) as f: + f.seek(42000) + f.write(b"bla") + f.flush() + f.close + + try: + self.checkdb_node( + backup_dir, 'node', + options=['-d', 'postgres', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of data corruption\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Checkdb failed", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Corruption detected in file "{0}", block 1'.format( + os.path.normpath(heap_full_path)), + e.message) + + self.assertIn( + 'WARNING: Corruption detected in file "{0}", block 5'.format( + os.path.normpath(heap_full_path)), + e.message) + + # Clean after yourself + node.stop() + + def test_checkdb_checkunique(self): + """Test checkunique parameter of amcheck.bt_index_check function""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + node.slow_start() + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + # Part of https://commitfest.postgresql.org/32/2976/ patch test + node.safe_psql( + "postgres", + "CREATE TABLE bttest_unique(a varchar(50), b varchar(1500), c bytea, d varchar(50)); " + "ALTER TABLE bttest_unique SET (autovacuum_enabled = false); " + "CREATE UNIQUE INDEX bttest_unique_idx ON bttest_unique(a,b); " + "UPDATE pg_catalog.pg_index SET indisunique = false " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "INSERT INTO bttest_unique " + " SELECT i::text::varchar, " + " array_to_string(array( " + " SELECT substr('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', ((random()*(36-1)+1)::integer), 1) " + " FROM generate_series(1,1300)),'')::varchar, " + " i::text::bytea, i::text::varchar " + " FROM generate_series(0,1) AS i, generate_series(0,30) AS x; " + "UPDATE pg_catalog.pg_index SET indisunique = true " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "DELETE FROM bttest_unique WHERE ctid::text='(0,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,3)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(9,3)';") + + # run without checkunique option (error will not detected) + output = self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + output) + self.assertIn( + 'All checked indexes are valid', + output) + + # run with checkunique option + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--checkunique', + '-d', 'postgres', '-p', str(node.port)]) + if (ProbackupTest.enterprise and + (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400)): + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of index corruption\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertRegex( + self.output, + r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") + except ProbackupException as e: + self.assertIn( + "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", + e.message) + + # Clean after yourself + node.stop() + + # @unittest.skip("skip") + def test_checkdb_sigint_handling(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + # FULL backup + gdb = self.checkdb_node( + backup_dir, 'node', gdb=True, + options=[ + '-d', 'postgres', '-j', '2', + '--skip-block-validation', + '--progress', + '--amcheck', '-p', str(node.port)]) + + gdb.set_breakpoint('amcheck_one_index') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + gdb.remove_all_breakpoints() + + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + with open(node.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('could not receive data from client', output) + self.assertNotIn('could not send data to client', output) + self.assertNotIn('connection to client lost', output) + + # Clean after yourself + gdb.kill() + node.stop() + + # @unittest.skip("skip") + def test_checkdb_with_least_privileges(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + try: + node.safe_psql( + "backupdb", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "backupdb", + "create extension amcheck_next") + + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC;") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function + + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' +# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # PG 10 + elif self.get_version(node) > 100000 and self.get_version(node) < 110000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') + + if ProbackupTest.enterprise: + # amcheck-1.1 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') + else: + # amcheck-1.0 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') + # >= 11 < 14 + elif self.get_version(node) > 110000 and self.get_version(node) < 140000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # checkunique parameter + if ProbackupTest.enterprise: + if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400): + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + # >= 14 + else: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # checkunique parameter + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + + if ProbackupTest.enterprise: + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + + # checkdb + try: + self.checkdb_node( + backup_dir, 'node', + options=[ + '--amcheck', '-U', 'backup', + '-d', 'backupdb', '-p', str(node.port)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because permissions are missing\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "INFO: Amcheck succeeded for database 'backupdb'", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Extension 'amcheck' or 'amcheck_next' are " + "not installed in database postgres", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "ERROR: Some databases were not amchecked", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # Clean after yourself + node.stop() diff --git a/tests/compatibility_test.py b/tests/compatibility_test.py new file mode 100644 index 000000000..591afb069 --- /dev/null +++ b/tests/compatibility_test.py @@ -0,0 +1,1500 @@ +import unittest +import subprocess +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from sys import exit +import shutil + + +def check_manual_tests_enabled(): + return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' + + +def check_ssh_agent_path_exists(): + return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ + + +class CompatibilityTest(ProbackupTest, unittest.TestCase): + + def setUp(self): + self.fname = self.id().split('.')[3] + + # @unittest.expectedFailure + @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') + @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') + # @unittest.skip("skip") + def test_catchup_with_different_remote_major_pg(self): + """ + Decription in jira issue PBCKP-236 + This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6 + + Prerequisites: + - pg_probackup git tag for PBCKP 2.5.1 + - master pg_probackup build should be made for PGPROEE11 + - agent pg_probackup build should be made for PGPROEE9_6 + + Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes + the PBCKP-236 problem + + Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH= + for the test + + Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + without pg_probackup executable + """ + + self.verbose = True + self.remote = True + # please use your own local path like + # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' + pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] + + src_pg = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'src'), + set_replication=True, + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # do full catchup + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + dst_options = {'port': str(dst_pg.port)} + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") + + # do delta catchup with remote pg_probackup agent with another postgres major version + # this DELTA backup should fail without PBCKP-236 patch. + self.catchup_node( + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] + ) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_page(self): + """Description in jira issue PGPRO-434""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with old binary + self.backup_node( + backup_dir, 'node', node, old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.show_pb(backup_dir) + + self.validate_pb(backup_dir) + + # RESTORE old FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Page BACKUP with old binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Page BACKUP with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'create table tmp as select * from pgbench_accounts where aid < 1000') + + node.safe_psql( + 'postgres', + 'delete from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM') + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'insert into pgbench_accounts select * from pgbench_accounts') + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_delta(self): + """Description in jira issue PGPRO-434""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with old binary + self.backup_node( + backup_dir, 'node', node, old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.show_pb(backup_dir) + + self.validate_pb(backup_dir) + + # RESTORE old FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta BACKUP with old binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta BACKUP with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'create table tmp as select * from pgbench_accounts where aid < 1000') + + node.safe_psql( + 'postgres', + 'delete from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM') + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.safe_psql( + 'postgres', + 'insert into pgbench_accounts select * from pgbench_accounts') + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_ptrack(self): + """Description in jira issue PGPRO-434""" + + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=10) + + # FULL backup with old binary + self.backup_node( + backup_dir, 'node', node, old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.show_pb(backup_dir) + + self.validate_pb(backup_dir) + + # RESTORE old FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # ptrack BACKUP with old binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--recovery-target=latest", + "--recovery-target-action=promote"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Ptrack BACKUP with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"] + ) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--recovery-target=latest", + "--recovery-target-action=promote"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_compression(self): + """Description in jira issue PGPRO-434""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=10) + + # FULL backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # restore OLD FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with OLD binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + old_binary=True, + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # PAGE backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with old binary + self.delete_pb(backup_dir, 'node', backup_id) + + self.backup_node( + backup_dir, 'node', node, + old_binary=True, + options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress'], + old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Delta backup with new binary + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"]) + + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=['--compress']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, + old_binary=True) + + node.pgbench_init(scale=1) + + # PAGE backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.merge_backup(backup_dir, "node", backup_id) + + self.show_pb(backup_dir, as_text=True, as_json=False) + + # restore OLD FULL with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_1(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + # PAGE2 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + # merge chain created by old binary with new binary + output = self.merge_backup(backup_dir, "node", backup_id) + + # check that in-place is disabled + self.assertIn( + "WARNING: In-place merge is disabled " + "because of storage format incompatibility", output) + + # restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_2(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=50) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + page1 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE from pgbench_accounts where ctid > '(10,1)'") + + # PAGE2 backup with OLD binary + page2 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata2 = self.pgdata_content(node.data_dir) + + # PAGE3 backup with OLD binary + page3 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True) + + pgdata3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE4 backup with NEW binary + page4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + pgdata4 = self.pgdata_content(node.data_dir) + + # merge backups one by one and check data correctness + # merge PAGE1 + self.merge_backup( + backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + + # check data correctness for PAGE1 + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, backup_id=page1, + options=['--log-level-file=VERBOSE']) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + # merge PAGE2 + self.merge_backup(backup_dir, "node", page2) + + # check data correctness for PAGE2 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + # merge PAGE3 + self.show_pb(backup_dir, 'node', page3) + self.merge_backup(backup_dir, "node", page3) + self.show_pb(backup_dir, 'node', page3) + + # check data correctness for PAGE3 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # merge PAGE4 + self.merge_backup(backup_dir, "node", page4) + + # check data correctness for PAGE4 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata4, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_3(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version =< 2.2.7 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=50) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, old_binary=True, options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + page1 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE from pgbench_accounts where ctid > '(10,1)'") + + # PAGE2 backup with OLD binary + page2 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata2 = self.pgdata_content(node.data_dir) + + # PAGE3 backup with OLD binary + page3 = self.backup_node( + backup_dir, 'node', node, + backup_type='page', old_binary=True, options=['--compress']) + + pgdata3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE4 backup with NEW binary + page4 = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + pgdata4 = self.pgdata_content(node.data_dir) + + # merge backups one by one and check data correctness + # merge PAGE1 + self.merge_backup( + backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + + # check data correctness for PAGE1 + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, backup_id=page1, + options=['--log-level-file=VERBOSE']) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + # merge PAGE2 + self.merge_backup(backup_dir, "node", page2) + + # check data correctness for PAGE2 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + # merge PAGE3 + self.show_pb(backup_dir, 'node', page3) + self.merge_backup(backup_dir, "node", page3) + self.show_pb(backup_dir, 'node', page3) + + # check data correctness for PAGE3 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # merge PAGE4 + self.merge_backup(backup_dir, "node", page4) + + # check data correctness for PAGE4 + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata4, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_4(self): + """ + Start merge between minor version, crash and retry it. + old binary version =< 2.4.0 + """ + if self.version_to_num(self.old_probackup_version) > self.version_to_num('2.4.0'): + self.assertTrue( + False, 'You need pg_probackup old_binary =< 2.4.0 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # FULL backup with OLD binary + self.backup_node( + backup_dir, 'node', node, old_binary=True, options=['--compress']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "20", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE backup with NEW binary + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + pgdata = self.pgdata_content(node.data_dir) + + # merge PAGE4 + gdb = self.merge_backup(backup_dir, "node", page_id, gdb=True) + + gdb.set_breakpoint('rename') + gdb.run_until_break() + gdb.continue_execution_until_break(500) + gdb._execute('signal SIGKILL') + + try: + self.merge_backup(backup_dir, "node", page_id) + self.assertEqual( + 1, 0, + "Expecting Error because of format changes.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Retry of failed merge for backups with different " + "between minor versions is forbidden to avoid data corruption " + "because of storage format changes introduced in 2.4.0 version, " + "please take a new full backup", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_5(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version >= STORAGE_FORMAT_VERSION (2.4.4) + """ + if self.version_to_num(self.old_probackup_version) < self.version_to_num('2.4.4'): + self.assertTrue( + False, 'OLD pg_probackup binary must be >= 2.4.4 for this test') + + self.assertNotEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num(self.probackup_version)) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + # PAGE2 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + # merge chain created by old binary with new binary + output = self.merge_backup(backup_dir, "node", backup_id) + + # check that in-place is disabled + self.assertNotIn( + "WARNING: In-place merge is disabled " + "because of storage format incompatibility", output) + + # restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id1 = self.backup_node(backup_dir, 'node', node, old_binary=True) + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id2 = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + pgdata2 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + id3 = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + pgdata3 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id1) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id2) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id3) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + # @unittest.skip("skip") + def test_page_vacuum_truncate_compression(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node',node, old_binary=True, options=['--compress']) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_page_vacuum_truncate_compressed_1(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup, + restore latest page backup using new binary + and check data correctness + old binary should be 2.2.x version + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id1 = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--compress']) + pgdata1 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + id2 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + pgdata2 = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + id3 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + old_binary=True, options=['--compress']) + pgdata3 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id1) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id2) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir, backup_id=id3) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + node_restored.cleanup() + + # @unittest.skip("skip") + def test_hidden_files(self): + """ + old_version should be < 2.3.0 + Create hidden file in pgdata, take backup + with old binary, then try to delete backup + with new binary + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close() + + backup_id = self.backup_node( + backup_dir, 'node',node, old_binary=True, options=['--stream']) + + self.delete_pb(backup_dir, 'node', backup_id) + + # @unittest.skip("skip") + def test_compatibility_tablespace(self): + """ + https://github.com/postgrespro/pg_probackup/issues/348 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"], old_binary=True) + + tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old') + + self.create_tblspace_in_node( + node, 'tblspace', + tblspc_path=tblspace_old_path) + + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace tblspace " + "as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"], old_binary=True) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/compression_test.py b/tests/compression_test.py new file mode 100644 index 000000000..94f2dffff --- /dev/null +++ b/tests/compression_test.py @@ -0,0 +1,495 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +class CompressionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_basic_compression_stream_zlib(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=[ + '--stream', + '--compress-algorithm=zlib']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=[ + '--stream', '--compress-algorithm=zlib']) + + # DELTA BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + + def test_compression_archive_zlib(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=["--compress-algorithm=zlib"]) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,2) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=["--compress-algorithm=zlib"]) + + # DELTA BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--compress-algorithm=zlib']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + def test_compression_stream_pglz(self): + """ + make archive node, make full and page stream backups, + check data correctness in restored instance + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--compress-algorithm=pglz']) + + # DELTA BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + def test_compression_archive_pglz(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--compress-algorithm=pglz']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--compress-algorithm=pglz']) + + # DELTA BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--compress-algorithm=pglz']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=page_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + def test_compression_wrong_algorithm(self): + """ + make archive node, make full and page backups, + check data correctness in restored instance + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--compress-algorithm=bla-blah']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because compress-algorithm is invalid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: invalid compress algorithm value "bla-blah"\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_incompressible_pages(self): + """ + make archive node, create table with incompressible toast pages, + take backup with compression, make sure that page was not compressed, + restore backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Full + self.backup_node( + backup_dir, 'node', node, + options=[ + '--compress-algorithm=zlib', + '--compress-level=0']) + + node.pgbench_init(scale=3) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + '--compress-algorithm=zlib', + '--compress-level=0']) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() diff --git a/tests/config_test.py b/tests/config_test.py new file mode 100644 index 000000000..b1a0f9295 --- /dev/null +++ b/tests/config_test.py @@ -0,0 +1,113 @@ +import unittest +import subprocess +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from sys import exit +from shutil import copyfile + + +class ConfigTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_remove_instance_config(self): + """remove pg_probackup.conself.f""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.show_pb(backup_dir) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + conf_file = os.path.join( + backup_dir, 'backups','node', 'pg_probackup.conf') + + os.unlink(os.path.join(backup_dir, 'backups','node', 'pg_probackup.conf')) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because pg_probackup.conf is missing. " + ".\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: could not open file "{0}": ' + 'No such file or directory'.format(conf_file), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_corrupt_backup_content(self): + """corrupt backup_content.control""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + full1_id = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'create table t1()') + + fulle2_id = self.backup_node(backup_dir, 'node', node) + + fulle1_conf_file = os.path.join( + backup_dir, 'backups','node', full1_id, 'backup_content.control') + + fulle2_conf_file = os.path.join( + backup_dir, 'backups','node', fulle2_id, 'backup_content.control') + + copyfile(fulle2_conf_file, fulle1_conf_file) + + try: + self.validate_pb(backup_dir, 'node') + self.assertEqual( + 1, 0, + "Expecting Error because pg_probackup.conf is missing. " + ".\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Invalid CRC of backup control file '{0}':".format(fulle1_conf_file), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Failed to get file list for backup {0}".format(full1_id), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WARNING: Backup {0} file list is corrupted".format(full1_id), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.show_pb(backup_dir, 'node', full1_id)['status'] + + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], 'CORRUPT') + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], 'OK') diff --git a/tests/delete_test.py b/tests/delete_test.py new file mode 100644 index 000000000..10100887d --- /dev/null +++ b/tests/delete_test.py @@ -0,0 +1,822 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess + + +class DeleteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_full_backups(self): + """delete full backups""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + id_1 = show_backups[0]['id'] + id_2 = show_backups[1]['id'] + id_3 = show_backups[2]['id'] + self.delete_pb(backup_dir, 'node', id_2) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_3) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_del_instance_archive(self): + """delete full backups""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + # full backup + self.backup_node(backup_dir, 'node', node) + + # restore + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Delete instance + self.del_instance(backup_dir, 'node') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_delete_archive_mix_compress_and_non_compressed_segments(self): + """delete full backups""" + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving( + backup_dir, 'node', node, compress=False) + node.slow_start() + + # full backup + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=10) + + # Restart archiving with compression + self.set_archiving(backup_dir, 'node', node, compress=True) + + node.restart() + + # full backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--retention-redundancy=3', + '--delete-expired']) + + # @unittest.skip("skip") + def test_delete_increment_page(self): + """delete increment and all after him""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # @unittest.skip("skip") + def test_delete_increment_ptrack(self): + """delete increment and all after him""" + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE EXTENSION ptrack') + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # ptrack backup mode + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # full backup mode + self.backup_node(backup_dir, 'node', node) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + # delete first page backup + self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 2) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['backup-mode'], "FULL") + self.assertEqual(show_backups[1]['status'], "OK") + + # @unittest.skip("skip") + def test_delete_orphaned_wal_segments(self): + """ + make archive node, make three full backups, + delete second backup without --wal option, + then delete orphaned wals via --wal option + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") + # first full backup + backup_1_id = self.backup_node(backup_dir, 'node', node) + # second full backup + backup_2_id = self.backup_node(backup_dir, 'node', node) + # third full backup + backup_3_id = self.backup_node(backup_dir, 'node', node) + node.stop() + + # Check wals + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] + original_wal_quantity = len(wals) + + # delete second full backup + self.delete_pb(backup_dir, 'node', backup_2_id) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + # try to delete wals for second backup + self.delete_pb(backup_dir, 'node', options=['--wal']) + # check wal quantity + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # delete first full backup + self.delete_pb(backup_dir, 'node', backup_1_id) + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + result = self.delete_pb(backup_dir, 'node', options=['--wal']) + # delete useless wals + self.assertTrue('On timeline 1 WAL segments between ' in result + and 'will be removed' in result) + + self.validate_pb(backup_dir) + self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + + # Check quantity, it should be lower than original + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] + self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + + # Delete last backup + self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] + self.assertEqual (0, len(wals), "Number of wals should be equal to 0") + + # @unittest.skip("skip") + def test_delete_wal_between_multiple_timelines(self): + """ + /-------B1-- + A1----------------A2---- + + delete A1 backup, check that WAL segments on [A1, A2) and + [A1, B1) are deleted and backups B1 and A2 keep + their WAL + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + A1 = self.backup_node(backup_dir, 'node', node) + + # load some data to node + node.pgbench_init(scale=3) + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + self.restore_node(backup_dir, 'node', node2) + self.set_auto_conf(node2, {'port': node2.port}) + node2.slow_start() + + # load some more data to node + node.pgbench_init(scale=3) + + # take A2 + A2 = self.backup_node(backup_dir, 'node', node) + + # load some more data to node2 + node2.pgbench_init(scale=2) + + B1 = self.backup_node( + backup_dir, 'node', + node2, data_dir=node2.data_dir) + + self.delete_pb(backup_dir, 'node', backup_id=A1, options=['--wal']) + + self.validate_pb(backup_dir) + + # @unittest.skip("skip") + def test_delete_backup_with_empty_control_file(self): + """ + take backup, truncate its control file, + try to delete it via 'delete' command + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + # page backup mode + self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + # page backup mode + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=['--stream']) + + with open( + os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.control'), + 'wt') as f: + f.flush() + f.close() + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 3) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # @unittest.skip("skip") + def test_delete_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 and FULLb status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa2 and FULla to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # Change PAGEb1 and FULlb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 and FULLa status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGEc1 OK + # FULLc OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Delete FULLb + self.delete_pb( + backup_dir, 'node', backup_id_b) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 5) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # @unittest.skip("skip") + def test_delete_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa should be deleted + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 and FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEb2, PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change FULLa to OK + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa3 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + # Delete FULLa + self.delete_pb(backup_dir, 'node', backup_id_a) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # @unittest.skip("skip") + def test_delete_multiple_descendants_dry_run(self): + """ + PAGEa3 + PAGEa2 / + \ / + PAGEa1 (delete target) + | + FULLa + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + node.pgbench_init(scale=1) + backup_id_a = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + + # Change PAGEa2 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + + # Delete PAGEa1 + output = self.delete_pb( + backup_dir, 'node', page_id_a1, + options=['--dry-run', '--log-level-console=LOG', '--delete-wal']) + + print(output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a3), + output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a2), + output) + self.assertIn( + 'LOG: Backup {0} can be deleted'.format(page_id_a1), + output) + + self.assertIn( + 'INFO: Resident data size to free by ' + 'delete of backup {0} :'.format(page_id_a1), + output) + + self.assertIn( + 'On timeline 1 WAL segments between 000000010000000000000001 ' + 'and 000000010000000000000003 can be removed', + output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + output = self.delete_pb( + backup_dir, 'node', page_id_a1, + options=['--log-level-console=LOG', '--delete-wal']) + + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a3), + output) + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a2), + output) + self.assertIn( + 'LOG: Backup {0} will be deleted'.format(page_id_a1), + output) + self.assertIn( + 'INFO: Resident data size to free by ' + 'delete of backup {0} :'.format(page_id_a1), + output) + + self.assertIn( + 'On timeline 1 WAL segments between 000000010000000000000001 ' + 'and 000000010000000000000003 will be removed', + output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + self.validate_pb(backup_dir, 'node') + + def test_delete_error_backups(self): + """delete increment and all after him""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # full backup mode + self.backup_node(backup_dir, 'node', node) + # page backup mode + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Take FULL BACKUP + backup_id_a = self.backup_node(backup_dir, 'node', node) + # Take PAGE BACKUP + backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page") + + backup_id_c = self.backup_node(backup_dir, 'node', node, backup_type="page") + + backup_id_d = self.backup_node(backup_dir, 'node', node, backup_type="page") + + # full backup mode + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_e = self.backup_node(backup_dir, 'node', node, backup_type="page") + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Change status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_c, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_e, 'ERROR') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 10) + + # delete error backups + output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR', '--dry-run']) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 10) + + self.assertIn( + "Deleting all backups with status 'ERROR' in dry run mode", + output) + + self.assertIn( + "INFO: Backup {0} with status OK can be deleted".format(backup_id_d), + output) + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + show_backups = self.show_pb(backup_dir, 'node') + output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) + print(output) + show_backups = self.show_pb(backup_dir, 'node') + self.assertEqual(len(show_backups), 4) + + self.assertEqual(show_backups[0]['status'], "OK") + self.assertEqual(show_backups[1]['status'], "OK") + self.assertEqual(show_backups[2]['status'], "OK") + self.assertEqual(show_backups[3]['status'], "OK") diff --git a/tests/delta_test.py b/tests/delta_test.py new file mode 100644 index 000000000..23583fd93 --- /dev/null +++ b/tests/delta_test.py @@ -0,0 +1,1201 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from testgres import QueryException +import subprocess +import time +from threading import Thread + + +class DeltaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_delta_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + ) + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, + 'node', + node_restored, + options=[ + "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_delta_vacuum_truncate_2(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take delta backup, take second delta backup, + restore latest delta backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + ) + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10100000) i;" + ) + filepath = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')" + ).decode('utf-8').rstrip() + + self.backup_node(backup_dir, 'node', node) + + print(os.path.join(node.data_dir, filepath + '.1')) + os.unlink(os.path.join(node.data_dir, filepath + '.1')) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_delta_stream(self): + """ + make archive node, take full and delta stream backups, + restore them and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + node.slow_start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # @unittest.skip("skip") + def test_delta_archive(self): + """ + make archive node, take full and delta archive backups, + restore them and check data correctness + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # delta BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") + delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check delta backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(delta_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=delta_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(delta_result, delta_result_new) + node.cleanup() + + # @unittest.skip("skip") + def test_delta_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check delta and data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'full_page_writes': 'off' + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init( + scale=100, + options=['--tablespace=somedata', '--no-vacuum']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.safe_psql("postgres", "checkpoint") + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") + # delta BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format( + tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + self.set_auto_conf(restored_node, {'port': restored_node.port}) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select count(*) from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_delta_vacuum_full(self): + """ + make node, make full and delta stream backups, + restore them and check data correctness + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + pg_connect = node.connect("postgres", autocommit=True) + + gdb = self.gdb_attach(pg_connect.pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + gdb.continue_execution_until_running() + + process = Thread( + target=pg_connect.execute, args=["VACUUM FULL t_heap"]) + process.start() + + while not gdb.stopped_in_breakpoint: + time.sleep(1) + + gdb.continue_execution_until_break(20) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + gdb.remove_all_breakpoints() + gdb._execute('detach') + process.join() + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take delta backup, + restore database and check it presense + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND DELTA BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_exists_in_previous_backup(self): + """ + Make node, take full backup, create table, take page backup, + take delta backup, check that file is no fully copied to delta backup + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB', + 'checkpoint_timeout': '5min', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + filepath = node.safe_psql( + "postgres", + "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + self.backup_node( + backup_dir, + 'node', + node, + options=["--stream"]) + + # PAGE BACKUP + backup_id = self.backup_node( + backup_dir, + 'node', + node, + backup_type='page' + ) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + +# if self.paranoia: +# pgdata_page = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) + + # DELTA BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) +# if self.paranoia: +# pgdata_delta = self.pgdata_content( +# os.path.join( +# backup_dir, 'backups', +# 'node', backup_id, 'database')) +# self.compare_pgdata( +# pgdata_page, pgdata_delta) + + fullpath = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', filepath) + self.assertFalse(os.path.exists(fullpath)) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + + node_restored.cleanup() + self.restore_node( + backup_dir, + 'node', + node_restored, + backup_id=backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_delta(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i") + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new") + + # DELTA BACKUP + result = node.safe_psql( + "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from t_heap") + + self.assertEqual(result, result_new, 'lost some data after restore') + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_delta(self): + """ + Make node, take full backup, create database, + take delta backup, alter database tablespace location, + take delta backup restore last delta backup. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql( + "postgres", + "create database db1 tablespace = 'somedata'") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter database db1 set tablespace somedata_new" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, restore database. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + def test_delta_nullified_heap_page_backup(self): + """ + make node, take full backup, nullify some heap block, + take delta backup, restore, physically compare pgdata`s + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "CHECKPOINT") + + self.backup_node( + backup_dir, 'node', node) + + # Nullify some block in PostgreSQL + file = os.path.join(node.data_dir, file_path).replace("\\", "/") + if os.name == 'nt': + file = file.replace("\\", "/") + + with open(file, 'r+b', 0) as f: + f.seek(8192) + f.write(b"\x00"*8192) + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["--log-level-file=verbose"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if not self.remote: + log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") + with open(log_file_path) as f: + content = f.read() + + self.assertIn( + 'VERBOSE: File: "{0}" blknum 1, empty page'.format(file), + content) + self.assertNotIn( + "Skipping blknum 1 in file: {0}".format(file), + content) + + # Restore DELTA backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_delta_backup_from_past(self): + """ + make node, take FULL stream backup, take DELTA stream backup, + restore FULL backup, try to take second DELTA stream backup + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=3) + + # First DELTA + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # Restore FULL backup + node.cleanup() + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + node.slow_start() + + # Second DELTA backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are backing up an instance from the past" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Current START LSN ' in e.message and + 'is lower than START LSN ' in e.message and + 'of previous backup ' in e.message and + 'It may indicate that we are trying ' + 'to backup PostgreSQL instance from the past' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_delta_pg_resetxlog(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='delta', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'Insert error message', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/exclude_test.py b/tests/exclude_test.py new file mode 100644 index 000000000..cb3530cd5 --- /dev/null +++ b/tests/exclude_test.py @@ -0,0 +1,338 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +class ExcludeTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_exclude_temp_files(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + oid = node.safe_psql( + 'postgres', + "select oid from pg_database where datname = 'postgres'").rstrip() + + pgsql_tmp_dir = os.path.join(node.data_dir, 'base', 'pgsql_tmp') + + os.mkdir(pgsql_tmp_dir) + + file = os.path.join(pgsql_tmp_dir, 'pgsql_tmp7351.16') + with open(file, 'w') as f: + f.write("HELLO") + f.flush() + f.close + + full_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + file = os.path.join( + backup_dir, 'backups', 'node', full_id, + 'database', 'base', 'pgsql_tmp', 'pgsql_tmp7351.16') + + self.assertFalse( + os.path.exists(file), + "File must be excluded: {0}".format(file)) + + # TODO check temporary tablespaces + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_exclude_temp_tables(self): + """ + make node without archiving, create temp table, take full backup, + check that temp table not present in backup catalogue + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as conn: + + conn.execute( + "create temp table test as " + "select generate_series(0,50050000)::text") + conn.commit() + + temp_schema_name = conn.execute( + "SELECT nspname FROM pg_namespace " + "WHERE oid = pg_my_temp_schema()")[0][0] + conn.commit() + + temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( + "pg_", "") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + heap_path = conn.execute( + "select pg_relation_filepath('test')")[0][0] + conn.commit() + + index_path = conn.execute( + "select pg_relation_filepath('test_idx')")[0][0] + conn.commit() + + heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] + conn.commit() + + toast_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] + conn.commit() + + toast_idx_path = conn.execute( + "select pg_relation_filepath('{0}.{1}')".format( + temp_toast_schema_name, + "pg_toast_" + str(heap_oid) + "_index"))[0][0] + conn.commit() + + temp_table_filename = os.path.basename(heap_path) + temp_idx_filename = os.path.basename(index_path) + temp_toast_filename = os.path.basename(toast_path) + temp_idx_toast_filename = os.path.basename(toast_idx_path) + + self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + for root, dirs, files in os.walk(backup_dir): + for file in files: + if file in [ + temp_table_filename, temp_table_filename + ".1", + temp_idx_filename, + temp_idx_filename + ".1", + temp_toast_filename, + temp_toast_filename + ".1", + temp_idx_toast_filename, + temp_idx_toast_filename + ".1" + ]: + self.assertEqual( + 1, 0, + "Found temp table file in backup catalogue.\n " + "Filepath: {0}".format(file)) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_1(self): + """ + make node without archiving, create unlogged table, take full backup, + alter table to unlogged, take delta backup, restore delta backup, + check that PGDATA`s are physically the same + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + "shared_buffers": "10MB"}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create unlogged table test as " + "select generate_series(0,5005000)::text") + conn.commit() + + conn.execute("create index test_idx on test (generate_series)") + conn.commit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + node.safe_psql('postgres', "alter table test set logged") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_exclude_unlogged_tables_2(self): + """ + 1. make node, create unlogged, take FULL, DELTA, PAGE, + check that unlogged table files was not backed up + 2. restore FULL, DELTA, PAGE to empty db, + ensure unlogged table exist and is epmty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + "shared_buffers": "10MB"}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_ids = [] + + for backup_type in ['full', 'delta', 'page']: + + if backup_type == 'full': + node.safe_psql( + 'postgres', + 'create unlogged table test as select generate_series(0,20050000)::text') + else: + node.safe_psql( + 'postgres', + 'insert into test select generate_series(0,20050000)::text') + + rel_path = node.execute( + 'postgres', + "select pg_relation_filepath('test')")[0][0] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type=backup_type, options=['--stream']) + + backup_ids.append(backup_id) + + filelist = self.get_backup_filelist( + backup_dir, 'node', backup_id) + + self.assertNotIn( + rel_path, filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.1', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.2', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.3', filelist, + "Unlogged table was not excluded") + + # ensure restoring retrieves back only empty unlogged table + for backup_id in backup_ids: + node.stop() + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + node.slow_start() + + self.assertEqual( + node.execute( + 'postgres', + 'select count(*) from test')[0][0], + 0) + + # @unittest.skip("skip") + def test_exclude_log_dir(self): + """ + check that by default 'log' and 'pg_log' directories are not backed up + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').decode('utf-8').rstrip() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertFalse(os.path.exists(log_file)) + + # @unittest.skip("skip") + def test_exclude_log_dir_1(self): + """ + check that "--backup-pg-log" works correctly + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + log_dir = node.safe_psql( + 'postgres', + 'show log_directory').decode('utf-8').rstrip() + + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream', '--backup-pg-log']) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + # check that PGDATA/log or PGDATA/pg_log do not exists + path = os.path.join(node.data_dir, log_dir) + log_file = os.path.join(path, 'postgresql.log') + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.exists(log_file)) diff --git a/tests/external_test.py b/tests/external_test.py new file mode 100644 index 000000000..53f3c5449 --- /dev/null +++ b/tests/external_test.py @@ -0,0 +1,2405 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name +import shutil + + +# TODO: add some ptrack tests +class ExternalTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_basic_external(self): + """ + make node, create external directory, take backup + with external directory, restore backup, check that + external directory was successfully copied + """ + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_dir = self.get_tblspace_path(node, 'somedirectory') + + # create directory in external_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup with external directory pointing to a file + file_path = os.path.join(core_dir, 'file') + with open(file_path, "w+") as f: + pass + + try: + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=[ + '--external-dirs={0}'.format(file_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir point to a file" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --external-dirs option' in e.message and + 'directory or symbolic link expected' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # Fill external directories + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir, options=["-j", "4"]) + + # Full backup with external dir + self.backup_node( + backup_dir, 'node', node, + options=[ + '--external-dirs={0}'.format(external_dir)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_external_none(self): + """ + make node, create external directory, take backup + with external directory, take delta backup with --external-dirs=none, + restore delta backup, check that + external directory was not copied + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_dir = self.get_tblspace_path(node, 'somedirectory') + + # create directory in external_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # Fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir, options=["-j", "4"]) + + # Full backup with external dir + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--external-dirs={0}'.format(external_dir)]) + + # Delta backup without external directory + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=['--external-dirs=none', '--stream']) + + shutil.rmtree(external_dir, ignore_errors=True) + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_external_dirs_overlapping(self): + """ + make node, create directory, + take backup with two external directories pointing to + the same directory, backup should fail + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # create directory in external_directory + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + os.mkdir(external_dir1) + os.mkdir(external_dir2) + + # Full backup with external dirs + try: + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}{1}{0}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir1)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: External directory path (-E option)' in e.message and + 'contain another external directory' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_external_dir_mapping(self): + """ + make node, take full backup, check that restore with + external-dir mapping will end with error, take page backup, + check that restore with external-dir mapping will end with + success + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # Fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: --external-mapping option' in e.message and + 'have an entry in list of external directories' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_backup_multiple_external(self): + """check that cmdline has priority over config""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=['-E', external_dir1]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", external_dir2]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir1']) + + node.cleanup() + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + restore delta backup, check that incremental chain + restored correctly + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories using new binary + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE chain with new binary + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_1(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup ajd restore it + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=3) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain chain with new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_backward_compatibility_merge_2(self): + """ + take backup with old binary without external dirs support + take delta backup with new binary and 2 external directories + merge delta backup and restore it + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.show_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node, old_binary=True) + self.show_pb(backup_dir) + + node.slow_start() + + node.pgbench_init(scale=3) + + # tmp FULL backup with old binary + tmp_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # delta backup with external directories using new binary + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1']) + pgbench.wait() + + # Fill external dirs with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, + options=['-j', '4', '--skip-external-dirs']) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, + options=['-j', '4', '--skip-external-dirs']) + + # delta backup without external directories using old binary + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge chain using new binary + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + external_dir1_new = self.get_tblspace_path( + node_restored, 'external_dir1') + external_dir2_new = self.get_tblspace_path( + node_restored, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge(self): + """""" + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=3) + + # take temp FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # FULL backup with old binary without external dirs support + self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=["-j", "4", "--stream"]) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + print(self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) + + # Merge + print(self.merge_backup(backup_dir, 'node', backup_id=backup_id, + options=['--log-level-file=VERBOSE'])) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_skip_external_dirs(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup with old data + tmp_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # fill external directories with old data + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup with external directories + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # drop old external data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + # fill external directories with new data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, + options=["-j", "4", "--skip-external-dirs"]) + + # DELTA backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # merge backups without external directories + self.merge_backup( + backup_dir, 'node', + backup_id=backup_id, options=['--skip-external-dirs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_1(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup with changed data + backup_id = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories using new binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_3(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["-j", "4"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # page backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=[ + "-j", "4", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # page backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page", + options=[ + "-j", "4", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.merge_backup( + backup_dir, 'node', backup_id=backup_id, + options=['--log-level-file=verbose']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_merge_2(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + # fill external directories with changed data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # delta backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # delta backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + # delta backup without external directories + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + external_dir1_new = self.get_tblspace_path(node, 'external_dir1') + external_dir2_new = self.get_tblspace_path(node, 'external_dir2') + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_external_changed_data(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + + # set externals + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Delta backup with external directories + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_external_changed_data_1(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '32MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # set externals + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Delta backup with only one external directory + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", external_dir1]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir2']) + + # Restore + node.cleanup() + shutil.rmtree(node._base_dir) + + # create empty file in external_dir2 + os.mkdir(node._base_dir) + os.mkdir(external_dir2) + with open(os.path.join(external_dir2, 'file'), 'w+') as f: + f.close() + + output = self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + self.assertNotIn( + 'externaldir2', + output) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs', 'external_dir2']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_merge_external_changed_data(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '32MB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + + # set externals + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', + node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # change data a bit + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # fill external directories with changed data + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, backup_id=backup_id, + options=["-j", "4", "--skip-external-dirs"]) + + # change data a bit more + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Delta backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_restore_skip_external(self): + """ + Check that --skip-external-dirs works correctly + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # FULL backup with external directories + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2)]) + + # delete first externals, so pgdata_compare + # will be capable of detecting redundant + # external files after restore + shutil.rmtree(external_dir1, ignore_errors=True) + shutil.rmtree(external_dir2, ignore_errors=True) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # RESTORE + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--skip-external-dirs"]) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_is_symlink(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + if os.name == 'nt': + self.skipTest('Skipped for Windows') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + os.symlink(symlinked_dir, external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_contain_symlink_on_dir(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + if os.name == 'nt': + self.skipTest('Skipped for Windows') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + dir_in_external_dir = os.path.join(external_dir, 'dir') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + os.mkdir(external_dir) + os.symlink(symlinked_dir, dir_in_external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_contain_symlink_on_file(self): + """ + Check that backup works correctly if external dir is symlink, + symlink pointing to external dir should be followed, + but restored as directory + """ + if os.name == 'nt': + self.skipTest('Skipped for Windows') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + file_in_external_dir = os.path.join(external_dir, 'file') + + # temp FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=["-j", "4", "--stream"]) + + # fill some directory with data + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + symlinked_dir = os.path.join(core_dir, 'symlinked') + + self.restore_node( + backup_dir, 'node', node, + data_dir=symlinked_dir, options=["-j", "4"]) + + # drop temp FULL backup + self.delete_pb(backup_dir, 'node', backup_id=backup_id) + + # create symlink to directory in external directory + src_file = os.path.join(symlinked_dir, 'postgresql.conf') + os.mkdir(external_dir) + os.chmod(external_dir, 0o0700) + os.symlink(src_file, file_in_external_dir) + + # FULL backup with external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + # RESTORE + node_restored.cleanup() + + external_dir_new = self.get_tblspace_path( + node_restored, 'external_dir') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", "--external-mapping={0}={1}".format( + external_dir, external_dir_new)]) + + pgdata_restored = self.pgdata_content( + node_restored.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertEqual( + external_dir, + self.show_pb( + backup_dir, 'node', + backup_id=backup_id)['external-dirs']) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_external_dir_is_tablespace(self): + """ + Check that backup fails with error + if external directory points to tablespace + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + self.create_tblspace_in_node( + node, 'tblspace1', tblspc_path=external_dir) + + node.pgbench_init(scale=1, tablespace='tblspace1') + + # FULL backup with external directories + try: + backup_id = self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir points to the tablespace" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'External directory path (-E option)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_restore_external_dir_not_empty(self): + """ + Check that backup fails with error + if external directory point to not empty tablespace and + if remapped directory also isn`t empty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + node.cleanup() + + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir is not empty" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'External directory is not empty', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + external_dir_new = self.get_tblspace_path(node, 'external_dir_new') + + # create empty file in directory, which will be a target of + # remapping + os.mkdir(external_dir_new) + with open(os.path.join(external_dir_new, 'file1'), 'w+') as f: + f.close() + + try: + self.restore_node( + backup_dir, 'node', node, + options=['--external-mapping={0}={1}'.format( + external_dir, external_dir_new)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped external dir is not empty" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'External directory is not empty', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_restore_external_dir_is_missing(self): + """ + take FULL backup with not empty external directory + delete external directory + take DELTA backup with external directory, which + should fail + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # drop external directory + shutil.rmtree(external_dir, ignore_errors=True) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: External directory is not found:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + # take DELTA without external directories + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_merge_external_dir_is_missing(self): + """ + take FULL backup with not empty external directory + delete external directory + take DELTA backup with external directory, which + should fail, + take DELTA backup without external directory, + merge it into FULL, restore and check + data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # drop external directory + shutil.rmtree(external_dir, ignore_errors=True) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because external dir is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: External directory is not found:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + # take DELTA without external directories + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["-j", "4", "--stream"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_restore_external_dir_is_empty(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored + external directory is empty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + os.chmod(external_dir, 0o0700) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # make external directory empty + os.remove(os.path.join(external_dir, 'file')) + + # take DELTA backup with empty external directory + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_merge_external_dir_is_empty(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + merge backups and restore FULL, check that restored + external directory is empty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # create empty file in external directory + # open(os.path.join(external_dir, 'file'), 'a').close() + os.mkdir(external_dir) + os.chmod(external_dir, 0o0700) + with open(os.path.join(external_dir, 'file'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + # make external directory empty + os.remove(os.path.join(external_dir, 'file')) + + # take DELTA backup with empty external directory + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", external_dir]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_restore_external_dir_string_order(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored + external directory is empty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + # create empty file in external directory + os.mkdir(external_dir_1) + os.chmod(external_dir_1, 0o0700) + with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: + f.close() + + os.mkdir(external_dir_2) + os.chmod(external_dir_2, 0o0700) + with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_2)]) + + with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: + f.close() + + with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: + f.close() + + # take DELTA backup and swap external_dir_2 and external_dir_1 + # in external_dir_str + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_1)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Restore Delta backup + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + def test_merge_external_dir_string_order(self): + """ + take FULL backup with not empty external directory + drop external directory content + take DELTA backup with the same external directory + restore DELRA backup, check that restored + external directory is empty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) + shutil.rmtree(core_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + # create empty file in external directory + os.mkdir(external_dir_1) + os.chmod(external_dir_1, 0o0700) + with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: + f.close() + + os.mkdir(external_dir_2) + os.chmod(external_dir_2, 0o0700) + with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: + f.close() + + # FULL backup with external directory + self.backup_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_2)]) + + with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: + f.close() + + with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: + f.close() + + # take DELTA backup and swap external_dir_2 and external_dir_1 + # in external_dir_str + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}".format( + external_dir_2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir_1)]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + # Merge backups + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + # Restore + node.cleanup() + shutil.rmtree(node.base_dir, ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_smart_restore_externals(self): + """ + make node, create database, take full backup with externals, + take incremental backup without externals and restore it, + make sure that files from externals are not copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # fill external directories with data + tmp_id = self.backup_node(backup_dir, 'node', node) + + external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') + external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir_1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir_2, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # restore PAGE backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=['--no-validate', '--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + for file in filelist_diff: + self.assertNotIn(file, logfile_content) + + # @unittest.skip("skip") + def test_external_validation(self): + """ + make node, create database, + take full backup with external directory, + corrupt external file in backup, + run validate which should fail + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take temp FULL backup + tmp_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + external_dir = self.get_tblspace_path(node, 'external_dir') + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, backup_id=tmp_id, + data_dir=external_dir, options=["-j", "4"]) + + self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', '-E', "{0}".format(external_dir)]) + + # Corrupt file + file = os.path.join( + backup_dir, 'backups', 'node', full_id, + 'external_directories', 'externaldir1', 'postgresql.auto.conf') + + with open(file, "r+b", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + try: + self.validate_pb(backup_dir) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because file in external dir is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Invalid CRC of backup file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "CORRUPT"') diff --git a/tests/false_positive_test.py b/tests/false_positive_test.py new file mode 100644 index 000000000..fbb785c60 --- /dev/null +++ b/tests/false_positive_test.py @@ -0,0 +1,337 @@ +import unittest +import os +from time import sleep + +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess + + +class FalsePositive(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_wal_lost_segment(self): + """ + Loose segment located between backups. ExpectedFailure. This is BUG + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=5) + + # delete last wal segment + wals_dir = os.path.join(backup_dir, "wal", 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile( + os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals = map(int, wals) + os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + + # We just lost a wal segment and know nothing about it + self.backup_node(backup_dir, 'node', node) + self.assertTrue( + 'validation completed successfully' in self.validate_pb( + backup_dir, 'node')) + ######## + + @unittest.expectedFailure + # Need to force validation of ancestor-chain + def test_incremental_backup_corrupt_full_1(self): + """page-level backup with corrupted full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + file = os.path.join( + backup_dir, "backups", "node", + backup_id.decode("utf-8"), "database", "postgresql.conf") + os.remove(file) + + try: + self.backup_node(backup_dir, 'node', node, backup_type="page") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid full backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertFalse( + True, + "Expecting Error because page backup should not be " + "possible without valid full backup.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertEqual( + e.message, + 'ERROR: Valid full backup on current timeline is not found. ' + 'Create new FULL backup before an incremental one.\n', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_pg_10_waldir(self): + """ + test group access for PG >= 11 + """ + if self.pg_config_version < self.version_to_num('10.0'): + self.skipTest('You need PostgreSQL >= 10 for this test') + + wal_dir = os.path.join( + os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') + import shutil + shutil.rmtree(wal_dir, ignore_errors=True) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--waldir={0}'.format(wal_dir)]) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # restore backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + # compare pgdata permissions + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.assertTrue( + os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), + 'pg_wal should be symlink') + + @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_time_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + target_time = node.safe_psql( + "postgres", + "select now()").rstrip() + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-time={0}'.format(target_time)]) + + @unittest.expectedFailure + # @unittest.skip("skip") + def test_recovery_target_lsn_backup_victim(self): + """ + Check that for validation to recovery target + probackup chooses valid backup + https://github.com/postgrespro/pg_probackup/issues/104 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + node.safe_psql( + "postgres", + "create table t_heap1 as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100) i") + + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--log-level-console=LOG'], gdb=True) + + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + self.switch_wal_segment(node) + + target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] + + self.validate_pb( + backup_dir, 'node', + options=['--recovery-target-lsn={0}'.format(target_lsn)]) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_streaming_timeout(self): + """ + Illustrate the problem of loosing exact error + message because our WAL streaming engine is "borrowed" + from pg_receivexlog + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_sender_timeout': '5s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, + options=['--stream', '--log-level-file=LOG']) + + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + + sleep(10) + gdb.continue_execution_until_error() + gdb._execute('detach') + sleep(2) + + log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file_path) as f: + log_content = f.read() + + self.assertIn( + 'could not receive data from WAL stream', + log_content) + + self.assertIn( + 'ERROR: Problem in receivexlog', + log_content) + + # @unittest.skip("skip") + @unittest.expectedFailure + def test_validate_all_empty_catalog(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup_dir is empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: This backup catalog contains no backup instances', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py new file mode 100644 index 000000000..613e4dd36 --- /dev/null +++ b/tests/incr_restore_test.py @@ -0,0 +1,2300 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from datetime import datetime +import sys +from time import sleep +from datetime import datetime, timedelta +import hashlib +import shutil +import json +from testgres import QueryException + + +class IncrRestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_incr_restore(self): + """incremental restore in CHECKSUM mode""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_basic_incr_restore_into_missing_directory(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_checksum_corruption_detection(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=lsn"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + tblspace = self.get_tblspace_path(node, 'tblspace') + some_directory = self.get_tblspace_path(node, 'some_directory') + + # stuff new destination with garbage + self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--incremental-mode=checksum", "--force", + "-T{0}={1}".format(tblspace, some_directory)]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_1(self): + """recovery to target timeline""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + tblspace = self.get_tblspace_path(node, 'tblspace') + some_directory = self.get_tblspace_path(node, 'some_directory') + + self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_2(self): + """ + If "--tablespace-mapping" option is used with incremental restore, + then new directory must be empty. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + + # fill node1 with data + out = self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '--force']) + + self.assertIn("WARNING: Backup catalog was initialized for system id", out) + + tblspace = self.get_tblspace_path(node, 'tblspace') + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') + + node.safe_psql( + 'postgres', + 'vacuum') + + self.backup_node(backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + try: + self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped directory is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=[ + '--force', '--incremental-mode=checksum', + '-T{0}={1}'.format(tblspace, tblspace)]) + + pgdata_restored = self.pgdata_content(node_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_3(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + + self.create_tblspace_in_node(node, 'tblspace2') + node.pgbench_init(scale=10, tablespace='tblspace2') + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_4(self): + """ + Check that system ID mismatch is detected, + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup of node1 with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + node.cleanup() + + # recreate node + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because destination directory has wrong system id.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup catalog was initialized for system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is not allowed', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_incr_restore_with_tablespace_5(self): + """ + More complicated case, we restore backup + with tablespace, which we remap into directory + with some old content, that belongs to an instance + with different system id. + """ + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + node1.slow_start() + + self.create_tblspace_in_node(node1, 'tblspace') + node1.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node1 with tblspace + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + pgdata = self.pgdata_content(node1.data_dir) + + node1.stop() + + # recreate node + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), + set_replication=True, + initdb_params=['--data-checksums']) + node2.slow_start() + + self.create_tblspace_in_node(node2, 'tblspace') + node2.pgbench_init(scale=10, tablespace='tblspace') + node2.stop() + + tblspc1_path = self.get_tblspace_path(node1, 'tblspace') + tblspc2_path = self.get_tblspace_path(node2, 'tblspace') + + out = self.restore_node( + backup_dir, 'node', node1, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum", + "-T{0}={1}".format(tblspc1_path, tblspc2_path)]) + + # check that tblspc1_path is empty + self.assertFalse( + os.listdir(tblspc1_path), + "Dir is not empty: '{0}'".format(tblspc1_path)) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_6(self): + """ + Empty pgdata, not empty tablespace + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because there is running postmaster " + "process in destination directory.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: PGDATA is empty, but tablespace destination is not', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + self.assertIn( + "INFO: Destination directory and tablespace directories are empty, " + "disable incremental restore", out) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_7(self): + """ + Restore backup without tablespace into + PGDATA with tablespace. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') + node.stop() + +# try: +# self.restore_node( +# backup_dir, 'node', node, +# options=[ +# "-j", "4", +# "--incremental-mode=checksum"]) +# # we should die here because exception is what we expect to happen +# self.assertEqual( +# 1, 0, +# "Expecting Error because there is running postmaster " +# "process in destination directory.\n " +# "Output: {0} \n CMD: {1}".format( +# repr(self.output), self.cmd)) +# except ProbackupException as e: +# self.assertIn( +# 'ERROR: PGDATA is empty, but tablespace destination is not', +# e.message, +# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( +# repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_basic_incr_restore_sanity(self): + """recovery to target timeline""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + try: + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because there is running postmaster " + "process in destination directory.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Postmaster with pid', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is not allowed', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + + try: + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=["-j", "4", "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because destination directory has wrong system id.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup catalog was initialized for system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is not allowed', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_incr_checksum_restore(self): + """ + /----C-----D + ------A----B---*--------X + + X - is instance, we want to return it to C state. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + xid = node.safe_psql( + 'postgres', + 'select txid_current()').decode('utf-8').rstrip() + + # --A-----B--------X + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=[ + '--recovery-target-action=promote', + '--recovery-target-xid={0}'.format(xid)]) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + # /-- + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # /--C + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + # /--C------ + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) + pgbench.wait() + + # /--C------D + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node_1.data_dir) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) + + + # @unittest.skip("skip") + def test_incr_lsn_restore(self): + """ + /----C-----D + ------A----B---*--------X + + X - is instance, we want to return it to C state. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=50) + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + xid = node.safe_psql( + 'postgres', + 'select txid_current()').decode('utf-8').rstrip() + + # --A-----B--------X + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir, + options=[ + '--recovery-target-action=promote', + '--recovery-target-xid={0}'.format(xid)]) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + # /-- + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # /--C + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + # /--C------ + # --A-----B----*----X + pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) + pgbench.wait() + + # /--C------D + # --A-----B----*----X + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node_1.data_dir) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + node.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_lsn_sanity(self): + """ + /----A-----B + F------*--------X + + X - is instance, we want to return it to state B. + fail is expected behaviour in case of lsn restore. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=10) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='full') + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-mode=lsn"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in lsn mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Cannot perform incremental restore of " + "backup chain {0} in 'lsn' mode".format(page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_incr_checksum_sanity(self): + """ + /----A-----B + F------*--------X + + X - is instance, we want to return it to state B. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1.cleanup() + + self.restore_node( + backup_dir, 'node', node_1, data_dir=node_1.data_dir) + + self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.slow_start() + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='full') + + pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node_1, + data_dir=node_1.data_dir, backup_type='page') + pgdata = self.pgdata_content(node_1.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_checksum_corruption_detection(self): + """ + check that corrupted page got detected and replaced + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), +# initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='full') + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: + f.seek(22000) + f.write(b"bla") + f.flush() + f.close + + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_lsn_corruption_detection(self): + """ + check that corrupted page got detected and replaced + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=20) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='full') + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node(backup_dir, 'node', node, + data_dir=node.data_dir, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + + path = os.path.join(node.data_dir, heap_path) + with open(path, "rb+", 0) as f: + f.seek(22000) + f.write(b"bla") + f.flush() + f.close + + self.restore_node( + backup_dir, 'node', node, data_dir=node.data_dir, + options=["-j", "4", "--incremental-mode=lsn"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_multiple_external(self): + """check that cmdline has priority over config""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + node.pgbench_init(scale=20) + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=['-E{0}{1}{2}'.format( + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", '--incremental-mode=checksum']) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_restore_multiple_external(self): + """check that cmdline has priority over config""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + external_dir1 = self.get_tblspace_path(node, 'external_dir1') + external_dir2 = self.get_tblspace_path(node, 'external_dir2') + + # FULL backup + node.pgbench_init(scale=20) + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + # fill external directories with data + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir1, options=["-j", "4"]) + + self.restore_node( + backup_dir, 'node', node, + data_dir=external_dir2, options=["-j", "4"]) + + self.set_config( + backup_dir, 'node', + options=['-E{0}{1}{2}'.format( + external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["-j", "4"]) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # cmdline option MUST override options in config + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + pgdata = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", '--incremental-mode=lsn']) + + pgdata_restored = self.pgdata_content( + node.base_dir, exclude_dirs=['logs']) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_restore_backward(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + node.pgbench_init(scale=2) + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + full_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + page_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["-j", "4"]) + + delta_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(full_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", '--incremental-mode=lsn', + '--recovery-target=immediate', '--recovery-target-action=pause']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in lsn mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "Cannot perform incremental restore of backup chain", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", '--incremental-mode=checksum', + '--recovery-target=immediate', '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(page_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, + options=[ + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(delta_pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_checksum_restore_backward(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'hot_standby': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + node.pgbench_init(scale=20) + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + + full_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + page_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + page_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=["-j", "4"]) + + delta_pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(full_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=[ + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(page_pgdata, pgdata_restored) + + node.slow_start(replica=True) + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=delta_id, + options=[ + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(delta_pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_make_replica_via_incr_checksum_restore(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + master.pgbench_init(scale=20) + + self.backup_node(backup_dir, 'node', master) + + self.restore_node( + backup_dir, 'node', replica, options=['-R']) + + # Settings for Replica + self.set_replica(master, replica, synchronous=False) + + replica.slow_start(replica=True) + + pgbench = master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PROMOTIONS + replica.promote() + new_master = replica + + # old master is going a bit further + old_master = master + pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + old_master.stop() + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # take backup from new master + self.backup_node( + backup_dir, 'node', new_master, + data_dir=new_master.data_dir, backup_type='page') + + # restore old master as replica + self.restore_node( + backup_dir, 'node', old_master, data_dir=old_master.data_dir, + options=['-R', '--incremental-mode=checksum']) + + self.set_replica(new_master, old_master, synchronous=True) + + old_master.slow_start(replica=True) + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # @unittest.skip("skip") + def test_make_replica_via_incr_lsn_restore(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + master.pgbench_init(scale=20) + + self.backup_node(backup_dir, 'node', master) + + self.restore_node( + backup_dir, 'node', replica, options=['-R']) + + # Settings for Replica + self.set_replica(master, replica, synchronous=False) + + replica.slow_start(replica=True) + + pgbench = master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PROMOTIONS + replica.promote() + new_master = replica + + # old master is going a bit further + old_master = master + pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + old_master.stop() + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # take backup from new master + self.backup_node( + backup_dir, 'node', new_master, + data_dir=new_master.data_dir, backup_type='page') + + # restore old master as replica + self.restore_node( + backup_dir, 'node', old_master, data_dir=old_master.data_dir, + options=['-R', '--incremental-mode=lsn']) + + self.set_replica(new_master, old_master, synchronous=True) + + old_master.slow_start(replica=True) + + pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_checksum_long_xact(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + + con.commit() + + node.safe_psql( + 'postgres', + 'select * from t1') + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=checksum']) + + node.slow_start() + + self.assertEqual( + node.safe_psql( + 'postgres', + 'select count(*) from t1').decode('utf-8').rstrip(), + '1') + + # @unittest.skip("skip") + # @unittest.expectedFailure + # This test will pass with Enterprise + # because it has checksums enabled by default + @unittest.skipIf(ProbackupTest.enterprise, 'skip') + def test_incr_lsn_long_xact_1(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + + con.commit() + + # when does LSN gets stamped when checksum gets updated ? + node.safe_psql( + 'postgres', + 'select * from t1') + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=lsn']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental restore in lsn mode is impossible\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Incremental restore in 'lsn' mode require data_checksums to be " + "enabled in destination data directory", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_lsn_long_xact_2(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'full_page_writes': 'off', + 'wal_log_hints': 'off'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + # FULL backup + con = node.connect("postgres") + con.execute("CREATE TABLE t1 (a int)") + con.commit() + + + con.execute("INSERT INTO t1 values (1)") + con.commit() + + # leave uncommited + con2 = node.connect("postgres") + con.execute("INSERT INTO t1 values (2)") + con2.execute("INSERT INTO t1 values (3)") + + full_id = self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4", "--stream"]) + + self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) + + con.commit() + + # when does LSN gets stamped when checksum gets updated ? + node.safe_psql( + 'postgres', + 'select * from t1') + +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) + + con2.commit() + node.safe_psql( + 'postgres', + 'select * from t1') + +# print(node.safe_psql( +# 'postgres', +# "select * from page_header(get_raw_page('t1', 0))")) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=lsn']) + + node.slow_start() + + self.assertEqual( + node.safe_psql( + 'postgres', + 'select count(*) from t1').decode('utf-8').rstrip(), + '1') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_zero_size_file_checksum(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + fullpath = os.path.join(node.data_dir, 'simple_file') + with open(fullpath, "w+b", 0) as f: + f.flush() + f.close + + # FULL backup + id1 = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + pgdata1 = self.pgdata_content(node.data_dir) + + with open(fullpath, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + id2 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata2 = self.pgdata_content(node.data_dir) + + with open(fullpath, "w") as f: + f.close() + + id3 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata3 = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id1, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + self.restore_node( + backup_dir, 'node', node, backup_id=id2, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + self.restore_node( + backup_dir, 'node', node, backup_id=id3, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_incr_restore_zero_size_file_lsn(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + fullpath = os.path.join(node.data_dir, 'simple_file') + with open(fullpath, "w+b", 0) as f: + f.flush() + f.close + + # FULL backup + id1 = self.backup_node( + backup_dir, 'node', node, + options=["-j", "4", "--stream"]) + + pgdata1 = self.pgdata_content(node.data_dir) + + with open(fullpath, "rb+", 0) as f: + f.seek(9000) + f.write(b"bla") + f.flush() + f.close + + id2 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata2 = self.pgdata_content(node.data_dir) + + with open(fullpath, "w") as f: + f.close() + + id3 = self.backup_node( + backup_dir, 'node', node, + backup_type="delta", options=["-j", "4", "--stream"]) + pgdata3 = self.pgdata_content(node.data_dir) + + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id1, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata1, pgdata_restored) + + node.slow_start() + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id2, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata2, pgdata_restored) + + node.slow_start() + node.stop() + + self.restore_node( + backup_dir, 'node', node, backup_id=id3, + options=["-j", "4", '-I', 'checksum']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata3, pgdata_restored) + + def test_incremental_partial_restore_exclude_checksum(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + node.pgbench_init(scale=20) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # restore FULL backup into second node2 + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1.cleanup() + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + # restore some data into node2 + self.restore_node(backup_dir, 'node', node2) + + # partial restore backup into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore backup into node2 + self.restore_node( + backup_dir, 'node', + node2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-I", "checksum"]) + + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_incremental_partial_restore_exclude_lsn(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + node.pgbench_init(scale=20) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1']) + pgbench.wait() + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.stop() + + # restore FULL backup into second node2 + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1.cleanup() + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + # restore some data into node2 + self.restore_node(backup_dir, 'node', node2) + + # partial restore backup into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore backup into node2 + node2.port = node.port + node2.slow_start() + node2.stop() + self.restore_node( + backup_dir, 'node', + node2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-I", "lsn"]) + + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_incremental_partial_restore_exclude_tablespace_checksum(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # cat_version = node.get_control_data()["Catalog version number"] + # version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version + + # PG_10_201707211 + # pg_tblspc/33172/PG_9.5_201510051/16386/ + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # node1 + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1.cleanup() + node1_tablespace = self.get_tblspace_path(node1, 'somedata') + + # node2 + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + node2_tablespace = self.get_tblspace_path(node2, 'somedata') + + # in node2 restore full backup + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + + # partial restore into node1 + self.restore_node( + backup_dir, 'node', + node1, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node1_tablespace)]) + + pgdata1 = self.pgdata_content(node1.data_dir) + + # partial incremental restore into node2 + try: + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped tablespace contain old data .\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-I", "checksum", "--force", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + + pgdata2 = self.pgdata_content(node2.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + + self.set_auto_conf(node2, {'port': node2.port}) + node2.slow_start() + + node2.safe_psql( + 'postgres', + 'select 1') + + try: + node2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_incremental_pg_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # in node1 restore full backup + self.restore_node(backup_dir, 'node', node1) + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + pgbench = node1.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node1.stop() + + # incremental restore into node1 + self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum"]) + + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + node1.safe_psql( + 'postgres', + 'select 1') + +# check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn diff --git a/tests/init_test.py b/tests/init_test.py new file mode 100644 index 000000000..94b076fef --- /dev/null +++ b/tests/init_test.py @@ -0,0 +1,138 @@ +import os +import unittest +from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException +import shutil + + +class InitTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_success(self): + """Success normal init""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) + self.init_pb(backup_dir) + self.assertEqual( + dir_files(backup_dir), + ['backups', 'wal'] + ) + self.add_instance(backup_dir, 'node', node) + self.assertIn( + "INFO: Instance 'node' successfully deleted", + self.del_instance(backup_dir, 'node'), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + + # Show non-existing instance + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' does not exist in this backup catalog", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Delete non-existing instance + try: + self.del_instance(backup_dir, 'node1') + self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node1' does not exist in this backup catalog", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # Add instance without pgdata + try: + self.run_pb([ + "add-instance", + "--instance=node1", + "-B", backup_dir + ]) + self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + + # @unittest.skip("skip") + def test_already_exist(self): + """Failure with backup catalog already existed""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) + self.init_pb(backup_dir) + try: + self.show_pb(backup_dir, 'node') + self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' does not exist in this backup catalog", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_abs_path(self): + """failure with backup catalog should be given as absolute path""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) + try: + self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) + self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: -B, --backup-path must be an absolute path", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_add_instance_idempotence(self): + """ + https://github.com/postgrespro/pg_probackup/issues/219 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) + self.init_pb(backup_dir) + + self.add_instance(backup_dir, 'node', node) + shutil.rmtree(os.path.join(backup_dir, 'backups', 'node')) + + dir_backups = os.path.join(backup_dir, 'backups', 'node') + dir_wal = os.path.join(backup_dir, 'wal', 'node') + + try: + self.add_instance(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' WAL archive directory already exists: ", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.add_instance(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Instance 'node' WAL archive directory already exists: ", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) diff --git a/tests/locking_test.py b/tests/locking_test.py new file mode 100644 index 000000000..5367c2610 --- /dev/null +++ b/tests/locking_test.py @@ -0,0 +1,629 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +class LockingTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_locking_running_validate_1(self): + """ + make node, take full backup, stop it in the middle + run validate, expect it to successfully executed, + concurrent RUNNING backup with pid file and active process is legal + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + validate_output = self.validate_pb( + backup_dir, options=['--log-level-console=LOG']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + self.assertIn( + "is using backup {0}, and is still running".format(backup_id), + validate_output, + '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + # Clean after yourself + gdb.kill() + + def test_locking_running_validate_2(self): + """ + make node, take full backup, stop it in the middle, + kill process so no cleanup is done - pid file is in place, + run validate, expect it to not successfully executed, + RUNNING backup with pid file AND without active pid is legal, + but his status must be changed to ERROR and pid file is deleted + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_error() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "which used backup {0} no longer exists".format( + backup_id) in e.message and + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + + # Clean after yourself + gdb.kill() + + def test_locking_running_validate_2_specific_id(self): + """ + make node, take full backup, stop it in the middle, + kill process so no cleanup is done - pid file is in place, + run validate on this specific backup, + expect it to not successfully executed, + RUNNING backup with pid file AND without active pid is legal, + but his status must be changed to ERROR and pid file is deleted + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_error() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "which used backup {0} no longer exists".format( + backup_id) in e.message and + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "ERROR: Backup {0} has status: ERROR".format(backup_id) in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has status: ERROR".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Backup {0} has status ERROR. Skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + gdb.kill() + + def test_locking_running_3(self): + """ + make node, take full backup, stop it in the middle, + terminate process, delete pid file, + run validate, expect it to not successfully executed, + RUNNING backup without pid file AND without active pid is legal, + his status must be changed to ERROR + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_error() + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + + backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.remove( + os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid')) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because RUNNING backup is no longer active.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "Backup {0} has status RUNNING, change it " + "to ERROR and skip validation".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + + # Clean after yourself + gdb.kill() + + def test_locking_restore_locked(self): + """ + make node, take full backup, take two page backups, + launch validate on PAGE1 and stop it in the middle, + launch restore of PAGE2. + Expect restore to sucseed because read-only locks + do not conflict + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.validate_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + # Clean after yourself + gdb.kill() + + def test_concurrent_delete_and_restore(self): + """ + make node, take full backup, take page backup, + launch validate on FULL and stop it in the middle, + launch restore of PAGE. + Expect restore to fail because validation of + intermediate backup is impossible + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.delete_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + # gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because restore without whole chain validation " + "is prohibited unless --no-validate provided.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "Backup {0} is used without validation".format( + restore_id) in e.message and + 'is using backup {0}, and is still running'.format( + backup_id) in e.message and + 'ERROR: Cannot lock backup' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + gdb.kill() + + def test_locking_concurrent_validate_and_backup(self): + """ + make node, take full backup, launch validate + and stop it in the middle, take page backup. + Expect PAGE backup to be successfully executed + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + # PAGE2 + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + gdb = self.validate_pb( + backup_dir, 'node', backup_id=backup_id, gdb=True) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + # This PAGE backup is expected to be successfull + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Clean after yourself + gdb.kill() + + def test_locking_concurren_restore_and_delete(self): + """ + make node, take full backup, launch restore + and stop it in the middle, delete full backup. + Expect it to fail. + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_id = self.backup_node(backup_dir, 'node', node) + + node.cleanup() + gdb = self.restore_node(backup_dir, 'node', node, gdb=True) + + gdb.set_breakpoint('create_data_directories') + gdb.run_until_break() + + try: + self.delete_pb(backup_dir, 'node', full_id) + self.assertEqual( + 1, 0, + "Expecting Error because backup is locked\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Cannot lock backup {0} directory".format(full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Clean after yourself + gdb.kill() + + def test_backup_directory_name(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_id_1 = self.backup_node(backup_dir, 'node', node) + page_id_1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + full_id_2 = self.backup_node(backup_dir, 'node', node) + page_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + old_path = os.path.join(backup_dir, 'backups', 'node', full_id_1) + new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') + + os.rename(old_path, new_path) + + # This PAGE backup is expected to be successfull + self.show_pb(backup_dir, 'node', full_id_1) + + self.validate_pb(backup_dir) + self.validate_pb(backup_dir, 'node') + self.validate_pb(backup_dir, 'node', full_id_1) + + self.restore_node(backup_dir, 'node', node, backup_id=full_id_1) + + self.delete_pb(backup_dir, 'node', full_id_1) + + old_path = os.path.join(backup_dir, 'backups', 'node', full_id_2) + new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') + + self.set_backup( + backup_dir, 'node', full_id_2, options=['--note=hello']) + + self.merge_backup(backup_dir, 'node', page_id_2, options=["-j", "4"]) + + self.assertNotIn( + 'note', + self.show_pb(backup_dir, 'node', page_id_2)) + + # Clean after yourself + + def test_empty_lock_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/308 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=100) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + with open(lockfile, "w+") as f: + f.truncate() + + out = self.validate_pb(backup_dir, 'node', backup_id) + + self.assertIn( + "Waiting 30 seconds on empty exclusive lock for backup", out) + +# lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') +# with open(lockfile, "w+") as f: +# f.truncate() +# +# p1 = self.validate_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=validate.log']) +# sleep(3) +# p2 = self.delete_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=delete.log']) +# +# p1.wait() +# p2.wait() + + def test_shared_lock(self): + """ + Make sure that shared lock leaves no files with pids + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile_excl = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + lockfile_shr = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup_ro.pid') + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + + gdb = self.validate_pb(backup_dir, 'node', backup_id, gdb=True) + + gdb.set_breakpoint('validate_one_page') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + os.path.exists(lockfile_shr), + "File should exist: {0}".format(lockfile_shr)) + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + diff --git a/tests/logging_test.py b/tests/logging_test.py new file mode 100644 index 000000000..c5cdfa344 --- /dev/null +++ b/tests/logging_test.py @@ -0,0 +1,345 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import datetime + +class LogTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + # PGPRO-2154 + def test_log_rotation(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1s', '--log-rotation-size=1MB']) + + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=verbose']) + + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=verbose'], gdb=True) + + gdb.set_breakpoint('open_logfile') + gdb.run_until_break() + gdb.continue_execution_until_exit() + + def test_log_filename_strftime(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE', + '--log-filename=pg_probackup-%a.log']) + + day_of_week = datetime.datetime.today().strftime("%a") + + path = os.path.join( + backup_dir, 'log', 'pg_probackup-{0}.log'.format(day_of_week)) + + self.assertTrue(os.path.isfile(path)) + + def test_truncate_rotation_file(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # truncate .rotation file + with open(rotation_file_path, "rb+", 0) as f: + f.truncate() + f.flush() + f.close + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=LOG'], + return_id=False) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + self.assertIn( + 'WARNING: cannot read creation timestamp from rotation file', + output) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=LOG'], + return_id=False) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + self.assertNotIn( + 'WARNING: cannot read creation timestamp from rotation file', + output) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + def test_unlink_rotation_file(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # unlink .rotation file + os.unlink(rotation_file_path) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=LOG'], + return_id=False) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + self.assertIn( + 'WARNING: missing rotation file:', + output) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE'], + return_id=False) + + self.assertNotIn( + 'WARNING: missing rotation file:', + output) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + def test_garbage_in_rotation_file(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', + options=['--log-rotation-age=1d']) + + self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=VERBOSE']) + + rotation_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log.rotation') + + log_file_path = os.path.join( + backup_dir, 'log', 'pg_probackup.log') + + log_file_size = os.stat(log_file_path).st_size + + self.assertTrue(os.path.isfile(rotation_file_path)) + + # mangle .rotation file + with open(rotation_file_path, "w+b", 0) as f: + f.write(b"blah") + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=LOG'], + return_id=False) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + self.assertIn( + 'WARNING: rotation file', + output) + + self.assertIn( + 'has wrong creation timestamp', + output) + + self.assertTrue(os.path.isfile(rotation_file_path)) + + output = self.backup_node( + backup_dir, 'node', node, + options=[ + '--stream', + '--log-level-file=LOG'], + return_id=False) + + self.assertNotIn( + 'WARNING: rotation file', + output) + + # check that log file wasn`t rotated + self.assertGreater( + os.stat(log_file_path).st_size, + log_file_size) + + def test_issue_274(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,45600) i") + + log_dir = os.path.join(backup_dir, "somedir") + + try: + self.backup_node( + backup_dir, 'node', replica, backup_type='page', + options=[ + '--log-level-console=verbose', '--log-level-file=verbose', + '--log-directory={0}'.format(log_dir), '-j1', + '--log-filename=somelog.txt', '--archive-timeout=5s', + '--no-validate', '--log-rotation-size=100KB']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of archiving timeout" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: WAL segment', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file_path = os.path.join( + log_dir, 'somelog.txt') + + self.assertTrue(os.path.isfile(log_file_path)) + + with open(log_file_path, "r+") as f: + log_content = f.read() + + self.assertIn('INFO: command:', log_content) diff --git a/tests/merge_test.py b/tests/merge_test.py new file mode 100644 index 000000000..ffa73263c --- /dev/null +++ b/tests/merge_test.py @@ -0,0 +1,2759 @@ +# coding: utf-8 + +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import QueryException +import shutil +from datetime import datetime, timedelta +import time +import subprocess + +class MergeTest(ProbackupTest, unittest.TestCase): + + def test_basic_merge_full_page(self): + """ + Test MERGE command, it merges FULL backup with target PAGE backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=["--data-checksums"]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Do full backup + self.backup_node(backup_dir, "node", node, options=['--compress']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + conn.commit() + + # Do first page backup + self.backup_node(backup_dir, "node", node, backup_type="page", options=['--compress']) + show_backup = self.show_pb(backup_dir, "node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do second page backup + self.backup_node( + backup_dir, "node", node, + backup_type="page", options=['--compress']) + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id, + options=["-j", "4"]) + show_backups = self.show_pb(backup_dir, "node") + + # sanity check + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # Check physical correctness + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + def test_merge_compressed_backups(self): + """ + Test MERGE command with compressed backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=["--data-checksums"]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Do full compressed backup + self.backup_node(backup_dir, "node", node, options=['--compress']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int)") + conn.execute( + "insert into test select i from generate_series(1,10) s(i)") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # Do compressed page backup + self.backup_node( + backup_dir, "node", node, backup_type="page", options=['--compress']) + show_backup = self.show_pb(backup_dir, "node")[1] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Check restored node + count2 = node.execute("postgres", "select count(*) from test") + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + + def test_merge_compressed_backups_1(self): + """ + Test MERGE command with compressed backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=["--data-checksums"]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=['--compress', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, + backup_type="delta", options=['--compress', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", options=['--compress']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + + def test_merge_compressed_and_uncompressed_backups(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=["--data-checksums"], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node(backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id, options=['-j2']) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + + def test_merge_compressed_and_uncompressed_backups_1(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=["--data-checksums"], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5) + + # Do compressed FULL backup + self.backup_node(backup_dir, "node", node, options=[ + '--compress-algorithm=zlib', '--stream']) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page", + options=['--compress-algorithm=zlib']) + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + + def test_merge_compressed_and_uncompressed_backups_2(self): + """ + Test MERGE command with compressed and uncompressed backups + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=["--data-checksums"], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, "node", node) + self.set_archiving(backup_dir, "node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=20) + + # Do uncompressed FULL backup + self.backup_node(backup_dir, "node", node) + show_backup = self.show_pb(backup_dir, "node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed DELTA backup + self.backup_node( + backup_dir, "node", node, backup_type="delta", + options=['--compress-algorithm=zlib', '--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.backup_node( + backup_dir, "node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + show_backup = self.show_pb(backup_dir, "node")[2] + page_id = show_backup["id"] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Merge all backups + self.merge_backup(backup_dir, "node", page_id) + show_backups = self.show_pb(backup_dir, "node") + + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_merge_tablespaces(self): + """ + Create tablespace with table, take FULL backup, + create another tablespace with another table and drop previous + tablespace, take page backup, merge it and restore + + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Create new tablespace + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + + # Drop old tablespace + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # PAGE backup + backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + + # this compare should fall because we lost some directories + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_merge_tablespaces_1(self): + """ + Create tablespace with table, take FULL backup, + create another tablespace with another table, take page backup, + drop first tablespace and take delta backup, + merge it and restore + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # CREATE NEW TABLESPACE + self.create_tblspace_in_node(node, 'somedata1') + + node.safe_psql( + "postgres", + "create table t_heap1 tablespace somedata1 as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + + node.safe_psql( + "postgres", + "drop table t_heap" + ) + node.safe_psql( + "postgres", + "drop tablespace somedata" + ) + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + pgdata = self.pgdata_content(node.data_dir) + + node.stop() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + shutil.rmtree( + self.get_tblspace_path(node, 'somedata1'), + ignore_errors=True) + node.cleanup() + + self.merge_backup(backup_dir, 'node', backup_id) + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_merge_page_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '300s'}) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + def test_merge_delta_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '300s'}) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + page_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + def test_merge_ptrack_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, merge full and page, + restore last page backup and check data correctness + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + ptrack_enable=True) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.merge_backup(backup_dir, "node", page_id) + + self.validate_pb(backup_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # @unittest.skip("skip") + def test_merge_delta_delete(self): + """ + Make node, create tablespace with table, take full backup, + alter tablespace location, take delta backup, merge full and delta, + restore database. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[1]["id"] + self.merge_backup(backup_dir, "node", backup_id, options=["-j", "4"]) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_continue_failed_merge(self): + """ + Check that failed MERGE can be continued + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join( + self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta' + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + + gdb.set_breakpoint('backup_non_data_file_internal') + gdb.run_until_break() + + gdb.continue_execution_until_break(5) + + gdb._execute('signal SIGKILL') + gdb._execute('detach') + time.sleep(1) + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + # Try to continue failed MERGE + self.merge_backup(backup_dir, "node", backup_id) + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # @unittest.skip("skip") + def test_continue_failed_merge_with_corrupted_delta_backup(self): + """ + Fail merge via gdb, corrupt DELTA backup, try to continue merge + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i") + + old_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + node.safe_psql( + "postgres", + "vacuum full t_heap") + + new_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + # DELTA BACKUP + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + backup_id = self.show_pb(backup_dir, "node")[1]["id"] + + # Failed MERGE + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('backup_non_data_file_internal') + gdb.run_until_break() + + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + # CORRUPT incremental backup + # read block from future + # block_size + backup_header = 8200 + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_2, 'database', new_path) + with open(file, 'rb') as f: + f.seek(8200) + block_1 = f.read(8200) + f.close + + # write block from future + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', old_path) + with open(file, 'r+b') as f: + f.seek(8200) + f.write(block_1) + f.close + + # Try to continue failed MERGE + try: + print(self.merge_backup(backup_dir, "node", backup_id)) + self.assertEqual( + 1, 0, + "Expecting Error because of incremental backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Backup {0} has status CORRUPT, merge is aborted".format( + backup_id) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_continue_failed_merge_2(self): + """ + Check that failed MERGE on delete can be continued + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + + gdb.set_breakpoint('pgFileDelete') + + gdb.run_until_break() + + gdb._execute('thread apply all bt') + + gdb.continue_execution_until_break(20) + + gdb._execute('thread apply all bt') + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] + + # TODO check that full backup has meta info is equal to DELETTING + + # Try to continue failed MERGE + self.merge_backup(backup_dir, "node", backup_id) + + def test_continue_failed_merge_3(self): + """ + Check that failed MERGE cannot be continued if intermediate + backup is missing. + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create test data + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100000) i" + ) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # CREATE FEW PAGE BACKUP + i = 0 + + while i < 2: + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200000) i" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page' + ) + i = i + 1 + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id_merge = self.show_pb(backup_dir, "node")[2]["id"] + backup_id_delete = self.show_pb(backup_dir, "node")[1]["id"] + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + + gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) + + gdb.set_breakpoint('backup_non_data_file_internal') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + print(self.show_pb(backup_dir, as_text=True, as_json=False)) + # print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # DELETE PAGE1 + shutil.rmtree( + os.path.join(backup_dir, "backups", "node", backup_id_delete)) + + # Try to continue failed MERGE + try: + self.merge_backup(backup_dir, "node", backup_id_merge) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Incremental chain is broken, " + "merge is impossible to finish" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_merge_different_compression_algo(self): + """ + Check that backups with different compression algorithms can be merged + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node( + backup_dir, 'node', node, options=['--compress-algorithm=zlib']) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--compress-algorithm=pglz']) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + backup_id = self.show_pb(backup_dir, "node")[2]["id"] + + self.merge_backup(backup_dir, "node", backup_id) + + def test_merge_different_wal_modes(self): + """ + Check that backups with different wal modes can be merged + correctly + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + # DELTA stream backup + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.assertEqual( + 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) + + def test_crash_after_opening_backup_control_1(self): + """ + check that crashing after opening backup.control + for writing will not result in losing backup metadata + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL stream backup + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # DELTA archive backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + + gdb.set_breakpoint('write_backup') + gdb.continue_execution_until_break() + gdb.set_breakpoint('pgBackupWriteControl') + gdb.continue_execution_until_break() + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # @unittest.skip("skip") + def test_crash_after_opening_backup_control_2(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + TODO: rewrite + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=3) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + pgbench = node.pgbench(options=['-T', '20', '-c', '2']) + pgbench.wait() + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + fsm_path = path + '_fsm' + + node.safe_psql( + 'postgres', + 'vacuum pgbench_accounts') + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(1) + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', fsm_path) + + # print(file_to_remove) + + os.remove(file_to_remove) + + # Continue failed merge + self.merge_backup(backup_dir, "node", backup_id) + + node.cleanup() + + # restore merge backup + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_losing_file_after_failed_merge(self): + """ + check that crashing after opening backup_content.control + for writing will not result in losing metadata about backup files + TODO: rewrite + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=1) + + # FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Change data + node.safe_psql( + 'postgres', + "update pgbench_accounts set aid = aid + 1005000") + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + node.safe_psql( + 'postgres', + "VACUUM pgbench_accounts") + + vm_path = path + '_vm' + + # DELTA backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb.set_breakpoint('write_backup_filelist') + gdb.run_until_break() + +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # In to_backup drop file that comes from from_backup + # emulate crash during previous merge + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', full_id, 'database', vm_path) + + os.remove(file_to_remove) + + # Try to continue failed MERGE + self.merge_backup(backup_dir, "node", backup_id) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_failed_merge_after_delete(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + dboid = node.safe_psql( + "postgres", + "select oid from pg_database where datname = 'testdb'").decode('utf-8').rstrip() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # drop database + node.safe_psql( + 'postgres', + 'DROP DATABASE testdb') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + + # backup half-merged + self.assertEqual( + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + db_path = os.path.join( + backup_dir, 'backups', 'node', + full_id, 'database', 'base', dboid) + + try: + self.merge_backup( + backup_dir, 'node', page_id_2, + options=['--log-level-console=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of missing parent.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Full backup {0} has unfinished merge with backup {1}".format( + full_id, page_id) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_failed_merge_after_delete_1(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + page_1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGE1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_1, 'ERROR') + + pgdata = self.pgdata_content(node.data_dir) + + # add data + pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # take PAGE2 backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGE1 backup status to OK + self.change_backup_status(backup_dir, 'node', page_1, 'OK') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + +# gdb.set_breakpoint('parray_bsearch') +# gdb.continue_execution_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(30) + gdb._execute('signal SIGKILL') + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + # restore + node.cleanup() + try: + #self.restore_node(backup_dir, 'node', node, backup_id=page_1) + self.restore_node(backup_dir, 'node', node) + self.assertEqual( + 1, 0, + "Expecting Error because of orphan status.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} is orphan".format(page_1), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_failed_merge_after_delete_2(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + page_1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # add data + pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + + # take PAGE2 backup + page_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_2, gdb=True, + options=['--log-level-console=VERBOSE']) + + gdb.set_breakpoint('pgFileDelete') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + gdb._execute('signal SIGKILL') + + self.delete_pb(backup_dir, 'node', backup_id=page_2) + + # rerun merge + try: + #self.restore_node(backup_dir, 'node', node, backup_id=page_1) + self.merge_backup(backup_dir, 'node', page_1) + self.assertEqual( + 1, 0, + "Expecting Error because of backup is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Full backup {0} has unfinished merge " + "with backup {1}".format(full_id, page_2), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_failed_merge_after_delete_3(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + dboid = node.safe_psql( + "postgres", + "select oid from pg_database where datname = 'testdb'").rstrip() + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # drop database + node.safe_psql( + 'postgres', + 'DROP DATABASE testdb') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb') + + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + + gdb.set_breakpoint('pgFileDelete') + gdb.continue_execution_until_break(20) + + gdb._execute('signal SIGKILL') + + # backup half-merged + self.assertEqual( + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + db_path = os.path.join( + backup_dir, 'backups', 'node', full_id) + + # FULL backup is missing now + shutil.rmtree(db_path) + + try: + self.merge_backup( + backup_dir, 'node', page_id_2, + options=['--log-level-console=verbose']) + self.assertEqual( + 1, 0, + "Expecting Error because of missing parent.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Failed to find parent full backup for {0}".format( + page_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") + def test_merge_backup_from_future(self): + """ + take FULL backup, table PAGE backup from future, + try to merge page with FULL + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=5) + + # Take PAGE from future + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + with open( + os.path.join( + backup_dir, 'backups', 'node', + backup_id, "backup.control"), "a") as conf: + conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3))) + + # rename directory + new_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.rename( + os.path.join(backup_dir, 'backups', 'node', backup_id), + os.path.join(backup_dir, 'backups', 'node', new_id)) + + pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) + pgbench.wait() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql( + 'postgres', + 'SELECT * from pgbench_accounts') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that merged backup has the same state as + node_restored.cleanup() + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.restore_node( + backup_dir, 'node', + node_restored, backup_id=backup_id) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf( + node_restored, + {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + 'postgres', + 'SELECT * from pgbench_accounts') + + self.assertTrue(result, result_new) + + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_merge_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa2 and FULL to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEb2, PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change FULLa to OK + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa3 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2, PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEa3, PAGEa2 and FULLa status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + self.merge_backup( + backup_dir, 'node', page_id_a2, + options=['--merge-expired', '--log-level-console=log']) + + try: + self.merge_backup( + backup_dir, 'node', page_id_a3, + options=['--merge-expired', '--log-level-console=log']) + self.assertEqual( + 1, 0, + "Expecting Error because of parent FULL backup is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "ERROR: Failed to find parent full backup for {0}".format( + page_id_a3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_smart_merge(self): + """ + make node, create database, take full backup, drop database, + take PAGE backup and merge it into FULL, + make sure that files from dropped database are not + copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + # merge PAGE backup + self.merge_backup( + backup_dir, 'node', page_id, + options=['--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + def test_idempotent_merge(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + full_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb2') + + page_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + gdb = self.merge_backup( + backup_dir, 'node', page_id_2, + gdb=True, options=['--log-level-console=verbose']) + + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + gdb.remove_all_breakpoints() + + gdb.set_breakpoint('rename') + gdb.continue_execution_until_break() + gdb.continue_execution_until_break(2) + + gdb._execute('signal SIGKILL') + + show_backups = self.show_pb(backup_dir, "node") + self.assertEqual(len(show_backups), 1) + + self.assertEqual( + 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + full_id, self.show_pb(backup_dir, 'node')[0]['id']) + + self.merge_backup(backup_dir, 'node', page_id_2) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) + + def test_merge_correct_inheritance(self): + """ + Make sure that backup metainformation fields + 'note' and 'expire-time' are correctly inherited + during merge + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.set_backup( + backup_dir, 'node', page_id, options=['--note=hello', '--ttl=20d']) + + page_meta = self.show_pb(backup_dir, 'node', page_id) + + self.merge_backup(backup_dir, 'node', page_id) + + print(self.show_pb(backup_dir, 'node', page_id)) + + self.assertEqual( + page_meta['note'], + self.show_pb(backup_dir, 'node', page_id)['note']) + + self.assertEqual( + page_meta['expire-time'], + self.show_pb(backup_dir, 'node', page_id)['expire-time']) + + def test_merge_correct_inheritance_1(self): + """ + Make sure that backup metainformation fields + 'note' and 'expire-time' are correctly inherited + during merge + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # add database + node.safe_psql( + 'postgres', + 'CREATE DATABASE testdb') + + # take FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note=hello', '--ttl=20d']) + + # create database + node.safe_psql( + 'postgres', + 'create DATABASE testdb1') + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.merge_backup(backup_dir, 'node', page_id) + + self.assertNotIn( + 'note', + self.show_pb(backup_dir, 'node', page_id)) + + self.assertNotIn( + 'expire-time', + self.show_pb(backup_dir, 'node', page_id)) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multi_timeline_merge(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t12 /---P--> + ... + t3 /----> + t2 /----> + t1 -F-----D-> + + P must have F as parent + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql("postgres", "create extension pageinspect") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + node.pgbench_init(scale=20) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create timelines + for i in range(2, 7): + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote', + '--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + + # at this point there is i+1 timeline + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create backup at 2, 4 and 6 timeline + if i % 2 == 0: + self.backup_node(backup_dir, 'node', node, backup_type='page') + + page_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + self.merge_backup(backup_dir, 'node', page_id) + + result = node.safe_psql( + "postgres", "select * from pgbench_accounts") + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from pgbench_accounts") + + self.assertEqual(result, result_new) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node.port)]) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node_restored.port)]) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_merge_page_header_map_retry(self): + """ + page header map cannot be trusted when + running retry + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=20) + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + delta_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + gdb = self.merge_backup(backup_dir, 'node', delta_id, gdb=True) + + # our goal here is to get full backup with merged data files, + # but with old page header map + gdb.set_breakpoint('cleanup_header_map') + gdb.run_until_break() + gdb._execute('signal SIGKILL') + + self.merge_backup(backup_dir, 'node', delta_id) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_missing_data_file(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Change data + pgbench = node.pgbench(options=['-T', '5', '-c', '1']) + pgbench.wait() + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', path) + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), + logfile_content) + + # @unittest.skip("skip") + def test_missing_non_data_file(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', 'backup_label') + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: File "{0}" is not found'.format(file_to_remove), + logfile_content) + + self.assertIn( + 'ERROR: Backup files merging failed', + logfile_content) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # @unittest.skip("skip") + def test_merge_remote_mode(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + self.set_config(backup_dir, 'node', options=['--retention-window=1']) + + backups = os.path.join(backup_dir, 'backups', 'node') + with open( + os.path.join( + backups, full_id, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=5))) + + gdb = self.backup_node( + backup_dir, "node", node, + options=['--log-level-file=VERBOSE', '--merge-expired'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + + with open(logfile, "w+") as f: + f.truncate() + + gdb.continue_execution_until_exit() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertNotIn( + 'SSH', logfile_content) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + def test_merge_pg_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + +# 1. Need new test with corrupted FULL backup +# 2. different compression levels diff --git a/tests/option_test.py b/tests/option_test.py new file mode 100644 index 000000000..eec1bab44 --- /dev/null +++ b/tests/option_test.py @@ -0,0 +1,231 @@ +import unittest +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import locale + + +class OptionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_help_1(self): + """help options""" + with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + + # @unittest.skip("skip") + def test_version_2(self): + """help options""" + with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: + self.assertIn( + version_out.read().decode("utf-8").strip(), + self.run_pb(["--version"]) + ) + + # @unittest.skip("skip") + def test_without_backup_path_3(self): + """backup command failure without backup mode option""" + try: + self.run_pb(["backup", "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_options_4(self): + """check options test""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # backup command failure without instance option + try: + self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) + self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: required parameter not specified: --instance', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure without backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) + self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # backup command failure with invalid backup mode option + try: + self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) + self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: invalid backup-mode "bad"', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # delete failure without delete options + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: You must specify at least one of the delete options: ' + '--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + + # delete failure without ID + try: + self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "option requires an argument -- 'i'", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_options_5(self): + """check options test""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + + output = self.init_pb(backup_dir) + self.assertIn( + "INFO: Backup catalog", + output) + + self.assertIn( + "successfully inited", + output) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # syntax error in pg_probackup.conf + conf_file = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") + with open(conf_file, "a") as conf: + conf.write(" = INFINITE\n") + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Syntax error in " = INFINITE', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid value in pg_probackup.conf + with open(conf_file, "a") as conf: + conf.write("BACKUP_MODE=\n") + + try: + self.backup_node(backup_dir, 'node', node, backup_type=None), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid option "BACKUP_MODE" in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # Command line parameters should override file values + with open(conf_file, "a") as conf: + conf.write("retention-redundancy=1\n") + + self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + + # User cannot send --system-identifier parameter via command line + try: + self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Option system-identifier cannot be specified in command line', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # invalid value in pg_probackup.conf + with open(conf_file, "a") as conf: + conf.write("SMOOTH_CHECKPOINT=FOO\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.clean_pb(backup_dir) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # invalid option in pg_probackup.conf + with open(conf_file, "a") as conf: + conf.write("TIMELINEID=1\n") + + try: + self.backup_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid option "TIMELINEID" in file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_help_6(self): + """help options""" + if ProbackupTest.enable_nls: + self.test_env['LC_ALL'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + else: + self.skipTest( + 'You need configure PostgreSQL with --enabled-nls option for this test') diff --git a/tests/page_test.py b/tests/page_test.py new file mode 100644 index 000000000..e77e5c827 --- /dev/null +++ b/tests/page_test.py @@ -0,0 +1,1424 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from testgres import QueryException +from datetime import datetime, timedelta +import subprocess +import gzip +import shutil + +class PageTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_page_vacuum_truncate(self): + """ + make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take page backup, take second page backup, + restore last page backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '300s'}) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node_restored.cleanup() + node.slow_start() + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + # TODO: make it dynamic + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Logical comparison + result1 = node.safe_psql( + "postgres", + "select * from t_heap") + + result2 = node_restored.safe_psql( + "postgres", + "select * from t_heap") + + self.assertEqual(result1, result2) + + # @unittest.skip("skip") + def test_page_vacuum_truncate_1(self): + """ + make node, create table, take full backup, + delete all data, vacuum relation, + take page backup, insert some data, + take second page backup and check data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_page_stream(self): + """ + make archive node, take full and page stream backups, + restore them and check data correctness + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=['--stream']) + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream', '-j', '4']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + + node.slow_start() + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(self.output), self.cmd)) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # @unittest.skip("skip") + def test_page_archive(self): + """ + make archive node, take full and page archive backups, + restore them and check data correctness + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(i::text)::tsvector as tsvector " + "from generate_series(100, 200) i") + page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn("INFO: Restore of backup {0} completed.".format( + full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check page backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(page_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=page_backup_id, + options=[ + "-j", "4", + "--immediate", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + self.assertEqual(page_result, page_result_new) + node.cleanup() + + # @unittest.skip("skip") + def test_page_multiple_segments(self): + """ + Make node, create table with multiple segments, + write some data to it, check page and data correctness + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'fsync': 'off', + 'shared_buffers': '1GB', + 'maintenance_work_mem': '1GB', + 'full_page_writes': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # PGBENCH STUFF + pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") + # PAGE BACKUP + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", + "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + pgdata_restored = self.pgdata_content(restored_node.data_dir) + + # START RESTORED NODE + self.set_auto_conf(restored_node, {'port': restored_node.port}) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select count(*) from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_page_delete(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + # FULL backup + self.backup_node(backup_dir, 'node', node) + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i") + + node.safe_psql( + "postgres", + "delete from t_heap") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # @unittest.skip("skip") + def test_page_delete_1(self): + """ + Make node, create tablespace with table, take full backup, + delete everything from table, vacuum table, take page backup, + restore page backup, compare . + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i" + ) + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "delete from t_heap" + ) + + node.safe_psql( + "postgres", + "vacuum t_heap" + ) + + # PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='page') + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + ) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata')) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + def test_parallel_pagemap(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + "hot_standby": "on" + } + ) + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node_restored.cleanup() + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + with node.connect() as conn: + conn.execute("create table test (id int)") + for x in range(0, 8): + conn.execute( + "insert into test select i from generate_series(1,100) s(i)") + conn.commit() + self.switch_wal_segment(conn) + count1 = conn.execute("select count(*) from test") + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore it + self.restore_node(backup_dir, 'node', node_restored) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # Check restored node + count2 = node_restored.execute("postgres", "select count(*) from test") + + self.assertEqual(count1, count2) + + # Clean after yourself + node.cleanup() + node_restored.cleanup() + + def test_parallel_pagemap_1(self): + """ + Test for parallel WAL segments reading, during which pagemap is built + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + # Initialize instance and backup directory + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={} + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Do full backup + self.backup_node(backup_dir, 'node', node) + show_backup = self.show_pb(backup_dir, 'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Fill instance with data and make several WAL segments ... + node.pgbench_init(scale=10) + + # do page backup in single thread + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + self.delete_pb(backup_dir, 'node', page_id) + + # ... and do page backup with parallel pagemap + self.backup_node( + backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.show_pb(backup_dir, 'node')[1] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + # Drop node and restore it + node.cleanup() + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + # Clean after yourself + node.cleanup() + + # @unittest.skip("skip") + def test_page_backup_with_lost_wal_segment(self): + """ + make node with archiving + make archive backup, then generate some wals with pgbench, + delete latest archived wal segment + run page backup, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=3) + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] + wals = map(str, wals) + file = os.path.join(wals_dir, max(wals)) + os.remove(file) + if self.archive_compress: + file = file[:-3] + + # Single-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Could not read WAL record at' in e.message and + 'is absent' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Multi-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Could not read WAL record at' in e.message and + 'is absent' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # @unittest.skip("skip") + def test_page_backup_with_corrupted_wal_segment(self): + """ + make node with archiving + make archive backup, then generate some wals with pgbench, + corrupt latest archived wal segment + run page backup, expecting error because of missing wal segment + make sure that backup status is 'ERROR' + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=10) + + # delete last wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + # file = os.path.join(wals_dir, max(wals)) + + if self.archive_compress: + original_file = os.path.join(wals_dir, '000000010000000000000004.gz') + tmp_file = os.path.join(backup_dir, '000000010000000000000004') + + with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + # drop healthy file + os.remove(original_file) + file = tmp_file + + else: + file = os.path.join(wals_dir, '000000010000000000000004') + + # corrupt file + print(file) + with open(file, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + if self.archive_compress: + # compress corrupted file and replace with it old file + with open(file, 'rb') as f_in, gzip.open(original_file, 'wb', compresslevel=1) as f_out: + shutil.copyfileobj(f_in, f_out) + + file = os.path.join(wals_dir, '000000010000000000000004.gz') + + #if self.archive_compress: + # file = file[:-3] + + # Single-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Could not read WAL record at' in e.message and + 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Multi-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Could not read WAL record at' in e.message and + 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( + file) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # @unittest.skip("skip") + def test_page_backup_with_alien_wal_segment(self): + """ + make two nodes with archiving + take archive full backup from both nodes, + generate some wals with pgbench on both nodes, + move latest archived wal segment from second node to first node`s archive + run page backup on first node + expecting error because of alien wal segment + make sure that backup status is 'ERROR' + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + alien_node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'alien_node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.add_instance(backup_dir, 'alien_node', alien_node) + self.set_archiving(backup_dir, 'alien_node', alien_node) + alien_node.slow_start() + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + self.backup_node( + backup_dir, 'alien_node', alien_node, options=['--stream']) + + # make some wals + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i;") + + alien_node.safe_psql( + "postgres", + "create database alien") + + alien_node.safe_psql( + "alien", + "create sequence t_seq; " + "create table t_heap_alien as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,100000) i;") + + # copy latest wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( + wals_dir, f)) and not f.endswith('.backup')] + wals = map(str, wals) + filename = max(wals) + file = os.path.join(wals_dir, filename) + file_destination = os.path.join( + os.path.join(backup_dir, 'wal', 'node'), filename) +# file = os.path.join(wals_dir, '000000010000000000000004') + print(file) + print(file_destination) + os.remove(file_destination) + os.rename(file, file_destination) + + # Single-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page') + self.assertEqual( + 1, 0, + "Expecting Error because of alien wal segment.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'Could not read WAL record at' in e.message and + 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # Multi-thread PAGE backup + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of alien wal segment.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn('Could not read WAL record at', e.message) + self.assertIn('WAL file is from different database system: ' + 'WAL file database system identifier is', e.message) + self.assertIn('pg_control database system identifier is', e.message) + self.assertIn('Possible WAL corruption. Error has occured ' + 'during reading WAL segment', e.message) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup {0} should have STATUS "ERROR"') + + # @unittest.skip("skip") + def test_multithread_page_backup_with_toast(self): + """ + make node, create toast, do multithread PAGE backup + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + # make some wals + node.safe_psql( + "postgres", + "create table t3 as select i, " + "repeat(md5(i::text),5006056) as fat_attr " + "from generate_series(0,70) i") + + # Multi-thread PAGE backup + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=["-j", "4"]) + + # @unittest.skip("skip") + def test_page_create_db(self): + """ + Make node, take full backup, create database db1, take page backup, + restore database and check it presense + """ + self.maxDiff = None + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB', + 'checkpoint_timeout': '5min', + } + ) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + self.backup_node( + backup_dir, 'node', node) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,1000) i") + + # PAGE BACKUP + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + node_restored.safe_psql('db1', 'select 1') + node_restored.cleanup() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PAGE BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PAGE BACKUP + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"] + ) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd) + ) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multi_timeline_page(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t12 /---P--> + ... + t3 /----> + t2 /----> + t1 -F-----D-> + + P must have F as parent + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql("postgres", "create extension pageinspect") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + node.pgbench_init(scale=20) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=full_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create timelines + for i in range(2, 7): + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote', + '--recovery-target-timeline={0}'.format(i)]) + node.slow_start() + + # at this point there is i+1 timeline + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # create backup at 2, 4 and 6 timeline + if i % 2 == 0: + self.backup_node(backup_dir, 'node', node, backup_type='page') + + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-file=VERBOSE']) + + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql( + "postgres", "select * from pgbench_accounts") + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + result_new = node_restored.safe_psql( + "postgres", "select * from pgbench_accounts") + + self.assertEqual(result, result_new) + + self.compare_pgdata(pgdata, pgdata_restored) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node.port)]) + + self.checkdb_node( + backup_dir, + 'node', + options=[ + '--amcheck', + '-d', 'postgres', '-p', str(node_restored.port)]) + + backup_list = self.show_pb(backup_dir, 'node') + + self.assertEqual( + backup_list[2]['parent-backup-id'], + backup_list[0]['id']) + self.assertEqual(backup_list[2]['current-tli'], 3) + + self.assertEqual( + backup_list[3]['parent-backup-id'], + backup_list[2]['id']) + self.assertEqual(backup_list[3]['current-tli'], 5) + + self.assertEqual( + backup_list[4]['parent-backup-id'], + backup_list[3]['id']) + self.assertEqual(backup_list[4]['current-tli'], 7) + + self.assertEqual( + backup_list[5]['parent-backup-id'], + backup_list[4]['id']) + self.assertEqual(backup_list[5]['current-tli'], 7) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_multitimeline_page_1(self): + """ + Check that backup in PAGE mode choose + parent backup correctly: + t2 /----> + t1 -F--P---D-> + + P must have F as parent + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql("postgres", "create extension pageinspect") + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + node.pgbench_init(scale=20) + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '20', '-c', '1']) + pgbench.wait() + + page1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + page1 = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page1, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench.wait() + + print(self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--log-level-console=LOG'], return_id=False)) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + self.compare_pgdata(pgdata, pgdata_restored) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_page_pg_resetxlog(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + self.switch_wal_segment(node) + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='page', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='page') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'Insert error message', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/pgpro2068_test.py b/tests/pgpro2068_test.py new file mode 100644 index 000000000..434ce2800 --- /dev/null +++ b/tests/pgpro2068_test.py @@ -0,0 +1,188 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from time import sleep +import shutil +import signal +from testgres import ProcessType + + +class BugTest(ProbackupTest, unittest.TestCase): + + def test_minrecpoint_on_replica(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-2068 + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + # 'checkpoint_timeout': '60min', + 'checkpoint_completion_target': '0.9', + 'bgwriter_delay': '10ms', + 'bgwriter_lru_maxpages': '1000', + 'bgwriter_lru_multiplier': '4.0', + 'max_wal_size': '256MB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take full backup and restore it as replica + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # start replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'node', replica, options=['-R']) + self.set_replica(node, replica) + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + self.set_auto_conf( + replica, + {'port': replica.port, 'restart_after_crash': 'off'}) + + # we need those later + node.safe_psql( + "postgres", + "CREATE EXTENSION plpython3u") + + node.safe_psql( + "postgres", + "CREATE EXTENSION pageinspect") + + replica.slow_start(replica=True) + + # generate some data + node.pgbench_init(scale=10) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "20"]) + pgbench.wait() + pgbench.stdout.close() + + # generate some more data and leave it in background + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-j 4", "-T", "100"]) + + # wait for shared buffer on replica to be filled with dirty data + sleep(20) + + # get pids of replica background workers + startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] + checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] + bgwriter_pid = replica.auxiliary_pids[ProcessType.BackgroundWriter][0] + + # break checkpointer on UpdateLastRemovedPtr + gdb_checkpointer = self.gdb_attach(checkpointer_pid) + gdb_checkpointer._execute('handle SIGINT noprint nostop pass') + gdb_checkpointer._execute('handle SIGUSR1 noprint nostop pass') + gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') + gdb_checkpointer.continue_execution_until_break() + + # break recovery on UpdateControlFile + gdb_recovery = self.gdb_attach(startup_pid) + gdb_recovery._execute('handle SIGINT noprint nostop pass') + gdb_recovery._execute('handle SIGUSR1 noprint nostop pass') + gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') + gdb_recovery.continue_execution_until_break() + gdb_recovery.set_breakpoint('UpdateControlFile') + gdb_recovery.continue_execution_until_break() + + # stop data generation + pgbench.wait() + pgbench.stdout.close() + + # kill someone, we need a crash + os.kill(int(bgwriter_pid), 9) + gdb_recovery._execute('detach') + gdb_checkpointer._execute('detach') + + # just to be sure + try: + replica.stop(['-m', 'immediate', '-D', replica.data_dir]) + except: + pass + + # MinRecLSN = replica.get_control_data()['Minimum recovery ending location'] + + # Promote replica with 'immediate' target action + if self.get_version(replica) >= self.version_to_num('12.0'): + recovery_config = 'postgresql.auto.conf' + else: + recovery_config = 'recovery.conf' + + replica.append_conf( + recovery_config, "recovery_target = 'immediate'") + replica.append_conf( + recovery_config, "recovery_target_action = 'pause'") + replica.slow_start(replica=True) + + if self.get_version(node) < 100000: + script = ''' +DO +$$ +relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") +current_xlog_lsn = plpy.execute("SELECT min_recovery_end_location as lsn FROM pg_control_recovery()")[0]['lsn'] +plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) +found_corruption = False +for relation in relations: + pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) + + if pages_from_future.nrows() == 0: + continue + + for page in pages_from_future: + plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) + found_corruption = True +if found_corruption: + plpy.error('Found Corruption') +$$ LANGUAGE plpython3u; +''' + else: + script = ''' +DO +$$ +relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") +current_xlog_lsn = plpy.execute("select pg_last_wal_replay_lsn() as lsn")[0]['lsn'] +plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) +found_corruption = False +for relation in relations: + pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) + + if pages_from_future.nrows() == 0: + continue + + for page in pages_from_future: + plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) + found_corruption = True +if found_corruption: + plpy.error('Found Corruption') +$$ LANGUAGE plpython3u; +''' + + # Find blocks from future + replica.safe_psql( + 'postgres', + script) + + # error is expected if version < 10.6 + # gdb_backup.continue_execution_until_exit() + + # do basebackup + + # do pg_probackup, expect error diff --git a/tests/pgpro560_test.py b/tests/pgpro560_test.py new file mode 100644 index 000000000..b665fd200 --- /dev/null +++ b/tests/pgpro560_test.py @@ -0,0 +1,123 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +import subprocess +from time import sleep + + +class CheckSystemID(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro560_control_file_loss(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node with stream support, delete control file + make backup + check that backup failed + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') + # Not delete this file permanently + os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) + + try: + self.backup_node(backup_dir, 'node', node, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because pg_control was deleted.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Could not open file' in e.message and + 'pg_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Return this file to avoid Postger fail + os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) + + def test_pgpro560_systemid_mismatch(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-560 + make node1 and node2 + feed to backup PGDATA from node1 and PGPORT from node2 + check that backup failed + """ + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + node1.slow_start() + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), + set_replication=True, + initdb_params=['--data-checksums']) + + node2.slow_start() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + try: + self.backup_node(backup_dir, 'node1', node2, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of SYSTEM ID mismatch.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + if self.get_version(node1) > 90600: + self.assertTrue( + 'ERROR: Backup data directory was ' + 'initialized for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'ERROR: System identifier mismatch. ' + 'Connected PostgreSQL instance has system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + sleep(1) + + try: + self.backup_node( + backup_dir, 'node1', node2, + data_dir=node1.data_dir, options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of of SYSTEM ID mismatch.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + if self.get_version(node1) > 90600: + self.assertTrue( + 'ERROR: Backup data directory was initialized ' + 'for system id' in e.message and + 'but connected instance system id is' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + 'ERROR: System identifier mismatch. ' + 'Connected PostgreSQL instance has system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) diff --git a/tests/pgpro589_test.py b/tests/pgpro589_test.py new file mode 100644 index 000000000..8ce8e1f56 --- /dev/null +++ b/tests/pgpro589_test.py @@ -0,0 +1,72 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess + + +class ArchiveCheck(ProbackupTest, unittest.TestCase): + + def test_pgpro589(self): + """ + https://jira.postgrespro.ru/browse/PGPRO-589 + make node without archive support, make backup which should fail + check that backup status equal to ERROR + check that no files where copied to backup catalogue + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + # make erroneous archive_command + self.set_auto_conf(node, {'archive_command': 'exit 0'}) + node.slow_start() + + node.pgbench_init(scale=5) + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "10"] + ) + pgbench.wait() + pgbench.stdout.close() + path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( + "utf-8") + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing archive wal " + "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Wait for WAL segment' in e.message and + 'ERROR: WAL segment' in e.message and + 'could not be archived in 10 seconds' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.assertEqual( + 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup should have ERROR status') + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', path) + self.assertFalse( + os.path.isfile(file), + "\n Start LSN was not found in archive but datafiles where " + "copied to backup catalogue.\n For example: {0}\n " + "It is not optimal".format(file)) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py new file mode 100644 index 000000000..6e5786f8c --- /dev/null +++ b/tests/ptrack_test.py @@ -0,0 +1,4407 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +from testgres import QueryException, StartNodeException +import shutil +import sys +from time import sleep +from threading import Thread + + +class PtrackTest(ProbackupTest, unittest.TestCase): + def setUp(self): + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + self.fname = self.id().split('.')[3] + + # @unittest.skip("skip") + def test_drop_rel_during_backup_ptrack(self): + """ + drop relation during ptrack backup + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PTRACK backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + gdb=True, options=['--log-level-file=LOG']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_without_full(self): + """ptrack backup without validated full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # @unittest.skip("skip") + def test_ptrack_stop_pg(self): + """ + create node, take full backup, + restart node, check that ptrack backup + can be taken + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.stop() + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup(self): + """ + t2 /------P2 + t1 ------F---*-----P1 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + sleep(15) + + xid = node.safe_psql( + 'postgres', + 'SELECT txid_current()').decode('utf-8').rstrip() + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + node.cleanup() + + # Restore from full backup to create Timeline 2 + print(self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(xid), + '--recovery-target-action=promote'])) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup_1(self): + """ + t2 /------ + t1 ---F---P1---* + + # delete P1 + t2 /------P2 + t1 ---F--------* + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # delete old PTRACK backup + self.delete_pb(backup_dir, 'node', backup_id=ptrack_id) + + # take new PTRACK backup + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').\ + decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # @unittest.skip("skip") + def test_ptrack_eat_my_data(self): + """ + PGPRO-4051 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=50) + + self.backup_node(backup_dir, 'node', node) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) + + for i in range(10): + print("Iteration: {0}".format(i)) + + sleep(2) + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored.cleanup() +# +# self.restore_node(backup_dir, 'node', node_restored) +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# +# self.compare_pgdata(pgdata, pgdata_restored) + + pgbench.terminate() + pgbench.wait() + + self.switch_wal_segment(node) + + result = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + balance = node_restored.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql( + 'postgres', + 'SELECT * FROM pgbench_accounts'), + 'Data loss') + + # @unittest.skip("skip") + def test_ptrack_simple(self): + """make node, make full and ptrack stream backups," + " restore them and check data correctness""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream']) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # Logical comparison + self.assertEqual( + result, + node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) + + # @unittest.skip("skip") + def test_ptrack_unprivileged(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE DATABASE backupdb") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack") + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") + + node.safe_psql( + "backupdb", + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + + self.backup_node( + backup_dir, 'node', node, + datname='backupdb', options=['--stream', "-U", "backup"]) + + self.backup_node( + backup_dir, 'node', node, datname='backupdb', + backup_type='ptrack', options=['--stream', "-U", "backup"]) + + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_enable(self): + """make ptrack without full backup, should result in error""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'shared_preload_libraries': 'ptrack'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack disabled.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled\n', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_disable(self): + """ + Take full backup, disable ptrack restart postgresql, + enable ptrack, restart postgresql, take ptrack backup + which should fail + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # DISABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") + node.stop() + node.slow_start() + + # ENABLE PTRACK + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") + node.stop() + node.slow_start() + + # PTRACK BACKUP + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack_enable was set to OFF at some" + " point after previous backup.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd + ) + ) + + # @unittest.skip("skip") + def test_ptrack_uncommitted_xact(self): + """make ptrack backup while there is uncommitted open transaction""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'wal_level': 'replica'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + con = node.connect("postgres") + con.execute( + "create table t_heap as select i" + " as id from generate_series(0,1) i") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + node_restored.data_dir, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # Physical comparison + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_vacuum_full(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i" + " as id from generate_series(0,1000000) i" + ) + + pg_connect = node.connect("postgres", autocommit=True) + + gdb = self.gdb_attach(pg_connect.pid) + gdb.set_breakpoint('reform_and_rewrite_tuple') + + gdb.continue_execution_until_running() + + process = Thread( + target=pg_connect.execute, args=["VACUUM FULL t_heap"]) + process.start() + + while not gdb.stopped_in_breakpoint: + sleep(1) + + gdb.continue_execution_until_break(20) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + gdb.remove_all_breakpoints() + gdb._execute('detach') + process.join() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # @unittest.skip("skip") + def test_ptrack_vacuum_truncate(self): + """make node, create table, take full backup, + delete last 3 pages, vacuum relation, + take ptrack backup, take second ptrack backup, + restore last ptrack backup and check data correctness""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql( + "postgres", + "create sequence t_seq; " + "create table t_heap tablespace somedata as select i as id, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1024) i;") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + "postgres", + "delete from t_heap where ctid >= '(11,0)'") + + node.safe_psql( + "postgres", + "vacuum t_heap") + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + old_tablespace = self.get_tblspace_path(node, 'somedata') + new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') + + self.restore_node( + backup_dir, 'node', node_restored, + options=["-j", "4", "-T", "{0}={1}".format( + old_tablespace, new_tablespace)] + ) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, + ignore_ptrack=False + ) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + # @unittest.skip("skip") + def test_ptrack_get_block(self): + """ + make node, make full and ptrack stream backups, + restore them and check data correctness + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,1) i") + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream'], + gdb=True) + + gdb.set_breakpoint('make_pagemap_from_ptrack_2') + gdb.run_until_break() + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + gdb.continue_execution_until_exit() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + result = node.safe_psql("postgres", "SELECT * FROM t_heap") + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + # Logical comparison + self.assertEqual( + result, + node.safe_psql("postgres", "SELECT * FROM t_heap")) + + # @unittest.skip("skip") + def test_ptrack_stream(self): + """make node, make full and ptrack stream backups, + restore them and check data correctness""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + node.safe_psql("postgres", "create sequence t_seq") + node.safe_psql( + "postgres", + "create table t_heap as select i as id, nextval('t_seq')" + " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" + " as tsvector from generate_series(0,100) i") + + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, nextval('t_seq') as t_seq," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i") + + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Drop Node + node.cleanup() + + # Restore and check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, options=["-j", "4"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Restore and check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, options=["-j", "4"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + # @unittest.skip("skip") + def test_ptrack_archive(self): + """make archive node, make full and ptrack backups, + check data correctness in restored instance""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as" + " select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i") + + full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_backup_id = self.backup_node(backup_dir, 'node', node) + full_target_time = self.show_pb( + backup_dir, 'node', full_backup_id)['recovery-time'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i") + + ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack') + ptrack_target_time = self.show_pb( + backup_dir, 'node', ptrack_backup_id)['recovery-time'] + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(200, 300) i") + + # Drop Node + node.cleanup() + + # Check full backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(full_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=full_backup_id, + options=[ + "-j", "4", "--recovery-target-action=promote", + "--time={0}".format(full_target_time)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + node.slow_start() + + full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(full_result, full_result_new) + node.cleanup() + + # Check ptrack backup + self.assertIn( + "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), + self.restore_node( + backup_dir, 'node', node, + backup_id=ptrack_backup_id, + options=[ + "-j", "4", + "--time={0}".format(ptrack_target_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd) + ) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(ptrack_result, ptrack_result_new) + + node.cleanup() + + @unittest.skip("skip") + def test_ptrack_pgpro417(self): + """ + Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail. Actual only for PTRACK 1.x + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql( + "postgres", + "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='full', options=["--stream"]) + + start_lsn_full = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + start_lsn_ptrack = self.show_pb( + backup_dir, 'node', backup_id)['start-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + # SECOND PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n" + " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + @unittest.skip("skip") + def test_page_pgpro417(self): + """ + Make archive node, take full backup, take page backup, + delete page backup. Try to take ptrack backup, which should fail. + Actual only for PTRACK 1.x + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + + # PAGE BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(100,200) i") + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.delete_pb(backup_dir, 'node', backup_id) +# sys.exit(1) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + + try: + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0}\n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + @unittest.skip("skip") + def test_full_pgpro417(self): + """ + Make node, take two full backups, delete full second backup. + Try to take ptrack backup, which should fail. + Relevant only for PTRACK 1.x + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector " + " from generate_series(0,100) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # SECOND FULL BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(100,200) i" + ) + node.safe_psql("postgres", "SELECT * FROM t_heap") + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + + self.delete_pb(backup_dir, 'node', backup_id) + + # PTRACK BACKUP + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(200,300) i") + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of LSN mismatch from ptrack_control " + "and previous backup start_lsn.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + "ERROR: LSN from ptrack_control" in e.message and + "Create new full backup before " + "an incremental one" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_create_db(self): + """ + Make node, take full backup, create database db1, take ptrack backup, + restore database and check it presense + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '10GB'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + node.safe_psql("postgres", "SELECT * FROM t_heap") + self.backup_node( + backup_dir, 'node', node, + options=["--stream"]) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + # DROP DATABASE DB1 + node.safe_psql( + "postgres", "drop database db1") + # SECOND PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"] + ) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE SECOND PTRACK BACKUP + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + node_restored.slow_start() + + try: + node_restored.safe_psql('db1', 'select 1') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are connecting to deleted database" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertTrue( + 'FATAL: database "db1" does not exist' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_create_db_on_replica(self): + """ + Make node, take full backup, create replica from it, + take full backup from replica, + create database db1, take ptrack backup from replica, + restore database and check it presense + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + self.restore_node(backup_dir, 'node', replica) + + # Add replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(node, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(node.port), + '--stream' + ] + ) + + # CREATE DATABASE DB1 + node.safe_psql("postgres", "create database db1") + node.safe_psql( + "db1", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + # Wait until replica catch up with master + self.wait_until_replica_catch_with_master(node, replica) + replica.safe_psql('postgres', 'checkpoint') + + # PTRACK BACKUP + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='ptrack', + options=[ + '-j10', + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(node.port) + ] + ) + + if self.paranoia: + pgdata = self.pgdata_content(replica.data_dir) + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'replica', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + # COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_alter_table_set_tablespace_ptrack(self): + """Make node, create tablespace with table, take full backup, + alter tablespace location, take ptrack backup, restore database.""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + self.create_tblspace_in_node(node, 'somedata') + node.safe_psql( + "postgres", + "create table t_heap tablespace somedata as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,100) i") + # FULL backup + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # ALTER TABLESPACE + self.create_tblspace_in_node(node, 'somedata_new') + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata_new") + + # sys.exit(1) + # PTRACK BACKUP + #result = node.safe_psql( + # "postgres", "select * from t_heap") + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', + options=["--stream"] + ) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + # node.stop() + # node.cleanup() + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata') + ), + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata_new'), + self.get_tblspace_path(node_restored, 'somedata_new') + ) + ] + ) + + # GET RESTORED PGDATA AND COMPARE + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + node_restored.slow_start() + +# result_new = node_restored.safe_psql( +# "postgres", "select * from t_heap") +# +# self.assertEqual(result, result_new, 'lost some data after restore') + + # @unittest.skip("skip") + def test_alter_database_set_tablespace_ptrack(self): + """Make node, create tablespace with database," + " take full backup, alter tablespace location," + " take ptrack backup, restore database.""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # CREATE TABLESPACE + self.create_tblspace_in_node(node, 'somedata') + + # ALTER DATABASE + node.safe_psql( + "template1", + "alter database postgres set tablespace somedata") + + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + node.stop() + + # RESTORE + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + self.restore_node( + backup_dir, 'node', + node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + self.get_tblspace_path(node, 'somedata'), + self.get_tblspace_path(node_restored, 'somedata'))]) + + # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + node_restored.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + node_restored.port = node.port + node_restored.slow_start() + + # @unittest.skip("skip") + def test_drop_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to tablespace 'somedata' + node.safe_psql( + "postgres", "alter table t_heap set tablespace somedata") + # PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # Move table back to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + # DROP TABLESPACE 'somedata' + node.safe_psql( + "postgres", "drop tablespace somedata") + # THIRD PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=["--stream"]) + + if self.paranoia: + pgdata = self.pgdata_content( + node.data_dir, ignore_ptrack=True) + + tblspace = self.get_tblspace_path(node, 'somedata') + node.cleanup() + shutil.rmtree(tblspace, ignore_errors=True) + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content( + node.data_dir, ignore_ptrack=True) + + node.slow_start() + + tblspc_exist = node.safe_psql( + "postgres", + "select exists(select 1 from " + "pg_tablespace where spcname = 'somedata')") + + if tblspc_exist.rstrip() == 't': + self.assertEqual( + 1, 0, + "Expecting Error because " + "tablespace 'somedata' should not be present") + + result_new = node.safe_psql("postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_alter_tablespace(self): + """ + Make node, create table, alter table tablespace, take ptrack backup, + move table from tablespace, take ptrack backup + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + tblspc_path = self.get_tblspace_path(node, 'somedata') + + # CREATE TABLE + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") + + result = node.safe_psql("postgres", "select * from t_heap") + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # Move table to separate tablespace + node.safe_psql( + "postgres", + "alter table t_heap set tablespace somedata") + # GET LOGICAL CONTENT FROM NODE + result = node.safe_psql("postgres", "select * from t_heap") + + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream"]) + + # GET PHYSICAL CONTENT FROM NODE + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore ptrack backup + restored_node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node.cleanup() + tblspc_path_new = self.get_tblspace_path( + restored_node, 'somedata_restored') + self.restore_node(backup_dir, 'node', restored_node, options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf( + restored_node, {'port': restored_node.port}) + restored_node.slow_start() + + # COMPARE LOGICAL CONTENT + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + restored_node.cleanup() + shutil.rmtree(tblspc_path_new, ignore_errors=True) + + # Move table to default tablespace + node.safe_psql( + "postgres", "alter table t_heap set tablespace pg_default") + # SECOND PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream"]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + # Restore second ptrack backup and check table consistency + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + self.compare_pgdata(pgdata, pgdata_restored) + + # START RESTORED NODE + self.set_auto_conf( + restored_node, {'port': restored_node.port}) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", "select * from t_heap") + self.assertEqual(result, result_new) + + # @unittest.skip("skip") + def test_ptrack_multiple_segments(self): + """ + Make node, create table, alter table tablespace, + take ptrack backup, move table from tablespace, take ptrack backup + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'full_page_writes': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # CREATE TABLE + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + # FULL BACKUP + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PTRACK STUFF + if node.major_version < 11: + idx_ptrack = {'type': 'heap'} + idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') + idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') + idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], idx_ptrack['old_size']) + + pgbench = node.pgbench( + options=['-T', '30', '-c', '1', '--no-vacuum']) + pgbench.wait() + + node.safe_psql("postgres", "checkpoint") + + if node.major_version < 11: + idx_ptrack['new_size'] = self.get_fork_size( + node, + 'pgbench_accounts') + + idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack['path'], + idx_ptrack['new_size']) + + idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, + idx_ptrack['path']) + + if not self.check_ptrack_sanity(idx_ptrack): + self.assertTrue( + False, 'Ptrack has failed to register changes in data files') + + # GET LOGICAL CONTENT FROM NODE + # it`s stupid, because hint`s are ignored by ptrack + result = node.safe_psql("postgres", "select * from pgbench_accounts") + # FIRTS PTRACK BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + # GET PHYSICAL CONTENT FROM NODE + pgdata = self.pgdata_content(node.data_dir) + + # RESTORE NODE + restored_node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node.cleanup() + tblspc_path = self.get_tblspace_path(node, 'somedata') + tblspc_path_new = self.get_tblspace_path( + restored_node, + 'somedata_restored') + + self.restore_node( + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format( + tblspc_path, tblspc_path_new)]) + + # GET PHYSICAL CONTENT FROM NODE_RESTORED + if self.paranoia: + pgdata_restored = self.pgdata_content( + restored_node.data_dir, ignore_ptrack=False) + + # START RESTORED NODE + self.set_auto_conf( + restored_node, {'port': restored_node.port}) + restored_node.slow_start() + + result_new = restored_node.safe_psql( + "postgres", + "select * from pgbench_accounts") + + # COMPARE RESTORED FILES + self.assertEqual(result, result_new, 'data is lost') + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + @unittest.skip("skip") + def test_atexit_fail(self): + """ + Take backups of every available types and check that PTRACK is clean. + Relevant only for PTRACK 1.x + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_connections': '15'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=["--stream", "-j 30"]) + + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because we are opening too many connections" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertIn( + 'setting its status to ERROR', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + + self.assertEqual( + node.safe_psql( + "postgres", + "select * from pg_is_in_backup()").rstrip(), + "f") + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_clean(self): + """ + Take backups of every available types and check that PTRACK is clean + Relevant only for PTRACK 1.x + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', options=['-j10', '--stream']) + + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + node.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + node.safe_psql('postgres', 'vacuum t_heap') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-j10', '--stream']) + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(node, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + @unittest.skip("skip") + def test_ptrack_clean_replica(self): + """ + Take backups of every available types from + master and check that PTRACK on replica is clean. + Relevant only for PTRACK 1.x + """ + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), " + "text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + + # Take PTRACK backup to clean every ptrack + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # Update everything and vacuum it + master.safe_psql( + 'postgres', + "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take PAGE backup to clean every ptrack + self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='page', + options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + master.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get new size of heap and indexes and calculate it in pages + idx_ptrack[i]['size'] = self.get_fork_size(replica, i) + # update path to heap and index files in case they`ve changed + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack bits are cleaned + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_cluster_on_btree(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, nextval('t_seq') as t_seq, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + if node.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_btree') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'cluster t_heap using t_gist') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_btree_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--stream', '--master-host=localhost', + '--master-db=postgres', '--master-port={0}'.format( + master.port)]) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'cluster t_heap using t_btree') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if master.major_version < 11: + self.check_ptrack_map_sanity(replica, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_cluster_on_gist_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "nextval('t_seq') as t_seq, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--stream', '--master-host=localhost', + '--master-db=postgres', '--master-port={0}'.format( + master.port)]) + + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1') + master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist') + + if master.major_version < 11: + master.safe_psql('postgres', 'CHECKPOINT') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + + if master.major_version < 11: + replica.safe_psql('postgres', 'CHECKPOINT') + self.check_ptrack_map_sanity(replica, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node) + + if self.paranoia: + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_empty(self): + """Take backups of every available types and check that PTRACK is clean""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector) " + "tablespace somedata") + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + options=['-j10', '--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'checkpoint') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + tblspace1 = self.get_tblspace_path(node, 'somedata') + tblspace2 = self.get_tblspace_path(node_restored, 'somedata') + + # Take PTRACK backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['-j10', '--stream']) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + self.restore_node( + backup_dir, 'node', node_restored, + backup_id=backup_id, + options=[ + "-j", "4", + "-T{0}={1}".format(tblspace1, tblspace2)]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_empty_replica(self): + """ + Take backups of every available types from master + and check that PTRACK on replica is clean + """ + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + replica.slow_start(replica=True) + + # Create table + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector)") + self.wait_until_replica_catch_with_master(master, replica) + + # Take FULL backup + self.backup_node( + backup_dir, + 'replica', + replica, + options=[ + '-j10', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + self.wait_until_replica_catch_with_master(master, replica) + + # Take PTRACK backup + backup_id = self.backup_node( + backup_dir, + 'replica', + replica, + backup_type='ptrack', + options=[ + '-j1', '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + if self.paranoia: + pgdata = self.pgdata_content(replica.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'replica', node_restored, + backup_id=backup_id, options=["-j", "4"]) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_truncate(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + if node.major_version < 11: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.safe_psql('postgres', 'truncate t_heap') + node.safe_psql('postgres', 'checkpoint') + + if node.major_version < 11: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + if node.major_version < 11: + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_basic_ptrack_truncate_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_wal_size': '32MB', + 'archive_timeout': '10s', + 'checkpoint_timeout': '5min'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) ".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + if replica.major_version < 11: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # Make backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + if replica.major_version < 11: + for i in idx_ptrack: + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'truncate t_heap') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + + if replica.major_version < 10: + replica.safe_psql( + "postgres", + "select pg_xlog_replay_pause()") + else: + replica.safe_psql( + "postgres", + "select pg_wal_replay_pause()") + + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '-j10', + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + + pgdata_restored = self.pgdata_content(node.data_dir) + + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # Make full backup to clean every ptrack + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + if node.major_version < 11: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) + + # @unittest.skip("skip") + def test_ptrack_vacuum_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Make FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, options=[ + '-j10', '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + if replica.major_version < 11: + for i in idx_ptrack: + # get fork size and calculate it in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums for every page of this fork + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) + self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) + + # Delete some rows, vacuum it and make checkpoint + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if replica.major_version < 11: + self.check_ptrack_map_sanity(master, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_frozen(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + node.safe_psql('postgres', 'vacuum freeze t_heap') + node.safe_psql('postgres', 'checkpoint') + + if node.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) + + # @unittest.skip("skip") + def test_ptrack_vacuum_bits_frozen_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Take backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + if replica.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'vacuum freeze t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if replica.major_version < 11: + self.check_ptrack_map_sanity(master, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + replica.cleanup() + + self.restore_node(backup_dir, 'replica', replica) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_bits_visibility(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + if node.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_2(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + pg_options={ 'wal_log_hints': 'on' }) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.create_tblspace_in_node(node, 'somedata') + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} " + "using {2}({3}) tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'vacuum t_heap') + node.safe_psql('postgres', 'checkpoint') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + if node.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + node.safe_psql('postgres', 'vacuum full t_heap') + node.safe_psql('postgres', 'checkpoint') + + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + shutil.rmtree( + self.get_tblspace_path(node, 'somedata'), + ignore_errors=True) + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_full_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " + "tsvector from generate_series(0,256000) i") + + if master.major_version < 11: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", + "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], + idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port), + '--stream']) + + if replica.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') + master.safe_psql('postgres', 'vacuum full t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'checkpoint') + + if replica.major_version < 11: + self.check_ptrack_map_sanity(master, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + replica.cleanup() + + self.restore_node(backup_dir, 'replica', replica) + + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_2(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table and indexes + res = node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + if node.major_version < 11: + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql('postgres', 'VACUUM t_heap') + + self.backup_node( + backup_dir, 'node', node, options=['-j10', '--stream']) + + if node.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + node.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128') + node.safe_psql('postgres', 'VACUUM t_heap') + node.safe_psql('postgres', 'CHECKPOINT') + + # CHECK PTRACK SANITY + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_vacuum_truncate_replica(self): + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + if master.major_version >= 11: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.restore_node(backup_dir, 'master', replica) + + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, 'replica', synchronous=True) + replica.slow_start(replica=True) + + # Create table and indexes + master.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " + "as tsvector from generate_series(0,2560) i") + + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + master.safe_psql( + "postgres", "create index {0} on {1} " + "using {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + master.safe_psql('postgres', 'vacuum t_heap') + master.safe_psql('postgres', 'checkpoint') + + # Take FULL backup to clean every ptrack + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '-j10', + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port) + ] + ) + + if master.major_version < 11: + for i in idx_ptrack: + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(replica, i) + # calculate md5sums of pages + idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( + idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) + + master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128;') + master.safe_psql('postgres', 'VACUUM t_heap') + master.safe_psql('postgres', 'CHECKPOINT') + + # Sync master and replica + self.wait_until_replica_catch_with_master(master, replica) + replica.safe_psql('postgres', 'CHECKPOINT') + + # CHECK PTRACK SANITY + if master.major_version < 11: + self.check_ptrack_map_sanity(master, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--log-level-file=INFO', + '--archive-timeout=30']) + + pgdata = self.pgdata_content(replica.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'replica', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + @unittest.skip("skip") + def test_ptrack_recovery(self): + """ + Check that ptrack map contain correct bits after recovery. + Actual only for PTRACK 1.x + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap tablespace somedata " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", "create index {0} on {1} using {2}({3}) " + "tablespace somedata".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + # get size of heap and indexes. size calculated in pages + idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) + # get path to heap and index files + idx_ptrack[i]['path'] = self.get_fork_path(node, i) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + for i in idx_ptrack: + # get ptrack for every idx + idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( + node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) + # check that ptrack has correct bits after recovery + self.check_ptrack_recovery(idx_ptrack[i]) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_recovery_1(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "CREATE INDEX {0} ON {1} USING {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + node.safe_psql( + 'postgres', + "create extension pg_buffercache") + + #print(node.safe_psql( + # 'postgres', + # "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_zero_changes(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create table t_heap " + "as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_ptrack_pg_resetxlog(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums'], + pg_options={ + 'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " +# "from generate_series(0,25600) i") + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Create indexes + for i in idx_ptrack: + if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': + node.safe_psql( + "postgres", + "CREATE INDEX {0} ON {1} USING {2}({3})".format( + i, idx_ptrack[i]['relation'], + idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + +# node.safe_psql( +# 'postgres', +# "create extension pg_buffercache") +# +# print(node.safe_psql( +# 'postgres', +# "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + # now smack it with sledgehammer + if node.major_version >= 10: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + node.data_dir, + '-o 42', + '-f' + ], + asynchronous=False) + + if not node.status(): + node.slow_start() + else: + print("Die! Die! Why won't you die?... Why won't you die?") + exit(1) + + # take ptrack backup +# self.backup_node( +# backup_dir, 'node', node, +# backup_type='ptrack', options=['--stream']) + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance was brutalized by pg_resetxlog" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd) + ) + except ProbackupException as e: + self.assertTrue( + 'ERROR: LSN from ptrack_control ' in e.message and + 'is greater than Start LSN of previous backup' in e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored = self.make_simple_node( +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) +# node_restored.cleanup() +# +# self.restore_node( +# backup_dir, 'node', node_restored) +# +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_ptrack_map(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + ptrack_version = self.get_ptrack_version(node) + + # Create table + node.safe_psql( + "postgres", + "create extension bloom; create sequence t_seq; " + "create table t_heap " + "as select nextval('t_seq')::int as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + # kill the bastard + if self.verbose: + print('Killing postmaster. Losing Ptrack changes') + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + ptrack_map = os.path.join(node.data_dir, 'global', 'ptrack.map') + + # Let`s do index corruption. ptrack.map + with open(ptrack_map, "rb+", 0) as f: + f.seek(42) + f.write(b"blablahblahs") + f.flush() + f.close + +# os.remove(os.path.join(node.logs_dir, node.pg_log_name)) + + if self.verbose: + print('Ptrack version:', ptrack_version) + if ptrack_version >= self.version_to_num("2.3"): + node.slow_start() + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'WARNING: ptrack read map: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) + + node.stop(['-D', node.data_dir]) + else: + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack.map is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + + self.assertIn( + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) + + self.set_auto_conf(node, {'ptrack.map_size': '0'}) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance ptrack is disabled" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Ptrack is disabled', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + node.stop(['-m', 'immediate', '-D', node.data_dir]) + + self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack map is from future" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: LSN from ptrack_control', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + node.safe_psql( + 'postgres', + "update t_heap set id = nextval('t_seq'), text = md5(text), " + "tsvector = md5(repeat(tsvector::text, 10))::tsvector") + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_horizon_lsn_ptrack(self): + """ + https://github.com/postgrespro/pg_probackup/pull/386 + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + self.assertLessEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num('2.4.15'), + 'You need pg_probackup old_binary =< 2.4.15 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.assertGreaterEqual( + self.get_ptrack_version(node), + self.version_to_num("2.1"), + "You need ptrack >=2.1 for this test") + + # set map_size to a minimal value + self.set_auto_conf(node, {'ptrack.map_size': '1'}) + node.restart() + + node.pgbench_init(scale=100) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node, options=['--stream'], old_binary=True) + + # enable archiving so the WAL size to do interfere with data bytes comparison later + self.set_archiving(backup_dir, 'node', node) + node.restart() + + # change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA is exemplar + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + delta_bytes = self.show_pb(backup_dir, 'node', backup_id=delta_id)["data-bytes"] + self.delete_pb(backup_dir, 'node', backup_id=delta_id) + + # PTRACK with current binary + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + ptrack_bytes = self.show_pb(backup_dir, 'node', backup_id=ptrack_id)["data-bytes"] + + # make sure that backup size is exactly the same + self.assertEqual(delta_bytes, ptrack_bytes) diff --git a/tests/remote_test.py b/tests/remote_test.py new file mode 100644 index 000000000..2d36d7346 --- /dev/null +++ b/tests/remote_test.py @@ -0,0 +1,43 @@ +import unittest +import os +from time import sleep +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.cfs_helpers import find_by_name + + +class RemoteTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_remote_sanity(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + output = self.backup_node( + backup_dir, 'node', node, + options=['--stream'], no_remote=True, return_id=False) + self.assertIn('remote: false', output) + + # try: + # self.backup_node( + # backup_dir, 'node', + # node, options=['--remote-proto=ssh', '--stream'], no_remote=True) + # # we should die here because exception is what we expect to happen + # self.assertEqual( + # 1, 0, + # "Expecting Error because remote-host option is missing." + # "\n Output: {0} \n CMD: {1}".format( + # repr(self.output), self.cmd)) + # except ProbackupException as e: + # self.assertIn( + # "Insert correct error", + # e.message, + # "\n Unexpected Error Message: {0}\n CMD: {1}".format( + # repr(e.message), self.cmd)) diff --git a/tests/replica_test.py b/tests/replica_test.py new file mode 100644 index 000000000..9c68de366 --- /dev/null +++ b/tests/replica_test.py @@ -0,0 +1,1654 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from datetime import datetime, timedelta +import subprocess +import time +from distutils.dir_util import copy_tree +from testgres import ProcessType +from time import sleep + + +class ReplicaTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_switchover(self): + """ + check that archiving on replica works correctly + over the course of several switchovers + https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + node1.slow_start() + + # take full backup and restore it + self.backup_node(backup_dir, 'node1', node1, options=['--stream']) + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + # create replica + self.restore_node(backup_dir, 'node1', node2) + + # setup replica + self.add_instance(backup_dir, 'node2', node2) + self.set_archiving(backup_dir, 'node2', node2, replica=True) + self.set_replica(node1, node2, synchronous=False) + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start(replica=True) + + # generate some data + node1.pgbench_init(scale=5) + + # take full backup on replica + self.backup_node(backup_dir, 'node2', node2, options=['--stream']) + + # first switchover + node1.stop() + node2.promote() + + self.set_replica(node2, node1, synchronous=False) + node2.reload() + node1.slow_start(replica=True) + + # take incremental backup from new master + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # second switchover + node2.stop() + node1.promote() + self.set_replica(node1, node2, synchronous=False) + node1.reload() + node2.slow_start(replica=True) + + # generate some more data + node1.pgbench_init(scale=5) + + # take incremental backup from replica + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # https://github.com/postgrespro/pg_probackup/issues/251 + self.validate_pb(backup_dir) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_stream_ptrack_backup(self): + """ + make node, take full backup, restore it and make replica from it, + take full stream backup from replica + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + if self.pg_config_version > self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + master.slow_start() + + if master.major_version >= 12: + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # CREATE TABLE + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + # take full backup and restore it + self.backup_node(backup_dir, 'master', master, options=['--stream']) + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + self.set_replica(master, replica) + + # Check data correctness on replica + replica.slow_start(replica=True) + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,512) i") + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + self.add_instance(backup_dir, 'replica', replica) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take PTRACK backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(512,768) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'replica', replica, backup_type='ptrack', + options=[ + '--stream', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PTRACK BACKUP TAKEN FROM replica + node.cleanup() + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_replica_archive_page_backup(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s', + 'max_wal_size': '32MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,2560) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + replica.slow_start(replica=True) + + # Check data correctness on replica + after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + + # Change data on master, take FULL backup from replica, + # restore taken backup and check that restored data + # equal to original data + master.psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(256,25120) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + self.wait_until_replica_catch_with_master(master, replica) + + backup_id = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=60', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE FULL BACKUP TAKEN FROM replica + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + + self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) + + node.slow_start() + + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM t_heap") + self.assertEqual(before, after) + node.cleanup() + + # Change data on master, make PAGE backup from replica, + # restore taken backup and check that restored data equal + # to original data + master.pgbench_init(scale=5) + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + backup_id = self.backup_node( + backup_dir, 'replica', + replica, backup_type='page', + options=[ + '--archive-timeout=60', + '--master-host=localhost', + '--master-db=postgres', + '--master-port={0}'.format(master.port)]) + + pgbench.wait() + + self.switch_wal_segment(master) + + before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + + self.validate_pb(backup_dir, 'replica') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + + # RESTORE PAGE BACKUP TAKEN FROM replica + self.restore_node( + backup_dir, 'replica', data_dir=node.data_dir, + backup_id=backup_id) + + self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) + + node.slow_start() + + # CHECK DATA CORRECTNESS + after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + self.assertEqual( + before, after, 'Restored data is not equal to original') + + self.add_instance(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # @unittest.skip("skip") + def test_basic_make_replica_via_restore(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,8192) i") + + before = master.safe_psql("postgres", "SELECT * FROM t_heap") + + backup_id = self.backup_node( + backup_dir, 'master', master, backup_type='page') + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_replica(master, replica, synchronous=True) + + replica.slow_start(replica=True) + + self.backup_node( + backup_dir, 'replica', replica, + options=['--archive-timeout=30s', '--stream']) + + # @unittest.skip("skip") + def test_take_backup_from_delayed_replica(self): + """ + make archive master, take full backups from master, + restore full backup as delayed replica, launch pgbench, + take FULL, PAGE and DELTA backups from replica + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '10s'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + master.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + self.wait_until_replica_catch_with_master(master, replica) + + if self.get_version(master) >= self.version_to_num('12.0'): + self.set_auto_conf( + replica, {'recovery_min_apply_delay': '300s'}) + else: + replica.append_conf( + 'recovery.conf', + 'recovery_min_apply_delay = 300s') + + replica.stop() + replica.slow_start(replica=True) + + master.pgbench_init(scale=10) + + pgbench = master.pgbench( + options=['-T', '60', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', + replica, options=['--archive-timeout=60s']) + + self.backup_node( + backup_dir, 'replica', replica, + data_dir=replica.data_dir, + backup_type='page', options=['--archive-timeout=60s']) + + sleep(1) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--archive-timeout=60s']) + + pgbench.wait() + + pgbench = master.pgbench( + options=['-T', '30', '-c', '2', '--no-vacuum']) + + self.backup_node( + backup_dir, 'replica', replica, + options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='page', options=['--stream']) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + pgbench.wait() + + # @unittest.skip("skip") + def test_replica_promote(self): + """ + start backup from replica, during backup promote replica + check that backup is failed + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '10s', + 'checkpoint_timeout': '30s', + 'max_wal_size': '32MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'master', master) + + master.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + self.restore_node( + backup_dir, 'master', replica, options=['-R']) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.set_replica( + master, replica, replica_name='replica', synchronous=True) + + replica.slow_start(replica=True) + + master.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,165000) i") + + self.wait_until_replica_catch_with_master(master, replica) + + # start backup from replica + gdb = self.backup_node( + backup_dir, 'replica', replica, gdb=True, + options=['--log-level-file=verbose']) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(20) + + replica.promote() + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + backup_id = self.show_pb( + backup_dir, 'replica')[0]["id"] + + # read log file content + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + f.close + + self.assertIn( + 'ERROR: the standby was promoted during online backup', + log_content) + + self.assertIn( + 'WARNING: Backup {0} is running, ' + 'setting its status to ERROR'.format(backup_id), + log_content) + + # @unittest.skip("skip") + def test_replica_stop_lsn_null_offset(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'node', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + output = self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + self.assertIn( + 'LOG: Invalid offset in stop_lsn value 0/4000000', + output) + + self.assertIn( + 'WARNING: WAL segment 000000010000000000000004 could not be streamed in 30 seconds', + output) + + self.assertIn( + 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', + output) + + self.assertIn( + 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', + output) + + self.assertIn( + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN 0/4000000', + output) + + self.assertIn( + 'LOG: Found prior LSN:', + output) + + # Clean after yourself + gdb_checkpointer.kill() + + # @unittest.skip("skip") + def test_replica_stop_lsn_null_offset_next_record(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + # open connection to master + conn = master.connect() + + gdb = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=40', + '--log-level-file=LOG', + '--no-validate', + '--stream'], + gdb=True) + + # Attention! this breakpoint is set to a probackup internal function, not a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb.continue_execution_until_running() + + sleep(5) + + conn.execute("create table t1()") + conn.commit() + + while 'RUNNING' in self.show_pb(backup_dir, 'replica')[0]['status']: + sleep(5) + + file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + + with open(file) as f: + log_content = f.read() + + self.assertIn( + 'LOG: Invalid offset in stop_lsn value 0/4000000', + log_content) + + self.assertIn( + 'LOG: Looking for segment: 000000010000000000000004', + log_content) + + self.assertIn( + 'LOG: First record in WAL segment "000000010000000000000004": 0/4000028', + log_content) + + self.assertIn( + 'INFO: stop_lsn: 0/4000000', + log_content) + + self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') + + # @unittest.skip("skip") + def test_archive_replica_null_offset(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) + master.slow_start() + + self.backup_node(backup_dir, 'node', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + # take backup from replica + output = self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate'], + return_id=False) + + self.assertIn( + 'LOG: Invalid offset in stop_lsn value 0/4000000', + output) + + self.assertIn( + 'WARNING: WAL segment 000000010000000000000004 could not be archived in 30 seconds', + output) + + self.assertIn( + 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', + output) + + self.assertIn( + 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', + output) + + self.assertIn( + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN 0/4000000', + output) + + self.assertIn( + 'LOG: Found prior LSN:', + output) + + print(output) + + # @unittest.skip("skip") + def test_archive_replica_not_null_offset(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master) + master.slow_start() + + self.backup_node(backup_dir, 'node', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + + replica.slow_start(replica=True) + + # take backup from replica + self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate'], + return_id=False) + + try: + self.backup_node( + backup_dir, 'node', replica, replica.data_dir, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of archive timeout. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + # vanilla -- 0/4000060 + # pgproee -- 0/4000078 + self.assertRegex( + e.message, + r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertRegex( + e.message, + r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + 'ERROR: WAL segment 000000010000000000000004 could not be archived in 30 seconds', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_replica_toast(self): + """ + make archive master, take full and page archive backups from master, + set replica, make archive backup from replica + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica', + 'shared_buffers': '128MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + self.set_archiving(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_checkpointer = self.gdb_attach(bgwriter_pid) + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'replica', replica, replica=True) + + copy_tree( + os.path.join(backup_dir, 'wal', 'master'), + os.path.join(backup_dir, 'wal', 'replica')) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + self.wait_until_replica_catch_with_master(master, replica) + + output = self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + pgdata = self.pgdata_content(replica.data_dir) + + self.assertIn( + 'WARNING: Could not read WAL record at', + output) + + self.assertIn( + 'LOG: Found prior LSN:', + output) + + res1 = replica.safe_psql( + 'postgres', + 'select md5(fat_attr) from t1') + + replica.cleanup() + + self.restore_node(backup_dir, 'replica', replica) + pgdata_restored = self.pgdata_content(replica.data_dir) + + replica.slow_start() + + res2 = replica.safe_psql( + 'postgres', + 'select md5(fat_attr) from t1') + + self.assertEqual(res1, res2) + + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + gdb_checkpointer.kill() + + # @unittest.skip("skip") + def test_start_stop_lsn_in_the_same_segno(self): + """ + """ + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica', + 'shared_buffers': '128MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + master.safe_psql( + 'postgres', + 'CHECKPOINT') + + self.wait_until_replica_catch_with_master(master, replica) + + sleep(60) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + @unittest.skip("skip") + def test_replica_promote_1(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + # set replica True, so archive_mode 'always' is used. + self.set_archiving(backup_dir, 'master', master, replica=True) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + self.wait_until_replica_catch_with_master(master, replica) + + wal_file = os.path.join( + backup_dir, 'wal', 'master', '000000010000000000000004') + + wal_file_partial = os.path.join( + backup_dir, 'wal', 'master', '000000010000000000000004.partial') + + self.assertFalse(os.path.exists(wal_file)) + + replica.promote() + + while not os.path.exists(wal_file_partial): + sleep(1) + + self.switch_wal_segment(master) + + # sleep to be sure, that any partial timeout is expired + sleep(70) + + self.assertTrue( + os.path.exists(wal_file_partial), + "File {0} disappeared".format(wal_file)) + + self.assertTrue( + os.path.exists(wal_file_partial), + "File {0} disappeared".format(wal_file_partial)) + + # @unittest.skip("skip") + def test_replica_promote_2(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + # set replica True, so archive_mode 'always' is used. + self.set_archiving( + backup_dir, 'master', master, replica=True) + master.slow_start() + + self.backup_node(backup_dir, 'master', master) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,1) i') + + self.wait_until_replica_catch_with_master(master, replica) + + replica.promote() + + self.backup_node( + backup_dir, 'master', replica, data_dir=replica.data_dir, + backup_type='page') + + # @unittest.skip("skip") + def test_replica_promote_archive_delta(self): + """ + t3 /---D3--> + t2 /-------> + t1 --F---D1--D2-- + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'archive_timeout': '30s'}) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + self.set_config( + backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_archiving(backup_dir, 'node', node1) + + node1.slow_start() + + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + + # Create replica + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + self.restore_node(backup_dir, 'node', node2, node2.data_dir) + + # Settings for Replica + self.set_replica(node1, node2) + self.set_auto_conf(node2, {'port': node2.port}) + self.set_archiving(backup_dir, 'node', node2, replica=True) + + node2.slow_start(replica=True) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + # delta backup on replica on timeline 1 + delta1_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, + 'delta', options=['--stream']) + + # delta backup on replica on timeline 1 + delta2_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, 'delta') + + self.change_backup_status( + backup_dir, 'node', delta2_id, 'ERROR') + + # node2 is now master + node2.promote() + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t3 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + + # node1 is now replica + node1.cleanup() + # kludge "backup_id=delta1_id" + self.restore_node( + backup_dir, 'node', node1, node1.data_dir, + backup_id=delta1_id, + options=[ + '--recovery-target-timeline=2', + '--recovery-target=latest']) + + # Settings for Replica + self.set_replica(node2, node1) + self.set_auto_conf(node1, {'port': node1.port}) + self.set_archiving(backup_dir, 'node', node1, replica=True) + + node1.slow_start(replica=True) + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t4 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,30) i') + self.wait_until_replica_catch_with_master(node2, node1) + + # node1 is back to be a master + node1.promote() + + sleep(5) + + # delta backup on timeline 3 + self.backup_node( + backup_dir, 'node', node1, node1.data_dir, 'delta', + options=['--archive-timeout=60']) + + pgdata = self.pgdata_content(node1.data_dir) + + node1.cleanup() + self.restore_node(backup_dir, 'node', node1, node1.data_dir) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_replica_promote_archive_page(self): + """ + t3 /---P3--> + t2 /-------> + t1 --F---P1--P2-- + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30s', + 'archive_timeout': '30s'}) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + self.set_archiving(backup_dir, 'node', node1) + self.set_config( + backup_dir, 'node', options=['--archive-timeout=60s']) + + node1.slow_start() + + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + + # Create replica + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + self.restore_node(backup_dir, 'node', node2, node2.data_dir) + + # Settings for Replica + self.set_replica(node1, node2) + self.set_auto_conf(node2, {'port': node2.port}) + self.set_archiving(backup_dir, 'node', node2, replica=True) + + node2.slow_start(replica=True) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + node1.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(node1, node2) + + # page backup on replica on timeline 1 + page1_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, + 'page', options=['--stream']) + + # page backup on replica on timeline 1 + page2_id = self.backup_node( + backup_dir, 'node', node2, node2.data_dir, 'page') + + self.change_backup_status( + backup_dir, 'node', page2_id, 'ERROR') + + # node2 is now master + node2.promote() + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t3 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + + # node1 is now replica + node1.cleanup() + # kludge "backup_id=page1_id" + self.restore_node( + backup_dir, 'node', node1, node1.data_dir, + backup_id=page1_id, + options=[ + '--recovery-target-timeline=2', + '--recovery-target=latest']) + + # Settings for Replica + self.set_replica(node2, node1) + self.set_auto_conf(node1, {'port': node1.port}) + self.set_archiving(backup_dir, 'node', node1, replica=True) + + node1.slow_start(replica=True) + + node2.safe_psql( + 'postgres', + 'CREATE TABLE t4 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,30) i') + self.wait_until_replica_catch_with_master(node2, node1) + + # node1 is back to be a master + node1.promote() + self.switch_wal_segment(node1) + + sleep(5) + + # delta3_id = self.backup_node( + # backup_dir, 'node', node2, node2.data_dir, 'delta') + # page backup on timeline 3 + page3_id = self.backup_node( + backup_dir, 'node', node1, node1.data_dir, 'page', + options=['--archive-timeout=60']) + + pgdata = self.pgdata_content(node1.data_dir) + + node1.cleanup() + self.restore_node(backup_dir, 'node', node1, node1.data_dir) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_parent_choosing(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + + master.slow_start() + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.set_replica(master, replica) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + self.add_instance(backup_dir, 'replica', replica) + + full_id = self.backup_node( + backup_dir, 'replica', + replica, options=['--stream']) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t2 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,20) i') + self.wait_until_replica_catch_with_master(master, replica) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + replica.promote() + + # failing, because without archving, it is impossible to + # take multi-timeline backup. + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) + + # @unittest.skip("skip") + def test_instance_from_the_past(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + full_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=10) + self.backup_node(backup_dir, 'node', node, options=['--stream']) + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=full_id) + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because instance is from the past " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Current START LSN' in e.message and + 'is lower than START LSN' in e.message and + 'It may indicate that we are trying to backup ' + 'PostgreSQL instance from the past' in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_replica_via_basebackup(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'hot_standby': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=10) + + #FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=['--recovery-target=latest', '--recovery-target-action=promote']) + node.slow_start() + + # Timeline 2 + # Take stream page backup from instance in timeline2 + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--log-level-file=verbose']) + + node.cleanup() + + # restore stream backup + self.restore_node(backup_dir, 'node', node) + + xlog_dir = 'pg_wal' + if self.get_version(node) < 100000: + xlog_dir = 'pg_xlog' + + filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") + self.assertTrue( + os.path.exists(filepath), + "History file do not exists: {0}".format(filepath)) + + node.slow_start() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + pg_basebackup_path = self.get_bin_path('pg_basebackup') + + self.run_binary( + [ + pg_basebackup_path, '-p', str(node.port), '-h', 'localhost', + '-R', '-X', 'stream', '-D', node_restored.data_dir + ]) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start(replica=True) + +# TODO: +# null offset STOP LSN and latest record in previous segment is conrecord (manual only) +# archiving from promoted delayed replica diff --git a/tests/restore_test.py b/tests/restore_test.py new file mode 100644 index 000000000..2de3ecc0f --- /dev/null +++ b/tests/restore_test.py @@ -0,0 +1,3822 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +import sys +from time import sleep +from datetime import datetime, timedelta, timezone +import hashlib +import shutil +import json +from shutil import copyfile +from testgres import QueryException, StartNodeException +from stat import S_ISDIR + + +class RestoreTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_restore_full_to_latest(self): + """recovery to latest from full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # 1 - Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # 2 - Test that recovery.conf was created + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + self.assertEqual(os.path.isfile(recovery_conf), True) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_full_page_to_latest(self): + """recovery to latest from full + page backups""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_to_specific_timeline(self): + """recovery to target timeline""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_tli = int( + node.get_control_data()["Latest checkpoint's TimeLineID"]) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Correct Backup must be choosen for restore + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--timeline={0}".format(target_tli)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + recovery_target_timeline = self.get_recovery_conf( + node)["recovery_target_timeline"] + self.assertEqual(int(recovery_target_timeline), target_tli) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_to_time(self): + """recovery to target time""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'TimeZone': 'GMT'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + backup_id = self.backup_node(backup_dir, 'node', node) + + target_time = node.execute( + "postgres", "SELECT to_char(now(), 'YYYY-MM-DD HH24:MI:SS+00')" + )[0][0] + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(target_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_to_xid_inclusive(self): + """recovery to target xid""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--xid={0}'.format(target_xid), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # @unittest.skip("skip") + def test_restore_to_xid_not_inclusive(self): + """recovery with target inclusive false""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = result[0][0] + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + '--xid={0}'.format(target_xid), + "--inclusive=false", + "--recovery-target-action=promote"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) + + # @unittest.skip("skip") + def test_restore_to_lsn_inclusive(self): + """recovery to target lsn""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + if self.get_version(node) < self.version_to_num('10.0'): + return + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) + + # @unittest.skip("skip") + def test_restore_to_lsn_not_inclusive(self): + """recovery to target lsn""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + if self.get_version(node) < self.version_to_num('10.0'): + return + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=2) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a int)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + res = con.execute("SELECT pg_current_wal_lsn()") + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "--inclusive=false", + "-j", "4", '--lsn={0}'.format(target_lsn), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + self.assertEqual( + len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) + + # @unittest.skip("skip") + def test_restore_full_ptrack_archive(self): + """recovery to latest from archive full+ptrack backups""" + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_ptrack(self): + """recovery to latest from archive full+ptrack+ptrack backups""" + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="ptrack") + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_full_ptrack_stream(self): + """recovery in stream mode to latest from full + ptrack backups""" + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + before = node.execute("postgres", "SELECT * FROM pgbench_branches") + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + after = node.execute("postgres", "SELECT * FROM pgbench_branches") + self.assertEqual(before, after) + + # @unittest.skip("skip") + def test_restore_full_ptrack_under_load(self): + """ + recovery to latest from full + ptrack backups + with loads when ptrack backup do + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=2) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + pgbench.wait() + pgbench.stdout.close() + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # @unittest.skip("skip") + def test_restore_full_under_load_ptrack(self): + """ + recovery to latest from full + page backups + with loads when full backup do + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + # wal_segment_size = self.guc_wal_segment_size(node) + node.pgbench_init(scale=2) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "4", "-T", "8"] + ) + + self.backup_node(backup_dir, 'node', node) + + pgbench.wait() + pgbench.stdout.close() + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["--stream"]) + + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + + self.assertEqual(bbalance, delta) + + node.stop() + node.cleanup() + # self.wrong_wal_clean(node, wal_segment_size) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + bbalance = node.execute( + "postgres", "SELECT sum(bbalance) FROM pgbench_branches") + delta = node.execute( + "postgres", "SELECT sum(delta) FROM pgbench_history") + self.assertEqual(bbalance, delta) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_1(self): + """recovery using tablespace-mapping option""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # 1 - Try to restore to existing directory + node.stop() + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore destination is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Restore destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 2 - Try to restore to existing tablespace directory + tblspc_path_tmp = os.path.join(node.base_dir, "tblspc_tmp") + os.rename(tblspc_path, tblspc_path_tmp) + node.cleanup() + os.rename(tblspc_path_tmp, tblspc_path) + try: + self.restore_node(backup_dir, 'node', node) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Restore tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 3 - Restore using tablespace-mapping to not empty directory + tblspc_path_temp = os.path.join(node.base_dir, "tblspc_temp") + os.mkdir(tblspc_path_temp) + with open(os.path.join(tblspc_path_temp, 'file'), 'w+') as f: + f.close() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["-T", "%s=%s" % (tblspc_path, tblspc_path_temp)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because restore tablespace destination is " + "not empty.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Restore tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # 4 - Restore using tablespace-mapping + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new)] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.execute("postgres", "SELECT id FROM test") + self.assertEqual(result[0][0], 1) + + # 4 - Restore using tablespace-mapping using page backup + self.backup_node(backup_dir, 'node', node) + with node.connect("postgres") as con: + con.execute("INSERT INTO test VALUES (2)") + con.commit() + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + + show_pb = self.show_pb(backup_dir, 'node') + self.assertEqual(show_pb[1]['status'], "OK") + self.assertEqual(show_pb[2]['status'], "OK") + + node.stop() + node.cleanup() + tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.execute("postgres", "SELECT id FROM test OFFSET 1") + self.assertEqual(result[0][0], 2) + + # @unittest.skip("skip") + def test_restore_with_tablespace_mapping_2(self): + """recovery using tablespace-mapping option and page backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Full backup + self.backup_node(backup_dir, 'node', node) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # First page backup + self.backup_node(backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + + # Create tablespace table + with node.connect("postgres") as con: +# con.connection.autocommit = True +# con.execute("CHECKPOINT") +# con.connection.autocommit = False + con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") + con.execute( + "INSERT INTO tbl1 SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Second page backup + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + + node.stop() + node.cleanup() + + tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + count = node.execute("postgres", "SELECT count(*) FROM tbl1") + self.assertEqual(count[0][0], 4) + + # @unittest.skip("skip") + def test_restore_with_missing_or_corrupted_tablespace_map(self): + """restore backup with missing or corrupted tablespace_map""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=1, tablespace='tblspace') + + # Full backup + self.backup_node(backup_dir, 'node', node) + + # Change some data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Page backup + page_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + olddir = self.get_tblspace_path(node, 'tblspace') + newdir = self.get_tblspace_path(node2, 'tblspace') + + # drop tablespace_map + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map') + + tablespace_map_tmp = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map_tmp') + + os.rename(tablespace_map, tablespace_map_tmp) + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + copyfile(tablespace_map_tmp, tablespace_map) + + with open(tablespace_map, "a") as f: + f.write("HELLO\n") + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # rename it back + os.rename(tablespace_map_tmp, tablespace_map) + + print(self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)])) + + pgdata_restored = self.pgdata_content(node2.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_restore_to_recovery_time(self): + """ + make node with archiving, make stream backup, + make PITR to Recovery Time + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + result = node.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_stream_pitr(self): + """ + make node with archiving, make stream backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + node.safe_psql("postgres", "create table t_heap(a int)") + node.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + + result = node.psql("postgres", 'select * from t_heap') + self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_node_backup_archive_pitr_2(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql("postgres", "create table t_heap(a int)") + node.stop() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote"] + ), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + result = node_restored.psql("postgres", 'select * from t_heap') + self.assertTrue('does not exist' in result[2].decode("utf-8")) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_archive_restore_to_restore_point(self): + """ + make node with archiving, make archive backup, + create table t_heap, make pitr to Recovery Time, + check that t_heap do not exists + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,10000)") + result = node.safe_psql( + "postgres", + "select * from t_heap") + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + node.safe_psql( + "postgres", + "create table t_heap_1 as select generate_series(0,10000)") + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + "--recovery-target-name=savepoint", + "--recovery-target-action=promote"]) + + node.slow_start() + + result_new = node.safe_psql("postgres", "select * from t_heap") + res = node.psql("postgres", "select * from t_heap_1") + self.assertEqual( + res[0], 1, + "Table t_heap_1 should not exist in restored instance") + + self.assertEqual(result, result_new) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_zags_block_corrupt(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + conn = node.connect() + with node.connect("postgres") as conn: + + conn.execute( + "create table tbl(i int)") + conn.commit() + conn.execute( + "create index idx ON tbl (i)") + conn.commit() + conn.execute( + "insert into tbl select i from generate_series(0,400) as i") + conn.commit() + conn.execute( + "select pg_relation_size('idx')") + conn.commit() + conn.execute( + "delete from tbl where i < 100") + conn.commit() + conn.execute( + "explain analyze select i from tbl order by i") + conn.commit() + conn.execute( + "select i from tbl order by i") + conn.commit() + conn.execute( + "create extension pageinspect") + conn.commit() + print(conn.execute( + "select * from bt_page_stats('idx',1)")) + conn.commit() + conn.execute( + "insert into tbl select i from generate_series(0,100) as i") + conn.commit() + conn.execute( + "insert into tbl select i from generate_series(0,100) as i") + conn.commit() + conn.execute( + "insert into tbl select i from generate_series(0,100) as i") + conn.commit() + conn.execute( + "insert into tbl select i from generate_series(0,100) as i") + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + initdb_params=['--data-checksums']) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + self.set_auto_conf( + node_restored, + {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) + + node_restored.slow_start() + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_zags_block_corrupt_1(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={ + 'full_page_writes': 'on'} + ) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + + node.safe_psql('postgres', 'create table tbl(i int)') + + node.safe_psql('postgres', 'create index idx ON tbl (i)') + + node.safe_psql( + 'postgres', + 'insert into tbl select i from generate_series(0,100000) as i') + + node.safe_psql( + 'postgres', + 'delete from tbl where i%2 = 0') + + node.safe_psql( + 'postgres', + 'explain analyze select i from tbl order by i') + + node.safe_psql( + 'postgres', + 'select i from tbl order by i') + + node.safe_psql( + 'postgres', + 'create extension pageinspect') + + node.safe_psql( + 'postgres', + 'insert into tbl select i from generate_series(0,100) as i') + + node.safe_psql( + 'postgres', + 'insert into tbl select i from generate_series(0,100) as i') + + node.safe_psql( + 'postgres', + 'insert into tbl select i from generate_series(0,100) as i') + + node.safe_psql( + 'postgres', + 'insert into tbl select i from generate_series(0,100) as i') + + self.switch_wal_segment(node) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + initdb_params=['--data-checksums']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + self.set_auto_conf( + node_restored, + {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) + + node_restored.slow_start() + + while True: + with open(node_restored.pg_log_file, 'r') as f: + if 'selected new timeline ID' in f.read(): + break + + # with open(node_restored.pg_log_file, 'r') as f: + # print(f.read()) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + self.compare_pgdata(pgdata, pgdata_restored) + +# pg_xlogdump_path = self.get_bin_path('pg_xlogdump') + +# pg_xlogdump = self.run_binary( +# [ +# pg_xlogdump_path, '-b', +# os.path.join(backup_dir, 'wal', 'node', '000000010000000000000003'), +# ' | ', 'grep', 'Btree', '' +# ], async=False) + + if pg_xlogdump.returncode: + self.assertFalse( + True, + 'Failed to start pg_wal_dump: {0}'.format( + pg_receivexlog.communicate()[1])) + + # @unittest.skip("skip") + def test_restore_chain(self): + """ + make node, take full backup, take several + ERROR delta backups, take valid delta backup, + restore must be successfull + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + # Take DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + # Take DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[3]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[4]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[5]['status'], + 'Backup STATUS should be "ERROR"') + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + # @unittest.skip("skip") + def test_restore_chain_with_corrupted_backup(self): + """more complex test_restore_chain()""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + # Take DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + # Take 1 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + # Take 2 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Take ERROR DELTA + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['-U', 'wrong_name']) + except ProbackupException as e: + pass + + # Take 3 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # Corrupted 4 DELTA + corrupt_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # ORPHAN 5 DELTA + restore_target_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # ORPHAN 6 DELTA + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # NEXT FULL BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='full') + + # Next Delta + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # do corrupt 6 DELTA backup + file = os.path.join( + backup_dir, 'backups', 'node', + corrupt_id, 'database', 'global', 'pg_control') + + file_new = os.path.join(backup_dir, 'pg_control') + os.rename(file, file_new) + + # RESTORE BACKUP + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, backup_id=restore_target_id) + self.assertEqual( + 1, 0, + "Expecting Error because restore backup is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is orphan'.format(restore_target_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[2]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[3]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[4]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[5]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node')[6]['status'], + 'Backup STATUS should be "ERROR"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[7]['status'], + 'Backup STATUS should be "OK"') + + # corruption victim + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node')[8]['status'], + 'Backup STATUS should be "CORRUPT"') + + # orphaned child + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node')[9]['status'], + 'Backup STATUS should be "ORPHAN"') + + # orphaned child + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node')[10]['status'], + 'Backup STATUS should be "ORPHAN"') + + # next FULL + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[11]['status'], + 'Backup STATUS should be "OK"') + + # next DELTA + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[12]['status'], + 'Backup STATUS should be "OK"') + + node.cleanup() + + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") + def test_restore_backup_from_future(self): + """more complex test_restore_chain()""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=5) + # pgbench = node.pgbench(options=['-T', '20', '-c', '2']) + # pgbench.wait() + + # Take PAGE from future + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + with open( + os.path.join( + backup_dir, 'backups', 'node', + backup_id, "backup.control"), "a") as conf: + conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3))) + + # rename directory + new_id = self.show_pb(backup_dir, 'node')[1]['id'] + + os.rename( + os.path.join(backup_dir, 'backups', 'node', backup_id), + os.path.join(backup_dir, 'backups', 'node', new_id)) + + pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) + pgbench.wait() + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_restore_target_immediate_stream(self): + """ + correct handling of immediate recovery target + for STREAM backups + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + # Take delta + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore delta backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--immediate']) + + self.assertTrue( + os.path.isfile(recovery_conf), + "File {0} do not exists".format(recovery_conf)) + + # restore delta backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--recovery-target=immediate']) + + self.assertTrue( + os.path.isfile(recovery_conf), + "File {0} do not exists".format(recovery_conf)) + + # @unittest.skip("skip") + def test_restore_target_immediate_archive(self): + """ + correct handling of immediate recovery target + for ARCHIVE backups + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node( + backup_dir, 'node', node) + + # Take delta + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--immediate']) + + # For archive backup with immediate recovery target + # recovery.conf is mandatory + with open(recovery_conf, 'r') as f: + self.assertIn("recovery_target = 'immediate'", f.read()) + + # restore page backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, options=['--recovery-target=immediate']) + + # For archive backup with immediate recovery target + # recovery.conf is mandatory + with open(recovery_conf, 'r') as f: + self.assertIn("recovery_target = 'immediate'", f.read()) + + # @unittest.skip("skip") + def test_restore_target_latest_archive(self): + """ + make sure that recovery_target 'latest' + is default recovery target + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # restore + node.cleanup() + self.restore_node(backup_dir, 'node', node) + + # hash_1 = hashlib.md5( + # open(recovery_conf, 'rb').read()).hexdigest() + + with open(recovery_conf, 'r') as f: + content_1 = '' + while True: + line = f.readline() + + if not line: + break + if line.startswith("#"): + continue + content_1 += line + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) + + # hash_2 = hashlib.md5( + # open(recovery_conf, 'rb').read()).hexdigest() + + with open(recovery_conf, 'r') as f: + content_2 = '' + while True: + line = f.readline() + + if not line: + break + if line.startswith("#"): + continue + content_2 += line + + self.assertEqual(content_1, content_2) + + # @unittest.skip("skip") + def test_restore_target_new_options(self): + """ + check that new --recovery-target-* + options are working correctly + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node) + + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + node.safe_psql( + "postgres", + "CREATE TABLE tbl0005 (a text)") + + node.safe_psql( + "postgres", "select pg_create_restore_point('savepoint')") + + target_name = 'savepoint' + + # in python-3.6+ it can be ...now()..astimezone()... + target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %z") + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + with node.connect("postgres") as con: + con.execute("INSERT INTO tbl0005 VALUES (1)") + con.commit() + if self.get_version(node) > self.version_to_num('10.0'): + res = con.execute("SELECT pg_current_wal_lsn()") + else: + res = con.execute("SELECT pg_current_xlog_location()") + + con.commit() + con.execute("INSERT INTO tbl0005 VALUES (2)") + con.commit() + xlogid, xrecoff = res[0][0].split('/') + xrecoff = hex(int(xrecoff, 16) + 1)[2:] + target_lsn = "{0}/{1}".format(xlogid, xrecoff) + + # Restore with recovery target time + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-time={0}'.format(target_time), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + self.assertIn( + "recovery_target_time = '{0}'".format(target_time), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target xid + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + self.assertIn( + "recovery_target_xid = '{0}'".format(target_xid), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target name + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-name={0}'.format(target_name), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + self.assertIn( + "recovery_target_name = '{0}'".format(target_name), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # Restore with recovery target lsn + if self.get_version(node) >= 100000: + + node.cleanup() + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) + + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() + + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) + + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) + + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) + + node.slow_start() + + # @unittest.skip("skip") + def test_smart_restore(self): + """ + make node, create database, take full backup, drop database, + take incremental backup and restore it, + make sure that files from dropped database are not + copied during restore + https://github.com/postgrespro/pg_probackup/issues/63 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create database + node.safe_psql( + "postgres", + "CREATE DATABASE testdb") + + # take FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # drop database + node.safe_psql( + "postgres", + "DROP DATABASE testdb") + + # take PAGE backup + page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # restore PAGE backup + node.cleanup() + self.restore_node( + backup_dir, 'node', node, backup_id=page_id, + options=['--no-validate', '--log-level-file=VERBOSE']) + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + # get delta between FULL and PAGE filelists + filelist_full = self.get_backup_filelist( + backup_dir, 'node', full_id) + + filelist_page = self.get_backup_filelist( + backup_dir, 'node', page_id) + + filelist_diff = self.get_backup_filelist_diff( + filelist_full, filelist_page) + + for file in filelist_diff: + self.assertNotIn(file, logfile_content) + + # @unittest.skip("skip") + def test_pg_11_group_access(self): + """ + test group access for PG >= 11 + """ + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=[ + '--data-checksums', + '--allow-group-access']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + # restore backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node( + backup_dir, 'node', node_restored) + + # compare pgdata permissions + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_restore_concurrent_drop_table(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress']) + + # DELTA backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress', '--no-validate'], + gdb=True) + + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + node.safe_psql( + 'postgres', + 'DROP TABLE pgbench_accounts') + + # do checkpoint to guarantee filenode removal + node.safe_psql( + 'postgres', + 'CHECKPOINT') + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_lost_non_data_file(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + os.remove(file) + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, options=['--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because of non-data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'No such file or directory', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup files restoring failed', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + def test_partial_restore_exclude(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node_restored_1) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + db1_path = os.path.join( + node_restored_1.data_dir, 'base', db_list['db1']) + db5_path = os.path.join( + node_restored_1.data_dir, 'base', db_list['db5']) + + self.truncate_every_file_in_dir(db1_path) + self.truncate_every_file_in_dir(db5_path) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-exclude=db1", + "--db-exclude=db5"]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'postgres', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_partial_restore_exclude_tablespace(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + cat_version = node.get_control_data()["Catalog version number"] + version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version + + # PG_10_201707211 + # pg_tblspc/33172/PG_9.5_201510051/16386/ + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").decode('utf-8').rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "-T", "{0}={1}".format( + node_tablespace, node1_tablespace)]) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + # truncate every db + for db in db_list: + # with exception below + if db in ['db1', 'db5']: + self.truncate_every_file_in_dir( + os.path.join( + node_restored_1.data_dir, 'pg_tblspc', + tbl_oid, version_specific_dir, db_list[db])) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2.cleanup() + node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace)]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'postgres', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db1', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db5', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_partial_restore_include(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node( + backup_dir, 'node', node_restored_1) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored_1) + + # truncate every db + for db in db_list: + # with exception below + if db in ['template0', 'template1', 'postgres', 'db1', 'db5']: + continue + self.truncate_every_file_in_dir( + os.path.join( + node_restored_1.data_dir, 'base', db_list[db])) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + node_restored_2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_2, options=[ + "--db-include=db1", + "--db-include=db5", + "--db-include=postgres"]) + + pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) + self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) + + self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + node_restored_2.slow_start() + + node_restored_2.safe_psql( + 'db1', + 'select 1') + + node_restored_2.safe_psql( + 'db5', + 'select 1') + + node_restored_2.safe_psql( + 'template1', + 'select 1') + + try: + node_restored_2.safe_psql( + 'db2', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + try: + node_restored_2.safe_psql( + 'db10', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + with open(node_restored_2.pg_log_file, 'r') as f: + output = f.read() + + self.assertNotIn('PANIC', output) + + def test_partial_restore_backward_compatibility_1(self): + """ + old binary should be of version < 2.2.0 + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with old binary, without partial restore support + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored, options=[ + "--db-exclude=db5"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup do not support partial restore.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # incremental backup with partial restore support + for i in range(11, 15, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # get db list + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + db_list_splitted = db_list_raw.splitlines() + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # get etalon + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db5'])) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db14'])) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + # get new node + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-exclude=db5", + "--db-exclude=db14"]) + + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + self.compare_pgdata(pgdata_restored, pgdata_restored_1) + + def test_partial_restore_backward_compatibility_merge(self): + """ + old binary should be of version < 2.2.0 + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with old binary, without partial restore support + backup_id = self.backup_node( + backup_dir, 'node', node, + old_binary=True, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', + node_restored, options=[ + "--db-exclude=db5"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup do not support partial restore.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # incremental backup with partial restore support + for i in range(11, 15, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # get db list + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + db_list_splitted = db_list_raw.splitlines() + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) + + # get etalon + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db5'])) + self.truncate_every_file_in_dir( + os.path.join( + node_restored.data_dir, 'base', db_list['db14'])) + pgdata_restored = self.pgdata_content(node_restored.data_dir) + + # get new node + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + # merge + self.merge_backup(backup_dir, 'node', backup_id=backup_id) + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=[ + "--db-exclude=db5", + "--db-exclude=db14"]) + pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) + + self.compare_pgdata(pgdata_restored, pgdata_restored_1) + + def test_empty_and_mangled_database_map(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with database_map + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + # truncate database_map + path = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'database_map') + with open(path, "w") as f: + f.close() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has empty or mangled database_map, " + "partial restore is impossible".format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has empty or mangled database_map, " + "partial restore is impossible".format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # mangle database_map + with open(path, "w") as f: + f.write("42") + f.close() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: field "dbOid" is not found in the line 42 of ' + 'the file backup_content.control', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db1", '--no-validate']) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: field "dbOid" is not found in the line 42 of ' + 'the file backup_content.control', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # check that simple restore is still possible + self.restore_node( + backup_dir, 'node', node_restored, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + def test_missing_database_map(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + node.safe_psql( + "postgres", + "CREATE DATABASE backupdb") + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if self.ptrack: + # TODO why backup works without these grants ? + # 'pg_ptrack_get_pagemapset(pg_lsn)', + # 'pg_ptrack_control_lsn()', + # because PUBLIC + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack; " + "GRANT USAGE ON SCHEMA ptrack TO backup; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + + if ProbackupTest.enterprise: + + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + # FULL backup without database_map + backup_id = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', "-U", "backup", '--log-level-file=verbose']) + + pgdata = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + # backup has missing database_map and that is legal + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-exclude=db5", "--db-exclude=db9"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=["--db-include=db1"]) + self.assertEqual( + 1, 0, + "Expecting Error because user do not have pg_database access.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} doesn't contain a database_map, " + "partial restore is impossible.".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # check that simple restore is still possible + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_stream_restore_command_option(self): + """ + correct handling of restore command options + when restoring STREAM backup + + 1. Restore STREAM backup with --restore-command only + parameter, check that PostgreSQL recovery uses + restore_command to obtain WAL from archive. + + 2. Restore STREAM backup wuth --restore-command + as replica, check that PostgreSQL recovery uses + restore_command to obtain WAL from archive. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'max_wal_size': '32MB'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(node.data_dir, 'recovery.conf') + + # Take FULL + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'create table t1()') + + # restore backup + node.cleanup() + shutil.rmtree(os.path.join(node.logs_dir)) + + restore_cmd = self.get_restore_command(backup_dir, 'node', node) + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--restore-command={0}'.format(restore_cmd)]) + + self.assertTrue( + os.path.isfile(recovery_conf), + "File '{0}' do not exists".format(recovery_conf)) + + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_signal = os.path.join(node.data_dir, 'recovery.signal') + self.assertTrue( + os.path.isfile(recovery_signal), + "File '{0}' do not exists".format(recovery_signal)) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select * from t1') + + timeline_id = node.safe_psql( + 'postgres', + 'select timeline_id from pg_control_checkpoint()').decode('utf-8').rstrip() + + self.assertEqual('2', timeline_id) + + # @unittest.skip("skip") + def test_restore_primary_conninfo(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' + + self.restore_node( + backup_dir, 'node', replica, + options=['-R', '--primary-conninfo={0}'.format(str_conninfo)]) + + if self.get_version(node) >= self.version_to_num('12.0'): + standby_signal = os.path.join(replica.data_dir, 'standby.signal') + self.assertTrue( + os.path.isfile(standby_signal), + "File '{0}' do not exists".format(standby_signal)) + + # TODO update test + if self.get_version(node) >= self.version_to_num('12.0'): + recovery_conf = os.path.join(replica.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) + else: + recovery_conf = os.path.join(replica.data_dir, 'recovery.conf') + + with open(os.path.join(replica.data_dir, recovery_conf), 'r') as f: + recovery_conf_content = f.read() + + self.assertIn(str_conninfo, recovery_conf_content) + + # @unittest.skip("skip") + def test_restore_primary_slot_info(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Take FULL + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + node.safe_psql( + "SELECT pg_create_physical_replication_slot('master_slot')") + + self.restore_node( + backup_dir, 'node', replica, + options=['-R', '--primary-slot-name=master_slot']) + + self.set_auto_conf(replica, {'port': replica.port}) + self.set_auto_conf(replica, {'hot_standby': 'on'}) + + if self.get_version(node) >= self.version_to_num('12.0'): + standby_signal = os.path.join(replica.data_dir, 'standby.signal') + self.assertTrue( + os.path.isfile(standby_signal), + "File '{0}' do not exists".format(standby_signal)) + + replica.slow_start(replica=True) + + def test_issue_249(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE database db1') + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'INSERT INTO pgbench_accounts SELECT * FROM t1') + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=["--db-include=db1"]) + + self.set_auto_conf( + node_restored_1, + {'port': node_restored_1.port, 'hot_standby': 'off'}) + + node_restored_1.slow_start() + + node_restored_1.safe_psql( + 'db1', + 'select 1') + + try: + node_restored_1.safe_psql( + 'postgres', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + def test_pg_12_probackup_recovery_conf_compatibility(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): + self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + time = node.safe_psql( + 'SELECT current_timestamp(0)::timestamptz;').decode('utf-8').rstrip() + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-time={0}".format(time), + "--recovery-target-action=promote"], + old_binary=True) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + xid = node.safe_psql( + 'SELECT txid_current()').decode('utf-8').rstrip() + node.pgbench_init(scale=1) + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-xid={0}".format(xid), + "--recovery-target-action=promote"]) + + node.slow_start() + + def test_drop_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # drop postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + os.remove(auto_path) + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + def test_truncate_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # truncate postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + with open(auto_path, "w+") as f: + f.truncate() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + # @unittest.skip("skip") + def test_concurrent_restore(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress']) + + pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress', '--no-validate']) + + pgdata1 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node.cleanup() + node_restored.cleanup() + + gdb = self.restore_node( + backup_dir, 'node', node, options=['--no-validate'], gdb=True) + + gdb.set_breakpoint('restore_data_file') + gdb.run_until_break() + + self.restore_node( + backup_dir, 'node', node_restored, options=['--no-validate']) + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + pgdata2 = self.pgdata_content(node.data_dir) + pgdata3 = self.pgdata_content(node_restored.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + self.compare_pgdata(pgdata2, pgdata3) + + # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 + @unittest.skip("skip") + def test_restore_issue_313(self): + """ + Check that partially restored PostgreSQL instance cannot be started + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + node.cleanup() + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: + count += 1 + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 2) + gdb.quit() + + # emulate the user or HA taking care of PG configuration + for fname in os.listdir(node_restored.data_dir): + if fname.endswith('.conf'): + os.rename( + os.path.join(node_restored.data_dir, fname), + os.path.join(node.data_dir, fname)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_restore_with_waldir(self): + """recovery using tablespace-mapping option and page backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + with node.connect("postgres") as con: + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Full backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Create waldir + waldir_path = os.path.join(node.base_dir, "waldir") + os.makedirs(waldir_path) + + # Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-X", "%s" % (waldir_path)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + + # check pg_wal is symlink + if node.major_version >= 10: + wal_path=os.path.join(node.data_dir, "pg_wal") + else: + wal_path=os.path.join(node.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) diff --git a/tests/retention_test.py b/tests/retention_test.py new file mode 100644 index 000000000..88432a00f --- /dev/null +++ b/tests/retention_test.py @@ -0,0 +1,2529 @@ +import os +import unittest +from datetime import datetime, timedelta +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from time import sleep +from distutils.dir_util import copy_tree + + +class RetentionTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_1(self): + """purge backups using redundancy-based retention policy""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + output_before = self.show_archive(backup_dir, 'node', tli=1) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + output_after = self.show_archive(backup_dir, 'node', tli=1) + + self.assertEqual( + output_before['max-segno'], + output_after['max-segno']) + + self.assertNotEqual( + output_before['min-segno'], + output_after['min-segno']) + + # Check that WAL segments were deleted + min_wal = output_after['min-segno'] + max_wal = output_after['max-segno'] + + for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): + if not wal_name.endswith(".backup"): + + if self.archive_compress: + wal_name = wal_name[-27:] + wal_name = wal_name[:-3] + else: + wal_name = wal_name[-24:] + + self.assertTrue(wal_name >= min_wal) + self.assertTrue(wal_name <= max_wal) + + # @unittest.skip("skip") + def test_retention_window_2(self): + """purge backups using window-based retention policy""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + with open( + os.path.join( + backup_dir, + 'backups', + 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + conf.write("retention-window = 1\n") + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + days_delta = 5 + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # Purge backups + self.delete_expired(backup_dir, 'node', options=['--expired']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # @unittest.skip("skip") + def test_retention_window_3(self): + """purge all backups using window-based retention policy""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # Take second FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # Take third FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--retention-window=1', '--expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # count wal files in ARCHIVE + + # @unittest.skip("skip") + def test_retention_window_4(self): + """purge all backups using window-based retention policy""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUPs + self.backup_node(backup_dir, 'node', node) + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + backup_id_3 = self.backup_node(backup_dir, 'node', node) + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_pb(backup_dir, 'node', backup_id_2) + self.delete_pb(backup_dir, 'node', backup_id_3) + + # Purge backups + self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--wal']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # count wal files in ARCHIVE + wals_dir = os.path.join(backup_dir, 'wal', 'node') + # n_wals = len(os.listdir(wals_dir)) + + # self.assertTrue(n_wals > 0) + + # self.delete_expired( + # backup_dir, 'node', + # options=['--retention-window=1', '--expired', '--wal']) + + # count again + n_wals = len(os.listdir(wals_dir)) + self.assertTrue(n_wals == 0) + + # @unittest.skip("skip") + def test_window_expire_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa ERROR + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa ERROR + + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change PAGEa1 and FULLa to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 and FULla to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # @unittest.skip("skip") + def test_redundancy_expire_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change FULLb backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 and FULLa backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa ERROR + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa ERROR + + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change PAGEa1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa2 and FULLa status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # Change PAGEb1 and FULLb status to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 and FULLa status to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + self.delete_expired( + backup_dir, 'node', + options=['--retention-redundancy=1', '--expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + print(self.show_pb( + backup_dir, 'node', as_json=False, as_text=True)) + + # @unittest.skip("skip") + def test_window_merge_interleaved_incremental_chains(self): + """complicated case of interleaved backup chains""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.backup_node(backup_dir, 'node', node) + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # FULLb ERROR + # FULLa OK + + # Take PAGEa1 backup + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Now we start to play with first generation of PAGE backups + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change PAGEa2 and FULLa to OK + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--merge-expired']) + + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_a, page_id_a2), + output) + + self.assertIn( + "Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a2), output) + + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_b, page_id_b2), + output) + + self.assertIn( + "Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b2), output) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # @unittest.skip("skip") + def test_window_merge_interleaved_incremental_chains_1(self): + """ + PAGEb3 + PAGEb2 + PAGEb1 + PAGEa1 + FULLb + FULLa + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # Take FULL BACKUPs + self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + # Change FULL B backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgdata_a1 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + # Change FULL B backup status to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + pgdata_b3 = self.pgdata_content(node.data_dir) + + pgbench = node.pgbench(options=['-t', '20', '-c', '1']) + pgbench.wait() + + # PAGEb3 OK + # PAGEb2 OK + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # Change PAGEa1 backup status to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # PAGEb3 OK + # PAGEb2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a1, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_expired( + backup_dir, 'node', + options=['--retention-window=1', '--expired', '--merge-expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a1) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'FULL') + + node.cleanup() + + # Data correctness of PAGEa3 + self.restore_node(backup_dir, 'node', node, backup_id=page_id_a1) + pgdata_restored_a1 = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata_a1, pgdata_restored_a1) + + node.cleanup() + + # Data correctness of PAGEb3 + self.restore_node(backup_dir, 'node', node, backup_id=page_id_b3) + pgdata_restored_b3 = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata_b3, pgdata_restored_b3) + + # @unittest.skip("skip") + def test_basic_window_merge_multiple_descendants(self): + """ + PAGEb3 + | PAGEa3 + -----------------------------retention window + PAGEb2 / + | PAGEa2 / should be deleted + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEb2 and PAGEb1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # and FULL stuff + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa3 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2, PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', '--delete-expired', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + # Merging chain A + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_a, page_id_a3), + output) + + self.assertIn( + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a3), output) + +# self.assertIn( +# "WARNING: Backup {0} has multiple valid descendants. " +# "Automatic merge is not possible.".format( +# page_id_a1), output) + + self.assertIn( + "LOG: Consider backup {0} for purge".format( + page_id_a2), output) + + # Merge chain B + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_b, page_id_b3), + output) + + self.assertIn( + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b3), output) + + self.assertIn( + "Delete: {0}".format(page_id_a2), output) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'FULL') + + # @unittest.skip("skip") + def test_basic_window_merge_multiple_descendants_1(self): + """ + PAGEb3 + | PAGEa3 + -----------------------------retention window + PAGEb2 / + | PAGEa2 / + PAGEb1 \ / + | PAGEa1 + FULLb | + FULLa + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Take FULL BACKUPs + backup_id_a = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + backup_id_b = self.backup_node(backup_dir, 'node', node) + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb backup status to ERROR + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + page_id_a1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change FULLb to OK + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') + + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + page_id_b1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb1 OK + # PAGEa1 ERROR + # FULLb OK + # FULLa OK + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # Change PAGEa1 to OK + self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') + + # Change PAGEb1 and FULLb to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa2 OK + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + # Change PAGEa2 and FULLa to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') + self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') + + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + page_id_b2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa ERROR + + # Change PAGEb2 and PAGEb1 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') + + # and FULL stuff + self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + page_id_a3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + # pgbench.wait() + + # PAGEa3 OK + # PAGEb2 ERROR + # PAGEa2 ERROR + # PAGEb1 ERROR + # PAGEa1 OK + # FULLb ERROR + # FULLa OK + + # Change PAGEa3 to ERROR + self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') + + # Change PAGEb2, PAGEb1 and FULLb to OK + self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGEb3 OK + # PAGEa3 ERROR + # PAGEb2 OK + # PAGEa2 ERROR + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Change PAGEa3, PAGEa2 and PAGEb1 status to OK + self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') + self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') + + # PAGEb3 OK + # PAGEa3 OK + # PAGEb2 OK + # PAGEa2 OK + # PAGEb1 OK + # PAGEa1 OK + # FULLb OK + # FULLa OK + + # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + page_id_a1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + page_id_a1) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + # Merging chain A + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_a, page_id_a3), + output) + + self.assertIn( + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_a, page_id_a3), output) + +# self.assertIn( +# "WARNING: Backup {0} has multiple valid descendants. " +# "Automatic merge is not possible.".format( +# page_id_a1), output) + + # Merge chain B + self.assertIn( + "Merge incremental chain between full backup {0} and backup {1}".format( + backup_id_b, page_id_b3), output) + + self.assertIn( + "INFO: Rename merged full backup {0} to {1}".format( + backup_id_b, page_id_b3), output) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['id'], + page_id_b3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['id'], + page_id_a3) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['id'], + page_id_a2) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[2]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[1]['backup-mode'], + 'FULL') + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['backup-mode'], + 'PAGE') + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--delete-expired', '--log-level-console=log']) + + # @unittest.skip("skip") + def test_window_chains(self): + """ + PAGE + -------window + PAGE + PAGE + FULL + PAGE + PAGE + FULL + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Chain A + self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Chain B + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', '--expired', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_window_chains_1(self): + """ + PAGE + -------window + PAGE + PAGE + FULL + PAGE + PAGE + FULL + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Chain A + self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Chain B + self.backup_node(backup_dir, 'node', node) + + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + page_id_b3 = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.pgdata_content(node.data_dir) + + # Purge backups + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup in [page_id_b3, 'pg_probackup.conf']: + continue + + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--merge-expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.assertIn( + "There are no backups to delete by retention policy", + output) + + self.assertIn( + "Retention merging finished", + output) + + output = self.delete_expired( + backup_dir, 'node', + options=[ + '--retention-window=1', + '--expired', '--log-level-console=log']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + + self.assertIn( + "There are no backups to merge by retention policy", + output) + + self.assertIn( + "Purging finished", + output) + + @unittest.skip("skip") + def test_window_error_backups(self): + """ + PAGE ERROR + -------window + PAGE ERROR + PAGE ERROR + PAGE ERROR + FULL ERROR + FULL + -------redundancy + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUPs + self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULLb backup status to ERROR + # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') + + # @unittest.skip("skip") + def test_window_error_backups_1(self): + """ + DELTA + PAGE ERROR + FULL + -------window + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # Take PAGE BACKUP + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', gdb=True) + + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + self.show_pb(backup_dir, 'node')[1]['id'] + + # Take DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-window=2', '--delete-expired']) + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + # @unittest.skip("skip") + def test_window_error_backups_2(self): + """ + DELTA + PAGE ERROR + FULL + -------window + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Take FULL BACKUP + self.backup_node(backup_dir, 'node', node) + + # Take PAGE BACKUP + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='page', gdb=True) + + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one + gdb.set_breakpoint('pg_stop_backup') + gdb.run_until_break() + gdb._execute('signal SIGKILL') + gdb.continue_execution_until_error() + + self.show_pb(backup_dir, 'node')[1]['id'] + + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'SELECT pg_catalog.pg_stop_backup()') + + # Take DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-window=2', '--delete-expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + + def test_retention_redundancy_overlapping_chains(self): + """""" + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + if self.get_version(node) < 90600: + self.skipTest('Skipped because ptrack support is disabled') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Make backups to be keeped + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + sleep(1) + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.validate_pb(backup_dir, 'node') + + def test_retention_redundancy_overlapping_chains_1(self): + """""" + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + if self.get_version(node) < 90600: + self.skipTest('Skipped because ptrack support is disabled') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Make backups to be keeped + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + sleep(1) + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + self.backup_node(backup_dir, 'node', node, backup_type="page") + + # Purge backups + self.delete_expired( + backup_dir, 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.validate_pb(backup_dir, 'node') + + def test_wal_purge_victim(self): + """ + https://github.com/postgrespro/pg_probackup/issues/103 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Make ERROR incremental backup + try: + self.backup_node(backup_dir, 'node', node, backup_type='page') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + page_id = self.show_pb(backup_dir, 'node')[0]['id'] + + sleep(1) + + # Make FULL backup + full_id = self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) + + try: + self.validate_pb(backup_dir, 'node') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "INFO: Backup {0} WAL segments are valid".format(full_id), + e.message) + self.assertIn( + "WARNING: Backup {0} has missing parent 0".format(page_id), + e.message) + + # @unittest.skip("skip") + def test_failed_merge_redundancy_retention(self): + """ + Check that retention purge works correctly with MERGING backups + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join( + self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL1 backup + full_id = self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # FULL2 backup + self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # FULL3 backup + self.backup_node(backup_dir, 'node', node) + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # DELTA BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=2']) + + self.set_config( + backup_dir, 'node', options=['--retention-window=2']) + + # create pair of MERGING backup as a result of failed merge + gdb = self.merge_backup( + backup_dir, 'node', delta_id, gdb=True) + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(2) + gdb._execute('signal SIGKILL') + + # "expire" first full backup + backups = os.path.join(backup_dir, 'backups', 'node') + with open( + os.path.join( + backups, full_id, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + # run retention merge + self.delete_expired( + backup_dir, 'node', options=['--delete-expired']) + + self.assertEqual( + 'MERGING', + self.show_pb(backup_dir, 'node', full_id)['status'], + 'Backup STATUS should be "MERGING"') + + self.assertEqual( + 'MERGING', + self.show_pb(backup_dir, 'node', delta_id)['status'], + 'Backup STATUS should be "MERGING"') + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) + + def test_wal_depth_1(self): + """ + |-------------B5----------> WAL timeline3 + |-----|-------------------------> WAL timeline2 + B1 B2---| B3 B4-------B6-----> WAL timeline1 + + wal-depth=2 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'archive_timeout': '30s', + 'checkpoint_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # FULL + node.pgbench_init(scale=1) + self.backup_node(backup_dir, 'node', node) + + # PAGE + node.pgbench_init(scale=1) + B2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # generate_some more data + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + + node.pgbench_init(scale=1) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.pgbench_init(scale=1) + + self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Timeline 2 + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node_restored.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_restored, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-action=promote']) + + self.assertIn( + 'Restore of backup {0} completed'.format(B2), + output) + + self.set_auto_conf(node_restored, options={'port': node_restored.port}) + + node_restored.slow_start() + + node_restored.pgbench_init(scale=1) + + target_xid = node_restored.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + + node_restored.pgbench_init(scale=2) + + # Timeline 3 + node_restored.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_restored, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'Restore of backup {0} completed'.format(B2), + output) + + self.set_auto_conf(node_restored, options={'port': node_restored.port}) + + node_restored.slow_start() + + node_restored.pgbench_init(scale=1) + self.backup_node( + backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) + + node.pgbench_init(scale=1) + self.backup_node(backup_dir, 'node', node) + + lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] + + self.validate_pb( + backup_dir, 'node', backup_id=B2, + options=['--recovery-target-lsn={0}'.format(lsn)]) + + self.validate_pb(backup_dir, 'node') + + def test_wal_purge(self): + """ + -------------------------------------> tli5 + ---------------------------B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + B* - backups + S* - switchpoints + + Expected result: + TLI5 will be purged entirely + B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + wal-depth=2 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # STREAM FULL + stream_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.stop() + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + B1 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + node.pgbench_init(scale=5) + + # B2 FULL on TLI1 + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + + self.delete_pb(backup_dir, 'node', options=['--delete-wal']) + + # TLI 2 + node_tli2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) + node_tli2.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_tli2, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=1', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + + self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.slow_start() + node_tli2.pgbench_init(scale=4) + + target_xid = node_tli2.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + node_tli2.pgbench_init(scale=1) + + self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=3) + + self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=1) + node_tli2.cleanup() + + # TLI3 + node_tli3 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) + node_tli3.cleanup() + + # Note, that successful validation here is a happy coincidence + output = self.restore_node( + backup_dir, 'node', node_tli3, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.slow_start() + node_tli3.pgbench_init(scale=5) + node_tli3.cleanup() + + # TLI4 + node_tli4 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) + node_tli4.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli4, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) + self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.slow_start() + + node_tli4.pgbench_init(scale=5) + + self.backup_node( + backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + node_tli4.pgbench_init(scale=5) + node_tli4.cleanup() + + # TLI5 + node_tli5 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) + node_tli5.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli5, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) + self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.slow_start() + node_tli5.pgbench_init(scale=10) + + # delete '.history' file of TLI4 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + # delete '.history' file of TLI5 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--delete-wal', '--dry-run', + '--log-level-console=verbose']) + + self.assertIn( + 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' + 'and 000000040000000000000006 can be removed', + output) + + self.assertIn( + 'INFO: On timeline 5 all files can be removed', + output) + + show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + + self.assertTrue(show_tli1_before) + self.assertTrue(show_tli2_before) + self.assertTrue(show_tli3_before) + self.assertTrue(show_tli4_before) + self.assertTrue(show_tli5_before) + + output = self.delete_pb( + backup_dir, 'node', + options=['--delete-wal', '--log-level-console=verbose']) + + self.assertIn( + 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' + 'and 000000040000000000000006 will be removed', + output) + + self.assertIn( + 'INFO: On timeline 5 all files will be removed', + output) + + show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + + self.assertEqual(show_tli1_before, show_tli1_after) + self.assertEqual(show_tli2_before, show_tli2_after) + self.assertEqual(show_tli3_before, show_tli3_after) + self.assertNotEqual(show_tli4_before, show_tli4_after) + self.assertNotEqual(show_tli5_before, show_tli5_after) + + self.assertEqual( + show_tli4_before['min-segno'], + '000000040000000000000002') + + self.assertEqual( + show_tli4_after['min-segno'], + '000000040000000000000006') + + self.assertFalse(show_tli5_after) + + self.validate_pb(backup_dir, 'node') + + def test_wal_depth_2(self): + """ + -------------------------------------> tli5 + ---------------------------B6--------> tli4 + S2`---------------> tli3 + S1`------------S2---B4-------B5--> tli2 + B1---S1-------------B2--------B3------> tli1 + + B* - backups + S* - switchpoints + wal-depth=2 + + Expected result: + TLI5 will be purged entirely + B6--------> tli4 + S2`---------------> tli3 + S1`------------S2 B4-------B5--> tli2 + B1---S1 B2--------B3------> tli1 + + wal-depth=2 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + + node.slow_start() + + # STREAM FULL + stream_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.stop() + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + B1 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=1) + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + node.pgbench_init(scale=5) + + # B2 FULL on TLI1 + B2 = self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + self.backup_node(backup_dir, 'node', node) + node.pgbench_init(scale=4) + + # TLI 2 + node_tli2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) + node_tli2.cleanup() + + output = self.restore_node( + backup_dir, 'node', node_tli2, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=1', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + + self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.slow_start() + node_tli2.pgbench_init(scale=4) + + target_xid = node_tli2.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + node_tli2.pgbench_init(scale=1) + + B4 = self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=3) + + self.backup_node( + backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + node_tli2.pgbench_init(scale=1) + node_tli2.cleanup() + + # TLI3 + node_tli3 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) + node_tli3.cleanup() + + # Note, that successful validation here is a happy coincidence + output = self.restore_node( + backup_dir, 'node', node_tli3, + options=[ + '--recovery-target-xid={0}'.format(target_xid), + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + self.assertIn( + 'INFO: Restore of backup {0} completed'.format(B1), + output) + self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.slow_start() + node_tli3.pgbench_init(scale=5) + node_tli3.cleanup() + + # TLI4 + node_tli4 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) + node_tli4.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli4, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) + self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.slow_start() + + node_tli4.pgbench_init(scale=5) + + self.backup_node( + backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + node_tli4.pgbench_init(scale=5) + node_tli4.cleanup() + + # TLI5 + node_tli5 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) + node_tli5.cleanup() + + self.restore_node( + backup_dir, 'node', node_tli5, backup_id=stream_id, + options=[ + '--recovery-target=immediate', + '--recovery-target-action=promote']) + + self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) + self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.slow_start() + node_tli5.pgbench_init(scale=10) + + # delete '.history' file of TLI4 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + # delete '.history' file of TLI5 + os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + + output = self.delete_pb( + backup_dir, 'node', + options=[ + '--delete-wal', '--dry-run', + '--wal-depth=2', '--log-level-console=verbose']) + + start_lsn_B2 = self.show_pb(backup_dir, 'node', B2)['start-lsn'] + self.assertIn( + 'On timeline 1 WAL is protected from purge at {0}'.format(start_lsn_B2), + output) + + self.assertIn( + 'LOG: Archive backup {0} to stay consistent protect from ' + 'purge WAL interval between 000000010000000000000004 ' + 'and 000000010000000000000005 on timeline 1'.format(B1), output) + + start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] + self.assertIn( + 'On timeline 2 WAL is protected from purge at {0}'.format(start_lsn_B4), + output) + + self.assertIn( + 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' + 'from purge WAL interval between 000000020000000000000006 and ' + '000000020000000000000009 on timeline 2', output) + + self.assertIn( + 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' + 'from purge WAL interval between 000000010000000000000004 and ' + '000000010000000000000006 on timeline 1', output) + + show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + + self.assertTrue(show_tli1_before) + self.assertTrue(show_tli2_before) + self.assertTrue(show_tli3_before) + self.assertTrue(show_tli4_before) + self.assertTrue(show_tli5_before) + + sleep(5) + + output = self.delete_pb( + backup_dir, 'node', + options=['--delete-wal', '--wal-depth=2', '--log-level-console=verbose']) + +# print(output) + + show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) + show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) + show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) + show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) + show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + + self.assertNotEqual(show_tli1_before, show_tli1_after) + self.assertNotEqual(show_tli2_before, show_tli2_after) + self.assertEqual(show_tli3_before, show_tli3_after) + self.assertNotEqual(show_tli4_before, show_tli4_after) + self.assertNotEqual(show_tli5_before, show_tli5_after) + + self.assertEqual( + show_tli4_before['min-segno'], + '000000040000000000000002') + + self.assertEqual( + show_tli4_after['min-segno'], + '000000040000000000000006') + + self.assertFalse(show_tli5_after) + + self.assertTrue(show_tli1_after['lost-segments']) + self.assertTrue(show_tli2_after['lost-segments']) + self.assertFalse(show_tli3_after['lost-segments']) + self.assertFalse(show_tli4_after['lost-segments']) + self.assertFalse(show_tli5_after) + + self.assertEqual(len(show_tli1_after['lost-segments']), 1) + self.assertEqual(len(show_tli2_after['lost-segments']), 1) + + self.assertEqual( + show_tli1_after['lost-segments'][0]['begin-segno'], + '000000010000000000000007') + + self.assertEqual( + show_tli1_after['lost-segments'][0]['end-segno'], + '00000001000000000000000A') + + self.assertEqual( + show_tli2_after['lost-segments'][0]['begin-segno'], + '00000002000000000000000A') + + self.assertEqual( + show_tli2_after['lost-segments'][0]['end-segno'], + '00000002000000000000000A') + + self.validate_pb(backup_dir, 'node') + + def test_basic_wal_depth(self): + """ + B1---B1----B3-----B4----B5------> tli1 + + Expected result with wal-depth=1: + B1 B1 B3 B4 B5------> tli1 + + wal-depth=1 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + node.pgbench_init(scale=1) + B1 = self.backup_node(backup_dir, 'node', node) + + + # B2 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B3 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B4 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # B5 + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + B5 = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--wal-depth=1', '--delete-wal']) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + target_xid = node.safe_psql( + "postgres", + "select txid_current()").decode('utf-8').rstrip() + + self.switch_wal_segment(node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '2']) + pgbench.wait() + + tli1 = self.show_archive(backup_dir, 'node', tli=1) + + # check that there are 4 lost_segments intervals + self.assertEqual(len(tli1['lost-segments']), 4) + + output = self.validate_pb( + backup_dir, 'node', B5, + options=['--recovery-target-xid={0}'.format(target_xid)]) + + print(output) + + self.assertIn( + 'INFO: Backup validation completed successfully on time', + output) + + self.assertIn( + 'xid {0} and LSN'.format(target_xid), + output) + + for backup_id in [B1, B2, B3, B4]: + try: + self.validate_pb( + backup_dir, 'node', backup_id, + options=['--recovery-target-xid={0}'.format(target_xid)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Not enough WAL records to xid {0}".format(target_xid), + e.message) + + self.validate_pb(backup_dir, 'node') + + def test_concurrent_running_full_backup(self): + """ + https://github.com/postgrespro/pg_probackup/issues/328 + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'RUNNING') + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired']) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired'], + return_id=False) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'OK') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[2]['status'], + 'OK') + + self.assertEqual( + len(self.show_pb(backup_dir, 'node')), + 6) diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py new file mode 100644 index 000000000..e789d174a --- /dev/null +++ b/tests/set_backup_test.py @@ -0,0 +1,476 @@ +import unittest +import subprocess +import os +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from sys import exit +from datetime import datetime, timedelta + + +class SetBackupTest(ProbackupTest, unittest.TestCase): + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_set_backup_sanity(self): + """general sanity for set-backup command""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + expire_time_1 = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=5)) + + try: + self.set_backup(backup_dir, False, options=['--ttl=30d']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing instance. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: required parameter not specified: --instance', + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.set_backup( + backup_dir, 'node', + options=[ + "--ttl=30d", + "--expire-time='{0}'".format(expire_time_1)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because options cannot be mixed. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--expire-time' " + "and '--ttl' options together", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + try: + self.set_backup(backup_dir, 'node', options=["--ttl=30d"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of missing backup_id. " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You must specify parameter (-i, --backup-id) " + "for 'set-backup' command", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.set_backup( + backup_dir, 'node', backup_id, options=["--ttl=30d"]) + + actual_expire_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['expire-time'] + + self.assertNotEqual(expire_time_1, actual_expire_time) + + expire_time_2 = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + + self.set_backup( + backup_dir, 'node', backup_id, + options=["--expire-time={0}".format(expire_time_2)]) + + actual_expire_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['expire-time'] + + self.assertIn(expire_time_2, actual_expire_time) + + # unpin backup + self.set_backup( + backup_dir, 'node', backup_id, options=["--ttl=0"]) + + attr_list = self.show_pb( + backup_dir, 'node', backup_id=backup_id) + + self.assertNotIn('expire-time', attr_list) + + self.set_backup( + backup_dir, 'node', backup_id, options=["--expire-time={0}".format(recovery_time)]) + + # parse string to datetime object + #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_retention_redundancy_pinning(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + with open(os.path.join( + backup_dir, 'backups', 'node', + "pg_probackup.conf"), "a") as conf: + conf.write("retention-redundancy = 1\n") + + self.set_config( + backup_dir, 'node', options=['--retention-redundancy=1']) + + # Make backups to be purged + full_id = self.backup_node(backup_dir, 'node', node) + page_id = self.backup_node( + backup_dir, 'node', node, backup_type="page") + # Make backups to be keeped + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type="page") + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.set_backup( + backup_dir, 'node', page_id, options=['--ttl=5d']) + + # Purge backups + log = self.delete_expired( + backup_dir, 'node', + options=['--delete-expired', '--log-level-console=LOG']) + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + + self.assertIn('Time Window: 0d/5d', log) + self.assertIn( + 'LOG: Backup {0} is pinned until'.format(page_id), + log) + self.assertIn( + 'LOG: Retain backup {0} because his descendant ' + '{1} is guarded by retention'.format(full_id, page_id), + log) + + # @unittest.skip("skip") + def test_retention_window_pinning(self): + """purge all backups using window-based retention policy""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUP + backup_id_1 = self.backup_node(backup_dir, 'node', node) + page1 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take second FULL BACKUP + backup_id_2 = self.backup_node(backup_dir, 'node', node) + page2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Take third FULL BACKUP + backup_id_3 = self.backup_node(backup_dir, 'node', node) + page2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + backups = os.path.join(backup_dir, 'backups', 'node') + for backup in os.listdir(backups): + if backup == 'pg_probackup.conf': + continue + with open( + os.path.join( + backups, backup, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3))) + + self.set_backup( + backup_dir, 'node', page1, options=['--ttl=30d']) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=LOG', + '--retention-window=1', + '--delete-expired']) + + self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + + self.assertIn( + 'LOG: Backup {0} is pinned until'.format(page1), out) + + self.assertIn( + 'LOG: Retain backup {0} because his descendant ' + '{1} is guarded by retention'.format(backup_id_1, page1), + out) + + # @unittest.skip("skip") + def test_wal_retention_and_pinning(self): + """ + B1---B2---P---B3---> + wal-depth=2 + P - pinned backup + + expected result after WAL purge: + B1 B2---P---B3---> + + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # take FULL BACKUP + self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + node.pgbench_init(scale=1) + + # Take PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--stream']) + + node.pgbench_init(scale=1) + + # Take DELTA BACKUP and pin it + expire_time = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + backup_id_pinned = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', + options=[ + '--stream', + '--expire-time={0}'.format(expire_time)]) + + node.pgbench_init(scale=1) + + # Take second PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + node.pgbench_init(scale=1) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=LOG', + '--delete-wal', '--wal-depth=2']) + + # print(out) + self.assertIn( + 'Pinned backup {0} is ignored for the ' + 'purpose of WAL retention'.format(backup_id_pinned), + out) + + for instance in self.show_archive(backup_dir): + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000004') + self.assertEqual(timeline['status'], 'OK') + + # @unittest.skip("skip") + def test_wal_retention_and_pinning_1(self): + """ + P---B1---> + wal-depth=2 + P - pinned backup + + expected result after WAL purge: + P---B1---> + + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + expire_time = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + + # take FULL BACKUP + backup_id_pinned = self.backup_node( + backup_dir, 'node', node, + options=['--expire-time={0}'.format(expire_time)]) + + node.pgbench_init(scale=2) + + # Take second PAGE BACKUP + self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + node.pgbench_init(scale=2) + + # Purge backups + out = self.delete_expired( + backup_dir, 'node', + options=[ + '--log-level-console=verbose', + '--delete-wal', '--wal-depth=2']) + + print(out) + self.assertIn( + 'Pinned backup {0} is ignored for the ' + 'purpose of WAL retention'.format(backup_id_pinned), + out) + + for instance in self.show_archive(backup_dir): + timelines = instance['timelines'] + + # sanity + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000002') + self.assertEqual(timeline['status'], 'OK') + + self.validate_pb(backup_dir) + + # @unittest.skip("skip") + def test_add_note_newlines(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format('hello\nhello')]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], "hello") + + self.set_backup(backup_dir, 'node', backup_id, options=['--note=hello\nhello']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], "hello") + + self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertNotIn('note', backup_meta) + + # @unittest.skip("skip") + def test_add_big_note(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + +# note = node.safe_psql( +# "postgres", +# "SELECT repeat('hello', 400)").rstrip() # TODO: investigate + + note = node.safe_psql( + "postgres", + "SELECT repeat('hello', 210)").rstrip() + + # FULL + try: + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format(note)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because note is too large " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup note cannot exceed 1024 bytes", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + note = node.safe_psql( + "postgres", + "SELECT repeat('hello', 200)").decode('utf-8').rstrip() + + backup_id = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--note={0}'.format(note)]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + self.assertEqual(backup_meta['note'], note) + + + # @unittest.skip("skip") + def test_add_big_note_1(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + note = node.safe_psql( + "postgres", + "SELECT repeat('q', 1024)").decode('utf-8').rstrip() + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + self.set_backup( + backup_dir, 'node', backup_id, + options=['--note={0}'.format(note)]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + + print(backup_meta) + self.assertEqual(backup_meta['note'], note) diff --git a/tests/show_test.py b/tests/show_test.py new file mode 100644 index 000000000..c4b96499d --- /dev/null +++ b/tests/show_test.py @@ -0,0 +1,509 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +class ShowTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_1(self): + """Status DONE and OK""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=off"]), + None + ) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_show_json(self): + """Status DONE and OK""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.assertEqual( + self.backup_node( + backup_dir, 'node', node, + options=["--log-level-console=off"]), + None + ) + self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + + # @unittest.skip("skip") + def test_corrupt_2(self): + """Status CORRUPT""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete file which belong to backup + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "database", "postgresql.conf") + os.remove(file) + + try: + self.validate_pb(backup_dir, 'node', backup_id) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup corrupted." + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd + ) + ) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd) + ) + self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + + # @unittest.skip("skip") + def test_no_control_file(self): + """backup.control doesn't exist""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # delete backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + os.remove(file) + + output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + + self.assertIn( + 'Control file', + output) + + self.assertIn( + 'doesn\'t exist', + output) + + # @unittest.skip("skip") + def test_empty_control_file(self): + """backup.control is empty""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # truncate backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'w') + fd.close() + + output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + + self.assertIn( + 'Control file', + output) + + self.assertIn( + 'is empty', + output) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_control_file(self): + """backup.control contains invalid option""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # corrupt backup.control file + file = os.path.join( + backup_dir, "backups", "node", + backup_id, "backup.control") + fd = open(file, 'a') + fd.write("statuss = OK") + fd.close() + + self.assertIn( + 'WARNING: Invalid option "statuss" in file', + self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness(self): + """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + backup_remote_id = self.backup_node(backup_dir, 'node', node) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness_1(self): + """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + backup_remote_id = self.backup_node(backup_dir, 'node', node) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # change data + pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_corrupt_correctness_2(self): + """backup.control contains invalid option""" + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + backup_local_id = self.backup_node( + backup_dir, 'node', node, + options=['--compress'], no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # change data + pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # DELTA + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='delta', options=['--compress'], no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # PAGE + backup_local_id = self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--compress'], no_remote=True) + + output_local = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_local_id) + self.delete_pb(backup_dir, 'node', backup_local_id) + + if self.remote: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', options=['--compress']) + else: + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) + + output_remote = self.show_pb( + backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + self.delete_pb(backup_dir, 'node', backup_remote_id) + + # check correctness + self.assertEqual( + output_local['data-bytes'], + output_remote['data-bytes']) + + self.assertEqual( + output_local['uncompressed-bytes'], + output_remote['uncompressed-bytes']) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_color_with_no_terminal(self): + """backup.control contains invalid option""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + try: + self.backup_node( + backup_dir, 'node', node, options=['--archive-timeout=1s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because archiving is disabled\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + '[0m', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) diff --git a/tests/time_consuming_test.py b/tests/time_consuming_test.py new file mode 100644 index 000000000..c0038c085 --- /dev/null +++ b/tests/time_consuming_test.py @@ -0,0 +1,77 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest +import subprocess +from time import sleep + + +class TimeConsumingTests(ProbackupTest, unittest.TestCase): + def test_pbckp150(self): + """ + https://jira.postgrespro.ru/browse/PBCKP-150 + create a node filled with pgbench + create FULL backup followed by PTRACK backup + run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel + """ + # init node + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={ + 'max_connections': 100, + 'log_statement': 'none', + 'log_checkpoints': 'on', + 'autovacuum': 'off', + 'ptrack.map_size': 1}) + + if node.major_version >= 13: + self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) + else: + self.set_auto_conf(node, {'wal_keep_segments': '1000'}) + + # init probackup and add an instance + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # run the node and init ptrack + node.slow_start() + node.safe_psql("postgres", "CREATE EXTENSION ptrack") + # populate it with pgbench + node.pgbench_init(scale=5) + + # FULL backup followed by PTRACK backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + # run ordinary pgbench scenario to imitate some activity and another pgbench for vacuuming in parallel + nBenchDuration = 30 + pgbench = node.pgbench(options=['-c', '20', '-j', '8', '-T', str(nBenchDuration)]) + with open('/tmp/pbckp150vacuum.sql', 'w') as f: + f.write('VACUUM (FULL) pgbench_accounts, pgbench_tellers, pgbench_history; SELECT pg_sleep(1);\n') + pgbenchval = node.pgbench(options=['-c', '1', '-f', '/tmp/pbckp150vacuum.sql', '-T', str(nBenchDuration)]) + + # several PTRACK backups + for i in range(nBenchDuration): + print("[{}] backing up PTRACK diff...".format(i+1)) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) + sleep(0.1) + # if the activity pgbench has finished, stop backing up + if pgbench.poll() is not None: + break + + pgbench.kill() + pgbenchval.kill() + pgbench.wait() + pgbenchval.wait() + + backups = self.show_pb(backup_dir, 'node') + for b in backups: + self.assertEqual("OK", b['status']) diff --git a/tests/time_stamp_test.py b/tests/time_stamp_test.py new file mode 100644 index 000000000..170c62cd4 --- /dev/null +++ b/tests/time_stamp_test.py @@ -0,0 +1,236 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from time import sleep + + +class TimeStamp(ProbackupTest, unittest.TestCase): + + def test_start_time_format(self): + """Test backup ID changing after start-time editing in backup.control. + We should convert local time in UTC format""" + # Create simple node + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) + show_backup = self.show_pb(backup_dir, 'node') + + i = 0 + while i < 2: + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: + output = "" + for line in f: + if line.startswith('start-time') is True: + if i == 0: + output = output + str(line[:-5])+'+00\''+'\n' + else: + output = output + str(line[:-5]) + '\'' + '\n' + else: + output = output + str(line) + f.close() + + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: + fw.write(output) + fw.flush() + show_backup = show_backup + self.show_pb(backup_dir, 'node') + i += 1 + + print(show_backup[1]['id']) + print(show_backup[2]['id']) + + self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + node.stop() + + def test_server_date_style(self): + """Issue #112""" + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={"datestyle": "GERMAN, DMY"}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2']) + + def test_handling_of_TZ_env_variable(self): + """Issue #284""" + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + my_env = os.environ.copy() + my_env["TZ"] = "America/Detroit" + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) + + output = self.show_pb(backup_dir, 'node', as_json=False, as_text=True, env=my_env) + + self.assertNotIn("backup ID in control file", output) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_dst_timezone_handling(self): + """for manual testing""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + print(subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + output = self.backup_node(backup_dir, 'node', node, return_id=False) + self.assertNotIn("backup ID in control file", output) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', return_id=False) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + @unittest.skip("skip") + def test_dst_timezone_handling_backward_compatibilty(self): + """for manual testing""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + self.backup_node(backup_dir, 'node', node, old_binary=True, return_id=False) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', old_binary=True, return_id=False) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() diff --git a/tests/validate_test.py b/tests/validate_test.py new file mode 100644 index 000000000..98a0fd13f --- /dev/null +++ b/tests/validate_test.py @@ -0,0 +1,4083 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from datetime import datetime, timedelta +from pathlib import Path +import subprocess +from sys import exit +import time +import hashlib + + +class ValidateTest(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_basic_validate_nullified_heap_page_backup(self): + """ + make node with nullified heap block + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "CHECKPOINT") + + # Nullify some block in PostgreSQL + file = os.path.join(node.data_dir, file_path) + with open(file, 'r+b') as f: + f.seek(8192) + f.write(b"\x00"*8192) + f.flush() + f.close + + self.backup_node( + backup_dir, 'node', node, options=['--log-level-file=verbose']) + + pgdata = self.pgdata_content(node.data_dir) + + if not self.remote: + log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") + with open(log_file_path) as f: + log_content = f.read() + self.assertIn( + 'File: "{0}" blknum 1, empty page'.format(Path(file).as_posix()), + log_content, + 'Failed to detect nullified block') + + self.validate_pb(backup_dir, options=["-j", "4"]) + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_validate_wal_unreal_values(self): + """ + make node with archiving, make archive backup + validate to both real and unreal values + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=3) + + target_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + after_backup_time = datetime.now().replace(second=0, microsecond=0) + + # Validate to real time + self.assertIn( + "INFO: Backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(target_time), "-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal time + unreal_time_1 = after_backup_time - timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format( + unreal_time_1), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup satisfying target options is not found', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to unreal time #2 + unreal_time_2 = after_backup_time + timedelta(days=2) + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(unreal_time_2), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal time.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Not enough WAL records to time' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate to real xid + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + self.switch_wal_segment(node) + time.sleep(5) + + self.assertIn( + "INFO: Backup validation completed successfully", + self.validate_pb( + backup_dir, 'node', options=["--xid={0}".format(target_xid), + "-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # Validate to unreal xid + unreal_xid = int(target_xid) + 1000 + try: + self.validate_pb( + backup_dir, 'node', options=["--xid={0}".format(unreal_xid), + "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of validation to unreal xid.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Not enough WAL records to xid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Validate with backup ID + output = self.validate_pb(backup_dir, 'node', backup_id, + options=["-j", "4"]) + self.assertIn( + "INFO: Validating backup {0}".format(backup_id), + output, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + self.assertIn( + "INFO: Backup {0} data files are valid".format(backup_id), + output, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + self.assertIn( + "INFO: Backup {0} WAL segments are valid".format(backup_id), + output, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + output, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + self.assertIn( + "INFO: Validate of backup {0} completed".format(backup_id), + output, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # @unittest.skip("skip") + def test_basic_validate_corrupted_intermediate_backup(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in PAGE1 backup, + run validate on PAGE1, expect PAGE1 to gain status CORRUPT + and PAGE2 gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_2, 'database', file_path) + with open(file, "r+b", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb( + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_2) in e.message and + 'ERROR: Backup {0} is corrupt'.format( + backup_id_2) in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + corrupt file in FULL and PAGE1 backups, run validate on PAGE1, + expect FULL and PAGE1 to gain status CORRUPT and + PAGE2 gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap_1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Corrupt some file in PAGE1 backup + file_page1 = os.path.join( + backup_dir, 'backups', 'node', + backup_id_2, 'database', file_path_t_heap_1) + with open(file_page1, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE1 + try: + self.validate_pb( + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his parent'.format( + backup_id_2) in e.message and + 'WARNING: Backup {0} is orphaned because his parent'.format( + backup_id_3) in e.message and + 'ERROR: Backup {0} is orphan.'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # @unittest.skip("skip") + def test_validate_specific_error_intermediate_backups(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + change backup status of FULL and PAGE1 to ERROR, + run validate on PAGE1 + purpose of this test is to be sure that not only + CORRUPT backup descendants can be orphanized + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULL backup status to ERROR + control_path = os.path.join( + backup_dir, 'backups', 'node', backup_id_1, 'backup.control') + + with open(control_path, 'r') as f: + actual_control = f.read() + + new_control_file = '' + for line in actual_control.splitlines(): + new_control_file += line.replace( + 'status = OK', 'status = ERROR') + new_control_file += '\n' + + with open(control_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + # Validate PAGE1 + try: + self.validate_pb( + backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: ERROR'.format( + backup_id_2, backup_id_1) in e.message and + 'INFO: Validating parents for backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Backup {0} has status ERROR. Skip validation.'.format( + backup_id_1) and + 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "ERROR"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # @unittest.skip("skip") + def test_validate_error_intermediate_backups(self): + """ + make archive node, take FULL, PAGE1, PAGE2 backups, + change backup status of FULL and PAGE1 to ERROR, + run validate on instance + purpose of this test is to be sure that not only + CORRUPT backup descendants can be orphanized + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Change FULL backup status to ERROR + control_path = os.path.join( + backup_dir, 'backups', 'node', backup_id_1, 'backup.control') + + with open(control_path, 'r') as f: + actual_control = f.read() + + new_control_file = '' + for line in actual_control.splitlines(): + new_control_file += line.replace( + 'status = OK', 'status = ERROR') + new_control_file += '\n' + + with open(control_path, 'wt') as f: + f.write(new_control_file) + f.flush() + f.close() + + # Validate instance + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because backup has status ERROR.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Backup {0} is orphaned because " + "his parent {1} has status: ERROR".format( + backup_id_2, backup_id_1) in e.message and + 'WARNING: Backup {0} has status ERROR. Skip validation'.format( + backup_id_1) in e.message and + "WARNING: Some backups are not valid" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'ERROR', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "ERROR"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', + self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + + # @unittest.skip("skip") + def test_validate_corrupted_intermediate_backups_1(self): + """ + make archive node, FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2, + corrupt file in PAGE1 and PAGE4, run validate on PAGE3, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 + to gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups', 'node', backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups', 'node', backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + backup_id=backup_id_4, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: CORRUPT'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: CORRUPT'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: CORRUPT'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: CORRUPT'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n ' + 'CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_specific_target_corrupted_intermediate_backups(self): + """ + make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 + corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, + expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to + gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + # PAGE1 + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_2 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(10000,20000) i") + backup_id_4 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE4 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + + target_xid = node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30001, 30001) i RETURNING (xmin)").decode('utf-8').rstrip() + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE5 + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_page_5 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() + backup_id_6 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE6 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_7 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL2 + backup_id_8 = self.backup_node(backup_dir, 'node', node) + + # Corrupt some file in PAGE2 and PAGE5 backups + file_page1 = os.path.join( + backup_dir, 'backups', 'node', + backup_id_3, 'database', file_page_2) + with open(file_page1, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + file_page4 = os.path.join( + backup_dir, 'backups', 'node', + backup_id_6, 'database', file_page_5) + with open(file_page4, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + # Validate PAGE3 + try: + self.validate_pb( + backup_dir, 'node', + options=[ + '-i', backup_id_4, '--xid={0}'.format(target_xid), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating parents for backup {0}'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his ' + 'parent {1} has status: CORRUPT'.format( + backup_id_4, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his ' + 'parent {1} has status: CORRUPT'.format( + backup_id_5, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his ' + 'parent {1} has status: CORRUPT'.format( + backup_id_6, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Backup {0} is orphaned because his ' + 'parent {1} has status: CORRUPT'.format( + backup_id_7, backup_id_3) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'ERROR: Backup {0} is orphan'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups(self): + """ + make archive node, take FULL1, PAGE1_1, FULL2, PAGE2_1 backups, FULL3 + corrupt file in FULL and FULL2 and run validate on instance, + expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN + FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_2, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_4, + 'database', rel_path)) + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups_interrupt(self): + """ + check that interrupt during validation is handled correctly + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_1, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_3, + 'database', rel_path)) + + # Validate Instance + gdb = self.validate_pb( + backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) + + gdb.set_breakpoint('validate_file_pages') + gdb.run_until_break() + gdb.continue_execution_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'Interrupted while locking backup', log_content) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_page(self): + """ + make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in PAGE1 backup and run validate on instance, + expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap1 = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + # PAGE2 + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + # PAGE3 + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups', 'node', backup_id_2, + 'database', file_path_t_heap1) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_5) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_5) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_4) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_4) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_3) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_3) in e.message and + 'WARNING: Backup {0} is orphaned because ' + 'his parent {1} has status: CORRUPT'.format( + backup_id_3, backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_2) in e.message and + 'WARNING: Invalid CRC of backup file' in e.message and + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id_2) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'INFO: Validating backup {0}'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} data files are valid'.format( + backup_id_1) in e.message and + 'INFO: Backup {0} WAL segments are valid'.format( + backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full_and_try_restore(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, + try to restore backup with --no-validation option""" + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + # PAGE1 + backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file' in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + node.cleanup() + restore_out = self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id_5), + restore_out, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # @unittest.skip("skip") + def test_validate_instance_with_corrupted_full(self): + """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, + corrupt file in FULL backup and run validate on instance, + expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + file_path_t_heap = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + # FULL1 + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + # PAGE1 + backup_id_2 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # PAGE2 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(20000,30000) i") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL1 + backup_id_4 = self.backup_node( + backup_dir, 'node', node) + + # PAGE3 + node.safe_psql( + "postgres", + "insert into t_heap select i as id, " + "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(30000,40000) i") + backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Corrupt some file in FULL backup + file_full = os.path.join( + backup_dir, 'backups', 'node', + backup_id_1, 'database', file_path_t_heap) + with open(file_full, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'INFO: Validating backup {0}'.format(backup_id_1) in e.message + and "INFO: Validate backups of the instance 'node'" in e.message + and 'WARNING: Invalid CRC of backup file' in e.message + and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_corrupt_wal_1(self): + """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id_1 = self.backup_node(backup_dir, 'node', node) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id_2 = self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(42) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Simple validate + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + + # @unittest.skip("skip") + def test_validate_corrupt_wal_2(self): + """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + backup_id = self.backup_node(backup_dir, 'node', node) + target_xid = None + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + for wal in wals: + with open(os.path.join(wals_dir, wal), "rb+", 0) as f: + f.seek(128) + f.write(b"blablablaadssaaaaaaaaaaaaaaa") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--xid={0}".format(target_xid), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: Backup' in e.message and + 'WAL segments are corrupted' in e.message and + "WARNING: There are not enough WAL " + "records to consistenly restore backup" in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "CORRUPT"') + + # @unittest.skip("skip") + def test_validate_wal_lost_segment_1(self): + """make archive node, make archive full backup, + delete from archive wal segment which belong to previous backup + run validate, expecting error because of missing wal segment + make sure that backup status is 'CORRUPT' + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + backup_id = self.backup_node(backup_dir, 'node', node) + + # Delete wal segment + wals_dir = os.path.join(backup_dir, 'wal', 'node') + wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] + wals.sort() + file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) + os.remove(file) + + # cut out '.gz' + if self.archive_compress: + file = file[:-3] + + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "is absent" in e.message and + "WARNING: There are not enough WAL records to consistenly " + "restore backup {0}".format(backup_id) in e.message and + "WARNING: Backup {0} WAL segments are corrupted".format( + backup_id) in e.message and + "WARNING: Some backups are not valid" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup {0} should have STATUS "CORRUPT"') + + # Run validate again + try: + self.validate_pb(backup_dir, 'node', backup_id, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Revalidating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_validate_corrupt_wal_between_backups(self): + """ + make archive node, make full backup, corrupt all wal files, + run validate to real xid, expect errors + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + # make some wals + node.pgbench_init(scale=3) + + with node.connect("postgres") as con: + con.execute("CREATE TABLE tbl0005 (a text)") + con.commit() + + with node.connect("postgres") as con: + res = con.execute( + "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") + con.commit() + target_xid = res[0][0] + + if self.get_version(node) < self.version_to_num('10.0'): + walfile = node.safe_psql( + 'postgres', + 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() + else: + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() + + if self.archive_compress: + walfile = walfile + '.gz' + self.switch_wal_segment(node) + + # generate some wals + node.pgbench_init(scale=3) + + self.backup_node(backup_dir, 'node', node) + + # Corrupt WAL + wals_dir = os.path.join(backup_dir, 'wal', 'node') + with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: + f.seek(9000) + f.write(b"b") + f.flush() + f.close + + # Validate to xid + try: + self.validate_pb( + backup_dir, + 'node', + backup_id, + options=[ + "--xid={0}".format(target_xid), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segments corruption.\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'ERROR: Not enough WAL records to xid' in e.message and + 'WARNING: Recovery can be done up to time' in e.message and + "ERROR: Not enough WAL records to xid {0}\n".format( + target_xid), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[0]['status'], + 'Backup STATUS should be "OK"') + + self.assertEqual( + 'OK', + self.show_pb(backup_dir, 'node')[1]['status'], + 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_pgpro702_688(self): + """ + make node without archiving, make stream backup, + get Recovery Time, validate to Recovery Time + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, options=["--stream"]) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id=backup_id)['recovery-time'] + + try: + self.validate_pb( + backup_dir, 'node', + options=["--time={0}".format(recovery_time), "-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of wal segment disappearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_pgpro688(self): + """ + make node with archiving, make backup, get Recovery Time, + validate to Recovery Time. Waiting PGPRO-688. RESOLVED + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + recovery_time = self.show_pb( + backup_dir, 'node', backup_id)['recovery-time'] + + self.validate_pb( + backup_dir, 'node', options=["--time={0}".format(recovery_time), + "-j", "4"]) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_pgpro561(self): + """ + make node with archiving, make stream backup, + restore it to node1, check that archiving is not successful on node1 + """ + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + self.set_archiving(backup_dir, 'node1', node1) + node1.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node1', node1, options=["--stream"]) + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + node1.psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,256) i") + + self.backup_node( + backup_dir, 'node1', node1, + backup_type='page', options=["--stream"]) + self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + + self.set_auto_conf( + node2, {'port': node2.port, 'archive_mode': 'off'}) + + node2.slow_start() + + self.set_auto_conf( + node2, {'archive_mode': 'on'}) + + node2.stop() + node2.slow_start() + + timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] + timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] + self.assertEqual( + timeline_node1, timeline_node2, + "Timelines on Master and Node1 should be equal. " + "This is unexpected") + + archive_command_node1 = node1.safe_psql( + "postgres", "show archive_command") + archive_command_node2 = node2.safe_psql( + "postgres", "show archive_command") + self.assertEqual( + archive_command_node1, archive_command_node2, + "Archive command on Master and Node should be equal. " + "This is unexpected") + + # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") + ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) + # if result == "": + # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') + + node1.psql( + "postgres", + "create table t_heap_1 as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10) i") + + self.switch_wal_segment(node1) + +# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] +# wals = map(str, wals) +# print(wals) + + self.switch_wal_segment(node2) + +# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( +# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] +# wals = map(str, wals) +# print(wals) + + time.sleep(5) + + log_file = os.path.join(node2.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertTrue( + 'LOG: archive command failed with exit code 1' in log_content and + 'DETAIL: The failed archive command was:' in log_content and + 'WAL file already exists in archive with different checksum' in log_content, + 'Expecting error messages about failed archive_command' + ) + self.assertFalse( + 'pg_probackup archive-push completed successfully' in log_content) + + # @unittest.skip("skip") + def test_validate_corrupted_full(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and three page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption and run valudate again, check that + second full backup and his page backups are OK + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '30'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.safe_psql( + "postgres", + "alter system set archive_command = 'false'") + node.reload() + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=1s']) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + pass + + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.set_archiving(backup_dir, 'node', node) + node.reload() + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue( + self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue( + self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + + os.rename(file_new, file) + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue( + self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + + # @unittest.skip("skip") + def test_validate_corrupted_full_1(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and corrupt his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + second page should be CORRUPT + third page should be ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + os.rename(file_new, file) + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id_page, 'database', 'backup_label') + + file_new = os.path.join(backup_dir, 'backup_label') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + + # @unittest.skip("skip") + def test_validate_corrupted_full_2(self): + """ + PAGE2_2b + PAGE2_2a + PAGE2_4 + PAGE2_4 <- validate + PAGE2_3 + PAGE2_2 <- CORRUPT + PAGE2_1 + FULL2 + PAGE1_1 + FULL1 + corrupt second page backup, run validate on PAGE2_3, check that + PAGE2_2 became CORRUPT and his descendants are ORPHANs, + take two more PAGE backups, which now trace their origin + to PAGE2_1 - latest OK backup, + run validate on PAGE2_3, check that PAGE2_2a and PAGE2_2b are OK, + + remove corruption from PAGE2_2 and run validate on PAGE2_4 + """ + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + corrupt_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + validate_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + corrupt_id, 'database', 'backup_label') + + file_new = os.path.join(backup_dir, 'backup_label') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'INFO: Validating parents for backup {0}'.format(validate_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[2]['id']), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[3]['id']), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'INFO: Validating backup {0}'.format( + corrupt_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + corrupt_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # THIS IS GOLD!!!! + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Backup {0} data files are valid'.format( + self.show_pb(backup_dir, 'node')[9]['id']), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'Backup {0} data files are valid'.format( + self.show_pb(backup_dir, 'node')[8]['id']), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Revalidating backup {0}'.format( + corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Some backups are not valid', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # revalidate again + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Validating parents for backup {0}'.format( + validate_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[2]['id']), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[3]['id']), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'INFO: Revalidating backup {0}'.format( + corrupt_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + corrupt_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'ERROR: Backup {0} is orphan.'.format( + validate_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # Fix CORRUPT + os.rename(file_new, file) + + output = self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + + self.assertIn( + 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), + output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), + output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), + output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), + output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Validating parents for backup {0}'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[2]['id']), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Validating backup {0}'.format( + self.show_pb(backup_dir, 'node')[3]['id']), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Revalidating backup {0}'.format( + corrupt_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} data files are valid'.format( + corrupt_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Revalidating backup {0}'.format( + self.show_pb(backup_dir, 'node')[5]['id']), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} data files are valid'.format( + self.show_pb(backup_dir, 'node')[5]['id']), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Revalidating backup {0}'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Backup {0} data files are valid'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Backup {0} WAL segments are valid'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Backup {0} is valid.'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'INFO: Validate of backup {0} completed.'.format( + validate_id), output, + '\n Unexpected Output Message: {0}\n'.format( + repr(output))) + + # Now we have two perfectly valid backup chains based on FULL2 + + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # @unittest.skip("skip") + def test_validate_corrupted_full_missing(self): + """ + make node with archiving, take full backup, and three page backups, + take another full backup and four page backups + corrupt second full backup, run validate, check that + second full backup became CORRUPT and his page backups are ORPHANs + remove corruption from full backup and remove his second page backup + run valudate again, check that + second full backup and his firts page backups are OK, + third page should be ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + backup_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_page = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + file = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'postgresql.auto.conf') + + file_new = os.path.join(backup_dir, 'postgresql.auto.conf') + os.rename(file, file_new) + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data file dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'Validating backup {0}'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} data files are corrupted'.format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} has status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[5]['id'], backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # Full backup is fixed + os.rename(file_new, file) + + # break PAGE + old_directory = os.path.join( + backup_dir, 'backups', 'node', backup_id_page) + new_directory = os.path.join(backup_dir, backup_id_page) + os.rename(old_directory, new_directory) + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[7]['id'], + backup_id_page), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[6]['id'], + backup_id_page), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( + self.show_pb(backup_dir, 'node')[5]['id'], backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + # missing backup is here + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # validate should be idempotent - user running validate + # second time must be provided with ID of missing backup + + try: + self.validate_pb(backup_dir, options=["-j", "4"]) + except ProbackupException as e: + self.assertIn( + 'WARNING: Some backups are not valid', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[7]['id'], + backup_id_page), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[6]['id'], + backup_id_page), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + # missing backup is here + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # fix missing PAGE backup + os.rename(new_directory, old_directory) + # exit(1) + + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + output = self.validate_pb(backup_dir, options=["-j", "4"]) + + self.assertIn( + 'INFO: All backups are valid', + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( + self.show_pb(backup_dir, 'node')[8]['id'], + self.show_pb(backup_dir, 'node')[6]['id']), + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( + self.show_pb(backup_dir, 'node')[7]['id'], + self.show_pb(backup_dir, 'node')[6]['id']), + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Revalidating backup {0}'.format( + self.show_pb(backup_dir, 'node')[6]['id']), + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Revalidating backup {0}'.format( + self.show_pb(backup_dir, 'node')[7]['id']), + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertIn( + 'Revalidating backup {0}'.format( + self.show_pb(backup_dir, 'node')[8]['id']), + output, + '\n Unexpected Error Message: {0}\n'.format( + repr(output))) + + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + def test_file_size_corruption_no_validate(self): + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + # initdb_params=['--data-checksums'], + ) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + heap_size = node.safe_psql( + "postgres", + "select pg_relation_size('t_heap')") + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4"], asynchronous=False, gdb=False) + + node.stop() + node.cleanup() + + # Let`s do file corruption + with open( + os.path.join( + backup_dir, "backups", 'node', backup_id, + "database", heap_path), "rb+", 0) as f: + f.truncate(int(heap_size) - 4096) + f.flush() + f.close + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=["--no-validate"]) + except ProbackupException as e: + self.assertTrue( + "ERROR: Backup files restoring failed" in e.message, + repr(e.message)) + + # @unittest.skip("skip") + def test_validate_specific_backup_with_missing_backup(self): + """ + PAGE3_2 + PAGE3_1 + FULL3 + PAGE2_5 + PAGE2_4 <- validate + PAGE2_3 + PAGE2_2 <- missing + PAGE2_1 + FULL2 + PAGE1_2 + PAGE1_1 + FULL1 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # CHAIN1 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN2 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + validate_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN3 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + old_directory = os.path.join(backup_dir, 'backups', 'node', missing_id) + new_directory = os.path.join(backup_dir, missing_id) + + os.rename(old_directory, new_directory) + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[7]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + # missing backup + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[7]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + os.rename(new_directory, old_directory) + + # Revalidate backup chain + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # @unittest.skip("skip") + def test_validate_specific_backup_with_missing_backup_1(self): + """ + PAGE3_2 + PAGE3_1 + FULL3 + PAGE2_5 + PAGE2_4 <- validate + PAGE2_3 + PAGE2_2 <- missing + PAGE2_1 + FULL2 <- missing + PAGE1_2 + PAGE1_1 + FULL1 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # CHAIN1 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN2 + missing_full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + validate_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN3 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + page_old_directory = os.path.join( + backup_dir, 'backups', 'node', missing_page_id) + page_new_directory = os.path.join(backup_dir, missing_page_id) + os.rename(page_old_directory, page_new_directory) + + full_old_directory = os.path.join( + backup_dir, 'backups', 'node', missing_full_id) + full_new_directory = os.path.join(backup_dir, missing_full_id) + os.rename(full_old_directory, full_new_directory) + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + # PAGE2_1 + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') # <- SHit + # FULL2 + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + os.rename(page_new_directory, page_old_directory) + os.rename(full_new_directory, full_old_directory) + + # Revalidate backup chain + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') # <- Fail + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # @unittest.skip("skip") + def test_validate_with_missing_backup_1(self): + """ + PAGE3_2 + PAGE3_1 + FULL3 + PAGE2_5 + PAGE2_4 <- validate + PAGE2_3 + PAGE2_2 <- missing + PAGE2_1 + FULL2 <- missing + PAGE1_2 + PAGE1_1 + FULL1 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # CHAIN1 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN2 + missing_full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + validate_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN3 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # Break PAGE + page_old_directory = os.path.join( + backup_dir, 'backups', 'node', missing_page_id) + page_new_directory = os.path.join(backup_dir, missing_page_id) + os.rename(page_old_directory, page_new_directory) + + # Break FULL + full_old_directory = os.path.join( + backup_dir, 'backups', 'node', missing_full_id) + full_new_directory = os.path.join(backup_dir, missing_full_id) + os.rename(full_old_directory, full_new_directory) + + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + # PAGE2_2 is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + # FULL1 - is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + os.rename(page_new_directory, page_old_directory) + + # Revalidate backup chain + try: + self.validate_pb(backup_dir, 'node', validate_id, + options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} has status: ORPHAN'.format( + validate_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[7]['id'], + missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[6]['id'], + missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[5]['id'], + missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[4]['id'], + missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[3]['id'], + missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + # FULL1 - is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + os.rename(full_new_directory, full_old_directory) + + # Revalidate chain + self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # @unittest.skip("skip") + def test_validate_with_missing_backup_2(self): + """ + PAGE3_2 + PAGE3_1 + FULL3 + PAGE2_5 + PAGE2_4 + PAGE2_3 + PAGE2_2 <- missing + PAGE2_1 + FULL2 <- missing + PAGE1_2 + PAGE1_1 + FULL1 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # CHAIN1 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN2 + missing_full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_page_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node( + backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # CHAIN3 + self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') + + page_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_page_id) + page_new_directory = os.path.join(backup_dir, missing_page_id) + os.rename(page_old_directory, page_new_directory) + + full_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_full_id) + full_new_directory = os.path.join(backup_dir, missing_full_id) + os.rename(full_old_directory, full_new_directory) + + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + # PAGE2_2 is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + # FULL1 - is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + os.rename(page_new_directory, page_old_directory) + + # Revalidate backup chain + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of backup dissapearance.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[7]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[6]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[5]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( + self.show_pb(backup_dir, 'node')[4]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'WARNING: Backup {0} has missing parent {1}'.format( + self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + # FULL1 - is missing + self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') + self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + + # @unittest.skip("skip") + def test_corrupt_pg_control_via_resetxlog(self): + """ PGPRO-2096 """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + if self.get_version(node) < 100000: + pg_resetxlog_path = self.get_bin_path('pg_resetxlog') + wal_dir = 'pg_xlog' + else: + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' + + os.mkdir( + os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) + + pg_control_path = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'global', 'pg_control') + + md5_before = hashlib.md5( + open(pg_control_path, 'rb').read()).hexdigest() + + self.run_binary( + [ + pg_resetxlog_path, + '-D', + os.path.join(backup_dir, 'backups', 'node', backup_id, 'database'), + '-o 42', + '-f' + ], + asynchronous=False) + + md5_after = hashlib.md5( + open(pg_control_path, 'rb').read()).hexdigest() + + if self.verbose: + print('\n MD5 BEFORE resetxlog: {0}\n MD5 AFTER resetxlog: {1}'.format( + md5_before, md5_after)) + + # Validate backup + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4"]) + self.assertEqual( + 1, 0, + "Expecting Error because of pg_control change.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'data files are corrupted', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_validation_after_backup(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + gdb = self.backup_node( + backup_dir, 'node', node, gdb=True, options=['--stream']) + + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + + file = os.path.join( + backup_dir, "backups", "node", backup_id, + "database", "postgresql.conf") + os.remove(file) + + gdb.continue_execution_until_exit() + + self.assertEqual( + 'CORRUPT', + self.show_pb(backup_dir, 'node', backup_id)['status'], + 'Backup STATUS should be "ERROR"') + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_corrupt_tablespace_map(self): + """ + Check that corruption in tablespace_map is detected + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'external_dir') + + node.safe_psql( + 'postgres', + 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'tablespace_map') + + # Corrupt tablespace_map file in FULL backup + with open(tablespace_map, "rb+", 0) as f: + f.seek(84) + f.write(b"blah") + f.flush() + f.close + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Invalid CRC of backup file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + #TODO fix the test + @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_target_lsn(self): + """ + Check validation to specific LSN + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,10000) i") + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + self.switch_wal_segment(node) + + backup_id = self.backup_node( + backup_dir, 'node', node_restored, + data_dir=node_restored.data_dir) + + target_lsn = self.show_pb(backup_dir, 'node')[1]['stop-lsn'] + + self.delete_pb(backup_dir, 'node', backup_id) + + self.validate_pb( + backup_dir, 'node', + options=[ + '--recovery-target-timeline=2', + '--recovery-target-lsn={0}'.format(target_lsn)]) + + @unittest.skip("skip") + def test_partial_validate_empty_and_mangled_database_map(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + # create databases + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup with database_map + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + # truncate database_map + path = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'database_map') + with open(path, "w") as f: + f.close() + + try: + self.validate_pb( + backup_dir, 'node', + options=["--db-include=db1"]) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Backup {0} data files are corrupted".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # mangle database_map + with open(path, "w") as f: + f.write("42") + f.close() + + try: + self.validate_pb( + backup_dir, 'node', + options=["--db-include=db1"]) + self.assertEqual( + 1, 0, + "Expecting Error because database_map is empty.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Backup {0} data files are corrupted".format( + backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + @unittest.skip("skip") + def test_partial_validate_exclude(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "--log-level-console=verbose"]) + self.assertEqual( + 1, 0, + "Expecting Error because of missing backup ID.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You must specify parameter (-i, --backup-id) for partial validation", + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + output = self.validate_pb( + backup_dir, 'node', backup_id, + options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "--log-level-console=verbose"]) + + self.assertIn( + "VERBOSE: Skip file validation due to partial restore", output) + + @unittest.skip("skip") + def test_partial_validate_include(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0}'.format(i)) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + try: + self.validate_pb( + backup_dir, 'node', + options=[ + "--db-include=db1", + "--db-exclude=db2"]) + self.assertEqual( + 1, 0, + "Expecting Error because of 'db-exclude' and 'db-include'.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + output = self.validate_pb( + backup_dir, 'node', backup_id, + options=[ + "--db-include=db1", + "--db-include=db5", + "--db-include=postgres", + "--log-level-console=verbose"]) + + self.assertIn( + "VERBOSE: Skip file validation due to partial restore", output) + + output = self.validate_pb( + backup_dir, 'node', backup_id, + options=["--log-level-console=verbose"]) + + self.assertNotIn( + "VERBOSE: Skip file validation due to partial restore", output) + + # @unittest.skip("skip") + def test_not_validate_diffenent_pg_version(self): + """Do not validate backup, if binary is compiled with different PG version""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_id = self.backup_node(backup_dir, 'node', node) + + control_file = os.path.join( + backup_dir, "backups", "node", backup_id, + "backup.control") + + pg_version = node.major_version + + if pg_version.is_integer(): + pg_version = int(pg_version) + + fake_new_pg_version = pg_version + 1 + + with open(control_file, 'r') as f: + data = f.read(); + + data = data.replace( + "server-version = {0}".format(str(pg_version)), + "server-version = {0}".format(str(fake_new_pg_version))) + + with open(control_file, 'w') as f: + f.write(data); + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because validation is forbidden if server version of backup " + "is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "ERROR: Backup {0} has server version".format(backup_id), + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_corrupt_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # Corrupt tablespace_map file in FULL backup + with open(page_header_map, "rb+", 0) as f: + f.seek(42) + f.write(b"blah") + f.flush() + f.close + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertTrue( + 'WARNING: An error occured during metadata decompression' in e.message and + 'data error' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + + self.assertIn("WARNING: Some backups are not valid", e.message) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_truncated_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # truncate page_header_map file + with open(page_header_map, "rb+", 0) as f: + f.truncate(121) + f.flush() + f.close + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + self.assertIn("WARNING: Some backups are not valid", e.message) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_validate_missing_page_header_map(self): + """ + Check that corruption in page_header_map is detected + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + page_header_map = os.path.join( + backup_dir, 'backups', 'node', backup_id, 'page_header_map') + + # unlink page_header_map file + os.remove(page_header_map) + + try: + self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.validate_pb(backup_dir) + self.assertEqual( + 1, 0, + "Expecting Error because page_header is corrupted.\n " + "Output: {0} \n CMD: {1}".format( + self.output, self.cmd)) + except ProbackupException as e: + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + self.assertIn("WARNING: Some backups are not valid", e.message) + + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_no_validate_tablespace_map(self): + """ + Check that --no-validate is propagated to tablespace_map + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'external_dir') + + node.safe_psql( + 'postgres', + 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') + + tblspace_new = self.get_tblspace_path(node, 'external_dir_new') + + oid = node.safe_psql( + 'postgres', + "select oid from pg_tablespace where spcname = 'external_dir'").decode('utf-8').rstrip() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'tablespace_map') + + # overwrite tablespace_map file + with open(tablespace_map, "w") as f: + f.write("{0} {1}".format(oid, tblspace_new)) + f.close + + node.cleanup() + + self.restore_node(backup_dir, 'node', node, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that tablespace restore as symlink + tablespace_link = os.path.join(node.data_dir, 'pg_tblspc', oid) + self.assertTrue( + os.path.islink(tablespace_link), + "'%s' is not a symlink" % tablespace_link) + + self.assertEqual( + os.readlink(tablespace_link), + tblspace_new, + "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) + +# validate empty backup list +# page from future during validate +# page from future during backup + +# corrupt block, so file become unaligned: +# 712 Assert(header.compressed_size <= BLCKSZ); +# 713 +# 714 read_len = fread(compressed_page.data, 1, +# 715 MAXALIGN(header.compressed_size), in); +# 716 if (read_len != MAXALIGN(header.compressed_size)) +# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", +# 718 blknum, file->path, read_len, header.compressed_size); From 4c09debe6f3f801bf856019767bb88e9e0c60a82 Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Thu, 24 Nov 2022 10:26:35 +0100 Subject: [PATCH 1976/2107] Revert "PBCKP-306 add '_test' to tests files" This reverts commit 2b8a1532350af09eed827f3024f8f4a30baf04fb. --- tests/CVE_2018_1058_test.py | 129 - tests/archive_test.py | 2707 ------------------ tests/backup_test.py | 3564 ----------------------- tests/cfs_backup_test.py | 1235 -------- tests/cfs_catchup_test.py | 117 - tests/cfs_restore_test.py | 450 --- tests/cfs_validate_backup_test.py | 24 - tests/checkdb_test.py | 849 ------ tests/compatibility_test.py | 1500 ---------- tests/compression_test.py | 495 ---- tests/config_test.py | 113 - tests/delete_test.py | 822 ------ tests/delta_test.py | 1201 -------- tests/exclude_test.py | 338 --- tests/external_test.py | 2405 ---------------- tests/false_positive_test.py | 337 --- tests/incr_restore_test.py | 2300 --------------- tests/init_test.py | 138 - tests/locking_test.py | 629 ---- tests/logging_test.py | 345 --- tests/merge_test.py | 2759 ------------------ tests/option_test.py | 231 -- tests/page_test.py | 1424 ---------- tests/pgpro2068_test.py | 188 -- tests/pgpro560_test.py | 123 - tests/pgpro589_test.py | 72 - tests/ptrack_test.py | 4407 ----------------------------- tests/remote_test.py | 43 - tests/replica_test.py | 1654 ----------- tests/restore_test.py | 3822 ------------------------- tests/retention_test.py | 2529 ----------------- tests/set_backup_test.py | 476 ---- tests/show_test.py | 509 ---- tests/time_consuming_test.py | 77 - tests/time_stamp_test.py | 236 -- tests/validate_test.py | 4083 -------------------------- 36 files changed, 42331 deletions(-) delete mode 100644 tests/CVE_2018_1058_test.py delete mode 100644 tests/archive_test.py delete mode 100644 tests/backup_test.py delete mode 100644 tests/cfs_backup_test.py delete mode 100644 tests/cfs_catchup_test.py delete mode 100644 tests/cfs_restore_test.py delete mode 100644 tests/cfs_validate_backup_test.py delete mode 100644 tests/checkdb_test.py delete mode 100644 tests/compatibility_test.py delete mode 100644 tests/compression_test.py delete mode 100644 tests/config_test.py delete mode 100644 tests/delete_test.py delete mode 100644 tests/delta_test.py delete mode 100644 tests/exclude_test.py delete mode 100644 tests/external_test.py delete mode 100644 tests/false_positive_test.py delete mode 100644 tests/incr_restore_test.py delete mode 100644 tests/init_test.py delete mode 100644 tests/locking_test.py delete mode 100644 tests/logging_test.py delete mode 100644 tests/merge_test.py delete mode 100644 tests/option_test.py delete mode 100644 tests/page_test.py delete mode 100644 tests/pgpro2068_test.py delete mode 100644 tests/pgpro560_test.py delete mode 100644 tests/pgpro589_test.py delete mode 100644 tests/ptrack_test.py delete mode 100644 tests/remote_test.py delete mode 100644 tests/replica_test.py delete mode 100644 tests/restore_test.py delete mode 100644 tests/retention_test.py delete mode 100644 tests/set_backup_test.py delete mode 100644 tests/show_test.py delete mode 100644 tests/time_consuming_test.py delete mode 100644 tests/time_stamp_test.py delete mode 100644 tests/validate_test.py diff --git a/tests/CVE_2018_1058_test.py b/tests/CVE_2018_1058_test.py deleted file mode 100644 index cfd55cc60..000000000 --- a/tests/CVE_2018_1058_test.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -class CVE_2018_1058(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_basic_default_search_path(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - "CREATE FUNCTION public.pgpro_edition() " - "RETURNS text " - "AS $$ " - "BEGIN " - " RAISE 'pg_probackup vulnerable!'; " - "END " - "$$ LANGUAGE plpgsql") - - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - - # @unittest.skip("skip") - def test_basic_backup_modified_search_path(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - "CREATE FUNCTION public.pg_control_checkpoint(OUT timeline_id integer, OUT dummy integer) " - "RETURNS record " - "AS $$ " - "BEGIN " - " RAISE '% vulnerable!', 'pg_probackup'; " - "END " - "$$ LANGUAGE plpgsql") - - node.safe_psql( - 'postgres', - "CREATE FUNCTION public.pg_proc(OUT proname name, OUT dummy integer) " - "RETURNS record " - "AS $$ " - "BEGIN " - " RAISE '% vulnerable!', 'pg_probackup'; " - "END " - "$$ LANGUAGE plpgsql; " - "CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()") - - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertFalse( - 'pg_probackup vulnerable!' in log_content) - - # @unittest.skip("skip") - def test_basic_checkdb_modified_search_path(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) - node.slow_start() - - node.safe_psql( - 'postgres', - "CREATE FUNCTION public.pg_database(OUT datname name, OUT oid oid, OUT dattablespace oid) " - "RETURNS record " - "AS $$ " - "BEGIN " - " RAISE 'pg_probackup vulnerable!'; " - "END " - "$$ LANGUAGE plpgsql; " - "CREATE VIEW public.pg_database AS SELECT * FROM public.pg_database()") - - node.safe_psql( - 'postgres', - "CREATE FUNCTION public.pg_extension(OUT extname name, OUT extnamespace oid, OUT extversion text) " - "RETURNS record " - "AS $$ " - "BEGIN " - " RAISE 'pg_probackup vulnerable!'; " - "END " - "$$ LANGUAGE plpgsql; " - "CREATE FUNCTION public.pg_namespace(OUT oid oid, OUT nspname name) " - "RETURNS record " - "AS $$ " - "BEGIN " - " RAISE 'pg_probackup vulnerable!'; " - "END " - "$$ LANGUAGE plpgsql; " - "CREATE VIEW public.pg_extension AS SELECT * FROM public.pg_extension();" - "CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();" - ) - - try: - self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - self.assertEqual( - 1, 0, - "Expecting Error because amcheck{,_next} not installed\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) diff --git a/tests/archive_test.py b/tests/archive_test.py deleted file mode 100644 index 5e59dd268..000000000 --- a/tests/archive_test.py +++ /dev/null @@ -1,2707 +0,0 @@ -import os -import shutil -import gzip -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException -from datetime import datetime, timedelta -import subprocess -from sys import exit -from time import sleep -from distutils.dir_util import copy_tree - - -class ArchiveTest(ProbackupTest, unittest.TestCase): - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_pgpro434_1(self): - """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector from " - "generate_series(0,100) i") - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.backup_node( - backup_dir, 'node', node) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node) - node.slow_start() - - # Recreate backup catalog - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # Make backup - self.backup_node(backup_dir, 'node', node) - node.cleanup() - - # Restore Database - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), - 'data after restore not equal to original data') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_pgpro434_2(self): - """ - Check that timelines are correct. - WAITING PGPRO-1053 for --immediate - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) - - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FIRST TIMELINE - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100) i") - backup_id = self.backup_node(backup_dir, 'node', node) - node.safe_psql( - "postgres", - "insert into t_heap select 100501 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1) i") - - # SECOND TIMELIN - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--immediate', '--recovery-target-action=promote']) - node.slow_start() - - if self.verbose: - print(node.safe_psql( - "postgres", - "select redo_wal_file from pg_control_checkpoint()")) - self.assertFalse( - node.execute( - "postgres", - "select exists(select 1 " - "from t_heap where id = 100501)")[0][0], - 'data after restore not equal to original data') - - node.safe_psql( - "postgres", - "insert into t_heap select 2 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(100,200) i") - - backup_id = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "insert into t_heap select 100502 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - - # THIRD TIMELINE - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--immediate', '--recovery-target-action=promote']) - node.slow_start() - - if self.verbose: - print( - node.safe_psql( - "postgres", - "select redo_wal_file from pg_control_checkpoint()")) - - node.safe_psql( - "postgres", - "insert into t_heap select 3 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(200,300) i") - - backup_id = self.backup_node(backup_dir, 'node', node) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.safe_psql( - "postgres", - "insert into t_heap select 100503 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - - # FOURTH TIMELINE - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--immediate', '--recovery-target-action=promote']) - node.slow_start() - - if self.verbose: - print('Fourth timeline') - print(node.safe_psql( - "postgres", - "select redo_wal_file from pg_control_checkpoint()")) - - # FIFTH TIMELINE - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--immediate', '--recovery-target-action=promote']) - node.slow_start() - - if self.verbose: - print('Fifth timeline') - print(node.safe_psql( - "postgres", - "select redo_wal_file from pg_control_checkpoint()")) - - # SIXTH TIMELINE - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--immediate', '--recovery-target-action=promote']) - node.slow_start() - - if self.verbose: - print('Sixth timeline') - print(node.safe_psql( - "postgres", - "select redo_wal_file from pg_control_checkpoint()")) - - self.assertFalse( - node.execute( - "postgres", - "select exists(select 1 from t_heap where id > 100500)")[0][0], - 'data after restore not equal to original data') - - self.assertEqual( - result, - node.safe_psql( - "postgres", - "SELECT * FROM t_heap"), - 'data after restore not equal to original data') - - # @unittest.skip("skip") - def test_pgpro434_3(self): - """ - Check pg_stop_backup_timeout, needed backup_timeout - Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - gdb = self.backup_node( - backup_dir, 'node', node, - options=[ - "--archive-timeout=60", - "--log-level-file=LOG"], - gdb=True) - - # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - - self.set_auto_conf(node, {'archive_command': 'exit 1'}) - node.reload() - - gdb.continue_execution_until_exit() - - sleep(1) - - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - - # in PG =< 9.6 pg_stop_backup always wait - if self.get_version(node) < 100000: - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) - else: - self.assertIn( - "ERROR: WAL segment 000000010000000000000003 could not be archived in 60 seconds", - log_content) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertNotIn( - 'FailedAssertion', - log_content, - 'PostgreSQL crashed because of a failed assert') - - # @unittest.skip("skip") - def test_pgpro434_4(self): - """ - Check pg_stop_backup_timeout, libpq-timeout requested. - Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - gdb = self.backup_node( - backup_dir, 'node', node, - options=[ - "--archive-timeout=60", - "--log-level-file=info"], - gdb=True) - - # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - - self.set_auto_conf(node, {'archive_command': 'exit 1'}) - node.reload() - - os.environ["PGAPPNAME"] = "foo" - - pid = node.safe_psql( - "postgres", - "SELECT pid " - "FROM pg_stat_activity " - "WHERE application_name = 'pg_probackup'").decode('utf-8').rstrip() - - os.environ["PGAPPNAME"] = "pg_probackup" - - postgres_gdb = self.gdb_attach(pid) - postgres_gdb.set_breakpoint('do_pg_stop_backup') - postgres_gdb.continue_execution_until_running() - - gdb.continue_execution_until_exit() - # gdb._execute('detach') - - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - - if self.get_version(node) < 150000: - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) - else: - self.assertIn( - "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", - log_content) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertNotIn( - 'FailedAssertion', - log_content, - 'PostgreSQL crashed because of a failed assert') - - # @unittest.skip("skip") - def test_archive_push_file_exists(self): - """Archive-push if file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = '000000010000000000000001.gz' - file = os.path.join(wals_dir, filename) - else: - filename = '000000010000000000000001' - file = os.path.join(wals_dir, filename) - - with open(file, 'a+b') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close() - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") - log_file = os.path.join(node.logs_dir, 'postgresql.log') - - self.switch_wal_segment(node) - sleep(1) - - with open(log_file, 'r') as f: - log_content = f.read() - self.assertIn( - 'LOG: archive command failed with exit code 1', - log_content) - - self.assertIn( - 'DETAIL: The failed archive command was:', - log_content) - - self.assertIn( - 'pg_probackup archive-push WAL file', - log_content) - - self.assertIn( - 'WAL file already exists in archive with different checksum', - log_content) - - self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) - - if self.get_version(node) < 100000: - wal_src = os.path.join( - node.data_dir, 'pg_xlog', '000000010000000000000001') - else: - wal_src = os.path.join( - node.data_dir, 'pg_wal', '000000010000000000000001') - - if self.archive_compress: - with open(wal_src, 'rb') as f_in, gzip.open( - file, 'wb', compresslevel=1) as f_out: - shutil.copyfileobj(f_in, f_out) - else: - shutil.copyfile(wal_src, file) - - self.switch_wal_segment(node) - sleep(5) - - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'pg_probackup archive-push completed successfully', - log_content) - - # btw check that console coloring codes are not slipped into log file - self.assertNotIn('[0m', log_content) - - print(log_content) - - # @unittest.skip("skip") - def test_archive_push_file_exists_overwrite(self): - """Archive-push if file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = '000000010000000000000001.gz' - file = os.path.join(wals_dir, filename) - else: - filename = '000000010000000000000001' - file = os.path.join(wals_dir, filename) - - with open(file, 'a+b') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close() - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100500) i") - log_file = os.path.join(node.logs_dir, 'postgresql.log') - - self.switch_wal_segment(node) - sleep(1) - - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'LOG: archive command failed with exit code 1', log_content) - self.assertIn( - 'DETAIL: The failed archive command was:', log_content) - self.assertIn( - 'pg_probackup archive-push WAL file', log_content) - self.assertNotIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum', log_content) - - self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) - - self.set_archiving(backup_dir, 'node', node, overwrite=True) - node.reload() - self.switch_wal_segment(node) - sleep(5) - - with open(log_file, 'r') as f: - log_content = f.read() - self.assertTrue( - 'pg_probackup archive-push completed successfully' in log_content, - 'Expecting messages about successfull execution archive_command') - - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) - - # @unittest.skip("skip") - def test_archive_push_partial_file_exists(self): - """Archive-push if stale '.part' file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving( - backup_dir, 'node', node, - log_level='verbose', archive_timeout=60) - - node.slow_start() - - # this backup is needed only for validation to xid - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t1(a int)") - - xid = node.safe_psql( - "postgres", - "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() - - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - - filename_orig = filename_orig.decode('utf-8') - - # form up path to next .part WAL segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = filename_orig + '.gz' + '.part' - file = os.path.join(wals_dir, filename) - else: - filename = filename_orig + '.part' - file = os.path.join(wals_dir, filename) - - # emulate stale .part file - with open(file, 'a+b') as f: - f.write(b"blahblah") - f.flush() - f.close() - - self.switch_wal_segment(node) - sleep(70) - - # check that segment is archived - if self.archive_compress: - filename_orig = filename_orig + '.gz' - - file = os.path.join(wals_dir, filename_orig) - self.assertTrue(os.path.isfile(file)) - - # successful validate means that archive-push reused stale wal segment - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-xid={0}'.format(xid)]) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'Reusing stale temp WAL file', - log_content) - - # @unittest.skip("skip") - def test_archive_push_part_file_exists_not_stale(self): - """Archive-push if .part file exists and it is not stale""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, archive_timeout=60) - - node.slow_start() - - node.safe_psql( - "postgres", - "create table t1()") - self.switch_wal_segment(node) - - node.safe_psql( - "postgres", - "create table t2()") - - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - - filename_orig = filename_orig.decode('utf-8') - - # form up path to next .part WAL segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = filename_orig + '.gz' + '.part' - file = os.path.join(wals_dir, filename) - else: - filename = filename_orig + '.part' - file = os.path.join(wals_dir, filename) - - with open(file, 'a+b') as f: - f.write(b"blahblah") - f.flush() - f.close() - - self.switch_wal_segment(node) - sleep(30) - - with open(file, 'a+b') as f: - f.write(b"blahblahblahblah") - f.flush() - f.close() - - sleep(40) - - # check that segment is NOT archived - if self.archive_compress: - filename_orig = filename_orig + '.gz' - - file = os.path.join(wals_dir, filename_orig) - - self.assertFalse(os.path.isfile(file)) - - # log_file = os.path.join(node.logs_dir, 'postgresql.log') - # with open(log_file, 'r') as f: - # log_content = f.read() - # self.assertIn( - # 'is not stale', - # log_content) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_replica_archive(self): - """ - make node without archiving, take stream backup and - turn it into replica, set replica with archiving, - make archive backup from replica - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s', - 'checkpoint_timeout': '30s', - 'max_wal_size': '32MB'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - # Settings for Replica - self.restore_node(backup_dir, 'master', replica) - self.set_replica(master, replica, synchronous=True) - - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Change data on master, take FULL backup from replica, - # restore taken backup and check that restored data equal - # to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE FULL BACKUP TAKEN FROM replica - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - - self.set_auto_conf(node, {'port': node.port}) - node.slow_start() - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Change data on master, make PAGE backup from replica, - # restore taken backup and check that restored data equal - # to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,80680) i") - - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - self.wait_until_replica_catch_with_master(master, replica) - - backup_id = self.backup_node( - backup_dir, 'replica', - replica, backup_type='page', - options=[ - '--archive-timeout=60', - '--master-db=postgres', - '--master-host=localhost', - '--master-port={0}'.format(master.port), - '--stream']) - - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE PAGE BACKUP TAKEN FROM replica - node.cleanup() - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) - - self.set_auto_conf(node, {'port': node.port}) - - node.slow_start() - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_master_and_replica_parallel_archiving(self): - """ - make node 'master 'with archiving, - take archive backup and turn it into replica, - set replica with archiving, make archive backup from replica, - make archive backup from master - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s'} - ) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.init_pb(backup_dir) - # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'master', master) - # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - # GET PHYSICAL CONTENT FROM MASTER - pgdata_master = self.pgdata_content(master.data_dir) - - # Settings for Replica - self.restore_node(backup_dir, 'master', replica) - # CHECK PHYSICAL CORRECTNESS on REPLICA - pgdata_replica = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata_master, pgdata_replica) - - self.set_replica(master, replica) - # ADD INSTANCE REPLICA - self.add_instance(backup_dir, 'replica', replica) - # SET ARCHIVING FOR REPLICA - self.set_archiving(backup_dir, 'replica', replica, replica=True) - replica.slow_start(replica=True) - - # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0, 60000) i") - - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # TAKE FULL ARCHIVE BACKUP FROM MASTER - backup_id = self.backup_node(backup_dir, 'master', master) - self.validate_pb(backup_dir, 'master') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_basic_master_and_replica_concurrent_archiving(self): - """ - make node 'master 'with archiving, - take archive backup and turn it into replica, - set replica with archiving, - make sure that archiving on both node is working. - """ - if self.pg_config_version < self.version_to_num('9.6.0'): - self.skipTest('You need PostgreSQL >= 9.6 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'archive_timeout': '10s'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.init_pb(backup_dir) - # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - master.pgbench_init(scale=5) - - # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'master', master) - # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - # GET PHYSICAL CONTENT FROM MASTER - pgdata_master = self.pgdata_content(master.data_dir) - - # Settings for Replica - self.restore_node( - backup_dir, 'master', replica) - # CHECK PHYSICAL CORRECTNESS on REPLICA - pgdata_replica = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata_master, pgdata_replica) - - self.set_replica(master, replica, synchronous=False) - # ADD INSTANCE REPLICA - # self.add_instance(backup_dir, 'replica', replica) - # SET ARCHIVING FOR REPLICA - self.set_archiving(backup_dir, 'master', replica, replica=True) - replica.slow_start(replica=True) - - # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - # TAKE FULL ARCHIVE BACKUP FROM REPLICA - backup_id = self.backup_node(backup_dir, 'master', replica) - - self.validate_pb(backup_dir, 'master') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - - # TAKE FULL ARCHIVE BACKUP FROM MASTER - backup_id = self.backup_node(backup_dir, 'master', master) - self.validate_pb(backup_dir, 'master') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - - master.pgbench_init(scale=10) - - sleep(10) - - replica.promote() - - master.pgbench_init(scale=10) - replica.pgbench_init(scale=10) - - self.backup_node(backup_dir, 'master', master) - self.backup_node(backup_dir, 'master', replica) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_concurrent_archiving(self): - """ - Concurrent archiving from master, replica and cascade replica - https://github.com/postgrespro/pg_probackup/issues/327 - - For PG >= 11 it is expected to pass this test - """ - - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) - master.slow_start() - - master.pgbench_init(scale=10) - - # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'node', master) - - # Settings for Replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) - replica.slow_start(replica=True) - - # create cascade replicas - replica1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica1')) - replica1.cleanup() - - # Settings for casaced replica - self.restore_node(backup_dir, 'node', replica1) - self.set_replica(replica, replica1, synchronous=False) - self.set_auto_conf(replica1, {'port': replica1.port}) - replica1.slow_start(replica=True) - - # Take full backup from master - self.backup_node(backup_dir, 'node', master) - - pgbench = master.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '30', '-c', '1']) - - # Take several incremental backups from master - self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) - - self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) - - pgbench.wait() - pgbench.stdout.close() - - with open(os.path.join(master.logs_dir, 'postgresql.log'), 'r') as f: - log_content = f.read() - self.assertNotIn('different checksum', log_content) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - log_content = f.read() - self.assertNotIn('different checksum', log_content) - - with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: - log_content = f.read() - self.assertNotIn('different checksum', log_content) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_pg_receivexlog(self): - """Test backup with pg_receivexlog wal delivary method""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - - pg_receivexlog = self.run_binary( - [ - pg_receivexlog_path, '-p', str(node.port), '--synchronous', - '-D', os.path.join(backup_dir, 'wal', 'node') - ], asynchronous=True) - - if pg_receivexlog.returncode: - self.assertFalse( - True, - 'Failed to start pg_receivexlog: {0}'.format( - pg_receivexlog.communicate()[1])) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - self.backup_node(backup_dir, 'node', node) - - # PAGE - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - - self.backup_node( - backup_dir, - 'node', - node, - backup_type='page' - ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.validate_pb(backup_dir) - - # Check data correctness - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - self.assertEqual( - result, - node.safe_psql( - "postgres", "SELECT * FROM t_heap" - ), - 'data after restore not equal to original data') - - # Clean after yourself - pg_receivexlog.kill() - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_pg_receivexlog_compression_pg10(self): - """Test backup with pg_receivewal compressed wal delivary method""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - - pg_receivexlog = self.run_binary( - [ - pg_receivexlog_path, '-p', str(node.port), '--synchronous', - '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') - ], asynchronous=True) - - if pg_receivexlog.returncode: - self.assertFalse( - True, - 'Failed to start pg_receivexlog: {0}'.format( - pg_receivexlog.communicate()[1])) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - self.backup_node(backup_dir, 'node', node) - - # PAGE - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - - self.backup_node( - backup_dir, 'node', node, - backup_type='page' - ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.validate_pb(backup_dir) - - # Check data correctness - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), - 'data after restore not equal to original data') - - # Clean after yourself - pg_receivexlog.kill() - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_catalog(self): - """ - ARCHIVE replica: - - t6 |----------------------- - t5 | |------- - | | - t4 | |-------------- - | | - t3 | |--B1--|/|--B2-|/|-B3--- - | | - t2 |--A1--------A2--- - t1 ---------Y1--Y2-- - - ARCHIVE master: - t1 -Z1--Z2--- - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - - master.slow_start() - - # FULL - master.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - self.backup_node(backup_dir, 'master', master) - - # PAGE - master.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - - self.backup_node( - backup_dir, 'master', master, backup_type='page') - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - self.set_replica(master, replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - - replica.slow_start(replica=True) - - # FULL backup replica - Y1 = self.backup_node( - backup_dir, 'replica', replica, - options=['--stream', '--archive-timeout=60s']) - - master.pgbench_init(scale=5) - - # PAGE backup replica - Y2 = self.backup_node( - backup_dir, 'replica', replica, - backup_type='page', options=['--stream', '--archive-timeout=60s']) - - # create timeline t2 - replica.promote() - - # FULL backup replica - A1 = self.backup_node( - backup_dir, 'replica', replica) - - replica.pgbench_init(scale=5) - - replica.safe_psql( - 'postgres', - "CREATE TABLE t1 (a text)") - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - # DELTA backup replica - A2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='delta') - - # create timeline t3 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - replica.slow_start() - - B1 = self.backup_node( - backup_dir, 'replica', replica) - - replica.pgbench_init(scale=2) - - B2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') - - replica.pgbench_init(scale=2) - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - B3 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') - - replica.pgbench_init(scale=2) - - # create timeline t4 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=3', - '--recovery-target-action=promote']) - - replica.slow_start() - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't2 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,6) i') - - target_xid = None - with replica.connect("postgres") as con: - res = con.execute( - "INSERT INTO t1 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't3 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,10) i') - - # create timeline t5 - replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=4', - '--recovery-target-action=promote']) - - replica.slow_start() - - replica.safe_psql( - 'postgres', - 'CREATE TABLE ' - 't4 as select i, ' - 'repeat(md5(i::text),5006056) as fat_attr ' - 'from generate_series(0,6) i') - - # create timeline t6 - replica.cleanup() - - self.restore_node( - backup_dir, 'replica', replica, backup_id=A1, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - replica.slow_start() - - replica.pgbench_init(scale=2) - - sleep(5) - - show = self.show_archive(backup_dir, as_text=True) - show = self.show_archive(backup_dir) - - for instance in show: - if instance['instance'] == 'replica': - replica_timelines = instance['timelines'] - - if instance['instance'] == 'master': - master_timelines = instance['timelines'] - - # check that all timelines are ok - for timeline in replica_timelines: - self.assertTrue(timeline['status'], 'OK') - - # check that all timelines are ok - for timeline in master_timelines: - self.assertTrue(timeline['status'], 'OK') - - # create holes in t3 - wals_dir = os.path.join(backup_dir, 'wal', 'replica') - wals = [ - f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) - and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') - ] - wals.sort() - - # check that t3 is ok - self.show_archive(backup_dir) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - # check that t3 is not OK - show = self.show_archive(backup_dir) - - show = self.show_archive(backup_dir) - - for instance in show: - if instance['instance'] == 'replica': - replica_timelines = instance['timelines'] - - # sanity - for timeline in replica_timelines: - if timeline['tli'] == 1: - timeline_1 = timeline - continue - - if timeline['tli'] == 2: - timeline_2 = timeline - continue - - if timeline['tli'] == 3: - timeline_3 = timeline - continue - - if timeline['tli'] == 4: - timeline_4 = timeline - continue - - if timeline['tli'] == 5: - timeline_5 = timeline - continue - - if timeline['tli'] == 6: - timeline_6 = timeline - continue - - self.assertEqual(timeline_6['status'], "OK") - self.assertEqual(timeline_5['status'], "OK") - self.assertEqual(timeline_4['status'], "OK") - self.assertEqual(timeline_3['status'], "DEGRADED") - self.assertEqual(timeline_2['status'], "OK") - self.assertEqual(timeline_1['status'], "OK") - - self.assertEqual(len(timeline_3['lost-segments']), 2) - self.assertEqual( - timeline_3['lost-segments'][0]['begin-segno'], - '000000030000000000000012') - self.assertEqual( - timeline_3['lost-segments'][0]['end-segno'], - '000000030000000000000013') - self.assertEqual( - timeline_3['lost-segments'][1]['begin-segno'], - '000000030000000000000017') - self.assertEqual( - timeline_3['lost-segments'][1]['end-segno'], - '000000030000000000000017') - - self.assertEqual(len(timeline_6['backups']), 0) - self.assertEqual(len(timeline_5['backups']), 0) - self.assertEqual(len(timeline_4['backups']), 0) - self.assertEqual(len(timeline_3['backups']), 3) - self.assertEqual(len(timeline_2['backups']), 2) - self.assertEqual(len(timeline_1['backups']), 2) - - # check closest backup correctness - self.assertEqual(timeline_6['closest-backup-id'], A1) - self.assertEqual(timeline_5['closest-backup-id'], B2) - self.assertEqual(timeline_4['closest-backup-id'], B2) - self.assertEqual(timeline_3['closest-backup-id'], A1) - self.assertEqual(timeline_2['closest-backup-id'], Y2) - - # check parent tli correctness - self.assertEqual(timeline_6['parent-tli'], 2) - self.assertEqual(timeline_5['parent-tli'], 4) - self.assertEqual(timeline_4['parent-tli'], 3) - self.assertEqual(timeline_3['parent-tli'], 2) - self.assertEqual(timeline_2['parent-tli'], 1) - self.assertEqual(timeline_1['parent-tli'], 0) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_catalog_1(self): - """ - double segment - compressed and not - """ - if not self.archive_compress: - self.skipTest('You need to enable ARCHIVE_COMPRESSION ' - 'for this test to run') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) - - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=2) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - original_file = os.path.join(wals_dir, '000000010000000000000001.gz') - tmp_file = os.path.join(wals_dir, '000000010000000000000001') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - os.rename( - os.path.join(wals_dir, '000000010000000000000001'), - os.path.join(wals_dir, '000000010000000000000002')) - - show = self.show_archive(backup_dir) - - for instance in show: - timelines = instance['timelines'] - - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000001') - self.assertEqual(timeline['status'], 'OK') - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_catalog_2(self): - """ - double segment - compressed and not - """ - if not self.archive_compress: - self.skipTest('You need to enable ARCHIVE_COMPRESSION ' - 'for this test to run') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) - - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=2) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - original_file = os.path.join(wals_dir, '000000010000000000000001.gz') - tmp_file = os.path.join(wals_dir, '000000010000000000000001') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - os.rename( - os.path.join(wals_dir, '000000010000000000000001'), - os.path.join(wals_dir, '000000010000000000000002')) - - os.remove(original_file) - - show = self.show_archive(backup_dir) - - for instance in show: - timelines = instance['timelines'] - - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000002') - self.assertEqual(timeline['status'], 'OK') - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_options(self): - """ - check that '--archive-host', '--archive-user', '--archiver-port' - and '--restore-command' are working as expected. - """ - if not self.remote: - self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) - - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=1) - - node.cleanup() - - wal_dir = os.path.join(backup_dir, 'wal', 'node') - self.restore_node( - backup_dir, 'node', node, - options=[ - '--restore-command="cp {0}/%f %p"'.format(wal_dir), - '--archive-host=localhost', - '--archive-port=22', - '--archive-user={0}'.format(self.user) - ]) - - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - with open(recovery_conf, 'r') as f: - recovery_content = f.read() - - self.assertIn( - 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), - recovery_content) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--archive-host=localhost', - '--archive-port=22', - '--archive-user={0}'.format(self.user)]) - - with open(recovery_conf, 'r') as f: - recovery_content = f.read() - - self.assertIn( - "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " - "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " - "--remote-port=22 --remote-user={3}'".format( - self.probackup_path, backup_dir, 'node', self.user), - recovery_content) - - node.slow_start() - - node.safe_psql( - 'postgres', - 'select 1') - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_options_1(self): - """ - check that '--archive-host', '--archive-user', '--archiver-port' - and '--restore-command' are working as expected with set-config - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) - - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=1) - - node.cleanup() - - wal_dir = os.path.join(backup_dir, 'wal', 'node') - self.set_config( - backup_dir, 'node', - options=[ - '--restore-command="cp {0}/%f %p"'.format(wal_dir), - '--archive-host=localhost', - '--archive-port=22', - '--archive-user={0}'.format(self.user)]) - self.restore_node(backup_dir, 'node', node) - - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - with open(recovery_conf, 'r') as f: - recovery_content = f.read() - - self.assertIn( - 'restore_command = \'"cp {0}/%f %p"\''.format(wal_dir), - recovery_content) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--restore-command=none', - '--archive-host=localhost1', - '--archive-port=23', - '--archive-user={0}'.format(self.user) - ]) - - with open(recovery_conf, 'r') as f: - recovery_content = f.read() - - self.assertIn( - "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " - "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " - "--remote-port=23 --remote-user={3}'".format( - self.probackup_path, backup_dir, 'node', self.user), - recovery_content) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_undefined_wal_file_path(self): - """ - check that archive-push works correct with undefined - --wal-file-path - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - if os.name == 'posix': - archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( - self.probackup_path, backup_dir, 'node') - elif os.name == 'nt': - archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( - self.probackup_path, backup_dir, 'node').replace("\\","\\\\") - else: - self.assertTrue(False, 'Unexpected os family') - - self.set_auto_conf( - node, - {'archive_command': archive_command}) - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0, 10) i") - self.switch_wal_segment(node) - - # check - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_intermediate_archiving(self): - """ - check that archive-push works correct with --wal-file-path setting by user - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - node_pg_options = {} - if node.major_version >= 13: - node_pg_options['wal_keep_size'] = '0MB' - else: - node_pg_options['wal_keep_segments'] = '0' - self.set_auto_conf(node, node_pg_options) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'intermediate_dir') - shutil.rmtree(wal_dir, ignore_errors=True) - os.makedirs(wal_dir) - if os.name == 'posix': - self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) - elif os.name == 'nt': - self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) - else: - self.assertTrue(False, 'Unexpected os family') - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0, 10) i") - self.switch_wal_segment(node) - - wal_segment = '000000010000000000000001' - - self.run_pb(["archive-push", "-B", backup_dir, - "--instance=node", "-D", node.data_dir, - "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment]) - - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_waldir_outside_pgdata_archiving(self): - """ - check that archive-push works correct with symlinked waldir - """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest( - 'Skipped because waldir outside pgdata is supported since PG 10') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') - shutil.rmtree(external_wal_dir, ignore_errors=True) - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0, 10) i") - self.switch_wal_segment(node) - - # check - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_hexadecimal_timeline(self): - """ - Check that timelines are correct. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, log_level='verbose') - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=2) - - # create timelines - for i in range(1, 13): - # print(i) - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=['--recovery-target-timeline={0}'.format(i)]) - node.slow_start() - node.pgbench_init(scale=2) - - sleep(5) - - show = self.show_archive(backup_dir) - - timelines = show[0]['timelines'] - - print(timelines[0]) - - tli13 = timelines[0] - - self.assertEqual( - 13, - tli13['tli']) - - self.assertEqual( - 12, - tli13['parent-tli']) - - self.assertEqual( - backup_id, - tli13['closest-backup-id']) - - self.assertEqual( - '0000000D000000000000001C', - tli13['max-segno']) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_archiving_and_slots(self): - """ - Check that archiving don`t break slot - guarantee. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'max_wal_size': '64MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, log_level='verbose') - node.slow_start() - - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - - # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " - # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" - - self.run_binary( - [ - pg_receivexlog_path, '-p', str(node.port), '--synchronous', - '--create-slot', '--slot', 'archive_slot', '--if-not-exists' - ]) - - node.pgbench_init(scale=10) - - pg_receivexlog = self.run_binary( - [ - pg_receivexlog_path, '-p', str(node.port), '--synchronous', - '-D', os.path.join(backup_dir, 'wal', 'node'), - '--no-loop', '--slot', 'archive_slot', - '-Z', '1' - ], asynchronous=True) - - if pg_receivexlog.returncode: - self.assertFalse( - True, - 'Failed to start pg_receivexlog: {0}'.format( - pg_receivexlog.communicate()[1])) - - sleep(2) - - pg_receivexlog.kill() - - backup_id = self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=20) - - exit(1) - - def test_archive_push_sanity(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_mode': 'on', - 'archive_command': 'exit 1'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - node.pgbench_init(scale=50) - node.stop() - - self.set_archiving(backup_dir, 'node', node) - os.remove(os.path.join(node.logs_dir, 'postgresql.log')) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - # print(postgres_log_content) - # make sure that .backup file is not compressed - self.assertNotIn('.backup.gz', postgres_log_content) - self.assertNotIn('WARNING', postgres_log_content) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node( - backup_dir, 'node', replica, - data_dir=replica.data_dir, options=['-R']) - - # self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) - self.set_auto_conf(replica, {'archive_mode': 'always'}) - self.set_auto_conf(replica, {'hot_standby': 'on'}) - replica.slow_start(replica=True) - - self.wait_until_replica_catch_with_master(node, replica) - - node.pgbench_init(scale=5) - - replica.promote() - replica.pgbench_init(scale=10) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - replica_log_content = f.read() - - # make sure that .partial file is not compressed - self.assertNotIn('.partial.gz', replica_log_content) - # make sure that .history file is not compressed - self.assertNotIn('.history.gz', replica_log_content) - self.assertNotIn('WARNING', replica_log_content) - - output = self.show_archive( - backup_dir, 'node', as_json=False, as_text=True, - options=['--log-level-console=INFO']) - - self.assertNotIn('WARNING', output) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_pg_receivexlog_partial_handling(self): - """check that archive-get delivers .partial and .gz.partial files""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - if self.get_version(node) < 100000: - app_name = 'pg_receivexlog' - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - app_name = 'pg_receivewal' - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - - cmdline = [ - pg_receivexlog_path, '-p', str(node.port), '--synchronous', - '-D', os.path.join(backup_dir, 'wal', 'node')] - - if self.archive_compress and node.major_version >= 10: - cmdline += ['-Z', '1'] - - env = self.test_env - env["PGAPPNAME"] = app_name - pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env=env) - - if pg_receivexlog.returncode: - self.assertFalse( - True, - 'Failed to start pg_receivexlog: {0}'.format( - pg_receivexlog.communicate()[1])) - - self.set_auto_conf(node, {'synchronous_standby_names': app_name}) - self.set_auto_conf(node, {'synchronous_commit': 'on'}) - node.reload() - - # FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000000) i") - - # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--stream']) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(1000000,2000000) i") - - pg_receivexlog.kill() - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, node_restored.data_dir, - options=['--recovery-target=latest', '--recovery-target-action=promote']) - self.set_auto_conf(node_restored, {'port': node_restored.port}) - self.set_auto_conf(node_restored, {'hot_standby': 'off'}) - - node_restored.slow_start() - - result = node.safe_psql( - "postgres", - "select sum(id) from t_heap").decode('utf-8').rstrip() - - result_new = node_restored.safe_psql( - "postgres", - "select sum(id) from t_heap").decode('utf-8').rstrip() - - self.assertEqual(result, result_new) - - @unittest.skip("skip") - def test_multi_timeline_recovery_prefetching(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=50) - - target_xid = node.safe_psql( - 'postgres', - 'select txid_current()').rstrip() - - node.pgbench_init(scale=20) - - node.stop() - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-action=promote']) - - node.slow_start() - - node.pgbench_init(scale=20) - - target_xid = node.safe_psql( - 'postgres', - 'select txid_current()').rstrip() - - node.stop(['-m', 'immediate', '-D', node.data_dir]) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ -# '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', -# '--recovery-target-action=promote', - '--no-validate']) - node.slow_start() - - node.pgbench_init(scale=20) - result = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') - node.stop() - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ -# '--recovery-target-xid=100500', - '--recovery-target-timeline=3', -# '--recovery-target-action=promote', - '--no-validate']) - os.remove(os.path.join(node.logs_dir, 'postgresql.log')) - - restore_command = self.get_restore_command(backup_dir, 'node', node) - restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' - - if node.major_version >= 12: - node.append_conf( - 'postgresql.auto.conf', "restore_command = '{0}'".format(restore_command)) - else: - node.append_conf( - 'recovery.conf', "restore_command = '{0}'".format(restore_command)) - - node.slow_start() - - result_new = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') - - self.assertEqual(result, result_new) - - with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - # check that requesting of non-existing segment do not - # throwns aways prefetch - self.assertIn( - 'pg_probackup archive-get failed to ' - 'deliver WAL file: 000000030000000000000006', - postgres_log_content) - - self.assertIn( - 'pg_probackup archive-get failed to ' - 'deliver WAL file: 000000020000000000000006', - postgres_log_content) - - self.assertIn( - 'pg_probackup archive-get used prefetched ' - 'WAL segment 000000010000000000000006, prefetch state: 5/10', - postgres_log_content) - - def test_archive_get_batching_sanity(self): - """ - Make sure that batching works. - .gz file is corrupted and uncompressed is not, check that both - corruption detected and uncompressed file is used. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=50) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node( - backup_dir, 'node', replica, replica.data_dir) - self.set_replica(node, replica, log_shipping=True) - - if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': 'exit 1'}) - else: - replica.append_conf('recovery.conf', "restore_command = 'exit 1'") - - replica.slow_start(replica=True) - - # at this point replica is consistent - restore_command = self.get_restore_command(backup_dir, 'node', replica) - - restore_command += ' -j 2 --batch-size=10' - - # print(restore_command) - - if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) - else: - replica.append_conf( - 'recovery.conf', "restore_command = '{0}'".format(restore_command)) - - replica.restart() - - sleep(5) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'pg_probackup archive-get completed successfully, fetched: 10/10', - postgres_log_content) - self.assertIn('used prefetched WAL segment', postgres_log_content) - self.assertIn('prefetch state: 9/10', postgres_log_content) - self.assertIn('prefetch state: 8/10', postgres_log_content) - - def test_archive_get_prefetch_corruption(self): - """ - Make sure that WAL corruption is detected. - And --prefetch-dir is honored. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=50) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node( - backup_dir, 'node', replica, replica.data_dir) - self.set_replica(node, replica, log_shipping=True) - - if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': 'exit 1'}) - else: - replica.append_conf('recovery.conf', "restore_command = 'exit 1'") - - replica.slow_start(replica=True) - - # at this point replica is consistent - restore_command = self.get_restore_command(backup_dir, 'node', replica) - - restore_command += ' -j5 --batch-size=10 --log-level-console=VERBOSE' - #restore_command += ' --batch-size=2 --log-level-console=VERBOSE' - - if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) - else: - replica.append_conf( - 'recovery.conf', "restore_command = '{0}'".format(restore_command)) - - replica.restart() - - sleep(5) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'pg_probackup archive-get completed successfully, fetched: 10/10', - postgres_log_content) - self.assertIn('used prefetched WAL segment', postgres_log_content) - self.assertIn('prefetch state: 9/10', postgres_log_content) - self.assertIn('prefetch state: 8/10', postgres_log_content) - - replica.stop() - - # generate WAL, copy it into prefetch directory, then corrupt - # some segment - node.pgbench_init(scale=20) - sleep(20) - - # now copy WAL files into prefetch directory and corrupt some of them - archive_dir = os.path.join(backup_dir, 'wal', 'node') - files = os.listdir(archive_dir) - files.sort() - - for filename in [files[-4], files[-3], files[-2], files[-1]]: - src_file = os.path.join(archive_dir, filename) - - if node.major_version >= 10: - wal_dir = 'pg_wal' - else: - wal_dir = 'pg_xlog' - - if filename.endswith('.gz'): - dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename[:-3]) - with gzip.open(src_file, 'rb') as f_in, open(dst_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - else: - dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) - shutil.copyfile(src_file, dst_file) - - # print(dst_file) - - # corrupt file - if files[-2].endswith('.gz'): - filename = files[-2][:-3] - else: - filename = files[-2] - - prefetched_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) - - with open(prefetched_file, "rb+", 0) as f: - f.seek(8192*2) - f.write(b"SURIKEN") - f.flush() - f.close - - # enable restore_command - restore_command = self.get_restore_command(backup_dir, 'node', replica) - restore_command += ' --batch-size=2 --log-level-console=VERBOSE' - - if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) - else: - replica.append_conf( - 'recovery.conf', "restore_command = '{0}'".format(restore_command)) - - os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) - replica.slow_start(replica=True) - - sleep(60) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), - postgres_log_content) - - self.assertIn( - 'LOG: restored log file "{0}" from archive'.format(filename), - postgres_log_content) - - # @unittest.skip("skip") - def test_archive_show_partial_files_handling(self): - """ - check that files with '.part', '.part.gz', '.partial' and '.partial.gz' - siffixes are handled correctly - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=False) - - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') - - # .part file - node.safe_psql( - "postgres", - "create table t1()") - - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - - filename = filename.decode('utf-8') - - self.switch_wal_segment(node) - - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.part'.format(filename))) - - # .gz.part file - node.safe_psql( - "postgres", - "create table t2()") - - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - - filename = filename.decode('utf-8') - - self.switch_wal_segment(node) - - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.gz.part'.format(filename))) - - # .partial file - node.safe_psql( - "postgres", - "create table t3()") - - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - - filename = filename.decode('utf-8') - - self.switch_wal_segment(node) - - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.partial'.format(filename))) - - # .gz.partial file - node.safe_psql( - "postgres", - "create table t4()") - - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - - filename = filename.decode('utf-8') - - self.switch_wal_segment(node) - - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.gz.partial'.format(filename))) - - self.show_archive(backup_dir, 'node', options=['--log-level-file=VERBOSE']) - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log'), 'r') as f: - log_content = f.read() - - self.assertNotIn( - 'WARNING', - log_content) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_archive_empty_history_file(self): - """ - https://github.com/postgrespro/pg_probackup/issues/326 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - node.pgbench_init(scale=5) - - # FULL - self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=5) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target=latest', - '--recovery-target-action=promote']) - - # Node in timeline 2 - node.slow_start() - - node.pgbench_init(scale=5) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target=latest', - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - # Node in timeline 3 - node.slow_start() - - node.pgbench_init(scale=5) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target=latest', - '--recovery-target-timeline=3', - '--recovery-target-action=promote']) - - # Node in timeline 4 - node.slow_start() - node.pgbench_init(scale=5) - - # Truncate history files - for tli in range(2, 5): - file = os.path.join( - backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) - with open(file, "w+") as f: - f.truncate() - - timelines = self.show_archive(backup_dir, 'node', options=['--log-level-file=INFO']) - - # check that all timelines has zero switchpoint - for timeline in timelines: - self.assertEqual(timeline['switchpoint'], '0/0') - - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - wal_dir = os.path.join(backup_dir, 'wal', 'node') - - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), - log_content) - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), - log_content) - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), - log_content) - -# TODO test with multiple not archived segments. -# TODO corrupted file in archive. - -# important - switchpoint may be NullOffset LSN and not actually existing in archive to boot. -# so write WAL validation code accordingly - -# change wal-seg-size -# -# -#t3 ---------------- -# / -#t2 ---------------- -# / -#t1 -A-------- -# -# - - -#t3 ---------------- -# / -#t2 ---------------- -# / -#t1 -A-------- -# diff --git a/tests/backup_test.py b/tests/backup_test.py deleted file mode 100644 index db7ccf5a0..000000000 --- a/tests/backup_test.py +++ /dev/null @@ -1,3564 +0,0 @@ -import unittest -import os -from time import sleep, time -from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException -import shutil -from distutils.dir_util import copy_tree -from testgres import ProcessType, QueryException -import subprocess - - -class BackupTest(ProbackupTest, unittest.TestCase): - - def test_full_backup(self): - """ - Just test full backup with at least two segments - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - # we need to write a lot. Lets speedup a bit. - pg_options={"fsync": "off", "synchronous_commit": "off"}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Fill with data - # Have to use scale=100 to create second segment. - node.pgbench_init(scale=100, no_vacuum=True) - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node) - - out = self.validate_pb(backup_dir, 'node', backup_id) - self.assertIn( - "INFO: Backup {0} is valid".format(backup_id), - out) - - def test_full_backup_stream(self): - """ - Just test full backup with at least two segments in stream mode - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - # we need to write a lot. Lets speedup a bit. - pg_options={"fsync": "off", "synchronous_commit": "off"}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Fill with data - # Have to use scale=100 to create second segment. - node.pgbench_init(scale=100, no_vacuum=True) - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node, - options=["--stream"]) - - out = self.validate_pb(backup_dir, 'node', backup_id) - self.assertIn( - "INFO: Backup {0} is valid".format(backup_id), - out) - - # @unittest.skip("skip") - # @unittest.expectedFailure - # PGPRO-707 - def test_backup_modes_archive(self): - """standart backup modes with ARCHIVE WAL method""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - full_backup_id = self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] - - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "FULL") - - # postmaster.pid and postmaster.opts shouldn't be copied - excluded = True - db_dir = os.path.join( - backup_dir, "backups", 'node', full_backup_id, "database") - - for f in os.listdir(db_dir): - if ( - os.path.isfile(os.path.join(db_dir, f)) and - ( - f == "postmaster.pid" or - f == "postmaster.opts" - ) - ): - excluded = False - self.assertEqual(excluded, True) - - # page backup mode - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - - show_backup_1 = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup_1['status'], "OK") - self.assertEqual(show_backup_1['backup-mode'], "PAGE") - - # delta backup mode - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta") - - show_backup_2 = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup_2['status'], "OK") - self.assertEqual(show_backup_2['backup-mode'], "DELTA") - - # Check parent backup - self.assertEqual( - full_backup_id, - self.show_pb( - backup_dir, 'node', - backup_id=show_backup_1['id'])["parent-backup-id"]) - - self.assertEqual( - page_backup_id, - self.show_pb( - backup_dir, 'node', - backup_id=show_backup_2['id'])["parent-backup-id"]) - - # @unittest.skip("skip") - def test_smooth_checkpoint(self): - """full backup with smooth checkpoint""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, - options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - node.stop() - - # @unittest.skip("skip") - def test_incremental_backup_without_full(self): - """page backup without validated full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], - "ERROR") - - # @unittest.skip("skip") - def test_incremental_backup_corrupt_full(self): - """page-level backup with corrupted full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join( - backup_dir, "backups", "node", backup_id, - "database", "postgresql.conf") - os.remove(file) - - try: - self.validate_pb(backup_dir, 'node') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of validation of corrupted backup.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message and - "WARNING: Backup file" in e.message and "is not found" in e.message and - "WARNING: Backup {0} data files are corrupted".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - - # @unittest.skip("skip") - def test_delta_threads_stream(self): - """delta multi thread backup mode and stream""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") - - # @unittest.skip("skip") - def test_page_detect_corruption(self): - """make node, corrupt some page, check that backup failed""" - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - node.safe_psql( - "postgres", - "CHECKPOINT") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - path = os.path.join(node.data_dir, heap_path) - with open(path, "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) - self.assertEqual( - 1, 0, - "Expecting Error because data file is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Corruption detected in file "{0}", ' - 'block 1: page verification failed, calculated checksum'.format(path), - e.message) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], - 'ERROR', - "Backup Status should be ERROR") - - # @unittest.skip("skip") - def test_backup_detect_corruption(self): - """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if self.ptrack: - node.safe_psql( - "postgres", - "create extension ptrack") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "select count(*) from t_heap") - - node.safe_psql( - "postgres", - "update t_heap set id = id + 10000") - - node.stop() - - heap_fullpath = os.path.join(node.data_dir, heap_path) - - with open(heap_fullpath, "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_backup_detect_invalid_block_header(self): - """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if self.ptrack: - node.safe_psql( - "postgres", - "create extension ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "select count(*) from t_heap") - - node.safe_psql( - "postgres", - "update t_heap set id = id + 10000") - - node.stop() - - heap_fullpath = os.path.join(node.data_dir, heap_path) - with open(heap_fullpath, "rb+", 0) as f: - f.seek(8193) - f.write(b"blahblahblahblah") - f.flush() - f.close - - node.slow_start() - -# self.backup_node( -# backup_dir, 'node', node, -# backup_type="full", options=["-j", "4", "--stream"]) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_backup_detect_missing_permissions(self): - """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if self.ptrack: - node.safe_psql( - "postgres", - "create extension ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "select count(*) from t_heap") - - node.safe_psql( - "postgres", - "update t_heap set id = id + 10000") - - node.stop() - - heap_fullpath = os.path.join(node.data_dir, heap_path) - with open(heap_fullpath, "rb+", 0) as f: - f.seek(8193) - f.write(b"blahblahblahblah") - f.flush() - f.close - - node.slow_start() - -# self.backup_node( -# backup_dir, 'node', node, -# backup_type="full", options=["-j", "4", "--stream"]) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_backup_truncate_misaligned(self): - """ - make node, truncate file to size not even to BLCKSIZE, - take backup - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100000) i") - - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - heap_size = node.safe_psql( - "postgres", - "select pg_relation_size('t_heap')") - - with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f: - f.truncate(int(heap_size) - 4096) - f.flush() - f.close - - output = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"], return_id=False) - - self.assertIn("WARNING: File", output) - self.assertIn("invalid file size", output) - - # @unittest.skip("skip") - def test_tablespace_in_pgdata_pgpro_1376(self): - """PGPRO-1376 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node( - node, 'tblspace1', - tblspc_path=( - os.path.join( - node.data_dir, 'somedirectory', '100500')) - ) - - self.create_tblspace_in_node( - node, 'tblspace2', - tblspc_path=(os.path.join(node.data_dir)) - ) - - node.safe_psql( - "postgres", - "create table t_heap1 tablespace tblspace1 as select 1 as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - node.safe_psql( - "postgres", - "create table t_heap2 tablespace tblspace2 as select 1 as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - backup_id_1 = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "drop table t_heap2") - node.safe_psql( - "postgres", - "drop tablespace tblspace2") - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - pgdata = self.pgdata_content(node.data_dir) - - relfilenode = node.safe_psql( - "postgres", - "select 't_heap1'::regclass::oid" - ).decode('utf-8').rstrip() - - list = [] - for root, dirs, files in os.walk(os.path.join( - backup_dir, 'backups', 'node', backup_id_1)): - for file in files: - if file == relfilenode: - path = os.path.join(root, file) - list = list + [path] - - # We expect that relfilenode can be encountered only once - if len(list) > 1: - message = "" - for string in list: - message = message + string + "\n" - self.assertEqual( - 1, 0, - "Following file copied twice by backup:\n {0}".format( - message) - ) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_basic_tablespace_handling(self): - """ - make node, take full backup, check that restore with - tablespace mapping will end with error, take page backup, - check that restore with tablespace mapping will end with - success - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') - tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') - - self.create_tblspace_in_node( - node, 'some_lame_tablespace') - - self.create_tblspace_in_node( - node, 'tblspace1', - tblspc_path=tblspace1_old_path) - - self.create_tblspace_in_node( - node, 'tblspace2', - tblspc_path=tblspace2_old_path) - - node.safe_psql( - "postgres", - "create table t_heap_lame tablespace some_lame_tablespace " - "as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - node.safe_psql( - "postgres", - "create table t_heap2 tablespace tblspace2 as select 1 as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - tblspace1_new_path = self.get_tblspace_path(node, 'tblspace1_new') - tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace1_new_path), - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace2_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - node.safe_psql( - "postgres", - "drop table t_heap_lame") - - node.safe_psql( - "postgres", - "drop tablespace some_lame_tablespace") - - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream"]) - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace1_new_path), - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace2_new_path)]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_tablespace_handling_1(self): - """ - make node with tablespace A, take full backup, check that restore with - tablespace mapping of tablespace B will end with error - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') - tblspace2_old_path = self.get_tblspace_path(node, 'tblspace2_old') - - tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - - self.create_tblspace_in_node( - node, 'tblspace1', - tblspc_path=tblspace1_old_path) - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_tablespace_handling_2(self): - """ - make node without tablespaces, take full backup, check that restore with - tablespace mapping will end with error - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') - tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_drop_rel_during_full_backup(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 512): - node.safe_psql( - "postgres", - "create table t_heap_{0} as select i" - " as id from generate_series(0,100) i".format(i)) - - node.safe_psql( - "postgres", - "VACUUM") - - node.pgbench_init(scale=10) - - relative_path_1 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() - - relative_path_2 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() - - absolute_path_1 = os.path.join(node.data_dir, relative_path_1) - absolute_path_2 = os.path.join(node.data_dir, relative_path_2) - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=LOG', '--log-level-console=LOG', '--progress'], - gdb=True) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - for i in range(1, 512): - node.safe_psql( - "postgres", - "drop table t_heap_{0}".format(i)) - - node.safe_psql( - "postgres", - "CHECKPOINT") - - node.safe_psql( - "postgres", - "CHECKPOINT") - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - # log_content = f.read() - # self.assertTrue( - # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, - # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - @unittest.skip("skip") - def test_drop_db_during_full_backup(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 2): - node.safe_psql( - "postgres", - "create database t_heap_{0}".format(i)) - - node.safe_psql( - "postgres", - "VACUUM") - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=[ - '--stream', '--log-level-file=LOG', - '--log-level-console=LOG', '--progress']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - for i in range(1, 2): - node.safe_psql( - "postgres", - "drop database t_heap_{0}".format(i)) - - node.safe_psql( - "postgres", - "CHECKPOINT") - - node.safe_psql( - "postgres", - "CHECKPOINT") - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - # log_content = f.read() - # self.assertTrue( - # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, - # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_drop_rel_during_backup_delta(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=10) - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - absolute_path = os.path.join(node.data_dir, relative_path) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # DELTA backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - gdb=True, options=['--log-level-file=LOG']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - node.safe_psql( - "postgres", - "DROP TABLE t_heap") - - node.safe_psql( - "postgres", - "CHECKPOINT") - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( - 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_drop_rel_during_backup_page(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - absolute_path = os.path.join(node.data_dir, relative_path) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "insert into t_heap select i" - " as id from generate_series(101,102) i") - - # PAGE backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', - gdb=True, options=['--log-level-file=LOG']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - os.remove(absolute_path) - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - gdb.kill() - - pgdata = self.pgdata_content(node.data_dir) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) - self.assertNotIn(relative_path, filelist) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_persistent_slot_for_stream_backup(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '40MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "SELECT pg_create_physical_replication_slot('slot_1')") - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1']) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1']) - - # @unittest.skip("skip") - def test_basic_temp_slot_for_stream_backup(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_size': '40MB'}) - - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--temp-slot']) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1', '--temp-slot']) - - # @unittest.skip("skip") - def test_backup_concurrent_drop_table(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--compress'], - gdb=True) - - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - - node.safe_psql( - 'postgres', - 'DROP TABLE pgbench_accounts') - - # do checkpoint to guarantee filenode removal - node.safe_psql( - 'postgres', - 'CHECKPOINT') - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - gdb.kill() - - show_backup = self.show_pb(backup_dir, 'node')[0] - - self.assertEqual(show_backup['status'], "OK") - - # @unittest.skip("skip") - def test_pg_11_adjusted_wal_segment_size(self): - """""" - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=[ - '--data-checksums', - '--wal-segsize=64'], - pg_options={ - 'min_wal_size': '128MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=5) - - # FULL STREAM backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - pgbench = node.pgbench(options=['-T', '5', '-c', '2']) - pgbench.wait() - - # PAGE STREAM backup - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--stream']) - - pgbench = node.pgbench(options=['-T', '5', '-c', '2']) - pgbench.wait() - - # DELTA STREAM backup - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - pgbench = node.pgbench(options=['-T', '5', '-c', '2']) - pgbench.wait() - - # FULL ARCHIVE backup - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '5', '-c', '2']) - pgbench.wait() - - # PAGE ARCHIVE backup - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '5', '-c', '2']) - pgbench.wait() - - # DELTA ARCHIVE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - pgdata = self.pgdata_content(node.data_dir) - - # delete - output = self.delete_pb( - backup_dir, 'node', - options=[ - '--expired', - '--delete-wal', - '--retention-redundancy=1']) - - # validate - self.validate_pb(backup_dir) - - # merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # restore - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=backup_id) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_sigint_handling(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=LOG']) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - gdb.kill() - - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - # @unittest.skip("skip") - def test_sigterm_handling(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=LOG']) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGTERM') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - # @unittest.skip("skip") - def test_sigquit_handling(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, options=['--stream']) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGQUIT') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - # @unittest.skip("skip") - def test_drop_table(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - connect_1 = node.connect("postgres") - connect_1.execute( - "create table t_heap as select i" - " as id from generate_series(0,100) i") - connect_1.commit() - - connect_2 = node.connect("postgres") - connect_2.execute("SELECT * FROM t_heap") - connect_2.commit() - - # DROP table - connect_2.execute("DROP TABLE t_heap") - connect_2.commit() - - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # @unittest.skip("skip") - def test_basic_missing_file_permissions(self): - """""" - if os.name == 'nt': - self.skipTest('Skipped because it is POSIX only test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pg_class')").decode('utf-8').rstrip() - - full_path = os.path.join(node.data_dir, relative_path) - - os.chmod(full_path, 000) - - try: - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - os.chmod(full_path, 700) - - # @unittest.skip("skip") - def test_basic_missing_dir_permissions(self): - """""" - if os.name == 'nt': - self.skipTest('Skipped because it is POSIX only test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - full_path = os.path.join(node.data_dir, 'pg_twophase') - - os.chmod(full_path, 000) - - try: - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open directory', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - os.rmdir(full_path) - - # @unittest.skip("skip") - def test_backup_with_least_privileges_role(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - if self.ptrack: - node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack; " - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 15 - else: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - if self.ptrack: - node.safe_psql( - "backupdb", - "GRANT USAGE ON SCHEMA ptrack TO backup") - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " - "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") - - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - datname='backupdb', options=['--stream', '-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, - datname='backupdb', options=['-U', 'backup']) - - # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='page', datname='backupdb', - options=['--stream', '-U', 'backup']) - - # DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - datname='backupdb', options=['--stream', '-U', 'backup']) - - # PTRACK - if self.ptrack: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - datname='backupdb', options=['--stream', '-U', 'backup']) - - # @unittest.skip("skip") - def test_parent_choosing(self): - """ - PAGE3 <- RUNNING(parent should be FULL) - PAGE2 <- OK - PAGE1 <- CORRUPT - FULL - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - full_id = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGE1 to ERROR - self.change_backup_status(backup_dir, 'node', page1_id, 'ERROR') - - # PAGE3 - page3_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=LOG']) - - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_file_content = f.read() - - self.assertIn( - "WARNING: Backup {0} has invalid parent: {1}. " - "Cannot be a parent".format(page2_id, page1_id), - log_file_content) - - self.assertIn( - "WARNING: Backup {0} has status: ERROR. " - "Cannot be a parent".format(page1_id), - log_file_content) - - self.assertIn( - "Parent backup: {0}".format(full_id), - log_file_content) - - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], - full_id) - - # @unittest.skip("skip") - def test_parent_choosing_1(self): - """ - PAGE3 <- RUNNING(parent should be FULL) - PAGE2 <- OK - PAGE1 <- (missing) - FULL - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - full_id = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Delete PAGE1 - shutil.rmtree( - os.path.join(backup_dir, 'backups', 'node', page1_id)) - - # PAGE3 - page3_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=LOG']) - - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_file_content = f.read() - - self.assertIn( - "WARNING: Backup {0} has missing parent: {1}. " - "Cannot be a parent".format(page2_id, page1_id), - log_file_content) - - self.assertIn( - "Parent backup: {0}".format(full_id), - log_file_content) - - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], - full_id) - - # @unittest.skip("skip") - def test_parent_choosing_2(self): - """ - PAGE3 <- RUNNING(backup should fail) - PAGE2 <- OK - PAGE1 <- OK - FULL <- (missing) - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - full_id = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Delete FULL - shutil.rmtree( - os.path.join(backup_dir, 'backups', 'node', full_id)) - - # PAGE3 - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=LOG']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Valid full backup on current timeline 1 is not found' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb( - backup_dir, 'node')[2]['status'], - 'ERROR') - - # @unittest.skip("skip") - def test_backup_with_less_privileges_role(self): - """ - check permissions correctness from documentation: - https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'archive_mode': 'always', - 'checkpoint_timeout': '60s', - 'wal_level': 'logical'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - if self.ptrack: - node.safe_psql( - 'backupdb', - 'CREATE EXTENSION ptrack') - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) - # >= 15 - else: - node.safe_psql( - 'backupdb', - "BEGIN; " - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) - - # enable STREAM backup - node.safe_psql( - 'backupdb', - 'ALTER ROLE backup WITH REPLICATION;') - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - datname='backupdb', options=['--stream', '-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, - datname='backupdb', options=['-U', 'backup']) - - # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='page', datname='backupdb', - options=['--stream', '-U', 'backup']) - - # DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - datname='backupdb', options=['--stream', '-U', 'backup']) - - # PTRACK - if self.ptrack: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - datname='backupdb', options=['--stream', '-U', 'backup']) - - if self.get_version(node) < 90600: - return - - # Restore as replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'node', replica) - self.set_replica(node, replica) - self.add_instance(backup_dir, 'replica', replica) - self.set_config( - backup_dir, 'replica', - options=['--archive-timeout=120s', '--log-level-console=LOG']) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_auto_conf(replica, {'hot_standby': 'on'}) - - # freeze bgwriter to get rid of RUNNING XACTS records - # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] - # gdb_checkpointer = self.gdb_attach(bgwriter_pid) - - copy_tree( - os.path.join(backup_dir, 'wal', 'node'), - os.path.join(backup_dir, 'wal', 'replica')) - - replica.slow_start(replica=True) - - # self.switch_wal_segment(node) - # self.switch_wal_segment(node) - - self.backup_node( - backup_dir, 'replica', replica, - datname='backupdb', options=['-U', 'backup']) - - # stream full backup from replica - self.backup_node( - backup_dir, 'replica', replica, - datname='backupdb', options=['--stream', '-U', 'backup']) - -# self.switch_wal_segment(node) - - # PAGE backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) - - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['--stream', '-U', 'backup']) - - # DELTA backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['--stream', '-U', 'backup']) - - # PTRACK backup from replica - if self.ptrack: - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - datname='backupdb', options=['--stream', '-U', 'backup']) - - @unittest.skip("skip") - def test_issue_132(self): - """ - https://github.com/postgrespro/pg_probackup/issues/132 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - with node.connect("postgres") as conn: - for i in range(50000): - conn.execute( - "CREATE TABLE t_{0} as select 1".format(i)) - conn.commit() - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - exit(1) - - @unittest.skip("skip") - def test_issue_132_1(self): - """ - https://github.com/postgrespro/pg_probackup/issues/132 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - with node.connect("postgres") as conn: - for i in range(30000): - conn.execute( - "CREATE TABLE t_{0} as select 1".format(i)) - conn.commit() - - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream'], old_binary=True) - - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream'], old_binary=True) - - node.cleanup() - - # make sure that new binary can detect corruption - try: - self.validate_pb(backup_dir, 'node', backup_id=full_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir, 'node', backup_id=delta_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], - 'Backup STATUS should be "CORRUPT"') - - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], - 'Backup STATUS should be "ORPHAN"') - - # check that revalidation is working correctly - try: - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], - 'Backup STATUS should be "CORRUPT"') - - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], - 'Backup STATUS should be "ORPHAN"') - - # check that '--no-validate' do not allow to restore ORPHAN backup -# try: -# self.restore_node( -# backup_dir, 'node', node, backup_id=delta_id, -# options=['--no-validate']) -# # we should die here because exception is what we expect to happen -# self.assertEqual( -# 1, 0, -# "Expecting Error because FULL backup is CORRUPT" -# "\n Output: {0} \n CMD: {1}".format( -# repr(self.output), self.cmd)) -# except ProbackupException as e: -# self.assertIn( -# 'Insert data', -# e.message, -# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( -# repr(e.message), self.cmd)) - - node.cleanup() - - output = self.restore_node( - backup_dir, 'node', node, backup_id=full_id, options=['--force']) - - self.assertIn( - 'WARNING: Backup {0} has status: CORRUPT'.format(full_id), - output) - - self.assertIn( - 'WARNING: Backup {0} is corrupt.'.format(full_id), - output) - - self.assertIn( - 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), - output) - - self.assertIn( - 'INFO: Restore of backup {0} completed.'.format(full_id), - output) - - node.cleanup() - - output = self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, options=['--force']) - - self.assertIn( - 'WARNING: Backup {0} is orphan.'.format(delta_id), - output) - - self.assertIn( - 'WARNING: Backup {0} is not valid, restore is forced'.format(full_id), - output) - - self.assertIn( - 'WARNING: Backup {0} is not valid, restore is forced'.format(delta_id), - output) - - self.assertIn( - 'INFO: Restore of backup {0} completed.'.format(delta_id), - output) - - def test_note_sanity(self): - """ - test that adding note to backup works as expected - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=LOG', '--note=test_note']) - - show_backups = self.show_pb(backup_dir, 'node') - - print(self.show_pb(backup_dir, as_text=True, as_json=True)) - - self.assertEqual(show_backups[0]['note'], "test_note") - - self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - - self.assertNotIn( - 'note', - backup_meta) - - # @unittest.skip("skip") - def test_parent_backup_made_by_newer_version(self): - """incremental backup with parent made by newer version""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - control_file = os.path.join( - backup_dir, "backups", "node", backup_id, - "backup.control") - - version = self.probackup_version - fake_new_version = str(int(version.split('.')[0]) + 1) + '.0.0' - - with open(control_file, 'r') as f: - data = f.read(); - - data = data.replace(version, fake_new_version) - - with open(control_file, 'w') as f: - f.write(data); - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "if parent made by newer version.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - - # @unittest.skip("skip") - def test_issue_289(self): - """ - https://github.com/postgrespro/pg_probackup/issues/289 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=10s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because full backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - "INFO: Wait for WAL segment", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "ERROR: Create new full backup before an incremental one", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - - # @unittest.skip("skip") - def test_issue_290(self): - """ - https://github.com/postgrespro/pg_probackup/issues/290 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - os.rmdir( - os.path.join(backup_dir, "wal", "node")) - - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - options=['--archive-timeout=10s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because full backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - "INFO: Wait for WAL segment", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WAL archive directory is not accessible", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - - @unittest.skip("skip") - def test_issue_203(self): - """ - https://github.com/postgrespro/pg_probackup/issues/203 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - with node.connect("postgres") as conn: - for i in range(1000000): - conn.execute( - "CREATE TABLE t_{0} as select 1".format(i)) - conn.commit() - - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j2']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', - node_restored, data_dir=node_restored.data_dir) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_issue_231(self): - """ - https://github.com/postgrespro/pg_probackup/issues/231 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - datadir = os.path.join(node.data_dir, '123') - - try: - self.backup_node( - backup_dir, 'node', node, data_dir='{0}'.format(datadir)) - except: - pass - - out = self.backup_node(backup_dir, 'node', node, options=['--stream'], return_id=False) - - # it is a bit racy - self.assertIn("WARNING: Cannot create directory", out) - - def test_incr_backup_filenode_map(self): - """ - https://github.com/postgrespro/pg_probackup/issues/320 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) - node1.cleanup() - - node.pgbench_init(scale=5) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.safe_psql( - 'postgres', - 'reindex index pg_type_oid_index') - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # incremental restore into node1 - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'select 1') - - # @unittest.skip("skip") - def test_missing_wal_segment(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=10) - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - # get segments in pg_wal, sort then and remove all but the latest - pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') - - if node.major_version >= 10: - pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') - else: - pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog') - - # Full backup in streaming mode - gdb = self.backup_node( - backup_dir, 'node', node, datname='backupdb', - options=['--stream', '--log-level-file=INFO'], gdb=True) - - # break at streaming start - gdb.set_breakpoint('start_WAL_streaming') - gdb.run_until_break() - - # generate some more data - node.pgbench_init(scale=3) - - # remove redundant WAL segments in pg_wal - files = os.listdir(pg_wal_dir) - files.sort(reverse=True) - - # leave first two files in list - del files[:2] - for filename in files: - os.remove(os.path.join(pg_wal_dir, filename)) - - gdb.continue_execution_until_exit() - - self.assertIn( - 'unexpected termination of replication stream: ERROR: requested WAL segment', - gdb.output) - - self.assertIn( - 'has already been removed', - gdb.output) - - self.assertIn( - 'ERROR: Interrupted during waiting for WAL streaming', - gdb.output) - - self.assertIn( - 'WARNING: backup in progress, stop backup', - gdb.output) - - # TODO: check the same for PAGE backup - - # @unittest.skip("skip") - def test_missing_replication_permission(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) -# self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(node, replica) - replica.slow_start(replica=True) - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 15 - else: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") - - sleep(2) - replica.promote() - - # Delta backup - try: - self.backup_node( - backup_dir, 'node', replica, backup_type='delta', - data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - # 9.5: ERROR: must be superuser or replication role to run a backup - # >=9.6: FATAL: must be superuser or replication role to start walsender - self.assertRegex( - e.message, - "ERROR: must be superuser or replication role to run a backup|FATAL: must be superuser or replication role to start walsender", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_missing_replication_permission_1(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(node, replica) - replica.slow_start(replica=True) - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # > 15 - else: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") - - replica.promote() - - # PAGE - output = self.backup_node( - backup_dir, 'node', replica, backup_type='page', - data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'], - return_id=False) - - self.assertIn( - 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', - output) - - # Messages before 14 - # 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender' - # Messages for >=14 - # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' - # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' - self.assertRegex( - output, - r'WARNING: could not connect to database backupdb: (connection to server (on socket "/tmp/.s.PGSQL.\d+"|at "localhost" \(127.0.0.1\), port \d+) failed: ){0,1}' - 'FATAL: must be superuser or replication role to start walsender') - - # @unittest.skip("skip") - def test_basic_backup_default_transaction_read_only(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'default_transaction_read_only': 'on'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - try: - node.safe_psql( - 'postgres', - 'create temp table t1()') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertIn( - "cannot execute CREATE TABLE in a read-only transaction", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream']) - - # DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) - - # PAGE backup - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # @unittest.skip("skip") - def test_backup_atexit(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=5) - - # Full backup in streaming mode - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=VERBOSE'], gdb=True) - - # break at streaming start - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - sleep(1) - - self.assertEqual( - self.show_pb( - backup_dir, 'node')[0]['status'], 'ERROR') - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - #print(log_content) - self.assertIn( - 'WARNING: backup in progress, stop backup', - log_content) - - if self.get_version(node) < 150000: - self.assertIn( - 'FROM pg_catalog.pg_stop_backup', - log_content) - else: - self.assertIn( - 'FROM pg_catalog.pg_backup_stop', - log_content) - - self.assertIn( - 'setting its status to ERROR', - log_content) - - # @unittest.skip("skip") - def test_pg_stop_backup_missing_permissions(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=5) - - self.simple_bootstrap(node, 'backup') - - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') - elif self.get_version(node) < 150000: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') - else: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') - - - # Full backup in streaming mode - try: - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '-U', 'backup']) - # we should die here because exception is what we expect to happen - if self.get_version(node) < 150000: - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_stop_backup " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - else: - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_backup_stop " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node) < 150000: - self.assertIn( - "ERROR: permission denied for function pg_stop_backup", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertIn( - "ERROR: permission denied for function pg_backup_stop", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "query was: SELECT pg_catalog.txid_snapshot_xmax", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_start_time(self): - """Test, that option --start-time allows to set backup_id and restore""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--start-time={0}'.format(str(startTime))]) - # restore FULL backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_full'), - backup_id=base36enc(startTime)) - - #FULL backup with incorrect start time - try: - startTime = str(int(time()-100000)) - self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--start-time={0}'.format(startTime)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - 'Expecting Error because start time for new backup must be newer ' - '\n Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # DELTA backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--start-time={0}'.format(str(startTime))]) - # restore DELTA backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_delta'), - backup_id=base36enc(startTime)) - - # PAGE backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--stream', '--start-time={0}'.format(str(startTime))]) - # restore PAGE backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_page'), - backup_id=base36enc(startTime)) - - # PTRACK backup - if self.ptrack: - node.safe_psql( - 'postgres', - 'create extension ptrack') - - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream', '--start-time={0}'.format(str(startTime))]) - # restore PTRACK backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_ptrack'), - backup_id=base36enc(startTime)) - - # @unittest.skip("skip") - def test_start_time_few_nodes(self): - """Test, that we can synchronize backup_id's for different DBs""" - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir1 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup1') - self.init_pb(backup_dir1) - self.add_instance(backup_dir1, 'node1', node1) - self.set_archiving(backup_dir1, 'node1', node1) - node1.slow_start() - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir2 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup2') - self.init_pb(backup_dir2) - self.add_instance(backup_dir2, 'node2', node2) - self.set_archiving(backup_dir2, 'node2', node2) - node2.slow_start() - - # FULL backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='full', - options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='full', - options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[0] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] - self.assertEqual(show_backup1['id'], show_backup2['id']) - - # DELTA backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='delta', - options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='delta', - options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[1] - show_backup2 = self.show_pb(backup_dir2, 'node2')[1] - self.assertEqual(show_backup1['id'], show_backup2['id']) - - # PAGE backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='page', - options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='page', - options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[2] - show_backup2 = self.show_pb(backup_dir2, 'node2')[2] - self.assertEqual(show_backup1['id'], show_backup2['id']) - - # PTRACK backup - if self.ptrack: - node1.safe_psql( - 'postgres', - 'create extension ptrack') - node2.safe_psql( - 'postgres', - 'create extension ptrack') - - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='ptrack', - options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='ptrack', - options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[3] - show_backup2 = self.show_pb(backup_dir2, 'node2')[3] - self.assertEqual(show_backup1['id'], show_backup2['id']) - diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py deleted file mode 100644 index 28ef275df..000000000 --- a/tests/cfs_backup_test.py +++ /dev/null @@ -1,1235 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): - # --- Begin --- # - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join( - self.tmp_path, self.module_name, self.fname, 'backup') - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'cfs_encryption': 'off', - 'max_wal_senders': '2', - 'shared_buffers': '200MB' - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - - self.node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name)) - - self.assertIn( - tblspace_name, str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertIn( - "compression=true", str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - - # --- Section: Full --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace(self): - """Case: Check fullbackup empty compressed tablespace""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_stream(self): - """Case: Check fullbackup empty compressed tablespace with options stream""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table(self): - """Case: Make full backup after created table in the tablespace""" - if not self.enterprise: - return - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "\n ERROR: {0}\n CMD: {1}".format( - repr(e.message), - repr(self.cmd) - ) - ) - return False - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in {0}".format( - os.path.join(self.backup_dir, 'node', backup_id)) - ) - - # check cfm size - cfms = find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table_stream(self): - """ - Case: Make full backup after created table in the tablespace with option --stream - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # --- Section: Incremental from empty tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_page_doesnt_store_unchanged_cfm(self): - """ - Case: Test page backup doesn't store cfm file if table were not modified - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files is found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # --- Section: Incremental from fill tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,25) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap', tblspace_name) - ) - - full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap') - ) - - page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_full, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - full_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_page, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - page_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments_in_multiple_tablespaces(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - tblspace_name_1 = 'tblspace_name_1' - tblspace_name_2 = 'tblspace_name_2' - - self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) - self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1)) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2)) - - full_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - full_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_1') - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_2') - ) - - page_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - page_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - full_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), - 'Lost data after restore') - self.assertEqual( - full_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - page_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), - 'Lost data after restore') - self.assertEqual( - page_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # --- Make backup with not valid data(broken .cfm) --- # - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_cfm_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - os.remove(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_file_pg_compression_from_tablespace_dir(self): - os.remove( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0]) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_data_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - os.remove(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_cfm_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - corrupt_file(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_data_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - corrupt_file(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_file_pg_compression_into_tablespace_dir(self): - - corrupted_file = find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0] - - self.assertTrue( - corrupt_file(corrupted_file), - "ERROR: File is not corrupted or it missing" - ) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - -# # --- End ---# - - -#class CfsBackupEncTest(CfsBackupNoEncTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py deleted file mode 100644 index 43c3f18f1..000000000 --- a/tests/cfs_catchup_test.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_full_catchup_with_tablespace(self): - """ - Test tablespace transfers - """ - # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), - set_replication = True - ) - src_pg.slow_start() - tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') - self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) - src_pg.safe_psql( - "postgres", - "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # 1st check: compare data directories - self.compare_pgdata( - self.pgdata_content(src_pg.data_dir), - self.pgdata_content(dst_pg.data_dir) - ) - - # check cfm size - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # make changes in master tablespace - src_pg.safe_psql( - "postgres", - "UPDATE ultimate_question SET answer = -1") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - - # and now delta backup - dst_pg.stop() - - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # check cfm size again - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - - # 3rd check: run verification query - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py deleted file mode 100644 index 6b69b4ffe..000000000 --- a/tests/cfs_restore_test.py +++ /dev/null @@ -1,450 +0,0 @@ -""" -restore - Syntax: - - pg_probackup restore -B backupdir --instance instance_name - [-D datadir] - [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] - [-j num_threads] [--progress] [-q] [-v] - -""" -import os -import unittest -import shutil - -from .helpers.cfs_helpers import find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' -tblspace_name_new = 'cfs_tblspace_new' - - -class CfsRestoreBase(ProbackupTest, unittest.TestCase): - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ -# 'ptrack_enable': 'on', - 'cfs_encryption': 'off', - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - self.add_data_in_cluster() - - self.backup_id = None - try: - self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - def add_data_in_cluster(self): - pass - - -class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_empty_tablespace_from_fullbackup(self): - """ - Case: Restore empty tablespace from valid full backup. - """ - self.node.stop(["-m", "immediate"]) - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), - "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." - ) - - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ).decode("UTF-8") - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, - "ERROR: The tablespace not restored or it restored without compressions" - ) - - -class CfsRestoreNoencTest(CfsRestoreBase): - def add_data_in_cluster(self): - self.node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,1e5) i'.format('t1', tblspace_name) - ) - self.table_t1 = self.node.safe_psql( - "postgres", - "SELECT * FROM t1" - ) - - # --- Restore from full backup ---# - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in tablespace dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_3_jobs(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location_5_jobs(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-j", "3", "-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) - ) - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_new_jobs(self): - pass - -# --------------------------------------------------------- # - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_new_jobs(self): - """ - Case: Restore from backup to new location, four jobs - """ - pass - - -#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() -# -# -#class CfsRestoreEncTest(CfsRestoreNoencTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup_test.py b/tests/cfs_validate_backup_test.py deleted file mode 100644 index 343020dfc..000000000 --- a/tests/cfs_validate_backup_test.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import unittest -import random - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): - def setUp(self): - pass - - def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): - pass - - def tearDown(self): - pass - - -#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py deleted file mode 100644 index 2caf4fcb2..000000000 --- a/tests/checkdb_test.py +++ /dev/null @@ -1,849 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -import subprocess -from testgres import QueryException -import shutil -import sys -import time - - -class CheckdbTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_checkdb_amcheck_only_sanity(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - node.safe_psql( - "postgres", - "create index on t_heap(id)") - - node.safe_psql( - "postgres", - "create table idxpart (a int) " - "partition by range (a)") - - node.safe_psql( - "postgres", - "create index on idxpart(a)") - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - # simple sanity - try: - self.checkdb_node( - options=['--skip-block-validation']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because --amcheck option is missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Option '--skip-block-validation' must be " - "used with '--amcheck' option", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # simple sanity - output = self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - output) - self.assertIn( - 'All checked indexes are valid', - output) - - # logging to file sanity - try: - self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because log_directory missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Cannot save checkdb logs to a file. " - "You must specify --log-directory option when " - "running checkdb with --log-level-file option enabled", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # If backup_dir provided, then instance name must be - # provided too - try: - self.checkdb_node( - backup_dir, - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because log_directory missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: required parameter not specified: --instance", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # checkdb can use default or set in config values, - # if backup_dir and instance name are provided - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '-d', 'postgres', '-p', str(node.port)]) - - # check that file present and full of messages - os.path.isfile(log_file_path) - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - log_file_content) - self.assertIn( - 'VERBOSE: (query)', - log_file_content) - os.unlink(log_file_path) - - # log-level-file and log-directory are provided - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), - '-d', 'postgres', '-p', str(node.port)]) - - # check that file present and full of messages - os.path.isfile(log_file_path) - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - log_file_content) - self.assertIn( - 'VERBOSE: (query)', - log_file_content) - os.unlink(log_file_path) - - gdb = self.checkdb_node( - gdb=True, - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), - '-d', 'postgres', '-p', str(node.port)]) - - gdb.set_breakpoint('amcheck_one_index') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "drop table t_heap") - - gdb.remove_all_breakpoints() - - gdb.continue_execution_until_exit() - - # check that message about missing index is present - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'ERROR: checkdb --amcheck finished with failure', - log_file_content) - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'postgres' " - "for index: 'public.t_heap_id_idx':", - log_file_content) - self.assertIn( - 'ERROR: could not open relation with OID', - log_file_content) - - # Clean after yourself - gdb.kill() - node.stop() - - # @unittest.skip("skip") - def test_basic_checkdb_amcheck_only_sanity(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # create two databases - node.safe_psql("postgres", "create database db1") - try: - node.safe_psql( - "db1", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "db1", - "create extension amcheck_next") - - node.safe_psql("postgres", "create database db2") - try: - node.safe_psql( - "db2", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "db2", - "create extension amcheck_next") - - # init pgbench in two databases and corrupt both indexes - node.pgbench_init(scale=5, dbname='db1') - node.pgbench_init(scale=5, dbname='db2') - - node.safe_psql( - "db2", - "alter index pgbench_accounts_pkey rename to some_index") - - index_path_1 = os.path.join( - node.data_dir, - node.safe_psql( - "db1", - "select pg_relation_filepath('pgbench_accounts_pkey')").decode('utf-8').rstrip()) - - index_path_2 = os.path.join( - node.data_dir, - node.safe_psql( - "db2", - "select pg_relation_filepath('some_index')").decode('utf-8').rstrip()) - - try: - self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because some db was not amchecked" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Some databases were not amchecked", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - node.stop() - - # Let`s do index corruption - with open(index_path_1, "rb+", 0) as f: - f.seek(42000) - f.write(b"blablahblahs") - f.flush() - f.close - - with open(index_path_2, "rb+", 0) as f: - f.seek(42000) - f.write(b"blablahblahs") - f.flush() - f.close - - node.slow_start() - - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - try: - self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because some db was not amchecked" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: checkdb --amcheck finished with failure", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # corruption of both indexes in db1 and db2 must be detected - # also the that amcheck is not installed in 'postgres' - # should be logged - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'db1' " - "for index: 'public.pgbench_accounts_pkey':", - log_file_content) - - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'db2' " - "for index: 'public.some_index':", - log_file_content) - - self.assertIn( - "ERROR: checkdb --amcheck finished with failure", - log_file_content) - - # Clean after yourself - node.stop() - - # @unittest.skip("skip") - def test_checkdb_block_validation_sanity(self): - """make node, corrupt some pages, check that checkdb failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - # sanity - try: - self.checkdb_node() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pgdata must be specified\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: required parameter not specified: PGDATA (-D, --pgdata)", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.checkdb_node( - data_dir=node.data_dir, - options=['-d', 'postgres', '-p', str(node.port)]) - - self.checkdb_node( - backup_dir, 'node', - options=['-d', 'postgres', '-p', str(node.port)]) - - heap_full_path = os.path.join(node.data_dir, heap_path) - - with open(heap_full_path, "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - with open(heap_full_path, "rb+", 0) as f: - f.seek(42000) - f.write(b"bla") - f.flush() - f.close - - try: - self.checkdb_node( - backup_dir, 'node', - options=['-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of data corruption\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Checkdb failed", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Corruption detected in file "{0}", block 1'.format( - os.path.normpath(heap_full_path)), - e.message) - - self.assertIn( - 'WARNING: Corruption detected in file "{0}", block 5'.format( - os.path.normpath(heap_full_path)), - e.message) - - # Clean after yourself - node.stop() - - def test_checkdb_checkunique(self): - """Test checkunique parameter of amcheck.bt_index_check function""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - node.slow_start() - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - # Part of https://commitfest.postgresql.org/32/2976/ patch test - node.safe_psql( - "postgres", - "CREATE TABLE bttest_unique(a varchar(50), b varchar(1500), c bytea, d varchar(50)); " - "ALTER TABLE bttest_unique SET (autovacuum_enabled = false); " - "CREATE UNIQUE INDEX bttest_unique_idx ON bttest_unique(a,b); " - "UPDATE pg_catalog.pg_index SET indisunique = false " - "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " - "INSERT INTO bttest_unique " - " SELECT i::text::varchar, " - " array_to_string(array( " - " SELECT substr('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', ((random()*(36-1)+1)::integer), 1) " - " FROM generate_series(1,1300)),'')::varchar, " - " i::text::bytea, i::text::varchar " - " FROM generate_series(0,1) AS i, generate_series(0,30) AS x; " - "UPDATE pg_catalog.pg_index SET indisunique = true " - "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " - "DELETE FROM bttest_unique WHERE ctid::text='(0,2)'; " - "DELETE FROM bttest_unique WHERE ctid::text='(4,2)'; " - "DELETE FROM bttest_unique WHERE ctid::text='(4,3)'; " - "DELETE FROM bttest_unique WHERE ctid::text='(9,3)';") - - # run without checkunique option (error will not detected) - output = self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - output) - self.assertIn( - 'All checked indexes are valid', - output) - - # run with checkunique option - try: - self.checkdb_node( - options=[ - '--amcheck', - '--skip-block-validation', - '--checkunique', - '-d', 'postgres', '-p', str(node.port)]) - if (ProbackupTest.enterprise and - (self.get_version(node) >= 111300 and self.get_version(node) < 120000 - or self.get_version(node) >= 120800 and self.get_version(node) < 130000 - or self.get_version(node) >= 130400)): - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of index corruption\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - else: - self.assertRegex( - self.output, - r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") - except ProbackupException as e: - self.assertIn( - "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", - e.message) - - # Clean after yourself - node.stop() - - # @unittest.skip("skip") - def test_checkdb_sigint_handling(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - # FULL backup - gdb = self.checkdb_node( - backup_dir, 'node', gdb=True, - options=[ - '-d', 'postgres', '-j', '2', - '--skip-block-validation', - '--progress', - '--amcheck', '-p', str(node.port)]) - - gdb.set_breakpoint('amcheck_one_index') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - with open(node.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('could not receive data from client', output) - self.assertNotIn('could not send data to client', output) - self.assertNotIn('connection to client lost', output) - - # Clean after yourself - gdb.kill() - node.stop() - - # @unittest.skip("skip") - def test_checkdb_with_least_privileges(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'CREATE DATABASE backupdb') - - try: - node.safe_psql( - "backupdb", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "backupdb", - "create extension amcheck_next") - - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC;") - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function - - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' -# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - - # PG 10 - elif self.get_version(node) > 100000 and self.get_version(node) < 110000: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') - - if ProbackupTest.enterprise: - # amcheck-1.1 - node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') - else: - # amcheck-1.0 - node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') - # >= 11 < 14 - elif self.get_version(node) > 110000 and self.get_version(node) < 140000: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - - # checkunique parameter - if ProbackupTest.enterprise: - if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 - or self.get_version(node) >= 120800 and self.get_version(node) < 130000 - or self.get_version(node) >= 130400): - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") - # >= 14 - else: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - - # checkunique parameter - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") - - if ProbackupTest.enterprise: - node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') - - # checkdb - try: - self.checkdb_node( - backup_dir, 'node', - options=[ - '--amcheck', '-U', 'backup', - '-d', 'backupdb', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because permissions are missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "INFO: Amcheck succeeded for database 'backupdb'", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Extension 'amcheck' or 'amcheck_next' are " - "not installed in database postgres", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "ERROR: Some databases were not amchecked", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # Clean after yourself - node.stop() diff --git a/tests/compatibility_test.py b/tests/compatibility_test.py deleted file mode 100644 index 591afb069..000000000 --- a/tests/compatibility_test.py +++ /dev/null @@ -1,1500 +0,0 @@ -import unittest -import subprocess -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from sys import exit -import shutil - - -def check_manual_tests_enabled(): - return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' - - -def check_ssh_agent_path_exists(): - return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ - - -class CompatibilityTest(ProbackupTest, unittest.TestCase): - - def setUp(self): - self.fname = self.id().split('.')[3] - - # @unittest.expectedFailure - @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') - @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') - # @unittest.skip("skip") - def test_catchup_with_different_remote_major_pg(self): - """ - Decription in jira issue PBCKP-236 - This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6 - - Prerequisites: - - pg_probackup git tag for PBCKP 2.5.1 - - master pg_probackup build should be made for PGPROEE11 - - agent pg_probackup build should be made for PGPROEE9_6 - - Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes - the PBCKP-236 problem - - Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH= - for the test - - Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' - without pg_probackup executable - """ - - self.verbose = True - self.remote = True - # please use your own local path like - # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' - pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] - - src_pg = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'src'), - set_replication=True, - ) - src_pg.slow_start() - src_pg.safe_psql( - "postgres", - "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - - # do full catchup - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( - backup_mode='FULL', - source_pgdata=src_pg.data_dir, - destination_node=dst_pg, - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] - ) - - dst_options = {'port': str(dst_pg.port)} - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - dst_pg.stop() - - src_pg.safe_psql( - "postgres", - "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") - - # do delta catchup with remote pg_probackup agent with another postgres major version - # this DELTA backup should fail without PBCKP-236 patch. - self.catchup_node( - backup_mode='DELTA', - source_pgdata=src_pg.data_dir, - destination_node=dst_pg, - # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version - options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_page(self): - """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=10) - - # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.show_pb(backup_dir) - - self.validate_pb(backup_dir) - - # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Page BACKUP with old binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='page', - old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Page BACKUP with new binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"]) - - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.safe_psql( - 'postgres', - 'create table tmp as select * from pgbench_accounts where aid < 1000') - - node.safe_psql( - 'postgres', - 'delete from pgbench_accounts') - - node.safe_psql( - 'postgres', - 'VACUUM') - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.safe_psql( - 'postgres', - 'insert into pgbench_accounts select * from pgbench_accounts') - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_delta(self): - """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=10) - - # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.show_pb(backup_dir) - - self.validate_pb(backup_dir) - - # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Delta BACKUP with old binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Delta BACKUP with new binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.safe_psql( - 'postgres', - 'create table tmp as select * from pgbench_accounts where aid < 1000') - - node.safe_psql( - 'postgres', - 'delete from pgbench_accounts') - - node.safe_psql( - 'postgres', - 'VACUUM') - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.safe_psql( - 'postgres', - 'insert into pgbench_accounts select * from pgbench_accounts') - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_ptrack(self): - """Description in jira issue PGPRO-434""" - - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=10) - - # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.show_pb(backup_dir) - - self.validate_pb(backup_dir) - - # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # ptrack BACKUP with old binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--recovery-target=latest", - "--recovery-target-action=promote"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Ptrack BACKUP with new binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"] - ) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--recovery-target=latest", - "--recovery-target-action=promote"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_compression(self): - """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=10) - - # FULL backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, - options=['--compress']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # restore OLD FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # PAGE backup with OLD binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"]) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, - backup_type='page', - old_binary=True, - options=['--compress']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # PAGE backup with new binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"]) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, - backup_type='page', - options=['--compress']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Delta backup with old binary - self.delete_pb(backup_dir, 'node', backup_id) - - self.backup_node( - backup_dir, 'node', node, - old_binary=True, - options=['--compress']) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"]) - - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=['--compress'], - old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Delta backup with new binary - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"]) - - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=['--compress']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge(self): - """ - Create node, take FULL and PAGE backups with old binary, - merge them with new binary - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, - old_binary=True) - - node.pgbench_init(scale=1) - - # PAGE backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.merge_backup(backup_dir, "node", backup_id) - - self.show_pb(backup_dir, as_text=True, as_json=False) - - # restore OLD FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge_1(self): - """ - Create node, take FULL and PAGE backups with old binary, - merge them with new binary. - old binary version =< 2.2.7 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=20) - - # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE1 backup with OLD binary - self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - - node.safe_psql( - 'postgres', - 'DELETE from pgbench_accounts') - - node.safe_psql( - 'postgres', - 'VACUUM pgbench_accounts') - - # PAGE2 backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - - pgdata = self.pgdata_content(node.data_dir) - - # merge chain created by old binary with new binary - output = self.merge_backup(backup_dir, "node", backup_id) - - # check that in-place is disabled - self.assertIn( - "WARNING: In-place merge is disabled " - "because of storage format incompatibility", output) - - # restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge_2(self): - """ - Create node, take FULL and PAGE backups with old binary, - merge them with new binary. - old binary version =< 2.2.7 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=50) - - node.safe_psql( - 'postgres', - 'VACUUM pgbench_accounts') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE1 backup with OLD binary - page1 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True) - - pgdata1 = self.pgdata_content(node.data_dir) - - node.safe_psql( - 'postgres', - "DELETE from pgbench_accounts where ctid > '(10,1)'") - - # PAGE2 backup with OLD binary - page2 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True) - - pgdata2 = self.pgdata_content(node.data_dir) - - # PAGE3 backup with OLD binary - page3 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True) - - pgdata3 = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE4 backup with NEW binary - page4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - pgdata4 = self.pgdata_content(node.data_dir) - - # merge backups one by one and check data correctness - # merge PAGE1 - self.merge_backup( - backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) - - # check data correctness for PAGE1 - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, backup_id=page1, - options=['--log-level-file=VERBOSE']) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - # merge PAGE2 - self.merge_backup(backup_dir, "node", page2) - - # check data correctness for PAGE2 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - # merge PAGE3 - self.show_pb(backup_dir, 'node', page3) - self.merge_backup(backup_dir, "node", page3) - self.show_pb(backup_dir, 'node', page3) - - # check data correctness for PAGE3 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - # merge PAGE4 - self.merge_backup(backup_dir, "node", page4) - - # check data correctness for PAGE4 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata4, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge_3(self): - """ - Create node, take FULL and PAGE backups with old binary, - merge them with new binary. - old binary version =< 2.2.7 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=50) - - node.safe_psql( - 'postgres', - 'VACUUM pgbench_accounts') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, old_binary=True, options=['--compress']) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE1 backup with OLD binary - page1 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--compress']) - - pgdata1 = self.pgdata_content(node.data_dir) - - node.safe_psql( - 'postgres', - "DELETE from pgbench_accounts where ctid > '(10,1)'") - - # PAGE2 backup with OLD binary - page2 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--compress']) - - pgdata2 = self.pgdata_content(node.data_dir) - - # PAGE3 backup with OLD binary - page3 = self.backup_node( - backup_dir, 'node', node, - backup_type='page', old_binary=True, options=['--compress']) - - pgdata3 = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE4 backup with NEW binary - page4 = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) - pgdata4 = self.pgdata_content(node.data_dir) - - # merge backups one by one and check data correctness - # merge PAGE1 - self.merge_backup( - backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) - - # check data correctness for PAGE1 - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, backup_id=page1, - options=['--log-level-file=VERBOSE']) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - # merge PAGE2 - self.merge_backup(backup_dir, "node", page2) - - # check data correctness for PAGE2 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - # merge PAGE3 - self.show_pb(backup_dir, 'node', page3) - self.merge_backup(backup_dir, "node", page3) - self.show_pb(backup_dir, 'node', page3) - - # check data correctness for PAGE3 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - # merge PAGE4 - self.merge_backup(backup_dir, "node", page4) - - # check data correctness for PAGE4 - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata4, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge_4(self): - """ - Start merge between minor version, crash and retry it. - old binary version =< 2.4.0 - """ - if self.version_to_num(self.old_probackup_version) > self.version_to_num('2.4.0'): - self.assertTrue( - False, 'You need pg_probackup old_binary =< 2.4.0 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=20) - - node.safe_psql( - 'postgres', - 'VACUUM pgbench_accounts') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, old_binary=True, options=['--compress']) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "20", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE backup with NEW binary - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) - pgdata = self.pgdata_content(node.data_dir) - - # merge PAGE4 - gdb = self.merge_backup(backup_dir, "node", page_id, gdb=True) - - gdb.set_breakpoint('rename') - gdb.run_until_break() - gdb.continue_execution_until_break(500) - gdb._execute('signal SIGKILL') - - try: - self.merge_backup(backup_dir, "node", page_id) - self.assertEqual( - 1, 0, - "Expecting Error because of format changes.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Retry of failed merge for backups with different " - "between minor versions is forbidden to avoid data corruption " - "because of storage format changes introduced in 2.4.0 version, " - "please take a new full backup", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_backward_compatibility_merge_5(self): - """ - Create node, take FULL and PAGE backups with old binary, - merge them with new binary. - old binary version >= STORAGE_FORMAT_VERSION (2.4.4) - """ - if self.version_to_num(self.old_probackup_version) < self.version_to_num('2.4.4'): - self.assertTrue( - False, 'OLD pg_probackup binary must be >= 2.4.4 for this test') - - self.assertNotEqual( - self.version_to_num(self.old_probackup_version), - self.version_to_num(self.probackup_version)) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=20) - - # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "1", "-T", "10", "--no-vacuum"]) - pgbench.wait() - pgbench.stdout.close() - - # PAGE1 backup with OLD binary - self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - - node.safe_psql( - 'postgres', - 'DELETE from pgbench_accounts') - - node.safe_psql( - 'postgres', - 'VACUUM pgbench_accounts') - - # PAGE2 backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - - pgdata = self.pgdata_content(node.data_dir) - - # merge chain created by old binary with new binary - output = self.merge_backup(backup_dir, "node", backup_id) - - # check that in-place is disabled - self.assertNotIn( - "WARNING: In-place merge is disabled " - "because of storage format incompatibility", output) - - # restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_page_vacuum_truncate(self): - """ - make node, create table, take full backup, - delete all data, vacuum relation, - take page backup, insert some data, - take second page backup, - restore latest page backup using new binary - and check data correctness - old binary should be 2.2.x version - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - id1 = self.backup_node(backup_dir, 'node', node, old_binary=True) - pgdata1 = self.pgdata_content(node.data_dir) - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - id2 = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - pgdata2 = self.pgdata_content(node.data_dir) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1) i") - - id3 = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) - pgdata3 = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id1) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id2) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id3) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - # @unittest.skip("skip") - def test_page_vacuum_truncate_compression(self): - """ - make node, create table, take full backup, - delete all data, vacuum relation, - take page backup, insert some data, - take second page backup, - restore latest page backup using new binary - and check data correctness - old binary should be 2.2.x version - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node',node, old_binary=True, options=['--compress']) - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='page', - old_binary=True, options=['--compress']) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1) i") - - self.backup_node( - backup_dir, 'node', node, backup_type='page', - old_binary=True, options=['--compress']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_page_vacuum_truncate_compressed_1(self): - """ - make node, create table, take full backup, - delete all data, vacuum relation, - take page backup, insert some data, - take second page backup, - restore latest page backup using new binary - and check data correctness - old binary should be 2.2.x version - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - id1 = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=['--compress']) - pgdata1 = self.pgdata_content(node.data_dir) - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - id2 = self.backup_node( - backup_dir, 'node', node, backup_type='page', - old_binary=True, options=['--compress']) - pgdata2 = self.pgdata_content(node.data_dir) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1) i") - - id3 = self.backup_node( - backup_dir, 'node', node, backup_type='page', - old_binary=True, options=['--compress']) - pgdata3 = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id1) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id2) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id3) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - node_restored.cleanup() - - # @unittest.skip("skip") - def test_hidden_files(self): - """ - old_version should be < 2.3.0 - Create hidden file in pgdata, take backup - with old binary, then try to delete backup - with new binary - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close() - - backup_id = self.backup_node( - backup_dir, 'node',node, old_binary=True, options=['--stream']) - - self.delete_pb(backup_dir, 'node', backup_id) - - # @unittest.skip("skip") - def test_compatibility_tablespace(self): - """ - https://github.com/postgrespro/pg_probackup/issues/348 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"], old_binary=True) - - tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old') - - self.create_tblspace_in_node( - node, 'tblspace', - tblspc_path=tblspace_old_path) - - node.safe_psql( - "postgres", - "create table t_heap_lame tablespace tblspace " - "as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - - tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=["-j", "4", "--stream"], old_binary=True) - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace_old_path, tblspace_new_path)]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/compression_test.py b/tests/compression_test.py deleted file mode 100644 index 94f2dffff..000000000 --- a/tests/compression_test.py +++ /dev/null @@ -1,495 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess - - -class CompressionTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_basic_compression_stream_zlib(self): - """ - make archive node, make full and page stream backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=[ - '--stream', - '--compress-algorithm=zlib']) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=[ - '--stream', '--compress-algorithm=zlib']) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress-algorithm=zlib']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - - def test_compression_archive_zlib(self): - """ - make archive node, make full and page backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=["--compress-algorithm=zlib"]) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(0,2) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=["--compress-algorithm=zlib"]) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--compress-algorithm=zlib']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() - - def test_compression_stream_pglz(self): - """ - make archive node, make full and page stream backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--compress-algorithm=pglz']) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--stream', '--compress-algorithm=pglz']) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress-algorithm=pglz']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() - - def test_compression_archive_pglz(self): - """ - make archive node, make full and page backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--compress-algorithm=pglz']) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--compress-algorithm=pglz']) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--compress-algorithm=pglz']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() - - def test_compression_wrong_algorithm(self): - """ - make archive node, make full and page backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--compress-algorithm=bla-blah']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because compress-algorithm is invalid.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, - 'ERROR: invalid compress algorithm value "bla-blah"\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_incompressible_pages(self): - """ - make archive node, create table with incompressible toast pages, - take backup with compression, make sure that page was not compressed, - restore backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Full - self.backup_node( - backup_dir, 'node', node, - options=[ - '--compress-algorithm=zlib', - '--compress-level=0']) - - node.pgbench_init(scale=3) - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - '--compress-algorithm=zlib', - '--compress-level=0']) - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() diff --git a/tests/config_test.py b/tests/config_test.py deleted file mode 100644 index b1a0f9295..000000000 --- a/tests/config_test.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest -import subprocess -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from sys import exit -from shutil import copyfile - - -class ConfigTest(ProbackupTest, unittest.TestCase): - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_remove_instance_config(self): - """remove pg_probackup.conself.f""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.show_pb(backup_dir) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - conf_file = os.path.join( - backup_dir, 'backups','node', 'pg_probackup.conf') - - os.unlink(os.path.join(backup_dir, 'backups','node', 'pg_probackup.conf')) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because pg_probackup.conf is missing. " - ".\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: could not open file "{0}": ' - 'No such file or directory'.format(conf_file), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_corrupt_backup_content(self): - """corrupt backup_content.control""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - full1_id = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - 'postgres', - 'create table t1()') - - fulle2_id = self.backup_node(backup_dir, 'node', node) - - fulle1_conf_file = os.path.join( - backup_dir, 'backups','node', full1_id, 'backup_content.control') - - fulle2_conf_file = os.path.join( - backup_dir, 'backups','node', fulle2_id, 'backup_content.control') - - copyfile(fulle2_conf_file, fulle1_conf_file) - - try: - self.validate_pb(backup_dir, 'node') - self.assertEqual( - 1, 0, - "Expecting Error because pg_probackup.conf is missing. " - ".\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Invalid CRC of backup control file '{0}':".format(fulle1_conf_file), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Failed to get file list for backup {0}".format(full1_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Backup {0} file list is corrupted".format(full1_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.show_pb(backup_dir, 'node', full1_id)['status'] - - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], 'CORRUPT') - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], 'OK') diff --git a/tests/delete_test.py b/tests/delete_test.py deleted file mode 100644 index 10100887d..000000000 --- a/tests/delete_test.py +++ /dev/null @@ -1,822 +0,0 @@ -import unittest -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import subprocess - - -class DeleteTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_delete_full_backups(self): - """delete full backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # full backup - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node) - - show_backups = self.show_pb(backup_dir, 'node') - id_1 = show_backups[0]['id'] - id_2 = show_backups[1]['id'] - id_3 = show_backups[2]['id'] - self.delete_pb(backup_dir, 'node', id_2) - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(show_backups[0]['id'], id_1) - self.assertEqual(show_backups[1]['id'], id_3) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_del_instance_archive(self): - """delete full backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # full backup - self.backup_node(backup_dir, 'node', node) - - # full backup - self.backup_node(backup_dir, 'node', node) - - # restore - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - # Delete instance - self.del_instance(backup_dir, 'node') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_delete_archive_mix_compress_and_non_compressed_segments(self): - """delete full backups""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving( - backup_dir, 'node', node, compress=False) - node.slow_start() - - # full backup - self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=10) - - # Restart archiving with compression - self.set_archiving(backup_dir, 'node', node, compress=True) - - node.restart() - - # full backup - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--retention-redundancy=3', - '--delete-expired']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--retention-redundancy=3', - '--delete-expired']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--retention-redundancy=3', - '--delete-expired']) - - # @unittest.skip("skip") - def test_delete_increment_page(self): - """delete increment and all after him""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # full backup mode - self.backup_node(backup_dir, 'node', node) - # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") - # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") - # full backup mode - self.backup_node(backup_dir, 'node', node) - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 4) - - # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['id']) - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 2) - - self.assertEqual(show_backups[0]['backup-mode'], "FULL") - self.assertEqual(show_backups[0]['status'], "OK") - self.assertEqual(show_backups[1]['backup-mode'], "FULL") - self.assertEqual(show_backups[1]['status'], "OK") - - # @unittest.skip("skip") - def test_delete_increment_ptrack(self): - """delete increment and all after him""" - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'CREATE EXTENSION ptrack') - - # full backup mode - self.backup_node(backup_dir, 'node', node) - # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # full backup mode - self.backup_node(backup_dir, 'node', node) - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 4) - - # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['id']) - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 2) - - self.assertEqual(show_backups[0]['backup-mode'], "FULL") - self.assertEqual(show_backups[0]['status'], "OK") - self.assertEqual(show_backups[1]['backup-mode'], "FULL") - self.assertEqual(show_backups[1]['status'], "OK") - - # @unittest.skip("skip") - def test_delete_orphaned_wal_segments(self): - """ - make archive node, make three full backups, - delete second backup without --wal option, - then delete orphaned wals via --wal option - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") - # first full backup - backup_1_id = self.backup_node(backup_dir, 'node', node) - # second full backup - backup_2_id = self.backup_node(backup_dir, 'node', node) - # third full backup - backup_3_id = self.backup_node(backup_dir, 'node', node) - node.stop() - - # Check wals - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] - original_wal_quantity = len(wals) - - # delete second full backup - self.delete_pb(backup_dir, 'node', backup_2_id) - # check wal quantity - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") - # try to delete wals for second backup - self.delete_pb(backup_dir, 'node', options=['--wal']) - # check wal quantity - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") - - # delete first full backup - self.delete_pb(backup_dir, 'node', backup_1_id) - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") - - result = self.delete_pb(backup_dir, 'node', options=['--wal']) - # delete useless wals - self.assertTrue('On timeline 1 WAL segments between ' in result - and 'will be removed' in result) - - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") - - # Check quantity, it should be lower than original - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] - self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") - - # Delete last backup - self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] - self.assertEqual (0, len(wals), "Number of wals should be equal to 0") - - # @unittest.skip("skip") - def test_delete_wal_between_multiple_timelines(self): - """ - /-------B1-- - A1----------------A2---- - - delete A1 backup, check that WAL segments on [A1, A2) and - [A1, B1) are deleted and backups B1 and A2 keep - their WAL - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - A1 = self.backup_node(backup_dir, 'node', node) - - # load some data to node - node.pgbench_init(scale=3) - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - self.restore_node(backup_dir, 'node', node2) - self.set_auto_conf(node2, {'port': node2.port}) - node2.slow_start() - - # load some more data to node - node.pgbench_init(scale=3) - - # take A2 - A2 = self.backup_node(backup_dir, 'node', node) - - # load some more data to node2 - node2.pgbench_init(scale=2) - - B1 = self.backup_node( - backup_dir, 'node', - node2, data_dir=node2.data_dir) - - self.delete_pb(backup_dir, 'node', backup_id=A1, options=['--wal']) - - self.validate_pb(backup_dir) - - # @unittest.skip("skip") - def test_delete_backup_with_empty_control_file(self): - """ - take backup, truncate its control file, - try to delete it via 'delete' command - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # full backup mode - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - # page backup mode - self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=['--stream']) - # page backup mode - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=['--stream']) - - with open( - os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.control'), - 'wt') as f: - f.flush() - f.close() - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 3) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # @unittest.skip("skip") - def test_delete_interleaved_incremental_chains(self): - """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULLb to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # FULLb ERROR - # FULLa OK - - # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change FULLb to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # Now we start to play with first generation of PAGE backups - # Change PAGEb1 and FULLb status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change PAGEa1 status to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa2 and FULla to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # Change PAGEb1 and FULlb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGEa2 and FULLa status to OK - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # PAGEc1 OK - # FULLc OK - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Delete FULLb - self.delete_pb( - backup_dir, 'node', backup_id_b) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 5) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - # @unittest.skip("skip") - def test_delete_multiple_descendants(self): - """ - PAGEb3 - | PAGEa3 - PAGEb2 / - | PAGEa2 / - PAGEb1 \ / - | PAGEa1 - FULLb | - FULLa should be deleted - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULLb to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change FULLb to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # Change PAGEa1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # Change PAGEb1 and FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa2 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - # Change PAGEb2, PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change FULLa to OK - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa3 OK - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa3 status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - - # Change PAGEb2 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb3 OK - # PAGEa3 ERROR - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Change PAGEa3, PAGEa2 and PAGEb1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - - # PAGEb3 OK - # PAGEa3 OK - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 - self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], - page_id_a1) - - self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], - page_id_a1) - - # Delete FULLa - self.delete_pb(backup_dir, 'node', backup_id_a) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - # @unittest.skip("skip") - def test_delete_multiple_descendants_dry_run(self): - """ - PAGEa3 - PAGEa2 / - \ / - PAGEa1 (delete target) - | - FULLa - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUP - node.pgbench_init(scale=1) - backup_id_a = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - - # Change PAGEa2 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGEa2 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - - # Delete PAGEa1 - output = self.delete_pb( - backup_dir, 'node', page_id_a1, - options=['--dry-run', '--log-level-console=LOG', '--delete-wal']) - - print(output) - self.assertIn( - 'LOG: Backup {0} can be deleted'.format(page_id_a3), - output) - self.assertIn( - 'LOG: Backup {0} can be deleted'.format(page_id_a2), - output) - self.assertIn( - 'LOG: Backup {0} can be deleted'.format(page_id_a1), - output) - - self.assertIn( - 'INFO: Resident data size to free by ' - 'delete of backup {0} :'.format(page_id_a1), - output) - - self.assertIn( - 'On timeline 1 WAL segments between 000000010000000000000001 ' - 'and 000000010000000000000003 can be removed', - output) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - output = self.delete_pb( - backup_dir, 'node', page_id_a1, - options=['--log-level-console=LOG', '--delete-wal']) - - self.assertIn( - 'LOG: Backup {0} will be deleted'.format(page_id_a3), - output) - self.assertIn( - 'LOG: Backup {0} will be deleted'.format(page_id_a2), - output) - self.assertIn( - 'LOG: Backup {0} will be deleted'.format(page_id_a1), - output) - self.assertIn( - 'INFO: Resident data size to free by ' - 'delete of backup {0} :'.format(page_id_a1), - output) - - self.assertIn( - 'On timeline 1 WAL segments between 000000010000000000000001 ' - 'and 000000010000000000000003 will be removed', - output) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) - - self.validate_pb(backup_dir, 'node') - - def test_delete_error_backups(self): - """delete increment and all after him""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # full backup mode - self.backup_node(backup_dir, 'node', node) - # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Take FULL BACKUP - backup_id_a = self.backup_node(backup_dir, 'node', node) - # Take PAGE BACKUP - backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page") - - backup_id_c = self.backup_node(backup_dir, 'node', node, backup_type="page") - - backup_id_d = self.backup_node(backup_dir, 'node', node, backup_type="page") - - # full backup mode - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - backup_id_e = self.backup_node(backup_dir, 'node', node, backup_type="page") - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Change status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_c, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_e, 'ERROR') - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 10) - - # delete error backups - output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR', '--dry-run']) - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 10) - - self.assertIn( - "Deleting all backups with status 'ERROR' in dry run mode", - output) - - self.assertIn( - "INFO: Backup {0} with status OK can be deleted".format(backup_id_d), - output) - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - - show_backups = self.show_pb(backup_dir, 'node') - output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) - print(output) - show_backups = self.show_pb(backup_dir, 'node') - self.assertEqual(len(show_backups), 4) - - self.assertEqual(show_backups[0]['status'], "OK") - self.assertEqual(show_backups[1]['status'], "OK") - self.assertEqual(show_backups[2]['status'], "OK") - self.assertEqual(show_backups[3]['status'], "OK") diff --git a/tests/delta_test.py b/tests/delta_test.py deleted file mode 100644 index 23583fd93..000000000 --- a/tests/delta_test.py +++ /dev/null @@ -1,1201 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from testgres import QueryException -import subprocess -import time -from threading import Thread - - -class DeltaTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_basic_delta_vacuum_truncate(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take delta backup, take second delta backup, - restore latest delta backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - self.restore_node( - backup_dir, 'node', node_restored) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_delta_vacuum_truncate_1(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take delta backup, take second delta backup, - restore latest delta backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) - - self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) - - pgdata = self.pgdata_content(node.data_dir) - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, - 'node', - node_restored, - options=[ - "-T", "{0}={1}".format( - old_tablespace, new_tablespace)] - ) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_delta_vacuum_truncate_2(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take delta backup, take second delta backup, - restore latest delta backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10100000) i;" - ) - filepath = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')" - ).decode('utf-8').rstrip() - - self.backup_node(backup_dir, 'node', node) - - print(os.path.join(node.data_dir, filepath + '.1')) - os.unlink(os.path.join(node.data_dir, filepath + '.1')) - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - self.restore_node( - backup_dir, 'node', node_restored) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_delta_stream(self): - """ - make archive node, take full and delta stream backups, - restore them and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(0,100) i") - - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream']) - - # delta BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) - node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() - - # @unittest.skip("skip") - def test_delta_archive(self): - """ - make archive node, take full and delta archive backups, - restore them and check data correctness - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full') - - # delta BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Drop Node - node.cleanup() - - # Restore and check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Restore and check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() - - # @unittest.skip("skip") - def test_delta_multiple_segments(self): - """ - Make node, create table with multiple segments, - write some data to it, check delta and data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'fsync': 'off', - 'shared_buffers': '1GB', - 'maintenance_work_mem': '1GB', - 'full_page_writes': 'off' - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # CREATE TABLE - node.pgbench_init( - scale=100, - options=['--tablespace=somedata', '--no-vacuum']) - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # PGBENCH STUFF - pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) - pgbench.wait() - node.safe_psql("postgres", "checkpoint") - - # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") - # delta BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # GET PHYSICAL CONTENT FROM NODE - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) - restored_node.cleanup() - tblspc_path = self.get_tblspace_path(node, 'somedata') - tblspc_path_new = self.get_tblspace_path( - restored_node, 'somedata_restored') - - self.restore_node( - backup_dir, 'node', restored_node, - options=[ - "-j", "4", "-T", "{0}={1}".format( - tblspc_path, tblspc_path_new)]) - - # GET PHYSICAL CONTENT FROM NODE_RESTORED - pgdata_restored = self.pgdata_content(restored_node.data_dir) - - # START RESTORED NODE - self.set_auto_conf(restored_node, {'port': restored_node.port}) - restored_node.slow_start() - - result_new = restored_node.safe_psql( - "postgres", - "select count(*) from pgbench_accounts") - - # COMPARE RESTORED FILES - self.assertEqual(result, result_new, 'data is lost') - - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_delta_vacuum_full(self): - """ - make node, make full and delta stream backups, - restore them and check data correctness - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i" - " as id from generate_series(0,1000000) i" - ) - - pg_connect = node.connect("postgres", autocommit=True) - - gdb = self.gdb_attach(pg_connect.pid) - gdb.set_breakpoint('reform_and_rewrite_tuple') - - gdb.continue_execution_until_running() - - process = Thread( - target=pg_connect.execute, args=["VACUUM FULL t_heap"]) - process.start() - - while not gdb.stopped_in_breakpoint: - time.sleep(1) - - gdb.continue_execution_until_break(20) - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - gdb.remove_all_breakpoints() - gdb._execute('detach') - process.join() - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "-T", "{0}={1}".format( - old_tablespace, new_tablespace)]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # @unittest.skip("skip") - def test_create_db(self): - """ - Make node, take full backup, create database db1, take delta backup, - restore database and check it presense - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '10GB', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - node.safe_psql("postgres", "SELECT * FROM t_heap") - self.backup_node( - backup_dir, 'node', node, - options=["--stream"]) - - # CREATE DATABASE DB1 - node.safe_psql("postgres", "create database db1") - node.safe_psql( - "db1", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - # DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - - node_restored.cleanup() - self.restore_node( - backup_dir, - 'node', - node_restored, - backup_id=backup_id, - options=[ - "-j", "4", - "--immediate", - "--recovery-target-action=promote"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # DROP DATABASE DB1 - node.safe_psql( - "postgres", "drop database db1") - # SECOND DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE SECOND DELTA BACKUP - node_restored.cleanup() - self.restore_node( - backup_dir, - 'node', - node_restored, - backup_id=backup_id, - options=[ - "-j", "4", - "--immediate", - "--recovery-target-action=promote"] - ) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_exists_in_previous_backup(self): - """ - Make node, take full backup, create table, take page backup, - take delta backup, check that file is no fully copied to delta backup - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '10GB', - 'checkpoint_timeout': '5min', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - node.safe_psql("postgres", "SELECT * FROM t_heap") - filepath = node.safe_psql( - "postgres", - "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - self.backup_node( - backup_dir, - 'node', - node, - options=["--stream"]) - - # PAGE BACKUP - backup_id = self.backup_node( - backup_dir, - 'node', - node, - backup_type='page' - ) - - fullpath = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', filepath) - self.assertFalse(os.path.exists(fullpath)) - -# if self.paranoia: -# pgdata_page = self.pgdata_content( -# os.path.join( -# backup_dir, 'backups', -# 'node', backup_id, 'database')) - - # DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) -# if self.paranoia: -# pgdata_delta = self.pgdata_content( -# os.path.join( -# backup_dir, 'backups', -# 'node', backup_id, 'database')) -# self.compare_pgdata( -# pgdata_page, pgdata_delta) - - fullpath = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', filepath) - self.assertFalse(os.path.exists(fullpath)) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - - node_restored.cleanup() - self.restore_node( - backup_dir, - 'node', - node_restored, - backup_id=backup_id, - options=[ - "-j", "4", - "--immediate", - "--recovery-target-action=promote"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_alter_table_set_tablespace_delta(self): - """ - Make node, create tablespace with table, take full backup, - alter tablespace location, take delta backup, restore database. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - self.create_tblspace_in_node(node, 'somedata') - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i") - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # ALTER TABLESPACE - self.create_tblspace_in_node(node, 'somedata_new') - node.safe_psql( - "postgres", - "alter table t_heap set tablespace somedata_new") - - # DELTA BACKUP - result = node.safe_psql( - "postgres", "select * from t_heap") - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata') - ), - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata_new'), - self.get_tblspace_path(node_restored, 'somedata_new') - ) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - result_new = node_restored.safe_psql( - "postgres", "select * from t_heap") - - self.assertEqual(result, result_new, 'lost some data after restore') - - # @unittest.skip("skip") - def test_alter_database_set_tablespace_delta(self): - """ - Make node, take full backup, create database, - take delta backup, alter database tablespace location, - take delta backup restore last delta backup. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # CREATE DATABASE DB1 - node.safe_psql( - "postgres", - "create database db1 tablespace = 'somedata'") - node.safe_psql( - "db1", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) - - # ALTER TABLESPACE - self.create_tblspace_in_node(node, 'somedata_new') - node.safe_psql( - "postgres", - "alter database db1 set tablespace somedata_new" - ) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata') - ), - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata_new'), - self.get_tblspace_path(node_restored, 'somedata_new') - ) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_delta_delete(self): - """ - Make node, create tablespace with table, take full backup, - alter tablespace location, take delta backup, restore database. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - - node.safe_psql( - "postgres", - "delete from t_heap" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata') - ) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - def test_delta_nullified_heap_page_backup(self): - """ - make node, take full backup, nullify some heap block, - take delta backup, restore, physically compare pgdata`s - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - file_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - node.safe_psql( - "postgres", - "CHECKPOINT") - - self.backup_node( - backup_dir, 'node', node) - - # Nullify some block in PostgreSQL - file = os.path.join(node.data_dir, file_path).replace("\\", "/") - if os.name == 'nt': - file = file.replace("\\", "/") - - with open(file, 'r+b', 0) as f: - f.seek(8192) - f.write(b"\x00"*8192) - f.flush() - f.close - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=["--log-level-file=verbose"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - if not self.remote: - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") - with open(log_file_path) as f: - content = f.read() - - self.assertIn( - 'VERBOSE: File: "{0}" blknum 1, empty page'.format(file), - content) - self.assertNotIn( - "Skipping blknum 1 in file: {0}".format(file), - content) - - # Restore DELTA backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_delta_backup_from_past(self): - """ - make node, take FULL stream backup, take DELTA stream backup, - restore FULL backup, try to take second DELTA stream backup - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=3) - - # First DELTA - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - # Restore FULL backup - node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) - node.slow_start() - - # Second DELTA backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are backing up an instance from the past" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Current START LSN ' in e.message and - 'is lower than START LSN ' in e.message and - 'of previous backup ' in e.message and - 'It may indicate that we are trying ' - 'to backup PostgreSQL instance from the past' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_delta_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'shared_buffers': '512MB', - 'max_wal_size': '3GB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select nextval('t_seq')::int as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(0,25600) i") - "from generate_series(0,2560) i") - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - # kill the bastard - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - # now smack it with sledgehammer - if node.major_version >= 10: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - - self.run_binary( - [ - pg_resetxlog_path, - '-D', - node.data_dir, - '-o 42', - '-f' - ], - asynchronous=False) - - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, -# backup_type='delta', options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'Insert error message', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - -# pgdata = self.pgdata_content(node.data_dir) -# -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) -# node_restored.cleanup() -# -# self.restore_node( -# backup_dir, 'node', node_restored) -# -# pgdata_restored = self.pgdata_content(node_restored.data_dir) -# self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/exclude_test.py b/tests/exclude_test.py deleted file mode 100644 index cb3530cd5..000000000 --- a/tests/exclude_test.py +++ /dev/null @@ -1,338 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class ExcludeTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_exclude_temp_files(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'logging_collector': 'on', - 'log_filename': 'postgresql.log'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - oid = node.safe_psql( - 'postgres', - "select oid from pg_database where datname = 'postgres'").rstrip() - - pgsql_tmp_dir = os.path.join(node.data_dir, 'base', 'pgsql_tmp') - - os.mkdir(pgsql_tmp_dir) - - file = os.path.join(pgsql_tmp_dir, 'pgsql_tmp7351.16') - with open(file, 'w') as f: - f.write("HELLO") - f.flush() - f.close - - full_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', options=['--stream']) - - file = os.path.join( - backup_dir, 'backups', 'node', full_id, - 'database', 'base', 'pgsql_tmp', 'pgsql_tmp7351.16') - - self.assertFalse( - os.path.exists(file), - "File must be excluded: {0}".format(file)) - - # TODO check temporary tablespaces - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_exclude_temp_tables(self): - """ - make node without archiving, create temp table, take full backup, - check that temp table not present in backup catalogue - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - with node.connect("postgres") as conn: - - conn.execute( - "create temp table test as " - "select generate_series(0,50050000)::text") - conn.commit() - - temp_schema_name = conn.execute( - "SELECT nspname FROM pg_namespace " - "WHERE oid = pg_my_temp_schema()")[0][0] - conn.commit() - - temp_toast_schema_name = "pg_toast_" + temp_schema_name.replace( - "pg_", "") - conn.commit() - - conn.execute("create index test_idx on test (generate_series)") - conn.commit() - - heap_path = conn.execute( - "select pg_relation_filepath('test')")[0][0] - conn.commit() - - index_path = conn.execute( - "select pg_relation_filepath('test_idx')")[0][0] - conn.commit() - - heap_oid = conn.execute("select 'test'::regclass::oid")[0][0] - conn.commit() - - toast_path = conn.execute( - "select pg_relation_filepath('{0}.{1}')".format( - temp_toast_schema_name, "pg_toast_" + str(heap_oid)))[0][0] - conn.commit() - - toast_idx_path = conn.execute( - "select pg_relation_filepath('{0}.{1}')".format( - temp_toast_schema_name, - "pg_toast_" + str(heap_oid) + "_index"))[0][0] - conn.commit() - - temp_table_filename = os.path.basename(heap_path) - temp_idx_filename = os.path.basename(index_path) - temp_toast_filename = os.path.basename(toast_path) - temp_idx_toast_filename = os.path.basename(toast_idx_path) - - self.backup_node( - backup_dir, 'node', node, backup_type='full', options=['--stream']) - - for root, dirs, files in os.walk(backup_dir): - for file in files: - if file in [ - temp_table_filename, temp_table_filename + ".1", - temp_idx_filename, - temp_idx_filename + ".1", - temp_toast_filename, - temp_toast_filename + ".1", - temp_idx_toast_filename, - temp_idx_toast_filename + ".1" - ]: - self.assertEqual( - 1, 0, - "Found temp table file in backup catalogue.\n " - "Filepath: {0}".format(file)) - - # @unittest.skip("skip") - def test_exclude_unlogged_tables_1(self): - """ - make node without archiving, create unlogged table, take full backup, - alter table to unlogged, take delta backup, restore delta backup, - check that PGDATA`s are physically the same - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - "shared_buffers": "10MB"}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - conn = node.connect() - with node.connect("postgres") as conn: - - conn.execute( - "create unlogged table test as " - "select generate_series(0,5005000)::text") - conn.commit() - - conn.execute("create index test_idx on test (generate_series)") - conn.commit() - - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream']) - - node.safe_psql('postgres', "alter table test set logged") - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_exclude_unlogged_tables_2(self): - """ - 1. make node, create unlogged, take FULL, DELTA, PAGE, - check that unlogged table files was not backed up - 2. restore FULL, DELTA, PAGE to empty db, - ensure unlogged table exist and is epmty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - "shared_buffers": "10MB"}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_ids = [] - - for backup_type in ['full', 'delta', 'page']: - - if backup_type == 'full': - node.safe_psql( - 'postgres', - 'create unlogged table test as select generate_series(0,20050000)::text') - else: - node.safe_psql( - 'postgres', - 'insert into test select generate_series(0,20050000)::text') - - rel_path = node.execute( - 'postgres', - "select pg_relation_filepath('test')")[0][0] - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type=backup_type, options=['--stream']) - - backup_ids.append(backup_id) - - filelist = self.get_backup_filelist( - backup_dir, 'node', backup_id) - - self.assertNotIn( - rel_path, filelist, - "Unlogged table was not excluded") - - self.assertNotIn( - rel_path + '.1', filelist, - "Unlogged table was not excluded") - - self.assertNotIn( - rel_path + '.2', filelist, - "Unlogged table was not excluded") - - self.assertNotIn( - rel_path + '.3', filelist, - "Unlogged table was not excluded") - - # ensure restoring retrieves back only empty unlogged table - for backup_id in backup_ids: - node.stop() - node.cleanup() - - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) - - node.slow_start() - - self.assertEqual( - node.execute( - 'postgres', - 'select count(*) from test')[0][0], - 0) - - # @unittest.skip("skip") - def test_exclude_log_dir(self): - """ - check that by default 'log' and 'pg_log' directories are not backed up - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'logging_collector': 'on', - 'log_filename': 'postgresql.log'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream']) - - log_dir = node.safe_psql( - 'postgres', - 'show log_directory').decode('utf-8').rstrip() - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - # check that PGDATA/log or PGDATA/pg_log do not exists - path = os.path.join(node.data_dir, log_dir) - log_file = os.path.join(path, 'postgresql.log') - self.assertTrue(os.path.exists(path)) - self.assertFalse(os.path.exists(log_file)) - - # @unittest.skip("skip") - def test_exclude_log_dir_1(self): - """ - check that "--backup-pg-log" works correctly - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'logging_collector': 'on', - 'log_filename': 'postgresql.log'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - log_dir = node.safe_psql( - 'postgres', - 'show log_directory').decode('utf-8').rstrip() - - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream', '--backup-pg-log']) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - # check that PGDATA/log or PGDATA/pg_log do not exists - path = os.path.join(node.data_dir, log_dir) - log_file = os.path.join(path, 'postgresql.log') - self.assertTrue(os.path.exists(path)) - self.assertTrue(os.path.exists(log_file)) diff --git a/tests/external_test.py b/tests/external_test.py deleted file mode 100644 index 53f3c5449..000000000 --- a/tests/external_test.py +++ /dev/null @@ -1,2405 +0,0 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name -import shutil - - -# TODO: add some ptrack tests -class ExternalTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_basic_external(self): - """ - make node, create external directory, take backup - with external directory, restore backup, check that - external directory was successfully copied - """ - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - external_dir = self.get_tblspace_path(node, 'somedirectory') - - # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup with external directory pointing to a file - file_path = os.path.join(core_dir, 'file') - with open(file_path, "w+") as f: - pass - - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=[ - '--external-dirs={0}'.format(file_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir point to a file" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --external-dirs option' in e.message and - 'directory or symbolic link expected' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - # Fill external directories - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir, options=["-j", "4"]) - - # Full backup with external dir - self.backup_node( - backup_dir, 'node', node, - options=[ - '--external-dirs={0}'.format(external_dir)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_external_none(self): - """ - make node, create external directory, take backup - with external directory, take delta backup with --external-dirs=none, - restore delta backup, check that - external directory was not copied - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - external_dir = self.get_tblspace_path(node, 'somedirectory') - - # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - # Fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir, options=["-j", "4"]) - - # Full backup with external dir - self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--external-dirs={0}'.format(external_dir)]) - - # Delta backup without external directory - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=['--external-dirs=none', '--stream']) - - shutil.rmtree(external_dir, ignore_errors=True) - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_external_dirs_overlapping(self): - """ - make node, create directory, - take backup with two external directories pointing to - the same directory, backup should fail - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - os.mkdir(external_dir1) - os.mkdir(external_dir2) - - # Full backup with external dirs - try: - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}{1}{0}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir1)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: External directory path (-E option)' in e.message and - 'contain another external directory' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_external_dir_mapping(self): - """ - make node, take full backup, check that restore with - external-dir mapping will end with error, take page backup, - check that restore with external-dir mapping will end with - success - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # Fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --external-mapping option' in e.message and - 'have an entry in list of external directories' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_backup_multiple_external(self): - """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.set_config( - backup_dir, 'node', - options=['-E', external_dir1]) - - # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", external_dir2]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs', 'external_dir1']) - - node.cleanup() - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_backward_compatibility(self): - """ - take backup with old binary without external dirs support - take delta backup with new binary and 2 external directories - restore delta backup, check that incremental chain - restored correctly - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - node.slow_start() - - node.pgbench_init(scale=3) - - # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - # fill external directories with changed data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # delta backup with external directories using new binary - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # RESTORE chain with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_backward_compatibility_merge_1(self): - """ - take backup with old binary without external dirs support - take delta backup with new binary and 2 external directories - merge delta backup ajd restore it - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - node.slow_start() - - node.pgbench_init(scale=3) - - # tmp FULL backup with old binary - tmp_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1']) - pgbench.wait() - - # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge chain chain with new binary - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # Restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_backward_compatibility_merge_2(self): - """ - take backup with old binary without external dirs support - take delta backup with new binary and 2 external directories - merge delta backup and restore it - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) - - node.slow_start() - - node.pgbench_init(scale=3) - - # tmp FULL backup with old binary - tmp_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1']) - pgbench.wait() - - # delta backup with external directories using new binary - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1']) - pgbench.wait() - - # Fill external dirs with changed data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, - options=['-j', '4', '--skip-external-dirs']) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, - options=['-j', '4', '--skip-external-dirs']) - - # delta backup without external directories using old binary - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge chain using new binary - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # Restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - external_dir1_new = self.get_tblspace_path( - node_restored, 'external_dir1') - external_dir2_new = self.get_tblspace_path( - node_restored, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_merge(self): - """""" - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node, old_binary=True) - node.slow_start() - - node.pgbench_init(scale=3) - - # take temp FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=["-j", "4", "--stream"]) - - # change data a bit - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - print(self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) - - # Merge - print(self.merge_backup(backup_dir, 'node', backup_id=backup_id, - options=['--log-level-file=VERBOSE'])) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - external_dir1_new = self.get_tblspace_path(node, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_merge_skip_external_dirs(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # FULL backup with old data - tmp_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # fill external directories with old data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # change data a bit - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup with external directories - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # drop old external data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - # fill external directories with new data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, - options=["-j", "4", "--skip-external-dirs"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, - options=["-j", "4", "--skip-external-dirs"]) - - # DELTA backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # merge backups without external directories - self.merge_backup( - backup_dir, 'node', - backup_id=backup_id, options=['--skip-external-dirs']) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_merge_1(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup with changed data - backup_id = self.backup_node( - backup_dir, 'node', node, - options=["-j", "4", "--stream"]) - - # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - external_dir1_new = self.get_tblspace_path(node, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_merge_3(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["-j", "4"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node) - - # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # page backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=[ - "-j", "4", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # page backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page", - options=[ - "-j", "4", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.merge_backup( - backup_dir, 'node', backup_id=backup_id, - options=['--log-level-file=verbose']) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - external_dir1_new = self.get_tblspace_path(node, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_merge_2(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - options=["-j", "4", "--stream"]) - - # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # delta backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # delta backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - # delta backup without external directories - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - external_dir1_new = self.get_tblspace_path(node, 'external_dir1') - external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format(external_dir2, external_dir2_new)]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_restore_external_changed_data(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - - # set externals - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', - node, options=["-j", "4", "--stream"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # change data a bit - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # fill external directories with changed data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - # change data a bit more - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Delta backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Restore - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_restore_external_changed_data_1(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '32MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # set externals - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', - node, options=["-j", "4", "--stream"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # change data a bit - pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # fill external directories with changed data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - # change data a bit more - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Delta backup with only one external directory - self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", external_dir1]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs', 'external_dir2']) - - # Restore - node.cleanup() - shutil.rmtree(node._base_dir) - - # create empty file in external_dir2 - os.mkdir(node._base_dir) - os.mkdir(external_dir2) - with open(os.path.join(external_dir2, 'file'), 'w+') as f: - f.close() - - output = self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - self.assertNotIn( - 'externaldir2', - output) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs', 'external_dir2']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_merge_external_changed_data(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '32MB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - - # set externals - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', - node, options=["-j", "4", "--stream"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # change data a bit - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # fill external directories with changed data - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, - options=["-j", "4", "--skip-external-dirs"]) - - # change data a bit more - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Delta backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge - self.merge_backup(backup_dir, 'node', backup_id) - - # Restore - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_restore_skip_external(self): - """ - Check that --skip-external-dirs works correctly - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # FULL backup with external directories - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2)]) - - # delete first externals, so pgdata_compare - # will be capable of detecting redundant - # external files after restore - shutil.rmtree(external_dir1, ignore_errors=True) - shutil.rmtree(external_dir2, ignore_errors=True) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # RESTORE - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--skip-external-dirs"]) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_dir_is_symlink(self): - """ - Check that backup works correctly if external dir is symlink, - symlink pointing to external dir should be followed, - but restored as directory - """ - if os.name == 'nt': - self.skipTest('Skipped for Windows') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') - - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) - - # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # create symlink to directory in external directory - os.symlink(symlinked_dir, external_dir) - - # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # RESTORE - node_restored.cleanup() - - external_dir_new = self.get_tblspace_path( - node_restored, 'external_dir') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", "--external-mapping={0}={1}".format( - external_dir, external_dir_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - self.assertEqual( - external_dir, - self.show_pb( - backup_dir, 'node', - backup_id=backup_id)['external-dirs']) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_dir_contain_symlink_on_dir(self): - """ - Check that backup works correctly if external dir is symlink, - symlink pointing to external dir should be followed, - but restored as directory - """ - if os.name == 'nt': - self.skipTest('Skipped for Windows') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - dir_in_external_dir = os.path.join(external_dir, 'dir') - - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') - - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) - - # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # create symlink to directory in external directory - os.mkdir(external_dir) - os.symlink(symlinked_dir, dir_in_external_dir) - - # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # RESTORE - node_restored.cleanup() - - external_dir_new = self.get_tblspace_path( - node_restored, 'external_dir') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", "--external-mapping={0}={1}".format( - external_dir, external_dir_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - self.assertEqual( - external_dir, - self.show_pb( - backup_dir, 'node', - backup_id=backup_id)['external-dirs']) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_dir_contain_symlink_on_file(self): - """ - Check that backup works correctly if external dir is symlink, - symlink pointing to external dir should be followed, - but restored as directory - """ - if os.name == 'nt': - self.skipTest('Skipped for Windows') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - file_in_external_dir = os.path.join(external_dir, 'file') - - # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) - - # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') - - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) - - # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) - - # create symlink to directory in external directory - src_file = os.path.join(symlinked_dir, 'postgresql.conf') - os.mkdir(external_dir) - os.chmod(external_dir, 0o0700) - os.symlink(src_file, file_in_external_dir) - - # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - # RESTORE - node_restored.cleanup() - - external_dir_new = self.get_tblspace_path( - node_restored, 'external_dir') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", "--external-mapping={0}={1}".format( - external_dir, external_dir_new)]) - - pgdata_restored = self.pgdata_content( - node_restored.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - self.assertEqual( - external_dir, - self.show_pb( - backup_dir, 'node', - backup_id=backup_id)['external-dirs']) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_external_dir_is_tablespace(self): - """ - Check that backup fails with error - if external directory points to tablespace - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - self.create_tblspace_in_node( - node, 'tblspace1', tblspc_path=external_dir) - - node.pgbench_init(scale=1, tablespace='tblspace1') - - # FULL backup with external directories - try: - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir points to the tablespace" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory path (-E option)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_restore_external_dir_not_empty(self): - """ - Check that backup fails with error - if external directory point to not empty tablespace and - if remapped directory also isn`t empty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # create empty file in external directory - # open(os.path.join(external_dir, 'file'), 'a').close() - os.mkdir(external_dir) - with open(os.path.join(external_dir, 'file'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - node.cleanup() - - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is not empty" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - external_dir_new = self.get_tblspace_path(node, 'external_dir_new') - - # create empty file in directory, which will be a target of - # remapping - os.mkdir(external_dir_new) - with open(os.path.join(external_dir_new, 'file1'), 'w+') as f: - f.close() - - try: - self.restore_node( - backup_dir, 'node', node, - options=['--external-mapping={0}={1}'.format( - external_dir, external_dir_new)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped external dir is not empty" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_restore_external_dir_is_missing(self): - """ - take FULL backup with not empty external directory - delete external directory - take DELTA backup with external directory, which - should fail - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # create empty file in external directory - # open(os.path.join(external_dir, 'file'), 'a').close() - os.mkdir(external_dir) - with open(os.path.join(external_dir, 'file'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - # drop external directory - shutil.rmtree(external_dir, ignore_errors=True) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: External directory is not found:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - # take DELTA without external directories - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["-j", "4", "--stream"]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Restore Delta backup - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_merge_external_dir_is_missing(self): - """ - take FULL backup with not empty external directory - delete external directory - take DELTA backup with external directory, which - should fail, - take DELTA backup without external directory, - merge it into FULL, restore and check - data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # create empty file in external directory - # open(os.path.join(external_dir, 'file'), 'a').close() - os.mkdir(external_dir) - with open(os.path.join(external_dir, 'file'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - # drop external directory - shutil.rmtree(external_dir, ignore_errors=True) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: External directory is not found:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - # take DELTA without external directories - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["-j", "4", "--stream"]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # Restore - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_restore_external_dir_is_empty(self): - """ - take FULL backup with not empty external directory - drop external directory content - take DELTA backup with the same external directory - restore DELRA backup, check that restored - external directory is empty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # create empty file in external directory - # open(os.path.join(external_dir, 'file'), 'a').close() - os.mkdir(external_dir) - os.chmod(external_dir, 0o0700) - with open(os.path.join(external_dir, 'file'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - # make external directory empty - os.remove(os.path.join(external_dir, 'file')) - - # take DELTA backup with empty external directory - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Restore Delta backup - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_merge_external_dir_is_empty(self): - """ - take FULL backup with not empty external directory - drop external directory content - take DELTA backup with the same external directory - merge backups and restore FULL, check that restored - external directory is empty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # create empty file in external directory - # open(os.path.join(external_dir, 'file'), 'a').close() - os.mkdir(external_dir) - os.chmod(external_dir, 0o0700) - with open(os.path.join(external_dir, 'file'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - # make external directory empty - os.remove(os.path.join(external_dir, 'file')) - - # take DELTA backup with empty external directory - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # Restore - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_restore_external_dir_string_order(self): - """ - take FULL backup with not empty external directory - drop external directory content - take DELTA backup with the same external directory - restore DELRA backup, check that restored - external directory is empty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') - external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') - - # create empty file in external directory - os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0o0700) - with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: - f.close() - - os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0o0700) - with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir_1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir_2)]) - - with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: - f.close() - - with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: - f.close() - - # take DELTA backup and swap external_dir_2 and external_dir_1 - # in external_dir_str - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir_2, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir_1)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Restore Delta backup - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - def test_merge_external_dir_string_order(self): - """ - take FULL backup with not empty external directory - drop external directory content - take DELTA backup with the same external directory - restore DELRA backup, check that restored - external directory is empty - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') - external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') - - # create empty file in external directory - os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0o0700) - with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: - f.close() - - os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0o0700) - with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: - f.close() - - # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir_1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir_2)]) - - with open(os.path.join(external_dir_1, 'fileB'), 'w+') as f: - f.close() - - with open(os.path.join(external_dir_2, 'fileY'), 'w+') as f: - f.close() - - # take DELTA backup and swap external_dir_2 and external_dir_1 - # in external_dir_str - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}".format( - external_dir_2, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir_1)]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - # Merge backups - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - # Restore - node.cleanup() - shutil.rmtree(node.base_dir, ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_smart_restore_externals(self): - """ - make node, create database, take full backup with externals, - take incremental backup without externals and restore it, - make sure that files from externals are not copied during restore - https://github.com/postgrespro/pg_probackup/issues/63 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # fill external directories with data - tmp_id = self.backup_node(backup_dir, 'node', node) - - external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') - external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') - - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir_1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir_2, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # create database - node.safe_psql( - "postgres", - "CREATE DATABASE testdb") - - # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - # drop database - node.safe_psql( - "postgres", - "DROP DATABASE testdb") - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # restore PAGE backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=['--no-validate', '--log-level-file=VERBOSE']) - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) - - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) - - filelist_diff = self.get_backup_filelist_diff( - filelist_full, filelist_page) - - for file in filelist_diff: - self.assertNotIn(file, logfile_content) - - # @unittest.skip("skip") - def test_external_validation(self): - """ - make node, create database, - take full backup with external directory, - corrupt external file in backup, - run validate which should fail - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # take temp FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - external_dir = self.get_tblspace_path(node, 'external_dir') - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir, options=["-j", "4"]) - - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', '-E', "{0}".format(external_dir)]) - - # Corrupt file - file = os.path.join( - backup_dir, 'backups', 'node', full_id, - 'external_directories', 'externaldir1', 'postgresql.auto.conf') - - with open(file, "r+b", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - try: - self.validate_pb(backup_dir) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because file in external dir is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Invalid CRC of backup file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', full_id)['status'], - 'Backup STATUS should be "CORRUPT"') diff --git a/tests/false_positive_test.py b/tests/false_positive_test.py deleted file mode 100644 index fbb785c60..000000000 --- a/tests/false_positive_test.py +++ /dev/null @@ -1,337 +0,0 @@ -import unittest -import os -from time import sleep - -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -import subprocess - - -class FalsePositive(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - @unittest.expectedFailure - def test_validate_wal_lost_segment(self): - """ - Loose segment located between backups. ExpectedFailure. This is BUG - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - # make some wals - node.pgbench_init(scale=5) - - # delete last wal segment - wals_dir = os.path.join(backup_dir, "wal", 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile( - os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals = map(int, wals) - os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) - - # We just lost a wal segment and know nothing about it - self.backup_node(backup_dir, 'node', node) - self.assertTrue( - 'validation completed successfully' in self.validate_pb( - backup_dir, 'node')) - ######## - - @unittest.expectedFailure - # Need to force validation of ancestor-chain - def test_incremental_backup_corrupt_full_1(self): - """page-level backup with corrupted full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join( - backup_dir, "backups", "node", - backup_id.decode("utf-8"), "database", "postgresql.conf") - os.remove(file) - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be " - "possible without valid full backup.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, - 'ERROR: Valid full backup on current timeline is not found. ' - 'Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertFalse( - True, - "Expecting Error because page backup should not be " - "possible without valid full backup.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, - 'ERROR: Valid full backup on current timeline is not found. ' - 'Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") - - # @unittest.skip("skip") - @unittest.expectedFailure - def test_pg_10_waldir(self): - """ - test group access for PG >= 11 - """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - - wal_dir = os.path.join( - os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') - import shutil - shutil.rmtree(wal_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=[ - '--data-checksums', - '--waldir={0}'.format(wal_dir)]) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - # restore backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - # compare pgdata permissions - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.assertTrue( - os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), - 'pg_wal should be symlink') - - @unittest.expectedFailure - # @unittest.skip("skip") - def test_recovery_target_time_backup_victim(self): - """ - Check that for validation to recovery target - probackup chooses valid backup - https://github.com/postgrespro/pg_probackup/issues/104 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - target_time = node.safe_psql( - "postgres", - "select now()").rstrip() - - node.safe_psql( - "postgres", - "create table t_heap1 as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100) i") - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - - # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-time={0}'.format(target_time)]) - - @unittest.expectedFailure - # @unittest.skip("skip") - def test_recovery_target_lsn_backup_victim(self): - """ - Check that for validation to recovery target - probackup chooses valid backup - https://github.com/postgrespro/pg_probackup/issues/104 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - node.safe_psql( - "postgres", - "create table t_heap1 as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100) i") - - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--log-level-console=LOG'], gdb=True) - - # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - self.switch_wal_segment(node) - - target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] - - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-lsn={0}'.format(target_lsn)]) - - # @unittest.skip("skip") - @unittest.expectedFailure - def test_streaming_timeout(self): - """ - Illustrate the problem of loosing exact error - message because our WAL streaming engine is "borrowed" - from pg_receivexlog - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_sender_timeout': '5s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, - options=['--stream', '--log-level-file=LOG']) - - # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - - sleep(10) - gdb.continue_execution_until_error() - gdb._execute('detach') - sleep(2) - - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_content = f.read() - - self.assertIn( - 'could not receive data from WAL stream', - log_content) - - self.assertIn( - 'ERROR: Problem in receivexlog', - log_content) - - # @unittest.skip("skip") - @unittest.expectedFailure - def test_validate_all_empty_catalog(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because backup_dir is empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: This backup catalog contains no backup instances', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py deleted file mode 100644 index 613e4dd36..000000000 --- a/tests/incr_restore_test.py +++ /dev/null @@ -1,2300 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import subprocess -from datetime import datetime -import sys -from time import sleep -from datetime import datetime, timedelta -import hashlib -import shutil -import json -from testgres import QueryException - - -class IncrRestoreTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_basic_incr_restore(self): - """incremental restore in CHECKSUM mode""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=50) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_basic_incr_restore_into_missing_directory(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=10) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_checksum_corruption_detection(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=10) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=lsn"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - tblspace = self.get_tblspace_path(node, 'tblspace') - some_directory = self.get_tblspace_path(node, 'some_directory') - - # stuff new destination with garbage - self.restore_node(backup_dir, 'node', node, data_dir=some_directory) - - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=10, tablespace='tblspace') - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--incremental-mode=checksum", "--force", - "-T{0}={1}".format(tblspace, some_directory)]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_1(self): - """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - tblspace = self.get_tblspace_path(node, 'tblspace') - some_directory = self.get_tblspace_path(node, 'some_directory') - - self.restore_node(backup_dir, 'node', node, data_dir=some_directory) - - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=10, tablespace='tblspace') - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_2(self): - """ - If "--tablespace-mapping" option is used with incremental restore, - then new directory must be empty. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - - # fill node1 with data - out = self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=['--incremental-mode=checksum', '--force']) - - self.assertIn("WARNING: Backup catalog was initialized for system id", out) - - tblspace = self.get_tblspace_path(node, 'tblspace') - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=5, tablespace='tblspace') - - node.safe_psql( - 'postgres', - 'vacuum') - - self.backup_node(backup_dir, 'node', node, backup_type='delta', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - try: - self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped directory is not empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Remapped tablespace destination is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - out = self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=[ - '--force', '--incremental-mode=checksum', - '-T{0}={1}'.format(tblspace, tblspace)]) - - pgdata_restored = self.pgdata_content(node_1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_3(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'tblspace1') - node.pgbench_init(scale=10, tablespace='tblspace1') - - # take backup with tblspace1 - self.backup_node(backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - self.drop_tblspace(node, 'tblspace1') - - self.create_tblspace_in_node(node, 'tblspace2') - node.pgbench_init(scale=10, tablespace='tblspace2') - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_4(self): - """ - Check that system ID mismatch is detected, - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'tblspace1') - node.pgbench_init(scale=10, tablespace='tblspace1') - - # take backup of node1 with tblspace1 - self.backup_node(backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - self.drop_tblspace(node, 'tblspace1') - node.cleanup() - - # recreate node - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - node.slow_start() - - self.create_tblspace_in_node(node, 'tblspace1') - node.pgbench_init(scale=10, tablespace='tblspace1') - node.stop() - - try: - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because destination directory has wrong system id.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup catalog was initialized for system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - out = self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--force", - "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_incr_restore_with_tablespace_5(self): - """ - More complicated case, we restore backup - with tablespace, which we remap into directory - with some old content, that belongs to an instance - with different system id. - """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) - node1.slow_start() - - self.create_tblspace_in_node(node1, 'tblspace') - node1.pgbench_init(scale=10, tablespace='tblspace') - - # take backup of node1 with tblspace - self.backup_node(backup_dir, 'node', node1, options=['--stream']) - pgdata = self.pgdata_content(node1.data_dir) - - node1.stop() - - # recreate node - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), - set_replication=True, - initdb_params=['--data-checksums']) - node2.slow_start() - - self.create_tblspace_in_node(node2, 'tblspace') - node2.pgbench_init(scale=10, tablespace='tblspace') - node2.stop() - - tblspc1_path = self.get_tblspace_path(node1, 'tblspace') - tblspc2_path = self.get_tblspace_path(node2, 'tblspace') - - out = self.restore_node( - backup_dir, 'node', node1, - options=[ - "-j", "4", "--force", - "--incremental-mode=checksum", - "-T{0}={1}".format(tblspc1_path, tblspc2_path)]) - - # check that tblspc1_path is empty - self.assertFalse( - os.listdir(tblspc1_path), - "Dir is not empty: '{0}'".format(tblspc1_path)) - - pgdata_restored = self.pgdata_content(node1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_6(self): - """ - Empty pgdata, not empty tablespace - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=10, tablespace='tblspace') - - # take backup of node with tblspace - self.backup_node(backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because there is running postmaster " - "process in destination directory.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: PGDATA is empty, but tablespace destination is not', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - out = self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--force", - "--incremental-mode=checksum"]) - - self.assertIn( - "INFO: Destination directory and tablespace directories are empty, " - "disable incremental restore", out) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_restore_with_tablespace_7(self): - """ - Restore backup without tablespace into - PGDATA with tablespace. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # take backup of node with tblspace - self.backup_node(backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=5, tablespace='tblspace') - node.stop() - -# try: -# self.restore_node( -# backup_dir, 'node', node, -# options=[ -# "-j", "4", -# "--incremental-mode=checksum"]) -# # we should die here because exception is what we expect to happen -# self.assertEqual( -# 1, 0, -# "Expecting Error because there is running postmaster " -# "process in destination directory.\n " -# "Output: {0} \n CMD: {1}".format( -# repr(self.output), self.cmd)) -# except ProbackupException as e: -# self.assertIn( -# 'ERROR: PGDATA is empty, but tablespace destination is not', -# e.message, -# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( -# repr(e.message), self.cmd)) - - out = self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_basic_incr_restore_sanity(self): - """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - try: - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because there is running postmaster " - "process in destination directory.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Postmaster with pid', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - - try: - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, - options=["-j", "4", "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because destination directory has wrong system id.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup catalog was initialized for system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_incr_checksum_restore(self): - """ - /----C-----D - ------A----B---*--------X - - X - is instance, we want to return it to C state. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - xid = node.safe_psql( - 'postgres', - 'select txid_current()').decode('utf-8').rstrip() - - # --A-----B--------X - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - node_1.cleanup() - - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, - options=[ - '--recovery-target-action=promote', - '--recovery-target-xid={0}'.format(xid)]) - - self.set_auto_conf(node_1, {'port': node_1.port}) - node_1.slow_start() - - # /-- - # --A-----B----*----X - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # /--C - # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - - # /--C------ - # --A-----B----*----X - pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) - pgbench.wait() - - # /--C------D - # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - - pgdata = self.pgdata_content(node_1.data_dir) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.set_auto_conf(node, {'port': node.port}) - node.slow_start() - - self.compare_pgdata(pgdata, pgdata_restored) - - - # @unittest.skip("skip") - def test_incr_lsn_restore(self): - """ - /----C-----D - ------A----B---*--------X - - X - is instance, we want to return it to C state. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - xid = node.safe_psql( - 'postgres', - 'select txid_current()').decode('utf-8').rstrip() - - # --A-----B--------X - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - node_1.cleanup() - - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, - options=[ - '--recovery-target-action=promote', - '--recovery-target-xid={0}'.format(xid)]) - - self.set_auto_conf(node_1, {'port': node_1.port}) - node_1.slow_start() - - # /-- - # --A-----B----*----X - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # /--C - # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - - # /--C------ - # --A-----B----*----X - pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) - pgbench.wait() - - # /--C------D - # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - - pgdata = self.pgdata_content(node_1.data_dir) - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.set_auto_conf(node, {'port': node.port}) - node.slow_start() - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_lsn_sanity(self): - """ - /----A-----B - F------*--------X - - X - is instance, we want to return it to state B. - fail is expected behaviour in case of lsn restore. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=10) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - node_1.cleanup() - - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir) - - self.set_auto_conf(node_1, {'port': node_1.port}) - node_1.slow_start() - - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='full') - - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - - node.stop() - - try: - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=lsn"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Cannot perform incremental restore of " - "backup chain {0} in 'lsn' mode".format(page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_incr_checksum_sanity(self): - """ - /----A-----B - F------*--------X - - X - is instance, we want to return it to state B. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=20) - - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) - node_1.cleanup() - - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir) - - self.set_auto_conf(node_1, {'port': node_1.port}) - node_1.slow_start() - - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='full') - - pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node(backup_dir, 'node', node_1, - data_dir=node_1.data_dir, backup_type='page') - pgdata = self.pgdata_content(node_1.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_checksum_corruption_detection(self): - """ - check that corrupted page got detected and replaced - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), -# initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=20) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, - data_dir=node.data_dir, backup_type='full') - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node(backup_dir, 'node', node, - data_dir=node.data_dir, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - - path = os.path.join(node.data_dir, heap_path) - with open(path, "rb+", 0) as f: - f.seek(22000) - f.write(b"bla") - f.flush() - f.close - - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=checksum"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_incr_lsn_corruption_detection(self): - """ - check that corrupted page got detected and replaced - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=20) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, - data_dir=node.data_dir, backup_type='full') - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node(backup_dir, 'node', node, - data_dir=node.data_dir, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - - path = os.path.join(node.data_dir, heap_path) - with open(path, "rb+", 0) as f: - f.seek(22000) - f.write(b"bla") - f.flush() - f.close - - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=lsn"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_restore_multiple_external(self): - """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - node.pgbench_init(scale=20) - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.set_config( - backup_dir, 'node', - options=['-E{0}{1}{2}'.format( - external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) - - # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=["-j", "4"]) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=checksum']) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_lsn_restore_multiple_external(self): - """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - external_dir1 = self.get_tblspace_path(node, 'external_dir1') - external_dir2 = self.get_tblspace_path(node, 'external_dir2') - - # FULL backup - node.pgbench_init(scale=20) - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - - # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) - - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) - - self.set_config( - backup_dir, 'node', - options=['-E{0}{1}{2}'.format( - external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) - - # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=["-j", "4"]) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - pgdata = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - node.stop() - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=lsn']) - - pgdata_restored = self.pgdata_content( - node.base_dir, exclude_dirs=['logs']) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_lsn_restore_backward(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - node.pgbench_init(scale=2) - full_id = self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - - full_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - page_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - delta_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=["-j", "4"]) - - delta_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=[ - "-j", "4", - '--incremental-mode=lsn', - '--recovery-target=immediate', - '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(full_pgdata, pgdata_restored) - - node.slow_start(replica=True) - node.stop() - - try: - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=[ - "-j", "4", '--incremental-mode=lsn', - '--recovery-target=immediate', '--recovery-target-action=pause']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "Cannot perform incremental restore of backup chain", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(page_pgdata, pgdata_restored) - - node.slow_start(replica=True) - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, - options=[ - "-j", "4", - '--incremental-mode=lsn', - '--recovery-target=immediate', - '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(delta_pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_checksum_restore_backward(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'hot_standby': 'on'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - node.pgbench_init(scale=20) - full_id = self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - - full_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - page_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - page_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - delta_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=["-j", "4"]) - - delta_pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=[ - "-j", "4", - '--incremental-mode=checksum', - '--recovery-target=immediate', - '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(full_pgdata, pgdata_restored) - - node.slow_start(replica=True) - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=[ - "-j", "4", - '--incremental-mode=checksum', - '--recovery-target=immediate', - '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(page_pgdata, pgdata_restored) - - node.slow_start(replica=True) - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, - options=[ - "-j", "4", - '--incremental-mode=checksum', - '--recovery-target=immediate', - '--recovery-target-action=pause']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(delta_pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_make_replica_via_incr_checksum_restore(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - master.pgbench_init(scale=20) - - self.backup_node(backup_dir, 'node', master) - - self.restore_node( - backup_dir, 'node', replica, options=['-R']) - - # Settings for Replica - self.set_replica(master, replica, synchronous=False) - - replica.slow_start(replica=True) - - pgbench = master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # PROMOTIONS - replica.promote() - new_master = replica - - # old master is going a bit further - old_master = master - pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - old_master.stop() - - pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # take backup from new master - self.backup_node( - backup_dir, 'node', new_master, - data_dir=new_master.data_dir, backup_type='page') - - # restore old master as replica - self.restore_node( - backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=checksum']) - - self.set_replica(new_master, old_master, synchronous=True) - - old_master.slow_start(replica=True) - - pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # @unittest.skip("skip") - def test_make_replica_via_incr_lsn_restore(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - master.pgbench_init(scale=20) - - self.backup_node(backup_dir, 'node', master) - - self.restore_node( - backup_dir, 'node', replica, options=['-R']) - - # Settings for Replica - self.set_replica(master, replica, synchronous=False) - - replica.slow_start(replica=True) - - pgbench = master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # PROMOTIONS - replica.promote() - new_master = replica - - # old master is going a bit further - old_master = master - pgbench = old_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - old_master.stop() - - pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # take backup from new master - self.backup_node( - backup_dir, 'node', new_master, - data_dir=new_master.data_dir, backup_type='page') - - # restore old master as replica - self.restore_node( - backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=lsn']) - - self.set_replica(new_master, old_master, synchronous=True) - - old_master.slow_start(replica=True) - - pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_checksum_long_xact(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'create extension pageinspect') - - # FULL backup - con = node.connect("postgres") - con.execute("CREATE TABLE t1 (a int)") - con.commit() - - - con.execute("INSERT INTO t1 values (1)") - con.commit() - - # leave uncommited - con2 = node.connect("postgres") - con.execute("INSERT INTO t1 values (2)") - con2.execute("INSERT INTO t1 values (3)") - - full_id = self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - - con.commit() - - node.safe_psql( - 'postgres', - 'select * from t1') - - con2.commit() - node.safe_psql( - 'postgres', - 'select * from t1') - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=checksum']) - - node.slow_start() - - self.assertEqual( - node.safe_psql( - 'postgres', - 'select count(*) from t1').decode('utf-8').rstrip(), - '1') - - # @unittest.skip("skip") - # @unittest.expectedFailure - # This test will pass with Enterprise - # because it has checksums enabled by default - @unittest.skipIf(ProbackupTest.enterprise, 'skip') - def test_incr_lsn_long_xact_1(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'create extension pageinspect') - - # FULL backup - con = node.connect("postgres") - con.execute("CREATE TABLE t1 (a int)") - con.commit() - - - con.execute("INSERT INTO t1 values (1)") - con.commit() - - # leave uncommited - con2 = node.connect("postgres") - con.execute("INSERT INTO t1 values (2)") - con2.execute("INSERT INTO t1 values (3)") - - full_id = self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - - con.commit() - - # when does LSN gets stamped when checksum gets updated ? - node.safe_psql( - 'postgres', - 'select * from t1') - - con2.commit() - node.safe_psql( - 'postgres', - 'select * from t1') - - node.stop() - - try: - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Incremental restore in 'lsn' mode require data_checksums to be " - "enabled in destination data directory", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_lsn_long_xact_2(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'full_page_writes': 'off', - 'wal_log_hints': 'off'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'create extension pageinspect') - - # FULL backup - con = node.connect("postgres") - con.execute("CREATE TABLE t1 (a int)") - con.commit() - - - con.execute("INSERT INTO t1 values (1)") - con.commit() - - # leave uncommited - con2 = node.connect("postgres") - con.execute("INSERT INTO t1 values (2)") - con2.execute("INSERT INTO t1 values (3)") - - full_id = self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - -# print(node.safe_psql( -# 'postgres', -# "select * from page_header(get_raw_page('t1', 0))")) - - con.commit() - - # when does LSN gets stamped when checksum gets updated ? - node.safe_psql( - 'postgres', - 'select * from t1') - -# print(node.safe_psql( -# 'postgres', -# "select * from page_header(get_raw_page('t1', 0))")) - - con2.commit() - node.safe_psql( - 'postgres', - 'select * from t1') - -# print(node.safe_psql( -# 'postgres', -# "select * from page_header(get_raw_page('t1', 0))")) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn']) - - node.slow_start() - - self.assertEqual( - node.safe_psql( - 'postgres', - 'select count(*) from t1').decode('utf-8').rstrip(), - '1') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_restore_zero_size_file_checksum(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w+b", 0) as f: - f.flush() - f.close - - # FULL backup - id1 = self.backup_node( - backup_dir, 'node', node, - options=["-j", "4", "--stream"]) - - pgdata1 = self.pgdata_content(node.data_dir) - - with open(fullpath, "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - id2 = self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - pgdata2 = self.pgdata_content(node.data_dir) - - with open(fullpath, "w") as f: - f.close() - - id3 = self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - pgdata3 = self.pgdata_content(node.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - self.restore_node( - backup_dir, 'node', node, backup_id=id2, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - self.restore_node( - backup_dir, 'node', node, backup_id=id3, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_incr_restore_zero_size_file_lsn(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w+b", 0) as f: - f.flush() - f.close - - # FULL backup - id1 = self.backup_node( - backup_dir, 'node', node, - options=["-j", "4", "--stream"]) - - pgdata1 = self.pgdata_content(node.data_dir) - - with open(fullpath, "rb+", 0) as f: - f.seek(9000) - f.write(b"bla") - f.flush() - f.close - - id2 = self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - pgdata2 = self.pgdata_content(node.data_dir) - - with open(fullpath, "w") as f: - f.close() - - id3 = self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - pgdata3 = self.pgdata_content(node.data_dir) - - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata1, pgdata_restored) - - node.slow_start() - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=id2, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata2, pgdata_restored) - - node.slow_start() - node.stop() - - self.restore_node( - backup_dir, 'node', node, backup_id=id3, - options=["-j", "4", '-I', 'checksum']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata3, pgdata_restored) - - def test_incremental_partial_restore_exclude_checksum(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - node.pgbench_init(scale=20) - - # FULL backup - self.backup_node(backup_dir, 'node', node) - pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # restore FULL backup into second node2 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) - node1.cleanup() - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - # restore some data into node2 - self.restore_node(backup_dir, 'node', node2) - - # partial restore backup into node1 - self.restore_node( - backup_dir, 'node', - node1, options=[ - "--db-exclude=db1", - "--db-exclude=db5"]) - - pgdata1 = self.pgdata_content(node1.data_dir) - - # partial incremental restore backup into node2 - self.restore_node( - backup_dir, 'node', - node2, options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "-I", "checksum"]) - - pgdata2 = self.pgdata_content(node2.data_dir) - - self.compare_pgdata(pgdata1, pgdata2) - - self.set_auto_conf(node2, {'port': node2.port}) - - node2.slow_start() - - node2.safe_psql( - 'postgres', - 'select 1') - - try: - node2.safe_psql( - 'db1', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node2.safe_psql( - 'db5', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_incremental_partial_restore_exclude_lsn(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - node.pgbench_init(scale=20) - - # FULL backup - self.backup_node(backup_dir, 'node', node) - pgdata = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1']) - pgbench.wait() - - # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - node.stop() - - # restore FULL backup into second node2 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) - node1.cleanup() - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - # restore some data into node2 - self.restore_node(backup_dir, 'node', node2) - - # partial restore backup into node1 - self.restore_node( - backup_dir, 'node', - node1, options=[ - "--db-exclude=db1", - "--db-exclude=db5"]) - - pgdata1 = self.pgdata_content(node1.data_dir) - - # partial incremental restore backup into node2 - node2.port = node.port - node2.slow_start() - node2.stop() - self.restore_node( - backup_dir, 'node', - node2, options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "-I", "lsn"]) - - pgdata2 = self.pgdata_content(node2.data_dir) - - self.compare_pgdata(pgdata1, pgdata2) - - self.set_auto_conf(node2, {'port': node2.port}) - - node2.slow_start() - - node2.safe_psql( - 'postgres', - 'select 1') - - try: - node2.safe_psql( - 'db1', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node2.safe_psql( - 'db5', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_incremental_partial_restore_exclude_tablespace_checksum(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # cat_version = node.get_control_data()["Catalog version number"] - # version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version - - # PG_10_201707211 - # pg_tblspc/33172/PG_9.5_201510051/16386/ - - self.create_tblspace_in_node(node, 'somedata') - - node_tablespace = self.get_tblspace_path(node, 'somedata') - - tbl_oid = node.safe_psql( - 'postgres', - "SELECT oid " - "FROM pg_tablespace " - "WHERE spcname = 'somedata'").rstrip() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0} tablespace somedata'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - # node1 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) - node1.cleanup() - node1_tablespace = self.get_tblspace_path(node1, 'somedata') - - # node2 - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - node2_tablespace = self.get_tblspace_path(node2, 'somedata') - - # in node2 restore full backup - self.restore_node( - backup_dir, 'node', - node2, options=[ - "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) - - # partial restore into node1 - self.restore_node( - backup_dir, 'node', - node1, options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "-T", "{0}={1}".format( - node_tablespace, node1_tablespace)]) - - pgdata1 = self.pgdata_content(node1.data_dir) - - # partial incremental restore into node2 - try: - self.restore_node( - backup_dir, 'node', - node2, options=[ - "-I", "checksum", - "--db-exclude=db1", - "--db-exclude=db5", - "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped tablespace contain old data .\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Remapped tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node( - backup_dir, 'node', - node2, options=[ - "-I", "checksum", "--force", - "--db-exclude=db1", - "--db-exclude=db5", - "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) - - pgdata2 = self.pgdata_content(node2.data_dir) - - self.compare_pgdata(pgdata1, pgdata2) - - self.set_auto_conf(node2, {'port': node2.port}) - node2.slow_start() - - node2.safe_psql( - 'postgres', - 'select 1') - - try: - node2.safe_psql( - 'db1', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node2.safe_psql( - 'db5', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_incremental_pg_filenode_map(self): - """ - https://github.com/postgrespro/pg_probackup/issues/320 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) - node1.cleanup() - - node.pgbench_init(scale=5) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - # in node1 restore full backup - self.restore_node(backup_dir, 'node', node1) - self.set_auto_conf(node1, {'port': node1.port}) - node1.slow_start() - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - - pgbench = node1.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - - node.safe_psql( - 'postgres', - 'reindex index pg_type_oid_index') - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - node1.stop() - - # incremental restore into node1 - self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum"]) - - self.set_auto_conf(node1, {'port': node1.port}) - node1.slow_start() - - node1.safe_psql( - 'postgres', - 'select 1') - -# check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn diff --git a/tests/init_test.py b/tests/init_test.py deleted file mode 100644 index 94b076fef..000000000 --- a/tests/init_test.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException -import shutil - - -class InitTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_success(self): - """Success normal init""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) - self.assertEqual( - dir_files(backup_dir), - ['backups', 'wal'] - ) - self.add_instance(backup_dir, 'node', node) - self.assertIn( - "INFO: Instance 'node' successfully deleted", - self.del_instance(backup_dir, 'node'), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) - - # Show non-existing instance - try: - self.show_pb(backup_dir, 'node') - self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) - - # Delete non-existing instance - try: - self.del_instance(backup_dir, 'node1') - self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node1' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) - - # Add instance without pgdata - try: - self.run_pb([ - "add-instance", - "--instance=node1", - "-B", backup_dir - ]) - self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) - - # @unittest.skip("skip") - def test_already_exist(self): - """Failure with backup catalog already existed""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) - try: - self.show_pb(backup_dir, 'node') - self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_abs_path(self): - """failure with backup catalog should be given as absolute path""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - try: - self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) - self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: -B, --backup-path must be an absolute path", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_add_instance_idempotence(self): - """ - https://github.com/postgrespro/pg_probackup/issues/219 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) - - self.add_instance(backup_dir, 'node', node) - shutil.rmtree(os.path.join(backup_dir, 'backups', 'node')) - - dir_backups = os.path.join(backup_dir, 'backups', 'node') - dir_wal = os.path.join(backup_dir, 'wal', 'node') - - try: - self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) diff --git a/tests/locking_test.py b/tests/locking_test.py deleted file mode 100644 index 5367c2610..000000000 --- a/tests/locking_test.py +++ /dev/null @@ -1,629 +0,0 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class LockingTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_locking_running_validate_1(self): - """ - make node, take full backup, stop it in the middle - run validate, expect it to successfully executed, - concurrent RUNNING backup with pid file and active process is legal - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - validate_output = self.validate_pb( - backup_dir, options=['--log-level-console=LOG']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - self.assertIn( - "is using backup {0}, and is still running".format(backup_id), - validate_output, - '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - # Clean after yourself - gdb.kill() - - def test_locking_running_validate_2(self): - """ - make node, take full backup, stop it in the middle, - kill process so no cleanup is done - pid file is in place, - run validate, expect it to not successfully executed, - RUNNING backup with pid file AND without active pid is legal, - but his status must be changed to ERROR and pid file is deleted - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - gdb.continue_execution_until_error() - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "which used backup {0} no longer exists".format( - backup_id) in e.message and - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) - - # Clean after yourself - gdb.kill() - - def test_locking_running_validate_2_specific_id(self): - """ - make node, take full backup, stop it in the middle, - kill process so no cleanup is done - pid file is in place, - run validate on this specific backup, - expect it to not successfully executed, - RUNNING backup with pid file AND without active pid is legal, - but his status must be changed to ERROR and pid file is deleted - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - gdb.continue_execution_until_error() - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - try: - self.validate_pb(backup_dir, 'node', backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "which used backup {0} no longer exists".format( - backup_id) in e.message and - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "ERROR: Backup {0} has status: ERROR".format(backup_id) in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) - - try: - self.validate_pb(backup_dir, 'node', backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has status: ERROR".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Backup {0} has status ERROR. Skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Clean after yourself - gdb.kill() - - def test_locking_running_3(self): - """ - make node, take full backup, stop it in the middle, - terminate process, delete pid file, - run validate, expect it to not successfully executed, - RUNNING backup without pid file AND without active pid is legal, - his status must be changed to ERROR - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - - gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - gdb.continue_execution_until_error() - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - os.remove( - os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid')) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) - - # Clean after yourself - gdb.kill() - - def test_locking_restore_locked(self): - """ - make node, take full backup, take two page backups, - launch validate on PAGE1 and stop it in the middle, - launch restore of PAGE2. - Expect restore to sucseed because read-only locks - do not conflict - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - full_id = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # PAGE2 - self.backup_node(backup_dir, 'node', node, backup_type='page') - - gdb = self.validate_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) - - gdb.set_breakpoint('pgBackupValidate') - gdb.run_until_break() - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - # Clean after yourself - gdb.kill() - - def test_concurrent_delete_and_restore(self): - """ - make node, take full backup, take page backup, - launch validate on FULL and stop it in the middle, - launch restore of PAGE. - Expect restore to fail because validation of - intermediate backup is impossible - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - gdb = self.delete_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) - - # gdb.set_breakpoint('pgFileDelete') - gdb.set_breakpoint('delete_backup_files') - gdb.run_until_break() - - node.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because restore without whole chain validation " - "is prohibited unless --no-validate provided.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "Backup {0} is used without validation".format( - restore_id) in e.message and - 'is using backup {0}, and is still running'.format( - backup_id) in e.message and - 'ERROR: Cannot lock backup' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Clean after yourself - gdb.kill() - - def test_locking_concurrent_validate_and_backup(self): - """ - make node, take full backup, launch validate - and stop it in the middle, take page backup. - Expect PAGE backup to be successfully executed - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - - # PAGE2 - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - gdb = self.validate_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) - - gdb.set_breakpoint('pgBackupValidate') - gdb.run_until_break() - - # This PAGE backup is expected to be successfull - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # Clean after yourself - gdb.kill() - - def test_locking_concurren_restore_and_delete(self): - """ - make node, take full backup, launch restore - and stop it in the middle, delete full backup. - Expect it to fail. - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - full_id = self.backup_node(backup_dir, 'node', node) - - node.cleanup() - gdb = self.restore_node(backup_dir, 'node', node, gdb=True) - - gdb.set_breakpoint('create_data_directories') - gdb.run_until_break() - - try: - self.delete_pb(backup_dir, 'node', full_id) - self.assertEqual( - 1, 0, - "Expecting Error because backup is locked\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Cannot lock backup {0} directory".format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Clean after yourself - gdb.kill() - - def test_backup_directory_name(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - full_id_1 = self.backup_node(backup_dir, 'node', node) - page_id_1 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - full_id_2 = self.backup_node(backup_dir, 'node', node) - page_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - node.cleanup() - - old_path = os.path.join(backup_dir, 'backups', 'node', full_id_1) - new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') - - os.rename(old_path, new_path) - - # This PAGE backup is expected to be successfull - self.show_pb(backup_dir, 'node', full_id_1) - - self.validate_pb(backup_dir) - self.validate_pb(backup_dir, 'node') - self.validate_pb(backup_dir, 'node', full_id_1) - - self.restore_node(backup_dir, 'node', node, backup_id=full_id_1) - - self.delete_pb(backup_dir, 'node', full_id_1) - - old_path = os.path.join(backup_dir, 'backups', 'node', full_id_2) - new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') - - self.set_backup( - backup_dir, 'node', full_id_2, options=['--note=hello']) - - self.merge_backup(backup_dir, 'node', page_id_2, options=["-j", "4"]) - - self.assertNotIn( - 'note', - self.show_pb(backup_dir, 'node', page_id_2)) - - # Clean after yourself - - def test_empty_lock_file(self): - """ - https://github.com/postgrespro/pg_probackup/issues/308 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=100) - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node) - - lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') - with open(lockfile, "w+") as f: - f.truncate() - - out = self.validate_pb(backup_dir, 'node', backup_id) - - self.assertIn( - "Waiting 30 seconds on empty exclusive lock for backup", out) - -# lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') -# with open(lockfile, "w+") as f: -# f.truncate() -# -# p1 = self.validate_pb(backup_dir, 'node', backup_id, asynchronous=True, -# options=['--log-level-file=LOG', '--log-filename=validate.log']) -# sleep(3) -# p2 = self.delete_pb(backup_dir, 'node', backup_id, asynchronous=True, -# options=['--log-level-file=LOG', '--log-filename=delete.log']) -# -# p1.wait() -# p2.wait() - - def test_shared_lock(self): - """ - Make sure that shared lock leaves no files with pids - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=1) - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node) - - lockfile_excl = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') - lockfile_shr = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup_ro.pid') - - self.validate_pb(backup_dir, 'node', backup_id) - - self.assertFalse( - os.path.exists(lockfile_excl), - "File should not exist: {0}".format(lockfile_excl)) - - self.assertFalse( - os.path.exists(lockfile_shr), - "File should not exist: {0}".format(lockfile_shr)) - - gdb = self.validate_pb(backup_dir, 'node', backup_id, gdb=True) - - gdb.set_breakpoint('validate_one_page') - gdb.run_until_break() - gdb.kill() - - self.assertTrue( - os.path.exists(lockfile_shr), - "File should exist: {0}".format(lockfile_shr)) - - self.validate_pb(backup_dir, 'node', backup_id) - - self.assertFalse( - os.path.exists(lockfile_excl), - "File should not exist: {0}".format(lockfile_excl)) - - self.assertFalse( - os.path.exists(lockfile_shr), - "File should not exist: {0}".format(lockfile_shr)) - diff --git a/tests/logging_test.py b/tests/logging_test.py deleted file mode 100644 index c5cdfa344..000000000 --- a/tests/logging_test.py +++ /dev/null @@ -1,345 +0,0 @@ -import unittest -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import datetime - -class LogTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - # PGPRO-2154 - def test_log_rotation(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', - options=['--log-rotation-age=1s', '--log-rotation-size=1MB']) - - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=verbose']) - - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=verbose'], gdb=True) - - gdb.set_breakpoint('open_logfile') - gdb.run_until_break() - gdb.continue_execution_until_exit() - - def test_log_filename_strftime(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', - options=['--log-rotation-age=1d']) - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=VERBOSE', - '--log-filename=pg_probackup-%a.log']) - - day_of_week = datetime.datetime.today().strftime("%a") - - path = os.path.join( - backup_dir, 'log', 'pg_probackup-{0}.log'.format(day_of_week)) - - self.assertTrue(os.path.isfile(path)) - - def test_truncate_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', - options=['--log-rotation-age=1d']) - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=VERBOSE']) - - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - log_file_size = os.stat(log_file_path).st_size - - self.assertTrue(os.path.isfile(rotation_file_path)) - - # truncate .rotation file - with open(rotation_file_path, "rb+", 0) as f: - f.truncate() - f.flush() - f.close - - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=LOG'], - return_id=False) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - self.assertIn( - 'WARNING: cannot read creation timestamp from rotation file', - output) - - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=LOG'], - return_id=False) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - self.assertNotIn( - 'WARNING: cannot read creation timestamp from rotation file', - output) - - self.assertTrue(os.path.isfile(rotation_file_path)) - - def test_unlink_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', - options=['--log-rotation-age=1d']) - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=VERBOSE']) - - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - log_file_size = os.stat(log_file_path).st_size - - self.assertTrue(os.path.isfile(rotation_file_path)) - - # unlink .rotation file - os.unlink(rotation_file_path) - - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=LOG'], - return_id=False) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - self.assertIn( - 'WARNING: missing rotation file:', - output) - - self.assertTrue(os.path.isfile(rotation_file_path)) - - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=VERBOSE'], - return_id=False) - - self.assertNotIn( - 'WARNING: missing rotation file:', - output) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - def test_garbage_in_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', - options=['--log-rotation-age=1d']) - - self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=VERBOSE']) - - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') - - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - log_file_size = os.stat(log_file_path).st_size - - self.assertTrue(os.path.isfile(rotation_file_path)) - - # mangle .rotation file - with open(rotation_file_path, "w+b", 0) as f: - f.write(b"blah") - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=LOG'], - return_id=False) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - self.assertIn( - 'WARNING: rotation file', - output) - - self.assertIn( - 'has wrong creation timestamp', - output) - - self.assertTrue(os.path.isfile(rotation_file_path)) - - output = self.backup_node( - backup_dir, 'node', node, - options=[ - '--stream', - '--log-level-file=LOG'], - return_id=False) - - self.assertNotIn( - 'WARNING: rotation file', - output) - - # check that log file wasn`t rotated - self.assertGreater( - os.stat(log_file_path).st_size, - log_file_size) - - def test_issue_274(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(node, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,45600) i") - - log_dir = os.path.join(backup_dir, "somedir") - - try: - self.backup_node( - backup_dir, 'node', replica, backup_type='page', - options=[ - '--log-level-console=verbose', '--log-level-file=verbose', - '--log-directory={0}'.format(log_dir), '-j1', - '--log-filename=somelog.txt', '--archive-timeout=5s', - '--no-validate', '--log-rotation-size=100KB']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of archiving timeout" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: WAL segment', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - log_file_path = os.path.join( - log_dir, 'somelog.txt') - - self.assertTrue(os.path.isfile(log_file_path)) - - with open(log_file_path, "r+") as f: - log_content = f.read() - - self.assertIn('INFO: command:', log_content) diff --git a/tests/merge_test.py b/tests/merge_test.py deleted file mode 100644 index ffa73263c..000000000 --- a/tests/merge_test.py +++ /dev/null @@ -1,2759 +0,0 @@ -# coding: utf-8 - -import unittest -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import QueryException -import shutil -from datetime import datetime, timedelta -import time -import subprocess - -class MergeTest(ProbackupTest, unittest.TestCase): - - def test_basic_merge_full_page(self): - """ - Test MERGE command, it merges FULL backup with target PAGE backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=["--data-checksums"]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Do full backup - self.backup_node(backup_dir, "node", node, options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Fill with data - with node.connect() as conn: - conn.execute("create table test (id int)") - conn.execute( - "insert into test select i from generate_series(1,10) s(i)") - conn.commit() - - # Do first page backup - self.backup_node(backup_dir, "node", node, backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[1] - - # sanity check - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Fill with data - with node.connect() as conn: - conn.execute( - "insert into test select i from generate_series(1,10) s(i)") - count1 = conn.execute("select count(*) from test") - conn.commit() - - # Do second page backup - self.backup_node( - backup_dir, "node", node, - backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # sanity check - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, - options=["-j", "4"]) - show_backups = self.show_pb(backup_dir, "node") - - # sanity check - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - # Check physical correctness - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - - # Check restored node - count2 = node.execute("postgres", "select count(*) from test") - self.assertEqual(count1, count2) - - def test_merge_compressed_backups(self): - """ - Test MERGE command with compressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=["--data-checksums"]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Do full compressed backup - self.backup_node(backup_dir, "node", node, options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Fill with data - with node.connect() as conn: - conn.execute("create table test (id int)") - conn.execute( - "insert into test select i from generate_series(1,10) s(i)") - count1 = conn.execute("select count(*) from test") - conn.commit() - - # Do compressed page backup - self.backup_node( - backup_dir, "node", node, backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[1] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - # Check restored node - count2 = node.execute("postgres", "select count(*) from test") - self.assertEqual(count1, count2) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_backups_1(self): - """ - Test MERGE command with compressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=10) - - # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=['--compress', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, - backup_type="delta", options=['--compress', '--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page", options=['--compress']) - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_and_uncompressed_backups(self): - """ - Test MERGE command with compressed and uncompressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=10) - - # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--compress', '--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do uncompressed PAGE backup - self.backup_node(backup_dir, "node", node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_and_uncompressed_backups_1(self): - """ - Test MERGE command with compressed and uncompressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=5) - - # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do uncompressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page", - options=['--compress-algorithm=zlib']) - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_and_uncompressed_backups_2(self): - """ - Test MERGE command with compressed and uncompressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=20) - - # Do uncompressed FULL backup - self.backup_node(backup_dir, "node", node) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--compress-algorithm=zlib', '--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do uncompressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_merge_tablespaces(self): - """ - Create tablespace with table, take FULL backup, - create another tablespace with another table and drop previous - tablespace, take page backup, merge it and restore - - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # Create new tablespace - self.create_tblspace_in_node(node, 'somedata1') - - node.safe_psql( - "postgres", - "create table t_heap1 tablespace somedata1 as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - - node.safe_psql( - "postgres", - "drop table t_heap" - ) - - # Drop old tablespace - node.safe_psql( - "postgres", - "drop tablespace somedata" - ) - - # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(node, 'somedata1'), - ignore_errors=True) - node.cleanup() - - self.merge_backup(backup_dir, 'node', backup_id) - - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - - # this compare should fall because we lost some directories - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_merge_tablespaces_1(self): - """ - Create tablespace with table, take FULL backup, - create another tablespace with another table, take page backup, - drop first tablespace and take delta backup, - merge it and restore - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # FULL backup - self.backup_node(backup_dir, 'node', node) - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - - # CREATE NEW TABLESPACE - self.create_tblspace_in_node(node, 'somedata1') - - node.safe_psql( - "postgres", - "create table t_heap1 tablespace somedata1 as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - - # PAGE backup - self.backup_node(backup_dir, 'node', node, backup_type="page") - - node.safe_psql( - "postgres", - "drop table t_heap" - ) - node.safe_psql( - "postgres", - "drop tablespace somedata" - ) - - # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta") - - pgdata = self.pgdata_content(node.data_dir) - - node.stop() - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(node, 'somedata1'), - ignore_errors=True) - node.cleanup() - - self.merge_backup(backup_dir, 'node', backup_id) - - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_merge_page_truncate(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take page backup, merge full and page, - restore last page backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s'}) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - page_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", page_id) - - self.validate_pb(backup_dir) - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") - - self.assertEqual(result1, result2) - - def test_merge_delta_truncate(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take page backup, merge full and page, - restore last page backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s'}) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - page_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", page_id) - - self.validate_pb(backup_dir) - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") - - self.assertEqual(result1, result2) - - def test_merge_ptrack_truncate(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take page backup, merge full and page, - restore last page backup and check data correctness - """ - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - ptrack_enable=True) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.merge_backup(backup_dir, "node", page_id) - - self.validate_pb(backup_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") - - self.assertEqual(result1, result2) - - # @unittest.skip("skip") - def test_merge_delta_delete(self): - """ - Make node, create tablespace with table, take full backup, - alter tablespace location, take delta backup, merge full and delta, - restore database. - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - - node.safe_psql( - "postgres", - "delete from t_heap" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - backup_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", backup_id, options=["-j", "4"]) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata') - ) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_continue_failed_merge(self): - """ - Check that failed MERGE can be continued - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join( - self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i" - ) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) - - node.safe_psql( - "postgres", - "delete from t_heap" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta' - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - backup_id = self.show_pb(backup_dir, "node")[2]["id"] - - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - - gdb.set_breakpoint('backup_non_data_file_internal') - gdb.run_until_break() - - gdb.continue_execution_until_break(5) - - gdb._execute('signal SIGKILL') - gdb._execute('detach') - time.sleep(1) - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - - # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - # @unittest.skip("skip") - def test_continue_failed_merge_with_corrupted_delta_backup(self): - """ - Fail merge via gdb, corrupt DELTA backup, try to continue merge - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i") - - old_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - node.safe_psql( - "postgres", - "vacuum full t_heap") - - new_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - # DELTA BACKUP - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - backup_id = self.show_pb(backup_dir, "node")[1]["id"] - - # Failed MERGE - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('backup_non_data_file_internal') - gdb.run_until_break() - - gdb.continue_execution_until_break(2) - - gdb._execute('signal SIGKILL') - - # CORRUPT incremental backup - # read block from future - # block_size + backup_header = 8200 - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', new_path) - with open(file, 'rb') as f: - f.seek(8200) - block_1 = f.read(8200) - f.close - - # write block from future - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', old_path) - with open(file, 'r+b') as f: - f.seek(8200) - f.write(block_1) - f.close - - # Try to continue failed MERGE - try: - print(self.merge_backup(backup_dir, "node", backup_id)) - self.assertEqual( - 1, 0, - "Expecting Error because of incremental backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Backup {0} has status CORRUPT, merge is aborted".format( - backup_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_continue_failed_merge_2(self): - """ - Check that failed MERGE on delete can be continued - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i") - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - backup_id = self.show_pb(backup_dir, "node")[2]["id"] - - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - - gdb.set_breakpoint('pgFileDelete') - - gdb.run_until_break() - - gdb._execute('thread apply all bt') - - gdb.continue_execution_until_break(20) - - gdb._execute('thread apply all bt') - - gdb._execute('signal SIGKILL') - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - - backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] - - # TODO check that full backup has meta info is equal to DELETTING - - # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) - - def test_continue_failed_merge_3(self): - """ - Check that failed MERGE cannot be continued if intermediate - backup is missing. - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Create test data - node.safe_psql("postgres", "create sequence t_seq") - node.safe_psql( - "postgres", - "create table t_heap as select i as id, nextval('t_seq')" - " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" - " as tsvector from generate_series(0,100000) i" - ) - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # CREATE FEW PAGE BACKUP - i = 0 - - while i < 2: - - node.safe_psql( - "postgres", - "delete from t_heap" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - node.safe_psql( - "postgres", - "insert into t_heap select i as id, nextval('t_seq') as t_seq," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(100,200000) i" - ) - - # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page' - ) - i = i + 1 - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - backup_id_merge = self.show_pb(backup_dir, "node")[2]["id"] - backup_id_delete = self.show_pb(backup_dir, "node")[1]["id"] - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - - gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) - - gdb.set_breakpoint('backup_non_data_file_internal') - gdb.run_until_break() - gdb.continue_execution_until_break(2) - - gdb._execute('signal SIGKILL') - - print(self.show_pb(backup_dir, as_text=True, as_json=False)) - # print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) - - # DELETE PAGE1 - shutil.rmtree( - os.path.join(backup_dir, "backups", "node", backup_id_delete)) - - # Try to continue failed MERGE - try: - self.merge_backup(backup_dir, "node", backup_id_merge) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Incremental chain is broken, " - "merge is impossible to finish" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_merge_different_compression_algo(self): - """ - Check that backups with different compression algorithms can be merged - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--compress-algorithm=zlib']) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,1000) i") - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--compress-algorithm=pglz']) - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - backup_id = self.show_pb(backup_dir, "node")[2]["id"] - - self.merge_backup(backup_dir, "node", backup_id) - - def test_merge_different_wal_modes(self): - """ - Check that backups with different wal modes can be merged - correctly - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL stream backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # DELTA archive backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - self.assertEqual( - 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) - - # DELTA stream backup - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - self.assertEqual( - 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) - - def test_crash_after_opening_backup_control_1(self): - """ - check that crashing after opening backup.control - for writing will not result in losing backup metadata - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL stream backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # DELTA archive backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('write_backup_filelist') - gdb.run_until_break() - - gdb.set_breakpoint('write_backup') - gdb.continue_execution_until_break() - gdb.set_breakpoint('pgBackupWriteControl') - gdb.continue_execution_until_break() - - gdb._execute('signal SIGKILL') - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - - # @unittest.skip("skip") - def test_crash_after_opening_backup_control_2(self): - """ - check that crashing after opening backup_content.control - for writing will not result in losing metadata about backup files - TODO: rewrite - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Add data - node.pgbench_init(scale=3) - - # FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '2']) - pgbench.wait() - - path = node.safe_psql( - 'postgres', - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - fsm_path = path + '_fsm' - - node.safe_psql( - 'postgres', - 'vacuum pgbench_accounts') - - # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('write_backup_filelist') - gdb.run_until_break() - -# gdb.set_breakpoint('sprintf') -# gdb.continue_execution_until_break(1) - - gdb._execute('signal SIGKILL') - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - - # In to_backup drop file that comes from from_backup - # emulate crash during previous merge - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', full_id, 'database', fsm_path) - - # print(file_to_remove) - - os.remove(file_to_remove) - - # Continue failed merge - self.merge_backup(backup_dir, "node", backup_id) - - node.cleanup() - - # restore merge backup - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_losing_file_after_failed_merge(self): - """ - check that crashing after opening backup_content.control - for writing will not result in losing metadata about backup files - TODO: rewrite - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Add data - node.pgbench_init(scale=1) - - # FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # Change data - node.safe_psql( - 'postgres', - "update pgbench_accounts set aid = aid + 1005000") - - path = node.safe_psql( - 'postgres', - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - node.safe_psql( - 'postgres', - "VACUUM pgbench_accounts") - - vm_path = path + '_vm' - - # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('write_backup_filelist') - gdb.run_until_break() - -# gdb.set_breakpoint('sprintf') -# gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - - # In to_backup drop file that comes from from_backup - # emulate crash during previous merge - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', full_id, 'database', vm_path) - - os.remove(file_to_remove) - - # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_failed_merge_after_delete(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # add database - node.safe_psql( - 'postgres', - 'CREATE DATABASE testdb') - - dboid = node.safe_psql( - "postgres", - "select oid from pg_database where datname = 'testdb'").decode('utf-8').rstrip() - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # drop database - node.safe_psql( - 'postgres', - 'DROP DATABASE testdb') - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - gdb = self.merge_backup( - backup_dir, 'node', page_id, - gdb=True, options=['--log-level-console=verbose']) - - gdb.set_breakpoint('delete_backup_files') - gdb.run_until_break() - - gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - - # backup half-merged - self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - db_path = os.path.join( - backup_dir, 'backups', 'node', - full_id, 'database', 'base', dboid) - - try: - self.merge_backup( - backup_dir, 'node', page_id_2, - options=['--log-level-console=verbose']) - self.assertEqual( - 1, 0, - "Expecting Error because of missing parent.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Full backup {0} has unfinished merge with backup {1}".format( - full_id, page_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_failed_merge_after_delete_1(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=1) - - page_1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGE1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_1, 'ERROR') - - pgdata = self.pgdata_content(node.data_dir) - - # add data - pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) - pgbench.wait() - - # take PAGE2 backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGE1 backup status to OK - self.change_backup_status(backup_dir, 'node', page_1, 'OK') - - gdb = self.merge_backup( - backup_dir, 'node', page_id, - gdb=True, options=['--log-level-console=verbose']) - - gdb.set_breakpoint('delete_backup_files') - gdb.run_until_break() - -# gdb.set_breakpoint('parray_bsearch') -# gdb.continue_execution_until_break() - - gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(30) - gdb._execute('signal SIGKILL') - - self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - # restore - node.cleanup() - try: - #self.restore_node(backup_dir, 'node', node, backup_id=page_1) - self.restore_node(backup_dir, 'node', node) - self.assertEqual( - 1, 0, - "Expecting Error because of orphan status.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} is orphan".format(page_1), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_failed_merge_after_delete_2(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=1) - - page_1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # add data - pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) - pgbench.wait() - - # take PAGE2 backup - page_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - gdb = self.merge_backup( - backup_dir, 'node', page_2, gdb=True, - options=['--log-level-console=VERBOSE']) - - gdb.set_breakpoint('pgFileDelete') - gdb.run_until_break() - gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') - - self.delete_pb(backup_dir, 'node', backup_id=page_2) - - # rerun merge - try: - #self.restore_node(backup_dir, 'node', node, backup_id=page_1) - self.merge_backup(backup_dir, 'node', page_1) - self.assertEqual( - 1, 0, - "Expecting Error because of backup is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Full backup {0} has unfinished merge " - "with backup {1}".format(full_id, page_2), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_failed_merge_after_delete_3(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # add database - node.safe_psql( - 'postgres', - 'CREATE DATABASE testdb') - - dboid = node.safe_psql( - "postgres", - "select oid from pg_database where datname = 'testdb'").rstrip() - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # drop database - node.safe_psql( - 'postgres', - 'DROP DATABASE testdb') - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # create database - node.safe_psql( - 'postgres', - 'create DATABASE testdb') - - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - gdb = self.merge_backup( - backup_dir, 'node', page_id, - gdb=True, options=['--log-level-console=verbose']) - - gdb.set_breakpoint('delete_backup_files') - gdb.run_until_break() - - gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(20) - - gdb._execute('signal SIGKILL') - - # backup half-merged - self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - db_path = os.path.join( - backup_dir, 'backups', 'node', full_id) - - # FULL backup is missing now - shutil.rmtree(db_path) - - try: - self.merge_backup( - backup_dir, 'node', page_id_2, - options=['--log-level-console=verbose']) - self.assertEqual( - 1, 0, - "Expecting Error because of missing parent.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Failed to find parent full backup for {0}".format( - page_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Skipped, because backups from the future are invalid. - # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" - # now (PBCKP-259). We can conduct such a test again when we - # untie 'backup_id' from 'start_time' - @unittest.skip("skip") - def test_merge_backup_from_future(self): - """ - take FULL backup, table PAGE backup from future, - try to merge page with FULL - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=5) - - # Take PAGE from future - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - with open( - os.path.join( - backup_dir, 'backups', 'node', - backup_id, "backup.control"), "a") as conf: - conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() + timedelta(days=3))) - - # rename directory - new_id = self.show_pb(backup_dir, 'node')[1]['id'] - - os.rename( - os.path.join(backup_dir, 'backups', 'node', backup_id), - os.path.join(backup_dir, 'backups', 'node', new_id)) - - pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) - pgbench.wait() - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', - node_restored, backup_id=backup_id) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # check that merged backup has the same state as - node_restored.cleanup() - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - self.restore_node( - backup_dir, 'node', - node_restored, backup_id=backup_id) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - self.set_auto_conf( - node_restored, - {'port': node_restored.port}) - node_restored.slow_start() - - result_new = node_restored.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') - - self.assertTrue(result, result_new) - - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_merge_multiple_descendants(self): - """ - PAGEb3 - | PAGEa3 - PAGEb2 / - | PAGEa2 / - PAGEb1 \ / - | PAGEa1 - FULLb | - FULLa - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change FULLb backup status to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # Change PAGEa1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # Change PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa2 and FULL to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - # Change PAGEb2, PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change FULLa to OK - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa3 OK - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa3 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - - # Change PAGEb2, PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb3 OK - # PAGEa3 ERROR - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - # Change PAGEa3, PAGEa2 and FULLa status to OK - self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb3 OK - # PAGEa3 OK - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 - self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], - page_id_a1) - - self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], - page_id_a1) - - self.merge_backup( - backup_dir, 'node', page_id_a2, - options=['--merge-expired', '--log-level-console=log']) - - try: - self.merge_backup( - backup_dir, 'node', page_id_a3, - options=['--merge-expired', '--log-level-console=log']) - self.assertEqual( - 1, 0, - "Expecting Error because of parent FULL backup is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Failed to find parent full backup for {0}".format( - page_id_a3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_smart_merge(self): - """ - make node, create database, take full backup, drop database, - take PAGE backup and merge it into FULL, - make sure that files from dropped database are not - copied during restore - https://github.com/postgrespro/pg_probackup/issues/63 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # create database - node.safe_psql( - "postgres", - "CREATE DATABASE testdb") - - # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - # drop database - node.safe_psql( - "postgres", - "DROP DATABASE testdb") - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) - - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) - - filelist_diff = self.get_backup_filelist_diff( - filelist_full, filelist_page) - - # merge PAGE backup - self.merge_backup( - backup_dir, 'node', page_id, - options=['--log-level-file=VERBOSE']) - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - def test_idempotent_merge(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # add database - node.safe_psql( - 'postgres', - 'CREATE DATABASE testdb') - - # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # create database - node.safe_psql( - 'postgres', - 'create DATABASE testdb1') - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # create database - node.safe_psql( - 'postgres', - 'create DATABASE testdb2') - - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - gdb = self.merge_backup( - backup_dir, 'node', page_id_2, - gdb=True, options=['--log-level-console=verbose']) - - gdb.set_breakpoint('delete_backup_files') - gdb.run_until_break() - gdb.remove_all_breakpoints() - - gdb.set_breakpoint('rename') - gdb.continue_execution_until_break() - gdb.continue_execution_until_break(2) - - gdb._execute('signal SIGKILL') - - show_backups = self.show_pb(backup_dir, "node") - self.assertEqual(len(show_backups), 1) - - self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - self.merge_backup(backup_dir, 'node', page_id_2) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) - - def test_merge_correct_inheritance(self): - """ - Make sure that backup metainformation fields - 'note' and 'expire-time' are correctly inherited - during merge - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # add database - node.safe_psql( - 'postgres', - 'CREATE DATABASE testdb') - - # take FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # create database - node.safe_psql( - 'postgres', - 'create DATABASE testdb1') - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.set_backup( - backup_dir, 'node', page_id, options=['--note=hello', '--ttl=20d']) - - page_meta = self.show_pb(backup_dir, 'node', page_id) - - self.merge_backup(backup_dir, 'node', page_id) - - print(self.show_pb(backup_dir, 'node', page_id)) - - self.assertEqual( - page_meta['note'], - self.show_pb(backup_dir, 'node', page_id)['note']) - - self.assertEqual( - page_meta['expire-time'], - self.show_pb(backup_dir, 'node', page_id)['expire-time']) - - def test_merge_correct_inheritance_1(self): - """ - Make sure that backup metainformation fields - 'note' and 'expire-time' are correctly inherited - during merge - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # add database - node.safe_psql( - 'postgres', - 'CREATE DATABASE testdb') - - # take FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--note=hello', '--ttl=20d']) - - # create database - node.safe_psql( - 'postgres', - 'create DATABASE testdb1') - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.merge_backup(backup_dir, 'node', page_id) - - self.assertNotIn( - 'note', - self.show_pb(backup_dir, 'node', page_id)) - - self.assertNotIn( - 'expire-time', - self.show_pb(backup_dir, 'node', page_id)) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_multi_timeline_merge(self): - """ - Check that backup in PAGE mode choose - parent backup correctly: - t12 /---P--> - ... - t3 /----> - t2 /----> - t1 -F-----D-> - - P must have F as parent - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql("postgres", "create extension pageinspect") - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - node.slow_start() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # create timelines - for i in range(2, 7): - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target=latest', - '--recovery-target-action=promote', - '--recovery-target-timeline={0}'.format(i)]) - node.slow_start() - - # at this point there is i+1 timeline - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # create backup at 2, 4 and 6 timeline - if i % 2 == 0: - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - pgdata = self.pgdata_content(node.data_dir) - - self.merge_backup(backup_dir, 'node', page_id) - - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") - - self.assertEqual(result, result_new) - - self.compare_pgdata(pgdata, pgdata_restored) - - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '-d', 'postgres', '-p', str(node.port)]) - - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '-d', 'postgres', '-p', str(node_restored.port)]) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_merge_page_header_map_retry(self): - """ - page header map cannot be trusted when - running retry - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=20) - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - delta_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - gdb = self.merge_backup(backup_dir, 'node', delta_id, gdb=True) - - # our goal here is to get full backup with merged data files, - # but with old page header map - gdb.set_breakpoint('cleanup_header_map') - gdb.run_until_break() - gdb._execute('signal SIGKILL') - - self.merge_backup(backup_dir, 'node', delta_id) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_missing_data_file(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Add data - node.pgbench_init(scale=1) - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # Change data - pgbench = node.pgbench(options=['-T', '5', '-c', '1']) - pgbench.wait() - - # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - path = node.safe_psql( - 'postgres', - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - gdb = self.merge_backup( - backup_dir, "node", delta_id, - options=['--log-level-file=VERBOSE'], gdb=True) - gdb.set_breakpoint('merge_files') - gdb.run_until_break() - - # remove data file in incremental backup - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', delta_id, 'database', path) - - os.remove(file_to_remove) - - gdb.continue_execution_until_error() - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - self.assertIn( - 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), - logfile_content) - - # @unittest.skip("skip") - def test_missing_non_data_file(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - gdb = self.merge_backup( - backup_dir, "node", delta_id, - options=['--log-level-file=VERBOSE'], gdb=True) - gdb.set_breakpoint('merge_files') - gdb.run_until_break() - - # remove data file in incremental backup - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', delta_id, 'database', 'backup_label') - - os.remove(file_to_remove) - - gdb.continue_execution_until_error() - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - self.assertIn( - 'ERROR: File "{0}" is not found'.format(file_to_remove), - logfile_content) - - self.assertIn( - 'ERROR: Backup files merging failed', - logfile_content) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) - - self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - - # @unittest.skip("skip") - def test_merge_remote_mode(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - self.set_config(backup_dir, 'node', options=['--retention-window=1']) - - backups = os.path.join(backup_dir, 'backups', 'node') - with open( - os.path.join( - backups, full_id, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=5))) - - gdb = self.backup_node( - backup_dir, "node", node, - options=['--log-level-file=VERBOSE', '--merge-expired'], gdb=True) - gdb.set_breakpoint('merge_files') - gdb.run_until_break() - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - - with open(logfile, "w+") as f: - f.truncate() - - gdb.continue_execution_until_exit() - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - self.assertNotIn( - 'SSH', logfile_content) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) - - def test_merge_pg_filenode_map(self): - """ - https://github.com/postgrespro/pg_probackup/issues/320 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) - node1.cleanup() - - node.pgbench_init(scale=5) - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '1']) - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.safe_psql( - 'postgres', - 'reindex index pg_type_oid_index') - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.merge_backup(backup_dir, 'node', backup_id) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'select 1') - -# 1. Need new test with corrupted FULL backup -# 2. different compression levels diff --git a/tests/option_test.py b/tests/option_test.py deleted file mode 100644 index eec1bab44..000000000 --- a/tests/option_test.py +++ /dev/null @@ -1,231 +0,0 @@ -import unittest -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import locale - - -class OptionTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_help_1(self): - """help options""" - with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: - self.assertEqual( - self.run_pb(["--help"]), - help_out.read().decode("utf-8") - ) - - # @unittest.skip("skip") - def test_version_2(self): - """help options""" - with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: - self.assertIn( - version_out.read().decode("utf-8").strip(), - self.run_pb(["--version"]) - ) - - # @unittest.skip("skip") - def test_without_backup_path_3(self): - """backup command failure without backup mode option""" - try: - self.run_pb(["backup", "-b", "full"]) - self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_options_4(self): - """check options test""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # backup command failure without instance option - try: - self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) - self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: required parameter not specified: --instance', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # backup command failure without backup mode option - try: - self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) - self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # backup command failure with invalid backup mode option - try: - self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) - self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: invalid backup-mode "bad"', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # delete failure without delete options - try: - self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: You must specify at least one of the delete options: ' - '--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - - # delete failure without ID - try: - self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "option requires an argument -- 'i'", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_options_5(self): - """check options test""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - - output = self.init_pb(backup_dir) - self.assertIn( - "INFO: Backup catalog", - output) - - self.assertIn( - "successfully inited", - output) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - # syntax error in pg_probackup.conf - conf_file = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") - with open(conf_file, "a") as conf: - conf.write(" = INFINITE\n") - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Syntax error in " = INFINITE', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # invalid value in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("BACKUP_MODE=\n") - - try: - self.backup_node(backup_dir, 'node', node, backup_type=None), - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "BACKUP_MODE" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # Command line parameters should override file values - with open(conf_file, "a") as conf: - conf.write("retention-redundancy=1\n") - - self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') - - # User cannot send --system-identifier parameter via command line - try: - self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Option system-identifier cannot be specified in command line', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # invalid value in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("SMOOTH_CHECKPOINT=FOO\n") - - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # invalid option in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("TIMELINEID=1\n") - - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "TIMELINEID" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_help_6(self): - """help options""" - if ProbackupTest.enable_nls: - self.test_env['LC_ALL'] = 'ru_RU.utf-8' - with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: - self.assertEqual( - self.run_pb(["--help"]), - help_out.read().decode("utf-8") - ) - else: - self.skipTest( - 'You need configure PostgreSQL with --enabled-nls option for this test') diff --git a/tests/page_test.py b/tests/page_test.py deleted file mode 100644 index e77e5c827..000000000 --- a/tests/page_test.py +++ /dev/null @@ -1,1424 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import QueryException -from datetime import datetime, timedelta -import subprocess -import gzip -import shutil - -class PageTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_basic_page_vacuum_truncate(self): - """ - make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take page backup, take second page backup, - restore last page backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '300s'}) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node_restored.cleanup() - node.slow_start() - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node) - - # TODO: make it dynamic - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") - - self.assertEqual(result1, result2) - - # @unittest.skip("skip") - def test_page_vacuum_truncate_1(self): - """ - make node, create table, take full backup, - delete all data, vacuum relation, - take page backup, insert some data, - take second page backup and check data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1) i") - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - # Physical comparison - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_page_stream(self): - """ - make archive node, take full and page stream backups, - restore them and check data correctness - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(0,100) i") - - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--stream']) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--stream', '-j', '4']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) - - node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=page_backup_id, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # @unittest.skip("skip") - def test_page_archive(self): - """ - make archive node, take full and page archive backups, - restore them and check data correctness - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'} - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full') - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, md5(i::text)::tsvector as tsvector " - "from generate_series(100, 200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Drop Node - node.cleanup() - - # Restore and check full backup - self.assertIn("INFO: Restore of backup {0} completed.".format( - full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, - options=[ - "-j", "4", - "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Restore and check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=page_backup_id, - options=[ - "-j", "4", - "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() - - # @unittest.skip("skip") - def test_page_multiple_segments(self): - """ - Make node, create table with multiple segments, - write some data to it, check page and data correctness - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'fsync': 'off', - 'shared_buffers': '1GB', - 'maintenance_work_mem': '1GB', - 'full_page_writes': 'off'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # CREATE TABLE - node.pgbench_init(scale=100, options=['--tablespace=somedata']) - # FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - # PGBENCH STUFF - pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") - # PAGE BACKUP - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # GET PHYSICAL CONTENT FROM NODE - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) - restored_node.cleanup() - tblspc_path = self.get_tblspace_path(node, 'somedata') - tblspc_path_new = self.get_tblspace_path( - restored_node, 'somedata_restored') - - self.restore_node( - backup_dir, 'node', restored_node, - options=[ - "-j", "4", - "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) - - # GET PHYSICAL CONTENT FROM NODE_RESTORED - pgdata_restored = self.pgdata_content(restored_node.data_dir) - - # START RESTORED NODE - self.set_auto_conf(restored_node, {'port': restored_node.port}) - restored_node.slow_start() - - result_new = restored_node.safe_psql( - "postgres", "select count(*) from pgbench_accounts") - - # COMPARE RESTORED FILES - self.assertEqual(result, result_new, 'data is lost') - - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_page_delete(self): - """ - Make node, create tablespace with table, take full backup, - delete everything from table, vacuum table, take page backup, - restore page backup, compare . - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - # FULL backup - self.backup_node(backup_dir, 'node', node) - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i") - - node.safe_psql( - "postgres", - "delete from t_heap") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page') - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata')) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # @unittest.skip("skip") - def test_page_delete_1(self): - """ - Make node, create tablespace with table, take full backup, - delete everything from table, vacuum table, take page backup, - restore page backup, compare . - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i" - ) - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "delete from t_heap" - ) - - node.safe_psql( - "postgres", - "vacuum t_heap" - ) - - # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page') - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata')) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - def test_parallel_pagemap(self): - """ - Test for parallel WAL segments reading, during which pagemap is built - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - "hot_standby": "on" - } - ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node_restored.cleanup() - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Do full backup - self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] - - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "FULL") - - # Fill instance with data and make several WAL segments ... - with node.connect() as conn: - conn.execute("create table test (id int)") - for x in range(0, 8): - conn.execute( - "insert into test select i from generate_series(1,100) s(i)") - conn.commit() - self.switch_wal_segment(conn) - count1 = conn.execute("select count(*) from test") - - # ... and do page backup with parallel pagemap - self.backup_node( - backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) - show_backup = self.show_pb(backup_dir, 'node')[1] - - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PAGE") - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Restore it - self.restore_node(backup_dir, 'node', node_restored) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # Check restored node - count2 = node_restored.execute("postgres", "select count(*) from test") - - self.assertEqual(count1, count2) - - # Clean after yourself - node.cleanup() - node_restored.cleanup() - - def test_parallel_pagemap_1(self): - """ - Test for parallel WAL segments reading, during which pagemap is built - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={} - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Do full backup - self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] - - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "FULL") - - # Fill instance with data and make several WAL segments ... - node.pgbench_init(scale=10) - - # do page backup in single thread - page_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - - self.delete_pb(backup_dir, 'node', page_id) - - # ... and do page backup with parallel pagemap - self.backup_node( - backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) - show_backup = self.show_pb(backup_dir, 'node')[1] - - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PAGE") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - node.slow_start() - - # Clean after yourself - node.cleanup() - - # @unittest.skip("skip") - def test_page_backup_with_lost_wal_segment(self): - """ - make node with archiving - make archive backup, then generate some wals with pgbench, - delete latest archived wal segment - run page backup, expecting error because of missing wal segment - make sure that backup status is 'ERROR' - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - # make some wals - node.pgbench_init(scale=3) - - # delete last wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] - wals = map(str, wals) - file = os.path.join(wals_dir, max(wals)) - os.remove(file) - if self.archive_compress: - file = file[:-3] - - # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'is absent' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'is absent' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # @unittest.skip("skip") - def test_page_backup_with_corrupted_wal_segment(self): - """ - make node with archiving - make archive backup, then generate some wals with pgbench, - corrupt latest archived wal segment - run page backup, expecting error because of missing wal segment - make sure that backup status is 'ERROR' - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - # make some wals - node.pgbench_init(scale=10) - - # delete last wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup')] - wals = map(str, wals) - # file = os.path.join(wals_dir, max(wals)) - - if self.archive_compress: - original_file = os.path.join(wals_dir, '000000010000000000000004.gz') - tmp_file = os.path.join(backup_dir, '000000010000000000000004') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - # drop healthy file - os.remove(original_file) - file = tmp_file - - else: - file = os.path.join(wals_dir, '000000010000000000000004') - - # corrupt file - print(file) - with open(file, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - if self.archive_compress: - # compress corrupted file and replace with it old file - with open(file, 'rb') as f_in, gzip.open(original_file, 'wb', compresslevel=1) as f_out: - shutil.copyfileobj(f_in, f_out) - - file = os.path.join(wals_dir, '000000010000000000000004.gz') - - #if self.archive_compress: - # file = file[:-3] - - # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( - file) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # @unittest.skip("skip") - def test_page_backup_with_alien_wal_segment(self): - """ - make two nodes with archiving - take archive full backup from both nodes, - generate some wals with pgbench on both nodes, - move latest archived wal segment from second node to first node`s archive - run page backup on first node - expecting error because of alien wal segment - make sure that backup status is 'ERROR' - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - alien_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'alien_node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.add_instance(backup_dir, 'alien_node', alien_node) - self.set_archiving(backup_dir, 'alien_node', alien_node) - alien_node.slow_start() - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - self.backup_node( - backup_dir, 'alien_node', alien_node, options=['--stream']) - - # make some wals - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i;") - - alien_node.safe_psql( - "postgres", - "create database alien") - - alien_node.safe_psql( - "alien", - "create sequence t_seq; " - "create table t_heap_alien as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100000) i;") - - # copy latest wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup')] - wals = map(str, wals) - filename = max(wals) - file = os.path.join(wals_dir, filename) - file_destination = os.path.join( - os.path.join(backup_dir, 'wal', 'node'), filename) -# file = os.path.join(wals_dir, '000000010000000000000004') - print(file) - print(file_destination) - os.remove(file_destination) - os.rename(file, file_destination) - - # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of alien wal segment.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of alien wal segment.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn('Could not read WAL record at', e.message) - self.assertIn('WAL file is from different database system: ' - 'WAL file database system identifier is', e.message) - self.assertIn('pg_control database system identifier is', e.message) - self.assertIn('Possible WAL corruption. Error has occured ' - 'during reading WAL segment', e.message) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], - 'Backup {0} should have STATUS "ERROR"') - - # @unittest.skip("skip") - def test_multithread_page_backup_with_toast(self): - """ - make node, create toast, do multithread PAGE backup - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - # make some wals - node.safe_psql( - "postgres", - "create table t3 as select i, " - "repeat(md5(i::text),5006056) as fat_attr " - "from generate_series(0,70) i") - - # Multi-thread PAGE backup - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - - # @unittest.skip("skip") - def test_page_create_db(self): - """ - Make node, take full backup, create database db1, take page backup, - restore database and check it presense - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '10GB', - 'checkpoint_timeout': '5min', - } - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - self.backup_node( - backup_dir, 'node', node) - - # CREATE DATABASE DB1 - node.safe_psql("postgres", "create database db1") - node.safe_psql( - "db1", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,1000) i") - - # PAGE BACKUP - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, options=["-j", "4"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - node_restored.safe_psql('db1', 'select 1') - node_restored.cleanup() - - # DROP DATABASE DB1 - node.safe_psql( - "postgres", "drop database db1") - # SECOND PAGE BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE SECOND PAGE BACKUP - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, options=["-j", "4"] - ) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd) - ) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_multi_timeline_page(self): - """ - Check that backup in PAGE mode choose - parent backup correctly: - t12 /---P--> - ... - t3 /----> - t2 /----> - t1 -F-----D-> - - P must have F as parent - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql("postgres", "create extension pageinspect") - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - node.slow_start() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # create timelines - for i in range(2, 7): - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target=latest', - '--recovery-target-action=promote', - '--recovery-target-timeline={0}'.format(i)]) - node.slow_start() - - # at this point there is i+1 timeline - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # create backup at 2, 4 and 6 timeline - if i % 2 == 0: - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-file=VERBOSE']) - - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") - - self.assertEqual(result, result_new) - - self.compare_pgdata(pgdata, pgdata_restored) - - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '-d', 'postgres', '-p', str(node.port)]) - - self.checkdb_node( - backup_dir, - 'node', - options=[ - '--amcheck', - '-d', 'postgres', '-p', str(node_restored.port)]) - - backup_list = self.show_pb(backup_dir, 'node') - - self.assertEqual( - backup_list[2]['parent-backup-id'], - backup_list[0]['id']) - self.assertEqual(backup_list[2]['current-tli'], 3) - - self.assertEqual( - backup_list[3]['parent-backup-id'], - backup_list[2]['id']) - self.assertEqual(backup_list[3]['current-tli'], 5) - - self.assertEqual( - backup_list[4]['parent-backup-id'], - backup_list[3]['id']) - self.assertEqual(backup_list[4]['current-tli'], 7) - - self.assertEqual( - backup_list[5]['parent-backup-id'], - backup_list[4]['id']) - self.assertEqual(backup_list[5]['current-tli'], 7) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_multitimeline_page_1(self): - """ - Check that backup in PAGE mode choose - parent backup correctly: - t2 /----> - t1 -F--P---D-> - - P must have F as parent - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql("postgres", "create extension pageinspect") - - try: - node.safe_psql( - "postgres", - "create extension amcheck") - except QueryException as e: - node.safe_psql( - "postgres", - "create extension amcheck_next") - - node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '20', '-c', '1']) - pgbench.wait() - - page1 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - page1 = self.backup_node(backup_dir, 'node', node, backup_type='delta') - - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page1, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - node.slow_start() - - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() - - print(self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--log-level-console=LOG'], return_id=False)) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - self.compare_pgdata(pgdata, pgdata_restored) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_page_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'shared_buffers': '512MB', - 'max_wal_size': '3GB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select nextval('t_seq')::int as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(0,25600) i") - "from generate_series(0,2560) i") - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - self.switch_wal_segment(node) - - # kill the bastard - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - # now smack it with sledgehammer - if node.major_version >= 10: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - - self.run_binary( - [ - pg_resetxlog_path, - '-D', - node.data_dir, - '-o 42', - '-f' - ], - asynchronous=False) - - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, -# backup_type='page', options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'Insert error message', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - -# pgdata = self.pgdata_content(node.data_dir) -# -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) -# node_restored.cleanup() -# -# self.restore_node( -# backup_dir, 'node', node_restored) -# -# pgdata_restored = self.pgdata_content(node_restored.data_dir) -# self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/pgpro2068_test.py b/tests/pgpro2068_test.py deleted file mode 100644 index 434ce2800..000000000 --- a/tests/pgpro2068_test.py +++ /dev/null @@ -1,188 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -from time import sleep -import shutil -import signal -from testgres import ProcessType - - -class BugTest(ProbackupTest, unittest.TestCase): - - def test_minrecpoint_on_replica(self): - """ - https://jira.postgrespro.ru/browse/PGPRO-2068 - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - # 'checkpoint_timeout': '60min', - 'checkpoint_completion_target': '0.9', - 'bgwriter_delay': '10ms', - 'bgwriter_lru_maxpages': '1000', - 'bgwriter_lru_multiplier': '4.0', - 'max_wal_size': '256MB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take full backup and restore it as replica - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # start replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'node', replica, options=['-R']) - self.set_replica(node, replica) - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - self.set_auto_conf( - replica, - {'port': replica.port, 'restart_after_crash': 'off'}) - - # we need those later - node.safe_psql( - "postgres", - "CREATE EXTENSION plpython3u") - - node.safe_psql( - "postgres", - "CREATE EXTENSION pageinspect") - - replica.slow_start(replica=True) - - # generate some data - node.pgbench_init(scale=10) - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"]) - pgbench.wait() - pgbench.stdout.close() - - # generate some more data and leave it in background - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-j 4", "-T", "100"]) - - # wait for shared buffer on replica to be filled with dirty data - sleep(20) - - # get pids of replica background workers - startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] - checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] - bgwriter_pid = replica.auxiliary_pids[ProcessType.BackgroundWriter][0] - - # break checkpointer on UpdateLastRemovedPtr - gdb_checkpointer = self.gdb_attach(checkpointer_pid) - gdb_checkpointer._execute('handle SIGINT noprint nostop pass') - gdb_checkpointer._execute('handle SIGUSR1 noprint nostop pass') - gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') - gdb_checkpointer.continue_execution_until_break() - - # break recovery on UpdateControlFile - gdb_recovery = self.gdb_attach(startup_pid) - gdb_recovery._execute('handle SIGINT noprint nostop pass') - gdb_recovery._execute('handle SIGUSR1 noprint nostop pass') - gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') - gdb_recovery.continue_execution_until_break() - gdb_recovery.set_breakpoint('UpdateControlFile') - gdb_recovery.continue_execution_until_break() - - # stop data generation - pgbench.wait() - pgbench.stdout.close() - - # kill someone, we need a crash - os.kill(int(bgwriter_pid), 9) - gdb_recovery._execute('detach') - gdb_checkpointer._execute('detach') - - # just to be sure - try: - replica.stop(['-m', 'immediate', '-D', replica.data_dir]) - except: - pass - - # MinRecLSN = replica.get_control_data()['Minimum recovery ending location'] - - # Promote replica with 'immediate' target action - if self.get_version(replica) >= self.version_to_num('12.0'): - recovery_config = 'postgresql.auto.conf' - else: - recovery_config = 'recovery.conf' - - replica.append_conf( - recovery_config, "recovery_target = 'immediate'") - replica.append_conf( - recovery_config, "recovery_target_action = 'pause'") - replica.slow_start(replica=True) - - if self.get_version(node) < 100000: - script = ''' -DO -$$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("SELECT min_recovery_end_location as lsn FROM pg_control_recovery()")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpython3u; -''' - else: - script = ''' -DO -$$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("select pg_last_wal_replay_lsn() as lsn")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpython3u; -''' - - # Find blocks from future - replica.safe_psql( - 'postgres', - script) - - # error is expected if version < 10.6 - # gdb_backup.continue_execution_until_exit() - - # do basebackup - - # do pg_probackup, expect error diff --git a/tests/pgpro560_test.py b/tests/pgpro560_test.py deleted file mode 100644 index b665fd200..000000000 --- a/tests/pgpro560_test.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -import subprocess -from time import sleep - - -class CheckSystemID(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_pgpro560_control_file_loss(self): - """ - https://jira.postgrespro.ru/browse/PGPRO-560 - make node with stream support, delete control file - make backup - check that backup failed - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') - # Not delete this file permanently - os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) - - try: - self.backup_node(backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pg_control was deleted.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Could not open file' in e.message and - 'pg_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Return this file to avoid Postger fail - os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) - - def test_pgpro560_systemid_mismatch(self): - """ - https://jira.postgrespro.ru/browse/PGPRO-560 - make node1 and node2 - feed to backup PGDATA from node1 and PGPORT from node2 - check that backup failed - """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) - - node1.slow_start() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), - set_replication=True, - initdb_params=['--data-checksums']) - - node2.slow_start() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) - - try: - self.backup_node(backup_dir, 'node1', node2, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of SYSTEM ID mismatch.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was ' - 'initialized for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node1', node2, - data_dir=node1.data_dir, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of of SYSTEM ID mismatch.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was initialized ' - 'for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) diff --git a/tests/pgpro589_test.py b/tests/pgpro589_test.py deleted file mode 100644 index 8ce8e1f56..000000000 --- a/tests/pgpro589_test.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess - - -class ArchiveCheck(ProbackupTest, unittest.TestCase): - - def test_pgpro589(self): - """ - https://jira.postgrespro.ru/browse/PGPRO-589 - make node without archive support, make backup which should fail - check that backup status equal to ERROR - check that no files where copied to backup catalogue - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - # make erroneous archive_command - self.set_auto_conf(node, {'archive_command': 'exit 0'}) - node.slow_start() - - node.pgbench_init(scale=5) - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "10"] - ) - pgbench.wait() - pgbench.stdout.close() - path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( - "utf-8") - - try: - self.backup_node( - backup_dir, 'node', node, - options=['--archive-timeout=10']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing archive wal " - "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'ERROR: WAL segment' in e.message and - 'could not be archived in 10 seconds' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] - self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup should have ERROR status') - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', path) - self.assertFalse( - os.path.isfile(file), - "\n Start LSN was not found in archive but datafiles where " - "copied to backup catalogue.\n For example: {0}\n " - "It is not optimal".format(file)) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py deleted file mode 100644 index 6e5786f8c..000000000 --- a/tests/ptrack_test.py +++ /dev/null @@ -1,4407 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -from testgres import QueryException, StartNodeException -import shutil -import sys -from time import sleep -from threading import Thread - - -class PtrackTest(ProbackupTest, unittest.TestCase): - def setUp(self): - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - self.fname = self.id().split('.')[3] - - # @unittest.skip("skip") - def test_drop_rel_during_backup_ptrack(self): - """ - drop relation during ptrack backup - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - absolute_path = os.path.join(node.data_dir, relative_path) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # PTRACK backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - gdb=True, options=['--log-level-file=LOG']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - os.remove(absolute_path) - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( - 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_ptrack_without_full(self): - """ptrack backup without validated full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - try: - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], - "ERROR") - - # @unittest.skip("skip") - def test_ptrack_threads(self): - """ptrack multi thread backup mode""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - # @unittest.skip("skip") - def test_ptrack_stop_pg(self): - """ - create node, take full backup, - restart node, check that ptrack backup - can be taken - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=1) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.stop() - node.slow_start() - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - # @unittest.skip("skip") - def test_ptrack_multi_timeline_backup(self): - """ - t2 /------P2 - t1 ------F---*-----P1 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=5) - - # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) - sleep(15) - - xid = node.safe_psql( - 'postgres', - 'SELECT txid_current()').decode('utf-8').rstrip() - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - - node.cleanup() - - # Restore from full backup to create Timeline 2 - print(self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-xid={0}'.format(xid), - '--recovery-target-action=promote'])) - - node.slow_start() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - - balance = node.safe_psql( - 'postgres', - 'select (select sum(tbalance) from pgbench_tellers) - ' - '( select sum(bbalance) from pgbench_branches) + ' - '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() - - self.assertEqual('0', balance) - - # @unittest.skip("skip") - def test_ptrack_multi_timeline_backup_1(self): - """ - t2 /------ - t1 ---F---P1---* - - # delete P1 - t2 /------P2 - t1 ---F--------* - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=5) - - # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - node.slow_start() - - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # delete old PTRACK backup - self.delete_pb(backup_dir, 'node', backup_id=ptrack_id) - - # take new PTRACK backup - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - - balance = node.safe_psql( - 'postgres', - 'select (select sum(tbalance) from pgbench_tellers) - ' - '( select sum(bbalance) from pgbench_branches) + ' - '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').\ - decode('utf-8').rstrip() - - self.assertEqual('0', balance) - - # @unittest.skip("skip") - def test_ptrack_eat_my_data(self): - """ - PGPRO-4051 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=50) - - self.backup_node(backup_dir, 'node', node) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) - - for i in range(10): - print("Iteration: {0}".format(i)) - - sleep(2) - - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') -# pgdata = self.pgdata_content(node.data_dir) -# -# node_restored.cleanup() -# -# self.restore_node(backup_dir, 'node', node_restored) -# pgdata_restored = self.pgdata_content(node_restored.data_dir) -# -# self.compare_pgdata(pgdata, pgdata_restored) - - pgbench.terminate() - pgbench.wait() - - self.switch_wal_segment(node) - - result = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") - - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - balance = node_restored.safe_psql( - 'postgres', - 'select (select sum(tbalance) from pgbench_tellers) - ' - '( select sum(bbalance) from pgbench_branches) + ' - '( select sum(abalance) from pgbench_accounts ) - ' - '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() - - self.assertEqual('0', balance) - - # Logical comparison - self.assertEqual( - result, - node_restored.safe_psql( - 'postgres', - 'SELECT * FROM pgbench_accounts'), - 'Data loss') - - # @unittest.skip("skip") - def test_ptrack_simple(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i") - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream']) - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # Logical comparison - self.assertEqual( - result, - node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) - - # @unittest.skip("skip") - def test_ptrack_unprivileged(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE DATABASE backupdb") - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 15 - else: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack") - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - node.safe_psql( - "backupdb", - "GRANT USAGE ON SCHEMA ptrack TO backup") - - node.safe_psql( - "backupdb", - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") - - if ProbackupTest.enterprise: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') - - self.backup_node( - backup_dir, 'node', node, - datname='backupdb', options=['--stream', "-U", "backup"]) - - self.backup_node( - backup_dir, 'node', node, datname='backupdb', - backup_type='ptrack', options=['--stream', "-U", "backup"]) - - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_enable(self): - """make ptrack without full backup, should result in error""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'shared_preload_libraries': 'ptrack'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # PTRACK BACKUP - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack disabled.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'ERROR: Ptrack is disabled\n', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_disable(self): - """ - Take full backup, disable ptrack restart postgresql, - enable ptrack, restart postgresql, take ptrack backup - which should fail - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # DISABLE PTRACK - node.safe_psql('postgres', "alter system set ptrack.map_size to 0") - node.stop() - node.slow_start() - - # ENABLE PTRACK - node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") - node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") - node.stop() - node.slow_start() - - # PTRACK BACKUP - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack_enable was set to OFF at some" - " point after previous backup.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'ERROR: LSN from ptrack_control', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd - ) - ) - - # @unittest.skip("skip") - def test_ptrack_uncommitted_xact(self): - """make ptrack backup while there is uncommitted open transaction""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'wal_level': 'replica'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - con = node.connect("postgres") - con.execute( - "create table t_heap as select i" - " as id from generate_series(0,1) i") - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - node_restored.data_dir, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # Physical comparison - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_ptrack_vacuum_full(self): - """make node, make full and ptrack stream backups, - restore them and check data correctness""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i" - " as id from generate_series(0,1000000) i" - ) - - pg_connect = node.connect("postgres", autocommit=True) - - gdb = self.gdb_attach(pg_connect.pid) - gdb.set_breakpoint('reform_and_rewrite_tuple') - - gdb.continue_execution_until_running() - - process = Thread( - target=pg_connect.execute, args=["VACUUM FULL t_heap"]) - process.start() - - while not gdb.stopped_in_breakpoint: - sleep(1) - - gdb.continue_execution_until_break(20) - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - gdb.remove_all_breakpoints() - gdb._execute('detach') - process.join() - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "-T", "{0}={1}".format( - old_tablespace, new_tablespace)] - ) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # @unittest.skip("skip") - def test_ptrack_vacuum_truncate(self): - """make node, create table, take full backup, - delete last 3 pages, vacuum relation, - take ptrack backup, take second ptrack backup, - restore last ptrack backup and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql( - "postgres", - "create sequence t_seq; " - "create table t_heap tablespace somedata as select i as id, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1024) i;") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - "postgres", - "delete from t_heap where ctid >= '(11,0)'") - - node.safe_psql( - "postgres", - "vacuum t_heap") - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - old_tablespace = self.get_tblspace_path(node, 'somedata') - new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - - self.restore_node( - backup_dir, 'node', node_restored, - options=["-j", "4", "-T", "{0}={1}".format( - old_tablespace, new_tablespace)] - ) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, - ignore_ptrack=False - ) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - # @unittest.skip("skip") - def test_ptrack_get_block(self): - """ - make node, make full and ptrack stream backups, - restore them and check data correctness - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i") - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'], - gdb=True) - - gdb.set_breakpoint('make_pagemap_from_ptrack_2') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - gdb.continue_execution_until_exit() - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - # Logical comparison - self.assertEqual( - result, - node.safe_psql("postgres", "SELECT * FROM t_heap")) - - # @unittest.skip("skip") - def test_ptrack_stream(self): - """make node, make full and ptrack stream backups, - restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - node.safe_psql("postgres", "create sequence t_seq") - node.safe_psql( - "postgres", - "create table t_heap as select i as id, nextval('t_seq')" - " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" - " as tsvector from generate_series(0,100) i") - - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, nextval('t_seq') as t_seq," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i") - - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Drop Node - node.cleanup() - - # Restore and check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, options=["-j", "4"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) - node.slow_start() - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Restore and check ptrack backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=ptrack_backup_id, options=["-j", "4"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) - - # @unittest.skip("skip") - def test_ptrack_archive(self): - """make archive node, make full and ptrack backups, - check data correctness in restored instance""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as" - " select i as id," - " md5(i::text) as text," - " md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i") - - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") - full_backup_id = self.backup_node(backup_dir, 'node', node) - full_target_time = self.show_pb( - backup_dir, 'node', full_backup_id)['recovery-time'] - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id," - " md5(i::text) as text," - " md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i") - - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') - ptrack_target_time = self.show_pb( - backup_dir, 'node', ptrack_backup_id)['recovery-time'] - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id," - " md5(i::text) as text," - " md5(i::text)::tsvector as tsvector" - " from generate_series(200, 300) i") - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, - options=[ - "-j", "4", "--recovery-target-action=promote", - "--time={0}".format(full_target_time)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) - node.slow_start() - - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check ptrack backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=ptrack_backup_id, - options=[ - "-j", "4", - "--time={0}".format(ptrack_target_time), - "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) - - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(ptrack_result, ptrack_result_new) - - node.cleanup() - - @unittest.skip("skip") - def test_ptrack_pgpro417(self): - """ - Make node, take full backup, take ptrack backup, - delete ptrack backup. Try to take ptrack backup, - which should fail. Actual only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql( - "postgres", - "SELECT * FROM t_heap") - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=["--stream"]) - - start_lsn_full = self.show_pb( - backup_dir, 'node', backup_id)['start-lsn'] - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - start_lsn_ptrack = self.show_pb( - backup_dir, 'node', backup_id)['start-lsn'] - - self.delete_pb(backup_dir, 'node', backup_id) - - # SECOND PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n" - " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - def test_page_pgpro417(self): - """ - Make archive node, take full backup, take page backup, - delete page backup. Try to take ptrack backup, which should fail. - Actual only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.delete_pb(backup_dir, 'node', backup_id) -# sys.exit(1) - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - - try: - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n " - "Output: {0}\n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - def test_full_pgpro417(self): - """ - Make node, take two full backups, delete full second backup. - Try to take ptrack backup, which should fail. - Relevant only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text," - " md5(i::text)::tsvector as tsvector " - " from generate_series(0,100) i" - ) - node.safe_psql("postgres", "SELECT * FROM t_heap") - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # SECOND FULL BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text," - " md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i" - ) - node.safe_psql("postgres", "SELECT * FROM t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - - self.delete_pb(backup_dir, 'node', backup_id) - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertTrue( - "ERROR: LSN from ptrack_control" in e.message and - "Create new full backup before " - "an incremental one" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_create_db(self): - """ - Make node, take full backup, create database db1, take ptrack backup, - restore database and check it presense - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '10GB'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - node.safe_psql("postgres", "SELECT * FROM t_heap") - self.backup_node( - backup_dir, 'node', node, - options=["--stream"]) - - # CREATE DATABASE DB1 - node.safe_psql("postgres", "create database db1") - node.safe_psql( - "db1", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - # PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, options=["-j", "4"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - # DROP DATABASE DB1 - node.safe_psql( - "postgres", "drop database db1") - # SECOND PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"] - ) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE SECOND PTRACK BACKUP - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, options=["-j", "4"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - node_restored.slow_start() - - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_create_db_on_replica(self): - """ - Make node, take full backup, create replica from it, - take full backup from replica, - create database db1, take ptrack backup from replica, - restore database and check it presense - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - self.restore_node(backup_dir, 'node', replica) - - # Add replica - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(node, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port), - '--stream' - ] - ) - - # CREATE DATABASE DB1 - node.safe_psql("postgres", "create database db1") - node.safe_psql( - "db1", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - # Wait until replica catch up with master - self.wait_until_replica_catch_with_master(node, replica) - replica.safe_psql('postgres', 'checkpoint') - - # PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'replica', - replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port) - ] - ) - - if self.paranoia: - pgdata = self.pgdata_content(replica.data_dir) - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'replica', node_restored, - backup_id=backup_id, options=["-j", "4"]) - - # COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_alter_table_set_tablespace_ptrack(self): - """Make node, create tablespace with table, take full backup, - alter tablespace location, take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - self.create_tblspace_in_node(node, 'somedata') - node.safe_psql( - "postgres", - "create table t_heap tablespace somedata as select i as id," - " md5(i::text) as text, md5(i::text)::tsvector as tsvector" - " from generate_series(0,100) i") - # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # ALTER TABLESPACE - self.create_tblspace_in_node(node, 'somedata_new') - node.safe_psql( - "postgres", - "alter table t_heap set tablespace somedata_new") - - # sys.exit(1) - # PTRACK BACKUP - #result = node.safe_psql( - # "postgres", "select * from t_heap") - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', - options=["--stream"] - ) - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - # node.stop() - # node.cleanup() - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata') - ), - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata_new'), - self.get_tblspace_path(node_restored, 'somedata_new') - ) - ] - ) - - # GET RESTORED PGDATA AND COMPARE - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - node_restored.slow_start() - -# result_new = node_restored.safe_psql( -# "postgres", "select * from t_heap") -# -# self.assertEqual(result, result_new, 'lost some data after restore') - - # @unittest.skip("skip") - def test_alter_database_set_tablespace_ptrack(self): - """Make node, create tablespace with database," - " take full backup, alter tablespace location," - " take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # CREATE TABLESPACE - self.create_tblspace_in_node(node, 'somedata') - - # ALTER DATABASE - node.safe_psql( - "template1", - "alter database postgres set tablespace somedata") - - # PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--stream"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - node.stop() - - # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - self.restore_node( - backup_dir, 'node', - node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - self.get_tblspace_path(node, 'somedata'), - self.get_tblspace_path(node_restored, 'somedata'))]) - - # GET PHYSICAL CONTENT and COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - node_restored.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - node_restored.port = node.port - node_restored.slow_start() - - # @unittest.skip("skip") - def test_drop_tablespace(self): - """ - Make node, create table, alter table tablespace, take ptrack backup, - move table from tablespace, take ptrack backup - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # CREATE TABLE - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - result = node.safe_psql("postgres", "select * from t_heap") - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # Move table to tablespace 'somedata' - node.safe_psql( - "postgres", "alter table t_heap set tablespace somedata") - # PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - # Move table back to default tablespace - node.safe_psql( - "postgres", "alter table t_heap set tablespace pg_default") - # SECOND PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - # DROP TABLESPACE 'somedata' - node.safe_psql( - "postgres", "drop tablespace somedata") - # THIRD PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - if self.paranoia: - pgdata = self.pgdata_content( - node.data_dir, ignore_ptrack=True) - - tblspace = self.get_tblspace_path(node, 'somedata') - node.cleanup() - shutil.rmtree(tblspace, ignore_errors=True) - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=True) - - node.slow_start() - - tblspc_exist = node.safe_psql( - "postgres", - "select exists(select 1 from " - "pg_tablespace where spcname = 'somedata')") - - if tblspc_exist.rstrip() == 't': - self.assertEqual( - 1, 0, - "Expecting Error because " - "tablespace 'somedata' should not be present") - - result_new = node.safe_psql("postgres", "select * from t_heap") - self.assertEqual(result, result_new) - - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_ptrack_alter_tablespace(self): - """ - Make node, create table, alter table tablespace, take ptrack backup, - move table from tablespace, take ptrack backup - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - tblspc_path = self.get_tblspace_path(node, 'somedata') - - # CREATE TABLE - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - result = node.safe_psql("postgres", "select * from t_heap") - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # Move table to separate tablespace - node.safe_psql( - "postgres", - "alter table t_heap set tablespace somedata") - # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from t_heap") - - # FIRTS PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--stream"]) - - # GET PHYSICAL CONTENT FROM NODE - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Restore ptrack backup - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) - restored_node.cleanup() - tblspc_path_new = self.get_tblspace_path( - restored_node, 'somedata_restored') - self.restore_node(backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) - - # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - restored_node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) - restored_node.slow_start() - - # COMPARE LOGICAL CONTENT - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") - self.assertEqual(result, result_new) - - restored_node.cleanup() - shutil.rmtree(tblspc_path_new, ignore_errors=True) - - # Move table to default tablespace - node.safe_psql( - "postgres", "alter table t_heap set tablespace pg_default") - # SECOND PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--stream"]) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - # Restore second ptrack backup and check table consistency - self.restore_node( - backup_dir, 'node', restored_node, - options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) - - # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT - if self.paranoia: - pgdata_restored = self.pgdata_content( - restored_node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) - restored_node.slow_start() - - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") - self.assertEqual(result, result_new) - - # @unittest.skip("skip") - def test_ptrack_multiple_segments(self): - """ - Make node, create table, alter table tablespace, - take ptrack backup, move table from tablespace, take ptrack backup - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'full_page_writes': 'off'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # CREATE TABLE - node.pgbench_init(scale=100, options=['--tablespace=somedata']) - # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # PTRACK STUFF - if node.major_version < 11: - idx_ptrack = {'type': 'heap'} - idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') - idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') - idx_ptrack['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack['path'], idx_ptrack['old_size']) - - pgbench = node.pgbench( - options=['-T', '30', '-c', '1', '--no-vacuum']) - pgbench.wait() - - node.safe_psql("postgres", "checkpoint") - - if node.major_version < 11: - idx_ptrack['new_size'] = self.get_fork_size( - node, - 'pgbench_accounts') - - idx_ptrack['new_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack['path'], - idx_ptrack['new_size']) - - idx_ptrack['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, - idx_ptrack['path']) - - if not self.check_ptrack_sanity(idx_ptrack): - self.assertTrue( - False, 'Ptrack has failed to register changes in data files') - - # GET LOGICAL CONTENT FROM NODE - # it`s stupid, because hint`s are ignored by ptrack - result = node.safe_psql("postgres", "select * from pgbench_accounts") - # FIRTS PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - # GET PHYSICAL CONTENT FROM NODE - pgdata = self.pgdata_content(node.data_dir) - - # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) - restored_node.cleanup() - tblspc_path = self.get_tblspace_path(node, 'somedata') - tblspc_path_new = self.get_tblspace_path( - restored_node, - 'somedata_restored') - - self.restore_node( - backup_dir, 'node', restored_node, - options=[ - "-j", "4", "-T", "{0}={1}".format( - tblspc_path, tblspc_path_new)]) - - # GET PHYSICAL CONTENT FROM NODE_RESTORED - if self.paranoia: - pgdata_restored = self.pgdata_content( - restored_node.data_dir, ignore_ptrack=False) - - # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) - restored_node.slow_start() - - result_new = restored_node.safe_psql( - "postgres", - "select * from pgbench_accounts") - - # COMPARE RESTORED FILES - self.assertEqual(result, result_new, 'data is lost') - - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - @unittest.skip("skip") - def test_atexit_fail(self): - """ - Take backups of every available types and check that PTRACK is clean. - Relevant only for PTRACK 1.x - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_connections': '15'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "-j 30"]) - - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are opening too many connections" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'setting its status to ERROR', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) - - self.assertEqual( - node.safe_psql( - "postgres", - "select * from pg_is_in_backup()").rstrip(), - "f") - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_clean(self): - """ - Take backups of every available types and check that PTRACK is clean - Relevant only for PTRACK 1.x - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, nextval('t_seq') as t_seq, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - options=['-j10', '--stream']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['-j10', '--stream']) - - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['-j10', '--stream']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - @unittest.skip("skip") - def test_ptrack_clean_replica(self): - """ - Take backups of every available types from - master and check that PTRACK on replica is clean. - Relevant only for PTRACK 1.x - """ - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='ptrack', - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='page', - options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_cluster_on_btree(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, nextval('t_seq') as t_seq, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - if node.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'cluster t_heap using t_btree') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_gist(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'cluster t_heap using t_gist') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_btree_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'cluster t_heap using t_btree') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if master.major_version < 11: - self.check_ptrack_map_sanity(replica, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(replica.data_dir) - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - - self.restore_node(backup_dir, 'replica', node) - - pgdata_restored = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_ptrack_cluster_on_gist_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) - - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1') - master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist') - - if master.major_version < 11: - master.safe_psql('postgres', 'CHECKPOINT') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - - if master.major_version < 11: - replica.safe_psql('postgres', 'CHECKPOINT') - self.check_ptrack_map_sanity(replica, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='ptrack', options=['-j10', '--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(replica.data_dir) - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - - self.restore_node(backup_dir, 'replica', node) - - if self.paranoia: - pgdata_restored = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_empty(self): - """Take backups of every available types and check that PTRACK is clean""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector) " - "tablespace somedata") - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - options=['-j10', '--stream']) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'checkpoint') - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - tblspace1 = self.get_tblspace_path(node, 'somedata') - tblspace2 = self.get_tblspace_path(node_restored, 'somedata') - - # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['-j10', '--stream']) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - self.restore_node( - backup_dir, 'node', node_restored, - backup_id=backup_id, - options=[ - "-j", "4", - "-T{0}={1}".format(tblspace1, tblspace2)]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_empty_replica(self): - """ - Take backups of every available types from master - and check that PTRACK on replica is clean - """ - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - replica.slow_start(replica=True) - - # Create table - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "(id int DEFAULT nextval('t_seq'), text text, tsvector tsvector)") - self.wait_until_replica_catch_with_master(master, replica) - - # Take FULL backup - self.backup_node( - backup_dir, - 'replica', - replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - self.wait_until_replica_catch_with_master(master, replica) - - # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='ptrack', - options=[ - '-j1', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - if self.paranoia: - pgdata = self.pgdata_content(replica.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'replica', node_restored, - backup_id=backup_id, options=["-j", "4"]) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_truncate(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - if node.major_version < 11: - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.safe_psql('postgres', 'truncate t_heap') - node.safe_psql('postgres', 'checkpoint') - - if node.major_version < 11: - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - if node.major_version < 11: - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - node.cleanup() - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_basic_ptrack_truncate_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '32MB', - 'archive_timeout': '10s', - 'checkpoint_timeout': '5min'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) ".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - if replica.major_version < 11: - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # Make backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - if replica.major_version < 11: - for i in idx_ptrack: - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'truncate t_heap') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - - if replica.major_version < 10: - replica.safe_psql( - "postgres", - "select pg_xlog_replay_pause()") - else: - replica.safe_psql( - "postgres", - "select pg_wal_replay_pause()") - - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - pgdata = self.pgdata_content(replica.data_dir) - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - - self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) - - pgdata_restored = self.pgdata_content(node.data_dir) - - if self.paranoia: - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node, {'port': node.port}) - - node.slow_start() - - node.safe_psql( - 'postgres', - 'select 1') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # Make full backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - if node.major_version < 11: - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - # Delete some rows, vacuum it and make checkpoint - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - - # @unittest.skip("skip") - def test_ptrack_vacuum_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Make FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - if replica.major_version < 11: - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums for every page of this fork - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['old_size']) - - # Delete some rows, vacuum it and make checkpoint - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if replica.major_version < 11: - self.check_ptrack_map_sanity(master, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(replica.data_dir) - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - - self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_bits_frozen(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - node.safe_psql('postgres', 'vacuum freeze t_heap') - node.safe_psql('postgres', 'checkpoint') - - if node.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - - # @unittest.skip("skip") - def test_ptrack_vacuum_bits_frozen_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Take backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - if replica.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'vacuum freeze t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if replica.major_version < 11: - self.check_ptrack_map_sanity(master, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=['-j10', '--stream']) - - pgdata = self.pgdata_content(replica.data_dir) - replica.cleanup() - - self.restore_node(backup_dir, 'replica', replica) - - pgdata_restored = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_bits_visibility(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - if node.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_full_2(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - pg_options={ 'wal_log_hints': 'on' }) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} " - "using {2}({3}) tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'vacuum t_heap') - node.safe_psql('postgres', 'checkpoint') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - if node.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - node.safe_psql('postgres', 'vacuum full t_heap') - node.safe_psql('postgres', 'checkpoint') - - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - - shutil.rmtree( - self.get_tblspace_path(node, 'somedata'), - ignore_errors=True) - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_full_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " - "tsvector from generate_series(0,256000) i") - - if master.major_version < 11: - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - - if replica.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'delete from t_heap where id%2 = 1') - master.safe_psql('postgres', 'vacuum full t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') - - if replica.major_version < 11: - self.check_ptrack_map_sanity(master, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='ptrack', options=['-j10', '--stream']) - - pgdata = self.pgdata_content(replica.data_dir) - replica.cleanup() - - self.restore_node(backup_dir, 'replica', replica) - - pgdata_restored = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_truncate_2(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # Create table and indexes - res = node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - if node.major_version < 11: - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql('postgres', 'VACUUM t_heap') - - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) - - if node.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - node.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128') - node.safe_psql('postgres', 'VACUUM t_heap') - node.safe_psql('postgres', 'CHECKPOINT') - - # CHECK PTRACK SANITY - if node.major_version < 11: - self.check_ptrack_map_sanity(node, idx_ptrack) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_vacuum_truncate_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - if master.major_version >= 11: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, 'replica', synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector " - "as tsvector from generate_series(0,2560) i") - - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", "create index {0} on {1} " - "using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port) - ] - ) - - if master.major_version < 11: - for i in idx_ptrack: - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # calculate md5sums of pages - idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( - idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - - master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id > 128;') - master.safe_psql('postgres', 'VACUUM t_heap') - master.safe_psql('postgres', 'CHECKPOINT') - - # Sync master and replica - self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'CHECKPOINT') - - # CHECK PTRACK SANITY - if master.major_version < 11: - self.check_ptrack_map_sanity(master, idx_ptrack) - - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '--stream', - '--log-level-file=INFO', - '--archive-timeout=30']) - - pgdata = self.pgdata_content(replica.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'replica', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - @unittest.skip("skip") - def test_ptrack_recovery(self): - """ - Check that ptrack map contain correct bits after recovery. - Actual only for PTRACK 1.x - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - # get size of heap and indexes. size calculated in pages - idx_ptrack[i]['size'] = int(self.get_fork_size(node, i)) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - for i in idx_ptrack: - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack has correct bits after recovery - self.check_ptrack_recovery(idx_ptrack[i]) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_recovery_1(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'shared_buffers': '512MB', - 'max_wal_size': '3GB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select nextval('t_seq')::int as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(0,25600) i") - "from generate_series(0,2560) i") - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "CREATE INDEX {0} ON {1} USING {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - node.safe_psql( - 'postgres', - "create extension pg_buffercache") - - #print(node.safe_psql( - # 'postgres', - # "SELECT count(*) FROM pg_buffercache WHERE isdirty")) - - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_zero_changes(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # Create table - node.safe_psql( - "postgres", - "create table t_heap " - "as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'shared_buffers': '512MB', - 'max_wal_size': '3GB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select nextval('t_seq')::int as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(0,25600) i") - "from generate_series(0,2560) i") - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # Create indexes - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "CREATE INDEX {0} ON {1} USING {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - -# node.safe_psql( -# 'postgres', -# "create extension pg_buffercache") -# -# print(node.safe_psql( -# 'postgres', -# "SELECT count(*) FROM pg_buffercache WHERE isdirty")) - - # kill the bastard - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - # now smack it with sledgehammer - if node.major_version >= 10: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - - self.run_binary( - [ - pg_resetxlog_path, - '-D', - node.data_dir, - '-o 42', - '-f' - ], - asynchronous=False) - - if not node.status(): - node.slow_start() - else: - print("Die! Die! Why won't you die?... Why won't you die?") - exit(1) - - # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, -# backup_type='ptrack', options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control ' in e.message and - 'is greater than Start LSN of previous backup' in e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - -# pgdata = self.pgdata_content(node.data_dir) -# -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) -# node_restored.cleanup() -# -# self.restore_node( -# backup_dir, 'node', node_restored) -# -# pgdata_restored = self.pgdata_content(node_restored.data_dir) -# self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_corrupt_ptrack_map(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - ptrack_version = self.get_ptrack_version(node) - - # Create table - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap " - "as select nextval('t_seq')::int as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - # kill the bastard - if self.verbose: - print('Killing postmaster. Losing Ptrack changes') - - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - ptrack_map = os.path.join(node.data_dir, 'global', 'ptrack.map') - - # Let`s do index corruption. ptrack.map - with open(ptrack_map, "rb+", 0) as f: - f.seek(42) - f.write(b"blablahblahs") - f.flush() - f.close - -# os.remove(os.path.join(node.logs_dir, node.pg_log_name)) - - if self.verbose: - print('Ptrack version:', ptrack_version) - if ptrack_version >= self.version_to_num("2.3"): - node.slow_start() - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'WARNING: ptrack read map: incorrect checksum of file "{0}"'.format(ptrack_map), - log_content) - - node.stop(['-D', node.data_dir]) - else: - try: - node.slow_start() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack.map is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except StartNodeException as e: - self.assertIn( - 'Cannot start node', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), - log_content) - - self.set_auto_conf(node, {'ptrack.map_size': '0'}) - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance ptrack is disabled" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Ptrack is disabled', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - node.stop(['-m', 'immediate', '-D', node.data_dir]) - - self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack map is from future" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: LSN from ptrack_control', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - node.safe_psql( - 'postgres', - "update t_heap set id = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_horizon_lsn_ptrack(self): - """ - https://github.com/postgrespro/pg_probackup/pull/386 - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - self.assertLessEqual( - self.version_to_num(self.old_probackup_version), - self.version_to_num('2.4.15'), - 'You need pg_probackup old_binary =< 2.4.15 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.assertGreaterEqual( - self.get_ptrack_version(node), - self.version_to_num("2.1"), - "You need ptrack >=2.1 for this test") - - # set map_size to a minimal value - self.set_auto_conf(node, {'ptrack.map_size': '1'}) - node.restart() - - node.pgbench_init(scale=100) - - # FULL backup - full_id = self.backup_node(backup_dir, 'node', node, options=['--stream'], old_binary=True) - - # enable archiving so the WAL size to do interfere with data bytes comparison later - self.set_archiving(backup_dir, 'node', node) - node.restart() - - # change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # DELTA is exemplar - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - delta_bytes = self.show_pb(backup_dir, 'node', backup_id=delta_id)["data-bytes"] - self.delete_pb(backup_dir, 'node', backup_id=delta_id) - - # PTRACK with current binary - ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - ptrack_bytes = self.show_pb(backup_dir, 'node', backup_id=ptrack_id)["data-bytes"] - - # make sure that backup size is exactly the same - self.assertEqual(delta_bytes, ptrack_bytes) diff --git a/tests/remote_test.py b/tests/remote_test.py deleted file mode 100644 index 2d36d7346..000000000 --- a/tests/remote_test.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name - - -class RemoteTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_remote_sanity(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - output = self.backup_node( - backup_dir, 'node', node, - options=['--stream'], no_remote=True, return_id=False) - self.assertIn('remote: false', output) - - # try: - # self.backup_node( - # backup_dir, 'node', - # node, options=['--remote-proto=ssh', '--stream'], no_remote=True) - # # we should die here because exception is what we expect to happen - # self.assertEqual( - # 1, 0, - # "Expecting Error because remote-host option is missing." - # "\n Output: {0} \n CMD: {1}".format( - # repr(self.output), self.cmd)) - # except ProbackupException as e: - # self.assertIn( - # "Insert correct error", - # e.message, - # "\n Unexpected Error Message: {0}\n CMD: {1}".format( - # repr(e.message), self.cmd)) diff --git a/tests/replica_test.py b/tests/replica_test.py deleted file mode 100644 index 9c68de366..000000000 --- a/tests/replica_test.py +++ /dev/null @@ -1,1654 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -import time -from distutils.dir_util import copy_tree -from testgres import ProcessType -from time import sleep - - -class ReplicaTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_replica_switchover(self): - """ - check that archiving on replica works correctly - over the course of several switchovers - https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) - - node1.slow_start() - - # take full backup and restore it - self.backup_node(backup_dir, 'node1', node1, options=['--stream']) - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - # create replica - self.restore_node(backup_dir, 'node1', node2) - - # setup replica - self.add_instance(backup_dir, 'node2', node2) - self.set_archiving(backup_dir, 'node2', node2, replica=True) - self.set_replica(node1, node2, synchronous=False) - self.set_auto_conf(node2, {'port': node2.port}) - - node2.slow_start(replica=True) - - # generate some data - node1.pgbench_init(scale=5) - - # take full backup on replica - self.backup_node(backup_dir, 'node2', node2, options=['--stream']) - - # first switchover - node1.stop() - node2.promote() - - self.set_replica(node2, node1, synchronous=False) - node2.reload() - node1.slow_start(replica=True) - - # take incremental backup from new master - self.backup_node( - backup_dir, 'node2', node2, - backup_type='delta', options=['--stream']) - - # second switchover - node2.stop() - node1.promote() - self.set_replica(node1, node2, synchronous=False) - node1.reload() - node2.slow_start(replica=True) - - # generate some more data - node1.pgbench_init(scale=5) - - # take incremental backup from replica - self.backup_node( - backup_dir, 'node2', node2, - backup_type='delta', options=['--stream']) - - # https://github.com/postgrespro/pg_probackup/issues/251 - self.validate_pb(backup_dir) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_replica_stream_ptrack_backup(self): - """ - make node, take full backup, restore it and make replica from it, - take full stream backup from replica - """ - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - if self.pg_config_version > self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - - master.slow_start() - - if master.major_version >= 12: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # CREATE TABLE - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - # take full backup and restore it - self.backup_node(backup_dir, 'master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - self.set_replica(master, replica) - - # Check data correctness on replica - replica.slow_start(replica=True) - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Change data on master, take FULL backup from replica, - # restore taken backup and check that restored data equal - # to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - self.add_instance(backup_dir, 'replica', replica) - - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - - self.set_auto_conf(node, {'port': node.port}) - - node.slow_start() - - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Change data on master, take PTRACK backup from replica, - # restore taken backup and check that restored data equal - # to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") - - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - backup_id = self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE PTRACK BACKUP TAKEN FROM replica - node.cleanup() - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) - - self.set_auto_conf(node, {'port': node.port}) - - node.slow_start() - - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_replica_archive_page_backup(self): - """ - make archive master, take full and page archive backups from master, - set replica, make archive backup from replica - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s', - 'checkpoint_timeout': '30s', - 'max_wal_size': '32MB'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node(backup_dir, 'master', master) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - backup_id = self.backup_node( - backup_dir, 'master', master, backup_type='page') - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - replica.slow_start(replica=True) - - # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - - # Change data on master, take FULL backup from replica, - # restore taken backup and check that restored data - # equal to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,25120) i") - - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - self.wait_until_replica_catch_with_master(master, replica) - - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE FULL BACKUP TAKEN FROM replica - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) - node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) - - self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) - - node.slow_start() - - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") - self.assertEqual(before, after) - node.cleanup() - - # Change data on master, make PAGE backup from replica, - # restore taken backup and check that restored data equal - # to original data - master.pgbench_init(scale=5) - - pgbench = master.pgbench( - options=['-T', '30', '-c', '2', '--no-vacuum']) - - backup_id = self.backup_node( - backup_dir, 'replica', - replica, backup_type='page', - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - - pgbench.wait() - - self.switch_wal_segment(master) - - before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts") - - self.validate_pb(backup_dir, 'replica') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) - - # RESTORE PAGE BACKUP TAKEN FROM replica - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, - backup_id=backup_id) - - self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) - - node.slow_start() - - # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") - self.assertEqual( - before, after, 'Restored data is not equal to original') - - self.add_instance(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # @unittest.skip("skip") - def test_basic_make_replica_via_restore(self): - """ - make archive master, take full and page archive backups from master, - set replica, make archive backup from replica - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node(backup_dir, 'master', master) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,8192) i") - - before = master.safe_psql("postgres", "SELECT * FROM t_heap") - - backup_id = self.backup_node( - backup_dir, 'master', master, backup_type='page') - self.restore_node( - backup_dir, 'master', replica, options=['-R']) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_replica(master, replica, synchronous=True) - - replica.slow_start(replica=True) - - self.backup_node( - backup_dir, 'replica', replica, - options=['--archive-timeout=30s', '--stream']) - - # @unittest.skip("skip") - def test_take_backup_from_delayed_replica(self): - """ - make archive master, take full backups from master, - restore full backup as delayed replica, launch pgbench, - take FULL, PAGE and DELTA backups from replica - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '10s'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node(backup_dir, 'master', master) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,165000) i") - - master.psql( - "postgres", - "create table t_heap_1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,165000) i") - - self.restore_node( - backup_dir, 'master', replica, options=['-R']) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - self.wait_until_replica_catch_with_master(master, replica) - - if self.get_version(master) >= self.version_to_num('12.0'): - self.set_auto_conf( - replica, {'recovery_min_apply_delay': '300s'}) - else: - replica.append_conf( - 'recovery.conf', - 'recovery_min_apply_delay = 300s') - - replica.stop() - replica.slow_start(replica=True) - - master.pgbench_init(scale=10) - - pgbench = master.pgbench( - options=['-T', '60', '-c', '2', '--no-vacuum']) - - self.backup_node( - backup_dir, 'replica', - replica, options=['--archive-timeout=60s']) - - self.backup_node( - backup_dir, 'replica', replica, - data_dir=replica.data_dir, - backup_type='page', options=['--archive-timeout=60s']) - - sleep(1) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--archive-timeout=60s']) - - pgbench.wait() - - pgbench = master.pgbench( - options=['-T', '30', '-c', '2', '--no-vacuum']) - - self.backup_node( - backup_dir, 'replica', replica, - options=['--stream']) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='page', options=['--stream']) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - - pgbench.wait() - - # @unittest.skip("skip") - def test_replica_promote(self): - """ - start backup from replica, during backup promote replica - check that backup is failed - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '10s', - 'checkpoint_timeout': '30s', - 'max_wal_size': '32MB'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.backup_node(backup_dir, 'master', master) - - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,165000) i") - - self.restore_node( - backup_dir, 'master', replica, options=['-R']) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_replica( - master, replica, replica_name='replica', synchronous=True) - - replica.slow_start(replica=True) - - master.psql( - "postgres", - "create table t_heap_1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,165000) i") - - self.wait_until_replica_catch_with_master(master, replica) - - # start backup from replica - gdb = self.backup_node( - backup_dir, 'replica', replica, gdb=True, - options=['--log-level-file=verbose']) - - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - gdb.continue_execution_until_break(20) - - replica.promote() - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - backup_id = self.show_pb( - backup_dir, 'replica')[0]["id"] - - # read log file content - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - f.close - - self.assertIn( - 'ERROR: the standby was promoted during online backup', - log_content) - - self.assertIn( - 'WARNING: Backup {0} is running, ' - 'setting its status to ERROR'.format(backup_id), - log_content) - - # @unittest.skip("skip") - def test_replica_stop_lsn_null_offset(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) - master.slow_start() - - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) - - self.backup_node(backup_dir, 'node', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - - replica.slow_start(replica=True) - - self.switch_wal_segment(master) - self.switch_wal_segment(master) - - output = self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate', - '--stream'], - return_id=False) - - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be streamed in 30 seconds', - output) - - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) - - self.assertIn( - 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', - output) - - self.assertIn( - 'has endpoint 0/4000000 which is ' - 'equal or greater than requested LSN 0/4000000', - output) - - self.assertIn( - 'LOG: Found prior LSN:', - output) - - # Clean after yourself - gdb_checkpointer.kill() - - # @unittest.skip("skip") - def test_replica_stop_lsn_null_offset_next_record(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - - self.backup_node(backup_dir, 'master', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - - replica.slow_start(replica=True) - - self.switch_wal_segment(master) - self.switch_wal_segment(master) - - # open connection to master - conn = master.connect() - - gdb = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=40', - '--log-level-file=LOG', - '--no-validate', - '--stream'], - gdb=True) - - # Attention! this breakpoint is set to a probackup internal function, not a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb.continue_execution_until_running() - - sleep(5) - - conn.execute("create table t1()") - conn.commit() - - while 'RUNNING' in self.show_pb(backup_dir, 'replica')[0]['status']: - sleep(5) - - file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - - with open(file) as f: - log_content = f.read() - - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - log_content) - - self.assertIn( - 'LOG: Looking for segment: 000000010000000000000004', - log_content) - - self.assertIn( - 'LOG: First record in WAL segment "000000010000000000000004": 0/4000028', - log_content) - - self.assertIn( - 'INFO: stop_lsn: 0/4000000', - log_content) - - self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') - - # @unittest.skip("skip") - def test_archive_replica_null_offset(self): - """ - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) - master.slow_start() - - self.backup_node(backup_dir, 'node', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) - - replica.slow_start(replica=True) - - self.switch_wal_segment(master) - self.switch_wal_segment(master) - - # take backup from replica - output = self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate'], - return_id=False) - - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - output) - - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) - - self.assertIn( - 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', - output) - - self.assertIn( - 'has endpoint 0/4000000 which is ' - 'equal or greater than requested LSN 0/4000000', - output) - - self.assertIn( - 'LOG: Found prior LSN:', - output) - - print(output) - - # @unittest.skip("skip") - def test_archive_replica_not_null_offset(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) - master.slow_start() - - self.backup_node(backup_dir, 'node', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'node', replica) - - # Settings for Replica - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - - replica.slow_start(replica=True) - - # take backup from replica - self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate'], - return_id=False) - - try: - self.backup_node( - backup_dir, 'node', replica, replica.data_dir, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of archive timeout. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - # vanilla -- 0/4000060 - # pgproee -- 0/4000078 - self.assertRegex( - e.message, - r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertRegex( - e.message, - r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - 'ERROR: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_replica_toast(self): - """ - make archive master, take full and page archive backups from master, - set replica, make archive backup from replica - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica', - 'shared_buffers': '128MB'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) - master.slow_start() - - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) - - self.backup_node(backup_dir, 'master', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) - - replica.slow_start(replica=True) - - self.switch_wal_segment(master) - self.switch_wal_segment(master) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,10) i') - - self.wait_until_replica_catch_with_master(master, replica) - - output = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate', - '--stream'], - return_id=False) - - pgdata = self.pgdata_content(replica.data_dir) - - self.assertIn( - 'WARNING: Could not read WAL record at', - output) - - self.assertIn( - 'LOG: Found prior LSN:', - output) - - res1 = replica.safe_psql( - 'postgres', - 'select md5(fat_attr) from t1') - - replica.cleanup() - - self.restore_node(backup_dir, 'replica', replica) - pgdata_restored = self.pgdata_content(replica.data_dir) - - replica.slow_start() - - res2 = replica.safe_psql( - 'postgres', - 'select md5(fat_attr) from t1') - - self.assertEqual(res1, res2) - - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - gdb_checkpointer.kill() - - # @unittest.skip("skip") - def test_start_stop_lsn_in_the_same_segno(self): - """ - """ - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica', - 'shared_buffers': '128MB'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - # freeze bgwriter to get rid of RUNNING XACTS records - bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - - replica.slow_start(replica=True) - - self.switch_wal_segment(master) - self.switch_wal_segment(master) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,10) i') - - master.safe_psql( - 'postgres', - 'CHECKPOINT') - - self.wait_until_replica_catch_with_master(master, replica) - - sleep(60) - - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate', - '--stream'], - return_id=False) - - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=30', - '--log-level-console=LOG', - '--no-validate', - '--stream'], - return_id=False) - - @unittest.skip("skip") - def test_replica_promote_1(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '1h', - 'wal_level': 'replica'}) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - # set replica True, so archive_mode 'always' is used. - self.set_archiving(backup_dir, 'master', master, replica=True) - master.slow_start() - - self.backup_node(backup_dir, 'master', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.set_replica(master, replica) - - replica.slow_start(replica=True) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,10) i') - - self.wait_until_replica_catch_with_master(master, replica) - - wal_file = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004') - - wal_file_partial = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004.partial') - - self.assertFalse(os.path.exists(wal_file)) - - replica.promote() - - while not os.path.exists(wal_file_partial): - sleep(1) - - self.switch_wal_segment(master) - - # sleep to be sure, that any partial timeout is expired - sleep(70) - - self.assertTrue( - os.path.exists(wal_file_partial), - "File {0} disappeared".format(wal_file)) - - self.assertTrue( - os.path.exists(wal_file_partial), - "File {0} disappeared".format(wal_file_partial)) - - # @unittest.skip("skip") - def test_replica_promote_2(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - # set replica True, so archive_mode 'always' is used. - self.set_archiving( - backup_dir, 'master', master, replica=True) - master.slow_start() - - self.backup_node(backup_dir, 'master', master) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,1) i') - - self.wait_until_replica_catch_with_master(master, replica) - - replica.promote() - - self.backup_node( - backup_dir, 'master', replica, data_dir=replica.data_dir, - backup_type='page') - - # @unittest.skip("skip") - def test_replica_promote_archive_delta(self): - """ - t3 /---D3--> - t2 /-------> - t1 --F---D1--D2-- - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'archive_timeout': '30s'}) - - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) - self.set_config( - backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node1) - - node1.slow_start() - - self.backup_node(backup_dir, 'node', node1, options=['--stream']) - - # Create replica - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - self.restore_node(backup_dir, 'node', node2, node2.data_dir) - - # Settings for Replica - self.set_replica(node1, node2) - self.set_auto_conf(node2, {'port': node2.port}) - self.set_archiving(backup_dir, 'node', node2, replica=True) - - node2.slow_start(replica=True) - - node1.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(node1, node2) - - node1.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(node1, node2) - - # delta backup on replica on timeline 1 - delta1_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, - 'delta', options=['--stream']) - - # delta backup on replica on timeline 1 - delta2_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, 'delta') - - self.change_backup_status( - backup_dir, 'node', delta2_id, 'ERROR') - - # node2 is now master - node2.promote() - - node2.safe_psql( - 'postgres', - 'CREATE TABLE t3 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - - # node1 is now replica - node1.cleanup() - # kludge "backup_id=delta1_id" - self.restore_node( - backup_dir, 'node', node1, node1.data_dir, - backup_id=delta1_id, - options=[ - '--recovery-target-timeline=2', - '--recovery-target=latest']) - - # Settings for Replica - self.set_replica(node2, node1) - self.set_auto_conf(node1, {'port': node1.port}) - self.set_archiving(backup_dir, 'node', node1, replica=True) - - node1.slow_start(replica=True) - - node2.safe_psql( - 'postgres', - 'CREATE TABLE t4 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,30) i') - self.wait_until_replica_catch_with_master(node2, node1) - - # node1 is back to be a master - node1.promote() - - sleep(5) - - # delta backup on timeline 3 - self.backup_node( - backup_dir, 'node', node1, node1.data_dir, 'delta', - options=['--archive-timeout=60']) - - pgdata = self.pgdata_content(node1.data_dir) - - node1.cleanup() - self.restore_node(backup_dir, 'node', node1, node1.data_dir) - - pgdata_restored = self.pgdata_content(node1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_replica_promote_archive_page(self): - """ - t3 /---P3--> - t2 /-------> - t1 --F---P1--P2-- - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s', - 'archive_timeout': '30s'}) - - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) - self.set_archiving(backup_dir, 'node', node1) - self.set_config( - backup_dir, 'node', options=['--archive-timeout=60s']) - - node1.slow_start() - - self.backup_node(backup_dir, 'node', node1, options=['--stream']) - - # Create replica - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - self.restore_node(backup_dir, 'node', node2, node2.data_dir) - - # Settings for Replica - self.set_replica(node1, node2) - self.set_auto_conf(node2, {'port': node2.port}) - self.set_archiving(backup_dir, 'node', node2, replica=True) - - node2.slow_start(replica=True) - - node1.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(node1, node2) - - node1.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(node1, node2) - - # page backup on replica on timeline 1 - page1_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, - 'page', options=['--stream']) - - # page backup on replica on timeline 1 - page2_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, 'page') - - self.change_backup_status( - backup_dir, 'node', page2_id, 'ERROR') - - # node2 is now master - node2.promote() - - node2.safe_psql( - 'postgres', - 'CREATE TABLE t3 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - - # node1 is now replica - node1.cleanup() - # kludge "backup_id=page1_id" - self.restore_node( - backup_dir, 'node', node1, node1.data_dir, - backup_id=page1_id, - options=[ - '--recovery-target-timeline=2', - '--recovery-target=latest']) - - # Settings for Replica - self.set_replica(node2, node1) - self.set_auto_conf(node1, {'port': node1.port}) - self.set_archiving(backup_dir, 'node', node1, replica=True) - - node1.slow_start(replica=True) - - node2.safe_psql( - 'postgres', - 'CREATE TABLE t4 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,30) i') - self.wait_until_replica_catch_with_master(node2, node1) - - # node1 is back to be a master - node1.promote() - self.switch_wal_segment(node1) - - sleep(5) - - # delta3_id = self.backup_node( - # backup_dir, 'node', node2, node2.data_dir, 'delta') - # page backup on timeline 3 - page3_id = self.backup_node( - backup_dir, 'node', node1, node1.data_dir, 'page', - options=['--archive-timeout=60']) - - pgdata = self.pgdata_content(node1.data_dir) - - node1.cleanup() - self.restore_node(backup_dir, 'node', node1, node1.data_dir) - - pgdata_restored = self.pgdata_content(node1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_parent_choosing(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.add_instance(backup_dir, 'replica', replica) - - full_id = self.backup_node( - backup_dir, 'replica', - replica, options=['--stream']) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - - replica.promote() - - # failing, because without archving, it is impossible to - # take multi-timeline backup. - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - - # @unittest.skip("skip") - def test_instance_from_the_past(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - full_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=10) - self.backup_node(backup_dir, 'node', node, options=['--stream']) - node.cleanup() - - self.restore_node(backup_dir, 'node', node, backup_id=full_id) - node.slow_start() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance is from the past " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Current START LSN' in e.message and - 'is lower than START LSN' in e.message and - 'It may indicate that we are trying to backup ' - 'PostgreSQL instance from the past' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_replica_via_basebackup(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'hot_standby': 'on'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - node.pgbench_init(scale=10) - - #FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=['--recovery-target=latest', '--recovery-target-action=promote']) - node.slow_start() - - # Timeline 2 - # Take stream page backup from instance in timeline2 - self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--log-level-file=verbose']) - - node.cleanup() - - # restore stream backup - self.restore_node(backup_dir, 'node', node) - - xlog_dir = 'pg_wal' - if self.get_version(node) < 100000: - xlog_dir = 'pg_xlog' - - filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") - self.assertTrue( - os.path.exists(filepath), - "History file do not exists: {0}".format(filepath)) - - node.slow_start() - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - pg_basebackup_path = self.get_bin_path('pg_basebackup') - - self.run_binary( - [ - pg_basebackup_path, '-p', str(node.port), '-h', 'localhost', - '-R', '-X', 'stream', '-D', node_restored.data_dir - ]) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - node_restored.slow_start(replica=True) - -# TODO: -# null offset STOP LSN and latest record in previous segment is conrecord (manual only) -# archiving from promoted delayed replica diff --git a/tests/restore_test.py b/tests/restore_test.py deleted file mode 100644 index 2de3ecc0f..000000000 --- a/tests/restore_test.py +++ /dev/null @@ -1,3822 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import subprocess -import sys -from time import sleep -from datetime import datetime, timedelta, timezone -import hashlib -import shutil -import json -from shutil import copyfile -from testgres import QueryException, StartNodeException -from stat import S_ISDIR - - -class RestoreTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_restore_full_to_latest(self): - """recovery to latest from full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - backup_id = self.backup_node(backup_dir, 'node', node) - - node.stop() - node.cleanup() - - # 1 - Test recovery from latest - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # 2 - Test that recovery.conf was created - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - self.assertEqual(os.path.isfile(recovery_conf), True) - - node.slow_start() - - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_full_page_to_latest(self): - """recovery to latest from full + page backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_to_specific_timeline(self): - """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - backup_id = self.backup_node(backup_dir, 'node', node) - - target_tli = int( - node.get_control_data()["Latest checkpoint's TimeLineID"]) - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T', '10', '-c', '2', '--no-vacuum']) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node) - - node.stop() - node.cleanup() - - # Correct Backup must be choosen for restore - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--timeline={0}".format(target_tli)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - recovery_target_timeline = self.get_recovery_conf( - node)["recovery_target_timeline"] - self.assertEqual(int(recovery_target_timeline), target_tli) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_to_time(self): - """recovery to target time""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'TimeZone': 'GMT'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - backup_id = self.backup_node(backup_dir, 'node', node) - - target_time = node.execute( - "postgres", "SELECT to_char(now(), 'YYYY-MM-DD HH24:MI:SS+00')" - )[0][0] - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--time={0}'.format(target_time), - "--recovery-target-action=promote" - ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_to_xid_inclusive(self): - """recovery to target xid""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - with node.connect("postgres") as con: - res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--xid={0}'.format(target_xid), - "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - self.assertEqual( - len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - - # @unittest.skip("skip") - def test_restore_to_xid_not_inclusive(self): - """recovery with target inclusive false""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - with node.connect("postgres") as con: - result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = result[0][0] - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - '--xid={0}'.format(target_xid), - "--inclusive=false", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - self.assertEqual( - len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) - - # @unittest.skip("skip") - def test_restore_to_lsn_inclusive(self): - """recovery to target lsn""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('10.0'): - return - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a int)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - with node.connect("postgres") as con: - con.execute("INSERT INTO tbl0005 VALUES (1)") - con.commit() - res = con.execute("SELECT pg_current_wal_lsn()") - con.commit() - con.execute("INSERT INTO tbl0005 VALUES (2)") - con.commit() - xlogid, xrecoff = res[0][0].split('/') - xrecoff = hex(int(xrecoff, 16) + 1)[2:] - target_lsn = "{0}/{1}".format(xlogid, xrecoff) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--lsn={0}'.format(target_lsn), - "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - self.assertEqual( - len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) - - # @unittest.skip("skip") - def test_restore_to_lsn_not_inclusive(self): - """recovery to target lsn""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('10.0'): - return - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=2) - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a int)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - with node.connect("postgres") as con: - con.execute("INSERT INTO tbl0005 VALUES (1)") - con.commit() - res = con.execute("SELECT pg_current_wal_lsn()") - con.commit() - con.execute("INSERT INTO tbl0005 VALUES (2)") - con.commit() - xlogid, xrecoff = res[0][0].split('/') - xrecoff = hex(int(xrecoff, 16) + 1)[2:] - target_lsn = "{0}/{1}".format(xlogid, xrecoff) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "--inclusive=false", - "-j", "4", '--lsn={0}'.format(target_lsn), - "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - self.assertEqual( - len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - - # @unittest.skip("skip") - def test_restore_full_ptrack_archive(self): - """recovery to latest from archive full+ptrack backups""" - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=2) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="ptrack") - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_ptrack(self): - """recovery to latest from archive full+ptrack+ptrack backups""" - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=2) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="ptrack") - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_full_ptrack_stream(self): - """recovery in stream mode to latest from full + ptrack backups""" - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=2) - - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["--stream"]) - - before = node.execute("postgres", "SELECT * FROM pgbench_branches") - - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") - self.assertEqual(before, after) - - # @unittest.skip("skip") - def test_restore_full_ptrack_under_load(self): - """ - recovery to latest from full + ptrack backups - with loads when ptrack backup do - """ - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.pgbench_init(scale=2) - - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "8"] - ) - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["--stream"]) - - pgbench.wait() - pgbench.stdout.close() - - bbalance = node.execute( - "postgres", "SELECT sum(bbalance) FROM pgbench_branches") - delta = node.execute( - "postgres", "SELECT sum(delta) FROM pgbench_history") - - self.assertEqual(bbalance, delta) - node.stop() - node.cleanup() - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - bbalance = node.execute( - "postgres", "SELECT sum(bbalance) FROM pgbench_branches") - delta = node.execute( - "postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - - # @unittest.skip("skip") - def test_restore_full_under_load_ptrack(self): - """ - recovery to latest from full + page backups - with loads when full backup do - """ - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - # wal_segment_size = self.guc_wal_segment_size(node) - node.pgbench_init(scale=2) - - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "8"] - ) - - self.backup_node(backup_dir, 'node', node) - - pgbench.wait() - pgbench.stdout.close() - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["--stream"]) - - bbalance = node.execute( - "postgres", "SELECT sum(bbalance) FROM pgbench_branches") - delta = node.execute( - "postgres", "SELECT sum(delta) FROM pgbench_history") - - self.assertEqual(bbalance, delta) - - node.stop() - node.cleanup() - # self.wrong_wal_clean(node, wal_segment_size) - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - bbalance = node.execute( - "postgres", "SELECT sum(bbalance) FROM pgbench_branches") - delta = node.execute( - "postgres", "SELECT sum(delta) FROM pgbench_history") - self.assertEqual(bbalance, delta) - - # @unittest.skip("skip") - def test_restore_with_tablespace_mapping_1(self): - """recovery using tablespace-mapping option""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Create tablespace - tblspc_path = os.path.join(node.base_dir, "tblspc") - os.makedirs(tblspc_path) - with node.connect("postgres") as con: - con.connection.autocommit = True - con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) - con.connection.autocommit = False - con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") - con.execute("INSERT INTO test VALUES (1)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - # 1 - Try to restore to existing directory - node.stop() - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore destination is not empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # 2 - Try to restore to existing tablespace directory - tblspc_path_tmp = os.path.join(node.base_dir, "tblspc_tmp") - os.rename(tblspc_path, tblspc_path_tmp) - node.cleanup() - os.rename(tblspc_path_tmp, tblspc_path) - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore tablespace destination is " - "not empty.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # 3 - Restore using tablespace-mapping to not empty directory - tblspc_path_temp = os.path.join(node.base_dir, "tblspc_temp") - os.mkdir(tblspc_path_temp) - with open(os.path.join(tblspc_path_temp, 'file'), 'w+') as f: - f.close() - - try: - self.restore_node( - backup_dir, 'node', node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_temp)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore tablespace destination is " - "not empty.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # 4 - Restore using tablespace-mapping - tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-T", "%s=%s" % (tblspc_path, tblspc_path_new)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - result = node.execute("postgres", "SELECT id FROM test") - self.assertEqual(result[0][0], 1) - - # 4 - Restore using tablespace-mapping using page backup - self.backup_node(backup_dir, 'node', node) - with node.connect("postgres") as con: - con.execute("INSERT INTO test VALUES (2)") - con.commit() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - - show_pb = self.show_pb(backup_dir, 'node') - self.assertEqual(show_pb[1]['status'], "OK") - self.assertEqual(show_pb[2]['status'], "OK") - - node.stop() - node.cleanup() - tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - result = node.execute("postgres", "SELECT id FROM test OFFSET 1") - self.assertEqual(result[0][0], 2) - - # @unittest.skip("skip") - def test_restore_with_tablespace_mapping_2(self): - """recovery using tablespace-mapping option and page backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Full backup - self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - # Create tablespace - tblspc_path = os.path.join(node.base_dir, "tblspc") - os.makedirs(tblspc_path) - with node.connect("postgres") as con: - con.connection.autocommit = True - con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) - con.connection.autocommit = False - con.execute( - "CREATE TABLE tbl AS SELECT * " - "FROM generate_series(0,3) AS integer") - con.commit() - - # First page backup - self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") - - # Create tablespace table - with node.connect("postgres") as con: -# con.connection.autocommit = True -# con.execute("CHECKPOINT") -# con.connection.autocommit = False - con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") - con.execute( - "INSERT INTO tbl1 SELECT * " - "FROM generate_series(0,3) AS integer") - con.commit() - - # Second page backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") - self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") - - node.stop() - node.cleanup() - - tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - count = node.execute("postgres", "SELECT count(*) FROM tbl") - self.assertEqual(count[0][0], 4) - count = node.execute("postgres", "SELECT count(*) FROM tbl1") - self.assertEqual(count[0][0], 4) - - # @unittest.skip("skip") - def test_restore_with_missing_or_corrupted_tablespace_map(self): - """restore backup with missing or corrupted tablespace_map""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Create tablespace - self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=1, tablespace='tblspace') - - # Full backup - self.backup_node(backup_dir, 'node', node) - - # Change some data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Page backup - page_id = self.backup_node(backup_dir, 'node', node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - olddir = self.get_tblspace_path(node, 'tblspace') - newdir = self.get_tblspace_path(node2, 'tblspace') - - # drop tablespace_map - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - page_id, 'database', 'tablespace_map') - - tablespace_map_tmp = os.path.join( - backup_dir, 'backups', 'node', - page_id, 'database', 'tablespace_map_tmp') - - os.rename(tablespace_map, tablespace_map_tmp) - - try: - self.restore_node( - backup_dir, 'node', node2, - options=["-T", "{0}={1}".format(olddir, newdir)]) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Tablespace map is missing: "{0}", ' - 'probably backup {1} is corrupt, validate it'.format( - tablespace_map, page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node(backup_dir, 'node', node2) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Tablespace map is missing: "{0}", ' - 'probably backup {1} is corrupt, validate it'.format( - tablespace_map, page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - copyfile(tablespace_map_tmp, tablespace_map) - - with open(tablespace_map, "a") as f: - f.write("HELLO\n") - - try: - self.restore_node( - backup_dir, 'node', node2, - options=["-T", "{0}={1}".format(olddir, newdir)]) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node(backup_dir, 'node', node2) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # rename it back - os.rename(tablespace_map_tmp, tablespace_map) - - print(self.restore_node( - backup_dir, 'node', node2, - options=["-T", "{0}={1}".format(olddir, newdir)])) - - pgdata_restored = self.pgdata_content(node2.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_archive_node_backup_stream_restore_to_recovery_time(self): - """ - make node with archiving, make stream backup, - make PITR to Recovery Time - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - node.safe_psql("postgres", "create table t_heap(a int)") - - node.stop() - node.cleanup() - - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote" - ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - result = node.psql("postgres", 'select * from t_heap') - self.assertTrue('does not exist' in result[2].decode("utf-8")) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_archive_node_backup_stream_restore_to_recovery_time(self): - """ - make node with archiving, make stream backup, - make PITR to Recovery Time - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - node.safe_psql("postgres", "create table t_heap(a int)") - node.stop() - node.cleanup() - - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote" - ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - result = node.psql("postgres", 'select * from t_heap') - self.assertTrue('does not exist' in result[2].decode("utf-8")) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_archive_node_backup_stream_pitr(self): - """ - make node with archiving, make stream backup, - create table t_heap, make pitr to Recovery Time, - check that t_heap do not exists - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - node.safe_psql("postgres", "create table t_heap(a int)") - node.cleanup() - - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote" - ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - node.slow_start() - - result = node.psql("postgres", 'select * from t_heap') - self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_archive_node_backup_archive_pitr_2(self): - """ - make node with archiving, make archive backup, - create table t_heap, make pitr to Recovery Time, - check that t_heap do not exists - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - node.safe_psql("postgres", "create table t_heap(a int)") - node.stop() - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - if self.paranoia: - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - self.set_auto_conf(node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - result = node_restored.psql("postgres", 'select * from t_heap') - self.assertTrue('does not exist' in result[2].decode("utf-8")) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_archive_restore_to_restore_point(self): - """ - make node with archiving, make archive backup, - create table t_heap, make pitr to Recovery Time, - check that t_heap do not exists - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select generate_series(0,10000)") - result = node.safe_psql( - "postgres", - "select * from t_heap") - node.safe_psql( - "postgres", "select pg_create_restore_point('savepoint')") - node.safe_psql( - "postgres", - "create table t_heap_1 as select generate_series(0,10000)") - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, - options=[ - "--recovery-target-name=savepoint", - "--recovery-target-action=promote"]) - - node.slow_start() - - result_new = node.safe_psql("postgres", "select * from t_heap") - res = node.psql("postgres", "select * from t_heap_1") - self.assertEqual( - res[0], 1, - "Table t_heap_1 should not exist in restored instance") - - self.assertEqual(result, result_new) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_zags_block_corrupt(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - conn = node.connect() - with node.connect("postgres") as conn: - - conn.execute( - "create table tbl(i int)") - conn.commit() - conn.execute( - "create index idx ON tbl (i)") - conn.commit() - conn.execute( - "insert into tbl select i from generate_series(0,400) as i") - conn.commit() - conn.execute( - "select pg_relation_size('idx')") - conn.commit() - conn.execute( - "delete from tbl where i < 100") - conn.commit() - conn.execute( - "explain analyze select i from tbl order by i") - conn.commit() - conn.execute( - "select i from tbl order by i") - conn.commit() - conn.execute( - "create extension pageinspect") - conn.commit() - print(conn.execute( - "select * from bt_page_stats('idx',1)")) - conn.commit() - conn.execute( - "insert into tbl select i from generate_series(0,100) as i") - conn.commit() - conn.execute( - "insert into tbl select i from generate_series(0,100) as i") - conn.commit() - conn.execute( - "insert into tbl select i from generate_series(0,100) as i") - conn.commit() - conn.execute( - "insert into tbl select i from generate_series(0,100) as i") - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - initdb_params=['--data-checksums']) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - self.set_auto_conf( - node_restored, - {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) - - node_restored.slow_start() - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_zags_block_corrupt_1(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={ - 'full_page_writes': 'on'} - ) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql('postgres', 'create table tbl(i int)') - - node.safe_psql('postgres', 'create index idx ON tbl (i)') - - node.safe_psql( - 'postgres', - 'insert into tbl select i from generate_series(0,100000) as i') - - node.safe_psql( - 'postgres', - 'delete from tbl where i%2 = 0') - - node.safe_psql( - 'postgres', - 'explain analyze select i from tbl order by i') - - node.safe_psql( - 'postgres', - 'select i from tbl order by i') - - node.safe_psql( - 'postgres', - 'create extension pageinspect') - - node.safe_psql( - 'postgres', - 'insert into tbl select i from generate_series(0,100) as i') - - node.safe_psql( - 'postgres', - 'insert into tbl select i from generate_series(0,100) as i') - - node.safe_psql( - 'postgres', - 'insert into tbl select i from generate_series(0,100) as i') - - node.safe_psql( - 'postgres', - 'insert into tbl select i from generate_series(0,100) as i') - - self.switch_wal_segment(node) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - initdb_params=['--data-checksums']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - self.set_auto_conf( - node_restored, - {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) - - node_restored.slow_start() - - while True: - with open(node_restored.pg_log_file, 'r') as f: - if 'selected new timeline ID' in f.read(): - break - - # with open(node_restored.pg_log_file, 'r') as f: - # print(f.read()) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - self.compare_pgdata(pgdata, pgdata_restored) - -# pg_xlogdump_path = self.get_bin_path('pg_xlogdump') - -# pg_xlogdump = self.run_binary( -# [ -# pg_xlogdump_path, '-b', -# os.path.join(backup_dir, 'wal', 'node', '000000010000000000000003'), -# ' | ', 'grep', 'Btree', '' -# ], async=False) - - if pg_xlogdump.returncode: - self.assertFalse( - True, - 'Failed to start pg_wal_dump: {0}'.format( - pg_receivexlog.communicate()[1])) - - # @unittest.skip("skip") - def test_restore_chain(self): - """ - make node, take full backup, take several - ERROR delta backups, take valid delta backup, - restore must be successfull - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node( - backup_dir, 'node', node) - - # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], - 'Backup STATUS should be "ERROR"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[3]['status'], - 'Backup STATUS should be "ERROR"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[4]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[5]['status'], - 'Backup STATUS should be "ERROR"') - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - # @unittest.skip("skip") - def test_restore_chain_with_corrupted_backup(self): - """more complex test_restore_chain()""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node( - backup_dir, 'node', node) - - # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - # Take 1 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - # Take 2 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass - - # Take 3 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # Corrupted 4 DELTA - corrupt_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # ORPHAN 5 DELTA - restore_target_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # ORPHAN 6 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # NEXT FULL BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='full') - - # Next Delta - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # do corrupt 6 DELTA backup - file = os.path.join( - backup_dir, 'backups', 'node', - corrupt_id, 'database', 'global', 'pg_control') - - file_new = os.path.join(backup_dir, 'pg_control') - os.rename(file, file_new) - - # RESTORE BACKUP - node.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node, backup_id=restore_target_id) - self.assertEqual( - 1, 0, - "Expecting Error because restore backup is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is orphan'.format(restore_target_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], - 'Backup STATUS should be "ERROR"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[3]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[4]['status'], - 'Backup STATUS should be "ERROR"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[5]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node')[6]['status'], - 'Backup STATUS should be "ERROR"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[7]['status'], - 'Backup STATUS should be "OK"') - - # corruption victim - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node')[8]['status'], - 'Backup STATUS should be "CORRUPT"') - - # orphaned child - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node')[9]['status'], - 'Backup STATUS should be "ORPHAN"') - - # orphaned child - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node')[10]['status'], - 'Backup STATUS should be "ORPHAN"') - - # next FULL - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[11]['status'], - 'Backup STATUS should be "OK"') - - # next DELTA - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[12]['status'], - 'Backup STATUS should be "OK"') - - node.cleanup() - - # Skipped, because backups from the future are invalid. - # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" - # now (PBCKP-259). We can conduct such a test again when we - # untie 'backup_id' from 'start_time' - @unittest.skip("skip") - def test_restore_backup_from_future(self): - """more complex test_restore_chain()""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=5) - # pgbench = node.pgbench(options=['-T', '20', '-c', '2']) - # pgbench.wait() - - # Take PAGE from future - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - with open( - os.path.join( - backup_dir, 'backups', 'node', - backup_id, "backup.control"), "a") as conf: - conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() + timedelta(days=3))) - - # rename directory - new_id = self.show_pb(backup_dir, 'node')[1]['id'] - - os.rename( - os.path.join(backup_dir, 'backups', 'node', backup_id), - os.path.join(backup_dir, 'backups', 'node', new_id)) - - pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) - pgbench.wait() - - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_restore_target_immediate_stream(self): - """ - correct handling of immediate recovery target - for STREAM backups - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # Take delta - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - # restore delta backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--immediate']) - - self.assertTrue( - os.path.isfile(recovery_conf), - "File {0} do not exists".format(recovery_conf)) - - # restore delta backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--recovery-target=immediate']) - - self.assertTrue( - os.path.isfile(recovery_conf), - "File {0} do not exists".format(recovery_conf)) - - # @unittest.skip("skip") - def test_restore_target_immediate_archive(self): - """ - correct handling of immediate recovery target - for ARCHIVE backups - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node( - backup_dir, 'node', node) - - # Take delta - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - # restore page backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--immediate']) - - # For archive backup with immediate recovery target - # recovery.conf is mandatory - with open(recovery_conf, 'r') as f: - self.assertIn("recovery_target = 'immediate'", f.read()) - - # restore page backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--recovery-target=immediate']) - - # For archive backup with immediate recovery target - # recovery.conf is mandatory - with open(recovery_conf, 'r') as f: - self.assertIn("recovery_target = 'immediate'", f.read()) - - # @unittest.skip("skip") - def test_restore_target_latest_archive(self): - """ - make sure that recovery_target 'latest' - is default recovery target - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node) - - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - # restore - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - # hash_1 = hashlib.md5( - # open(recovery_conf, 'rb').read()).hexdigest() - - with open(recovery_conf, 'r') as f: - content_1 = '' - while True: - line = f.readline() - - if not line: - break - if line.startswith("#"): - continue - content_1 += line - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) - - # hash_2 = hashlib.md5( - # open(recovery_conf, 'rb').read()).hexdigest() - - with open(recovery_conf, 'r') as f: - content_2 = '' - while True: - line = f.readline() - - if not line: - break - if line.startswith("#"): - continue - content_2 += line - - self.assertEqual(content_1, content_2) - - # @unittest.skip("skip") - def test_restore_target_new_options(self): - """ - check that new --recovery-target-* - options are working correctly - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node) - - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - node.pgbench_init(scale=2) - pgbench = node.pgbench( - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pgbench.wait() - pgbench.stdout.close() - - node.safe_psql( - "postgres", - "CREATE TABLE tbl0005 (a text)") - - node.safe_psql( - "postgres", "select pg_create_restore_point('savepoint')") - - target_name = 'savepoint' - - # in python-3.6+ it can be ...now()..astimezone()... - target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %z") - with node.connect("postgres") as con: - res = con.execute( - "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - with node.connect("postgres") as con: - con.execute("INSERT INTO tbl0005 VALUES (1)") - con.commit() - if self.get_version(node) > self.version_to_num('10.0'): - res = con.execute("SELECT pg_current_wal_lsn()") - else: - res = con.execute("SELECT pg_current_xlog_location()") - - con.commit() - con.execute("INSERT INTO tbl0005 VALUES (2)") - con.commit() - xlogid, xrecoff = res[0][0].split('/') - xrecoff = hex(int(xrecoff, 16) + 1)[2:] - target_lsn = "{0}/{1}".format(xlogid, xrecoff) - - # Restore with recovery target time - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-time={0}'.format(target_time), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) - - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() - - self.assertIn( - "recovery_target_time = '{0}'".format(target_time), - recovery_conf_content) - - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) - - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) - - node.slow_start() - - # Restore with recovery target xid - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) - - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() - - self.assertIn( - "recovery_target_xid = '{0}'".format(target_xid), - recovery_conf_content) - - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) - - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) - - node.slow_start() - - # Restore with recovery target name - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-name={0}'.format(target_name), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) - - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() - - self.assertIn( - "recovery_target_name = '{0}'".format(target_name), - recovery_conf_content) - - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) - - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) - - node.slow_start() - - # Restore with recovery target lsn - if self.get_version(node) >= 100000: - - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-lsn={0}'.format(target_lsn), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) - - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() - - self.assertIn( - "recovery_target_lsn = '{0}'".format(target_lsn), - recovery_conf_content) - - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) - - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) - - node.slow_start() - - # @unittest.skip("skip") - def test_smart_restore(self): - """ - make node, create database, take full backup, drop database, - take incremental backup and restore it, - make sure that files from dropped database are not - copied during restore - https://github.com/postgrespro/pg_probackup/issues/63 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # create database - node.safe_psql( - "postgres", - "CREATE DATABASE testdb") - - # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) - - # drop database - node.safe_psql( - "postgres", - "DROP DATABASE testdb") - - # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # restore PAGE backup - node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=['--no-validate', '--log-level-file=VERBOSE']) - - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() - - # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) - - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) - - filelist_diff = self.get_backup_filelist_diff( - filelist_full, filelist_page) - - for file in filelist_diff: - self.assertNotIn(file, logfile_content) - - # @unittest.skip("skip") - def test_pg_11_group_access(self): - """ - test group access for PG >= 11 - """ - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=[ - '--data-checksums', - '--allow-group-access']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # take FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - # restore backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node( - backup_dir, 'node', node_restored) - - # compare pgdata permissions - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_restore_concurrent_drop_table(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--compress']) - - # DELTA backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress', '--no-validate'], - gdb=True) - - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - - node.safe_psql( - 'postgres', - 'DROP TABLE pgbench_accounts') - - # do checkpoint to guarantee filenode removal - node.safe_psql( - 'postgres', - 'CHECKPOINT') - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - node.cleanup() - - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_lost_non_data_file(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - os.remove(file) - - node.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because of non-data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'No such file or directory', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Backup files restoring failed', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - def test_partial_restore_exclude(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - pgdata = self.pgdata_content(node.data_dir) - - # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - try: - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node( - backup_dir, 'node', node_restored_1) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored_1) - - db1_path = os.path.join( - node_restored_1.data_dir, 'base', db_list['db1']) - db5_path = os.path.join( - node_restored_1.data_dir, 'base', db_list['db5']) - - self.truncate_every_file_in_dir(db1_path) - self.truncate_every_file_in_dir(db5_path) - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) - node_restored_2.cleanup() - - self.restore_node( - backup_dir, 'node', - node_restored_2, options=[ - "--db-exclude=db1", - "--db-exclude=db5"]) - - pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) - self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) - - node_restored_2.slow_start() - - node_restored_2.safe_psql( - 'postgres', - 'select 1') - - try: - node_restored_2.safe_psql( - 'db1', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node_restored_2.safe_psql( - 'db5', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node_restored_2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_partial_restore_exclude_tablespace(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - cat_version = node.get_control_data()["Catalog version number"] - version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version - - # PG_10_201707211 - # pg_tblspc/33172/PG_9.5_201510051/16386/ - - self.create_tblspace_in_node(node, 'somedata') - - node_tablespace = self.get_tblspace_path(node, 'somedata') - - tbl_oid = node.safe_psql( - 'postgres', - "SELECT oid " - "FROM pg_tablespace " - "WHERE spcname = 'somedata'").decode('utf-8').rstrip() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0} tablespace somedata'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - pgdata = self.pgdata_content(node.data_dir) - - # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') - - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "-T", "{0}={1}".format( - node_tablespace, node1_tablespace)]) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored_1) - - # truncate every db - for db in db_list: - # with exception below - if db in ['db1', 'db5']: - self.truncate_every_file_in_dir( - os.path.join( - node_restored_1.data_dir, 'pg_tblspc', - tbl_oid, version_specific_dir, db_list[db])) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) - node_restored_2.cleanup() - node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') - - self.restore_node( - backup_dir, 'node', - node_restored_2, options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) - - pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) - self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) - - node_restored_2.slow_start() - - node_restored_2.safe_psql( - 'postgres', - 'select 1') - - try: - node_restored_2.safe_psql( - 'db1', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node_restored_2.safe_psql( - 'db5', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node_restored_2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_partial_restore_include(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() - - db_list_splitted = db_list_raw.splitlines() - - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - pgdata = self.pgdata_content(node.data_dir) - - # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - try: - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node( - backup_dir, 'node', node_restored_1) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - self.compare_pgdata(pgdata, pgdata_restored_1) - - # truncate every db - for db in db_list: - # with exception below - if db in ['template0', 'template1', 'postgres', 'db1', 'db5']: - continue - self.truncate_every_file_in_dir( - os.path.join( - node_restored_1.data_dir, 'base', db_list[db])) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) - node_restored_2.cleanup() - - self.restore_node( - backup_dir, 'node', - node_restored_2, options=[ - "--db-include=db1", - "--db-include=db5", - "--db-include=postgres"]) - - pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) - self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) - node_restored_2.slow_start() - - node_restored_2.safe_psql( - 'db1', - 'select 1') - - node_restored_2.safe_psql( - 'db5', - 'select 1') - - node_restored_2.safe_psql( - 'template1', - 'select 1') - - try: - node_restored_2.safe_psql( - 'db2', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - try: - node_restored_2.safe_psql( - 'db10', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - with open(node_restored_2.pg_log_file, 'r') as f: - output = f.read() - - self.assertNotIn('PANIC', output) - - def test_partial_restore_backward_compatibility_1(self): - """ - old binary should be of version < 2.2.0 - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup with old binary, without partial restore support - backup_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', - node_restored, options=[ - "--db-exclude=db5"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup do not support partial restore.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # incremental backup with partial restore support - for i in range(11, 15, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # get db list - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() - db_list_splitted = db_list_raw.splitlines() - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - # get etalon - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - self.truncate_every_file_in_dir( - os.path.join( - node_restored.data_dir, 'base', db_list['db5'])) - self.truncate_every_file_in_dir( - os.path.join( - node_restored.data_dir, 'base', db_list['db14'])) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - # get new node - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-exclude=db5", - "--db-exclude=db14"]) - - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - - self.compare_pgdata(pgdata_restored, pgdata_restored_1) - - def test_partial_restore_backward_compatibility_merge(self): - """ - old binary should be of version < 2.2.0 - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup with old binary, without partial restore support - backup_id = self.backup_node( - backup_dir, 'node', node, - old_binary=True, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', - node_restored, options=[ - "--db-exclude=db5"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup do not support partial restore.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # incremental backup with partial restore support - for i in range(11, 15, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # get db list - db_list_raw = node.safe_psql( - 'postgres', - 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() - db_list_splitted = db_list_raw.splitlines() - db_list = {} - for line in db_list_splitted: - line = json.loads(line) - db_list[line['datname']] = line['oid'] - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - - # get etalon - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - self.truncate_every_file_in_dir( - os.path.join( - node_restored.data_dir, 'base', db_list['db5'])) - self.truncate_every_file_in_dir( - os.path.join( - node_restored.data_dir, 'base', db_list['db14'])) - pgdata_restored = self.pgdata_content(node_restored.data_dir) - - # get new node - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - # merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-exclude=db5", - "--db-exclude=db14"]) - pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - - self.compare_pgdata(pgdata_restored, pgdata_restored_1) - - def test_empty_and_mangled_database_map(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup with database_map - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - # truncate database_map - path = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'database_map') - with open(path, "w") as f: - f.close() - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has empty or mangled database_map, " - "partial restore is impossible".format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has empty or mangled database_map, " - "partial restore is impossible".format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # mangle database_map - with open(path, "w") as f: - f.write("42") - f.close() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # check that simple restore is still possible - self.restore_node( - backup_dir, 'node', node_restored, options=['--no-validate']) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - def test_missing_database_map(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - node.safe_psql( - "postgres", - "CREATE DATABASE backupdb") - - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 15 - else: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - - if self.ptrack: - # TODO why backup works without these grants ? - # 'pg_ptrack_get_pagemapset(pg_lsn)', - # 'pg_ptrack_control_lsn()', - # because PUBLIC - node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack; " - "GRANT USAGE ON SCHEMA ptrack TO backup; " - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - - if ProbackupTest.enterprise: - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") - - # FULL backup without database_map - backup_id = self.backup_node( - backup_dir, 'node', node, datname='backupdb', - options=['--stream', "-U", "backup", '--log-level-file=verbose']) - - pgdata = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - # backup has missing database_map and that is legal - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db5", "--db-exclude=db9"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # check that simple restore is still possible - self.restore_node(backup_dir, 'node', node_restored) - - pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_stream_restore_command_option(self): - """ - correct handling of restore command options - when restoring STREAM backup - - 1. Restore STREAM backup with --restore-command only - parameter, check that PostgreSQL recovery uses - restore_command to obtain WAL from archive. - - 2. Restore STREAM backup wuth --restore-command - as replica, check that PostgreSQL recovery uses - restore_command to obtain WAL from archive. - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'max_wal_size': '32MB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(node.data_dir, 'recovery.conf') - - # Take FULL - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=5) - - node.safe_psql( - 'postgres', - 'create table t1()') - - # restore backup - node.cleanup() - shutil.rmtree(os.path.join(node.logs_dir)) - - restore_cmd = self.get_restore_command(backup_dir, 'node', node) - - self.restore_node( - backup_dir, 'node', node, - options=[ - '--restore-command={0}'.format(restore_cmd)]) - - self.assertTrue( - os.path.isfile(recovery_conf), - "File '{0}' do not exists".format(recovery_conf)) - - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_signal = os.path.join(node.data_dir, 'recovery.signal') - self.assertTrue( - os.path.isfile(recovery_signal), - "File '{0}' do not exists".format(recovery_signal)) - - node.slow_start() - - node.safe_psql( - 'postgres', - 'select * from t1') - - timeline_id = node.safe_psql( - 'postgres', - 'select timeline_id from pg_control_checkpoint()').decode('utf-8').rstrip() - - self.assertEqual('2', timeline_id) - - # @unittest.skip("skip") - def test_restore_primary_conninfo(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=1) - - #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' - - self.restore_node( - backup_dir, 'node', replica, - options=['-R', '--primary-conninfo={0}'.format(str_conninfo)]) - - if self.get_version(node) >= self.version_to_num('12.0'): - standby_signal = os.path.join(replica.data_dir, 'standby.signal') - self.assertTrue( - os.path.isfile(standby_signal), - "File '{0}' do not exists".format(standby_signal)) - - # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(replica.data_dir, 'postgresql.auto.conf') - with open(recovery_conf, 'r') as f: - print(f.read()) - else: - recovery_conf = os.path.join(replica.data_dir, 'recovery.conf') - - with open(os.path.join(replica.data_dir, recovery_conf), 'r') as f: - recovery_conf_content = f.read() - - self.assertIn(str_conninfo, recovery_conf_content) - - # @unittest.skip("skip") - def test_restore_primary_slot_info(self): - """ - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Take FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=1) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - node.safe_psql( - "SELECT pg_create_physical_replication_slot('master_slot')") - - self.restore_node( - backup_dir, 'node', replica, - options=['-R', '--primary-slot-name=master_slot']) - - self.set_auto_conf(replica, {'port': replica.port}) - self.set_auto_conf(replica, {'hot_standby': 'on'}) - - if self.get_version(node) >= self.version_to_num('12.0'): - standby_signal = os.path.join(replica.data_dir, 'standby.signal') - self.assertTrue( - os.path.isfile(standby_signal), - "File '{0}' do not exists".format(standby_signal)) - - replica.slow_start(replica=True) - - def test_issue_249(self): - """ - https://github.com/postgrespro/pg_probackup/issues/249 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - 'postgres', - 'CREATE database db1') - - node.pgbench_init(scale=5) - - node.safe_psql( - 'postgres', - 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') - - node.safe_psql( - 'postgres', - 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') - - node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - 'postgres', - 'INSERT INTO pgbench_accounts SELECT * FROM t1') - - # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) - node_restored_1.cleanup() - - self.restore_node( - backup_dir, 'node', - node_restored_1, options=["--db-include=db1"]) - - self.set_auto_conf( - node_restored_1, - {'port': node_restored_1.port, 'hot_standby': 'off'}) - - node_restored_1.slow_start() - - node_restored_1.safe_psql( - 'db1', - 'select 1') - - try: - node_restored_1.safe_psql( - 'postgres', - 'select 1') - except QueryException as e: - self.assertIn('FATAL', e.message) - - def test_pg_12_probackup_recovery_conf_compatibility(self): - """ - https://github.com/postgrespro/pg_probackup/issues/249 - - pg_probackup version must be 12 or greater - """ - if not self.probackup_old_path: - self.skipTest("You must specify PGPROBACKUPBIN_OLD" - " for run this test") - if self.pg_config_version < self.version_to_num('12.0'): - self.skipTest('You need PostgreSQL >= 12 for this test') - - if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): - self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node, old_binary=True) - - node.pgbench_init(scale=5) - - node.safe_psql( - 'postgres', - 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') - - time = node.safe_psql( - 'SELECT current_timestamp(0)::timestamptz;').decode('utf-8').rstrip() - - node.safe_psql( - 'postgres', - 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') - - node.cleanup() - - self.restore_node( - backup_dir, 'node',node, - options=[ - "--recovery-target-time={0}".format(time), - "--recovery-target-action=promote"], - old_binary=True) - - node.slow_start() - - self.backup_node(backup_dir, 'node', node, old_binary=True) - - node.pgbench_init(scale=5) - - xid = node.safe_psql( - 'SELECT txid_current()').decode('utf-8').rstrip() - node.pgbench_init(scale=1) - - node.cleanup() - - self.restore_node( - backup_dir, 'node',node, - options=[ - "--recovery-target-xid={0}".format(xid), - "--recovery-target-action=promote"]) - - node.slow_start() - - def test_drop_postgresql_auto_conf(self): - """ - https://github.com/postgrespro/pg_probackup/issues/249 - - pg_probackup version must be 12 or greater - """ - - if self.pg_config_version < self.version_to_num('12.0'): - self.skipTest('You need PostgreSQL >= 12 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # drop postgresql.auto.conf - auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") - os.remove(auto_path) - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - node.cleanup() - - self.restore_node( - backup_dir, 'node',node, - options=[ - "--recovery-target=latest", - "--recovery-target-action=promote"]) - - node.slow_start() - - self.assertTrue(os.path.exists(auto_path)) - - def test_truncate_postgresql_auto_conf(self): - """ - https://github.com/postgrespro/pg_probackup/issues/249 - - pg_probackup version must be 12 or greater - """ - - if self.pg_config_version < self.version_to_num('12.0'): - self.skipTest('You need PostgreSQL >= 12 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - # truncate postgresql.auto.conf - auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") - with open(auto_path, "w+") as f: - f.truncate() - - self.backup_node(backup_dir, 'node', node, backup_type='page') - - node.cleanup() - - self.restore_node( - backup_dir, 'node',node, - options=[ - "--recovery-target=latest", - "--recovery-target-action=promote"]) - node.slow_start() - - self.assertTrue(os.path.exists(auto_path)) - - # @unittest.skip("skip") - def test_concurrent_restore(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--compress']) - - pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress', '--no-validate']) - - pgdata1 = self.pgdata_content(node.data_dir) - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node.cleanup() - node_restored.cleanup() - - gdb = self.restore_node( - backup_dir, 'node', node, options=['--no-validate'], gdb=True) - - gdb.set_breakpoint('restore_data_file') - gdb.run_until_break() - - self.restore_node( - backup_dir, 'node', node_restored, options=['--no-validate']) - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - pgdata2 = self.pgdata_content(node.data_dir) - pgdata3 = self.pgdata_content(node_restored.data_dir) - - self.compare_pgdata(pgdata1, pgdata2) - self.compare_pgdata(pgdata2, pgdata3) - - # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 - @unittest.skip("skip") - def test_restore_issue_313(self): - """ - Check that partially restored PostgreSQL instance cannot be started - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - node.cleanup() - - count = 0 - filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) - for file in filelist: - # count only nondata files - if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: - count += 1 - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - - gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) - gdb.verbose = False - gdb.set_breakpoint('restore_non_data_file') - gdb.run_until_break() - gdb.continue_execution_until_break(count - 2) - gdb.quit() - - # emulate the user or HA taking care of PG configuration - for fname in os.listdir(node_restored.data_dir): - if fname.endswith('.conf'): - os.rename( - os.path.join(node_restored.data_dir, fname), - os.path.join(node.data_dir, fname)) - - try: - node.slow_start() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because backup is not fully restored") - except StartNodeException as e: - self.assertIn( - 'Cannot start node', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_restore_with_waldir(self): - """recovery using tablespace-mapping option and page backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - - with node.connect("postgres") as con: - con.execute( - "CREATE TABLE tbl AS SELECT * " - "FROM generate_series(0,3) AS integer") - con.commit() - - # Full backup - backup_id = self.backup_node(backup_dir, 'node', node) - - node.stop() - node.cleanup() - - # Create waldir - waldir_path = os.path.join(node.base_dir, "waldir") - os.makedirs(waldir_path) - - # Test recovery from latest - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-X", "%s" % (waldir_path)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - count = node.execute("postgres", "SELECT count(*) FROM tbl") - self.assertEqual(count[0][0], 4) - - # check pg_wal is symlink - if node.major_version >= 10: - wal_path=os.path.join(node.data_dir, "pg_wal") - else: - wal_path=os.path.join(node.data_dir, "pg_xlog") - - self.assertEqual(os.path.islink(wal_path), True) diff --git a/tests/retention_test.py b/tests/retention_test.py deleted file mode 100644 index 88432a00f..000000000 --- a/tests/retention_test.py +++ /dev/null @@ -1,2529 +0,0 @@ -import os -import unittest -from datetime import datetime, timedelta -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from time import sleep -from distutils.dir_util import copy_tree - - -class RetentionTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_retention_redundancy_1(self): - """purge backups using redundancy-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) - - # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - # Make backups to be keeped - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - output_before = self.show_archive(backup_dir, 'node', tli=1) - - # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - output_after = self.show_archive(backup_dir, 'node', tli=1) - - self.assertEqual( - output_before['max-segno'], - output_after['max-segno']) - - self.assertNotEqual( - output_before['min-segno'], - output_after['min-segno']) - - # Check that WAL segments were deleted - min_wal = output_after['min-segno'] - max_wal = output_after['max-segno'] - - for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): - if not wal_name.endswith(".backup"): - - if self.archive_compress: - wal_name = wal_name[-27:] - wal_name = wal_name[:-3] - else: - wal_name = wal_name[-24:] - - self.assertTrue(wal_name >= min_wal) - self.assertTrue(wal_name <= max_wal) - - # @unittest.skip("skip") - def test_retention_window_2(self): - """purge backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - with open( - os.path.join( - backup_dir, - 'backups', - 'node', - "pg_probackup.conf"), "a") as conf: - conf.write("retention-redundancy = 1\n") - conf.write("retention-window = 1\n") - - # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - # Make backup to be keeped - self.backup_node(backup_dir, 'node', node) - - backups = os.path.join(backup_dir, 'backups', 'node') - days_delta = 5 - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=days_delta))) - days_delta -= 1 - - # Make backup to be keeped - self.backup_node(backup_dir, 'node', node, backup_type="page") - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - # Purge backups - self.delete_expired(backup_dir, 'node', options=['--expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - # @unittest.skip("skip") - def test_retention_window_3(self): - """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - # Take second FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - # Take third FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--retention-window=1', '--expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - # count wal files in ARCHIVE - - # @unittest.skip("skip") - def test_retention_window_4(self): - """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) - - backup_id_2 = self.backup_node(backup_dir, 'node', node) - - backup_id_3 = self.backup_node(backup_dir, 'node', node) - - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_pb(backup_dir, 'node', backup_id_2) - self.delete_pb(backup_dir, 'node', backup_id_3) - - # Purge backups - self.delete_expired( - backup_dir, 'node', - options=['--retention-window=1', '--expired', '--wal']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - # count wal files in ARCHIVE - wals_dir = os.path.join(backup_dir, 'wal', 'node') - # n_wals = len(os.listdir(wals_dir)) - - # self.assertTrue(n_wals > 0) - - # self.delete_expired( - # backup_dir, 'node', - # options=['--retention-window=1', '--expired', '--wal']) - - # count again - n_wals = len(os.listdir(wals_dir)) - self.assertTrue(n_wals == 0) - - # @unittest.skip("skip") - def test_window_expire_interleaved_incremental_chains(self): - """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # FULLb ERROR - # FULLa OK - - # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change FULLb backup status to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa ERROR - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa ERROR - - # Now we start to play with first generation of PAGE backups - # Change PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change PAGEa1 and FULLa to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa2 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGEa2 and FULla to OK - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_expired( - backup_dir, 'node', - options=['--retention-window=1', '--expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - # @unittest.skip("skip") - def test_redundancy_expire_interleaved_incremental_chains(self): - """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULL B backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # FULLb ERROR - # FULLa OK - # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change FULLb backup status to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 and FULLa backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa ERROR - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa ERROR - - # Now we start to play with first generation of PAGE backups - # Change PAGEb1 status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change PAGEa1 status to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa2 and FULLa status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # Change PAGEb1 and FULLb status to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # Change PAGEa2 and FULLa status to OK - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - self.delete_expired( - backup_dir, 'node', - options=['--retention-redundancy=1', '--expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) - - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) - - # @unittest.skip("skip") - def test_window_merge_interleaved_incremental_chains(self): - """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) - - # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # FULLb ERROR - # FULLa OK - - # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change FULLb to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # Now we start to play with first generation of PAGE backups - # Change PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Change PAGEa1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa2 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change PAGEa2 and FULLa to OK - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', - options=['--retention-window=1', '--expired', '--merge-expired']) - - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_a, page_id_a2), - output) - - self.assertIn( - "Rename merged full backup {0} to {1}".format( - backup_id_a, page_id_a2), output) - - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_b, page_id_b2), - output) - - self.assertIn( - "Rename merged full backup {0} to {1}".format( - backup_id_b, page_id_b2), output) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - # @unittest.skip("skip") - def test_window_merge_interleaved_incremental_chains_1(self): - """ - PAGEb3 - PAGEb2 - PAGEb1 - PAGEa1 - FULLb - FULLa - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=5) - - # Take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - backup_id_b = self.backup_node(backup_dir, 'node', node) - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - # Change FULL B backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgdata_a1 = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - # Change FULL B backup status to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - pgdata_b3 = self.pgdata_content(node.data_dir) - - pgbench = node.pgbench(options=['-t', '20', '-c', '1']) - pgbench.wait() - - # PAGEb3 OK - # PAGEb2 OK - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # Change PAGEa1 backup status to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # PAGEb3 OK - # PAGEb2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a1, page_id_b3, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_expired( - backup_dir, 'node', - options=['--retention-window=1', '--expired', '--merge-expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], - page_id_b3) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], - page_id_a1) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], - 'FULL') - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], - 'FULL') - - node.cleanup() - - # Data correctness of PAGEa3 - self.restore_node(backup_dir, 'node', node, backup_id=page_id_a1) - pgdata_restored_a1 = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata_a1, pgdata_restored_a1) - - node.cleanup() - - # Data correctness of PAGEb3 - self.restore_node(backup_dir, 'node', node, backup_id=page_id_b3) - pgdata_restored_b3 = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata_b3, pgdata_restored_b3) - - # @unittest.skip("skip") - def test_basic_window_merge_multiple_descendants(self): - """ - PAGEb3 - | PAGEa3 - -----------------------------retention window - PAGEb2 / - | PAGEa2 / should be deleted - PAGEb1 \ / - | PAGEa1 - FULLb | - FULLa - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - backup_id_b = self.backup_node(backup_dir, 'node', node) - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change FULLb to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change PAGEa1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # Change PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa2 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - # Change PAGEb2 and PAGEb1 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - - # and FULL stuff - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEa3 OK - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa3 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - - # Change PAGEb2, PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb3 OK - # PAGEa3 ERROR - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Change PAGEa3, PAGEa2 and PAGEb1 status to OK - self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - - # PAGEb3 OK - # PAGEa3 OK - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], - page_id_a1) - - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], - page_id_a1) - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', '--delete-expired', - '--merge-expired', '--log-level-console=log']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - # Merging chain A - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_a, page_id_a3), - output) - - self.assertIn( - "INFO: Rename merged full backup {0} to {1}".format( - backup_id_a, page_id_a3), output) - -# self.assertIn( -# "WARNING: Backup {0} has multiple valid descendants. " -# "Automatic merge is not possible.".format( -# page_id_a1), output) - - self.assertIn( - "LOG: Consider backup {0} for purge".format( - page_id_a2), output) - - # Merge chain B - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_b, page_id_b3), - output) - - self.assertIn( - "INFO: Rename merged full backup {0} to {1}".format( - backup_id_b, page_id_b3), output) - - self.assertIn( - "Delete: {0}".format(page_id_a2), output) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], - page_id_b3) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], - page_id_a3) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], - 'FULL') - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], - 'FULL') - - # @unittest.skip("skip") - def test_basic_window_merge_multiple_descendants_1(self): - """ - PAGEb3 - | PAGEa3 - -----------------------------retention window - PAGEb2 / - | PAGEa2 / - PAGEb1 \ / - | PAGEa1 - FULLb | - FULLa - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - backup_id_b = self.backup_node(backup_dir, 'node', node) - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change FULLb to OK - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa1 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a1, 'ERROR') - - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb1 OK - # PAGEa1 ERROR - # FULLb OK - # FULLa OK - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # Change PAGEa1 to OK - self.change_backup_status(backup_dir, 'node', page_id_a1, 'OK') - - # Change PAGEb1 and FULLb to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEa2 OK - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - # Change PAGEa2 and FULLa to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a2, 'ERROR') - self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') - - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa ERROR - - # Change PAGEb2 and PAGEb1 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_b2, 'ERROR') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'ERROR') - - # and FULL stuff - self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - # pgbench.wait() - - # PAGEa3 OK - # PAGEb2 ERROR - # PAGEa2 ERROR - # PAGEb1 ERROR - # PAGEa1 OK - # FULLb ERROR - # FULLa OK - - # Change PAGEa3 to ERROR - self.change_backup_status(backup_dir, 'node', page_id_a3, 'ERROR') - - # Change PAGEb2, PAGEb1 and FULLb to OK - self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGEb3 OK - # PAGEa3 ERROR - # PAGEb2 OK - # PAGEa2 ERROR - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Change PAGEa3, PAGEa2 and PAGEb1 status to OK - self.change_backup_status(backup_dir, 'node', page_id_a3, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') - self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') - - # PAGEb3 OK - # PAGEa3 OK - # PAGEb2 OK - # PAGEa2 OK - # PAGEb1 OK - # PAGEa1 OK - # FULLb OK - # FULLa OK - - # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], - page_id_a1) - - self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], - page_id_a1) - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', - '--merge-expired', '--log-level-console=log']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) - - # Merging chain A - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_a, page_id_a3), - output) - - self.assertIn( - "INFO: Rename merged full backup {0} to {1}".format( - backup_id_a, page_id_a3), output) - -# self.assertIn( -# "WARNING: Backup {0} has multiple valid descendants. " -# "Automatic merge is not possible.".format( -# page_id_a1), output) - - # Merge chain B - self.assertIn( - "Merge incremental chain between full backup {0} and backup {1}".format( - backup_id_b, page_id_b3), output) - - self.assertIn( - "INFO: Rename merged full backup {0} to {1}".format( - backup_id_b, page_id_b3), output) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['id'], - page_id_b3) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], - page_id_a3) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], - page_id_a2) - - self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['backup-mode'], - 'FULL') - - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], - 'FULL') - - self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], - 'PAGE') - - output = self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', - '--delete-expired', '--log-level-console=log']) - - # @unittest.skip("skip") - def test_window_chains(self): - """ - PAGE - -------window - PAGE - PAGE - FULL - PAGE - PAGE - FULL - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # Chain A - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Chain B - self.backup_node(backup_dir, 'node', node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - pgdata = self.pgdata_content(node.data_dir) - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_b3, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', '--expired', - '--merge-expired', '--log-level-console=log']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) - - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - def test_window_chains_1(self): - """ - PAGE - -------window - PAGE - PAGE - FULL - PAGE - PAGE - FULL - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - # Chain A - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Chain B - self.backup_node(backup_dir, 'node', node) - - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.pgdata_content(node.data_dir) - - # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_b3, 'pg_probackup.conf']: - continue - - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', - '--merge-expired', '--log-level-console=log']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - self.assertIn( - "There are no backups to delete by retention policy", - output) - - self.assertIn( - "Retention merging finished", - output) - - output = self.delete_expired( - backup_dir, 'node', - options=[ - '--retention-window=1', - '--expired', '--log-level-console=log']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) - - self.assertIn( - "There are no backups to merge by retention policy", - output) - - self.assertIn( - "Purging finished", - output) - - @unittest.skip("skip") - def test_window_error_backups(self): - """ - PAGE ERROR - -------window - PAGE ERROR - PAGE ERROR - PAGE ERROR - FULL ERROR - FULL - -------redundancy - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change FULLb backup status to ERROR - # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # @unittest.skip("skip") - def test_window_error_backups_1(self): - """ - DELTA - PAGE ERROR - FULL - -------window - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - # Take PAGE BACKUP - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) - - # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - self.show_pb(backup_dir, 'node')[1]['id'] - - # Take DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--retention-window=2', '--delete-expired']) - - # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - # @unittest.skip("skip") - def test_window_error_backups_2(self): - """ - DELTA - PAGE ERROR - FULL - -------window - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) - - # Take PAGE BACKUP - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) - - # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one - gdb.set_breakpoint('pg_stop_backup') - gdb.run_until_break() - gdb._execute('signal SIGKILL') - gdb.continue_execution_until_error() - - self.show_pb(backup_dir, 'node')[1]['id'] - - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'SELECT pg_catalog.pg_stop_backup()') - - # Take DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--retention-window=2', '--delete-expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) - - def test_retention_redundancy_overlapping_chains(self): - """""" - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) - - # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Make backups to be keeped - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - sleep(1) - - self.backup_node(backup_dir, 'node', node, backup_type="page") - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - self.validate_pb(backup_dir, 'node') - - def test_retention_redundancy_overlapping_chains_1(self): - """""" - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) - - # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Make backups to be keeped - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - sleep(1) - - self.backup_node(backup_dir, 'node', node, backup_type="page") - - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() - - self.backup_node(backup_dir, 'node', node, backup_type="page") - - # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - self.validate_pb(backup_dir, 'node') - - def test_wal_purge_victim(self): - """ - https://github.com/postgrespro/pg_probackup/issues/103 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # Make ERROR incremental backup - try: - self.backup_node(backup_dir, 'node', node, backup_type='page') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - page_id = self.show_pb(backup_dir, 'node')[0]['id'] - - sleep(1) - - # Make FULL backup - full_id = self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) - - try: - self.validate_pb(backup_dir, 'node') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "INFO: Backup {0} WAL segments are valid".format(full_id), - e.message) - self.assertIn( - "WARNING: Backup {0} has missing parent 0".format(page_id), - e.message) - - # @unittest.skip("skip") - def test_failed_merge_redundancy_retention(self): - """ - Check that retention purge works correctly with MERGING backups - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join( - self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL1 backup - full_id = self.backup_node(backup_dir, 'node', node) - - # DELTA BACKUP - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # FULL2 backup - self.backup_node(backup_dir, 'node', node) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # FULL3 backup - self.backup_node(backup_dir, 'node', node) - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=2']) - - self.set_config( - backup_dir, 'node', options=['--retention-window=2']) - - # create pair of MERGING backup as a result of failed merge - gdb = self.merge_backup( - backup_dir, 'node', delta_id, gdb=True) - gdb.set_breakpoint('backup_non_data_file') - gdb.run_until_break() - gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') - - # "expire" first full backup - backups = os.path.join(backup_dir, 'backups', 'node') - with open( - os.path.join( - backups, full_id, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - # run retention merge - self.delete_expired( - backup_dir, 'node', options=['--delete-expired']) - - self.assertEqual( - 'MERGING', - self.show_pb(backup_dir, 'node', full_id)['status'], - 'Backup STATUS should be "MERGING"') - - self.assertEqual( - 'MERGING', - self.show_pb(backup_dir, 'node', delta_id)['status'], - 'Backup STATUS should be "MERGING"') - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) - - def test_wal_depth_1(self): - """ - |-------------B5----------> WAL timeline3 - |-----|-------------------------> WAL timeline2 - B1 B2---| B3 B4-------B6-----> WAL timeline1 - - wal-depth=2 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s', - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - - node.slow_start() - - # FULL - node.pgbench_init(scale=1) - self.backup_node(backup_dir, 'node', node) - - # PAGE - node.pgbench_init(scale=1) - B2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # generate_some more data - node.pgbench_init(scale=1) - - target_xid = node.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - - node.pgbench_init(scale=1) - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.pgbench_init(scale=1) - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Timeline 2 - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - - node_restored.cleanup() - - output = self.restore_node( - backup_dir, 'node', node_restored, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-action=promote']) - - self.assertIn( - 'Restore of backup {0} completed'.format(B2), - output) - - self.set_auto_conf(node_restored, options={'port': node_restored.port}) - - node_restored.slow_start() - - node_restored.pgbench_init(scale=1) - - target_xid = node_restored.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - - node_restored.pgbench_init(scale=2) - - # Timeline 3 - node_restored.cleanup() - - output = self.restore_node( - backup_dir, 'node', node_restored, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - self.assertIn( - 'Restore of backup {0} completed'.format(B2), - output) - - self.set_auto_conf(node_restored, options={'port': node_restored.port}) - - node_restored.slow_start() - - node_restored.pgbench_init(scale=1) - self.backup_node( - backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) - - node.pgbench_init(scale=1) - self.backup_node(backup_dir, 'node', node) - - lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] - - self.validate_pb( - backup_dir, 'node', backup_id=B2, - options=['--recovery-target-lsn={0}'.format(lsn)]) - - self.validate_pb(backup_dir, 'node') - - def test_wal_purge(self): - """ - -------------------------------------> tli5 - ---------------------------B6--------> tli4 - S2`---------------> tli3 - S1`------------S2---B4-------B5--> tli2 - B1---S1-------------B2--------B3------> tli1 - - B* - backups - S* - switchpoints - - Expected result: - TLI5 will be purged entirely - B6--------> tli4 - S2`---------------> tli3 - S1`------------S2---B4-------B5--> tli2 - B1---S1-------------B2--------B3------> tli1 - - wal-depth=2 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - - node.slow_start() - - # STREAM FULL - stream_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.stop() - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - B1 = self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=1) - - target_xid = node.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - node.pgbench_init(scale=5) - - # B2 FULL on TLI1 - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=4) - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=4) - - self.delete_pb(backup_dir, 'node', options=['--delete-wal']) - - # TLI 2 - node_tli2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) - node_tli2.cleanup() - - output = self.restore_node( - backup_dir, 'node', node_tli2, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1', - '--recovery-target-action=promote']) - - self.assertIn( - 'INFO: Restore of backup {0} completed'.format(B1), - output) - - self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) - node_tli2.slow_start() - node_tli2.pgbench_init(scale=4) - - target_xid = node_tli2.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - node_tli2.pgbench_init(scale=1) - - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) - node_tli2.pgbench_init(scale=3) - - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) - node_tli2.pgbench_init(scale=1) - node_tli2.cleanup() - - # TLI3 - node_tli3 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) - node_tli3.cleanup() - - # Note, that successful validation here is a happy coincidence - output = self.restore_node( - backup_dir, 'node', node_tli3, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - self.assertIn( - 'INFO: Restore of backup {0} completed'.format(B1), - output) - self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) - node_tli3.slow_start() - node_tli3.pgbench_init(scale=5) - node_tli3.cleanup() - - # TLI4 - node_tli4 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) - node_tli4.cleanup() - - self.restore_node( - backup_dir, 'node', node_tli4, backup_id=stream_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) - self.set_archiving(backup_dir, 'node', node_tli4) - node_tli4.slow_start() - - node_tli4.pgbench_init(scale=5) - - self.backup_node( - backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) - node_tli4.pgbench_init(scale=5) - node_tli4.cleanup() - - # TLI5 - node_tli5 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) - node_tli5.cleanup() - - self.restore_node( - backup_dir, 'node', node_tli5, backup_id=stream_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) - self.set_archiving(backup_dir, 'node', node_tli5) - node_tli5.slow_start() - node_tli5.pgbench_init(scale=10) - - # delete '.history' file of TLI4 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) - # delete '.history' file of TLI5 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) - - output = self.delete_pb( - backup_dir, 'node', - options=[ - '--delete-wal', '--dry-run', - '--log-level-console=verbose']) - - self.assertIn( - 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' - 'and 000000040000000000000006 can be removed', - output) - - self.assertIn( - 'INFO: On timeline 5 all files can be removed', - output) - - show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) - - self.assertTrue(show_tli1_before) - self.assertTrue(show_tli2_before) - self.assertTrue(show_tli3_before) - self.assertTrue(show_tli4_before) - self.assertTrue(show_tli5_before) - - output = self.delete_pb( - backup_dir, 'node', - options=['--delete-wal', '--log-level-console=verbose']) - - self.assertIn( - 'INFO: On timeline 4 WAL segments between 000000040000000000000002 ' - 'and 000000040000000000000006 will be removed', - output) - - self.assertIn( - 'INFO: On timeline 5 all files will be removed', - output) - - show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) - - self.assertEqual(show_tli1_before, show_tli1_after) - self.assertEqual(show_tli2_before, show_tli2_after) - self.assertEqual(show_tli3_before, show_tli3_after) - self.assertNotEqual(show_tli4_before, show_tli4_after) - self.assertNotEqual(show_tli5_before, show_tli5_after) - - self.assertEqual( - show_tli4_before['min-segno'], - '000000040000000000000002') - - self.assertEqual( - show_tli4_after['min-segno'], - '000000040000000000000006') - - self.assertFalse(show_tli5_after) - - self.validate_pb(backup_dir, 'node') - - def test_wal_depth_2(self): - """ - -------------------------------------> tli5 - ---------------------------B6--------> tli4 - S2`---------------> tli3 - S1`------------S2---B4-------B5--> tli2 - B1---S1-------------B2--------B3------> tli1 - - B* - backups - S* - switchpoints - wal-depth=2 - - Expected result: - TLI5 will be purged entirely - B6--------> tli4 - S2`---------------> tli3 - S1`------------S2 B4-------B5--> tli2 - B1---S1 B2--------B3------> tli1 - - wal-depth=2 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - - node.slow_start() - - # STREAM FULL - stream_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.stop() - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - B1 = self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=1) - - target_xid = node.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - node.pgbench_init(scale=5) - - # B2 FULL on TLI1 - B2 = self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=4) - self.backup_node(backup_dir, 'node', node) - node.pgbench_init(scale=4) - - # TLI 2 - node_tli2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) - node_tli2.cleanup() - - output = self.restore_node( - backup_dir, 'node', node_tli2, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1', - '--recovery-target-action=promote']) - - self.assertIn( - 'INFO: Restore of backup {0} completed'.format(B1), - output) - - self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) - node_tli2.slow_start() - node_tli2.pgbench_init(scale=4) - - target_xid = node_tli2.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - node_tli2.pgbench_init(scale=1) - - B4 = self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) - node_tli2.pgbench_init(scale=3) - - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) - node_tli2.pgbench_init(scale=1) - node_tli2.cleanup() - - # TLI3 - node_tli3 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) - node_tli3.cleanup() - - # Note, that successful validation here is a happy coincidence - output = self.restore_node( - backup_dir, 'node', node_tli3, - options=[ - '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=2', - '--recovery-target-action=promote']) - - self.assertIn( - 'INFO: Restore of backup {0} completed'.format(B1), - output) - self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) - node_tli3.slow_start() - node_tli3.pgbench_init(scale=5) - node_tli3.cleanup() - - # TLI4 - node_tli4 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) - node_tli4.cleanup() - - self.restore_node( - backup_dir, 'node', node_tli4, backup_id=stream_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) - self.set_archiving(backup_dir, 'node', node_tli4) - node_tli4.slow_start() - - node_tli4.pgbench_init(scale=5) - - self.backup_node( - backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) - node_tli4.pgbench_init(scale=5) - node_tli4.cleanup() - - # TLI5 - node_tli5 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) - node_tli5.cleanup() - - self.restore_node( - backup_dir, 'node', node_tli5, backup_id=stream_id, - options=[ - '--recovery-target=immediate', - '--recovery-target-action=promote']) - - self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) - self.set_archiving(backup_dir, 'node', node_tli5) - node_tli5.slow_start() - node_tli5.pgbench_init(scale=10) - - # delete '.history' file of TLI4 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) - # delete '.history' file of TLI5 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) - - output = self.delete_pb( - backup_dir, 'node', - options=[ - '--delete-wal', '--dry-run', - '--wal-depth=2', '--log-level-console=verbose']) - - start_lsn_B2 = self.show_pb(backup_dir, 'node', B2)['start-lsn'] - self.assertIn( - 'On timeline 1 WAL is protected from purge at {0}'.format(start_lsn_B2), - output) - - self.assertIn( - 'LOG: Archive backup {0} to stay consistent protect from ' - 'purge WAL interval between 000000010000000000000004 ' - 'and 000000010000000000000005 on timeline 1'.format(B1), output) - - start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] - self.assertIn( - 'On timeline 2 WAL is protected from purge at {0}'.format(start_lsn_B4), - output) - - self.assertIn( - 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' - 'from purge WAL interval between 000000020000000000000006 and ' - '000000020000000000000009 on timeline 2', output) - - self.assertIn( - 'LOG: Timeline 3 to stay reachable from timeline 1 protect ' - 'from purge WAL interval between 000000010000000000000004 and ' - '000000010000000000000006 on timeline 1', output) - - show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) - - self.assertTrue(show_tli1_before) - self.assertTrue(show_tli2_before) - self.assertTrue(show_tli3_before) - self.assertTrue(show_tli4_before) - self.assertTrue(show_tli5_before) - - sleep(5) - - output = self.delete_pb( - backup_dir, 'node', - options=['--delete-wal', '--wal-depth=2', '--log-level-console=verbose']) - -# print(output) - - show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) - - self.assertNotEqual(show_tli1_before, show_tli1_after) - self.assertNotEqual(show_tli2_before, show_tli2_after) - self.assertEqual(show_tli3_before, show_tli3_after) - self.assertNotEqual(show_tli4_before, show_tli4_after) - self.assertNotEqual(show_tli5_before, show_tli5_after) - - self.assertEqual( - show_tli4_before['min-segno'], - '000000040000000000000002') - - self.assertEqual( - show_tli4_after['min-segno'], - '000000040000000000000006') - - self.assertFalse(show_tli5_after) - - self.assertTrue(show_tli1_after['lost-segments']) - self.assertTrue(show_tli2_after['lost-segments']) - self.assertFalse(show_tli3_after['lost-segments']) - self.assertFalse(show_tli4_after['lost-segments']) - self.assertFalse(show_tli5_after) - - self.assertEqual(len(show_tli1_after['lost-segments']), 1) - self.assertEqual(len(show_tli2_after['lost-segments']), 1) - - self.assertEqual( - show_tli1_after['lost-segments'][0]['begin-segno'], - '000000010000000000000007') - - self.assertEqual( - show_tli1_after['lost-segments'][0]['end-segno'], - '00000001000000000000000A') - - self.assertEqual( - show_tli2_after['lost-segments'][0]['begin-segno'], - '00000002000000000000000A') - - self.assertEqual( - show_tli2_after['lost-segments'][0]['end-segno'], - '00000002000000000000000A') - - self.validate_pb(backup_dir, 'node') - - def test_basic_wal_depth(self): - """ - B1---B1----B3-----B4----B5------> tli1 - - Expected result with wal-depth=1: - B1 B1 B3 B4 B5------> tli1 - - wal-depth=1 - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - node.pgbench_init(scale=1) - B1 = self.backup_node(backup_dir, 'node', node) - - - # B2 - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - B2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # B3 - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - B3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # B4 - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - B4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # B5 - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - B5 = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--wal-depth=1', '--delete-wal']) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - target_xid = node.safe_psql( - "postgres", - "select txid_current()").decode('utf-8').rstrip() - - self.switch_wal_segment(node) - - pgbench = node.pgbench(options=['-T', '10', '-c', '2']) - pgbench.wait() - - tli1 = self.show_archive(backup_dir, 'node', tli=1) - - # check that there are 4 lost_segments intervals - self.assertEqual(len(tli1['lost-segments']), 4) - - output = self.validate_pb( - backup_dir, 'node', B5, - options=['--recovery-target-xid={0}'.format(target_xid)]) - - print(output) - - self.assertIn( - 'INFO: Backup validation completed successfully on time', - output) - - self.assertIn( - 'xid {0} and LSN'.format(target_xid), - output) - - for backup_id in [B1, B2, B3, B4]: - try: - self.validate_pb( - backup_dir, 'node', backup_id, - options=['--recovery-target-xid={0}'.format(target_xid)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Not enough WAL records to xid {0}".format(target_xid), - e.message) - - self.validate_pb(backup_dir, 'node') - - def test_concurrent_running_full_backup(self): - """ - https://github.com/postgrespro/pg_probackup/issues/328 - """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - gdb.kill() - - self.assertTrue( - self.show_pb(backup_dir, 'node')[0]['status'], - 'RUNNING') - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--retention-redundancy=2', '--delete-expired']) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'], - 'RUNNING') - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - gdb.kill() - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - gdb.kill() - - self.backup_node(backup_dir, 'node', node) - - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) - gdb.set_breakpoint('backup_data_file') - gdb.run_until_break() - gdb.kill() - - self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--retention-redundancy=2', '--delete-expired'], - return_id=False) - - self.assertTrue( - self.show_pb(backup_dir, 'node')[0]['status'], - 'OK') - - self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'], - 'RUNNING') - - self.assertTrue( - self.show_pb(backup_dir, 'node')[2]['status'], - 'OK') - - self.assertEqual( - len(self.show_pb(backup_dir, 'node')), - 6) diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py deleted file mode 100644 index e789d174a..000000000 --- a/tests/set_backup_test.py +++ /dev/null @@ -1,476 +0,0 @@ -import unittest -import subprocess -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from sys import exit -from datetime import datetime, timedelta - - -class SetBackupTest(ProbackupTest, unittest.TestCase): - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_set_backup_sanity(self): - """general sanity for set-backup command""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - recovery_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['recovery-time'] - - expire_time_1 = "{:%Y-%m-%d %H:%M:%S}".format( - datetime.now() + timedelta(days=5)) - - try: - self.set_backup(backup_dir, False, options=['--ttl=30d']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing instance. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: required parameter not specified: --instance', - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.set_backup( - backup_dir, 'node', - options=[ - "--ttl=30d", - "--expire-time='{0}'".format(expire_time_1)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because options cannot be mixed. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--expire-time' " - "and '--ttl' options together", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.set_backup(backup_dir, 'node', options=["--ttl=30d"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing backup_id. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You must specify parameter (-i, --backup-id) " - "for 'set-backup' command", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.set_backup( - backup_dir, 'node', backup_id, options=["--ttl=30d"]) - - actual_expire_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['expire-time'] - - self.assertNotEqual(expire_time_1, actual_expire_time) - - expire_time_2 = "{:%Y-%m-%d %H:%M:%S}".format( - datetime.now() + timedelta(days=6)) - - self.set_backup( - backup_dir, 'node', backup_id, - options=["--expire-time={0}".format(expire_time_2)]) - - actual_expire_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['expire-time'] - - self.assertIn(expire_time_2, actual_expire_time) - - # unpin backup - self.set_backup( - backup_dir, 'node', backup_id, options=["--ttl=0"]) - - attr_list = self.show_pb( - backup_dir, 'node', backup_id=backup_id) - - self.assertNotIn('expire-time', attr_list) - - self.set_backup( - backup_dir, 'node', backup_id, options=["--expire-time={0}".format(recovery_time)]) - - # parse string to datetime object - #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_retention_redundancy_pinning(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - with open(os.path.join( - backup_dir, 'backups', 'node', - "pg_probackup.conf"), "a") as conf: - conf.write("retention-redundancy = 1\n") - - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) - - # Make backups to be purged - full_id = self.backup_node(backup_dir, 'node', node) - page_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - # Make backups to be keeped - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - self.set_backup( - backup_dir, 'node', page_id, options=['--ttl=5d']) - - # Purge backups - log = self.delete_expired( - backup_dir, 'node', - options=['--delete-expired', '--log-level-console=LOG']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - - self.assertIn('Time Window: 0d/5d', log) - self.assertIn( - 'LOG: Backup {0} is pinned until'.format(page_id), - log) - self.assertIn( - 'LOG: Retain backup {0} because his descendant ' - '{1} is guarded by retention'.format(full_id, page_id), - log) - - # @unittest.skip("skip") - def test_retention_window_pinning(self): - """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUP - backup_id_1 = self.backup_node(backup_dir, 'node', node) - page1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Take second FULL BACKUP - backup_id_2 = self.backup_node(backup_dir, 'node', node) - page2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Take third FULL BACKUP - backup_id_3 = self.backup_node(backup_dir, 'node', node) - page2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.set_backup( - backup_dir, 'node', page1, options=['--ttl=30d']) - - # Purge backups - out = self.delete_expired( - backup_dir, 'node', - options=[ - '--log-level-console=LOG', - '--retention-window=1', - '--delete-expired']) - - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - - self.assertIn( - 'LOG: Backup {0} is pinned until'.format(page1), out) - - self.assertIn( - 'LOG: Retain backup {0} because his descendant ' - '{1} is guarded by retention'.format(backup_id_1, page1), - out) - - # @unittest.skip("skip") - def test_wal_retention_and_pinning(self): - """ - B1---B2---P---B3---> - wal-depth=2 - P - pinned backup - - expected result after WAL purge: - B1 B2---P---B3---> - - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take FULL BACKUP - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - node.pgbench_init(scale=1) - - # Take PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--stream']) - - node.pgbench_init(scale=1) - - # Take DELTA BACKUP and pin it - expire_time = "{:%Y-%m-%d %H:%M:%S}".format( - datetime.now() + timedelta(days=6)) - backup_id_pinned = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - '--stream', - '--expire-time={0}'.format(expire_time)]) - - node.pgbench_init(scale=1) - - # Take second PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) - - node.pgbench_init(scale=1) - - # Purge backups - out = self.delete_expired( - backup_dir, 'node', - options=[ - '--log-level-console=LOG', - '--delete-wal', '--wal-depth=2']) - - # print(out) - self.assertIn( - 'Pinned backup {0} is ignored for the ' - 'purpose of WAL retention'.format(backup_id_pinned), - out) - - for instance in self.show_archive(backup_dir): - timelines = instance['timelines'] - - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000004') - self.assertEqual(timeline['status'], 'OK') - - # @unittest.skip("skip") - def test_wal_retention_and_pinning_1(self): - """ - P---B1---> - wal-depth=2 - P - pinned backup - - expected result after WAL purge: - P---B1---> - - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - expire_time = "{:%Y-%m-%d %H:%M:%S}".format( - datetime.now() + timedelta(days=6)) - - # take FULL BACKUP - backup_id_pinned = self.backup_node( - backup_dir, 'node', node, - options=['--expire-time={0}'.format(expire_time)]) - - node.pgbench_init(scale=2) - - # Take second PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - node.pgbench_init(scale=2) - - # Purge backups - out = self.delete_expired( - backup_dir, 'node', - options=[ - '--log-level-console=verbose', - '--delete-wal', '--wal-depth=2']) - - print(out) - self.assertIn( - 'Pinned backup {0} is ignored for the ' - 'purpose of WAL retention'.format(backup_id_pinned), - out) - - for instance in self.show_archive(backup_dir): - timelines = instance['timelines'] - - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000002') - self.assertEqual(timeline['status'], 'OK') - - self.validate_pb(backup_dir) - - # @unittest.skip("skip") - def test_add_note_newlines(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL - backup_id = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--note={0}'.format('hello\nhello')]) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual(backup_meta['note'], "hello") - - self.set_backup(backup_dir, 'node', backup_id, options=['--note=hello\nhello']) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual(backup_meta['note'], "hello") - - self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - self.assertNotIn('note', backup_meta) - - # @unittest.skip("skip") - def test_add_big_note(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - -# note = node.safe_psql( -# "postgres", -# "SELECT repeat('hello', 400)").rstrip() # TODO: investigate - - note = node.safe_psql( - "postgres", - "SELECT repeat('hello', 210)").rstrip() - - # FULL - try: - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--note={0}'.format(note)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because note is too large " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup note cannot exceed 1024 bytes", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - note = node.safe_psql( - "postgres", - "SELECT repeat('hello', 200)").decode('utf-8').rstrip() - - backup_id = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--note={0}'.format(note)]) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - self.assertEqual(backup_meta['note'], note) - - - # @unittest.skip("skip") - def test_add_big_note_1(self): - """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - note = node.safe_psql( - "postgres", - "SELECT repeat('q', 1024)").decode('utf-8').rstrip() - - # FULL - backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - self.set_backup( - backup_dir, 'node', backup_id, - options=['--note={0}'.format(note)]) - - backup_meta = self.show_pb(backup_dir, 'node', backup_id) - - print(backup_meta) - self.assertEqual(backup_meta['note'], note) diff --git a/tests/show_test.py b/tests/show_test.py deleted file mode 100644 index c4b96499d..000000000 --- a/tests/show_test.py +++ /dev/null @@ -1,509 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class ShowTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_show_1(self): - """Status DONE and OK""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.assertEqual( - self.backup_node( - backup_dir, 'node', node, - options=["--log-level-console=off"]), - None - ) - self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_show_json(self): - """Status DONE and OK""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.assertEqual( - self.backup_node( - backup_dir, 'node', node, - options=["--log-level-console=off"]), - None - ) - self.backup_node(backup_dir, 'node', node) - self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - - # @unittest.skip("skip") - def test_corrupt_2(self): - """Status CORRUPT""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - # delete file which belong to backup - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "database", "postgresql.conf") - os.remove(file) - - try: - self.validate_pb(backup_dir, 'node', backup_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because backup corrupted." - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'data files are corrupted', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) - self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) - - # @unittest.skip("skip") - def test_no_control_file(self): - """backup.control doesn't exist""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - # delete backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - os.remove(file) - - output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) - - self.assertIn( - 'Control file', - output) - - self.assertIn( - 'doesn\'t exist', - output) - - # @unittest.skip("skip") - def test_empty_control_file(self): - """backup.control is empty""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - # truncate backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - fd = open(file, 'w') - fd.close() - - output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) - - self.assertIn( - 'Control file', - output) - - self.assertIn( - 'is empty', - output) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_corrupt_control_file(self): - """backup.control contains invalid option""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - # corrupt backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - fd = open(file, 'a') - fd.write("statuss = OK") - fd.close() - - self.assertIn( - 'WARNING: Invalid option "statuss" in file', - self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_corrupt_correctness(self): - """backup.control contains invalid option""" - if not self.remote: - self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - - backup_remote_id = self.backup_node(backup_dir, 'node', node) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_corrupt_correctness_1(self): - """backup.control contains invalid option""" - if not self.remote: - self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - - backup_remote_id = self.backup_node(backup_dir, 'node', node) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # change data - pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) - pgbench.wait() - - # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_corrupt_correctness_2(self): - """backup.control contains invalid option""" - if not self.remote: - self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" - " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, - options=['--compress'], no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, options=['--compress']) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # change data - pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) - pgbench.wait() - - # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--compress'], no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--compress']) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--compress'], no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) - - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) - - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_color_with_no_terminal(self): - """backup.control contains invalid option""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=1) - - # FULL - try: - self.backup_node( - backup_dir, 'node', node, options=['--archive-timeout=1s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because archiving is disabled\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - '[0m', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) diff --git a/tests/time_consuming_test.py b/tests/time_consuming_test.py deleted file mode 100644 index c0038c085..000000000 --- a/tests/time_consuming_test.py +++ /dev/null @@ -1,77 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest -import subprocess -from time import sleep - - -class TimeConsumingTests(ProbackupTest, unittest.TestCase): - def test_pbckp150(self): - """ - https://jira.postgrespro.ru/browse/PBCKP-150 - create a node filled with pgbench - create FULL backup followed by PTRACK backup - run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel - """ - # init node - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - if not self.ptrack: - self.skipTest('Skipped because ptrack support is disabled') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={ - 'max_connections': 100, - 'log_statement': 'none', - 'log_checkpoints': 'on', - 'autovacuum': 'off', - 'ptrack.map_size': 1}) - - if node.major_version >= 13: - self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) - else: - self.set_auto_conf(node, {'wal_keep_segments': '1000'}) - - # init probackup and add an instance - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - # run the node and init ptrack - node.slow_start() - node.safe_psql("postgres", "CREATE EXTENSION ptrack") - # populate it with pgbench - node.pgbench_init(scale=5) - - # FULL backup followed by PTRACK backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) - - # run ordinary pgbench scenario to imitate some activity and another pgbench for vacuuming in parallel - nBenchDuration = 30 - pgbench = node.pgbench(options=['-c', '20', '-j', '8', '-T', str(nBenchDuration)]) - with open('/tmp/pbckp150vacuum.sql', 'w') as f: - f.write('VACUUM (FULL) pgbench_accounts, pgbench_tellers, pgbench_history; SELECT pg_sleep(1);\n') - pgbenchval = node.pgbench(options=['-c', '1', '-f', '/tmp/pbckp150vacuum.sql', '-T', str(nBenchDuration)]) - - # several PTRACK backups - for i in range(nBenchDuration): - print("[{}] backing up PTRACK diff...".format(i+1)) - self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) - sleep(0.1) - # if the activity pgbench has finished, stop backing up - if pgbench.poll() is not None: - break - - pgbench.kill() - pgbenchval.kill() - pgbench.wait() - pgbenchval.wait() - - backups = self.show_pb(backup_dir, 'node') - for b in backups: - self.assertEqual("OK", b['status']) diff --git a/tests/time_stamp_test.py b/tests/time_stamp_test.py deleted file mode 100644 index 170c62cd4..000000000 --- a/tests/time_stamp_test.py +++ /dev/null @@ -1,236 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -import subprocess -from time import sleep - - -class TimeStamp(ProbackupTest, unittest.TestCase): - - def test_start_time_format(self): - """Test backup ID changing after start-time editing in backup.control. - We should convert local time in UTC format""" - # Create simple node - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.start() - - backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) - show_backup = self.show_pb(backup_dir, 'node') - - i = 0 - while i < 2: - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: - output = "" - for line in f: - if line.startswith('start-time') is True: - if i == 0: - output = output + str(line[:-5])+'+00\''+'\n' - else: - output = output + str(line[:-5]) + '\'' + '\n' - else: - output = output + str(line) - f.close() - - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: - fw.write(output) - fw.flush() - show_backup = show_backup + self.show_pb(backup_dir, 'node') - i += 1 - - print(show_backup[1]['id']) - print(show_backup[2]['id']) - - self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") - - output = self.show_pb(backup_dir, as_json=False, as_text=True) - self.assertNotIn("backup ID in control file", output) - - node.stop() - - def test_server_date_style(self): - """Issue #112""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={"datestyle": "GERMAN, DMY"}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.start() - - self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j 2']) - - def test_handling_of_TZ_env_variable(self): - """Issue #284""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.start() - - my_env = os.environ.copy() - my_env["TZ"] = "America/Detroit" - - self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) - - output = self.show_pb(backup_dir, 'node', as_json=False, as_text=True, env=my_env) - - self.assertNotIn("backup ID in control file", output) - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_dst_timezone_handling(self): - """for manual testing""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - print(subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate()) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-ntp', 'false'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # FULL - output = self.backup_node(backup_dir, 'node', node, return_id=False) - self.assertNotIn("backup ID in control file", output) - - # move to dst - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # DELTA - output = self.backup_node( - backup_dir, 'node', node, backup_type='delta', return_id=False) - self.assertNotIn("backup ID in control file", output) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # DELTA - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - output = self.show_pb(backup_dir, as_json=False, as_text=True) - self.assertNotIn("backup ID in control file", output) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-ntp', 'true'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - sleep(10) - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - output = self.show_pb(backup_dir, as_json=False, as_text=True) - self.assertNotIn("backup ID in control file", output) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - @unittest.skip("skip") - def test_dst_timezone_handling_backward_compatibilty(self): - """for manual testing""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-ntp', 'false'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # FULL - self.backup_node(backup_dir, 'node', node, old_binary=True, return_id=False) - - # move to dst - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # DELTA - output = self.backup_node( - backup_dir, 'node', node, backup_type='delta', old_binary=True, return_id=False) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - # DELTA - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - output = self.show_pb(backup_dir, as_json=False, as_text=True) - self.assertNotIn("backup ID in control file", output) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-ntp', 'true'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() - - sleep(10) - - self.backup_node(backup_dir, 'node', node, backup_type='delta') - - output = self.show_pb(backup_dir, as_json=False, as_text=True) - self.assertNotIn("backup ID in control file", output) - - subprocess.Popen( - ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() diff --git a/tests/validate_test.py b/tests/validate_test.py deleted file mode 100644 index 98a0fd13f..000000000 --- a/tests/validate_test.py +++ /dev/null @@ -1,4083 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from pathlib import Path -import subprocess -from sys import exit -import time -import hashlib - - -class ValidateTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_basic_validate_nullified_heap_page_backup(self): - """ - make node with nullified heap block - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - file_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - - node.safe_psql( - "postgres", - "CHECKPOINT") - - # Nullify some block in PostgreSQL - file = os.path.join(node.data_dir, file_path) - with open(file, 'r+b') as f: - f.seek(8192) - f.write(b"\x00"*8192) - f.flush() - f.close - - self.backup_node( - backup_dir, 'node', node, options=['--log-level-file=verbose']) - - pgdata = self.pgdata_content(node.data_dir) - - if not self.remote: - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") - with open(log_file_path) as f: - log_content = f.read() - self.assertIn( - 'File: "{0}" blknum 1, empty page'.format(Path(file).as_posix()), - log_content, - 'Failed to detect nullified block') - - self.validate_pb(backup_dir, options=["-j", "4"]) - node.cleanup() - - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_validate_wal_unreal_values(self): - """ - make node with archiving, make archive backup - validate to both real and unreal values - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - - node.pgbench_init(scale=3) - - target_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - after_backup_time = datetime.now().replace(second=0, microsecond=0) - - # Validate to real time - self.assertIn( - "INFO: Backup validation completed successfully", - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(target_time), "-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # Validate to unreal time - unreal_time_1 = after_backup_time - timedelta(days=2) - try: - self.validate_pb( - backup_dir, 'node', options=["--time={0}".format( - unreal_time_1), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal time.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup satisfying target options is not found', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Validate to unreal time #2 - unreal_time_2 = after_backup_time + timedelta(days=2) - try: - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(unreal_time_2), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal time.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to time' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Validate to real xid - target_xid = None - with node.connect("postgres") as con: - res = con.execute( - "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - self.switch_wal_segment(node) - time.sleep(5) - - self.assertIn( - "INFO: Backup validation completed successfully", - self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(target_xid), - "-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # Validate to unreal xid - unreal_xid = int(target_xid) + 1000 - try: - self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(unreal_xid), - "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal xid.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to xid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Validate with backup ID - output = self.validate_pb(backup_dir, 'node', backup_id, - options=["-j", "4"]) - self.assertIn( - "INFO: Validating backup {0}".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} data files are valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} WAL segments are valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} is valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Validate of backup {0} completed".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # @unittest.skip("skip") - def test_basic_validate_corrupted_intermediate_backup(self): - """ - make archive node, take FULL, PAGE1, PAGE2 backups, - corrupt file in PAGE1 backup, - run validate on PAGE1, expect PAGE1 to gain status CORRUPT - and PAGE2 gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Corrupt some file - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', file_path) - with open(file, "r+b", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - # Simple validate - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message and - 'ERROR: Backup {0} is corrupt'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - - # @unittest.skip("skip") - def test_validate_corrupted_intermediate_backups(self): - """ - make archive node, take FULL, PAGE1, PAGE2 backups, - corrupt file in FULL and PAGE1 backups, run validate on PAGE1, - expect FULL and PAGE1 to gain status CORRUPT and - PAGE2 gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_path_t_heap = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap_1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_path_t_heap_1 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - # Corrupt some file in PAGE1 backup - file_page1 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', file_path_t_heap_1) - with open(file_page1, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - # Validate PAGE1 - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} is orphaned because his parent'.format( - backup_id_3) in e.message and - 'ERROR: Backup {0} is orphan.'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - - # @unittest.skip("skip") - def test_validate_specific_error_intermediate_backups(self): - """ - make archive node, take FULL, PAGE1, PAGE2 backups, - change backup status of FULL and PAGE1 to ERROR, - run validate on PAGE1 - purpose of this test is to be sure that not only - CORRUPT backup descendants can be orphanized - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change FULL backup status to ERROR - control_path = os.path.join( - backup_dir, 'backups', 'node', backup_id_1, 'backup.control') - - with open(control_path, 'r') as f: - actual_control = f.read() - - new_control_file = '' - for line in actual_control.splitlines(): - new_control_file += line.replace( - 'status = OK', 'status = ERROR') - new_control_file += '\n' - - with open(control_path, 'wt') as f: - f.write(new_control_file) - f.flush() - f.close() - - # Validate PAGE1 - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: ERROR'.format( - backup_id_2, backup_id_1) in e.message and - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} has status ERROR. Skip validation.'.format( - backup_id_1) and - 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "ERROR"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - - # @unittest.skip("skip") - def test_validate_error_intermediate_backups(self): - """ - make archive node, take FULL, PAGE1, PAGE2 backups, - change backup status of FULL and PAGE1 to ERROR, - run validate on instance - purpose of this test is to be sure that not only - CORRUPT backup descendants can be orphanized - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Change FULL backup status to ERROR - control_path = os.path.join( - backup_dir, 'backups', 'node', backup_id_1, 'backup.control') - - with open(control_path, 'r') as f: - actual_control = f.read() - - new_control_file = '' - for line in actual_control.splitlines(): - new_control_file += line.replace( - 'status = OK', 'status = ERROR') - new_control_file += '\n' - - with open(control_path, 'wt') as f: - f.write(new_control_file) - f.flush() - f.close() - - # Validate instance - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Backup {0} is orphaned because " - "his parent {1} has status: ERROR".format( - backup_id_2, backup_id_1) in e.message and - 'WARNING: Backup {0} has status ERROR. Skip validation'.format( - backup_id_1) in e.message and - "WARNING: Some backups are not valid" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'ERROR', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "ERROR"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - - # @unittest.skip("skip") - def test_validate_corrupted_intermediate_backups_1(self): - """ - make archive node, FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2, - corrupt file in PAGE1 and PAGE4, run validate on PAGE3, - expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 - to gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_page_2 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE3 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - backup_id_4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE4 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE5 - node.safe_psql( - "postgres", - "create table t_heap1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_page_5 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() - backup_id_6 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE6 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30000,40000) i") - backup_id_7 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL2 - backup_id_8 = self.backup_node(backup_dir, 'node', node) - - # Corrupt some file in PAGE2 and PAGE5 backups - file_page1 = os.path.join( - backup_dir, 'backups', 'node', backup_id_3, 'database', file_page_2) - with open(file_page1, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - file_page4 = os.path.join( - backup_dir, 'backups', 'node', backup_id_6, 'database', file_page_5) - with open(file_page4, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - # Validate PAGE3 - try: - self.validate_pb( - backup_dir, 'node', - backup_id=backup_id_4, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_4, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_5, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_6, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_7, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format(repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], - 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_validate_specific_target_corrupted_intermediate_backups(self): - """ - make archive node, take FULL1, PAGE1, PAGE2, PAGE3, PAGE4, PAGE5, FULL2 - corrupt file in PAGE1 and PAGE4, run validate on PAGE3 to specific xid, - expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to - gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - # PAGE1 - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_page_2 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE3 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(10000,20000) i") - backup_id_4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE4 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - - target_xid = node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30001, 30001) i RETURNING (xmin)").decode('utf-8').rstrip() - - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE5 - node.safe_psql( - "postgres", - "create table t_heap1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_page_5 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() - backup_id_6 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE6 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30000,40000) i") - backup_id_7 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL2 - backup_id_8 = self.backup_node(backup_dir, 'node', node) - - # Corrupt some file in PAGE2 and PAGE5 backups - file_page1 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_3, 'database', file_page_2) - with open(file_page1, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - file_page4 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_6, 'database', file_page_5) - with open(file_page4, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - # Validate PAGE3 - try: - self.validate_pb( - backup_dir, 'node', - options=[ - '-i', backup_id_4, '--xid={0}'.format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_4, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_5, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_6, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_7, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'ERROR: Backup {0} is orphan'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_validate_instance_with_several_corrupt_backups(self): - """ - make archive node, take FULL1, PAGE1_1, FULL2, PAGE2_1 backups, FULL3 - corrupt file in FULL and FULL2 and run validate on instance, - expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN - FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select generate_series(0,1) i") - # FULL1 - backup_id_1 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) - - # FULL2 - backup_id_2 = self.backup_node(backup_dir, 'node', node) - rel_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - node.safe_psql( - "postgres", - "insert into t_heap values(2)") - - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL3 - backup_id_4 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "insert into t_heap values(3)") - - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL4 - backup_id_6 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) - - # Corrupt some files in FULL2 and FULL3 backup - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_2, - 'database', rel_path)) - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_4, - 'database', rel_path)) - - # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message, - "\n Unexpected Error Message: {0}\n " - "CMD: {1}".format(repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Some backups are not valid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_4)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], - 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_validate_instance_with_several_corrupt_backups_interrupt(self): - """ - check that interrupt during validation is handled correctly - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select generate_series(0,1) i") - # FULL1 - backup_id_1 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) - - # FULL2 - backup_id_2 = self.backup_node(backup_dir, 'node', node) - rel_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - node.safe_psql( - "postgres", - "insert into t_heap values(2)") - - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL3 - backup_id_4 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "insert into t_heap values(3)") - - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL4 - backup_id_6 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) - - # Corrupt some files in FULL2 and FULL3 backup - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_1, - 'database', rel_path)) - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_3, - 'database', rel_path)) - - # Validate Instance - gdb = self.validate_pb( - backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) - - gdb.set_breakpoint('validate_file_pages') - gdb.run_until_break() - gdb.continue_execution_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - gdb.continue_execution_until_error() - - self.assertEqual( - 'DONE', self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'DONE', self.show_pb(backup_dir, 'node', backup_id_6)['status'], - 'Backup STATUS should be "OK"') - - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertNotIn( - 'Interrupted while locking backup', log_content) - - # @unittest.skip("skip") - def test_validate_instance_with_corrupted_page(self): - """ - make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, - corrupt file in PAGE1 backup and run validate on instance, - expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_path_t_heap1 = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - # FULL1 - backup_id_4 = self.backup_node( - backup_dir, 'node', node) - # PAGE3 - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', backup_id_2, - 'database', file_path_t_heap1) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message, - "\n Unexpected Error Message: {0}\n " - "CMD: {1}".format(repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_5) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_5) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_5) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_4) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_4) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_3) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_3) in e.message and - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_3, backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Some backups are not valid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], - 'Backup STATUS should be "OK"') - self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], - 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_validate_instance_with_corrupted_full_and_try_restore(self): - """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, - corrupt file in FULL backup and run validate on instance, - expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, - try to restore backup with --no-validation option""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - file_path_t_heap = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - # PAGE1 - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # PAGE2 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # FULL1 - backup_id_4 = self.backup_node(backup_dir, 'node', node) - - # PAGE3 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30000,40000) i") - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file' in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - - node.cleanup() - restore_out = self.restore_node( - backup_dir, 'node', node, - options=["--no-validate"]) - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id_5), - restore_out, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - - # @unittest.skip("skip") - def test_validate_instance_with_corrupted_full(self): - """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, - corrupt file in FULL backup and run validate on instance, - expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - file_path_t_heap = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # PAGE2 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(20000,30000) i") - - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - # FULL1 - backup_id_4 = self.backup_node( - backup_dir, 'node', node) - - # PAGE3 - node.safe_psql( - "postgres", - "insert into t_heap select i as id, " - "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30000,40000) i") - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') - - # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file' in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_validate_corrupt_wal_1(self): - """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id_1 = self.backup_node(backup_dir, 'node', node) - - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - backup_id_2 = self.backup_node(backup_dir, 'node', node) - - # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - for wal in wals: - with open(os.path.join(wals_dir, wal), "rb+", 0) as f: - f.seek(42) - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close - - # Simple validate - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup' in e.message and - 'WAL segments are corrupted' in e.message and - "WARNING: There are not enough WAL " - "records to consistenly restore backup" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "CORRUPT"') - - # @unittest.skip("skip") - def test_validate_corrupt_wal_2(self): - """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - backup_id = self.backup_node(backup_dir, 'node', node) - target_xid = None - with node.connect("postgres") as con: - res = con.execute( - "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - for wal in wals: - with open(os.path.join(wals_dir, wal), "rb+", 0) as f: - f.seek(128) - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close - - # Validate to xid - try: - self.validate_pb( - backup_dir, - 'node', - backup_id, - options=[ - "--xid={0}".format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup' in e.message and - 'WAL segments are corrupted' in e.message and - "WARNING: There are not enough WAL " - "records to consistenly restore backup" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "CORRUPT"') - - # @unittest.skip("skip") - def test_validate_wal_lost_segment_1(self): - """make archive node, make archive full backup, - delete from archive wal segment which belong to previous backup - run validate, expecting error because of missing wal segment - make sure that backup status is 'CORRUPT' - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.pgbench_init(scale=3) - - backup_id = self.backup_node(backup_dir, 'node', node) - - # Delete wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) - os.remove(file) - - # cut out '.gz' - if self.archive_compress: - file = file[:-3] - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "is absent" in e.message and - "WARNING: There are not enough WAL records to consistenly " - "restore backup {0}".format(backup_id) in e.message and - "WARNING: Backup {0} WAL segments are corrupted".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup {0} should have STATUS "CORRUPT"') - - # Run validate again - try: - self.validate_pb(backup_dir, 'node', backup_id, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'INFO: Revalidating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_validate_corrupt_wal_between_backups(self): - """ - make archive node, make full backup, corrupt all wal files, - run validate to real xid, expect errors - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - # make some wals - node.pgbench_init(scale=3) - - with node.connect("postgres") as con: - con.execute("CREATE TABLE tbl0005 (a text)") - con.commit() - - with node.connect("postgres") as con: - res = con.execute( - "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") - con.commit() - target_xid = res[0][0] - - if self.get_version(node) < self.version_to_num('10.0'): - walfile = node.safe_psql( - 'postgres', - 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() - else: - walfile = node.safe_psql( - 'postgres', - 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() - - if self.archive_compress: - walfile = walfile + '.gz' - self.switch_wal_segment(node) - - # generate some wals - node.pgbench_init(scale=3) - - self.backup_node(backup_dir, 'node', node) - - # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: - f.seek(9000) - f.write(b"b") - f.flush() - f.close - - # Validate to xid - try: - self.validate_pb( - backup_dir, - 'node', - backup_id, - options=[ - "--xid={0}".format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to xid' in e.message and - 'WARNING: Recovery can be done up to time' in e.message and - "ERROR: Not enough WAL records to xid {0}\n".format( - target_xid), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], - 'Backup STATUS should be "OK"') - - self.assertEqual( - 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], - 'Backup STATUS should be "OK"') - - # @unittest.skip("skip") - def test_pgpro702_688(self): - """ - make node without archiving, make stream backup, - get Recovery Time, validate to Recovery Time - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - recovery_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['recovery-time'] - - try: - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(recovery_time), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WAL archive is empty. You cannot restore backup to a ' - 'recovery target without WAL archive', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_pgpro688(self): - """ - make node with archiving, make backup, get Recovery Time, - validate to Recovery Time. Waiting PGPRO-688. RESOLVED - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.validate_pb( - backup_dir, 'node', options=["--time={0}".format(recovery_time), - "-j", "4"]) - - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_pgpro561(self): - """ - make node with archiving, make stream backup, - restore it to node1, check that archiving is not successful on node1 - """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) - self.set_archiving(backup_dir, 'node1', node1) - node1.slow_start() - - backup_id = self.backup_node( - backup_dir, 'node1', node1, options=["--stream"]) - - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) - node2.cleanup() - - node1.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - - self.backup_node( - backup_dir, 'node1', node1, - backup_type='page', options=["--stream"]) - self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) - - self.set_auto_conf( - node2, {'port': node2.port, 'archive_mode': 'off'}) - - node2.slow_start() - - self.set_auto_conf( - node2, {'archive_mode': 'on'}) - - node2.stop() - node2.slow_start() - - timeline_node1 = node1.get_control_data()["Latest checkpoint's TimeLineID"] - timeline_node2 = node2.get_control_data()["Latest checkpoint's TimeLineID"] - self.assertEqual( - timeline_node1, timeline_node2, - "Timelines on Master and Node1 should be equal. " - "This is unexpected") - - archive_command_node1 = node1.safe_psql( - "postgres", "show archive_command") - archive_command_node2 = node2.safe_psql( - "postgres", "show archive_command") - self.assertEqual( - archive_command_node1, archive_command_node2, - "Archive command on Master and Node should be equal. " - "This is unexpected") - - # result = node2.safe_psql("postgres", "select last_failed_wal from pg_stat_get_archiver() where last_failed_wal is not NULL") - ## self.assertEqual(res, six.b(""), 'Restored Node1 failed to archive segment {0} due to having the same archive command as Master'.format(res.rstrip())) - # if result == "": - # self.assertEqual(1, 0, 'Error is expected due to Master and Node1 having the common archive and archive_command') - - node1.psql( - "postgres", - "create table t_heap_1 as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10) i") - - self.switch_wal_segment(node1) - -# wals_dir = os.path.join(backup_dir, 'wal', 'node1') -# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( -# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] -# wals = map(str, wals) -# print(wals) - - self.switch_wal_segment(node2) - -# wals_dir = os.path.join(backup_dir, 'wal', 'node1') -# wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( -# wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] -# wals = map(str, wals) -# print(wals) - - time.sleep(5) - - log_file = os.path.join(node2.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertTrue( - 'LOG: archive command failed with exit code 1' in log_content and - 'DETAIL: The failed archive command was:' in log_content and - 'WAL file already exists in archive with different checksum' in log_content, - 'Expecting error messages about failed archive_command' - ) - self.assertFalse( - 'pg_probackup archive-push completed successfully' in log_content) - - # @unittest.skip("skip") - def test_validate_corrupted_full(self): - """ - make node with archiving, take full backup, and three page backups, - take another full backup and three page backups - corrupt second full backup, run validate, check that - second full backup became CORRUPT and his page backups are ORPHANs - remove corruption and run valudate again, check that - second full backup and his page backups are OK - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - "postgres", - "alter system set archive_command = 'false'") - node.reload() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=1s']) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - pass - - self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') - self.set_archiving(backup_dir, 'node', node) - node.reload() - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue( - self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') - self.assertTrue( - self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue( - self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') - self.assertTrue( - self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - - os.rename(file_new, file) - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - - # @unittest.skip("skip") - def test_validate_corrupted_full_1(self): - """ - make node with archiving, take full backup, and three page backups, - take another full backup and four page backups - corrupt second full backup, run validate, check that - second full backup became CORRUPT and his page backups are ORPHANs - remove corruption from full backup and corrupt his second page backup - run valudate again, check that - second full backup and his firts page backups are OK, - second page should be CORRUPT - third page should be ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - backup_id_page = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(file_new, file) - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_page, 'database', 'backup_label') - - file_new = os.path.join(backup_dir, 'backup_label') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - - # @unittest.skip("skip") - def test_validate_corrupted_full_2(self): - """ - PAGE2_2b - PAGE2_2a - PAGE2_4 - PAGE2_4 <- validate - PAGE2_3 - PAGE2_2 <- CORRUPT - PAGE2_1 - FULL2 - PAGE1_1 - FULL1 - corrupt second page backup, run validate on PAGE2_3, check that - PAGE2_2 became CORRUPT and his descendants are ORPHANs, - take two more PAGE backups, which now trace their origin - to PAGE2_1 - latest OK backup, - run validate on PAGE2_3, check that PAGE2_2a and PAGE2_2b are OK, - - remove corruption from PAGE2_2 and run validate on PAGE2_4 - """ - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - corrupt_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - corrupt_id, 'database', 'backup_label') - - file_new = os.path.join(backup_dir, 'backup_label') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'INFO: Validating parents for backup {0}'.format(validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # THIS IS GOLD!!!! - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[9]['id']), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[8]['id']), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # revalidate again - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating parents for backup {0}'.format( - validate_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'ERROR: Backup {0} is orphan.'.format( - validate_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # Fix CORRUPT - os.rename(file_new, file) - - output = self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating parents for backup {0}'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - corrupt_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[5]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[5]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Backup {0} WAL segments are valid'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Backup {0} is valid.'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validate of backup {0} completed.'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - # Now we have two perfectly valid backup chains based on FULL2 - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # @unittest.skip("skip") - def test_validate_corrupted_full_missing(self): - """ - make node with archiving, take full backup, and three page backups, - take another full backup and four page backups - corrupt second full backup, run validate, check that - second full backup became CORRUPT and his page backups are ORPHANs - remove corruption from full backup and remove his second page backup - run valudate again, check that - second full backup and his firts page backups are OK, - third page should be ORPHAN - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - backup_id_page = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} has status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # Full backup is fixed - os.rename(file_new, file) - - # break PAGE - old_directory = os.path.join( - backup_dir, 'backups', 'node', backup_id_page) - new_directory = os.path.join(backup_dir, backup_id_page) - os.rename(old_directory, new_directory) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - backup_id_page), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - backup_id_page), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - # missing backup is here - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # validate should be idempotent - user running validate - # second time must be provided with ID of missing backup - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - backup_id_page), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - backup_id_page), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - # missing backup is here - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # fix missing PAGE backup - os.rename(new_directory, old_directory) - # exit(1) - - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - output = self.validate_pb(backup_dir, options=["-j", "4"]) - - self.assertIn( - 'INFO: All backups are valid', - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( - self.show_pb(backup_dir, 'node')[8]['id'], - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[7]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[8]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - def test_file_size_corruption_no_validate(self): - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - # initdb_params=['--data-checksums'], - ) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i") - node.safe_psql( - "postgres", - "CHECKPOINT;") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - heap_size = node.safe_psql( - "postgres", - "select pg_relation_size('t_heap')") - - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4"], asynchronous=False, gdb=False) - - node.stop() - node.cleanup() - - # Let`s do file corruption - with open( - os.path.join( - backup_dir, "backups", 'node', backup_id, - "database", heap_path), "rb+", 0) as f: - f.truncate(int(heap_size) - 4096) - f.flush() - f.close - - node.cleanup() - - try: - self.restore_node( - backup_dir, 'node', node, - options=["--no-validate"]) - except ProbackupException as e: - self.assertTrue( - "ERROR: Backup files restoring failed" in e.message, - repr(e.message)) - - # @unittest.skip("skip") - def test_validate_specific_backup_with_missing_backup(self): - """ - PAGE3_2 - PAGE3_1 - FULL3 - PAGE2_5 - PAGE2_4 <- validate - PAGE2_3 - PAGE2_2 <- missing - PAGE2_1 - FULL2 - PAGE1_2 - PAGE1_1 - FULL1 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN2 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - old_directory = os.path.join(backup_dir, 'backups', 'node', missing_id) - new_directory = os.path.join(backup_dir, missing_id) - - os.rename(old_directory, new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - # missing backup - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - os.rename(new_directory, old_directory) - - # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # @unittest.skip("skip") - def test_validate_specific_backup_with_missing_backup_1(self): - """ - PAGE3_2 - PAGE3_1 - FULL3 - PAGE2_5 - PAGE2_4 <- validate - PAGE2_3 - PAGE2_2 <- missing - PAGE2_1 - FULL2 <- missing - PAGE1_2 - PAGE1_1 - FULL1 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) - - full_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - # PAGE2_1 - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') # <- SHit - # FULL2 - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(page_new_directory, page_old_directory) - os.rename(full_new_directory, full_old_directory) - - # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') # <- Fail - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # @unittest.skip("skip") - def test_validate_with_missing_backup_1(self): - """ - PAGE3_2 - PAGE3_1 - FULL3 - PAGE2_5 - PAGE2_4 <- validate - PAGE2_3 - PAGE2_2 <- missing - PAGE2_1 - FULL2 <- missing - PAGE1_2 - PAGE1_1 - FULL1 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # Break PAGE - page_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) - - # Break FULL - full_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - # PAGE2_2 is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(page_new_directory, page_old_directory) - - # Revalidate backup chain - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format( - validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[3]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') - # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(full_new_directory, full_old_directory) - - # Revalidate chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # @unittest.skip("skip") - def test_validate_with_missing_backup_2(self): - """ - PAGE3_2 - PAGE3_1 - FULL3 - PAGE2_5 - PAGE2_4 - PAGE2_3 - PAGE2_2 <- missing - PAGE2_1 - FULL2 <- missing - PAGE1_2 - PAGE1_1 - FULL1 - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) - - full_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - # PAGE2_2 is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') - # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(page_new_directory, page_old_directory) - - # Revalidate backup chain - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') - # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - # @unittest.skip("skip") - def test_corrupt_pg_control_via_resetxlog(self): - """ PGPRO-2096 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - if self.get_version(node) < 100000: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' - - os.mkdir( - os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) - - pg_control_path = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'global', 'pg_control') - - md5_before = hashlib.md5( - open(pg_control_path, 'rb').read()).hexdigest() - - self.run_binary( - [ - pg_resetxlog_path, - '-D', - os.path.join(backup_dir, 'backups', 'node', backup_id, 'database'), - '-o 42', - '-f' - ], - asynchronous=False) - - md5_after = hashlib.md5( - open(pg_control_path, 'rb').read()).hexdigest() - - if self.verbose: - print('\n MD5 BEFORE resetxlog: {0}\n MD5 AFTER resetxlog: {1}'.format( - md5_before, md5_after)) - - # Validate backup - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of pg_control change.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'data files are corrupted', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_validation_after_backup(self): - """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, options=['--stream']) - - gdb.set_breakpoint('pgBackupValidate') - gdb.run_until_break() - - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] - - file = os.path.join( - backup_dir, "backups", "node", backup_id, - "database", "postgresql.conf") - os.remove(file) - - gdb.continue_execution_until_exit() - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], - 'Backup STATUS should be "ERROR"') - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_validate_corrupt_tablespace_map(self): - """ - Check that corruption in tablespace_map is detected - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'external_dir') - - node.safe_psql( - 'postgres', - 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'tablespace_map') - - # Corrupt tablespace_map file in FULL backup - with open(tablespace_map, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Invalid CRC of backup file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - #TODO fix the test - @unittest.expectedFailure - # @unittest.skip("skip") - def test_validate_target_lsn(self): - """ - Check validation to specific LSN - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - - self.restore_node(backup_dir, 'node', node_restored) - - self.set_auto_conf( - node_restored, {'port': node_restored.port}) - - node_restored.slow_start() - - self.switch_wal_segment(node) - - backup_id = self.backup_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir) - - target_lsn = self.show_pb(backup_dir, 'node')[1]['stop-lsn'] - - self.delete_pb(backup_dir, 'node', backup_id) - - self.validate_pb( - backup_dir, 'node', - options=[ - '--recovery-target-timeline=2', - '--recovery-target-lsn={0}'.format(target_lsn)]) - - @unittest.skip("skip") - def test_partial_validate_empty_and_mangled_database_map(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - - node.slow_start() - - # create databases - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup with database_map - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) - - # truncate database_map - path = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'database_map') - with open(path, "w") as f: - f.close() - - try: - self.validate_pb( - backup_dir, 'node', - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Backup {0} data files are corrupted".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # mangle database_map - with open(path, "w") as f: - f.write("42") - f.close() - - try: - self.validate_pb( - backup_dir, 'node', - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Backup {0} data files are corrupted".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - def test_partial_validate_exclude(self): - """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "--log-level-console=verbose"]) - self.assertEqual( - 1, 0, - "Expecting Error because of missing backup ID.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You must specify parameter (-i, --backup-id) for partial validation", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - output = self.validate_pb( - backup_dir, 'node', backup_id, - options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "--log-level-console=verbose"]) - - self.assertIn( - "VERBOSE: Skip file validation due to partial restore", output) - - @unittest.skip("skip") - def test_partial_validate_include(self): - """ - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - for i in range(1, 10, 1): - node.safe_psql( - 'postgres', - 'CREATE database db{0}'.format(i)) - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - output = self.validate_pb( - backup_dir, 'node', backup_id, - options=[ - "--db-include=db1", - "--db-include=db5", - "--db-include=postgres", - "--log-level-console=verbose"]) - - self.assertIn( - "VERBOSE: Skip file validation due to partial restore", output) - - output = self.validate_pb( - backup_dir, 'node', backup_id, - options=["--log-level-console=verbose"]) - - self.assertNotIn( - "VERBOSE: Skip file validation due to partial restore", output) - - # @unittest.skip("skip") - def test_not_validate_diffenent_pg_version(self): - """Do not validate backup, if binary is compiled with different PG version""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - backup_id = self.backup_node(backup_dir, 'node', node) - - control_file = os.path.join( - backup_dir, "backups", "node", backup_id, - "backup.control") - - pg_version = node.major_version - - if pg_version.is_integer(): - pg_version = int(pg_version) - - fake_new_pg_version = pg_version + 1 - - with open(control_file, 'r') as f: - data = f.read(); - - data = data.replace( - "server-version = {0}".format(str(pg_version)), - "server-version = {0}".format(str(fake_new_pg_version))) - - with open(control_file, 'w') as f: - f.write(data); - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because validation is forbidden if server version of backup " - "is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has server version".format(backup_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_validate_corrupt_page_header_map(self): - """ - Check that corruption in page_header_map is detected - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') - - # Corrupt tablespace_map file in FULL backup - with open(page_header_map, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - - self.assertIn("WARNING: Some backups are not valid", e.message) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_validate_truncated_page_header_map(self): - """ - Check that corruption in page_header_map is detected - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') - - # truncate page_header_map file - with open(page_header_map, "rb+", 0) as f: - f.truncate(121) - f.flush() - f.close - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - self.assertIn("WARNING: Some backups are not valid", e.message) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_validate_missing_page_header_map(self): - """ - Check that corruption in page_header_map is detected - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') - - # unlink page_header_map file - os.remove(page_header_map) - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - self.assertIn("WARNING: Some backups are not valid", e.message) - - # @unittest.expectedFailure - # @unittest.skip("skip") - def test_no_validate_tablespace_map(self): - """ - Check that --no-validate is propagated to tablespace_map - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'external_dir') - - node.safe_psql( - 'postgres', - 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') - - tblspace_new = self.get_tblspace_path(node, 'external_dir_new') - - oid = node.safe_psql( - 'postgres', - "select oid from pg_tablespace where spcname = 'external_dir'").decode('utf-8').rstrip() - - # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - pgdata = self.pgdata_content(node.data_dir) - - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'tablespace_map') - - # overwrite tablespace_map file - with open(tablespace_map, "w") as f: - f.write("{0} {1}".format(oid, tblspace_new)) - f.close - - node.cleanup() - - self.restore_node(backup_dir, 'node', node, options=['--no-validate']) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # check that tablespace restore as symlink - tablespace_link = os.path.join(node.data_dir, 'pg_tblspc', oid) - self.assertTrue( - os.path.islink(tablespace_link), - "'%s' is not a symlink" % tablespace_link) - - self.assertEqual( - os.readlink(tablespace_link), - tblspace_new, - "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) - -# validate empty backup list -# page from future during validate -# page from future during backup - -# corrupt block, so file become unaligned: -# 712 Assert(header.compressed_size <= BLCKSZ); -# 713 -# 714 read_len = fread(compressed_page.data, 1, -# 715 MAXALIGN(header.compressed_size), in); -# 716 if (read_len != MAXALIGN(header.compressed_size)) -# -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", -# 718 blknum, file->path, read_len, header.compressed_size); From 08557855e2df28e07584725114aade6993ab004f Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 24 Nov 2022 12:29:44 +0300 Subject: [PATCH 1977/2107] fix error During refactoring base36enc it were mistakenly changed from current_backup->start_time to current_backup->parent_backup --- src/validate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validate.c b/src/validate.c index b89b67b84..9372b082c 100644 --- a/src/validate.c +++ b/src/validate.c @@ -508,7 +508,7 @@ do_validate_instance(InstanceState *instanceState) /* determine missing backup ID */ parent_backup_id = base36enc(tmp_backup->parent_backup); - current_backup_id = base36enc(current_backup->parent_backup); + current_backup_id = backup_id_of(current_backup); corrupted_backup_found = true; /* orphanize current_backup */ From 8488b289ba220440c53bd7dfd8bc682be8df953e Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 24 Nov 2022 14:23:45 +0300 Subject: [PATCH 1978/2107] fix test_backup_via_unprivileged_user with ptrack enabled It is garbage remained from old ptrack version support. --- tests/auth_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index 7e0b6fcfb..525dee258 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -148,8 +148,6 @@ def test_backup_via_unprivileged_user(self): node.safe_psql( "test1", "create table t1 as select generate_series(0,100)") - if self.ptrack: - self.set_auto_conf(node, {'ptrack_enable': 'on'}) node.stop() node.slow_start() From e674202eac14e3ef65a63c4f15ba6ad0969bef04 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 24 Nov 2022 14:42:12 +0300 Subject: [PATCH 1979/2107] test_backup_via_unprivileged_user - test with ptrack correctly --- tests/auth_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index 525dee258..d0be9f344 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -33,6 +33,7 @@ def test_backup_via_unprivileged_user(self): node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, + ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') @@ -41,6 +42,11 @@ def test_backup_via_unprivileged_user(self): self.set_archiving(backup_dir, 'node', node) node.slow_start() + if self.ptrack: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") try: @@ -160,9 +166,10 @@ def test_backup_via_unprivileged_user(self): backup_dir, 'node', node, options=['-U', 'backup']) # PTRACK -# self.backup_node( -# backup_dir, 'node', node, -# backup_type='ptrack', options=['-U', 'backup']) + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) class AuthTest(unittest.TestCase): From 73cce507c2aeafc20437b2b0c69328120f8a5d15 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 03:37:19 +0300 Subject: [PATCH 1980/2107] [PBCKP-358] fix CatchupTest.test_unclean_(delta|ptrack)_catchup and BugTest.test_minrecpoint_on_replica as well Tests were broken with introduction of "startness" handling in 9924ab014 [PBCKP-304] extended testgres.PosgresNode to ... since tests uses os.kill directly. --- tests/catchup.py | 4 ++-- tests/helpers/ptrack_helpers.py | 9 +++++++++ tests/pgpro2068.py | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/catchup.py b/tests/catchup.py index 88170c807..c94a5300d 100644 --- a/tests/catchup.py +++ b/tests/catchup.py @@ -972,7 +972,7 @@ def test_unclean_delta_catchup(self): self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") - os.kill(dst_pg.pid, signal.SIGKILL) + dst_pg.kill() # preparation 3: make changes on master (source) src_pg.pgbench_init(scale = 10) @@ -1061,7 +1061,7 @@ def test_unclean_ptrack_catchup(self): self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") - os.kill(dst_pg.pid, signal.SIGKILL) + dst_pg.kill() # preparation 3: make changes on master (source) src_pg.pgbench_init(scale = 10) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index e35f57bce..ab6bdda68 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -3,6 +3,7 @@ import gc import unittest from sys import exit, argv, version_info +import signal import subprocess import shutil import six @@ -190,6 +191,14 @@ def stop(self, *args, **kwargs): self.is_started = False return result + def kill(self, someone = None): + if self.is_started: + sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK + if someone == None: + os.kill(self.pid, sig) + else: + os.kill(self.auxiliary_pids[someone][0], sig) + self.is_started = False class ProbackupTest(object): # Class attributes diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py index 434ce2800..da76a8815 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068.py @@ -85,7 +85,6 @@ def test_minrecpoint_on_replica(self): # get pids of replica background workers startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] - bgwriter_pid = replica.auxiliary_pids[ProcessType.BackgroundWriter][0] # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) @@ -108,7 +107,7 @@ def test_minrecpoint_on_replica(self): pgbench.stdout.close() # kill someone, we need a crash - os.kill(int(bgwriter_pid), 9) + replica.kill(someone=ProcessType.BackgroundWriter) gdb_recovery._execute('detach') gdb_checkpointer._execute('detach') From b7a183081ecee1622cc6f66f59d65abded73abae Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 06:40:07 +0300 Subject: [PATCH 1981/2107] [PBCKP-357] fix UnicodeDecodeError Got in some tests: ``` Traceback (most recent call last): File "pg_probackup/tests/replica.py", line 625, in test_replica_stop_lsn_null_offset gdb_checkpointer = self.gdb_attach(bgwriter_pid) File "pg_probackup/tests/helpers/ptrack_helpers.py", line 1984, in gdb_attach return GDBobj([str(pid)], self, attach=True) File "pg_probackup/tests/helpers/ptrack_helpers.py", line 2054, in __init__ line = self.get_line() File "pg_probackup/tests/helpers/ptrack_helpers.py", line 2065, in get_line line = self.proc.stdout.readline() File "/usr/lib/python3.10/codecs.py", line 322, in decode (result, consumed) = self._buffer_decode(data, self.errors, final) UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 189: invalid continuation byte ``` Fixed with `errors='replace'` --- tests/helpers/ptrack_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index ab6bdda68..a141d3f44 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -2045,7 +2045,8 @@ def __init__(self, cmd, env, attach=False): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0, - universal_newlines=True + text=True, + errors='replace', ) self.gdb_pid = self.proc.pid From 7eab6e346353349fedfed3891babd29590fbc647 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 07:56:12 +0300 Subject: [PATCH 1982/2107] [PBCKP-358] sort directories to not skip 'base/1' because of 'base/12699' Looks like os.walk may yield dirs unsorted. This way "check if directory already here as part of larger directory" man mistakenly skip short 'base/1' as "part of" 'base/12699' --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index a141d3f44..4484d3167 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1799,7 +1799,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): ) for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True): - for directory in dirs: + for directory in sorted(dirs): directory_path = os.path.join(root, directory) directory_relpath = os.path.relpath(directory_path, pgdata) From 19a7c5b01f678cc2ef458e4fd3cd017378de79bf Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 07:59:14 +0300 Subject: [PATCH 1983/2107] tests: do not read whole file at once for digest. Iterate instead. --- tests/helpers/ptrack_helpers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 4484d3167..b0b997616 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1774,9 +1774,9 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_relpath = os.path.relpath(file_fullpath, pgdata) directory_dict['files'][file_relpath] = {'is_datafile': False} with open(file_fullpath, 'rb') as f: - content = f.read() # truncate cfm's content's zero tail if file_relpath.endswith('.cfm'): + content = f.read() zero64 = b"\x00"*64 l = len(content) while l > 64: @@ -1785,9 +1785,14 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): break l = s content = content[:l] - directory_dict['files'][file_relpath]['md5'] = hashlib.md5(content).hexdigest() -# directory_dict['files'][file_relpath]['md5'] = hashlib.md5( -# f = open(file_fullpath, 'rb').read()).hexdigest() + digest = hashlib.md5(content) + else: + digest = hashlib.md5() + while True: + b = f.read(64*1024) + if not b: break + digest.update(b) + directory_dict['files'][file_relpath]['md5'] = digest.hexdigest() # crappy algorithm if file.isdigit(): From 20667e959478ec0108afecec65b077fce708a7f0 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 08:30:45 +0300 Subject: [PATCH 1984/2107] tests: better directory collection in pgdata_content There is no need to walk pgdata twice. We could delete parent directories instead of skipping them. --- tests/helpers/ptrack_helpers.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index b0b997616..dafa1e0bb 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1803,27 +1803,13 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_fullpath, size_in_pages ) - for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True): - for directory in sorted(dirs): + for directory in dirs: directory_path = os.path.join(root, directory) directory_relpath = os.path.relpath(directory_path, pgdata) - - found = False - for d in dirs_to_ignore: - if d in directory_relpath: - found = True - break - - # check if directory already here as part of larger directory - if not found: - for d in directory_dict['dirs']: - # print("OLD dir {0}".format(d)) - if directory_relpath in d: - found = True - break - - if not found: - directory_dict['dirs'][directory_relpath] = {} + parent = os.path.dirname(directory_relpath) + if parent in directory_dict['dirs']: + del directory_dict['dirs'][parent] + directory_dict['dirs'][directory_relpath] = {} # get permissions for every file and directory for file in directory_dict['dirs']: From 1617eb34ecccd8a5f408b7a4712611bb0f7681d1 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 10:54:41 +0300 Subject: [PATCH 1985/2107] tests: prettify pgdata_content and compare_pgdata --- tests/helpers/ptrack_helpers.py | 234 ++++++++++++++++---------------- 1 file changed, 116 insertions(+), 118 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index dafa1e0bb..fc193fba4 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1772,7 +1772,8 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_fullpath = os.path.join(root, file) file_relpath = os.path.relpath(file_fullpath, pgdata) - directory_dict['files'][file_relpath] = {'is_datafile': False} + cfile = ContentFile(file.isdigit()) + directory_dict['files'][file_relpath] = cfile with open(file_fullpath, 'rb') as f: # truncate cfm's content's zero tail if file_relpath.endswith('.cfm'): @@ -1792,14 +1793,12 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): b = f.read(64*1024) if not b: break digest.update(b) - directory_dict['files'][file_relpath]['md5'] = digest.hexdigest() + cfile.md5 = digest.hexdigest() # crappy algorithm - if file.isdigit(): - directory_dict['files'][file_relpath]['is_datafile'] = True + if cfile.is_datafile: size_in_pages = os.path.getsize(file_fullpath)/8192 - directory_dict['files'][file_relpath][ - 'md5_per_page'] = self.get_md5_per_page_for_fork( + cfile.md5_per_page = self.get_md5_per_page_for_fork( file_fullpath, size_in_pages ) @@ -1809,18 +1808,16 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): parent = os.path.dirname(directory_relpath) if parent in directory_dict['dirs']: del directory_dict['dirs'][parent] - directory_dict['dirs'][directory_relpath] = {} + directory_dict['dirs'][directory_relpath] = ContentDir() # get permissions for every file and directory - for file in directory_dict['dirs']: + for file, cfile in directory_dict['dirs'].items(): full_path = os.path.join(pgdata, file) - directory_dict['dirs'][file]['mode'] = os.stat( - full_path).st_mode + cfile.mode = os.stat(full_path).st_mode - for file in directory_dict['files']: + for file, cdir in directory_dict['files'].items(): full_path = os.path.join(pgdata, file) - directory_dict['files'][file]['mode'] = os.stat( - full_path).st_mode + cdir.mode = os.stat(full_path).st_mode return directory_dict @@ -1852,123 +1849,117 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict error_message = 'Restored PGDATA is not equal to original!\n' # Compare directories - for directory in restored_pgdata['dirs']: - if directory not in original_pgdata['dirs']: + restored_dirs = set(restored_pgdata['dirs']) + original_dirs = set(restored_pgdata['dirs']) + + for directory in sorted(restored_dirs - original_dirs): + fail = True + error_message += '\nDirectory was not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in sorted(original_dirs - restored_dirs): + fail = True + error_message += '\nDirectory dissappeared' + error_message += ' in restored PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in sorted(original_dirs & restored_dirs): + original = original_pgdata['dirs'][directory] + restored = restored_pgdata['dirs'][directory] + if original.mode != restored.mode: fail = True - error_message += '\nDirectory was not present' - error_message += ' in original PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory)) - else: - if ( - restored_pgdata['dirs'][directory]['mode'] != - original_pgdata['dirs'][directory]['mode'] - ): - fail = True - error_message += '\nDir permissions mismatch:\n' - error_message += ' Dir old: {0} Permissions: {1}\n'.format( - os.path.join(original_pgdata['pgdata'], directory), - original_pgdata['dirs'][directory]['mode']) - error_message += ' Dir new: {0} Permissions: {1}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory), - restored_pgdata['dirs'][directory]['mode']) - - for directory in original_pgdata['dirs']: - if directory not in restored_pgdata['dirs']: - fail = True - error_message += '\nDirectory dissappeared' - error_message += ' in restored PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory)) - - for file in restored_pgdata['files']: + error_message += '\nDir permissions mismatch:\n' + error_message += ' Dir old: {0} Permissions: {1}\n'.format( + os.path.join(original_pgdata['pgdata'], directory), + original.mode) + error_message += ' Dir new: {0} Permissions: {1}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory), + restored.mode) + + restored_files = set(restored_pgdata['files']) + original_files = set(restored_pgdata['files']) + + for file in sorted(restored_files - original_files): # File is present in RESTORED PGDATA # but not present in ORIGINAL # only backup_label is allowed - if file not in original_pgdata['files']: - fail = True - error_message += '\nFile is not present' - error_message += ' in original PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], file)) - - for file in original_pgdata['files']: - if file in restored_pgdata['files']: + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in sorted(original_files - restored_files): + error_message += ( + '\nFile disappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True - if ( - restored_pgdata['files'][file]['mode'] != - original_pgdata['files'][file]['mode'] - ): + for file in sorted(original_files & restored_files): + original = original_pgdata['files'][file] + restored = restored_pgdata['files'][file] + if restored.mode != original.mode: + fail = True + error_message += '\nFile permissions mismatch:\n' + error_message += ' File_old: {0} Permissions: {1:o}\n'.format( + os.path.join(original_pgdata['pgdata'], file), + original.mode) + error_message += ' File_new: {0} Permissions: {1:o}\n'.format( + os.path.join(restored_pgdata['pgdata'], file), + restored.mode) + + if original.md5 != restored.md5: + if file not in exclusion_dict: fail = True - error_message += '\nFile permissions mismatch:\n' - error_message += ' File_old: {0} Permissions: {1:o}\n'.format( + error_message += ( + '\nFile Checksum mismatch.\n' + 'File_old: {0}\nChecksum_old: {1}\n' + 'File_new: {2}\nChecksum_new: {3}\n').format( os.path.join(original_pgdata['pgdata'], file), - original_pgdata['files'][file]['mode']) - error_message += ' File_new: {0} Permissions: {1:o}\n'.format( + original.md5, os.path.join(restored_pgdata['pgdata'], file), - restored_pgdata['files'][file]['mode']) + restored.md5 + ) - if ( - original_pgdata['files'][file]['md5'] != - restored_pgdata['files'][file]['md5'] - ): - if file not in exclusion_dict: - fail = True - error_message += ( - '\nFile Checksum mismatch.\n' - 'File_old: {0}\nChecksum_old: {1}\n' - 'File_new: {2}\nChecksum_new: {3}\n').format( - os.path.join(original_pgdata['pgdata'], file), - original_pgdata['files'][file]['md5'], - os.path.join(restored_pgdata['pgdata'], file), - restored_pgdata['files'][file]['md5'] - ) + if not original.is_datafile: + continue - if original_pgdata['files'][file]['is_datafile']: - for page in original_pgdata['files'][file]['md5_per_page']: - if page not in restored_pgdata['files'][file]['md5_per_page']: - error_message += ( - '\n Page {0} dissappeared.\n ' - 'File: {1}\n').format( - page, - os.path.join( - restored_pgdata['pgdata'], - file - ) - ) - continue - - if not (file in exclusion_dict and page in exclusion_dict[file]): - if ( - original_pgdata['files'][file]['md5_per_page'][page] != - restored_pgdata['files'][file]['md5_per_page'][page] - ): - fail = True - error_message += ( - '\n Page checksum mismatch: {0}\n ' - ' PAGE Checksum_old: {1}\n ' - ' PAGE Checksum_new: {2}\n ' - ' File: {3}\n' - ).format( - page, - original_pgdata['files'][file][ - 'md5_per_page'][page], - restored_pgdata['files'][file][ - 'md5_per_page'][page], - os.path.join( - restored_pgdata['pgdata'], file) - ) - for page in restored_pgdata['files'][file]['md5_per_page']: - if page not in original_pgdata['files'][file]['md5_per_page']: - error_message += '\n Extra page {0}\n File: {1}\n'.format( - page, - os.path.join( - restored_pgdata['pgdata'], file)) + original_pages = set(original.md5_per_page) + restored_pages = set(restored.md5_per_page) - else: - error_message += ( - '\nFile disappearance.\n ' - 'File: {0}\n').format( - os.path.join(restored_pgdata['pgdata'], file) + for page in sorted(original_pages - restored_pages): + error_message += '\n Page {0} dissappeared.\n File: {1}\n'.format( + page, + os.path.join(restored_pgdata['pgdata'], file) ) - fail = True + + + for page in sorted(restored_pages - original_pages): + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join(restored_pgdata['pgdata'], file)) + + for page in sorted(original_pages & restored_pages): + if file in exclusion_dict and page in exclusion_dict[file]: + continue + + if original.md5_per_page[page] != restored.md5_per_page[page]: + fail = True + error_message += ( + '\n Page checksum mismatch: {0}\n ' + ' PAGE Checksum_old: {1}\n ' + ' PAGE Checksum_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original.md5_per_page[page], + restored.md5_per_page[page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + self.assertFalse(fail, error_message) def gdb_attach(self, pid): @@ -2221,3 +2212,10 @@ def _execute(self, cmd, running=True): # if running and line.startswith('*running'): break return output +class ContentFile(object): + __slots__ = ('is_datafile', 'mode', 'md5', 'md5_per_page') + def __init__(self, is_datafile: bool): + self.is_datafile = is_datafile + +class ContentDir(object): + __slots__ = ('mode') \ No newline at end of file From 440441dc6fdb0562775c400e6109912ff371cd70 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 11:26:54 +0300 Subject: [PATCH 1986/2107] ... fix names --- tests/helpers/ptrack_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index fc193fba4..ab164855a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1811,13 +1811,13 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): directory_dict['dirs'][directory_relpath] = ContentDir() # get permissions for every file and directory - for file, cfile in directory_dict['dirs'].items(): - full_path = os.path.join(pgdata, file) - cfile.mode = os.stat(full_path).st_mode + for dir, cdir in directory_dict['dirs'].items(): + full_path = os.path.join(pgdata, dir) + cdir.mode = os.stat(full_path).st_mode - for file, cdir in directory_dict['files'].items(): + for file, cfile in directory_dict['files'].items(): full_path = os.path.join(pgdata, file) - cdir.mode = os.stat(full_path).st_mode + cfile.mode = os.stat(full_path).st_mode return directory_dict From d0494662875b9d324e15c6fa346da8e29858239b Mon Sep 17 00:00:00 2001 From: Viktoria Shepard Date: Fri, 25 Nov 2022 12:57:43 +0300 Subject: [PATCH 1987/2107] PBCKP-306 add '_test' to test files --- ...CVE_2018_1058.py => CVE_2018_1058_test.py} | 0 tests/__init__.py | 86 +++++++++---------- tests/{archive.py => archive_test.py} | 0 tests/{backup.py => backup_test.py} | 0 tests/{catchup.py => catchup_test.py} | 0 tests/{cfs_backup.py => cfs_backup_test.py} | 0 tests/{cfs_catchup.py => cfs_catchup_test.py} | 0 tests/{cfs_restore.py => cfs_restore_test.py} | 0 ..._backup.py => cfs_validate_backup_test.py} | 0 tests/{checkdb.py => checkdb_test.py} | 0 ...compatibility.py => compatibility_test.py} | 0 tests/{compression.py => compression_test.py} | 0 tests/{config.py => config_test.py} | 0 tests/{delete.py => delete_test.py} | 0 tests/{delta.py => delta_test.py} | 0 tests/{exclude.py => exclude_test.py} | 0 tests/{external.py => external_test.py} | 0 ...lse_positive.py => false_positive_test.py} | 0 .../{incr_restore.py => incr_restore_test.py} | 0 tests/{init.py => init_test.py} | 0 tests/{locking.py => locking_test.py} | 0 tests/{logging.py => logging_test.py} | 0 tests/{merge.py => merge_test.py} | 0 tests/{option.py => option_test.py} | 0 tests/{page.py => page_test.py} | 0 tests/{pgpro2068.py => pgpro2068_test.py} | 0 tests/{pgpro560.py => pgpro560_test.py} | 0 tests/{pgpro589.py => pgpro589_test.py} | 0 tests/{ptrack.py => ptrack_test.py} | 0 tests/{remote.py => remote_test.py} | 0 tests/{replica.py => replica_test.py} | 0 tests/{restore.py => restore_test.py} | 0 tests/{retention.py => retention_test.py} | 0 tests/{set_backup.py => set_backup_test.py} | 0 tests/{show.py => show_test.py} | 0 ...me_consuming.py => time_consuming_test.py} | 0 tests/{time_stamp.py => time_stamp_test.py} | 0 tests/{validate.py => validate_test.py} | 0 38 files changed, 43 insertions(+), 43 deletions(-) rename tests/{CVE_2018_1058.py => CVE_2018_1058_test.py} (100%) rename tests/{archive.py => archive_test.py} (100%) rename tests/{backup.py => backup_test.py} (100%) rename tests/{catchup.py => catchup_test.py} (100%) rename tests/{cfs_backup.py => cfs_backup_test.py} (100%) rename tests/{cfs_catchup.py => cfs_catchup_test.py} (100%) rename tests/{cfs_restore.py => cfs_restore_test.py} (100%) rename tests/{cfs_validate_backup.py => cfs_validate_backup_test.py} (100%) rename tests/{checkdb.py => checkdb_test.py} (100%) rename tests/{compatibility.py => compatibility_test.py} (100%) rename tests/{compression.py => compression_test.py} (100%) rename tests/{config.py => config_test.py} (100%) rename tests/{delete.py => delete_test.py} (100%) rename tests/{delta.py => delta_test.py} (100%) rename tests/{exclude.py => exclude_test.py} (100%) rename tests/{external.py => external_test.py} (100%) rename tests/{false_positive.py => false_positive_test.py} (100%) rename tests/{incr_restore.py => incr_restore_test.py} (100%) rename tests/{init.py => init_test.py} (100%) rename tests/{locking.py => locking_test.py} (100%) rename tests/{logging.py => logging_test.py} (100%) rename tests/{merge.py => merge_test.py} (100%) rename tests/{option.py => option_test.py} (100%) rename tests/{page.py => page_test.py} (100%) rename tests/{pgpro2068.py => pgpro2068_test.py} (100%) rename tests/{pgpro560.py => pgpro560_test.py} (100%) rename tests/{pgpro589.py => pgpro589_test.py} (100%) rename tests/{ptrack.py => ptrack_test.py} (100%) rename tests/{remote.py => remote_test.py} (100%) rename tests/{replica.py => replica_test.py} (100%) rename tests/{restore.py => restore_test.py} (100%) rename tests/{retention.py => retention_test.py} (100%) rename tests/{set_backup.py => set_backup_test.py} (100%) rename tests/{show.py => show_test.py} (100%) rename tests/{time_consuming.py => time_consuming_test.py} (100%) rename tests/{time_stamp.py => time_stamp_test.py} (100%) rename tests/{validate.py => validate_test.py} (100%) diff --git a/tests/CVE_2018_1058.py b/tests/CVE_2018_1058_test.py similarity index 100% rename from tests/CVE_2018_1058.py rename to tests/CVE_2018_1058_test.py diff --git a/tests/__init__.py b/tests/__init__.py index 40d5faf65..c8d2c70c3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,13 +1,13 @@ import unittest import os -from . import init, merge, option, show, compatibility, \ - backup, delete, delta, restore, validate, \ - retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ - compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ - cfs_validate_backup, auth_test, time_stamp, logging, \ - locking, remote, external, config, checkdb, set_backup, incr_restore, \ - catchup, CVE_2018_1058, time_consuming +from . import init_test, merge_test, option_test, show_test, compatibility_test, \ + backup_test, delete_test, delta_test, restore_test, validate_test, \ + retention_test, pgpro560_test, pgpro589_test, pgpro2068_test, false_positive_test, replica_test, \ + compression_test, page_test, ptrack_test, archive_test, exclude_test, cfs_backup_test, cfs_restore_test, \ + cfs_validate_backup_test, auth_test, time_stamp_test, logging_test, \ + locking_test, remote_test, external_test, config_test, checkdb_test, set_backup_test, incr_restore_test, \ + catchup_test, CVE_2018_1058_test, time_consuming_test def load_tests(loader, tests, pattern): @@ -19,50 +19,50 @@ def load_tests(loader, tests, pattern): if 'PG_PROBACKUP_PTRACK' in os.environ: if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': - suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_test)) # PG_PROBACKUP_LONG section for tests that are long # by design e.g. they contain loops, sleeps and so on if 'PG_PROBACKUP_LONG' in os.environ: if os.environ['PG_PROBACKUP_LONG'] == 'ON': - suite.addTests(loader.loadTestsFromModule(time_consuming)) + suite.addTests(loader.loadTestsFromModule(time_consuming_test)) suite.addTests(loader.loadTestsFromModule(auth_test)) - suite.addTests(loader.loadTestsFromModule(archive)) - suite.addTests(loader.loadTestsFromModule(backup)) - suite.addTests(loader.loadTestsFromModule(catchup)) + suite.addTests(loader.loadTestsFromModule(archive_test)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(catchup_test)) if 'PGPROBACKUPBIN_OLD' in os.environ and os.environ['PGPROBACKUPBIN_OLD']: - suite.addTests(loader.loadTestsFromModule(compatibility)) - suite.addTests(loader.loadTestsFromModule(checkdb)) - suite.addTests(loader.loadTestsFromModule(config)) - suite.addTests(loader.loadTestsFromModule(cfs_backup)) - suite.addTests(loader.loadTestsFromModule(cfs_restore)) - suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) - suite.addTests(loader.loadTestsFromModule(compression)) - suite.addTests(loader.loadTestsFromModule(delete)) - suite.addTests(loader.loadTestsFromModule(delta)) - suite.addTests(loader.loadTestsFromModule(exclude)) - suite.addTests(loader.loadTestsFromModule(external)) - suite.addTests(loader.loadTestsFromModule(false_positive)) - suite.addTests(loader.loadTestsFromModule(init)) - suite.addTests(loader.loadTestsFromModule(incr_restore)) - suite.addTests(loader.loadTestsFromModule(locking)) - suite.addTests(loader.loadTestsFromModule(logging)) - suite.addTests(loader.loadTestsFromModule(merge)) - suite.addTests(loader.loadTestsFromModule(option)) - suite.addTests(loader.loadTestsFromModule(page)) - suite.addTests(loader.loadTestsFromModule(pgpro560)) - suite.addTests(loader.loadTestsFromModule(pgpro589)) - suite.addTests(loader.loadTestsFromModule(pgpro2068)) - suite.addTests(loader.loadTestsFromModule(remote)) - suite.addTests(loader.loadTestsFromModule(replica)) - suite.addTests(loader.loadTestsFromModule(restore)) - suite.addTests(loader.loadTestsFromModule(retention)) - suite.addTests(loader.loadTestsFromModule(set_backup)) - suite.addTests(loader.loadTestsFromModule(show)) - suite.addTests(loader.loadTestsFromModule(time_stamp)) - suite.addTests(loader.loadTestsFromModule(validate)) - suite.addTests(loader.loadTestsFromModule(CVE_2018_1058)) + suite.addTests(loader.loadTestsFromModule(compatibility_test)) + suite.addTests(loader.loadTestsFromModule(checkdb_test)) + suite.addTests(loader.loadTestsFromModule(config_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_restore_test)) + suite.addTests(loader.loadTestsFromModule(cfs_validate_backup_test)) + suite.addTests(loader.loadTestsFromModule(compression_test)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(delta_test)) + suite.addTests(loader.loadTestsFromModule(exclude_test)) + suite.addTests(loader.loadTestsFromModule(external_test)) + suite.addTests(loader.loadTestsFromModule(false_positive_test)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(incr_restore_test)) + suite.addTests(loader.loadTestsFromModule(locking_test)) + suite.addTests(loader.loadTestsFromModule(logging_test)) + suite.addTests(loader.loadTestsFromModule(merge_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560_test)) + suite.addTests(loader.loadTestsFromModule(pgpro589_test)) + suite.addTests(loader.loadTestsFromModule(pgpro2068_test)) + suite.addTests(loader.loadTestsFromModule(remote_test)) + suite.addTests(loader.loadTestsFromModule(replica_test)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(set_backup_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(time_stamp_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(CVE_2018_1058_test)) return suite diff --git a/tests/archive.py b/tests/archive_test.py similarity index 100% rename from tests/archive.py rename to tests/archive_test.py diff --git a/tests/backup.py b/tests/backup_test.py similarity index 100% rename from tests/backup.py rename to tests/backup_test.py diff --git a/tests/catchup.py b/tests/catchup_test.py similarity index 100% rename from tests/catchup.py rename to tests/catchup_test.py diff --git a/tests/cfs_backup.py b/tests/cfs_backup_test.py similarity index 100% rename from tests/cfs_backup.py rename to tests/cfs_backup_test.py diff --git a/tests/cfs_catchup.py b/tests/cfs_catchup_test.py similarity index 100% rename from tests/cfs_catchup.py rename to tests/cfs_catchup_test.py diff --git a/tests/cfs_restore.py b/tests/cfs_restore_test.py similarity index 100% rename from tests/cfs_restore.py rename to tests/cfs_restore_test.py diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup_test.py similarity index 100% rename from tests/cfs_validate_backup.py rename to tests/cfs_validate_backup_test.py diff --git a/tests/checkdb.py b/tests/checkdb_test.py similarity index 100% rename from tests/checkdb.py rename to tests/checkdb_test.py diff --git a/tests/compatibility.py b/tests/compatibility_test.py similarity index 100% rename from tests/compatibility.py rename to tests/compatibility_test.py diff --git a/tests/compression.py b/tests/compression_test.py similarity index 100% rename from tests/compression.py rename to tests/compression_test.py diff --git a/tests/config.py b/tests/config_test.py similarity index 100% rename from tests/config.py rename to tests/config_test.py diff --git a/tests/delete.py b/tests/delete_test.py similarity index 100% rename from tests/delete.py rename to tests/delete_test.py diff --git a/tests/delta.py b/tests/delta_test.py similarity index 100% rename from tests/delta.py rename to tests/delta_test.py diff --git a/tests/exclude.py b/tests/exclude_test.py similarity index 100% rename from tests/exclude.py rename to tests/exclude_test.py diff --git a/tests/external.py b/tests/external_test.py similarity index 100% rename from tests/external.py rename to tests/external_test.py diff --git a/tests/false_positive.py b/tests/false_positive_test.py similarity index 100% rename from tests/false_positive.py rename to tests/false_positive_test.py diff --git a/tests/incr_restore.py b/tests/incr_restore_test.py similarity index 100% rename from tests/incr_restore.py rename to tests/incr_restore_test.py diff --git a/tests/init.py b/tests/init_test.py similarity index 100% rename from tests/init.py rename to tests/init_test.py diff --git a/tests/locking.py b/tests/locking_test.py similarity index 100% rename from tests/locking.py rename to tests/locking_test.py diff --git a/tests/logging.py b/tests/logging_test.py similarity index 100% rename from tests/logging.py rename to tests/logging_test.py diff --git a/tests/merge.py b/tests/merge_test.py similarity index 100% rename from tests/merge.py rename to tests/merge_test.py diff --git a/tests/option.py b/tests/option_test.py similarity index 100% rename from tests/option.py rename to tests/option_test.py diff --git a/tests/page.py b/tests/page_test.py similarity index 100% rename from tests/page.py rename to tests/page_test.py diff --git a/tests/pgpro2068.py b/tests/pgpro2068_test.py similarity index 100% rename from tests/pgpro2068.py rename to tests/pgpro2068_test.py diff --git a/tests/pgpro560.py b/tests/pgpro560_test.py similarity index 100% rename from tests/pgpro560.py rename to tests/pgpro560_test.py diff --git a/tests/pgpro589.py b/tests/pgpro589_test.py similarity index 100% rename from tests/pgpro589.py rename to tests/pgpro589_test.py diff --git a/tests/ptrack.py b/tests/ptrack_test.py similarity index 100% rename from tests/ptrack.py rename to tests/ptrack_test.py diff --git a/tests/remote.py b/tests/remote_test.py similarity index 100% rename from tests/remote.py rename to tests/remote_test.py diff --git a/tests/replica.py b/tests/replica_test.py similarity index 100% rename from tests/replica.py rename to tests/replica_test.py diff --git a/tests/restore.py b/tests/restore_test.py similarity index 100% rename from tests/restore.py rename to tests/restore_test.py diff --git a/tests/retention.py b/tests/retention_test.py similarity index 100% rename from tests/retention.py rename to tests/retention_test.py diff --git a/tests/set_backup.py b/tests/set_backup_test.py similarity index 100% rename from tests/set_backup.py rename to tests/set_backup_test.py diff --git a/tests/show.py b/tests/show_test.py similarity index 100% rename from tests/show.py rename to tests/show_test.py diff --git a/tests/time_consuming.py b/tests/time_consuming_test.py similarity index 100% rename from tests/time_consuming.py rename to tests/time_consuming_test.py diff --git a/tests/time_stamp.py b/tests/time_stamp_test.py similarity index 100% rename from tests/time_stamp.py rename to tests/time_stamp_test.py diff --git a/tests/validate.py b/tests/validate_test.py similarity index 100% rename from tests/validate.py rename to tests/validate_test.py From be949fd91e9ef37d8e68b2cec8210348939b0a29 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 25 Nov 2022 16:23:56 +0300 Subject: [PATCH 1988/2107] fix memory leak in config_get_opt --- src/utils/configuration.c | 3 +++ src/utils/pgut.c | 6 ++++++ src/utils/pgut.h | 1 + 3 files changed, 10 insertions(+) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 93f29c488..193d1c680 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -538,6 +538,9 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], assign_option(opt, optarg, SOURCE_CMD); } + pgut_free(optstring); + pgut_free(longopts); + return optind; } diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 2cf0ccbe7..6123c18d8 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -993,6 +993,12 @@ pgut_str_strip_trailing_filename(const char *filepath, const char *filename) return pgut_strndup(filepath, fp_len); } +void +pgut_free(void *p) +{ + free(p); +} + FILE * pgut_fopen(const char *path, const char *mode, bool missing_ok) { diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 116ee41c0..f8554f9d0 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -64,6 +64,7 @@ extern void *pgut_realloc(void *p, size_t size); extern char *pgut_strdup(const char *str); extern char *pgut_strndup(const char *str, size_t n); extern char *pgut_str_strip_trailing_filename(const char *filepath, const char *filename); +extern void pgut_free(void *p); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) #define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) From 15a5c5dad7fcb441883046e4caa1f01089f35005 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 29 Nov 2022 11:12:20 +0300 Subject: [PATCH 1989/2107] [PBCKP-354] Pg15: continue reading if error "missing contrecord" is met. Pg15 now reports if it didn't met expected contrecord. Absence of this message was long standing bug in previous postgres versions. This situation could happen if WAL segment was rewritten after restart. It causes "tests.validate.ValidateTest.test_validate_wal_unreal_values" to hang but (looks like) for other reason: test tries to read "in future". Probably we should stop reading logs here. But since we always did continue here, lets continue as well. --- src/parsexlog.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index f12aae904..bcdd814d6 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1443,7 +1443,14 @@ XLogThreadWorker(void *arg) * Usually SimpleXLogPageRead() does it by itself. But here we need * to do it manually to support threads. */ - if (reader_data->need_switch && errormsg == NULL) + if (reader_data->need_switch && ( + errormsg == NULL || + /* + * Pg15 now informs if "contrecord" is missing. + * TODO: probably we should abort reading logs at this moment. + * But we continue as we did with bug present in Pg < 15. + */ + strncmp(errormsg, "missing contrecord", 18) == 0)) { if (SwitchThreadToNextWal(xlogreader, thread_arg)) continue; From 0a5fc87dbd704e0ee5e430b0406bb0328c553055 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 29 Nov 2022 15:12:32 +0300 Subject: [PATCH 1990/2107] [PBCKP-360] fix exception check in test_validate_corrupt_page_header_map zlib decompression could mark error either as "data error" or "buffer error". One of check did consider it, other didn't. Make them same. And use `assertRaises` for good (requires python 3.2 at least) --- tests/validate.py | 51 +++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/tests/validate.py b/tests/validate.py index 966ad81a8..a6776d571 100644 --- a/tests/validate.py +++ b/tests/validate.py @@ -4007,43 +4007,34 @@ def test_validate_corrupt_page_header_map(self): f.seek(42) f.write(b"blah") f.flush() - f.close - try: + with self.assertRaises(ProbackupException) as cm: self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + e = cm.exception + self.assertRegex( + cm.exception.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - try: + self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + + with self.assertRaises(ProbackupException) as cm: self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + e = cm.exception + self.assertRegex( + e.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.assertIn("WARNING: Some backups are not valid", e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + + self.assertIn("WARNING: Some backups are not valid", e.message) # Clean after yourself self.del_test_dir(module_name, fname) From 3b72dd66af0bae871184cbf969c18e70b1b0b4b8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 29 Nov 2022 16:24:49 +0300 Subject: [PATCH 1991/2107] fix github tests.init_test --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab1a5888d..6f99d0f27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,7 +87,7 @@ jobs: If (!$Env:MODE -Or $Env:MODE -Eq "basic") { $Env:PG_PROBACKUP_TEST_BASIC = "ON" python -m unittest -v tests - python -m unittest -v tests.init + python -m unittest -v tests.init_test } else { python -m unittest -v tests.$Env:MODE } From 3b2efe63a23363fe3b6fea9c7c441aede39a6581 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 30 Nov 2022 14:58:36 +0300 Subject: [PATCH 1992/2107] and again try fix travis tests.init_test --- travis/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/run_tests.sh b/travis/run_tests.sh index 1823b05de..84d7aa173 100755 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -111,7 +111,7 @@ if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests - python3 -m unittest -v tests.init + python3 -m unittest -v tests.init_test else echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests.$MODE From 7e59a19df1f71796d0e154e259732f7ac8c4a4c3 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Wed, 30 Nov 2022 15:56:41 +0300 Subject: [PATCH 1993/2107] [DOC] {PBCKP-320] Remove duplicate descriptions of backup/catchup modes [skip-travis] --- doc/pgprobackup.xml | 121 +++++++++----------------------------------- 1 file changed, 23 insertions(+), 98 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 6babf00f7..7c8610681 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -312,7 +312,7 @@ doc/src/sgml/pgprobackup.sgml - + FULL backups contain all the data files required to restore the database cluster. @@ -328,7 +328,7 @@ doc/src/sgml/pgprobackup.sgml - + DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that have changed since the previous backup. This @@ -337,7 +337,7 @@ doc/src/sgml/pgprobackup.sgml - + PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups @@ -352,7 +352,7 @@ doc/src/sgml/pgprobackup.sgml - + PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, @@ -443,7 +443,7 @@ doc/src/sgml/pgprobackup.sgml parameters and have the same major release number. Depending on cluster configuration, PostgreSQL itself may apply additional restrictions, such as CPU architecture - or libc/libicu versions. + or libc/icu versions. @@ -1274,36 +1274,11 @@ pg_probackup backup -B backup_dir --instance Where backup_mode can take one of the following values: + FULL, + DELTA, + PAGE, and + PTRACK. - - - - FULL — creates a full backup that contains all the data - files of the cluster to be restored. - - - - - DELTA — reads all data files in the data directory and - creates an incremental backup for pages that have changed - since the previous backup. - - - - - PAGE — creates an incremental backup based on the WAL - files that have been generated since the previous full or - incremental backup was taken. Only changed blocks are read - from data files. - - - - - PTRACK — creates an incremental backup tracking page - changes on the fly. - - - When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the @@ -3532,25 +3507,25 @@ pg_probackup catchup -b catchup_mode --source-pgdata= Where catchup_mode can take one of the - following values: FULL, DELTA, or PTRACK. + following values: - - FULL — creates a full copy of the PostgreSQL instance. + + FULL — creates a full copy of the PostgreSQL instance. The data directory of the destination instance must be empty for this mode. - - DELTA — reads all data files in the data directory and + + DELTA — reads all data files in the data directory and creates an incremental copy for pages that have changed since the destination instance was shut down. - - PTRACK — tracking page changes on the fly, + + PTRACK — tracking page changes on the fly, only reads and copies pages that have changed since the point of divergence of the source and destination instances. @@ -3817,35 +3792,10 @@ pg_probackup backup -B backup_dir -b bac Specifies the backup mode to use. Possible values are: - - - - - FULL — creates a full backup that contains all the data - files of the cluster to be restored. - - - - - DELTA — reads all data files in the data directory and - creates an incremental backup for pages that have changed - since the previous backup. - - - - - PAGE — creates an incremental PAGE backup based on the WAL - files that have changed since the previous full or - incremental backup was taken. - - - - - PTRACK — creates an incremental PTRACK backup tracking - page changes on the fly. - - - + FULL, + DELTA, + PAGE, and + PTRACK. @@ -4540,34 +4490,9 @@ pg_probackup catchup -b catchup_mode Specifies the catchup mode to use. Possible values are: - - - - - FULL — creates a full copy of the PostgreSQL instance. - - - - - DELTA — reads all data files in the data directory and - creates an incremental copy for pages that have changed - since the destination instance was shut down. - - - - - PTRACK — tracking page changes on the fly, - only reads and copies pages that have changed since the point of divergence - of the source and destination instances. - - - PTRACK catchup mode requires PTRACK - not earlier than 2.0 and hence, PostgreSQL not earlier than 11. - - - - - + FULL, + DELTA, and + PTRACK. From 8fa063f688c8de69e785ad0188feae36586b7a30 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 30 Nov 2022 16:34:00 +0300 Subject: [PATCH 1994/2107] travis: and backup_test --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8315f7842..17e6d2579 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,8 +35,8 @@ env: - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE - - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup - - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup_stream + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup_stream # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup # - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb From 619816012a8c71ab7d6362d60947f97066b9a98d Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 30 Nov 2022 21:50:20 +0300 Subject: [PATCH 1995/2107] fix ArchiveTest.test_pgpro434_4 for Pg15 --- tests/archive_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/archive_test.py b/tests/archive_test.py index 5e59dd268..f6cd50a9f 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -317,7 +317,10 @@ def test_pgpro434_4(self): os.environ["PGAPPNAME"] = "pg_probackup" postgres_gdb = self.gdb_attach(pid) - postgres_gdb.set_breakpoint('do_pg_stop_backup') + if self.get_version(node) < 150000: + postgres_gdb.set_breakpoint('do_pg_stop_backup') + else: + postgres_gdb.set_breakpoint('do_pg_backup_stop') postgres_gdb.continue_execution_until_running() gdb.continue_execution_until_exit() From 9bcefb2569c33e58e623bdcb9c8a991d1721126f Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 2 Dec 2022 15:10:22 +0300 Subject: [PATCH 1996/2107] [PBCKP-327] test_ptrack_multiple_segments: try to avoid memory consumption --- tests/helpers/ptrack_helpers.py | 28 ++++++++++++++++++++++++++++ tests/ptrack_test.py | 10 ++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index ab164855a..2a4d4c271 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -15,6 +15,8 @@ from time import sleep import re import json +from hashlib import md5 +import random idx_ptrack = { 't_heap': { @@ -200,6 +202,32 @@ def kill(self, someone = None): os.kill(self.auxiliary_pids[someone][0], sig) self.is_started = False + def table_checksum(self, table, sort, dbname="postgres"): + curname = "cur_"+str(random.randint(0,2**48)) + + sum = md5(b"\x01") + + con = self.connect(dbname=dbname) + + con.execute(f""" + DECLARE {curname} NO SCROLL CURSOR FOR + SELECT t::text FROM {table} as t ORDER BY {sort}; + """) + + while True: + rows = con.execute(f"FETCH FORWARD 10000 FROM {curname}") + if not rows: + break + for row in rows: + sum.update(row[0].encode('utf8')) + sum.update(b'\x00') + + con.execute(f"CLOSE {curname}; ROLLBACK;") + + con.close() + sum.update(b'\x02') + return sum.hexdigest() + class ProbackupTest(object): # Class attributes enterprise = is_enterprise() diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 6e5786f8c..ed4498a61 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -2039,6 +2039,8 @@ def test_ptrack_multiple_segments(self): # CREATE TABLE node.pgbench_init(scale=100, options=['--tablespace=somedata']) + result = node.table_checksum("pgbench_accounts", "aid", + dbname="postgres") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -2075,7 +2077,8 @@ def test_ptrack_multiple_segments(self): # GET LOGICAL CONTENT FROM NODE # it`s stupid, because hint`s are ignored by ptrack - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts", "aid", + dbname="postgres") # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) @@ -2108,9 +2111,8 @@ def test_ptrack_multiple_segments(self): restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", - "select * from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts", "aid", + dbname="postgres") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') From 8d8a92c1d14e7ddeb98a671fbbe1ce2e0e590cf9 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 2 Dec 2022 16:55:38 +0300 Subject: [PATCH 1997/2107] tests: more usages for table_checksum and reduce batch twice for sanity --- tests/archive_test.py | 8 ++------ tests/cfs_backup_test.py | 28 ++++++++++++---------------- tests/helpers/ptrack_helpers.py | 2 +- tests/page_test.py | 6 ++---- tests/ptrack_test.py | 6 ++---- tests/replica_test.py | 4 ++-- 6 files changed, 21 insertions(+), 33 deletions(-) diff --git a/tests/archive_test.py b/tests/archive_test.py index f6cd50a9f..1bf8a1d45 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -2215,9 +2215,7 @@ def test_multi_timeline_recovery_prefetching(self): node.slow_start() node.pgbench_init(scale=20) - result = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') + result = node.table_checksum("pgbench_accounts", "aid") node.stop() node.cleanup() @@ -2242,9 +2240,7 @@ def test_multi_timeline_recovery_prefetching(self): node.slow_start() - result_new = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') + result_new = node.table_checksum("pgbench_accounts", "aid") self.assertEqual(result, result_new) diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py index 28ef275df..adfcaef19 100644 --- a/tests/cfs_backup_test.py +++ b/tests/cfs_backup_test.py @@ -761,7 +761,7 @@ def test_multiple_segments(self): 't_heap', tblspace_name) ) - full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = self.node.table_checksum("t_heap", "id") try: backup_id_full = self.backup_node( @@ -783,7 +783,7 @@ def test_multiple_segments(self): 't_heap') ) - page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + page_result = self.node.table_checksum("t_heap", "id") try: backup_id_page = self.backup_node( @@ -824,7 +824,7 @@ def test_multiple_segments(self): self.node.slow_start() self.assertEqual( full_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + self.node.table_checksum("t_heap", "id"), 'Lost data after restore') # CHECK PAGE BACKUP @@ -843,7 +843,7 @@ def test_multiple_segments(self): self.node.slow_start() self.assertEqual( page_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + self.node.table_checksum("t_heap", "id"), 'Lost data after restore') # @unittest.expectedFailure @@ -877,10 +877,8 @@ def test_multiple_segments_in_multiple_tablespaces(self): "FROM generate_series(0,1005000) i".format( 't_heap_2', tblspace_name_2)) - full_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - full_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") + full_result_1 = self.node.table_checksum("t_heap_1", "id") + full_result_2 = self.node.table_checksum("t_heap_2", "id") try: backup_id_full = self.backup_node( @@ -911,10 +909,8 @@ def test_multiple_segments_in_multiple_tablespaces(self): 't_heap_2') ) - page_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - page_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") + page_result_1 = self.node.table_checksum("t_heap_1", "id") + page_result_2 = self.node.table_checksum("t_heap_2", "id") try: backup_id_page = self.backup_node( @@ -955,11 +951,11 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.assertEqual( full_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + self.node.table_checksum("t_heap_1", "id"), 'Lost data after restore') self.assertEqual( full_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + self.node.table_checksum("t_heap_2", "id"), 'Lost data after restore') # CHECK PAGE BACKUP @@ -976,11 +972,11 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.assertEqual( page_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + self.node.table_checksum("t_heap_1", "id"), 'Lost data after restore') self.assertEqual( page_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + self.node.table_checksum("t_heap_2", "id"), 'Lost data after restore') # @unittest.expectedFailure diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 2a4d4c271..6fe3d6333 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -215,7 +215,7 @@ def table_checksum(self, table, sort, dbname="postgres"): """) while True: - rows = con.execute(f"FETCH FORWARD 10000 FROM {curname}") + rows = con.execute(f"FETCH FORWARD 5000 FROM {curname}") if not rows: break for row in rows: diff --git a/tests/page_test.py b/tests/page_test.py index e77e5c827..4c5ba7f87 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -1191,8 +1191,7 @@ def test_multi_timeline_page(self): pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts", "aid") node_restored = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) @@ -1204,8 +1203,7 @@ def test_multi_timeline_page(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = node_restored.table_checksum("pgbench_accounts", "aid") self.assertEqual(result, result_new) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index ed4498a61..14688fc11 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -375,7 +375,7 @@ def test_ptrack_eat_my_data(self): self.switch_wal_segment(node) - result = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + result = node.table_checksum("pgbench_accounts", "aid") node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -396,9 +396,7 @@ def test_ptrack_eat_my_data(self): # Logical comparison self.assertEqual( result, - node_restored.safe_psql( - 'postgres', - 'SELECT * FROM pgbench_accounts'), + node.table_checksum("pgbench_accounts", "aid"), 'Data loss') # @unittest.skip("skip") diff --git a/tests/replica_test.py b/tests/replica_test.py index 9c68de366..ecc92e19f 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -326,7 +326,7 @@ def test_replica_archive_page_backup(self): self.switch_wal_segment(master) - before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + before = master.table_checksum("pgbench_accounts", "aid") self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -342,7 +342,7 @@ def test_replica_archive_page_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + after = master.table_checksum("pgbench_accounts", "aid") self.assertEqual( before, after, 'Restored data is not equal to original') From 02e3fb0477f44f58b4e05115aa41709b2ec7ad8d Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sun, 4 Dec 2022 02:26:54 +0300 Subject: [PATCH 1998/2107] tests: table_checksum needs no sorting in fact since we are compare table content exactly --- tests/archive_test.py | 4 ++-- tests/cfs_backup_test.py | 24 ++++++++++++------------ tests/helpers/ptrack_helpers.py | 23 ++++++++++------------- tests/page_test.py | 4 ++-- tests/ptrack_test.py | 13 +++++-------- tests/replica_test.py | 4 ++-- 6 files changed, 33 insertions(+), 39 deletions(-) diff --git a/tests/archive_test.py b/tests/archive_test.py index 1bf8a1d45..fb2600c4a 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -2215,7 +2215,7 @@ def test_multi_timeline_recovery_prefetching(self): node.slow_start() node.pgbench_init(scale=20) - result = node.table_checksum("pgbench_accounts", "aid") + result = node.table_checksum("pgbench_accounts") node.stop() node.cleanup() @@ -2240,7 +2240,7 @@ def test_multi_timeline_recovery_prefetching(self): node.slow_start() - result_new = node.table_checksum("pgbench_accounts", "aid") + result_new = node.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py index adfcaef19..cd2826d21 100644 --- a/tests/cfs_backup_test.py +++ b/tests/cfs_backup_test.py @@ -761,7 +761,7 @@ def test_multiple_segments(self): 't_heap', tblspace_name) ) - full_result = self.node.table_checksum("t_heap", "id") + full_result = self.node.table_checksum("t_heap") try: backup_id_full = self.backup_node( @@ -783,7 +783,7 @@ def test_multiple_segments(self): 't_heap') ) - page_result = self.node.table_checksum("t_heap", "id") + page_result = self.node.table_checksum("t_heap") try: backup_id_page = self.backup_node( @@ -824,7 +824,7 @@ def test_multiple_segments(self): self.node.slow_start() self.assertEqual( full_result, - self.node.table_checksum("t_heap", "id"), + self.node.table_checksum("t_heap"), 'Lost data after restore') # CHECK PAGE BACKUP @@ -843,7 +843,7 @@ def test_multiple_segments(self): self.node.slow_start() self.assertEqual( page_result, - self.node.table_checksum("t_heap", "id"), + self.node.table_checksum("t_heap"), 'Lost data after restore') # @unittest.expectedFailure @@ -877,8 +877,8 @@ def test_multiple_segments_in_multiple_tablespaces(self): "FROM generate_series(0,1005000) i".format( 't_heap_2', tblspace_name_2)) - full_result_1 = self.node.table_checksum("t_heap_1", "id") - full_result_2 = self.node.table_checksum("t_heap_2", "id") + full_result_1 = self.node.table_checksum("t_heap_1") + full_result_2 = self.node.table_checksum("t_heap_2") try: backup_id_full = self.backup_node( @@ -909,8 +909,8 @@ def test_multiple_segments_in_multiple_tablespaces(self): 't_heap_2') ) - page_result_1 = self.node.table_checksum("t_heap_1", "id") - page_result_2 = self.node.table_checksum("t_heap_2", "id") + page_result_1 = self.node.table_checksum("t_heap_1") + page_result_2 = self.node.table_checksum("t_heap_2") try: backup_id_page = self.backup_node( @@ -951,11 +951,11 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.assertEqual( full_result_1, - self.node.table_checksum("t_heap_1", "id"), + self.node.table_checksum("t_heap_1"), 'Lost data after restore') self.assertEqual( full_result_2, - self.node.table_checksum("t_heap_2", "id"), + self.node.table_checksum("t_heap_2"), 'Lost data after restore') # CHECK PAGE BACKUP @@ -972,11 +972,11 @@ def test_multiple_segments_in_multiple_tablespaces(self): self.assertEqual( page_result_1, - self.node.table_checksum("t_heap_1", "id"), + self.node.table_checksum("t_heap_1"), 'Lost data after restore') self.assertEqual( page_result_2, - self.node.table_checksum("t_heap_2", "id"), + self.node.table_checksum("t_heap_2"), 'Lost data after restore') # @unittest.expectedFailure diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6fe3d6333..555c0a73e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -15,7 +15,6 @@ from time import sleep import re import json -from hashlib import md5 import random idx_ptrack = { @@ -202,30 +201,28 @@ def kill(self, someone = None): os.kill(self.auxiliary_pids[someone][0], sig) self.is_started = False - def table_checksum(self, table, sort, dbname="postgres"): - curname = "cur_"+str(random.randint(0,2**48)) - - sum = md5(b"\x01") - + def table_checksum(self, table, dbname="postgres"): con = self.connect(dbname=dbname) - con.execute(f""" - DECLARE {curname} NO SCROLL CURSOR FOR - SELECT t::text FROM {table} as t ORDER BY {sort}; - """) + curname = "cur_"+str(random.randint(0,2**48)) + + con.execute(""" + DECLARE %s NO SCROLL CURSOR FOR + SELECT t::text FROM %s as t + """ % (curname, table)) + sum = hashlib.md5() while True: - rows = con.execute(f"FETCH FORWARD 5000 FROM {curname}") + rows = con.execute("FETCH FORWARD 5000 FROM %s" % curname) if not rows: break for row in rows: + # hash uses SipHash since Python3.4, therefore it is good enough sum.update(row[0].encode('utf8')) - sum.update(b'\x00') con.execute(f"CLOSE {curname}; ROLLBACK;") con.close() - sum.update(b'\x02') return sum.hexdigest() class ProbackupTest(object): diff --git a/tests/page_test.py b/tests/page_test.py index 4c5ba7f87..be6116bbe 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -1191,7 +1191,7 @@ def test_multi_timeline_page(self): pgdata = self.pgdata_content(node.data_dir) - result = node.table_checksum("pgbench_accounts", "aid") + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) @@ -1203,7 +1203,7 @@ def test_multi_timeline_page(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.table_checksum("pgbench_accounts", "aid") + result_new = node_restored.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 14688fc11..b8a4065b0 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -375,7 +375,7 @@ def test_ptrack_eat_my_data(self): self.switch_wal_segment(node) - result = node.table_checksum("pgbench_accounts", "aid") + result = node.table_checksum("pgbench_accounts") node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -396,7 +396,7 @@ def test_ptrack_eat_my_data(self): # Logical comparison self.assertEqual( result, - node.table_checksum("pgbench_accounts", "aid"), + node.table_checksum("pgbench_accounts"), 'Data loss') # @unittest.skip("skip") @@ -2037,8 +2037,7 @@ def test_ptrack_multiple_segments(self): # CREATE TABLE node.pgbench_init(scale=100, options=['--tablespace=somedata']) - result = node.table_checksum("pgbench_accounts", "aid", - dbname="postgres") + result = node.table_checksum("pgbench_accounts") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -2075,8 +2074,7 @@ def test_ptrack_multiple_segments(self): # GET LOGICAL CONTENT FROM NODE # it`s stupid, because hint`s are ignored by ptrack - result = node.table_checksum("pgbench_accounts", "aid", - dbname="postgres") + result = node.table_checksum("pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) @@ -2109,8 +2107,7 @@ def test_ptrack_multiple_segments(self): restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.table_checksum("pgbench_accounts", "aid", - dbname="postgres") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') diff --git a/tests/replica_test.py b/tests/replica_test.py index ecc92e19f..577dcd3a5 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -326,7 +326,7 @@ def test_replica_archive_page_backup(self): self.switch_wal_segment(master) - before = master.table_checksum("pgbench_accounts", "aid") + before = master.table_checksum("pgbench_accounts") self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -342,7 +342,7 @@ def test_replica_archive_page_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = master.table_checksum("pgbench_accounts", "aid") + after = master.table_checksum("pgbench_accounts") self.assertEqual( before, after, 'Restored data is not equal to original') From fc50cf0ddfc116460da742e658ad50a08775bf3f Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sun, 4 Dec 2022 04:24:55 +0300 Subject: [PATCH 1999/2107] tests: fix travis uses old image with python3.5 --- tests/helpers/ptrack_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 555c0a73e..706506432 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -220,7 +220,7 @@ def table_checksum(self, table, dbname="postgres"): # hash uses SipHash since Python3.4, therefore it is good enough sum.update(row[0].encode('utf8')) - con.execute(f"CLOSE {curname}; ROLLBACK;") + con.execute("CLOSE %s; ROLLBACK;" % curname) con.close() return sum.hexdigest() From f2f47f77345d62e127019323f0cfbd8f06e2379b Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sun, 4 Dec 2022 05:33:52 +0300 Subject: [PATCH 2000/2107] get rid of plpython usage --- tests/pgpro2068_test.py | 72 ++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/tests/pgpro2068_test.py b/tests/pgpro2068_test.py index da76a8815..04f0eb6fa 100644 --- a/tests/pgpro2068_test.py +++ b/tests/pgpro2068_test.py @@ -53,11 +53,6 @@ def test_minrecpoint_on_replica(self): replica, {'port': replica.port, 'restart_after_crash': 'off'}) - # we need those later - node.safe_psql( - "postgres", - "CREATE EXTENSION plpython3u") - node.safe_psql( "postgres", "CREATE EXTENSION pageinspect") @@ -131,48 +126,37 @@ def test_minrecpoint_on_replica(self): recovery_config, "recovery_target_action = 'pause'") replica.slow_start(replica=True) + current_xlog_lsn_query = 'SELECT pg_last_wal_replay_lsn() INTO current_xlog_lsn' if self.get_version(node) < 100000: - script = ''' -DO -$$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("SELECT min_recovery_end_location as lsn FROM pg_control_recovery()")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpython3u; -''' - else: - script = ''' + current_xlog_lsn_query = 'SELECT min_recovery_end_location INTO current_xlog_lsn FROM pg_control_recovery()' + + script = f''' DO $$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("select pg_last_wal_replay_lsn() as lsn")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpython3u; -''' +DECLARE + roid oid; + current_xlog_lsn pg_lsn; + pages_from_future RECORD; + found_corruption bool := false; +BEGIN + {current_xlog_lsn_query}; + RAISE NOTICE 'CURRENT LSN: %', current_xlog_lsn; + FOR roid IN select oid from pg_class class where relkind IN ('r', 'i', 't', 'm') and relpersistence = 'p' LOOP + FOR pages_from_future IN + with number_of_blocks as (select blknum from generate_series(0, pg_relation_size(roid) / 8192 -1) as blknum ) + select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid + from number_of_blocks, page_header(get_raw_page(roid::regclass::text, number_of_blocks.blknum::int)) + where lsn > current_xlog_lsn LOOP + RAISE NOTICE 'Found page from future. OID: %, BLKNUM: %, LSN: %', roid, pages_from_future.blknum, pages_from_future.lsn; + found_corruption := true; + END LOOP; + END LOOP; + IF found_corruption THEN + RAISE 'Found Corruption'; + END IF; +END; +$$ LANGUAGE plpgsql; +'''.format(current_xlog_lsn_query=current_xlog_lsn_query) # Find blocks from future replica.safe_psql( From 0f7e01b7f392d12a22d385e1d6241bce754b2cd8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sun, 4 Dec 2022 09:08:26 +0300 Subject: [PATCH 2001/2107] try travis without docker --- .travis.yml | 55 +++++++++++++--- travis/Dockerfile.in | 30 --------- travis/backup_restore.sh | 66 ------------------- travis/before-install.sh | 6 ++ travis/before-script-user.sh | 7 ++ travis/before-script.sh | 19 ++++++ travis/docker-compose.yml | 17 ----- travis/install.sh | 66 +++++++++++++++++++ travis/make_dockerfile.sh | 37 ----------- travis/run_tests.sh | 124 ----------------------------------- travis/script.sh | 41 ++++++++++++ 11 files changed, 185 insertions(+), 283 deletions(-) delete mode 100644 travis/Dockerfile.in delete mode 100644 travis/backup_restore.sh create mode 100755 travis/before-install.sh create mode 100755 travis/before-script-user.sh create mode 100755 travis/before-script.sh delete mode 100644 travis/docker-compose.yml create mode 100755 travis/install.sh delete mode 100755 travis/make_dockerfile.sh delete mode 100755 travis/run_tests.sh create mode 100755 travis/script.sh diff --git a/.travis.yml b/.travis.yml index 17e6d2579..074ae3d02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,60 @@ os: linux -dist: bionic +dist: jammy language: c -services: - - docker +cache: ccache + +addons: + apt: + packages: + - sudo + - libc-dev + - bison + - flex + - libreadline-dev + - zlib1g-dev + - libzstd-dev + - libssl-dev + - perl + - libperl-dev + - libdbi-perl + - cpanminus + - locales + - python3 + - python3-dev + - python3-pip + - libicu-dev + - libgss-dev + - libkrb5-dev + - libxml2-dev + - libxslt1-dev + - libldap2-dev + - tcl-dev + - diffutils + - gdb + - gettext + - lcov + - openssh-client + - openssh-server + - libipc-run-perl + - libtime-hires-perl + - libtimedate-perl + - libdbd-pg-perl before_install: - - cp travis/* . + - sudo travis/before-install.sh install: - - ./make_dockerfile.sh - - docker-compose build + - travis/install.sh + +before_script: + - sudo travis/before-script.sh + - travis/before-script-user.sh script: - - docker-compose run tests - # - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - # - docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh + - travis/script.sh notifications: email: diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in deleted file mode 100644 index a67663d3b..000000000 --- a/travis/Dockerfile.in +++ /dev/null @@ -1,30 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python3 python3-pip - -# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -# RUN python2 get-pip.py -RUN python3 -m pip install virtualenv - -# Environment -ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} -ENV PTRACK_PATCH_PG_BRANCH=${PTRACK_PATCH_PG_BRANCH} -ENV PGPROBACKUP_GDB=${PGPROBACKUP_GDB} -ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin - -# Make directories -RUN mkdir -p /pg/testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /pg/testdir -WORKDIR /pg/testdir - -# Grant privileges -RUN chown -R postgres:postgres /pg/testdir - -USER postgres -ENTRYPOINT MODE=${MODE} /run.sh diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh deleted file mode 100644 index b3c9df1ed..000000000 --- a/travis/backup_restore.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -ex - -# vars -export PGVERSION=9.5.4 -export PATH=$PATH:/usr/pgsql-9.5/bin -export PGUSER=pgbench -export PGDATABASE=pgbench -export PGDATA=/var/lib/pgsql/9.5/data -export BACKUP_PATH=/backups -export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog -export PGDATA2=/var/lib/pgsql/9.5/data2 -export PGBENCH_SCALE=100 -export PGBENCH_TIME=60 - -# prepare directory -cp -a /tests /build -pushd /build - -# download postgresql -yum install -y wget -wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz -tar xf postgresql.tar.gz - -# install pg_probackup -yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm -yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel -make top_srcdir=postgresql-$PGVERSION -make install top_srcdir=postgresql-$PGVERSION - -# initialize cluster and database -yum install -y postgresql95-server -su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" -cat < $PGDATA/pg_hba.conf -local all all trust -host all all 127.0.0.1/32 trust -local replication pgbench trust -host replication pgbench 127.0.0.1/32 trust -EOF -cat < $PGDATA/postgresql.auto.conf -max_wal_senders = 2 -wal_level = logical -wal_log_hints = on -EOF -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" -su postgres -c "createdb -U postgres $PGUSER" -su postgres -c "createuser -U postgres -a -d -E $PGUSER" -pgbench -i -s $PGBENCH_SCALE - -# Count current -COUNT=$(psql -Atc "select count(*) from pgbench_accounts") -pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & - -# create backup -pg_probackup init -pg_probackup backup -b full --disable-ptrack-clear --stream -v -pg_probackup show -sleep $PGBENCH_TIME - -# restore from backup -chown -R postgres:postgres $BACKUP_PATH -su postgres -c "pg_probackup restore -D $PGDATA2" - -# start backup server -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" -( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/travis/before-install.sh b/travis/before-install.sh new file mode 100755 index 000000000..376de5e6e --- /dev/null +++ b/travis/before-install.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -xe + +mkdir /pg +chown travis /pg \ No newline at end of file diff --git a/travis/before-script-user.sh b/travis/before-script-user.sh new file mode 100755 index 000000000..d9c07f1e4 --- /dev/null +++ b/travis/before-script-user.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -xe + +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N "" +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +ssh-keyscan -H localhost >> ~/.ssh/known_hosts diff --git a/travis/before-script.sh b/travis/before-script.sh new file mode 100755 index 000000000..ca59bcf23 --- /dev/null +++ b/travis/before-script.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -xe + +/etc/init.d/ssh start + +# Show pg_config path (just in case) +echo "############### pg_config path:" +which pg_config + +# Show pg_config just in case +echo "############### pg_config:" +pg_config + +# Show kernel parameters +echo "############### kernel params:" +cat /proc/sys/kernel/yama/ptrace_scope +sudo sysctl kernel.yama.ptrace_scope=0 +cat /proc/sys/kernel/yama/ptrace_scope diff --git a/travis/docker-compose.yml b/travis/docker-compose.yml deleted file mode 100644 index fc6545567..000000000 --- a/travis/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.7" -services: - tests: - build: - context: . - - cap_add: - - SYS_PTRACE - - security_opt: - - seccomp=unconfined - - # don't work - #sysctls: - # kernel.yama.ptrace_scope: 0 - privileged: true - diff --git a/travis/install.sh b/travis/install.sh new file mode 100755 index 000000000..43ada47b7 --- /dev/null +++ b/travis/install.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -xe + +if [ -z ${PG_VERSION+x} ]; then + echo PG_VERSION is not set! + exit 1 +fi + +if [ -z ${PG_BRANCH+x} ]; then + echo PG_BRANCH is not set! + exit 1 +fi + +if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then + PTRACK_PATCH_PG_BRANCH=OFF +fi + +# fix +sudo chown -R travis /home/travis/.ccache + +export PGHOME=/pg + +# Clone Postgres +echo "############### Getting Postgres sources:" +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 + +# Clone ptrack +if [ "$PTRACK_PATCH_PG_BRANCH" != "OFF" ]; then + git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 postgres/contrib/ptrack + export PG_PROBACKUP_PTRACK=ON +else + export PG_PROBACKUP_PTRACK=OFF +fi + +# Compile and install Postgres +echo "############### Compiling Postgres:" +cd postgres # Go to postgres dir +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then + git apply -3 contrib/ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff +fi +CC='ccache gcc' CFLAGS="-Og" ./configure --prefix=$PGHOME \ + --cache-file=~/.ccache/configure-cache \ + --enable-debug --enable-cassert --enable-depend \ + --enable-tap-tests --enable-nls +make -s -j$(nproc) install +make -s -j$(nproc) -C contrib/ install + +# Override default Postgres instance +export PATH=$PGHOME/bin:$PATH +export LD_LIBRARY_PATH=$PGHOME/lib +export PG_CONFIG=$(which pg_config) + +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then + echo "############### Compiling Ptrack:" + make -C contrib/ptrack install +fi + +# Get amcheck if missing +if [ ! -d "contrib/amcheck" ]; then + echo "############### Getting missing amcheck:" + git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck + make -C contrib/amcheck install +fi + +pip3 install testgres \ No newline at end of file diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh deleted file mode 100755 index e780649d9..000000000 --- a/travis/make_dockerfile.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_VERSION+x} ]; then - echo PG_VERSION is not set! - exit 1 -fi - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -fi - -if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then - PTRACK_PATCH_PG_BRANCH=OFF -fi - -if [ -z ${PGPROBACKUP_GDB+x} ]; then - PGPROBACKUP_GDB=ON -fi - -echo PG_VERSION=${PG_VERSION} -echo PG_BRANCH=${PG_BRANCH} -echo MODE=${MODE} -echo PTRACK_PATCH_PG_BRANCH=${PTRACK_PATCH_PG_BRANCH} -echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} - -sed \ - -e 's/${PG_VERSION}/'${PG_VERSION}/g \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ - -e 's/${PTRACK_PATCH_PG_BRANCH}/'${PTRACK_PATCH_PG_BRANCH}/g \ - -e 's/${PGPROBACKUP_GDB}/'${PGPROBACKUP_GDB}/g \ -Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh deleted file mode 100755 index 84d7aa173..000000000 --- a/travis/run_tests.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2022, Postgres Professional -# -set -xe - -sudo su -c 'mkdir /run/sshd' -sudo su -c 'apt-get update -y' -sudo su -c 'apt-get install openssh-client openssh-server -y' -sudo su -c '/etc/init.d/ssh start' - -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N "" -cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys -ssh-keyscan -H localhost >> ~/.ssh/known_hosts - -PG_SRC=$PWD/postgres - -# # Here PG_VERSION is provided by postgres:X-alpine docker image -# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2 -# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c - - -# mkdir $PG_SRC - -# tar \ -# --extract \ -# --file postgresql.tar.bz2 \ -# --directory $PG_SRC \ -# --strip-components 1 - -# Clone Postgres -echo "############### Getting Postgres sources:" -git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 - -# Clone ptrack -if [ "$PTRACK_PATCH_PG_BRANCH" != "OFF" ]; then - git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 - export PG_PROBACKUP_PTRACK=ON -else - export PG_PROBACKUP_PTRACK=OFF -fi - - -# Compile and install Postgres -echo "############### Compiling Postgres:" -cd postgres # Go to postgres dir -if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then - git apply -3 ../ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff -fi -CFLAGS="-O0" ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests --enable-nls --with-python -make -s -j$(nproc) install -#make -s -j$(nproc) -C 'src/common' install -#make -s -j$(nproc) -C 'src/port' install -#make -s -j$(nproc) -C 'src/interfaces' install -make -s -j$(nproc) -C contrib/ install - -# Override default Postgres instance -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib -export PG_CONFIG=$(which pg_config) - -if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then - echo "############### Compiling Ptrack:" - make USE_PGXS=1 -C ../ptrack install -fi - -# Get amcheck if missing -if [ ! -d "contrib/amcheck" ]; then - echo "############### Getting missing amcheck:" - git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck - make USE_PGXS=1 -C contrib/amcheck install -fi - -# Get back to testdir -cd .. - -# Show pg_config path (just in case) -echo "############### pg_config path:" -which pg_config - -# Show pg_config just in case -echo "############### pg_config:" -pg_config - -# Show kernel parameters -echo "############### kernel params:" -cat /proc/sys/kernel/yama/ptrace_scope -sudo sysctl kernel.yama.ptrace_scope=0 -cat /proc/sys/kernel/yama/ptrace_scope - -# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) -echo "############### Compiling and installing pg_probackup:" -# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install -make USE_PGXS=1 top_srcdir=$PG_SRC install - -# Setup python environment -echo "############### Setting up python env:" -python3 -m virtualenv pyenv -source pyenv/bin/activate -pip3 install testgres - -echo "############### Testing:" -echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA} -echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION} -echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD} -echo PGPROBACKUPBIN=${PGPROBACKUPBIN} -echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} -echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} -echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} -if [ "$MODE" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - python3 -m unittest -v tests - python3 -m unittest -v tests.init_test -else - echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} - python3 -m unittest -v tests.$MODE -fi - -# Generate *.gcov files -# gcov src/*.c src/*.h - -# Send coverage stats to Codecov -# bash <(curl -s https://codecov.io/bash) diff --git a/travis/script.sh b/travis/script.sh new file mode 100755 index 000000000..31ef09726 --- /dev/null +++ b/travis/script.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -xe + +export PGHOME=/pg +export PG_SRC=$PWD/postgres +export PATH=$PGHOME/bin:$PATH +export LD_LIBRARY_PATH=$PGHOME/lib +export PG_CONFIG=$(which pg_config) + +# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) +echo "############### Compiling and installing pg_probackup:" +# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install +make USE_PGXS=1 top_srcdir=$PG_SRC install + +if [ -z ${MODE+x} ]; then + MODE=basic +fi + +if [ -z ${PGPROBACKUP_GDB+x} ]; then + PGPROBACKUP_GDB=ON +fi + +echo "############### Testing:" +echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA} +echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION} +echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD} +echo PGPROBACKUPBIN=${PGPROBACKUPBIN} +echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} +echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} +echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} + +if [ "$MODE" = "basic" ]; then + export PG_PROBACKUP_TEST_BASIC=ON + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} + python3 -m unittest -v tests + python3 -m unittest -v tests.init_test +else + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} + python3 -m unittest -v tests.$MODE +fi From c42f68ecca86de49aa55fbd4c98d3099062d54b0 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Mon, 5 Dec 2022 11:21:19 +0300 Subject: [PATCH 2002/2107] [PBCKP-382] version 15 compatibility bug --- tests/auth_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index d0be9f344..65c30e6ee 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -140,7 +140,6 @@ def test_backup_via_unprivileged_user(self): else: node.safe_psql( "postgres", - "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup;") self.backup_node( From 3109634ecb4c5995cd76a2dfb122afa08e404916 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 5 Dec 2022 15:49:12 +0300 Subject: [PATCH 2003/2107] [PBCKP-325] change test_issue_231 to check backup id are different --- tests/backup_test.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index db7ccf5a0..31f0b427a 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -2790,16 +2790,10 @@ def test_issue_231(self): datadir = os.path.join(node.data_dir, '123') - try: - self.backup_node( - backup_dir, 'node', node, data_dir='{0}'.format(datadir)) - except: - pass - - out = self.backup_node(backup_dir, 'node', node, options=['--stream'], return_id=False) + pb1 = self.backup_node(backup_dir, 'node', node, data_dir='{0}'.format(datadir)) + pb2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) - # it is a bit racy - self.assertIn("WARNING: Cannot create directory", out) + self.assertNotEqual(pb1, pb2) def test_incr_backup_filenode_map(self): """ From 6bd71d866dda40c2d6697f3e86f488b13648bd61 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 6 Dec 2022 01:51:22 +0300 Subject: [PATCH 2004/2107] [PBCKP-382] and another one Pg15 pg_backup_stop() --- tests/auth_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/auth_test.py b/tests/auth_test.py index 65c30e6ee..52d7e1544 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -240,7 +240,6 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_backup_stop() TO backup; " "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " From bef73b8ba61dce7e4c4704eaa8b2d31c4ec08fd8 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sun, 4 Dec 2022 03:31:32 +0300 Subject: [PATCH 2005/2107] more usages for table_checksum --- tests/archive_test.py | 51 +++++++++++++-------------------- tests/catchup_test.py | 52 +++++++++++++++++----------------- tests/cfs_catchup_test.py | 8 +++--- tests/cfs_restore_test.py | 29 +++++++++---------- tests/compression_test.py | 48 +++++++++++++++---------------- tests/delta_test.py | 32 ++++++++++----------- tests/merge_test.py | 43 ++++++++-------------------- tests/page_test.py | 30 ++++++++------------ tests/ptrack_test.py | 59 +++++++++++++++++---------------------- tests/replica_test.py | 22 +++++++-------- tests/restore_test.py | 50 ++++++++++++++++----------------- 11 files changed, 185 insertions(+), 239 deletions(-) diff --git a/tests/archive_test.py b/tests/archive_test.py index fb2600c4a..b2217a7bf 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -35,7 +35,7 @@ def test_pgpro434_1(self): "md5(repeat(i::text,10))::tsvector as tsvector from " "generate_series(0,100) i") - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node) node.cleanup() @@ -58,7 +58,7 @@ def test_pgpro434_1(self): node.slow_start() self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + result, node.table_checksum("t_heap"), 'data after restore not equal to original data') # @unittest.skip("skip") @@ -152,7 +152,7 @@ def test_pgpro434_2(self): backup_id = self.backup_node(backup_dir, 'node', node) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node.safe_psql( "postgres", "insert into t_heap select 100503 as id, md5(i::text) as text, " @@ -204,11 +204,7 @@ def test_pgpro434_2(self): "select exists(select 1 from t_heap where id > 100500)")[0][0], 'data after restore not equal to original data') - self.assertEqual( - result, - node.safe_psql( - "postgres", - "SELECT * FROM t_heap"), + self.assertEqual(result, node.table_checksum("t_heap"), 'data after restore not equal to original data') # @unittest.skip("skip") @@ -702,7 +698,7 @@ def test_replica_archive(self): "from generate_series(0,2560) i") self.backup_node(backup_dir, 'master', master, options=['--stream']) - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # Settings for Replica self.restore_node(backup_dir, 'master', replica) @@ -713,7 +709,7 @@ def test_replica_archive(self): replica.slow_start(replica=True) # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -724,7 +720,7 @@ def test_replica_archive(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'replica', replica, @@ -748,7 +744,7 @@ def test_replica_archive(self): self.set_auto_conf(node, {'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, make PAGE backup from replica, @@ -760,7 +756,7 @@ def test_replica_archive(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,80680) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.wait_until_replica_catch_with_master(master, replica) @@ -787,7 +783,7 @@ def test_replica_archive(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # @unittest.expectedFailure @@ -831,7 +827,7 @@ def test_master_and_replica_parallel_archiving(self): # TAKE FULL ARCHIVE BACKUP FROM MASTER self.backup_node(backup_dir, 'master', master) # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # GET PHYSICAL CONTENT FROM MASTER pgdata_master = self.pgdata_content(master.data_dir) @@ -849,7 +845,7 @@ def test_master_and_replica_parallel_archiving(self): replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) master.psql( @@ -923,7 +919,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): # TAKE FULL ARCHIVE BACKUP FROM MASTER self.backup_node(backup_dir, 'master', master) # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # GET PHYSICAL CONTENT FROM MASTER pgdata_master = self.pgdata_content(master.data_dir) @@ -942,7 +938,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) master.psql( @@ -1107,7 +1103,7 @@ def test_archive_pg_receivexlog(self): node, backup_type='page' ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.validate_pb(backup_dir) # Check data correctness @@ -1117,9 +1113,7 @@ def test_archive_pg_receivexlog(self): self.assertEqual( result, - node.safe_psql( - "postgres", "SELECT * FROM t_heap" - ), + node.table_checksum("t_heap"), 'data after restore not equal to original data') # Clean after yourself @@ -1176,7 +1170,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): backup_dir, 'node', node, backup_type='page' ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.validate_pb(backup_dir) # Check data correctness @@ -1185,7 +1179,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): node.slow_start() self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + result, node.table_checksum("t_heap"), 'data after restore not equal to original data') # Clean after yourself @@ -2150,13 +2144,8 @@ def test_archive_pg_receivexlog_partial_handling(self): node_restored.slow_start() - result = node.safe_psql( - "postgres", - "select sum(id) from t_heap").decode('utf-8').rstrip() - - result_new = node_restored.safe_psql( - "postgres", - "select sum(id) from t_heap").decode('utf-8').rstrip() + result = node.table_checksum("t_heap") + result_new = node_restored.table_checksum("t_heap") self.assertEqual(result, result_new) diff --git a/tests/catchup_test.py b/tests/catchup_test.py index c94a5300d..21bcd7973 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -22,7 +22,7 @@ def test_basic_full_catchup(self): src_pg.safe_psql( "postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do full catchup dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) @@ -47,7 +47,7 @@ def test_basic_full_catchup(self): dst_pg.slow_start() # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -69,7 +69,7 @@ def test_full_catchup_with_tablespace(self): src_pg.safe_psql( "postgres", "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do full catchup with tablespace mapping dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) @@ -105,7 +105,7 @@ def test_full_catchup_with_tablespace(self): dst_pg.slow_start() # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -146,7 +146,7 @@ def test_basic_delta_catchup(self): pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup self.catchup_node( @@ -171,7 +171,7 @@ def test_basic_delta_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -218,7 +218,7 @@ def test_basic_ptrack_catchup(self): pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do ptrack catchup self.catchup_node( @@ -243,7 +243,7 @@ def test_basic_ptrack_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -282,7 +282,7 @@ def test_tli_delta_catchup(self): src_pg.slow_start(replica = True) src_pg.promote() src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do catchup (src_tli = 2, dst_tli = 1) self.catchup_node( @@ -306,7 +306,7 @@ def test_tli_delta_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') dst_pg.stop() @@ -364,7 +364,7 @@ def test_tli_ptrack_catchup(self): self.assertEqual(src_tli, "2", "Postgres didn't update TLI after promote") src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do catchup (src_tli = 2, dst_tli = 1) self.catchup_node( @@ -388,7 +388,7 @@ def test_tli_ptrack_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') dst_pg.stop() @@ -818,7 +818,7 @@ def test_tli_destination_mismatch(self): # preparation 3: "useful" changes src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # try catchup try: @@ -832,7 +832,7 @@ def test_tli_destination_mismatch(self): dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") dst_pg.stop() self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') except ProbackupException as e: @@ -896,7 +896,7 @@ def test_tli_source_mismatch(self): # preparation 4: "useful" changes src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # try catchup try: @@ -910,7 +910,7 @@ def test_tli_source_mismatch(self): dst_options['port'] = str(dst_pg.port) self.set_auto_conf(dst_pg, dst_options) dst_pg.slow_start() - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") dst_pg.stop() self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') except ProbackupException as e: @@ -979,7 +979,7 @@ def test_unclean_delta_catchup(self): pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup self.catchup_node( @@ -1004,7 +1004,7 @@ def test_unclean_delta_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -1068,7 +1068,7 @@ def test_unclean_ptrack_catchup(self): pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup self.catchup_node( @@ -1093,7 +1093,7 @@ def test_unclean_ptrack_catchup(self): dst_pg.slow_start(replica = True) # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup @@ -1367,9 +1367,9 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") dst_pg.catchup() # wait for replication - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # preparation 4: make changes on master (source) @@ -1397,9 +1397,9 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") dst_pg.catchup() # wait for replication - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # preparation 5: make changes on master (source) @@ -1426,9 +1426,9 @@ def test_config_exclusion(self): # check: run verification query src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") dst_pg.catchup() # wait for replication - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # Cleanup diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py index 43c3f18f1..f6760b72c 100644 --- a/tests/cfs_catchup_test.py +++ b/tests/cfs_catchup_test.py @@ -25,7 +25,7 @@ def test_full_catchup_with_tablespace(self): src_pg.safe_psql( "postgres", "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") src_pg.safe_psql( "postgres", "CHECKPOINT") @@ -76,7 +76,7 @@ def test_full_catchup_with_tablespace(self): dst_pg.slow_start() # 2nd check: run verification query - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') # and now delta backup @@ -112,6 +112,6 @@ def test_full_catchup_with_tablespace(self): # 3rd check: run verification query - src_query_result = src_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") - dst_query_result = dst_pg.safe_psql("postgres", "SELECT * FROM ultimate_question") + src_query_result = src_pg.table_checksum("ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py index 6b69b4ffe..e70af39b4 100644 --- a/tests/cfs_restore_test.py +++ b/tests/cfs_restore_test.py @@ -112,10 +112,7 @@ def add_data_in_cluster(self): MD5(repeat(i::text,10))::tsvector AS tsvector \ FROM generate_series(0,1e5) i'.format('t1', tblspace_name) ) - self.table_t1 = self.node.safe_psql( - "postgres", - "SELECT * FROM t1" - ) + self.table_t1 = self.node.table_checksum("t1") # --- Restore from full backup ---# # @unittest.expectedFailure @@ -154,8 +151,8 @@ def test_restore_from_fullbackup_to_old_location(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure @@ -193,8 +190,8 @@ def test_restore_from_fullbackup_to_old_location_3_jobs(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure @@ -236,8 +233,8 @@ def test_restore_from_fullbackup_to_new_location(self): ) self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) node_new.cleanup() @@ -280,8 +277,8 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): ) self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) node_new.cleanup() @@ -328,8 +325,8 @@ def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure @@ -375,8 +372,8 @@ def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs( ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure diff --git a/tests/compression_test.py b/tests/compression_test.py index 94f2dffff..e779f6472 100644 --- a/tests/compression_test.py +++ b/tests/compression_test.py @@ -32,7 +32,7 @@ def test_basic_compression_stream_zlib(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=[ @@ -45,7 +45,7 @@ def test_basic_compression_stream_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=[ @@ -57,7 +57,7 @@ def test_basic_compression_stream_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=zlib']) @@ -77,7 +77,7 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -93,7 +93,7 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -109,7 +109,7 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) def test_compression_archive_zlib(self): @@ -134,7 +134,7 @@ def test_compression_archive_zlib(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=["--compress-algorithm=zlib"]) @@ -145,7 +145,7 @@ def test_compression_archive_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(0,2) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=["--compress-algorithm=zlib"]) @@ -155,7 +155,7 @@ def test_compression_archive_zlib(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=zlib']) @@ -175,7 +175,7 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -191,7 +191,7 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -207,7 +207,7 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -234,7 +234,7 @@ def test_compression_stream_pglz(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream', '--compress-algorithm=pglz']) @@ -245,7 +245,7 @@ def test_compression_stream_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--stream', '--compress-algorithm=pglz']) @@ -256,7 +256,7 @@ def test_compression_stream_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=pglz']) @@ -276,7 +276,7 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -292,7 +292,7 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -308,7 +308,7 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -335,7 +335,7 @@ def test_compression_archive_pglz(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--compress-algorithm=pglz']) @@ -346,7 +346,7 @@ def test_compression_archive_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--compress-algorithm=pglz']) @@ -357,7 +357,7 @@ def test_compression_archive_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(200,300) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=pglz']) @@ -377,7 +377,7 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -393,7 +393,7 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -409,7 +409,7 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() diff --git a/tests/delta_test.py b/tests/delta_test.py index 23583fd93..8736a079c 100644 --- a/tests/delta_test.py +++ b/tests/delta_test.py @@ -239,7 +239,7 @@ def test_delta_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream']) @@ -250,7 +250,7 @@ def test_delta_stream(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream']) @@ -270,7 +270,7 @@ def test_delta_stream(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -286,7 +286,7 @@ def test_delta_stream(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -313,7 +313,7 @@ def test_delta_archive(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full') @@ -322,7 +322,7 @@ def test_delta_archive(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta') @@ -341,7 +341,7 @@ def test_delta_archive(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -357,7 +357,7 @@ def test_delta_archive(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() @@ -400,7 +400,7 @@ def test_delta_multiple_segments(self): node.safe_psql("postgres", "checkpoint") # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") # delta BACKUP self.backup_node( backup_dir, 'node', node, @@ -429,9 +429,7 @@ def test_delta_multiple_segments(self): self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", - "select count(*) from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') @@ -540,7 +538,7 @@ def test_create_db(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -663,7 +661,7 @@ def test_exists_in_previous_backup(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") filepath = node.safe_psql( "postgres", "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() @@ -774,8 +772,7 @@ def test_alter_table_set_tablespace_delta(self): "alter table t_heap set tablespace somedata_new") # DELTA BACKUP - result = node.safe_psql( - "postgres", "select * from t_heap") + result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, backup_type='delta', @@ -813,8 +810,7 @@ def test_alter_table_set_tablespace_delta(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from t_heap") + result_new = node_restored.table_checksum("t_heap") self.assertEqual(result, result_new, 'lost some data after restore') diff --git a/tests/merge_test.py b/tests/merge_test.py index ffa73263c..c789298fd 100644 --- a/tests/merge_test.py +++ b/tests/merge_test.py @@ -659,13 +659,8 @@ def test_merge_page_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) @@ -744,13 +739,8 @@ def test_merge_delta_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) @@ -836,13 +826,8 @@ def test_merge_ptrack_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) @@ -1931,9 +1916,7 @@ def test_merge_backup_from_future(self): backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) @@ -1959,11 +1942,9 @@ def test_merge_backup_from_future(self): {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') + result_new = node_restored.table_checksum("pgbench_accounts") - self.assertTrue(result, result_new) + self.assertEqual(result, result_new) self.compare_pgdata(pgdata, pgdata_restored) @@ -2458,8 +2439,7 @@ def test_multi_timeline_merge(self): self.merge_backup(backup_dir, 'node', page_id) - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) @@ -2471,8 +2451,7 @@ def test_multi_timeline_merge(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = node_restored.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) diff --git a/tests/page_test.py b/tests/page_test.py index be6116bbe..786374bdb 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -84,13 +84,8 @@ def test_basic_page_vacuum_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) @@ -191,7 +186,7 @@ def test_page_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream']) @@ -202,7 +197,7 @@ def test_page_stream(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--stream', '-j', '4']) @@ -223,7 +218,7 @@ def test_page_stream(self): ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -242,7 +237,7 @@ def test_page_stream(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -272,7 +267,7 @@ def test_page_archive(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full') @@ -282,7 +277,7 @@ def test_page_archive(self): "insert into t_heap select i as id, " "md5(i::text) as text, md5(i::text)::tsvector as tsvector " "from generate_series(100, 200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=["-j", "4"]) @@ -308,7 +303,7 @@ def test_page_archive(self): node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -332,7 +327,7 @@ def test_page_archive(self): node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -370,7 +365,7 @@ def test_page_multiple_segments(self): pgbench.wait() # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select count(*) from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") # PAGE BACKUP self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -398,8 +393,7 @@ def test_page_multiple_segments(self): self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", "select count(*) from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index b8a4065b0..7b5bc416b 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -440,7 +440,7 @@ def test_ptrack_simple(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node_restored = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) @@ -463,7 +463,7 @@ def test_ptrack_simple(self): # Logical comparison self.assertEqual( result, - node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) + node_restored.table_checksum("t_heap")) # @unittest.skip("skip") def test_ptrack_unprivileged(self): @@ -1030,7 +1030,7 @@ def test_ptrack_get_block(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) @@ -1044,7 +1044,7 @@ def test_ptrack_get_block(self): # Logical comparison self.assertEqual( result, - node.safe_psql("postgres", "SELECT * FROM t_heap")) + node.table_checksum("t_heap")) # @unittest.skip("skip") def test_ptrack_stream(self): @@ -1075,7 +1075,7 @@ def test_ptrack_stream(self): " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" " as tsvector from generate_series(0,100) i") - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, options=['--stream']) @@ -1086,7 +1086,7 @@ def test_ptrack_stream(self): " md5(i::text) as text, md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i") - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result = node.table_checksum("t_heap") ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) @@ -1108,7 +1108,7 @@ def test_ptrack_stream(self): repr(self.output), self.cmd) ) node.slow_start() - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -1128,7 +1128,7 @@ def test_ptrack_stream(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result_new = node.table_checksum("t_heap") self.assertEqual(ptrack_result, ptrack_result_new) # @unittest.skip("skip") @@ -1162,7 +1162,7 @@ def test_ptrack_archive(self): " md5(i::text)::tsvector as tsvector" " from generate_series(0,100) i") - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node(backup_dir, 'node', node) full_target_time = self.show_pb( backup_dir, 'node', full_backup_id)['recovery-time'] @@ -1175,7 +1175,7 @@ def test_ptrack_archive(self): " md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i") - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result = node.table_checksum("t_heap") ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack') ptrack_target_time = self.show_pb( @@ -1208,7 +1208,7 @@ def test_ptrack_archive(self): ) node.slow_start() - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -1233,7 +1233,7 @@ def test_ptrack_archive(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result_new = node.table_checksum("t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() @@ -1263,9 +1263,6 @@ def test_ptrack_pgpro417(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql( - "postgres", - "SELECT * FROM t_heap") backup_id = self.backup_node( backup_dir, 'node', node, @@ -1280,7 +1277,7 @@ def test_ptrack_pgpro417(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=["--stream"]) @@ -1339,7 +1336,7 @@ def test_page_pgpro417(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") # PAGE BACKUP node.safe_psql( @@ -1347,7 +1344,7 @@ def test_page_pgpro417(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1403,7 +1400,7 @@ def test_full_pgpro417(self): " md5(i::text)::tsvector as tsvector " " from generate_series(0,100) i" ) - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node(backup_dir, 'node', node, options=["--stream"]) # SECOND FULL BACKUP @@ -1413,7 +1410,7 @@ def test_full_pgpro417(self): " md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i" ) - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1474,7 +1471,7 @@ def test_create_db(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1694,8 +1691,7 @@ def test_alter_table_set_tablespace_ptrack(self): # sys.exit(1) # PTRACK BACKUP - #result = node.safe_psql( - # "postgres", "select * from t_heap") + #result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -1737,8 +1733,7 @@ def test_alter_table_set_tablespace_ptrack(self): node_restored, {'port': node_restored.port}) node_restored.slow_start() -# result_new = node_restored.safe_psql( -# "postgres", "select * from t_heap") +# result_new = node_restored.table_checksum("t_heap") # # self.assertEqual(result, result_new, 'lost some data after restore') @@ -1838,7 +1833,7 @@ def test_drop_tablespace(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1892,7 +1887,7 @@ def test_drop_tablespace(self): "Expecting Error because " "tablespace 'somedata' should not be present") - result_new = node.safe_psql("postgres", "select * from t_heap") + result_new = node.table_checksum("t_heap") self.assertEqual(result, result_new) if self.paranoia: @@ -1930,7 +1925,7 @@ def test_ptrack_alter_tablespace(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1939,7 +1934,7 @@ def test_ptrack_alter_tablespace(self): "postgres", "alter table t_heap set tablespace somedata") # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FIRTS PTRACK BACKUP self.backup_node( @@ -1971,8 +1966,7 @@ def test_ptrack_alter_tablespace(self): restored_node.slow_start() # COMPARE LOGICAL CONTENT - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") + result_new = restored_node.table_checksum("t_heap") self.assertEqual(result, result_new) restored_node.cleanup() @@ -2006,8 +2000,7 @@ def test_ptrack_alter_tablespace(self): restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") + result_new = restored_node.table_checksum("t_heap") self.assertEqual(result, result_new) # @unittest.skip("skip") diff --git a/tests/replica_test.py b/tests/replica_test.py index 577dcd3a5..17fc5a823 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -125,7 +125,7 @@ def test_replica_stream_ptrack_backup(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # take full backup and restore it self.backup_node(backup_dir, 'master', master, options=['--stream']) @@ -137,7 +137,7 @@ def test_replica_stream_ptrack_backup(self): # Check data correctness on replica replica.slow_start(replica=True) - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -148,7 +148,7 @@ def test_replica_stream_ptrack_backup(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.add_instance(backup_dir, 'replica', replica) backup_id = self.backup_node( @@ -173,7 +173,7 @@ def test_replica_stream_ptrack_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take PTRACK backup from replica, @@ -185,7 +185,7 @@ def test_replica_stream_ptrack_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', @@ -208,7 +208,7 @@ def test_replica_stream_ptrack_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # @unittest.skip("skip") @@ -248,7 +248,7 @@ def test_replica_archive_page_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') @@ -262,7 +262,7 @@ def test_replica_archive_page_backup(self): replica.slow_start(replica=True) # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -274,7 +274,7 @@ def test_replica_archive_page_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,25120) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.wait_until_replica_catch_with_master(master, replica) @@ -301,7 +301,7 @@ def test_replica_archive_page_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) node.cleanup() @@ -385,7 +385,7 @@ def test_basic_make_replica_via_restore(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,8192) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') diff --git a/tests/restore_test.py b/tests/restore_test.py index 2de3ecc0f..da3ebffb4 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -34,7 +34,7 @@ def test_restore_full_to_latest(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) node.stop() @@ -60,7 +60,7 @@ def test_restore_full_to_latest(self): node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -88,7 +88,7 @@ def test_restore_full_page_to_latest(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -102,7 +102,7 @@ def test_restore_full_page_to_latest(self): node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -120,7 +120,7 @@ def test_restore_to_specific_timeline(self): node.pgbench_init(scale=2) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) @@ -164,7 +164,7 @@ def test_restore_to_specific_timeline(self): self.assertEqual(int(recovery_target_timeline), target_tli) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -182,7 +182,7 @@ def test_restore_to_time(self): node.slow_start() node.pgbench_init(scale=2) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) @@ -210,7 +210,7 @@ def test_restore_to_time(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -238,7 +238,7 @@ def test_restore_to_xid_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() @@ -264,7 +264,7 @@ def test_restore_to_xid_inclusive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) @@ -294,7 +294,7 @@ def test_restore_to_xid_not_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() @@ -321,7 +321,7 @@ def test_restore_to_xid_not_inclusive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) @@ -354,7 +354,7 @@ def test_restore_to_lsn_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() @@ -387,7 +387,7 @@ def test_restore_to_lsn_inclusive(self): node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) @@ -420,7 +420,7 @@ def test_restore_to_lsn_not_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() @@ -454,7 +454,7 @@ def test_restore_to_lsn_not_inclusive(self): node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) @@ -492,7 +492,7 @@ def test_restore_full_ptrack_archive(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="ptrack") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -506,7 +506,7 @@ def test_restore_full_ptrack_archive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -549,7 +549,7 @@ def test_restore_ptrack(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="ptrack") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -563,7 +563,7 @@ def test_restore_ptrack(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -601,7 +601,7 @@ def test_restore_full_ptrack_stream(self): backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -614,7 +614,7 @@ def test_restore_full_ptrack_stream(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) # @unittest.skip("skip") @@ -1289,9 +1289,7 @@ def test_archive_restore_to_restore_point(self): node.safe_psql( "postgres", "create table t_heap as select generate_series(0,10000)") - result = node.safe_psql( - "postgres", - "select * from t_heap") + result = node.table_checksum("t_heap") node.safe_psql( "postgres", "select pg_create_restore_point('savepoint')") node.safe_psql( @@ -1307,7 +1305,7 @@ def test_archive_restore_to_restore_point(self): node.slow_start() - result_new = node.safe_psql("postgres", "select * from t_heap") + result_new = node.table_checksum("t_heap") res = node.psql("postgres", "select * from t_heap_1") self.assertEqual( res[0], 1, From 29a9efb4d499dbf8ae93f842cf730f2c1c6f0ed4 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 6 Dec 2022 14:17:04 +0300 Subject: [PATCH 2006/2107] [PBCKP-325] refix test_issue_231 to make two backups in one second we have to fail them. Therefore we have to fetch backup_id from log in exception's message. Retry for 20 seconds to have a chance to start in one second. If we couldn't, lets skip the test. --- tests/backup_test.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 31f0b427a..8810108a5 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -1,5 +1,6 @@ import unittest import os +import re from time import sleep, time from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil @@ -2780,18 +2781,33 @@ def test_issue_231(self): """ backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - node.slow_start() datadir = os.path.join(node.data_dir, '123') - pb1 = self.backup_node(backup_dir, 'node', node, data_dir='{0}'.format(datadir)) - pb2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + t0 = time() + while True: + with self.assertRaises(ProbackupException) as ctx: + self.backup_node(backup_dir, 'node', node) + pb1 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] + + t = time() + if int(pb1, 36) == int(t) and t % 1 < 0.5: + # ok, we have a chance to start next backup in same second + break + elif t - t0 > 20: + # Oops, we are waiting for too long. Looks like this runner + # is too slow. Lets skip the test. + self.skipTest("runner is too slow") + # sleep to the second's end so backup will not sleep for a second. + sleep(1 - t % 1) + + with self.assertRaises(ProbackupException) as ctx: + self.backup_node(backup_dir, 'node', node) + pb2 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] self.assertNotEqual(pb1, pb2) From 3bc0fc4b8169d53ee93913f21538926fc4463d36 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 7 Dec 2022 13:35:25 +0300 Subject: [PATCH 2007/2107] Documentation hot fix --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7c8610681..2cb10e379 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -653,7 +653,7 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; - For PostgreSQL 10: + For PostgreSQL versions 10 — 14: BEGIN; From 15c304ad6cec482ac2ed7f9680f3379d49504086 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Sun, 4 Dec 2022 18:15:22 +0300 Subject: [PATCH 2008/2107] [PBCKP-375] Prepared for moving RelFileNode to RelFileLocator in the PG16. --- src/datapagemap.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/datapagemap.h b/src/datapagemap.h index 6af54713b..6ad7a6204 100644 --- a/src/datapagemap.h +++ b/src/datapagemap.h @@ -9,7 +9,12 @@ #ifndef DATAPAGEMAP_H #define DATAPAGEMAP_H +#if PG_VERSION_NUM < 160000 #include "storage/relfilenode.h" +#else +#include "storage/relfilelocator.h" +#define RelFileNode RelFileLocator +#endif #include "storage/block.h" From 25e63c5a7c0f290290216bd9783632ed06ea7435 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 8 Dec 2022 07:44:12 +0300 Subject: [PATCH 2009/2107] raw strings in python regex; ignore generated transation files Author: Sergey Fukanchik --- .gitignore | 3 +++ tests/helpers/ptrack_helpers.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 502473605..97d323ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ # Binaries /pg_probackup +# Generated translated file +/po/ru.mo + # Generated by test suite /regression.diffs /regression.out diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 706506432..067225d66 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1685,7 +1685,7 @@ def version_to_num(self, version): parts.append('0') num = 0 for part in parts: - num = num * 100 + int(re.sub("[^\d]", "", part)) + num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num def switch_wal_segment(self, node): @@ -2038,7 +2038,7 @@ def __init__(self, cmd, env, attach=False): # Get version gdb_version_number = re.search( - b"^GNU gdb [^\d]*(\d+)\.(\d)", + br"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) self.major_version = int(gdb_version_number.group(1)) self.minor_version = int(gdb_version_number.group(2)) From 822fbbfe503f52a3b898a9e8deb54f4b28b9f56c Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Thu, 3 Nov 2022 10:50:23 +0300 Subject: [PATCH 2010/2107] [PBCKP-326] regex fixed in test_missing_replication_permission_1 Everything between WARNING and FATAL sections is now handled with the [\s\S]*? regex: * [\s\S] is a group that handles any whitespace and non-whitespace character including new lines which are important in this case. * "*" quantifier means zero or more characters. There may as well be nothing between these two sections. * "?" quantifies in this case means greedy search so that we don't match more than we need. --- tests/backup_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 8810108a5..fc1135cab 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3226,10 +3226,11 @@ def test_missing_replication_permission_1(self): # Messages for >=14 # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' + # OS-dependant messages: + # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (::1), port 12101 failed: Connection refused\n\tIs the server running on that host and accepting TCP/IP connections?\nconnection to server at "localhost" (127.0.0.1), port 12101 failed: FATAL: must be superuser or replication role to start walsender' self.assertRegex( output, - r'WARNING: could not connect to database backupdb: (connection to server (on socket "/tmp/.s.PGSQL.\d+"|at "localhost" \(127.0.0.1\), port \d+) failed: ){0,1}' - 'FATAL: must be superuser or replication role to start walsender') + r'WARNING: could not connect to database backupdb:[\s\S]*?FATAL: must be superuser or replication role to start walsender') # @unittest.skip("skip") def test_basic_backup_default_transaction_read_only(self): From 076e3fdae97d75a3a5c1e859202762864b087971 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Mon, 12 Dec 2022 15:46:17 +0300 Subject: [PATCH 2011/2107] [PBCKP-394] skip creating partitioned index on < 11 versions on test_checkdb_amcheck_only_sanity --- tests/checkdb_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index 2caf4fcb2..1e6daefdb 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -40,9 +40,11 @@ def test_checkdb_amcheck_only_sanity(self): "create table idxpart (a int) " "partition by range (a)") - node.safe_psql( - "postgres", - "create index on idxpart(a)") + # there aren't partitioned indexes on 10 and lesser versions + if self.get_version(node) >= 110000: + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( From b74ec9046d635dbe9ce6a133494bfa4995f56fa1 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Mon, 5 Dec 2022 22:31:47 +0700 Subject: [PATCH 2012/2107] Update Readme --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c5b01ced2..5b9e094a2 100644 --- a/README.md +++ b/README.md @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}-dbg +sudo apt-get install pg-probackup-{15,14,13,12,11,10} +sudo apt-get install pg-probackup-{15,14,13,12,11,10}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{14,13,12,11,10,9.6} +sudo apt-get source pg-probackup-{15,14,13,12,11,10} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{14,13,12,11,10,9.6}{-dbg,} +sudo apt-get install pg-probackup-{15,14,13,12,11,10}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10} +yum install pg_probackup-{15,14,13,12,11,10}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10} +yum install pg_probackup-{15,14,13,12,11,10}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{14,13,12,11,10,9.6} -yum install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +yum install pg_probackup-{15,14,13,12,11,10} +yum install pg_probackup-{15,14,13,12,11,10}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{14,13,12,11,10,9.6} +yumdownloader --source pg_probackup-{15,14,13,12,11,10} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{14,13,12,11,10,9.6} -zypper install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10} +zypper install pg_probackup-{15,14,13,12,11,10}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{14,13,12,11,10,9.6} - -#RPM ALT Linux 7 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' -sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +zypper si pg_probackup-{15,14,13,12,11,10} #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10} +sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11,10} +sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo + +#RPM ALT Linux 10 +sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' +sudo apt-get update +sudo apt-get install pg_probackup-{15,14,13,12,11,10} +sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise From 9391a1b6768c8913497f99642c1c138657cdb159 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 13 Dec 2022 00:01:03 +0700 Subject: [PATCH 2013/2107] Increment the version --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 843fb3522..fa3bc4123 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -347,7 +347,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.10" +#define PROGRAM_VERSION "2.5.11" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20509 From 24f12d98d98601cffb195d22342162f877a6a1c7 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 13 Dec 2022 02:52:18 +0300 Subject: [PATCH 2014/2107] [PBCKP-402,PBCKP-354] refix "missing contrecord" detection. Error message is translated according to current locale. So we can't compare it as a string. But `abortedRecPtr` exists exactly for this case, so we can rely on it. --- src/parsexlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parsexlog.c b/src/parsexlog.c index 284b610f6..b7743f6c7 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1446,7 +1446,7 @@ XLogThreadWorker(void *arg) * TODO: probably we should abort reading logs at this moment. * But we continue as we did with bug present in Pg < 15. */ - strncmp(errormsg, "missing contrecord", 18) == 0)) + !XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))) { if (SwitchThreadToNextWal(xlogreader, thread_arg)) continue; From 10ac3c9918907c827e2b14687e2fe12629a415b4 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 13 Dec 2022 13:06:09 +0300 Subject: [PATCH 2015/2107] [PBCKP-402] bound check for abortedRecPtr to Pg15 It is not really needed in previous versions. It doesn't harm, but we want to reduce tests amount. --- src/parsexlog.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index b7743f6c7..7c4b5b349 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1439,6 +1439,7 @@ XLogThreadWorker(void *arg) * Usually SimpleXLogPageRead() does it by itself. But here we need * to do it manually to support threads. */ +#if PG_VERSION_NUM >= 150000 if (reader_data->need_switch && ( errormsg == NULL || /* @@ -1447,6 +1448,9 @@ XLogThreadWorker(void *arg) * But we continue as we did with bug present in Pg < 15. */ !XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))) +#else + if (reader_data->need_switch && errormsg == NULL) +#endif { if (SwitchThreadToNextWal(xlogreader, thread_arg)) continue; From b90273fe143c4b2d2d198f2e29b61547c316a561 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 13 Dec 2022 22:50:17 +0700 Subject: [PATCH 2016/2107] Increment the expected test version --- tests/expected/option_version.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index 8abfe7fdd..e0d6924b9 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.10 +pg_probackup 2.5.11 From 30e3e37c7b92d629982cf375ccc4ccc0fd518fb9 Mon Sep 17 00:00:00 2001 From: Alexander Burtsev Date: Wed, 14 Dec 2022 16:09:53 +0300 Subject: [PATCH 2017/2107] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5b9e094a2..fb3c5b79c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) # pg_probackup From 640e7a5dcd8b6ad1884b4c0b8698fc38fec18d14 Mon Sep 17 00:00:00 2001 From: Alexander Burtsev Date: Wed, 14 Dec 2022 17:42:15 +0300 Subject: [PATCH 2018/2107] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fb3c5b79c..7486a6ca6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) +[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) # pg_probackup From bbe41a403de67db1a6d44b22c7372faaad411a7a Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Thu, 15 Dec 2022 13:37:15 +0300 Subject: [PATCH 2019/2107] Fix tests test_restore_from_fullbackup_to_new_location and test_restore_from_fullbackup_to_new_location_5_jobs --- tests/cfs_restore_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py index e70af39b4..2fa35e71a 100644 --- a/tests/cfs_restore_test.py +++ b/tests/cfs_restore_test.py @@ -233,7 +233,7 @@ def test_restore_from_fullbackup_to_new_location(self): ) self.assertEqual( - self.node.table_checksum("t1"), + node_new.table_checksum("t1"), self.table_t1 ) node_new.cleanup() @@ -277,7 +277,7 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): ) self.assertEqual( - self.node.table_checksum("t1"), + node_new.table_checksum("t1"), self.table_t1 ) node_new.cleanup() From 8f4e7d6e5f163744baa340143d9b65682fb64fd9 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 22 Dec 2022 05:49:52 +0300 Subject: [PATCH 2020/2107] [PBCKP-346] archive-get doesn't need -D/--pgdata argument at all --- src/pg_probackup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 849685278..ed48178b4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -677,6 +677,7 @@ main(int argc, char *argv[]) if (instance_config.pgdata != NULL) canonicalize_path(instance_config.pgdata); if (instance_config.pgdata != NULL && + backup_subcmd != ARCHIVE_GET_CMD && !is_absolute_path(instance_config.pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); From 22bdbb391549d41c581e85b5eebd07404d7bbc3c Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Wed, 11 Jan 2023 23:37:45 +0300 Subject: [PATCH 2021/2107] [PBCKP-423] test_archive_push_sanity: wait logs with tail_file (and some other test as well). --- tests/archive_test.py | 95 ++++++++++++++++------------------- tests/helpers/__init__.py | 2 +- tests/helpers/data_helpers.py | 78 ++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 52 deletions(-) create mode 100644 tests/helpers/data_helpers.py diff --git a/tests/archive_test.py b/tests/archive_test.py index b2217a7bf..dd6959290 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -3,6 +3,7 @@ import gzip import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException +from .helpers.data_helpers import tail_file from datetime import datetime, timedelta import subprocess from sys import exit @@ -383,26 +384,31 @@ def test_archive_push_file_exists(self): self.switch_wal_segment(node) sleep(1) - with open(log_file, 'r') as f: - log_content = f.read() + log = tail_file(log_file, linetimeout=30, totaltimeout=120, + collect=True) + log.wait(contains = 'The failed archive command was') + self.assertIn( 'LOG: archive command failed with exit code 1', - log_content) + log.content) self.assertIn( 'DETAIL: The failed archive command was:', - log_content) + log.content) self.assertIn( 'pg_probackup archive-push WAL file', - log_content) + log.content) self.assertIn( 'WAL file already exists in archive with different checksum', - log_content) + log.content) self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) + 'pg_probackup archive-push completed successfully', log.content) + + # btw check that console coloring codes are not slipped into log file + self.assertNotIn('[0m', log.content) if self.get_version(node) < 100000: wal_src = os.path.join( @@ -419,19 +425,9 @@ def test_archive_push_file_exists(self): shutil.copyfile(wal_src, file) self.switch_wal_segment(node) - sleep(5) - - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'pg_probackup archive-push completed successfully', - log_content) - - # btw check that console coloring codes are not slipped into log file - self.assertNotIn('[0m', log_content) - print(log_content) + log.stop_collect() + log.wait(contains = 'pg_probackup archive-push completed successfully') # @unittest.skip("skip") def test_archive_push_file_exists_overwrite(self): @@ -471,39 +467,35 @@ def test_archive_push_file_exists_overwrite(self): self.switch_wal_segment(node) sleep(1) - with open(log_file, 'r') as f: - log_content = f.read() + log = tail_file(log_file, linetimeout=30, collect=True) + log.wait(contains = 'The failed archive command was') self.assertIn( - 'LOG: archive command failed with exit code 1', log_content) + 'LOG: archive command failed with exit code 1', log.content) self.assertIn( - 'DETAIL: The failed archive command was:', log_content) + 'DETAIL: The failed archive command was:', log.content) self.assertIn( - 'pg_probackup archive-push WAL file', log_content) + 'pg_probackup archive-push WAL file', log.content) self.assertNotIn( 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) + 'different checksum, overwriting', log.content) self.assertIn( 'WAL file already exists in archive with ' - 'different checksum', log_content) + 'different checksum', log.content) self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) + 'pg_probackup archive-push completed successfully', log.content) self.set_archiving(backup_dir, 'node', node, overwrite=True) node.reload() self.switch_wal_segment(node) - sleep(5) - with open(log_file, 'r') as f: - log_content = f.read() - self.assertTrue( - 'pg_probackup archive-push completed successfully' in log_content, - 'Expecting messages about successfull execution archive_command') + log.drop_content() + log.wait(contains = 'pg_probackup archive-push completed successfully') self.assertIn( 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) + 'different checksum, overwriting', log.content) # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): @@ -2049,14 +2041,22 @@ def test_archive_push_sanity(self): replica.promote() replica.pgbench_init(scale=10) - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - replica_log_content = f.read() + log = tail_file(os.path.join(replica.logs_dir, 'postgresql.log'), + collect=True) + log.wait(regex=r"pushing file.*history") + log.wait(contains='archive-push completed successfully') + log.wait(regex=r"pushing file.*partial") + log.wait(contains='archive-push completed successfully') # make sure that .partial file is not compressed - self.assertNotIn('.partial.gz', replica_log_content) + self.assertNotIn('.partial.gz', log.content) # make sure that .history file is not compressed - self.assertNotIn('.history.gz', replica_log_content) - self.assertNotIn('WARNING', replica_log_content) + self.assertNotIn('.history.gz', log.content) + + replica.stop() + log.wait_shutdown() + + self.assertNotIn('WARNING', log.content) output = self.show_archive( backup_dir, 'node', as_json=False, as_text=True, @@ -2440,18 +2440,11 @@ def test_archive_get_prefetch_corruption(self): os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) replica.slow_start(replica=True) - sleep(60) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), - postgres_log_content) - - self.assertIn( - 'LOG: restored log file "{0}" from archive'.format(filename), - postgres_log_content) + prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) + restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log')) + tailer.wait(contains=prefetch_line) + tailer.wait(contains=restored_line) # @unittest.skip("skip") def test_archive_show_partial_files_handling(self): diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 4ae3ef8c4..2e5ed40e8 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] +__all__ = ['ptrack_helpers', 'cfs_helpers', 'data_helpers'] import unittest diff --git a/tests/helpers/data_helpers.py b/tests/helpers/data_helpers.py new file mode 100644 index 000000000..27cb66c3d --- /dev/null +++ b/tests/helpers/data_helpers.py @@ -0,0 +1,78 @@ +import re +import unittest +import functools +import time + +def _tail_file(file, linetimeout, totaltimeout): + start = time.time() + with open(file, 'r') as f: + waits = 0 + while waits < linetimeout: + line = f.readline() + if line == '': + waits += 1 + time.sleep(1) + continue + waits = 0 + yield line + if time.time() - start > totaltimeout: + raise TimeoutError("total timeout tailing %s" % (file,)) + else: + raise TimeoutError("line timeout tailing %s" % (file,)) + + +class tail_file(object): # snake case to immitate function + def __init__(self, filename, *, linetimeout=10, totaltimeout=60, collect=False): + self.filename = filename + self.tailer = _tail_file(filename, linetimeout, totaltimeout) + self.collect = collect + self.lines = [] + self._content = None + + def __iter__(self): + return self + + def __next__(self): + line = next(self.tailer) + if self.collect: + self.lines.append(line) + self._content = None + return line + + @property + def content(self): + if not self.collect: + raise AttributeError("content collection is not enabled", + name="content", obj=self) + if not self._content: + self._content = "".join(self.lines) + return self._content + + def drop_content(self): + self.lines.clear() + self._content = None + + def stop_collect(self): + self.drop_content() + self.collect = False + + def wait(self, *, contains:str = None, regex:str = None): + assert contains != None or regex != None + assert contains == None or regex == None + try: + for line in self: + if contains is not None and contains in line: + break + if regex is not None and re.search(regex, line): + break + except TimeoutError: + msg = "Didn't found expected " + if contains is not None: + msg += repr(contains) + elif regex is not None: + msg += f"/{regex}/" + msg += f" in {self.filename}" + raise unittest.TestCase.failureException(msg) + + def wait_shutdown(self): + self.wait(contains='database system is shut down') From fa2902090ac2c9e4f7a006c6e88461424757cac1 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 12 Jan 2023 00:04:47 +0300 Subject: [PATCH 2022/2107] [PBCKP-423] and backport cleanup_ptrack for test_archive_push_sanity --- tests/archive_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/archive_test.py b/tests/archive_test.py index dd6959290..00fd1f592 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -2013,7 +2013,7 @@ def test_archive_push_sanity(self): self.backup_node(backup_dir, 'node', node) with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() + postgres_log_content = cleanup_ptrack(f.read()) # print(postgres_log_content) # make sure that .backup file is not compressed @@ -2056,7 +2056,7 @@ def test_archive_push_sanity(self): replica.stop() log.wait_shutdown() - self.assertNotIn('WARNING', log.content) + self.assertNotIn('WARNING', cleanup_ptrack(log.content)) output = self.show_archive( backup_dir, 'node', as_json=False, as_text=True, @@ -2662,6 +2662,17 @@ def test_archive_empty_history_file(self): 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), log_content) + +def cleanup_ptrack(log_content): + # PBCKP-423 - need to clean ptrack warning + ptrack_is_not = 'Ptrack 1.X is not supported anymore' + if ptrack_is_not in log_content: + lines = [line for line in log_content.splitlines() + if ptrack_is_not not in line] + log_content = "".join(lines) + return log_content + + # TODO test with multiple not archived segments. # TODO corrupted file in archive. From 343c6a0f98f936727b7154d3c0ba6b687db7bff4 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 12 Jan 2023 00:26:45 +0300 Subject: [PATCH 2023/2107] [PBCKP-423] add '.partial.part' detection as well Purposes are: - to not issue WARNING - to remove file properly in delete_walfiles_in_tli --- src/catalog.c | 3 ++- src/pg_probackup.h | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 92a2d84b7..afbac28ab 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1623,7 +1623,8 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) } /* temp WAL segment */ else if (IsTempXLogFileName(file->name) || - IsTempCompressXLogFileName(file->name)) + IsTempCompressXLogFileName(file->name) || + IsTempPartialXLogFileName(file->name)) { elog(VERBOSE, "temp WAL file \"%s\"", file->name); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fa3bc4123..6ff8a41b3 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -777,6 +777,11 @@ typedef struct StopBackupCallbackParams strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0) +#define IsTempPartialXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".partial.part") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".partial.part") == 0) + #define IsTempCompressXLogFileName(fname) \ (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \ strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ From 612f530de0df16616d01529551ab24c011ad64bb Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Fri, 9 Dec 2022 11:51:48 +0300 Subject: [PATCH 2024/2107] [PBCKP-287] fix. added cfs files grouping and processing cfs segment in single thread manner --- src/backup.c | 284 +++++++++++++++++++++++++++++++++------------ src/dir.c | 14 ++- src/pg_probackup.h | 7 +- src/utils/parray.c | 27 +++++ src/utils/parray.h | 4 + 5 files changed, 257 insertions(+), 79 deletions(-) diff --git a/src/backup.c b/src/backup.c index 35fc98092..415f4a02a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -65,7 +65,11 @@ static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); -static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +static size_t rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +static void group_cfs_segments(parray *files, size_t first, size_t last); +static bool remove_excluded_files_criterion(void *value, void *exclude_args); +static void backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments); +static void process_file(int i, pgFile *file, backup_files_arg *arguments); static StopBackupCallbackParams stop_callback_params; @@ -2054,8 +2058,6 @@ static void * backup_files(void *arg) { int i; - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; static time_t prev_time; backup_files_arg *arguments = (backup_files_arg *) arg; @@ -2067,7 +2069,6 @@ backup_files(void *arg) for (i = 0; i < n_backup_files_list; i++) { pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - pgFile *prev_file = NULL; /* We have already copied all directories */ if (S_ISDIR(file->mode)) @@ -2087,6 +2088,9 @@ backup_files(void *arg) } } + if (file->skip_cfs_nested) + continue; + if (!pg_atomic_test_set_flag(&file->lock)) continue; @@ -2097,89 +2101,146 @@ backup_files(void *arg) elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_backup_files_list, file->rel_path); - /* Handle zero sized files */ - if (file->size == 0) + if (file->is_cfs) { - file->write_size = 0; - continue; - } - - /* construct destination filepath */ - if (file->external_dir_num == 0) - { - join_path_components(from_fullpath, arguments->from_root, file->rel_path); - join_path_components(to_fullpath, arguments->to_root, file->rel_path); + backup_cfs_segment(i, file, arguments); } else { - char external_dst[MAXPGPATH]; - char *external_path = parray_get(arguments->external_dirs, - file->external_dir_num - 1); + process_file(i, file, arguments); + } + } + + /* ssh connection to longer needed */ + fio_disconnect(); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +static void +process_file(int i, pgFile *file, backup_files_arg *arguments) +{ + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + pgFile *prev_file = NULL; + + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Process file \"%s\"", + i + 1, parray_num(arguments->files_list), file->rel_path); - makeExternalDirPathByNum(external_dst, + /* Handle zero sized files */ + if (file->size == 0) + { + file->write_size = 0; + return; + } + + /* construct from_fullpath & to_fullpath */ + if (file->external_dir_num == 0) + { + join_path_components(from_fullpath, arguments->from_root, file->rel_path); + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + } + else + { + char external_dst[MAXPGPATH]; + char *external_path = parray_get(arguments->external_dirs, + file->external_dir_num - 1); + + makeExternalDirPathByNum(external_dst, arguments->external_prefix, file->external_dir_num); - join_path_components(to_fullpath, external_dst, file->rel_path); - join_path_components(from_fullpath, external_path, file->rel_path); - } - - /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + join_path_components(to_fullpath, external_dst, file->rel_path); + join_path_components(from_fullpath, external_path, file->rel_path); + } - /* Check that file exist in previous backup */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - pgFile **prev_file_tmp = NULL; - prev_file_tmp = (pgFile **) parray_bsearch(arguments->prev_filelist, - file, pgFileCompareRelPathWithExternal); - if (prev_file_tmp) - { - /* File exists in previous backup */ - file->exists_in_prev = true; - prev_file = *prev_file_tmp; - } - } + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) + { + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); + return; + } - /* backup file */ - if (file->is_datafile && !file->is_cfs) - { - backup_data_file(file, from_fullpath, to_fullpath, - arguments->prev_start_lsn, - current.backup_mode, - instance_config.compress_alg, - instance_config.compress_level, - arguments->nodeInfo->checksum_version, - arguments->hdr_map, false); - } - else + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + pgFile **prevFileTmp = NULL; + prevFileTmp = (pgFile **) parray_bsearch(arguments->prev_filelist, + file, pgFileCompareRelPathWithExternal); + if (prevFileTmp) { - backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, - current.backup_mode, current.parent_backup, true); + /* File exists in previous backup */ + file->exists_in_prev = true; + prev_file = *prevFileTmp; } + } - if (file->write_size == FILE_NOT_FOUND) - continue; + /* backup file */ + if (file->is_datafile && !file->is_cfs) + { + backup_data_file(file, from_fullpath, to_fullpath, + arguments->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arguments->nodeInfo->checksum_version, + arguments->hdr_map, false); + } + else + { + backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, true); + } - if (file->write_size == BYTES_INVALID) - { - elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath); - continue; - } + if (file->write_size == FILE_NOT_FOUND) + return; - elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", - from_fullpath, file->write_size); + if (file->write_size == BYTES_INVALID) + { + elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath); + return; } - /* ssh connection to longer needed */ - fio_disconnect(); + elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", + from_fullpath, file->write_size); - /* Data files transferring is successful */ - arguments->ret = 0; +} - return NULL; +static void +backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { + pgFile *data_file = file; + pgFile *cfm_file = NULL; + pgFile *data_bck_file = NULL; + pgFile *cfm_bck_file = NULL; + + while (data_file->cfs_chain) + { + data_file = data_file->cfs_chain; + if (data_file->forkName == cfm) + cfm_file = data_file; + if (data_file->forkName == cfs_bck) + data_bck_file = data_file; + if (data_file->forkName == cfm_bck) + cfm_bck_file = data_file; + } + data_file = file; + Assert(cfm_file); /* ensure we always have cfm exist */ + + elog(LOG, "backup CFS segment %s, data_file=%s, cfm_file=%s, data_bck_file=%s, cfm_bck_file=%s", + data_file->name, data_file->name, cfm_file->name, data_bck_file == NULL? "NULL": data_bck_file->name, cfm_bck_file == NULL? "NULL": cfm_bck_file->name); + + /* storing cfs in order data_bck_file -> cfm_bck -> data_file -> map */ + if (cfm_bck_file) + process_file(i, cfm_bck_file, arguments); + if (data_bck_file) + process_file(i, data_bck_file, arguments); + process_file(i, cfm_file, arguments); + process_file(i, data_file, arguments); + elog(LOG, "Backup CFS segment %s done", data_file->name); } /* @@ -2209,11 +2270,12 @@ parse_filelist_filenames(parray *files, const char *root) */ if (strcmp(file->name, "pg_compression") == 0) { + /* processing potential cfs tablespace */ Oid tblspcOid; Oid dbOid; char tmp_rel_path[MAXPGPATH]; /* - * Check that the file is located under + * Check that pg_compression is located under * TABLESPACE_VERSION_DIRECTORY */ sscanf_result = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s/%u", @@ -2222,8 +2284,12 @@ parse_filelist_filenames(parray *files, const char *root) /* Yes, it is */ if (sscanf_result == 2 && strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY, - strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) - set_cfs_datafiles(files, root, file->rel_path, i); + strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) { + /* rewind index to the beginning of cfs tablespace */ + size_t start = rewind_and_mark_cfs_datafiles(files, root, file->rel_path, i); + /* group every to cfs segments chains */ + group_cfs_segments(files, start, i); + } } } @@ -2238,7 +2304,7 @@ parse_filelist_filenames(parray *files, const char *root) */ int unlogged_file_num = i - 1; pgFile *unlogged_file = (pgFile *) parray_get(files, - unlogged_file_num); + unlogged_file_num); unlogged_file_reloid = file->relOid; @@ -2246,11 +2312,10 @@ parse_filelist_filenames(parray *files, const char *root) (unlogged_file_reloid != 0) && (unlogged_file->relOid == unlogged_file_reloid)) { - pgFileFree(unlogged_file); - parray_remove(files, unlogged_file_num); + /* flagged to remove from list on stage 2 */ + unlogged_file->remove_from_list = true; unlogged_file_num--; - i--; unlogged_file = (pgFile *) parray_get(files, unlogged_file_num); @@ -2260,6 +2325,68 @@ parse_filelist_filenames(parray *files, const char *root) i++; } + + /* stage 2. clean up from temporary tables */ + parray_remove_if(files, remove_excluded_files_criterion, NULL, pgFileFree); +} + +static bool +remove_excluded_files_criterion(void *value, void *exclude_args) { + pgFile *file = (pgFile*)value; + return file->remove_from_list; +} + +/* + * For every cfs segment do group its files to linked list, datafile on the head. + * All non data files of segment moved to linked list and marked to skip in backup processing threads. + * @param first - first index of cfs tablespace files + * @param last - last index of cfs tablespace files + */ +void group_cfs_segments(parray *files, size_t first, size_t last) {/* grouping cfs files by relOid.segno, removing leafs of group */ + + for (;first <= last; first++) + { + pgFile *file = parray_get(files, first); + + if (file->is_cfs) + { + pgFile *cfs_file = file; + size_t counter = first + 1; + pgFile *chain_file = parray_get(files, counter); + + bool has_cfm = false; /* flag for later assertion the cfm file also exist */ + + elog(LOG, "Preprocessing cfs file %s, %u.%d", cfs_file->name, cfs_file->relOid, cfs_file->segno); + + elog(LOG, "Checking file %s, %u.%d as cfs chain", chain_file->name, chain_file->relOid, chain_file->segno); + + /* scanning cfs segment files */ + while (cfs_file->relOid == chain_file->relOid && + cfs_file->segno == chain_file->segno) + { + elog(LOG, "Grouping cfs chain file %s, %d.%d", chain_file->name, chain_file->relOid, chain_file->segno); + chain_file->skip_cfs_nested = true; + cfs_file->cfs_chain = chain_file; /* adding to cfs group */ + cfs_file = chain_file; + + /* next file */ + counter++; + chain_file = parray_get(files, counter); + elog(LOG, "Checking file %s, %u.%d as cfs chain", chain_file->name, chain_file->relOid, chain_file->segno); + } + + /* assertion - we always have cfs data + cfs map files */ + cfs_file = file; + for (; cfs_file; cfs_file = cfs_file->cfs_chain) { + elog(LOG, "searching cfm in %s, chain is %s", cfs_file->name, cfs_file->cfs_chain == NULL? "NULL": cfs_file->cfs_chain->name); + has_cfm = cfs_file->forkName == cfm; + } + Assert(has_cfm); + + /* shifting to last cfs segment file */ + first = counter-1; + } + } } /* If file is equal to pg_compression, then we consider this tablespace as @@ -2273,9 +2400,11 @@ parse_filelist_filenames(parray *files, const char *root) * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + * + * @returns index of first tablespace entry, i.e tblspcOid/TABLESPACE_VERSION_DIRECTORY */ -static void -set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +static size_t +rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) { int len; int p; @@ -2311,6 +2440,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) } } free(cfs_tblspc_path); + return p+1; } /* diff --git a/src/dir.c b/src/dir.c index 0a55c0f67..4bae25de2 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1837,7 +1837,19 @@ set_forkname(pgFile *file) return false; } - /* CFS "fork name" */ + /* CFS family fork names */ + if (file->forkName == none && + is_forkname(file->name, &i, ".cfm.bck")) + { + /* /^\d+(\.\d+)?\.cfm\.bck$/ */ + file->forkName = cfm_bck; + } + if (file->forkName == none && + is_forkname(file->name, &i, ".bck")) + { + /* /^\d+(\.\d+)?\.bck$/ */ + file->forkName = cfs_bck; + } if (file->forkName == none && is_forkname(file->name, &i, ".cfm")) { diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 6ff8a41b3..a5c17d9f8 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -222,7 +222,9 @@ typedef enum ForkName fsm, cfm, init, - ptrack + ptrack, + cfs_bck, + cfm_bck } ForkName; #define INIT_FILE_CRC32(use_crc32c, crc) \ @@ -278,6 +280,7 @@ typedef struct pgFile int segno; /* Segment number for ptrack */ int n_blocks; /* number of blocks in the data file in data directory */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ + struct pgFile *cfs_chain; /* linked list of CFS segment's cfm, bck, cfm_bck related files */ int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -292,6 +295,8 @@ typedef struct pgFile pg_off_t hdr_off; /* offset in header map */ int hdr_size; /* length of headers */ bool excluded; /* excluded via --exclude-path option */ + bool skip_cfs_nested; /* mark to skip in processing treads as nested to cfs_chain */ + bool remove_from_list; /* tmp flag to clean up files list from temp and unlogged tables */ } pgFile; typedef struct page_map_entry diff --git a/src/utils/parray.c b/src/utils/parray.c index 792e26907..65377c001 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -217,3 +217,30 @@ bool parray_contains(parray *array, void *elem) } return false; } + +/* effectively remove elements that satisfy certain criterion */ +void +parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean) { + int i = 0; + int j = 0; + + /* removing certain elements */ + while(j < parray_num(array)) { + void *value = array->data[j]; + // if the value satisfies the criterion, clean it up + if(criterion(value, args)) { + clean(value); + j++; + continue; + } + + if(i != j) + array->data[i] = array->data[j]; + + i++; + j++; + } + + /* adjust the number of used elements */ + array->used -= j - i; +} diff --git a/src/utils/parray.h b/src/utils/parray.h index e92ad728c..08846f252 100644 --- a/src/utils/parray.h +++ b/src/utils/parray.h @@ -16,6 +16,9 @@ */ typedef struct parray parray; +typedef bool (*criterion_fn)(void *value, void *args); +typedef void (*cleanup_fn)(void *ref); + extern parray *parray_new(void); extern void parray_expand(parray *array, size_t newnum); extern void parray_free(parray *array); @@ -32,6 +35,7 @@ extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const extern int parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)); extern void parray_walk(parray *array, void (*action)(void *)); extern bool parray_contains(parray *array, void *elem); +extern void parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean); #endif /* PARRAY_H */ From 2f27142009dc6c6c1783e07622b5f285b43577bc Mon Sep 17 00:00:00 2001 From: Ivan Lazarev Date: Thu, 12 Jan 2023 13:47:28 +0300 Subject: [PATCH 2025/2107] [PBCKP-287] skipping data_bck_file+cfm_bck_file on backup when they both exist --- src/backup.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/backup.c b/src/backup.c index 415f4a02a..6831ec8f6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2233,11 +2233,24 @@ backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { elog(LOG, "backup CFS segment %s, data_file=%s, cfm_file=%s, data_bck_file=%s, cfm_bck_file=%s", data_file->name, data_file->name, cfm_file->name, data_bck_file == NULL? "NULL": data_bck_file->name, cfm_bck_file == NULL? "NULL": cfm_bck_file->name); - /* storing cfs in order data_bck_file -> cfm_bck -> data_file -> map */ - if (cfm_bck_file) - process_file(i, cfm_bck_file, arguments); + /* storing cfs segment. processing corner case [PBCKP-287] stage 1. + * - when we do have data_bck_file we should skip both data_bck_file and cfm_bck_file if exists. + * they are removed by cfs_recover() during postgres start. + */ if (data_bck_file) - process_file(i, data_bck_file, arguments); + { + if (cfm_bck_file) + cfm_bck_file->write_size = FILE_NOT_FOUND; + data_bck_file->write_size = FILE_NOT_FOUND; + } + /* else we store cfm_bck_file. processing corner case [PBCKP-287] stage 2. + * - when we do have cfm_bck_file only we should store it. + * it will replace cfm_file after postgres start. + */ + else if (cfm_bck_file) + process_file(i, cfm_bck_file, arguments); + + /* storing cfs segment in order cfm_file -> datafile to guarantee their consistency */ process_file(i, cfm_file, arguments); process_file(i, data_file, arguments); elog(LOG, "Backup CFS segment %s done", data_file->name); From d3babee2ae5caafcff0ec106070af4a2955f5458 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 12 Jan 2023 15:23:34 +0300 Subject: [PATCH 2026/2107] [PBCKP-287] simplify and fix cfs chaining - properly chain only main fork and cfs related forks - properly chain datafile with its cfm even sort order places other segments between - don't raise error for system tables which has no companion cfm. + fix couple of tests --- src/backup.c | 142 ++++++++++++++++++++------------------- src/utils/pgut.h | 10 +++ tests/cfs_backup_test.py | 27 ++------ 3 files changed, 90 insertions(+), 89 deletions(-) diff --git a/src/backup.c b/src/backup.c index 6831ec8f6..af225017a 100644 --- a/src/backup.c +++ b/src/backup.c @@ -65,8 +65,7 @@ static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); -static size_t rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); -static void group_cfs_segments(parray *files, size_t first, size_t last); +static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); static bool remove_excluded_files_criterion(void *value, void *exclude_args); static void backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments); static void process_file(int i, pgFile *file, backup_files_arg *arguments); @@ -2228,7 +2227,11 @@ backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { cfm_bck_file = data_file; } data_file = file; - Assert(cfm_file); /* ensure we always have cfm exist */ + if (data_file->relOid >= FirstNormalObjectId && cfm_file == NULL) + { + elog(ERROR, "'CFS' file '%s' have to have '%s.cfm' companion file", + data_file->rel_path, data_file->name); + } elog(LOG, "backup CFS segment %s, data_file=%s, cfm_file=%s, data_bck_file=%s, cfm_bck_file=%s", data_file->name, data_file->name, cfm_file->name, data_bck_file == NULL? "NULL": data_bck_file->name, cfm_bck_file == NULL? "NULL": cfm_bck_file->name); @@ -2251,7 +2254,10 @@ backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { process_file(i, cfm_bck_file, arguments); /* storing cfs segment in order cfm_file -> datafile to guarantee their consistency */ - process_file(i, cfm_file, arguments); + /* cfm_file could be NULL for system tables. But we don't clear is_cfs flag + * for compatibility with older pg_probackup. */ + if (cfm_file) + process_file(i, cfm_file, arguments); process_file(i, data_file, arguments); elog(LOG, "Backup CFS segment %s done", data_file->name); } @@ -2299,9 +2305,7 @@ parse_filelist_filenames(parray *files, const char *root) strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY, strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) { /* rewind index to the beginning of cfs tablespace */ - size_t start = rewind_and_mark_cfs_datafiles(files, root, file->rel_path, i); - /* group every to cfs segments chains */ - group_cfs_segments(files, start, i); + rewind_and_mark_cfs_datafiles(files, root, file->rel_path, i); } } } @@ -2349,57 +2353,11 @@ remove_excluded_files_criterion(void *value, void *exclude_args) { return file->remove_from_list; } -/* - * For every cfs segment do group its files to linked list, datafile on the head. - * All non data files of segment moved to linked list and marked to skip in backup processing threads. - * @param first - first index of cfs tablespace files - * @param last - last index of cfs tablespace files - */ -void group_cfs_segments(parray *files, size_t first, size_t last) {/* grouping cfs files by relOid.segno, removing leafs of group */ - - for (;first <= last; first++) - { - pgFile *file = parray_get(files, first); - - if (file->is_cfs) - { - pgFile *cfs_file = file; - size_t counter = first + 1; - pgFile *chain_file = parray_get(files, counter); - - bool has_cfm = false; /* flag for later assertion the cfm file also exist */ - - elog(LOG, "Preprocessing cfs file %s, %u.%d", cfs_file->name, cfs_file->relOid, cfs_file->segno); - - elog(LOG, "Checking file %s, %u.%d as cfs chain", chain_file->name, chain_file->relOid, chain_file->segno); - - /* scanning cfs segment files */ - while (cfs_file->relOid == chain_file->relOid && - cfs_file->segno == chain_file->segno) - { - elog(LOG, "Grouping cfs chain file %s, %d.%d", chain_file->name, chain_file->relOid, chain_file->segno); - chain_file->skip_cfs_nested = true; - cfs_file->cfs_chain = chain_file; /* adding to cfs group */ - cfs_file = chain_file; - - /* next file */ - counter++; - chain_file = parray_get(files, counter); - elog(LOG, "Checking file %s, %u.%d as cfs chain", chain_file->name, chain_file->relOid, chain_file->segno); - } - - /* assertion - we always have cfs data + cfs map files */ - cfs_file = file; - for (; cfs_file; cfs_file = cfs_file->cfs_chain) { - elog(LOG, "searching cfm in %s, chain is %s", cfs_file->name, cfs_file->cfs_chain == NULL? "NULL": cfs_file->cfs_chain->name); - has_cfm = cfs_file->forkName == cfm; - } - Assert(has_cfm); - - /* shifting to last cfs segment file */ - first = counter-1; - } - } +static uint32_t +hash_rel_seg(pgFile* file) +{ + uint32 hash = hash_mix32_2(file->relOid, file->segno); + return hash_mix32_2(hash, 0xcf5); } /* If file is equal to pg_compression, then we consider this tablespace as @@ -2416,13 +2374,24 @@ void group_cfs_segments(parray *files, size_t first, size_t last) {/* grouping c * * @returns index of first tablespace entry, i.e tblspcOid/TABLESPACE_VERSION_DIRECTORY */ -static size_t +static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) { int len; int p; + int j; pgFile *prev_file; + pgFile *tmp_file; char *cfs_tblspc_path; + uint32_t h; + + /* hash table for cfm files */ +#define HASHN 128 + parray *hashtab[HASHN] = {NULL}; + parray *bucket; + for (p = 0; p < HASHN; p++) + hashtab[p] = parray_new(); + cfs_tblspc_path = strdup(relative); if(!cfs_tblspc_path) @@ -2437,23 +2406,60 @@ rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, s elog(LOG, "Checking file in cfs tablespace %s", prev_file->rel_path); - if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) + if (strstr(prev_file->rel_path, cfs_tblspc_path) == NULL) + { + elog(LOG, "Breaking on %s", prev_file->rel_path); + break; + } + + if (!S_ISREG(prev_file->mode)) + continue; + + h = hash_rel_seg(prev_file); + bucket = hashtab[h % HASHN]; + + if (prev_file->forkName == cfm || prev_file->forkName == cfm_bck || + prev_file->forkName == cfs_bck) { - if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + parray_append(bucket, prev_file); + } + else if (prev_file->is_datafile && prev_file->forkName == none) + { + elog(LOG, "Processing 'cfs' file %s", prev_file->rel_path); + /* have to mark as is_cfs even for system-tables for compatibility + * with older pg_probackup */ + prev_file->is_cfs = true; + prev_file->cfs_chain = NULL; + for (j = 0; j < parray_num(bucket); j++) { - elog(LOG, "Setting 'is_cfs' on file %s, name %s", - prev_file->rel_path, prev_file->name); - prev_file->is_cfs = true; + tmp_file = parray_get(bucket, j); + elog(LOG, "Linking 'cfs' file '%s' to '%s'", + tmp_file->rel_path, prev_file->rel_path); + if (tmp_file->relOid == prev_file->relOid && + tmp_file->segno == prev_file->segno) + { + tmp_file->cfs_chain = prev_file->cfs_chain; + prev_file->cfs_chain = tmp_file; + parray_remove(bucket, j); + j--; + } } } - else + } + + for (p = 0; p < HASHN; p++) + { + bucket = hashtab[p]; + for (j = 0; j < parray_num(bucket); j++) { - elog(LOG, "Breaking on %s", prev_file->rel_path); - break; + tmp_file = parray_get(bucket, j); + elog(WARNING, "Orphaned cfs related file '%s'", tmp_file->rel_path); } + parray_free(bucket); + hashtab[p] = NULL; } +#undef HASHN free(cfs_tblspc_path); - return p+1; } /* diff --git a/src/utils/pgut.h b/src/utils/pgut.h index f8554f9d0..4fd659b82 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -115,4 +115,14 @@ extern int usleep(unsigned int usec); #define ARG_SIZE_HINT static #endif +static inline uint32_t hash_mix32_2(uint32_t a, uint32_t b) +{ + b ^= (a<<7)|(a>>25); + a *= 0xdeadbeef; + b *= 0xcafeabed; + a ^= a >> 16; + b ^= b >> 15; + return a^b; +} + #endif /* PGUT_H */ diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py index cd2826d21..fb4a6c6b8 100644 --- a/tests/cfs_backup_test.py +++ b/tests/cfs_backup_test.py @@ -431,16 +431,10 @@ def test_page_doesnt_store_unchanged_cfm(self): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) + self.node.safe_psql("postgres", "checkpoint") + + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') self.assertTrue( find_by_extensions( @@ -449,16 +443,8 @@ def test_page_doesnt_store_unchanged_cfm(self): "ERROR: .cfm files not found in backup dir" ) - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') show_backup = self.show_pb(self.backup_dir, 'node', backup_id) self.assertEqual( @@ -1046,7 +1032,6 @@ def test_fullbackup_after_create_table_page_after_create_table_stream(self): ) # --- Make backup with not valid data(broken .cfm) --- # - @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_delete_random_cfm_file_from_tablespace_dir(self): From b240b9077263aed673d6cdd0bafe5a7032e904e0 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 13 Jan 2023 09:38:48 +0300 Subject: [PATCH 2027/2107] fix for < Pg12 --- src/backup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backup.c b/src/backup.c index af225017a..78a679244 100644 --- a/src/backup.c +++ b/src/backup.c @@ -13,6 +13,9 @@ #if PG_VERSION_NUM < 110000 #include "catalog/catalog.h" #endif +#if PG_VERSION_NUM < 120000 +#include "access/transam.h" +#endif #include "catalog/pg_tablespace.h" #include "pgtar.h" #include "streamutil.h" From 610216c6f8b1036bf6de94d92d6acef4ed71fb87 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 13 Jan 2023 15:31:03 +0300 Subject: [PATCH 2028/2107] ptrack_helpers.py: fix compare_pgdata --- tests/helpers/ptrack_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 067225d66..c96007448 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1875,7 +1875,7 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict # Compare directories restored_dirs = set(restored_pgdata['dirs']) - original_dirs = set(restored_pgdata['dirs']) + original_dirs = set(original_pgdata['dirs']) for directory in sorted(restored_dirs - original_dirs): fail = True @@ -1903,7 +1903,7 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict restored.mode) restored_files = set(restored_pgdata['files']) - original_files = set(restored_pgdata['files']) + original_files = set(original_pgdata['files']) for file in sorted(restored_files - original_files): # File is present in RESTORED PGDATA From 49bb374d0c9be08152318613a579625e83bc8b2b Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Sat, 24 Dec 2022 14:50:14 +0300 Subject: [PATCH 2029/2107] [PBCKP-365] Fixed test help_6 test. Added check_locale function for check that locale is installed. --- tests/option_test.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/option_test.py b/tests/option_test.py index eec1bab44..af4b12b71 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -3,7 +3,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import locale - class OptionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -220,12 +219,28 @@ def test_options_5(self): def test_help_6(self): """help options""" if ProbackupTest.enable_nls: - self.test_env['LC_ALL'] = 'ru_RU.utf-8' - with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: - self.assertEqual( - self.run_pb(["--help"]), - help_out.read().decode("utf-8") - ) + if check_locale('ru_RU.utf-8'): + self.test_env['LC_ALL'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + else: + self.skipTest( + "Locale ru_RU.utf-8 doesn't work. You need install ru_RU.utf-8 locale for this test") else: self.skipTest( 'You need configure PostgreSQL with --enabled-nls option for this test') + + +def check_locale(locale_name): + ret=True + old_locale = locale.setlocale(locale.LC_CTYPE,"") + try: + locale.setlocale(locale.LC_CTYPE, locale_name) + except locale.Error: + ret=False + finally: + locale.setlocale(locale.LC_CTYPE, old_locale) + return ret From 137814aa6fd3d4d050ec78d0412c4ed5c6574c93 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 19 Jan 2023 13:45:34 +0300 Subject: [PATCH 2030/2107] do_retention_merge: fix removing from to_keep_list --- src/delete.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/delete.c b/src/delete.c index 3f299d78b..c8a8c22ec 100644 --- a/src/delete.c +++ b/src/delete.c @@ -552,7 +552,12 @@ do_retention_merge(InstanceState *instanceState, parray *backup_list, /* Try to remove merged incremental backup from both keep and purge lists */ parray_rm(to_purge_list, tmp_backup, pgBackupCompareId); - parray_set(to_keep_list, i, NULL); + for (i = 0; i < parray_num(to_keep_list); i++) + if (parray_get(to_keep_list, i) == tmp_backup) + { + parray_set(to_keep_list, i, NULL); + break; + } } if (!no_validate) pgBackupValidate(full_backup, NULL); From 1f5991dadbd383f6bb8ff36de1d504531491a25f Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 20 Jan 2023 04:06:41 +0300 Subject: [PATCH 2031/2107] fix cfs handling: forgot to set skip_cfs_nested --- src/backup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backup.c b/src/backup.c index 78a679244..7e6e33c53 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2424,6 +2424,7 @@ rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, s if (prev_file->forkName == cfm || prev_file->forkName == cfm_bck || prev_file->forkName == cfs_bck) { + prev_file->skip_cfs_nested = true; parray_append(bucket, prev_file); } else if (prev_file->is_datafile && prev_file->forkName == none) From c8909b825ed973731342fe7369e2dd00e48c2c13 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 9 Feb 2023 15:33:10 +0300 Subject: [PATCH 2032/2107] Fix warning in get_backup_filelist pg_multixact contains files which looks like db files, but they are not. get_backup_filelist should not pass to set_forkname files not in db folder nor `global` folder. --- src/catalog.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index afbac28ab..aadc47bb1 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1142,7 +1142,10 @@ get_backup_filelist(pgBackup *backup, bool strict) if (!file->is_datafile || file->is_cfs) file->size = file->uncompressed_size; - if (file->external_dir_num == 0 && S_ISREG(file->mode)) + if (file->external_dir_num == 0 && + (file->dbOid != 0 || + path_is_prefix_of_path("global", file->rel_path)) && + S_ISREG(file->mode)) { bool is_datafile = file->is_datafile; set_forkname(file); From 297dd2a7eb23821bdb9c53aa4d137639103851fc Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 10 Feb 2023 15:07:25 +0300 Subject: [PATCH 2033/2107] test for previous commit --- tests/backup_test.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/backup_test.py b/tests/backup_test.py index fc1135cab..32a2cee50 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3573,3 +3573,36 @@ def test_start_time_few_nodes(self): show_backup2 = self.show_pb(backup_dir2, 'node2')[3] self.assertEqual(show_backup1['id'], show_backup2['id']) + def test_regress_issue_585(self): + """https://github.com/postgrespro/pg_probackup/issues/585""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # create couple of files that looks like db files + with open(os.path.join(node.data_dir, 'pg_multixact/offsets/1000'),'wb') as f: + pass + with open(os.path.join(node.data_dir, 'pg_multixact/members/1000'),'wb') as f: + pass + + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream']) + + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream'], + return_id=False, + ) + self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') + + node.cleanup() + + output = self.restore_node(backup_dir, 'node', node) + self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') From a2387b5134bc6f56b54079903b1845494d29252e Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 28 Feb 2023 20:53:02 +0300 Subject: [PATCH 2034/2107] [PBCKP-211] got rid of timezone hack, tzset() allowed on every platform Now if UTC is true in time2iso we just use strftime to format string and add +00 timezone --- src/utils/configuration.c | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 193d1c680..24c6febbf 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1294,9 +1294,7 @@ parse_time(const char *value, time_t *result, bool utc_default) { /* set timezone to UTC */ pgut_setenv("TZ", "UTC"); -#ifdef WIN32 tzset(); -#endif } /* convert time to utc unix time */ @@ -1308,9 +1306,7 @@ parse_time(const char *value, time_t *result, bool utc_default) else pgut_unsetenv("TZ"); -#ifdef WIN32 tzset(); -#endif /* adjust time zone */ if (tz_set || utc_default) @@ -1546,33 +1542,19 @@ time2iso(char *buf, size_t len, time_t time, bool utc) time_t gmt; time_t offset; char *ptr = buf; - char *local_tz = getenv("TZ"); /* set timezone to UTC if requested */ if (utc) { - pgut_setenv("TZ", "UTC"); -#ifdef WIN32 - tzset(); -#endif + ptm = gmtime(&time); + strftime(ptr, len, "%Y-%m-%d %H:%M:%S+00", ptm); + return; } ptm = gmtime(&time); gmt = mktime(ptm); ptm = localtime(&time); - if (utc) - { - /* return old timezone back if any */ - if (local_tz) - pgut_setenv("TZ", local_tz); - else - pgut_unsetenv("TZ"); -#ifdef WIN32 - tzset(); -#endif - } - /* adjust timezone offset */ offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); From 4c001e86f46c75619c03e96c3d2dc2a3bb8f4695 Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Fri, 13 Jan 2023 01:03:03 +0100 Subject: [PATCH 2035/2107] Style and typo fixes in pg_probackup.c, pg_probackup.h --- src/pg_probackup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index ed48178b4..0e371ef42 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -1186,8 +1186,8 @@ opt_datname_exclude_list(ConfigOption *opt, const char *arg) void opt_datname_include_list(ConfigOption *opt, const char *arg) { - if (strcmp(arg, "tempate0") == 0 || - strcmp(arg, "tempate1") == 0) + if (strcmp(arg, "template0") == 0 || + strcmp(arg, "template1") == 0) elog(ERROR, "Databases 'template0' and 'template1' cannot be used for partial restore or validation"); opt_parser_add_to_parray_helper(&datname_include_list, arg); From 7249b10be0d3a92cd983088c71abe397d3974eee Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Fri, 19 Aug 2022 06:10:58 +0300 Subject: [PATCH 2036/2107] [PBCKP-247]: typo in the option_get_value function. --- src/utils/configuration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 24c6febbf..61d153baa 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -689,7 +689,7 @@ option_get_value(ConfigOption *opt) if (opt->type == 'i') convert_from_base_unit(*((int32 *) opt->var), opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'i') + else if (opt->type == 'I') convert_from_base_unit(*((int64 *) opt->var), opt->flags & OPTION_UNIT, &value, &unit); else if (opt->type == 'u') From be2b90bd3fc96a20dee4a5755311923ca3b6aa11 Mon Sep 17 00:00:00 2001 From: Sergey Fukanchik Date: Mon, 6 Mar 2023 13:34:53 +0300 Subject: [PATCH 2037/2107] PBCKP-553 fix a typo in merge preconditions check --- src/merge.c | 2 +- tests/merge_test.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/merge.c b/src/merge.c index 0017c9e9c..3b8321e97 100644 --- a/src/merge.c +++ b/src/merge.c @@ -337,7 +337,7 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool else { if ((full_backup->status == BACKUP_STATUS_MERGED || - full_backup->status == BACKUP_STATUS_MERGED) && + full_backup->status == BACKUP_STATUS_MERGING) && dest_backup->start_time != full_backup->merge_dest_backup) { elog(ERROR, "Full backup %s has unfinished merge with backup %s", diff --git a/tests/merge_test.py b/tests/merge_test.py index c789298fd..a9bc6fe68 100644 --- a/tests/merge_test.py +++ b/tests/merge_test.py @@ -2734,5 +2734,46 @@ def test_merge_pg_filenode_map(self): 'postgres', 'select 1') + def test_unfinished_merge(self): + """ Test when parent has unfinished merge with a different backup. """ + self._check_gdb_flag_or_skip_test() + cases = [('fail_merged', 'write_backup_filelist', ['MERGED', 'MERGING', 'OK']), + ('fail_merging', 'pgBackupWriteControl', ['MERGING', 'OK', 'OK'])] + + for name, terminate_at, states in cases: + node_name = 'node_' + name + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, name) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, node_name), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, node_name, node) + self.set_archiving(backup_dir, node_name, node) + node.slow_start() + + full_id=self.backup_node(backup_dir, node_name, node, options=['--stream']) + + backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') + second_backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') + + gdb = self.merge_backup(backup_dir, node_name, backup_id, gdb=True) + gdb.set_breakpoint(terminate_at) + gdb.run_until_break() + + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + print(self.show_pb(backup_dir, node_name, as_json=False, as_text=True)) + + for expected, real in zip(states, self.show_pb(backup_dir, node_name), strict=True): + self.assertEqual(expected, real['status']) + + with self.assertRaisesRegex(ProbackupException, + f"Full backup {full_id} has unfinished merge with backup {backup_id}"): + self.merge_backup(backup_dir, node_name, second_backup_id, gdb=False) + # 1. Need new test with corrupted FULL backup # 2. different compression levels From 3c111262af9aa867eff56add03f2fc2366cadb91 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 3 Jan 2023 07:52:16 +0300 Subject: [PATCH 2038/2107] compatibility tests: skip if PGPROBACKUPBIN_OLD is not set --- tests/compatibility_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/compatibility_test.py b/tests/compatibility_test.py index 591afb069..7ae8baf9f 100644 --- a/tests/compatibility_test.py +++ b/tests/compatibility_test.py @@ -14,12 +14,7 @@ def check_ssh_agent_path_exists(): return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ -class CompatibilityTest(ProbackupTest, unittest.TestCase): - - def setUp(self): - self.fname = self.id().split('.')[3] - - # @unittest.expectedFailure +class CrossCompatibilityTest(ProbackupTest, unittest.TestCase): @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') # @unittest.skip("skip") @@ -86,6 +81,14 @@ def test_catchup_with_different_remote_major_pg(self): options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] ) + +class CompatibilityTest(ProbackupTest, unittest.TestCase): + + def setUp(self): + super().setUp() + if not self.probackup_old_path: + self.skipTest('PGPROBACKUPBIN_OLD is not set') + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): From 9727a98976f706be9348ac638667821f226d89bb Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Fri, 10 Mar 2023 18:24:46 +0300 Subject: [PATCH 2039/2107] PBCKP-422: Fix is_enterprise checking for upstream and pgpro. Add is_pgpro checking --- tests/checkdb_test.py | 2 +- tests/helpers/ptrack_helpers.py | 34 ++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index 1e6daefdb..4d3a4cbbf 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -808,7 +808,7 @@ def test_checkdb_with_least_privileges(self): "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( 'backupdb', 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index c96007448..f8044a814 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -90,27 +90,34 @@ def dir_files(base_dir): return out_list +def is_pgpro(): + # pg_config --help + cmd = [os.environ['PG_CONFIG'], '--help'] + + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return b'postgrespro' in result.stdout + + def is_enterprise(): # pg_config --help cmd = [os.environ['PG_CONFIG'], '--help'] - p = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - return b'postgrespro.ru' in p.communicate()[0] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + # PostgresPro std or ent + if b'postgrespro' in p.stdout: + cmd = [os.environ['PG_CONFIG'], '--pgpro-edition'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + + return b'enterprise' in p.stdout + else: # PostgreSQL + return False + - def is_nls_enabled(): cmd = [os.environ['PG_CONFIG'], '--configure'] - p = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - return b'enable-nls' in p.communicate()[0] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return b'enable-nls' in result.stdout def base36enc(number): @@ -229,6 +236,7 @@ class ProbackupTest(object): # Class attributes enterprise = is_enterprise() enable_nls = is_nls_enabled() + pgpro = is_pgpro() def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) From 2e6d20dba059fe2faf006ad01289263debab11b8 Mon Sep 17 00:00:00 2001 From: "s.fukanchik" Date: Sun, 12 Mar 2023 19:59:17 +0300 Subject: [PATCH 2040/2107] PBCKP-191 serialize wal segment push finalization --- src/archive.c | 135 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 120 insertions(+), 15 deletions(-) diff --git a/src/archive.c b/src/archive.c index 734602cac..e06d01b68 100644 --- a/src/archive.c +++ b/src/archive.c @@ -13,14 +13,6 @@ #include "utils/thread.h" #include "instr_time.h" -static int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout); -#ifdef HAVE_LIBZ -static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout); -#endif static void *push_files(void *arg); static void *get_files(void *arg); static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, @@ -91,8 +83,19 @@ typedef struct WALSegno { char name[MAXFNAMELEN]; volatile pg_atomic_flag lock; + volatile pg_atomic_uint32 done; + struct WALSegno* prev; } WALSegno; +static int push_file_internal_uncompressed(WALSegno *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + uint32 archive_timeout); +#ifdef HAVE_LIBZ +static int push_file_internal_gz(WALSegno *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int compress_level, uint32 archive_timeout); +#endif + static int push_file(WALSegno *xlogfile, const char *archive_status_dir, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout, @@ -337,16 +340,18 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, /* If compression is not required, then just copy it as is */ if (!is_compress) - rc = push_file_internal_uncompressed(xlogfile->name, pg_xlog_dir, + rc = push_file_internal_uncompressed(xlogfile, pg_xlog_dir, archive_dir, overwrite, no_sync, archive_timeout); #ifdef HAVE_LIBZ else - rc = push_file_internal_gz(xlogfile->name, pg_xlog_dir, archive_dir, + rc = push_file_internal_gz(xlogfile, pg_xlog_dir, archive_dir, overwrite, no_sync, compress_level, archive_timeout); #endif + pg_atomic_write_u32(&xlogfile->done, 1); + /* take '--no-ready-rename' flag into account */ if (!no_ready_rename && archive_status_dir != NULL) { @@ -381,13 +386,14 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, * has the same checksum */ int -push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, +push_file_internal_uncompressed(WALSegno *wal_file, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout) { FILE *in = NULL; int out = -1; char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ + const char *wal_file_name = wal_file->name; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; /* partial handling */ @@ -409,7 +415,10 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); + } /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -422,8 +431,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } /* Already existing destination temp file is not an error condition */ } else @@ -453,15 +465,21 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } else /* Successfully created partial file */ break; } else + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } /* first round */ @@ -492,8 +510,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (!partial_is_stale) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", to_fullpath_part, archive_timeout); + } /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); @@ -501,7 +522,10 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } part_opened: @@ -536,6 +560,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath); } @@ -553,6 +578,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (ferror(in)) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot read source file \"%s\": %s", from_fullpath, strerror(errno)); } @@ -560,6 +586,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to destination temp file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -575,14 +602,29 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (fio_check_error_fd(out, &errmsg)) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath_part, errmsg); } + if (wal_file->prev != NULL) + { + while (!pg_atomic_read_u32(&wal_file->prev->done)) + { + if (thread_interrupted || interrupted) + { + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "terminated while waiting for prev file"); + } + usleep(250); + } + } + /* close temp file */ if (fio_close(out) != 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot close temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -591,8 +633,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (!no_sync) { if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); @@ -603,6 +648,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", to_fullpath_part, to_fullpath, strerror(errno)); } @@ -620,13 +666,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d * has the same checksum */ int -push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, +push_file_internal_gz(WALSegno *wal_file, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, int compress_level, uint32 archive_timeout) { FILE *in = NULL; gzFile out = NULL; char *buf = pgut_malloc(OUT_BUF_SIZE); + const char *wal_file_name = wal_file->name; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; char to_fullpath_gz[MAXPGPATH]; @@ -656,8 +703,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_fullpath, strerror(errno)); + } /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -667,8 +717,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } /* Already existing destination temp file is not an error condition */ } else @@ -698,16 +751,22 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } else /* Successfully created partial file */ break; } else + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } /* first round */ @@ -738,8 +797,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (!partial_is_stale) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", to_fullpath_gz_part, archive_timeout); + } /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); @@ -747,8 +809,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (out == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } part_opened: @@ -784,6 +849,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath_gz); } @@ -801,6 +867,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (ferror(in)) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot read from source file \"%s\": %s", from_fullpath, strerror(errno)); } @@ -808,6 +875,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, get_gz_error(out, errno)); } @@ -823,14 +891,29 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (fio_check_error_fd_gz(out, &errmsg)) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", to_fullpath_gz_part, errmsg); } + if (wal_file->prev != NULL) + { + while (!pg_atomic_read_u32(&wal_file->prev->done)) + { + if (thread_interrupted || interrupted) + { + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "terminated while waiting for prev file"); + } + usleep(250); + } + } + /* close temp file, TODO: make it synchronous */ if (fio_gzclose(out) != 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); } @@ -839,8 +922,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (!no_sync) { if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } elog(LOG, "Rename \"%s\" to \"%s\"", @@ -852,6 +938,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); } @@ -905,6 +992,15 @@ get_gz_error(gzFile gzf, int errnum) // } //} +static int +walSegnoCompareName(const void *f1, const void *f2) +{ + WALSegno *w1 = *(WALSegno**)f1; + WALSegno *w2 = *(WALSegno**)f2; + + return strcmp(w1->name, w2->name); +} + /* Look for files with '.ready' suffix in archive_status directory * and pack such files into batch sized array. */ @@ -912,14 +1008,15 @@ parray * setup_push_filelist(const char *archive_status_dir, const char *first_file, int batch_size) { - int i; WALSegno *xlogfile = NULL; parray *status_files = NULL; parray *batch_files = parray_new(); + size_t i; /* guarantee that first filename is in batch list */ - xlogfile = palloc(sizeof(WALSegno)); + xlogfile = palloc0(sizeof(WALSegno)); pg_atomic_init_flag(&xlogfile->lock); + pg_atomic_init_u32(&xlogfile->done, 0); snprintf(xlogfile->name, MAXFNAMELEN, "%s", first_file); parray_append(batch_files, xlogfile); @@ -950,8 +1047,9 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, if (strcmp(filename, first_file) == 0) continue; - xlogfile = palloc(sizeof(WALSegno)); + xlogfile = palloc0(sizeof(WALSegno)); pg_atomic_init_flag(&xlogfile->lock); + pg_atomic_init_u32(&xlogfile->done, 0); snprintf(xlogfile->name, MAXFNAMELEN, "%s", filename); parray_append(batch_files, xlogfile); @@ -960,6 +1058,13 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, break; } + parray_qsort(batch_files, walSegnoCompareName); + for (i = 1; i < parray_num(batch_files); i++) + { + xlogfile = (WALSegno*) parray_get(batch_files, i); + xlogfile->prev = (WALSegno*) parray_get(batch_files, i-1); + } + /* cleanup */ parray_walk(status_files, pgFileFree); parray_free(status_files); From 91ebe718ba3653b1a1212429d68f4ae77e097611 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 16 Mar 2023 12:47:03 +0300 Subject: [PATCH 2041/2107] [PBCKP-528] Fixed memory leaks and some minor bugs. --- src/catalog.c | 13 ++++++++++--- src/delete.c | 8 ++++++++ src/dir.c | 4 ++-- src/merge.c | 4 ++-- src/show.c | 4 +++- src/stream.c | 2 +- src/utils/configuration.c | 26 ++++++++++++++++++++++++-- src/utils/pgut.c | 5 ++++- 8 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index aadc47bb1..1cf86cd24 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -891,7 +891,7 @@ catalog_get_instance_list(CatalogState *catalogState) instanceState = pgut_new(InstanceState); - strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); + strlcpy(instanceState->instance_name, dent->d_name, MAXPGPATH); join_path_components(instanceState->instance_backup_subdir_path, catalogState->backup_subdir_path, instanceState->instance_name); join_path_components(instanceState->instance_wal_subdir_path, @@ -2245,6 +2245,12 @@ do_set_backup(InstanceState *instanceState, time_t backup_id, if (set_backup_params->note) add_note(target_backup, set_backup_params->note); + /* Cleanup */ + if (backup_list) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } } /* @@ -2310,6 +2316,7 @@ add_note(pgBackup *target_backup, char *note) { char *note_string; + char *p; /* unset note */ if (pg_strcasecmp(note, "none") == 0) @@ -2326,8 +2333,8 @@ add_note(pgBackup *target_backup, char *note) * we save only "aaa" * Example: tests.set_backup.SetBackupTest.test_add_note_newlines */ - note_string = pgut_malloc(MAX_NOTE_SIZE); - sscanf(note, "%[^\n]", note_string); + p = strchr(note, '\n'); + note_string = pgut_strndup(note, p ? (p-note) : MAX_NOTE_SIZE); target_backup->note = note_string; elog(INFO, "Adding note to backup %s: '%s'", diff --git a/src/delete.c b/src/delete.c index c8a8c22ec..f48ecc95f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -158,7 +158,13 @@ void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync) /* Retention is disabled but we still can cleanup wal */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + parray_free(to_keep_list); + parray_free(to_purge_list); return; + } } else /* At least one retention policy is active */ @@ -1047,6 +1053,8 @@ do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, if (parray_num(backup_list) == 0) { elog(WARNING, "Instance '%s' has no backups", instanceState->instance_name); + parray_free(delete_list); + parray_free(backup_list); return; } diff --git a/src/dir.c b/src/dir.c index 4bae25de2..c6701929a 100644 --- a/src/dir.c +++ b/src/dir.c @@ -151,7 +151,7 @@ dir_create_dir(const char *dir, mode_t mode, bool strict) { char parent[MAXPGPATH]; - strncpy(parent, dir, MAXPGPATH); + strlcpy(parent, dir, MAXPGPATH); get_parent_directory(parent); /* Create parent first */ @@ -964,7 +964,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba if (links) { /* get parent dir of rel_path */ - strncpy(parent_dir, dir->rel_path, MAXPGPATH); + strlcpy(parent_dir, dir->rel_path, MAXPGPATH); get_parent_directory(parent_dir); /* check if directory is actually link to tablespace */ diff --git a/src/merge.c b/src/merge.c index 3b8321e97..c2751cde3 100644 --- a/src/merge.c +++ b/src/merge.c @@ -887,7 +887,7 @@ merge_chain(InstanceState *instanceState, pfree(threads); } - if (result_filelist && parray_num(result_filelist) > 0) + if (result_filelist) { parray_walk(result_filelist, pgFileFree); parray_free(result_filelist); @@ -1067,7 +1067,7 @@ merge_files(void *arg) tmp_file->hdr_crc = file->hdr_crc; } else - tmp_file->uncompressed_size = tmp_file->uncompressed_size; + tmp_file->uncompressed_size = file->uncompressed_size; /* Copy header metadata from old map into a new one */ tmp_file->n_headers = file->n_headers; diff --git a/src/show.c b/src/show.c index 2e06582ed..cc22a2acb 100644 --- a/src/show.c +++ b/src/show.c @@ -452,7 +452,7 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) appendPQExpBuffer(buf, INT64_FORMAT, backup->uncompressed_bytes); } - if (backup->uncompressed_bytes >= 0) + if (backup->pgdata_bytes >= 0) { json_add_key(buf, "pgdata-bytes", json_level); appendPQExpBuffer(buf, INT64_FORMAT, backup->pgdata_bytes); @@ -514,6 +514,8 @@ show_backup(InstanceState *instanceState, time_t requested_backup_id) elog(INFO, "Requested backup \"%s\" is not found.", /* We do not need free base36enc's result, we exit anyway */ base36enc(requested_backup_id)); + parray_walk(backups, pgBackupFree); + parray_free(backups); /* This is not error */ return 0; } diff --git a/src/stream.c b/src/stream.c index f7bbeae5a..73bea6780 100644 --- a/src/stream.c +++ b/src/stream.c @@ -648,7 +648,7 @@ start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOption //TODO Add a comment about this calculation stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; - strncpy(stream_thread_arg.basedir, stream_dst_path, sizeof(stream_thread_arg.basedir)); + strlcpy(stream_thread_arg.basedir, stream_dst_path, sizeof(stream_thread_arg.basedir)); /* * Connect in replication mode to the server. diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 61d153baa..08d024516 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -1177,7 +1177,8 @@ parse_time(const char *value, time_t *result, bool utc_default) char *local_tz = getenv("TZ"); /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + + 1); + tmp = pgut_malloc(strlen(value) + 1); + if(!tmp) return false; len = 0; fields_num = 1; @@ -1205,7 +1206,10 @@ parse_time(const char *value, time_t *result, bool utc_default) errno = 0; hr = strtol(value + 1, &cp, 10); if ((value + 1) == cp || errno == ERANGE) + { + pfree(tmp); return false; + } /* explicit delimiter? */ if (*cp == ':') @@ -1213,13 +1217,19 @@ parse_time(const char *value, time_t *result, bool utc_default) errno = 0; min = strtol(cp + 1, &cp, 10); if (errno == ERANGE) + { + pfree(tmp); return false; + } if (*cp == ':') { errno = 0; sec = strtol(cp + 1, &cp, 10); if (errno == ERANGE) + { + pfree(tmp); return false; + } } } /* otherwise, might have run things together... */ @@ -1234,11 +1244,20 @@ parse_time(const char *value, time_t *result, bool utc_default) /* Range-check the values; see notes in datatype/timestamp.h */ if (hr < 0 || hr > MAX_TZDISP_HOUR) + { + pfree(tmp); return false; + } if (min < 0 || min >= MINS_PER_HOUR) + { + pfree(tmp); return false; + } if (sec < 0 || sec >= SECS_PER_MINUTE) + { + pfree(tmp); return false; + } tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; if (*value == '-') @@ -1251,7 +1270,10 @@ parse_time(const char *value, time_t *result, bool utc_default) } /* wrong format */ else if (!IsSpace(*value)) + { + pfree(tmp); return false; + } else value++; } @@ -1268,7 +1290,7 @@ parse_time(const char *value, time_t *result, bool utc_default) i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", &tm.tm_year, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); - free(tmp); + pfree(tmp); if (i < 3 || i > 6) return false; diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6123c18d8..9559fa644 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1215,13 +1215,16 @@ pgut_pgfnames(const char *path, bool strict) } } + filenames[numnames] = NULL; + if (errno) { elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path); + pgut_pgfnames_cleanup(filenames); + closedir(dir); return NULL; } - filenames[numnames] = NULL; if (closedir(dir)) { From c0b8eb7c20a65240d6be448a48f426dba3b17477 Mon Sep 17 00:00:00 2001 From: "s.fukanchik" Date: Mon, 27 Mar 2023 21:47:17 +0300 Subject: [PATCH 2042/2107] PBCKP-91 fix grammar in probackup's log messages --- src/archive.c | 4 ++-- src/backup.c | 33 ++++++++++++++++--------------- src/catalog.c | 8 ++++---- src/catchup.c | 2 +- src/checkdb.c | 4 ++-- src/dir.c | 22 ++++++++++----------- src/fetch.c | 2 +- src/init.c | 13 +++++++------ src/merge.c | 4 ++-- src/pg_probackup.c | 20 ++++++++++++------- src/ptrack.c | 2 +- src/restore.c | 41 ++++++++++++++++++++------------------- src/util.c | 4 ++-- src/utils/configuration.c | 12 +++++++++--- src/validate.c | 4 ++-- tests/auth_test.py | 4 ++-- tests/backup_test.py | 8 ++++---- tests/checkdb_test.py | 4 ++-- tests/compression_test.py | 2 +- tests/init_test.py | 3 ++- tests/option_test.py | 19 ++++++++---------- tests/restore_test.py | 4 ++-- tests/set_backup_test.py | 2 +- 23 files changed, 117 insertions(+), 104 deletions(-) diff --git a/src/archive.c b/src/archive.c index e06d01b68..7d753c8b3 100644 --- a/src/archive.c +++ b/src/archive.c @@ -614,7 +614,7 @@ push_file_internal_uncompressed(WALSegno *wal_file, const char *pg_xlog_dir, if (thread_interrupted || interrupted) { pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "terminated while waiting for prev file"); + elog(ERROR, "Terminated while waiting for prev file"); } usleep(250); } @@ -903,7 +903,7 @@ push_file_internal_gz(WALSegno *wal_file, const char *pg_xlog_dir, if (thread_interrupted || interrupted) { pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "terminated while waiting for prev file"); + elog(ERROR, "Terminated while waiting for prev file"); } usleep(250); } diff --git a/src/backup.c b/src/backup.c index 7e6e33c53..4c2454558 100644 --- a/src/backup.c +++ b/src/backup.c @@ -84,7 +84,7 @@ backup_stopbackup_callback(bool fatal, void *userdata) */ if (backup_in_progress) { - elog(WARNING, "backup in progress, stop backup"); + elog(WARNING, "A backup is in progress, stopping it."); /* don't care about stop_lsn in case of error */ pg_stop_backup_send(st->conn, st->server_version, current.from_replica, exclusive_backup, NULL); } @@ -711,8 +711,9 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, char pretty_bytes[20]; if (!instance_config.pgdata) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); @@ -936,12 +937,12 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) if (nodeInfo->server_version < 90500) elog(ERROR, - "server version is %s, must be %s or higher", + "Server version is %s, must be %s or higher", nodeInfo->server_version_str, "9.5"); if (current.from_replica && nodeInfo->server_version < 90600) elog(ERROR, - "server version is %s, must be %s or higher for backup from replica", + "Server version is %s, must be %s or higher for backup from replica", nodeInfo->server_version_str, "9.6"); if (nodeInfo->pgpro_support) @@ -1050,7 +1051,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) res = pgut_execute(conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) - elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(conn)); + elog(ERROR, "Cannot get %s: %s", name, PQerrorMessage(conn)); block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); if ((endp && *endp) || block_size != blcksz) @@ -1439,7 +1440,7 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l } if (!current.stream && is_start_lsn && try_count == 30) - elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " + elog(WARNING, "By default pg_probackup assumes that WAL delivery method to be ARCHIVE. " "If continuous archiving is not set up, use '--stream' option to make autonomous backup. " "Otherwise check that continuous archiving works correctly."); @@ -1775,9 +1776,9 @@ pg_stop_backup_consume(PGconn *conn, int server_version, { pgut_cancel(conn); #if PG_VERSION_NUM >= 150000 - elog(ERROR, "interrupted during waiting for pg_backup_stop"); + elog(ERROR, "Interrupted during waiting for pg_backup_stop"); #else - elog(ERROR, "interrupted during waiting for pg_stop_backup"); + elog(ERROR, "Interrupted during waiting for pg_stop_backup"); #endif } @@ -1823,7 +1824,7 @@ pg_stop_backup_consume(PGconn *conn, int server_version, case PGRES_TUPLES_OK: break; default: - elog(ERROR, "query failed: %s query was: %s", + elog(ERROR, "Query failed: %s query was: %s", PQerrorMessage(conn), query_text); } backup_in_progress = false; @@ -1834,13 +1835,13 @@ pg_stop_backup_consume(PGconn *conn, int server_version, /* get&check recovery_xid */ if (sscanf(PQgetvalue(query_result, 0, recovery_xid_colno), XID_FMT, &result->snapshot_xid) != 1) elog(ERROR, - "result of txid_snapshot_xmax() is invalid: %s", + "Result of txid_snapshot_xmax() is invalid: %s", PQgetvalue(query_result, 0, recovery_xid_colno)); /* get&check recovery_time */ if (!parse_time(PQgetvalue(query_result, 0, recovery_time_colno), &result->invocation_time, true)) elog(ERROR, - "result of current_timestamp is invalid: %s", + "Result of current_timestamp is invalid: %s", PQgetvalue(query_result, 0, recovery_time_colno)); /* get stop_backup_lsn */ @@ -1898,13 +1899,13 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c join_path_components(full_filename, path, filename); fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "can't open %s file \"%s\": %s", + elog(ERROR, "Can't open %s file \"%s\": %s", error_msg_filename, full_filename, strerror(errno)); if (fio_fwrite(fp, data, len) != len || fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "can't write %s file \"%s\": %s", + elog(ERROR, "Can't write %s file \"%s\": %s", error_msg_filename, full_filename, strerror(errno)); /* @@ -1943,7 +1944,7 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb /* Remove it ? */ if (!backup_in_progress) - elog(ERROR, "backup is not in progress"); + elog(ERROR, "Backup is not in progress"); pg_silent_client_messages(pg_startbackup_conn); @@ -2098,7 +2099,7 @@ backup_files(void *arg) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during backup"); + elog(ERROR, "Interrupted during backup"); elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_backup_files_list, file->rel_path); diff --git a/src/catalog.c b/src/catalog.c index 1cf86cd24..b29090789 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1055,7 +1055,7 @@ get_backup_filelist(pgBackup *backup, bool strict) fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); + elog(ERROR, "Cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); /* enable stdio buffering for local file */ if (!fio_is_remote(FIO_BACKUP_HOST)) @@ -2841,7 +2841,7 @@ parse_backup_mode(const char *value) return BACKUP_MODE_DIFF_DELTA; /* Backup mode is invalid, so leave with an error */ - elog(ERROR, "invalid backup-mode \"%s\"", value); + elog(ERROR, "Invalid backup-mode \"%s\"", value); return BACKUP_MODE_INVALID; } @@ -2876,7 +2876,7 @@ parse_compress_alg(const char *arg) len = strlen(arg); if (len == 0) - elog(ERROR, "compress algorithm is empty"); + elog(ERROR, "Compress algorithm is empty"); if (pg_strncasecmp("zlib", arg, len) == 0) return ZLIB_COMPRESS; @@ -2885,7 +2885,7 @@ parse_compress_alg(const char *arg) else if (pg_strncasecmp("none", arg, len) == 0) return NONE_COMPRESS; else - elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + elog(ERROR, "Invalid compress algorithm value \"%s\"", arg); return NOT_DEFINED_COMPRESS; } diff --git a/src/catchup.c b/src/catchup.c index 79e3361a8..427542dda 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -184,7 +184,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (source_node_info->ptrack_version_num == 0) elog(ERROR, "This PostgreSQL instance does not support ptrack"); else if (source_node_info->ptrack_version_num < 200) - elog(ERROR, "ptrack extension is too old.\n" + elog(ERROR, "Ptrack extension is too old.\n" "Upgrade ptrack to version >= 2"); else if (!source_node_info->is_ptrack_enabled) elog(ERROR, "Ptrack is disabled"); diff --git a/src/checkdb.c b/src/checkdb.c index 1133a7b5d..2a7d4e9eb 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -145,7 +145,7 @@ check_files(void *arg) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during checkdb"); + elog(ERROR, "Interrupted during checkdb"); /* No need to check directories */ if (S_ISDIR(file->mode)) @@ -750,7 +750,7 @@ do_checkdb(bool need_amcheck, if (!skip_block_validation) { if (!pgdata) - elog(ERROR, "required parameter not specified: PGDATA " + elog(ERROR, "Required parameter not specified: PGDATA " "(-D, --pgdata)"); /* get node info */ diff --git a/src/dir.c b/src/dir.c index c6701929a..a16e0f396 100644 --- a/src/dir.c +++ b/src/dir.c @@ -182,7 +182,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, /* file not found is not an error case */ if (errno == ENOENT) return NULL; - elog(ERROR, "cannot stat file \"%s\": %s", path, + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); } @@ -787,14 +787,14 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, for (arg_ptr = arg; *arg_ptr; arg_ptr++) { if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "directory name too long"); + elog(ERROR, "Directory name too long"); if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') ; /* skip backslash escaping = */ else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) { if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in %s mapping\n", type); + elog(ERROR, "Multiple \"=\" signs in %s mapping\n", type); else dst = dst_ptr = cell->new_dir; } @@ -803,7 +803,7 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, } if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid %s mapping format \"%s\", " + elog(ERROR, "Invalid %s mapping format \"%s\", " "must be \"OLDDIR=NEWDIR\"", type, arg); canonicalize_path(cell->old_dir); canonicalize_path(cell->new_dir); @@ -815,11 +815,11 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, * consistent with the new_dir check. */ if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path in %s mapping: %s\n", + elog(ERROR, "Old directory is not an absolute path in %s mapping: %s\n", type, cell->old_dir); if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path in %s mapping: %s\n", + elog(ERROR, "New directory is not an absolute path in %s mapping: %s\n", type, cell->new_dir); if (list->tail) @@ -1046,7 +1046,7 @@ read_tablespace_map(parray *links, const char *backup_dir) int i = 0; if (sscanf(buf, "%s %n", link_name, &n) != 1) - elog(ERROR, "invalid format found in \"%s\"", map_path); + elog(ERROR, "Invalid format found in \"%s\"", map_path); path = buf + n; @@ -1438,7 +1438,7 @@ get_control_value_str(const char *str, const char *name, { /* verify if value_str not exceeds value_str_size limits */ if (value_str - value_str_start >= value_str_size - 1) { - elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", + elog(ERROR, "Field \"%s\" is out of range in the line %s of the file %s", name, str, DATABASE_FILE_LIST); } *value_str = *buf; @@ -1463,7 +1463,7 @@ get_control_value_str(const char *str, const char *name, /* Did not find target field */ if (is_mandatory) - elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + elog(ERROR, "Field \"%s\" is not found in the line %s of the file %s", name, str, DATABASE_FILE_LIST); return false; } @@ -1490,7 +1490,7 @@ dir_is_empty(const char *path, fio_location location) /* Directory in path doesn't exist */ if (errno == ENOENT) return true; - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", path, strerror(errno)); } errno = 0; @@ -1506,7 +1506,7 @@ dir_is_empty(const char *path, fio_location location) return false; } if (errno) - elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot read directory \"%s\": %s", path, strerror(errno)); fio_closedir(dir); diff --git a/src/fetch.c b/src/fetch.c index bef30dac6..5401d815e 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -92,7 +92,7 @@ fetchFile(PGconn *conn, const char *filename, size_t *filesize) /* sanity check the result set */ if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) - elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + elog(ERROR, "Unexpected result set while fetching remote file \"%s\"", filename); /* Read result to local variables */ diff --git a/src/init.c b/src/init.c index 8773016b5..837e2bad0 100644 --- a/src/init.c +++ b/src/init.c @@ -24,11 +24,11 @@ do_init(CatalogState *catalogState) results = pg_check_dir(catalogState->catalog_path); if (results == 4) /* exists and not empty*/ - elog(ERROR, "backup catalog already exist and it's not empty"); + elog(ERROR, "The backup catalog already exists and is not empty"); else if (results == -1) /*trouble accessing directory*/ { int errno_tmp = errno; - elog(ERROR, "cannot open backup catalog directory \"%s\": %s", + elog(ERROR, "Cannot open backup catalog directory \"%s\": %s", catalogState->catalog_path, strerror(errno_tmp)); } @@ -41,7 +41,7 @@ do_init(CatalogState *catalogState) /* create backup catalog wal directory */ dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); - elog(INFO, "Backup catalog '%s' successfully inited", catalogState->catalog_path); + elog(INFO, "Backup catalog '%s' successfully initialized", catalogState->catalog_path); return 0; } @@ -53,8 +53,9 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) /* PGDATA is always required */ if (instance->pgdata == NULL) - elog(ERROR, "Required parameter not specified: PGDATA " - "(-D, --pgdata)"); + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); /* Read system_identifier from PGDATA */ instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); @@ -121,6 +122,6 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) /* pgdata was set through command line */ do_set_config(instanceState, true); - elog(INFO, "Instance '%s' successfully inited", instanceState->instance_name); + elog(INFO, "Instance '%s' successfully initialized", instanceState->instance_name); return 0; } diff --git a/src/merge.c b/src/merge.c index c2751cde3..e8f926795 100644 --- a/src/merge.c +++ b/src/merge.c @@ -79,10 +79,10 @@ do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool int i; if (backup_id == INVALID_BACKUP_ID) - elog(ERROR, "required parameter is not specified: --backup-id"); + elog(ERROR, "Required parameter is not specified: --backup-id"); if (instanceState == NULL) - elog(ERROR, "required parameter is not specified: --instance"); + elog(ERROR, "Required parameter is not specified: --instance"); elog(INFO, "Merge started"); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 0e371ef42..505dff89b 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -491,7 +491,10 @@ main(int argc, char *argv[]) backup_subcmd != HELP_CMD && backup_subcmd != VERSION_CMD && backup_subcmd != CATCHUP_CMD) - elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); + elog(ERROR, + "No backup catalog path specified.\n" + "Please specify it either using environment variable BACKUP_PATH or\n" + "command line option --backup-path (-B)"); /* ===== catalogState (END) ======*/ @@ -505,7 +508,7 @@ main(int argc, char *argv[]) { if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD && backup_subcmd != CATCHUP_CMD) - elog(ERROR, "required parameter not specified: --instance"); + elog(ERROR, "Required parameter not specified: --instance"); } else { @@ -618,7 +621,7 @@ main(int argc, char *argv[]) backup_path != NULL && instance_name == NULL && instance_config.pgdata == NULL) - elog(ERROR, "required parameter not specified: --instance"); + elog(ERROR, "Required parameter not specified: --instance"); /* Check checkdb command options consistency */ if (backup_subcmd == CHECKDB_CMD && @@ -831,14 +834,16 @@ main(int argc, char *argv[]) if (catchup_destination_pgdata == NULL) elog(ERROR, "You must specify \"--destination-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "Required parameter not specified: BACKUP_MODE (-b, --backup-mode)"); + elog(ERROR, "No backup mode specified.\n" + "Please specify it either using environment variable BACKUP_MODE or\n" + "command line option --backup-mode (-b)"); if (current.backup_mode != BACKUP_MODE_FULL && current.backup_mode != BACKUP_MODE_DIFF_PTRACK && current.backup_mode != BACKUP_MODE_DIFF_DELTA) elog(ERROR, "Only \"FULL\", \"PTRACK\" and \"DELTA\" modes are supported with the \"%s\" command", get_subcmd_name(backup_subcmd)); if (!stream_wal) elog(INFO, "--stream is required, forcing stream mode"); current.stream = stream_wal = true; if (instance_config.external_dir_str) - elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); + elog(ERROR, "External directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); // TODO check instance_config.conn_opt } @@ -985,8 +990,9 @@ main(int argc, char *argv[]) /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "required parameter not specified: BACKUP_MODE " - "(-b, --backup-mode)"); + elog(ERROR, "No backup mode specified.\n" + "Please specify it either using environment variable BACKUP_MODE or\n" + "command line option --backup-mode (-b)"); return do_backup(instanceState, set_backup_params, no_validate, no_sync, backup_logs, start_time); diff --git a/src/ptrack.c b/src/ptrack.c index ebcba1dd4..d27629e45 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -214,7 +214,7 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, pfree(params[0]); if (PQnfields(res) != 2) - elog(ERROR, "cannot get ptrack pagemapset"); + elog(ERROR, "Cannot get ptrack pagemapset"); /* sanity ? */ diff --git a/src/restore.c b/src/restore.c index 6c0e1881f..bb38e8d7e 100644 --- a/src/restore.c +++ b/src/restore.c @@ -131,13 +131,14 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg XLogRecPtr shift_lsn = InvalidXLogRecPtr; if (instanceState == NULL) - elog(ERROR, "required parameter not specified: --instance"); + elog(ERROR, "Required parameter not specified: --instance"); if (params->is_restore) { if (instance_config.pgdata == NULL) - elog(ERROR, - "required parameter not specified: PGDATA (-D, --pgdata)"); + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); /* Check if restore destination empty */ if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) @@ -290,7 +291,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (!satisfy_timeline(timelines, current_backup->tli, current_backup->stop_lsn)) { if (target_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "target backup %s does not satisfy target timeline", + elog(ERROR, "Target backup %s does not satisfy target timeline", base36enc(target_backup_id)); else /* Try to find another backup that satisfies target timeline */ @@ -776,7 +777,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, use_bitmap = false; if (params->incremental_mode != INCR_NONE) - elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version"); + elog(ERROR, "Incremental restore is not possible for backups older than 2.3.0 version"); } /* There is no point in bitmap restore, when restoring a single FULL backup, @@ -1479,7 +1480,7 @@ update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backu fp = fio_fopen(path, "w", FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, + elog(ERROR, "Cannot open file \"%s\": %s", path, strerror(errno)); if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) @@ -1499,7 +1500,7 @@ update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backu if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(ERROR, "Cannot write file \"%s\": %s", path, strerror(errno)); } #endif @@ -1538,7 +1539,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, { /* file not found is not an error case */ if (errno != ENOENT) - elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path, + elog(ERROR, "Cannot stat file \"%s\": %s", postgres_auto_path, strerror(errno)); st.st_size = 0; } @@ -1548,13 +1549,13 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, { fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); + elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); } sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); if (fp_tmp == NULL) - elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); + elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); while (fp && fgets(line, lengthof(line), fp)) { @@ -1612,7 +1613,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, { fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", postgres_auto_path, + elog(ERROR, "Cannot open file \"%s\": %s", postgres_auto_path, strerror(errno)); fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", @@ -1626,7 +1627,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", postgres_auto_path, + elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path, strerror(errno)); /* @@ -1646,12 +1647,12 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, + elog(ERROR, "Cannot open file \"%s\": %s", path, strerror(errno)); if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(ERROR, "Cannot write file \"%s\": %s", path, strerror(errno)); } @@ -1662,12 +1663,12 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, + elog(ERROR, "Cannot open file \"%s\": %s", path, strerror(errno)); if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(ERROR, "Cannot write file \"%s\": %s", path, strerror(errno)); } } @@ -1704,12 +1705,12 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict if (fd == NULL) { if (errno != ENOENT) - elog(ERROR, "could not open file \"%s\": %s", path, + elog(ERROR, "Could not open file \"%s\": %s", path, strerror(errno)); /* There is no history file for target timeline */ if (strict) - elog(ERROR, "recovery target timeline %u does not exist", + elog(ERROR, "Recovery target timeline %u does not exist", targetTLI); else return NULL; @@ -1743,12 +1744,12 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict { /* expect a numeric timeline ID as first field of line */ elog(ERROR, - "syntax error in history file: %s. Expected a numeric timeline ID.", + "Syntax error in history file: %s. Expected a numeric timeline ID.", fline); } if (nfields != 3) elog(ERROR, - "syntax error in history file: %s. Expected a transaction log switchpoint location.", + "Syntax error in history file: %s. Expected a transaction log switchpoint location.", fline); if (last_timeline && tli <= last_timeline->tli) diff --git a/src/util.c b/src/util.c index e371d2c6d..1407f03cc 100644 --- a/src/util.c +++ b/src/util.c @@ -74,7 +74,7 @@ checkControlFile(ControlFileData *ControlFile) if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) && ControlFile->pg_control_version / 65536 != 0) - elog(ERROR, "possible byte ordering mismatch\n" + elog(ERROR, "Possible byte ordering mismatch\n" "The byte ordering used to store the pg_control file might not match the one\n" "used by this program. In that case the results below would be incorrect, and\n" "the PostgreSQL installation would be incompatible with this data directory."); @@ -93,7 +93,7 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) #endif if (size != ControlFileSize) - elog(ERROR, "unexpected control file size %d, expected %d", + elog(ERROR, "Unexpected control file size %d, expected %d", (int) size, ControlFileSize); memcpy(ControlFile, src, sizeof(ControlFileData)); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 08d024516..921555350 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -521,11 +521,17 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], optstring = longopts_to_optstring(longopts, cmd_len + len); + opterr = 0; /* Assign named options */ while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { ConfigOption *opt; + if (c == '?') + { + elog(ERROR, "Option '%s' requires an argument. Try \"%s --help\" for more information.", + argv[optind-1], PROGRAM_NAME); + } opt = option_find(c, cmd_options); if (opt == NULL) opt = option_find(c, options); @@ -1439,16 +1445,16 @@ parse_lsn(const char *value, XLogRecPtr *result) len1 = strspn(value, "0123456789abcdefABCDEF"); if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; else { - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); return false; } diff --git a/src/validate.c b/src/validate.c index 9372b082c..471351678 100644 --- a/src/validate.c +++ b/src/validate.c @@ -394,7 +394,7 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) /* open directory and list contents */ dir = opendir(catalogState->backup_subdir_path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); errno = 0; while ((dent = readdir(dir))) @@ -412,7 +412,7 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) join_path_components(child, catalogState->backup_subdir_path, dent->d_name); if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; diff --git a/tests/auth_test.py b/tests/auth_test.py index 52d7e1544..32cabc4a1 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -117,13 +117,13 @@ def test_backup_via_unprivileged_user(self): except ProbackupException as e: if self.get_version(node) < 150000: self.assertIn( - "ERROR: query failed: ERROR: permission denied " + "ERROR: Query failed: ERROR: permission denied " "for function pg_stop_backup", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) else: self.assertIn( - "ERROR: query failed: ERROR: permission denied " + "ERROR: Query failed: ERROR: permission denied " "for function pg_backup_stop", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/backup_test.py b/tests/backup_test.py index 32a2cee50..9e911893e 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -2926,9 +2926,9 @@ def test_missing_wal_segment(self): gdb.output) self.assertIn( - 'WARNING: backup in progress, stop backup', + 'WARNING: A backup is in progress, stopping it', gdb.output) - + # TODO: check the same for PAGE backup # @unittest.skip("skip") @@ -3316,7 +3316,7 @@ def test_backup_atexit(self): log_content = f.read() #print(log_content) self.assertIn( - 'WARNING: backup in progress, stop backup', + 'WARNING: A backup is in progress, stopping it.', log_content) if self.get_version(node) < 150000: @@ -3327,7 +3327,7 @@ def test_backup_atexit(self): self.assertIn( 'FROM pg_catalog.pg_backup_stop', log_content) - + self.assertIn( 'setting its status to ERROR', log_content) diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index 4d3a4cbbf..eb46aea19 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -131,7 +131,7 @@ def test_checkdb_amcheck_only_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: required parameter not specified: --instance", + "ERROR: Required parameter not specified: --instance", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -397,7 +397,7 @@ def test_checkdb_block_validation_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: required parameter not specified: PGDATA (-D, --pgdata)", + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) diff --git a/tests/compression_test.py b/tests/compression_test.py index e779f6472..55924b9d2 100644 --- a/tests/compression_test.py +++ b/tests/compression_test.py @@ -443,7 +443,7 @@ def test_compression_wrong_algorithm(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: invalid compress algorithm value "bla-blah"\n', + 'ERROR: Invalid compress algorithm value "bla-blah"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/init_test.py b/tests/init_test.py index 94b076fef..4e000c78f 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -56,7 +56,8 @@ def test_success(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", + "ERROR: No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\ncommand line option --pgdata (-D)", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) diff --git a/tests/option_test.py b/tests/option_test.py index af4b12b71..66cc13746 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -33,7 +33,9 @@ def test_without_backup_path_3(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)', + 'ERROR: No backup catalog path specified.\n' + \ + 'Please specify it either using environment variable BACKUP_PATH or\n' + \ + 'command line option --backup-path (-B)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -54,7 +56,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: --instance', + 'ERROR: Required parameter not specified: --instance', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -65,7 +67,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + 'ERROR: No backup mode specified.\nPlease specify it either using environment variable BACKUP_MODE or\ncommand line option --backup-mode (-b)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -76,7 +78,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: invalid backup-mode "bad"', + 'ERROR: Invalid backup-mode "bad"', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -102,7 +104,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "option requires an argument -- 'i'", + "Option '-i' requires an argument", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -114,13 +116,8 @@ def test_options_5(self): base_dir=os.path.join(self.module_name, self.fname, 'node')) output = self.init_pb(backup_dir) - self.assertIn( - "INFO: Backup catalog", - output) + self.assertIn(f"INFO: Backup catalog '{backup_dir}' successfully initialized", output) - self.assertIn( - "successfully inited", - output) self.add_instance(backup_dir, 'node', node) node.slow_start() diff --git a/tests/restore_test.py b/tests/restore_test.py index da3ebffb4..67e99515c 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -2995,7 +2995,7 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' + 'ERROR: Field "dbOid" is not found in the line 42 of ' 'the file backup_content.control', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3011,7 +3011,7 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' + 'ERROR: Field "dbOid" is not found in the line 42 of ' 'the file backup_content.control', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py index e789d174a..31334cfba 100644 --- a/tests/set_backup_test.py +++ b/tests/set_backup_test.py @@ -41,7 +41,7 @@ def test_set_backup_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: --instance', + 'ERROR: Required parameter not specified: --instance', e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) From a04f00aacc548ad1ff1ee457bc0e2b8707bba4b3 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 28 Mar 2023 17:23:44 +0700 Subject: [PATCH 2043/2107] Up version --- src/pg_probackup.h | 2 +- tests/expected/option_version.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index a5c17d9f8..bcea92804 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -352,7 +352,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.11" +#define PROGRAM_VERSION "2.5.12" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20509 diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out index e0d6924b9..0d50cb268 100644 --- a/tests/expected/option_version.out +++ b/tests/expected/option_version.out @@ -1 +1 @@ -pg_probackup 2.5.11 +pg_probackup 2.5.12 From 279c98140ffcc38425e31ad17abe1b980d3154c6 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Sat, 15 Apr 2023 14:47:38 +0300 Subject: [PATCH 2044/2107] PGPRO-552: use uint32 in some places --- src/backup.c | 4 ++-- src/utils/pgut.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.c b/src/backup.c index 4c2454558..41f035a86 100644 --- a/src/backup.c +++ b/src/backup.c @@ -2357,7 +2357,7 @@ remove_excluded_files_criterion(void *value, void *exclude_args) { return file->remove_from_list; } -static uint32_t +static uint32 hash_rel_seg(pgFile* file) { uint32 hash = hash_mix32_2(file->relOid, file->segno); @@ -2387,7 +2387,7 @@ rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, s pgFile *prev_file; pgFile *tmp_file; char *cfs_tblspc_path; - uint32_t h; + uint32 h; /* hash table for cfm files */ #define HASHN 128 diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 4fd659b82..1b7b7864c 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -115,7 +115,7 @@ extern int usleep(unsigned int usec); #define ARG_SIZE_HINT static #endif -static inline uint32_t hash_mix32_2(uint32_t a, uint32_t b) +static inline uint32 hash_mix32_2(uint32 a, uint32 b) { b ^= (a<<7)|(a>>25); a *= 0xdeadbeef; From b7551bd2bf5f0173225111ed946a7e221ea93647 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 18 Apr 2023 18:25:36 +0300 Subject: [PATCH 2045/2107] PBCKP-91: delete "\r" in windows command output. --- tests/helpers/ptrack_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f8044a814..6b665097c 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -982,7 +982,8 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8'), self.cmd) + raise ProbackupException(e.output.decode('utf-8').replace("\r",""), + self.cmd) def run_binary(self, command, asynchronous=False, env=None): From d6721662ec76257d9470b1d20d75b7bc6bb1501c Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Tue, 18 Apr 2023 17:24:34 +0300 Subject: [PATCH 2046/2107] Use _chsize_s on windows to grow file Looks like "seek after file end and write" doesn't work correctly in Windows. We have to grow file before. Met occasionally on GitHub actions run. Could not reproduce locally. --- src/utils/file.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/utils/file.c b/src/utils/file.c index c4ed9c721..e062a2133 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2717,6 +2717,14 @@ fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len) if (len == 0) return true; +#ifdef WIN32 + if (st->read_size > st->write_size && + _chsize_s(fileno(out), st->read_size) != 0) + { + elog(WARNING, "Could not change file size to %lld: %m", st->read_size); + return false; + } +#endif if (st->read_size > st->write_size && fseeko(out, st->read_size, SEEK_SET) != 0) { From 0690f8d10eef4926bbe236c88327be55eba242f2 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Mon, 24 Apr 2023 18:22:28 +0300 Subject: [PATCH 2047/2107] [PBCKP-602] Added saving full size for non data files. --- src/data.c | 2 ++ tests/backup_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/data.c b/src/data.c index 490faf9b6..21c41e0b6 100644 --- a/src/data.c +++ b/src/data.c @@ -815,6 +815,8 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) { file->write_size = BYTES_INVALID; + /* get full size from previous backup for unchanged file */ + file->uncompressed_size = prev_file->uncompressed_size; return; /* ...skip copying file. */ } } diff --git a/tests/backup_test.py b/tests/backup_test.py index 9e911893e..86a2124dc 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3606,3 +3606,36 @@ def test_regress_issue_585(self): output = self.restore_node(backup_dir, 'node', node) self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') + + def test_2_delta_backups(self): + """https://github.com/postgrespro/pg_probackup/issues/596""" + node = self.make_simple_node('node', + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # delta backup mode + delta_backup_id1 = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + + delta_backup_id2 = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + + # postgresql.conf and pg_hba.conf shouldn't be copied + conf_file = os.path.join(backup_dir, 'backups', 'node', delta_backup_id1, 'database', 'postgresql.conf') + self.assertFalse( + os.path.exists(conf_file), + "File should not exist: {0}".format(conf_file)) + conf_file = os.path.join(backup_dir, 'backups', 'node', delta_backup_id2, 'database', 'postgresql.conf') + print(conf_file) + self.assertFalse( + os.path.exists(conf_file), + "File should not exist: {0}".format(conf_file)) From 754f02228e84ac791b66b494e13d523e590490b7 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 24 May 2023 15:13:01 +0300 Subject: [PATCH 2048/2107] PBCKP-624 Remove outdated installation instructions for Standard and Enterprise --- LICENSE | 2 +- README.md | 91 ++++++++++++++----------------------------------------- 2 files changed, 24 insertions(+), 69 deletions(-) diff --git a/LICENSE b/LICENSE index 0ba831507..66476e8a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2020, Postgres Professional +Copyright (c) 2015-2023, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/README.md b/README.md index 7486a6ca6..b804eb1fb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15; +* PostgreSQL 11, 12, 13, 14, 15; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -74,113 +74,68 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10} -sudo apt-get install pg-probackup-{15,14,13,12,11,10}-dbg +sudo apt-get install pg-probackup-{15,14,13,12,11} +sudo apt-get install pg-probackup-{15,14,13,12,11}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{15,14,13,12,11,10} +sudo apt-get source pg-probackup-{15,14,13,12,11} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10}{-dbg,} +sudo apt-get install pg-probackup-{15,14,13,12,11}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{15,14,13,12,11,10} +yumdownloader --source pg_probackup-{15,14,13,12,11} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10} -zypper install pg_probackup-{15,14,13,12,11,10}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11} +zypper install pg_probackup-{15,14,13,12,11}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{15,14,13,12,11,10} +zypper si pg_probackup-{15,14,13,12,11} #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo #RPM ALT Linux 10 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo -``` - -#### pg_probackup for PostgresPro Standard and Enterprise -```shell -#DEB Ubuntu|Debian Packages -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg - -#DEB Astra Linix Orel -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} - - -#RPM Centos Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM RHEL Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM Oracle Linux Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 7 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 8 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 9 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). +For users of Postgres Pro products, commercial editions of pg_probackup are available for installation from the corresponding Postgres Pro product repository. + ## Building from source ### Linux From c9fc20b898610207a23f29fc2ec535d49c265e18 Mon Sep 17 00:00:00 2001 From: Yuriy Sokolov Date: Thu, 25 May 2023 18:12:11 +0300 Subject: [PATCH 2049/2107] PBCKP-604: Allow partial incremental restore only with a flag --destroy-all-other-dbs --- src/help.c | 5 ++ src/pg_probackup.c | 3 + src/pg_probackup.h | 8 +- src/restore.c | 25 +++++- tests/expected/option_help.out | 1 + tests/expected/option_help_ru.out | 1 + tests/incr_restore_test.py | 136 +++++++++++++++++++++++++++++- 7 files changed, 170 insertions(+), 9 deletions(-) diff --git a/src/help.c b/src/help.c index 116a0711c..954ba6416 100644 --- a/src/help.c +++ b/src/help.c @@ -175,6 +175,7 @@ help_pg_probackup(void) printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include | --db-exclude]\n")); + printf(_(" [--destroy-all-other-dbs]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -450,6 +451,7 @@ help_restore(void) printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); + printf(_(" [--destroy-all-other-dbs]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); @@ -497,6 +499,9 @@ help_restore(void) printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); printf(_(" --db-exclude dbname do not restore specified databases\n")); + printf(_(" --destroy-all-other-dbs\n")); + printf(_(" allows to do partial restore that is prohibited by default,\n")); + printf(_(" because it might remove all other databases.\n")); printf(_("\n Recovery options:\n")); printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 505dff89b..17beff55a 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -124,6 +124,7 @@ static parray *datname_include_list = NULL; static parray *exclude_absolute_paths_list = NULL; static parray *exclude_relative_paths_list = NULL; static char* gl_waldir_path = NULL; +static bool allow_partial_incremental = false; /* checkdb options */ bool need_amcheck = false; @@ -242,6 +243,7 @@ static ConfigOption cmd_options[] = { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, { 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT }, + { 'b', 242, "destroy-all-other-dbs", &allow_partial_incremental, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, @@ -764,6 +766,7 @@ main(int argc, char *argv[]) restore_params->partial_restore_type = NONE; restore_params->primary_conninfo = primary_conninfo; restore_params->incremental_mode = incremental_mode; + restore_params->allow_partial_incremental = allow_partial_incremental; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bcea92804..5ee612e6f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -179,6 +179,7 @@ typedef enum DestDirIncrCompatibility POSTMASTER_IS_RUNNING, SYSTEM_ID_MISMATCH, BACKUP_LABEL_EXISTS, + PARTIAL_INCREMENTAL_FORBIDDEN, DEST_IS_NOT_OK, DEST_OK } DestDirIncrCompatibility; @@ -585,7 +586,8 @@ typedef struct pgRestoreParams /* options for partial restore */ PartialRestoreType partial_restore_type; parray *partial_db_list; - + bool allow_partial_incremental; + char* waldir; } pgRestoreParams; @@ -903,7 +905,9 @@ extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode); + IncrRestoreMode incremental_mode, + parray *partial_db_list, + bool allow_partial_incremental); /* in remote.c */ extern void check_remote_agent_compatibility(int agent_version, diff --git a/src/restore.c b/src/restore.c index bb38e8d7e..5b1585024 100644 --- a/src/restore.c +++ b/src/restore.c @@ -150,6 +150,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (params->incremental_mode != INCR_NONE) { DestDirIncrCompatibility rc; + const char *message = NULL; bool ok_to_go = true; elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", @@ -157,12 +158,15 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg rc = check_incremental_compatibility(instance_config.pgdata, instance_config.system_identifier, - params->incremental_mode); + params->incremental_mode, + params->partial_db_list, + params->allow_partial_incremental); if (rc == POSTMASTER_IS_RUNNING) { /* Even with force flag it is unwise to run * incremental restore over running instance */ + message = "Postmaster is running."; ok_to_go = false; } else if (rc == SYSTEM_ID_MISMATCH) @@ -174,7 +178,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (params->incremental_mode != INCR_NONE && params->force) cleanup_pgdata = true; else + { + message = "System ID mismatch."; ok_to_go = false; + } } else if (rc == BACKUP_LABEL_EXISTS) { @@ -187,7 +194,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg * to calculate switchpoint. */ if (params->incremental_mode == INCR_LSN) + { + message = "Backup label exists. Cannot use incremental restore in LSN mode."; ok_to_go = false; + } } else if (rc == DEST_IS_NOT_OK) { @@ -196,11 +206,16 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg * so we cannot be sure that postmaster is running or not. * It is better to just error out. */ + message = "We cannot be sure about the database state."; + ok_to_go = false; + } else if (rc == PARTIAL_INCREMENTAL_FORBIDDEN) + { + message = "Partial incremental restore into non-empty PGDATA is forbidden."; ok_to_go = false; } if (!ok_to_go) - elog(ERROR, "Incremental restore is not allowed"); + elog(ERROR, "Incremental restore is not allowed: %s", message); } else elog(ERROR, "Restore destination is not empty: \"%s\"", @@ -2142,7 +2157,9 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, */ DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode) + IncrRestoreMode incremental_mode, + parray *partial_db_list, + bool allow_partial_incremental) { uint64 system_id_pgdata; bool system_id_match = false; @@ -2226,6 +2243,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, if (backup_label_exists) return BACKUP_LABEL_EXISTS; + if (partial_db_list && !allow_partial_incremental) + return PARTIAL_INCREMENTAL_FORBIDDEN; /* some other error condition */ if (!success) return DEST_IS_NOT_OK; diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 5948d0503..49f79607f 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -92,6 +92,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-X WALDIR | --waldir=WALDIR] [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] + [--destroy-all-other-dbs] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 358c49428..976932b9d 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -92,6 +92,7 @@ pg_probackup - утилита для управления резервным к [-X WALDIR | --waldir=WALDIR] [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] + [--destroy-all-other-dbs] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index 613e4dd36..f17ee95d1 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -1962,7 +1962,9 @@ def test_incremental_partial_restore_exclude_checksum(self): node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "checksum"]) + "-I", "checksum", + "--destroy-all-other-dbs", + ]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -2068,7 +2070,9 @@ def test_incremental_partial_restore_exclude_lsn(self): node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "lsn"]) + "-I", "lsn", + "--destroy-all-other-dbs", + ]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -2188,7 +2192,8 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): "--db-exclude=db1", "--db-exclude=db5", "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) + node_tablespace, node2_tablespace), + "--destroy-all-other-dbs"]) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -2209,7 +2214,9 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): "--db-exclude=db1", "--db-exclude=db5", "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) + node_tablespace, node2_tablespace), + "--destroy-all-other-dbs", + ]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -2241,6 +2248,127 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.assertNotIn('PANIC', output) + def test_incremental_partial_restore_deny(self): + """ + Do now allow partial incremental restore into non-empty PGDATA + becase we can't limit WAL replay to a single database. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 3): + node.safe_psql('postgres', f'CREATE database db{i}') + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + try: + self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) + self.fail("incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Postmaster is running.", e.message) + + node.safe_psql('db2', 'create table x (id int)') + node.safe_psql('db2', 'insert into x values (42)') + + node.stop() + + try: + self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) + self.fail("because incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + + node.slow_start() + value = node.execute('db2', 'select * from x')[0][0] + self.assertEqual(42, value) + + def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self): + """ + Do now allow partial incremental restore into non-empty PGDATA + becase we can't limit WAL replay to a single database. + (case of tablespaces) + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # node2 + node2 = self.make_simple_node('node2') + node2.cleanup() + node2_tablespace = self.get_tblspace_path(node2, 'somedata') + + # in node2 restore full backup + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-T", f"{node_tablespace}={node2_tablespace}"]) + + # partial incremental restore into node2 + try: + self.restore_node(backup_dir, 'node', node2, + options=["-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"]) + self.fail("remapped tablespace contain old data") + except ProbackupException as e: + pass + + try: + self.restore_node(backup_dir, 'node', node2, + options=[ + "-I", "checksum", "--force", + "--db-exclude=db1", "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"]) + self.fail("incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + def test_incremental_pg_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 From db12a039f6c794d4d3e09ee65e8cb8a5e346658a Mon Sep 17 00:00:00 2001 From: Viktoria Shepard Date: Thu, 25 May 2023 18:12:41 +0300 Subject: [PATCH 2050/2107] PBCKP-604 doc add option --destroy-all-other-dbs --- doc/pgprobackup.xml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2cb10e379..297b124f0 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -5850,6 +5850,34 @@ pg_probackup catchup -b catchup_mode + + + Testing and Debugging Options + + This section describes options useful only in a test or development environment. + + + + + + + + By default, pg_probackup exits with an + error if an attempt is made to perform a partial incremental restore + since this destroys databases not included in the restore set. This + flag allows you to suppress the error and proceed with the partial + incremental restore (e.g., to keep a development database snapshot + up-to-date with a production one). This option can be used with the + command. + + + Never use this flag in a production cluster. + + + + + + From a29e378f34d12495f1b64da08f76fb7ad17aab89 Mon Sep 17 00:00:00 2001 From: Sergey Fukanchik Date: Mon, 18 Sep 2023 19:48:22 +0300 Subject: [PATCH 2051/2107] PBCKP-698 allow relative paths in dir_create_dir --- src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dir.c b/src/dir.c index a16e0f396..7ba831c3b 100644 --- a/src/dir.c +++ b/src/dir.c @@ -155,7 +155,7 @@ dir_create_dir(const char *dir, mode_t mode, bool strict) get_parent_directory(parent); /* Create parent first */ - if (access(parent, F_OK) == -1) + if (strlen(parent) > 0 && access(parent, F_OK) == -1) dir_create_dir(parent, mode, false); /* Create directory */ From 9762426ce90c33ffb9fa3896228fe4e58f878aa9 Mon Sep 17 00:00:00 2001 From: Sergey Fukanchik Date: Mon, 18 Sep 2023 18:12:06 +0300 Subject: [PATCH 2052/2107] PBCKP-732 ignore PGDATA setting in catchup mode as we use --source-pgdata instead --- src/pg_probackup.c | 2 +- tests/catchup_test.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 17beff55a..30b4212b4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -682,7 +682,7 @@ main(int argc, char *argv[]) if (instance_config.pgdata != NULL) canonicalize_path(instance_config.pgdata); if (instance_config.pgdata != NULL && - backup_subcmd != ARCHIVE_GET_CMD && + (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) && !is_absolute_path(instance_config.pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); diff --git a/tests/catchup_test.py b/tests/catchup_test.py index 21bcd7973..cf8388dd2 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1585,3 +1585,42 @@ def test_dry_run_catchup_delta(self): # Cleanup src_pg.stop() + + def test_pgdata_is_ignored(self): + """ In catchup we still allow PGDATA to be set either from command line + or from the env var. This test that PGDATA is actually ignored and + --source-pgadta is used instead + """ + node = self.make_simple_node('node', + set_replication = True + ) + node.slow_start() + + # do full catchup + dest = self.make_empty_node('dst') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = node.data_dir, + destination_node = dest, + options = ['-d', 'postgres', '-p', str(node.port), '--stream', '--pgdata=xxx'] + ) + + self.compare_pgdata( + self.pgdata_content(node.data_dir), + self.pgdata_content(dest.data_dir) + ) + + os.environ['PGDATA']='xxx' + + dest2 = self.make_empty_node('dst') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = node.data_dir, + destination_node = dest2, + options = ['-d', 'postgres', '-p', str(node.port), '--stream'] + ) + + self.compare_pgdata( + self.pgdata_content(node.data_dir), + self.pgdata_content(dest2.data_dir) + ) From eb5ccf91b88a92f86631a11a916bd425da86c717 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Thu, 21 Sep 2023 17:28:47 +0300 Subject: [PATCH 2053/2107] PBCKP-751: Fixed for PG 16 build and removed some compilation warnings. --- po/LINGUAS | 1 + src/data.c | 8 ++++---- src/dir.c | 1 - src/show.c | 10 +++++----- src/stream.c | 8 ++++++-- src/validate.c | 4 +--- 6 files changed, 17 insertions(+), 15 deletions(-) create mode 100644 po/LINGUAS diff --git a/po/LINGUAS b/po/LINGUAS new file mode 100644 index 000000000..562ba4cf0 --- /dev/null +++ b/po/LINGUAS @@ -0,0 +1 @@ +ru diff --git a/src/data.c b/src/data.c index 21c41e0b6..a287218ea 100644 --- a/src/data.c +++ b/src/data.c @@ -142,7 +142,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) phdr = (PageHeader) page; /* First check if page header is valid (it seems to be fast enough check) */ - if (!(PageGetPageSize(phdr) == BLCKSZ && + if (!(PageGetPageSize(page) == BLCKSZ && // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && @@ -181,7 +181,7 @@ parse_page(Page page, XLogRecPtr *lsn) /* Get lsn from page header */ *lsn = PageXLogRecPtrGet(phdr->pd_lsn); - if (PageGetPageSize(phdr) == BLCKSZ && + if (PageGetPageSize(page) == BLCKSZ && // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && @@ -203,10 +203,10 @@ get_header_errormsg(Page page, char **errormsg) PageHeader phdr = (PageHeader) page; *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - if (PageGetPageSize(phdr) != BLCKSZ) + if (PageGetPageSize(page) != BLCKSZ) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "page size %lu is not equal to block size %u", - PageGetPageSize(phdr), BLCKSZ); + PageGetPageSize(page), BLCKSZ); else if (phdr->pd_lower < SizeOfPageHeaderData) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " diff --git a/src/dir.c b/src/dir.c index 7ba831c3b..353ed2d43 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1175,7 +1175,6 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pg { pgFile *link = (pgFile *) parray_get(links, i); const char *linked_path = link->linked; - TablespaceListCell *cell; bool remapped = false; for (cell = tablespace_dirs.head; cell; cell = cell->next) diff --git a/src/show.c b/src/show.c index cc22a2acb..86a122698 100644 --- a/src/show.c +++ b/src/show.c @@ -137,7 +137,7 @@ do_show(CatalogState *catalogState, InstanceState *instanceState, show_instance_start(); for (i = 0; i < parray_num(instances); i++) { - InstanceState *instanceState = parray_get(instances, i); + instanceState = parray_get(instances, i); if (interrupted) elog(ERROR, "Interrupted during show"); @@ -202,22 +202,22 @@ pretty_size(int64 size, char *buf, size_t len) return; } - if (Abs(size) < limit) + if (size < limit) snprintf(buf, len, "%dB", (int) size); else { size >>= 9; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dkB", (int) half_rounded(size)); else { size >>= 10; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dMB", (int) half_rounded(size)); else { size >>= 10; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dGB", (int) half_rounded(size)); else { diff --git a/src/stream.c b/src/stream.c index 73bea6780..77453e997 100644 --- a/src/stream.c +++ b/src/stream.c @@ -307,7 +307,11 @@ StreamLog(void *arg) } #if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 160000 + if (!ctl.walmethod->ops->finish(ctl.walmethod)) +#else if (!ctl.walmethod->finish()) +#endif { interrupted = true; elog(ERROR, "Could not finish writing WAL files: %s", @@ -529,7 +533,7 @@ get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backu /* link parent to child */ for (i = 0; i < parray_num(tli_list); i++) { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); + tlinfo = (timelineInfo *) parray_get(tli_list, i); for (j = 0; j < parray_num(tli_list); j++) { @@ -546,7 +550,7 @@ get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backu /* add backups to each timeline info */ for (i = 0; i < parray_num(tli_list); i++) { - timelineInfo *tlinfo = parray_get(tli_list, i); + tlinfo = parray_get(tli_list, i); for (j = 0; j < parray_num(backup_list); j++) { pgBackup *backup = parray_get(backup_list, j); diff --git a/src/validate.c b/src/validate.c index 471351678..0887b2e7a 100644 --- a/src/validate.c +++ b/src/validate.c @@ -401,8 +401,6 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) { char child[MAXPGPATH]; struct stat st; - InstanceState *instanceState; - /* skip entries point current dir or parent dir */ if (strcmp(dent->d_name, ".") == 0 || @@ -420,7 +418,7 @@ do_validate_all(CatalogState *catalogState, InstanceState *instanceState) /* * Initialize instance configuration. */ - instanceState = pgut_new(InstanceState); + instanceState = pgut_new(InstanceState); /* memory leak */ strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); join_path_components(instanceState->instance_backup_subdir_path, From 4868daae347efaf3bbd46ee31f974a023015f41a Mon Sep 17 00:00:00 2001 From: "z.kasymalieva" Date: Wed, 18 Oct 2023 15:50:14 +0300 Subject: [PATCH 2054/2107] [PBCKP-770] The line informing that the remote mode parameters are added to PG_PROBACKUP.CONF has been removed; the set_config command must be used instead. --- doc/pgprobackup.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 297b124f0..22ed9dfd3 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -569,10 +569,9 @@ pg_probackup add-instance -B backup_dir -D backups/instance_name directory contains the pg_probackup.conf configuration file that controls - pg_probackup settings for this backup instance. If you run this - command with the - remote_options, the specified - parameters will be added to pg_probackup.conf. + pg_probackup settings for this backup instance. To add + remote_options to the configuration file, use the + command. For details on how to fine-tune pg_probackup configuration, see From 915d06655445c47e8a6764ed5e32e5e69fe87534 Mon Sep 17 00:00:00 2001 From: Viktoriia Shepard Date: Fri, 27 Oct 2023 19:12:16 +0200 Subject: [PATCH 2055/2107] fix tests for Pg16 and EE16 --- tests/backup_test.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/backup_test.py b/tests/backup_test.py index 86a2124dc..dc60228b5 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3075,11 +3075,20 @@ def test_missing_replication_permission(self): except ProbackupException as e: # 9.5: ERROR: must be superuser or replication role to run a backup # >=9.6: FATAL: must be superuser or replication role to start walsender - self.assertRegex( - e.message, - "ERROR: must be superuser or replication role to run a backup|FATAL: must be superuser or replication role to start walsender", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + if self.pg_config_version < 160000: + self.assertRegex( + e.message, + "ERROR: must be superuser or replication role to run a backup|" + "FATAL: must be superuser or replication role to start walsender", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertRegex( + e.message, + "FATAL: permission denied to start WAL sender\n" + "DETAIL: Only roles with the REPLICATION", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) # @unittest.skip("skip") def test_missing_replication_permission_1(self): @@ -3228,9 +3237,17 @@ def test_missing_replication_permission_1(self): # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' # OS-dependant messages: # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (::1), port 12101 failed: Connection refused\n\tIs the server running on that host and accepting TCP/IP connections?\nconnection to server at "localhost" (127.0.0.1), port 12101 failed: FATAL: must be superuser or replication role to start walsender' - self.assertRegex( - output, - r'WARNING: could not connect to database backupdb:[\s\S]*?FATAL: must be superuser or replication role to start walsender') + + if self.pg_config_version < 160000: + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb:[\s\S]*?' + r'FATAL: must be superuser or replication role to start walsender') + else: + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb:[\s\S]*?' + r'FATAL: permission denied to start WAL sender') # @unittest.skip("skip") def test_basic_backup_default_transaction_read_only(self): From c7ca6cb9c7a859c199c55c57963c5abd5357b52f Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Sat, 28 Oct 2023 09:05:31 +0700 Subject: [PATCH 2056/2107] Up version --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 5ee612e6f..7bbee7cd2 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -353,7 +353,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.12" +#define PROGRAM_VERSION "2.5.13" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20509 From 48efe9086f9a9171b1e51fdf9105edcd667332a4 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Sat, 28 Oct 2023 19:28:33 +0700 Subject: [PATCH 2057/2107] Add the 16th version in Readme --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7486a6ca6..396699805 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15; +* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15, 16 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -41,9 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL 11, 12, 13, 14, 15 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 11, 12, 13, 14 -* Postgres Pro Enterprise 11, 12, 13, 14 +* vanilla PostgreSQL 11, 12, 13, 14, 15, 16 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 11, 12, 13, 14, 15, 16 +* Postgres Pro Enterprise 11, 12, 13, 14, 15, 16 ## Limitations @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10} -sudo apt-get install pg-probackup-{15,14,13,12,11,10}-dbg +sudo apt-get install pg-probackup-{16,15,14,13,12,11,10} +sudo apt-get install pg-probackup-{16,15,14,13,12,11,10}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{15,14,13,12,11,10} +sudo apt-get source pg-probackup-{16,15,14,13,12,11,10} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11,10}{-dbg,} +sudo apt-get install pg-probackup-{16,15,14,13,12,11,10}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11,10} +yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11,10} +yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{15,14,13,12,11,10} -yum install pg_probackup-{15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11,10} +yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{15,14,13,12,11,10} +yumdownloader --source pg_probackup-{16,15,14,13,12,11,10} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11,10} -zypper install pg_probackup-{15,14,13,12,11,10}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{16,15,14,13,12,11,10} +zypper install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{15,14,13,12,11,10} +zypper si pg_probackup-{16,15,14,13,12,11,10} #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo #RPM ALT Linux 10 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11,10} -sudo apt-get install pg_probackup-{15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} +sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo ``` #### pg_probackup for PostgresPro Standard and Enterprise From eb160266253f6fa059f59e52a19d9e215018c431 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 1 Nov 2023 23:22:51 +0700 Subject: [PATCH 2058/2107] Add psycopg2 to the testing environment --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f99d0f27..c3ad89568 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,8 +72,7 @@ jobs: - name: Install Testgres run: | git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git - cd testgres - python setup.py install + pip3 install psycopg2 ./testgres # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder - name: Test Probackup From 3207d6c636a04b5849f2ab0892c90237562d0564 Mon Sep 17 00:00:00 2001 From: oleg gurev Date: Wed, 8 Nov 2023 13:44:51 +0300 Subject: [PATCH 2059/2107] PBCKP-782 doc added note in Remote Mode section --- doc/Readme.md | 3 +++ doc/pgprobackup.xml | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/doc/Readme.md b/doc/Readme.md index 756c6aaa0..0e1d64590 100644 --- a/doc/Readme.md +++ b/doc/Readme.md @@ -3,3 +3,6 @@ xmllint --noout --valid probackup.xml xsltproc stylesheet.xsl probackup.xml >pg-probackup.html ``` +> [!NOTE] +>Install ```docbook-xsl``` if you got +>``` "xsl:import : unable to load http://docbook.sourceforge.net/release/xsl/current/xhtml/docbook.xsl"``` \ No newline at end of file diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 22ed9dfd3..70586c478 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1970,6 +1970,14 @@ pg_probackup restore -B backup_dir --instance + + + In addition to SSH connection, pg_probackup uses + a regular connection to the database to manage the remote operation. + See for details of how to set up + a database connection. + + The typical workflow is as follows: From 70b97d851635ad3fb2fea3d025880c6319276a1c Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 13 Nov 2023 15:23:07 +0300 Subject: [PATCH 2060/2107] declare XID_FMT for PGPRO_EE if not defined --- src/pg_probackup.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7bbee7cd2..48b9bf884 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -110,6 +110,8 @@ extern const char *PROGRAM_EMAIL; /* 64-bit xid support for PGPRO_EE */ #ifndef PGPRO_EE #define XID_FMT "%u" +#elif !defined(XID_FMT) +#define XID_FMT UINT64_FORMAT #endif #ifndef STDIN_FILENO From de531e62e4e25526a21b9d57652518a034d815fe Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 13 Nov 2023 15:26:23 +0300 Subject: [PATCH 2061/2107] PBCKP-797: fix race condition by waiting and filling same amount of rows. --- tests/page_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/page_test.py b/tests/page_test.py index 786374bdb..99f3ce992 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -6,6 +6,7 @@ import subprocess import gzip import shutil +import time class PageTest(ProbackupTest, unittest.TestCase): @@ -893,7 +894,7 @@ def test_page_backup_with_alien_wal_segment(self): "create table t_heap as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i;") + "from generate_series(0,10000) i;") alien_node.safe_psql( "postgres", @@ -905,7 +906,7 @@ def test_page_backup_with_alien_wal_segment(self): "create table t_heap_alien as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100000) i;") + "from generate_series(0,10000) i;") # copy latest wal segment wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') @@ -916,9 +917,9 @@ def test_page_backup_with_alien_wal_segment(self): file = os.path.join(wals_dir, filename) file_destination = os.path.join( os.path.join(backup_dir, 'wal', 'node'), filename) -# file = os.path.join(wals_dir, '000000010000000000000004') - print(file) - print(file_destination) + start = time.time() + while not os.path.exists(file_destination) and time.time() - start < 20: + time.sleep(0.1) os.remove(file_destination) os.rename(file, file_destination) From 3963f39ce670400d2965545b1984ca9bf7cf5c53 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 24 May 2023 15:13:01 +0300 Subject: [PATCH 2062/2107] PBCKP-624 Remove outdated installation instructions for Standard and Enterprise --- LICENSE | 2 +- README.md | 91 ++++++++++++++----------------------------------------- 2 files changed, 24 insertions(+), 69 deletions(-) diff --git a/LICENSE b/LICENSE index 0ba831507..66476e8a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2020, Postgres Professional +Copyright (c) 2015-2023, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/README.md b/README.md index 396699805..973816c26 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.6, 10, 11, 12, 13, 14, 15, 16 +* PostgreSQL 11, 12, 13, 14, 15; As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -74,113 +74,68 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{16,15,14,13,12,11,10} -sudo apt-get install pg-probackup-{16,15,14,13,12,11,10}-dbg +sudo apt-get install pg-probackup-{15,14,13,12,11} +sudo apt-get install pg-probackup-{15,14,13,12,11}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{16,15,14,13,12,11,10} +sudo apt-get source pg-probackup-{15,14,13,12,11} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{16,15,14,13,12,11,10}{-dbg,} +sudo apt-get install pg-probackup-{15,14,13,12,11}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11,10} -yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11,10} -yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11,10} -yum install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +yum install pg_probackup-{15,14,13,12,11} +yum install pg_probackup-{15,14,13,12,11}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{16,15,14,13,12,11,10} +yumdownloader --source pg_probackup-{15,14,13,12,11} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{16,15,14,13,12,11,10} -zypper install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11} +zypper install pg_probackup-{15,14,13,12,11}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{16,15,14,13,12,11,10} +zypper si pg_probackup-{15,14,13,12,11} #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo #RPM ALT Linux 10 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10} -sudo apt-get install pg_probackup-{16,15,14,13,12,11,10}-debuginfo -``` - -#### pg_probackup for PostgresPro Standard and Enterprise -```shell -#DEB Ubuntu|Debian Packages -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{14,13,12,11,10,9.6}-dbg - -#DEB Astra Linix Orel -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}{-dbg,} - - -#RPM Centos Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM RHEL Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM Oracle Linux Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -yum install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 7 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 8 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo - -#RPM ALT Linux 9 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list' && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{14,13,12,11,10,9.6}-debuginfo +sudo apt-get install pg_probackup-{15,14,13,12,11} +sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). +For users of Postgres Pro products, commercial editions of pg_probackup are available for installation from the corresponding Postgres Pro product repository. + ## Building from source ### Linux From 8408753f8d176a04ad25f5a0ad1d317b16db9bf6 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Sat, 28 Oct 2023 19:28:33 +0700 Subject: [PATCH 2063/2107] Cherry-pick Add the 16th version in Readme --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 973816c26..c72b65dab 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 11, 12, 13, 14, 15; +* PostgreSQL 11, 12, 13, 14, 15, 16 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -74,62 +74,62 @@ Installers are available in release **assets**. [Latests](https://github.com/pos #DEB Ubuntu|Debian Packages sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11} -sudo apt-get install pg-probackup-{15,14,13,12,11}-dbg +sudo apt-get install pg-probackup-{16,15,14,13,12,11} +sudo apt-get install pg-probackup-{16,15,14,13,12,11}-dbg #DEB-SRC Packages sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{15,14,13,12,11} +sudo apt-get source pg-probackup-{16,15,14,13,12,11,10} #DEB Astra Linix Orel sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{15,14,13,12,11}{-dbg,} +sudo apt-get install pg-probackup-{16,15,14,13,12,11}{-dbg,} #RPM Centos Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{15,14,13,12,11} -yum install pg_probackup-{15,14,13,12,11}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11} +yum install pg_probackup-{16,15,14,13,12,11}-debuginfo #RPM RHEL Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{15,14,13,12,11} -yum install pg_probackup-{15,14,13,12,11}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11} +yum install pg_probackup-{16,15,14,13,12,11}-debuginfo #RPM Oracle Linux Packages rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{15,14,13,12,11} -yum install pg_probackup-{15,14,13,12,11}-debuginfo +yum install pg_probackup-{16,15,14,13,12,11} +yum install pg_probackup-{16,15,14,13,12,11}-debuginfo #SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{15,14,13,12,11} +yumdownloader --source pg_probackup-{16,15,14,13,12,11} #RPM SUSE|SLES Packages zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{15,14,13,12,11} -zypper install pg_probackup-{15,14,13,12,11}-debuginfo +zypper --gpg-auto-import-keys install -y pg_probackup-{16,15,14,13,12,11} +zypper install pg_probackup-{16,15,14,13,12,11}-debuginfo #SRPM SUSE|SLES Packages -zypper si pg_probackup-{15,14,13,12,11} +zypper si pg_probackup-{16,15,14,13,12,11} #RPM ALT Linux 8 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11} -sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11} +sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo #RPM ALT Linux 9 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11} -sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11} +sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo #RPM ALT Linux 10 sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' sudo apt-get update -sudo apt-get install pg_probackup-{15,14,13,12,11} -sudo apt-get install pg_probackup-{15,14,13,12,11}-debuginfo +sudo apt-get install pg_probackup-{16,15,14,13,12,11} +sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo ``` Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). From 623b659fe8071bc79006996d6511d90dcd09f9db Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 22 Nov 2023 11:35:21 +0700 Subject: [PATCH 2064/2107] Add the installation steps on various systems --- README.md | 64 +------------ doc/pgprobackup.xml | 219 +++++++++++++++++++++++++++++++++++++++++++- doc/stylesheet.css | 3 +- 3 files changed, 218 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index c72b65dab..1cd518849 100644 --- a/README.md +++ b/README.md @@ -69,68 +69,8 @@ For detailed release plans check [Milestones](https://github.com/postgrespro/pg_ Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/latest). ### Linux Installation -#### pg_probackup for vanilla PostgreSQL -```shell -#DEB Ubuntu|Debian Packages -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{16,15,14,13,12,11} -sudo apt-get install pg-probackup-{16,15,14,13,12,11}-dbg - -#DEB-SRC Packages -sudo sh -c 'echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ - /etc/apt/sources.list.d/pg_probackup.list' && sudo apt-get update -sudo apt-get source pg-probackup-{16,15,14,13,12,11,10} - -#DEB Astra Linix Orel -sudo sh -c 'echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ stretch main-stretch" > /etc/apt/sources.list.d/pg_probackup.list' -sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{16,15,14,13,12,11}{-dbg,} - -#RPM Centos Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11} -yum install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#RPM RHEL Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11} -yum install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#RPM Oracle Linux Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{16,15,14,13,12,11} -yum install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#SRPM Centos|RHEL|OracleLinux Packages -yumdownloader --source pg_probackup-{16,15,14,13,12,11} - -#RPM SUSE|SLES Packages -zypper install --allow-unsigned-rpm -y https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm -zypper --gpg-auto-import-keys install -y pg_probackup-{16,15,14,13,12,11} -zypper install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#SRPM SUSE|SLES Packages -zypper si pg_probackup-{16,15,14,13,12,11} - -#RPM ALT Linux 8 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' -sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11} -sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#RPM ALT Linux 9 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' -sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11} -sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo - -#RPM ALT Linux 10 -sudo sh -c 'echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p10 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list' -sudo apt-get update -sudo apt-get install pg_probackup-{16,15,14,13,12,11} -sudo apt-get install pg_probackup-{16,15,14,13,12,11}-debuginfo -``` + +See the [Installation](https://postgrespro.github.io/pg_probackup/#pbk-install) section in the documentation. Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 70586c478..2ec4258cb 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -172,7 +172,10 @@ doc/src/sgml/pgprobackup.sgml Overview - Installation and Setup + Installation + + + Setup Command-Line Reference @@ -451,8 +454,215 @@ doc/src/sgml/pgprobackup.sgml - - Installation and Setup + + Installation + + Installation on Debian family systems (Debian, Ubuntu etc.) + + You may need to use apt-get instead of apt on older systems in the commands below. + + + + Add the pg_probackup repository GPG key + +sudo apt install gpg wget +wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | \ +sudo apt-key add - + + + + Setup the binary package repository + +. /etc/os-release +echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + Optionally setup the source package repository for rebuilding the binaries + +echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ +sudo tee -a /etc/apt/sources.list.d/pg_probackup.list + + + + List the available pg_probackup packages + + + Using apt: + +sudo apt update +apt search pg_probackup + + + + Using apt-get: + +sudo apt-get update +apt-cache search pg_probackup + + + + + + Install or upgrade a pg_probackup version of your choice + +sudo apt install pg-probackup-15 + + + + Optionally install the debug package + +sudo apt install pg-probackup-15-dbg + + + + Optionally install the source package (provided you have set up the source package repository as described above) + +sudo apt install dpkg-dev +sudo apt source pg-probackup-15 + + + + + + Installation on Red Hat family systems (CentOS, Oracle Linux etc.) + + You may need to use yum instead of dnf on older systems in the commands below. + + + + Install the pg_probackup repository + +dnf install https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm + + + + List the available pg_probackup packages + +dnf search pg_probackup + + + + Install or upgrade a pg_probackup version of your choice + +dnf install pg_probackup-15 + + + + Optionally install the debug package + +dnf install pg_probackup-15-debuginfo + + + + Optionally install the source package for rebuilding the binaries + + + Using dnf: + +dnf install 'dnf-command(download)' +dnf download --source pg_probackup-15 + + + + Using yum: + +yumdownloader --source pg_probackup-15 + + + + + + + + Installation on ALT Linux + + You may need to use yum instead of dnf on older systems in the commands below. + + + + Setup the repository + + + On ALT Linux 10: + +. /etc/os-release +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p$VERSION_ID x86_64 vanilla" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + On ALT Linux 8 and 9: + +. /etc/os-release +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-$VERSION_ID x86_64 vanilla" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + + + List the available pg_probackup packages + +sudo apt-get update +apt-cache search pg_probackup + + + + Install or upgrade a pg_probackup version of your choice + +sudo apt-get install pg_probackup-15 + + + + Optionally install the debug package + +sudo apt-get install pg_probackup-15-debuginfo + + + + + + Installation on SUSE Linux + + + Add the pg_probackup repository GPG key + +zypper in -y gpg wget +wget -O GPG-KEY-PG_PROBACKUP https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP +rpm --import GPG-KEY-PG_PROBACKUP + + + + Setup the repository + +zypper in https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm + + + + List the available pg_probackup packages + +zypper se pg_probackup + + + + Install or upgrade a pg_probackup version of your choice + +zypper in pg_probackup-15 + + + + Optionally install the source package for rebuilding the binaries + +zypper si pg_probackup-15 + + + + + + + Setup Once you have pg_probackup installed, complete the following setup: @@ -1986,8 +2196,7 @@ pg_probackup restore -B backup_dir --instance On your backup host, configure pg_probackup as explained in the section - Installation and - Setup. For the + Setup. For the and commands, make sure to specify remote diff --git a/doc/stylesheet.css b/doc/stylesheet.css index 4d84058f5..31464154b 100644 --- a/doc/stylesheet.css +++ b/doc/stylesheet.css @@ -119,7 +119,8 @@ body { } .book code, kbd, pre, samp { - font-family: monospace,monospace; + font-family: monospace,monospace; + font-size: 90%; } .book .txtCommentsWrap { From 7c29b63e2f5308fd5d77b45c9d1d28f1b16e286c Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Mon, 28 Nov 2022 15:52:13 +0300 Subject: [PATCH 2065/2107] [DOC] [PGPRO-7104] Remove outdated options and PostgreSQL versions from documentation [skip travis] --- doc/pgprobackup.xml | 172 ++------------------------------------------ 1 file changed, 4 insertions(+), 168 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 2ec4258cb..3dadc0aad 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -164,7 +164,7 @@ doc/src/sgml/pgprobackup.sgml recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. - pg_probackup supports PostgreSQL 9.5 or higher. + pg_probackup supports PostgreSQL 10 or higher. @@ -416,7 +416,7 @@ doc/src/sgml/pgprobackup.sgml - On Unix systems, for PostgreSQL 10 or lower, + On Unix systems, for PostgreSQL 10, a backup can be made only by the same OS user that has started the PostgreSQL server. For example, if PostgreSQL server is started by user postgres, the backup command must also be run @@ -821,49 +821,6 @@ pg_probackup add-instance -B backup_dir -D used for connection to the PostgreSQL server: - - For PostgreSQL 9.5: - - -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -COMMIT; - - - For PostgreSQL 9.6: - - -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; - - - For PostgreSQL versions 10 — 14: - BEGIN; CREATE ROLE backup WITH LOGIN; @@ -1110,7 +1067,7 @@ archive_command = '"install_dir/pg_probackup" archive Setting up Backup from Standby - For PostgreSQL 9.6 or higher, pg_probackup can take backups from + pg_probackup can take backups from a standby server. This requires the following additional setup: @@ -5013,8 +4970,7 @@ pg_probackup catchup -b catchup_mode Specifies the LSN of the write-ahead log location up to which - recovery will proceed. Can be used only when restoring - a database cluster of major version 10 or higher. + recovery will proceed. @@ -5976,124 +5932,6 @@ pg_probackup catchup -b catchup_mode - - Replica Options - - This section describes the options related to taking a backup - from standby. - - - - Starting from pg_probackup 2.0.24, backups can be - taken from standby without connecting to the master server, - so these options are no longer required. In lower versions, - pg_probackup had to connect to the master to determine - recovery time — the earliest moment for which you can - restore a consistent state of the database cluster. - - - - - - - - - Deprecated. Specifies the name of the database on the master - server to connect to. The connection is used only for managing - the backup process, so you can connect to any existing - database. Can be set in the pg_probackup.conf using the - command. - - - Default: postgres, the default PostgreSQL database - - - - - - - - - Deprecated. Specifies the host name of the system on which the - master server is running. - - - - - - - - - Deprecated. Specifies the TCP port or the local Unix domain - socket file extension on which the master server is listening - for connections. - - - Default: 5432, the PostgreSQL default port - - - - - - - - - Deprecated. User name to connect as. - - - Default: postgres, - the PostgreSQL default user name - - - - - - - - - - Deprecated. Wait time for WAL segment streaming via - replication, in seconds. By default, pg_probackup waits 300 - seconds. You can also define this parameter in the - pg_probackup.conf configuration file using the - command. - - - Default: 300 sec - - - - - - - - - Testing and Debugging Options - - This section describes options useful only in a test or development environment. - - - - - - - - By default, pg_probackup exits with an - error if an attempt is made to perform a partial incremental restore - since this destroys databases not included in the restore set. This - flag allows you to suppress the error and proceed with the partial - incremental restore (e.g., to keep a development database snapshot - up-to-date with a production one). This option can be used with the - command. - - - Never use this flag in a production cluster. - - - - - - @@ -6305,8 +6143,6 @@ xlog-seg-size = 16777216 pgdatabase = backupdb pghost = postgres_host pguser = backup -# Replica parameters -replica-timeout = 5min # Archive parameters archive-timeout = 5min # Logging parameters From 363024d5f83b3381d8bb80d1b1bbd5a907d4d091 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 7 Dec 2022 13:35:25 +0300 Subject: [PATCH 2066/2107] Documentation hot fix --- doc/pgprobackup.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 3dadc0aad..590f18d62 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -821,6 +821,9 @@ pg_probackup add-instance -B backup_dir -D used for connection to the PostgreSQL server: + + For PostgreSQL versions 10 — 14: + BEGIN; CREATE ROLE backup WITH LOGIN; From 72a9605217ab1437d3e8ee36b93f54ebd4ec2913 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Thu, 23 Nov 2023 11:08:30 +0700 Subject: [PATCH 2067/2107] Fix a typo --- doc/pgprobackup.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 590f18d62..5610cd150 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -577,9 +577,6 @@ yumdownloader --source pg_probackup-15 Installation on ALT Linux - - You may need to use yum instead of dnf on older systems in the commands below. - Setup the repository From b52b4d9fcbbb5d414d4c2d50771e5abd7e025cbd Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Fri, 24 Nov 2023 13:56:27 +0700 Subject: [PATCH 2068/2107] PBCKP-816: Remove version 10 from pg_probackup docs --- README.md | 2 +- doc/pgprobackup.xml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1cd518849..6f7ba5df9 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ cd && git clone https://github.com/postgrespro/ ### Windows Currently pg_probackup can be build using only MSVC 2013. -Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/10/install-windows-full.html) with MSVC 2013. +Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/current/install-windows-full.html) with MSVC 2013. If zlib support is needed, src/tools/msvc/config.pl must contain path to directory with compiled zlib. [Example](https://gist.githubusercontent.com/gsmol/80989f976ce9584824ae3b1bfb00bd87/raw/240032950d4ac4801a79625dd00c8f5d4ed1180c/gistfile1.txt) ```shell diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 5610cd150..875250566 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -164,7 +164,7 @@ doc/src/sgml/pgprobackup.sgml recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. - pg_probackup supports PostgreSQL 10 or higher. + pg_probackup supports PostgreSQL 11 or higher. @@ -416,7 +416,7 @@ doc/src/sgml/pgprobackup.sgml - On Unix systems, for PostgreSQL 10, + On Unix systems, for PostgreSQL 11, a backup can be made only by the same OS user that has started the PostgreSQL server. For example, if PostgreSQL server is started by user postgres, the backup command must also be run @@ -819,7 +819,7 @@ pg_probackup add-instance -B backup_dir -D to the PostgreSQL server: - For PostgreSQL versions 10 — 14: + For PostgreSQL versions 11 — 14: BEGIN; @@ -1807,7 +1807,7 @@ pg_probackup restore -B backup_dir --instance primary_conninfo parameter; you have to add the password manually or use the --primary-conninfo option, if required. - For PostgreSQL 11 or lower, + For PostgreSQL 11, recovery settings are written into the recovery.conf file. Starting from PostgreSQL 12, pg_probackup writes these settings into From b5fca40c1a8d059304ae0854cc64b69ac6588b08 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 24 Nov 2023 16:21:08 +0700 Subject: [PATCH 2069/2107] PBCKP-816 Store the GPG key in the recommended way --- doc/pgprobackup.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 875250566..31f0c5292 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -466,8 +466,8 @@ doc/src/sgml/pgprobackup.sgml Add the pg_probackup repository GPG key sudo apt install gpg wget -wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | \ -sudo apt-key add - +wget -qO - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | \ +sudo tee /etc/apt/trusted.gpg.d/pg_probackup.asc From f6f5bfa2916d155f459f5f771c9c6051b95b9c68 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 24 Nov 2023 18:09:01 +0700 Subject: [PATCH 2070/2107] PBCKP-816 Update a link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f7ba5df9..2279b97a4 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Installers are available in release **assets**. [Latests](https://github.com/pos See the [Installation](https://postgrespro.github.io/pg_probackup/#pbk-install) section in the documentation. -Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-install-and-setup). +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-setup). For users of Postgres Pro products, commercial editions of pg_probackup are available for installation from the corresponding Postgres Pro product repository. From 41f5baae72e7615a6bad6325476ec6498327cb9e Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Sun, 26 Nov 2023 21:42:06 +0700 Subject: [PATCH 2071/2107] Switch to the new GPG key in the Debian installation --- doc/pgprobackup.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 31f0c5292..1f764a432 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -466,7 +466,7 @@ doc/src/sgml/pgprobackup.sgml Add the pg_probackup repository GPG key sudo apt install gpg wget -wget -qO - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | \ +wget -qO - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG-PROBACKUP | \ sudo tee /etc/apt/trusted.gpg.d/pg_probackup.asc From 1ee26f9912e83353ccf0db0c0055ee3a131b4bde Mon Sep 17 00:00:00 2001 From: Nikolay Zakharov Date: Wed, 13 Dec 2023 15:23:57 +0300 Subject: [PATCH 2072/2107] Fix bug: terminate program if file is not exists (#839) --- src/utils/file.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index e062a2133..d39d3e320 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1159,24 +1159,35 @@ fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location lo bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) { + char *abs_name1 = make_absolute_path(filename1); + char *abs_name2 = make_absolute_path(filename2); + bool result = strcmp(abs_name1, abs_name2) == 0; + #ifndef WIN32 - struct stat stat1, stat2; + if (!result) + { + struct stat stat1, stat2; - if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); + if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) + { + if (errno == ENOENT) + return false; + elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); + } - if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); + if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) + { + if (errno == ENOENT) + return false; + elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); + } - return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; -#else - char *abs_name1 = make_absolute_path(filename1); - char *abs_name2 = make_absolute_path(filename2); - bool result = strcmp(abs_name1, abs_name2) == 0; + result = (stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev); + } +#endif free(abs_name2); free(abs_name1); return result; -#endif } /* From d26df12019d6a00e27645d7eb84be71b2b968138 Mon Sep 17 00:00:00 2001 From: Victor Spirin Date: Wed, 6 Dec 2023 20:32:35 +0300 Subject: [PATCH 2073/2107] PBCKP-819: Fixed the recovery-target-timeline command line parameter. It may contain 'current' or 'latest' keywords. recovery-target can be 'latest' too for compatibility with previous versions --- src/pg_probackup.c | 8 ++-- src/pg_probackup.h | 3 +- src/restore.c | 38 +++++++++++++-- tests/restore_test.py | 109 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 147 insertions(+), 11 deletions(-) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 30b4212b4..6653898e4 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -98,7 +98,7 @@ static char *target_time = NULL; static char *target_xid = NULL; static char *target_lsn = NULL; static char *target_inclusive = NULL; -static TimeLineID target_tli; +static char *target_tli_string; /* timeline number, "current" or "latest"*/ static char *target_stop; static bool target_immediate; static char *target_name = NULL; @@ -227,7 +227,7 @@ static ConfigOption cmd_options[] = { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 144, "recovery-target-lsn", &target_lsn, SOURCE_CMD_STRICT }, { 's', 138, "recovery-target-inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "recovery-target-timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 139, "recovery-target-timeline", &target_tli_string, SOURCE_CMD_STRICT }, { 's', 157, "recovery-target", &target_stop, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, @@ -285,7 +285,7 @@ static ConfigOption cmd_options[] = { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 139, "timeline", &target_tli_string, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, @@ -739,7 +739,7 @@ main(int argc, char *argv[]) */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_lsn, + target_inclusive, target_tli_string, target_lsn, (target_stop != NULL) ? target_stop : (target_immediate) ? "immediate" : NULL, target_name, target_action); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 48b9bf884..8246c517d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -564,6 +564,7 @@ typedef struct pgRecoveryTarget const char *target_stop; const char *target_name; const char *target_action; + const char *target_tli_string; /* timeline number, "current" or "latest" from recovery_target_timeline option*/ } pgRecoveryTarget; /* Options needed for restore and validate commands */ @@ -893,7 +894,7 @@ extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, - const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + const char *target_inclusive, const char *target_tli_string, const char* target_lsn, const char *target_stop, const char *target_name, const char *target_action); diff --git a/src/restore.c b/src/restore.c index 5b1585024..707dca0c8 100644 --- a/src/restore.c +++ b/src/restore.c @@ -1332,8 +1332,10 @@ create_recovery_conf(InstanceState *instanceState, time_t backup_id, } /* restore-target='latest' support */ - target_latest = rt->target_stop != NULL && - strcmp(rt->target_stop, "latest") == 0; + target_latest = (rt->target_tli_string != NULL && + strcmp(rt->target_tli_string, "latest") == 0) || + (rt->target_stop != NULL && + strcmp(rt->target_stop, "latest") == 0); target_immediate = rt->target_stop != NULL && strcmp(rt->target_stop, "immediate") == 0; @@ -1359,6 +1361,13 @@ create_recovery_conf(InstanceState *instanceState, time_t backup_id, rt->xid_string || rt->lsn_string || rt->target_name || target_immediate || target_latest || restore_command_provided) params->recovery_settings_mode = PITR_REQUESTED; + /* + * The recovery-target-timeline option can be 'latest' for streaming backups. + * This operation requires a WAL archive for PITR. + */ + if (rt->target_tli && backup->stream && params->recovery_settings_mode != PITR_REQUESTED) + elog(WARNING, "The '--recovery-target-timeline' option applied for STREAM backup. " + "The timeline number will be ignored."); elog(LOG, "----------------------------------------"); @@ -1438,14 +1447,20 @@ print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); else { + if (rt->target_tli_string) + fio_fprintf(fp, "recovery_target_timeline = '%s'\n", rt->target_tli_string); + else if (rt->target_stop && (strcmp(rt->target_stop, "latest") == 0)) + fio_fprintf(fp, "recovery_target_timeline = 'latest'\n"); #if PG_VERSION_NUM >= 120000 - + else + { /* * In PG12 default recovery target timeline was changed to 'latest', which * is extremely risky. Explicitly preserve old behavior of recovering to current * timneline for PG12. */ fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); + } #endif } @@ -1877,7 +1892,7 @@ pgRecoveryTarget * parseRecoveryTargetOptions(const char *target_time, const char *target_xid, const char *target_inclusive, - TimeLineID target_tli, + const char *target_tli_string, const char *target_lsn, const char *target_stop, const char *target_name, @@ -1950,7 +1965,20 @@ parseRecoveryTargetOptions(const char *target_time, target_inclusive); } - rt->target_tli = target_tli; + rt->target_tli_string = target_tli_string; + rt->target_tli = 0; + /* target_tli can contains timeline number, "current" or "latest" */ + if(target_tli_string && strcmp(target_tli_string, "current") != 0 && strcmp(target_tli_string, "latest") != 0) + { + errno = 0; + rt->target_tli = strtoul(target_tli_string, NULL, 10); + if (errno == EINVAL || errno == ERANGE || !rt->target_tli) + { + elog(ERROR, "Invalid value for '--recovery-target-timeline' option '%s'", + target_tli_string); + } + } + if (target_stop) { if ((strcmp(target_stop, "immediate") != 0) diff --git a/tests/restore_test.py b/tests/restore_test.py index 67e99515c..df836aada 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1916,7 +1916,9 @@ def test_restore_target_immediate_archive(self): with open(recovery_conf, 'r') as f: self.assertIn("recovery_target = 'immediate'", f.read()) - # @unittest.skip("skip") + # Skipped, because default recovery_target_timeline is 'current' + # Before PBCKP-598 the --recovery-target=latest' option did not work and this test allways passed + @unittest.skip("skip") def test_restore_target_latest_archive(self): """ make sure that recovery_target 'latest' @@ -3818,3 +3820,108 @@ def test_restore_with_waldir(self): wal_path=os.path.join(node.data_dir, "pg_xlog") self.assertEqual(os.path.islink(wal_path), True) + + # @unittest.skip("skip") + def test_restore_to_latest_timeline(self): + """recovery to latest timeline""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + node.pgbench_init(scale=2) + + before1 = node.table_checksum("pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + before2 = node.table_checksum("pgbench_branches") + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + # restore from first backup + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-timeline=latest", "-i", backup_id] + ) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + # check recovery-target=latest option for compatibility with previous versions + node.cleanup() + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target=latest", "-i", backup_id] + ) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + + # start postgres and promote wal files to latest timeline + node.slow_start() + + # check for the latest updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before2, after) + + # checking recovery_target_timeline=current is the default option + if self.pg_config_version >= self.version_to_num('12.0'): + node.stop() + node.cleanup() + + # restore from first backup + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "-i", backup_id] + ) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "current") + + # start postgres with current timeline + node.slow_start() + + # check for the current updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before1, after) From 52e47fe19659f6f5be59fc36acac53aba0e4033d Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Sun, 3 Dec 2023 23:59:35 +0300 Subject: [PATCH 2074/2107] [PBCKP-218] Incremental restore and missing pg_control (issue #304) - pg_control file backup after all other files in backup - pg_control file restore last in full restore - rename pg_control to pg_control.pbk.bak at start of non-full restore - remove pg_control.pbk.bak in the end of successfull non-full restore - use pg_control.pbk.bak after failed non-full restore - added tests for full and incremental restore Tags: backup, catchup, restore --- src/backup.c | 58 +++++++++++--- src/catchup.c | 46 ++++++++++- src/dir.c | 2 +- src/pg_probackup.h | 3 + src/restore.c | 83 +++++++++++++++++++- src/util.c | 20 +++++ tests/helpers/ptrack_helpers.py | 2 +- tests/incr_restore_test.py | 88 ++++++++++++++++++++- tests/restore_test.py | 135 +++++++++++++++++--------------- 9 files changed, 354 insertions(+), 83 deletions(-) diff --git a/src/backup.c b/src/backup.c index 41f035a86..78c3512e9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -122,6 +122,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, char pretty_time[20]; char pretty_bytes[20]; + pgFile *src_pg_control_file = NULL; + elog(INFO, "Database backup start"); if(current.external_dir_str) { @@ -424,6 +426,24 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, } + /* + * find pg_control file + * We'll copy it last + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(backup_files_list, &search_key, pgFileCompareRelPathWithExternal); + + if (control_file_elem_index < 0) + elog(ERROR, "File \"%s\" not found in PGDATA %s", XLOG_CONTROL_FILE, current.database_dir); + src_pg_control_file = (pgFile *)parray_get(backup_files_list, control_file_elem_index); + } + /* setup thread locks */ pfilearray_clear_locks(backup_files_list); @@ -483,6 +503,26 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, backup_isok = false; } + /* copy pg_control at very end */ + if (backup_isok) + { + + elog(progress ? INFO : LOG, "Progress: Backup file \"%s\"", + src_pg_control_file->rel_path); + + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + join_path_components(from_fullpath, instance_config.pgdata, src_pg_control_file->rel_path); + join_path_components(to_fullpath, current.database_dir, src_pg_control_file->rel_path); + + backup_non_data_file(src_pg_control_file, NULL, + from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, + true); + } + + + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -510,17 +550,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { pgFile *pg_control = NULL; - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); + pg_control = src_pg_control_file; - if (tmp_file->external_dir_num == 0 && - (strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0)) - { - pg_control = tmp_file; - break; - } - } if (!pg_control) elog(ERROR, "Failed to find file \"%s\" in backup filelist.", @@ -2076,6 +2107,13 @@ backup_files(void *arg) /* We have already copied all directories */ if (S_ISDIR(file->mode)) continue; + /* + * Don't copy the pg_control file now, we'll copy it last + */ + if(file->external_dir_num == 0 && pg_strcasecmp(file->rel_path, XLOG_CONTROL_FILE) == 0) + { + continue; + } if (arguments->thread_num == 1) { diff --git a/src/catchup.c b/src/catchup.c index 427542dda..00752b194 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -171,10 +171,13 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { - dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false); + ControlFileData dst_control; + get_control_file_or_back_file(dest_pgdata, FIO_LOCAL_HOST, &dst_control); + dest_id = dst_control.system_identifier; + if (source_conn_id != dest_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, dest_pgdata, dest_id); + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, dest_pgdata, (long long)dest_id); } } @@ -640,6 +643,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, ssize_t transfered_walfiles_bytes = 0; char pretty_source_bytes[20]; + char dest_pg_control_fullpath[MAXPGPATH]; + char dest_pg_control_bak_fullpath[MAXPGPATH]; + source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata); catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); @@ -935,6 +941,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, Assert(file->external_dir_num == 0); if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) redundant = true; + /* global/pg_control.pbk.bak is always keeped, because it's needed for restart failed incremental restore */ + if (pg_strcasecmp(file->rel_path, XLOG_CONTROL_BAK_FILE) == 0) + redundant = false; /* if file does not exists in destination list, then we can safely unlink it */ if (redundant) @@ -966,6 +975,28 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, if (dest_filelist) parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal); + join_path_components(dest_pg_control_fullpath, dest_pgdata, XLOG_CONTROL_FILE); + join_path_components(dest_pg_control_bak_fullpath, dest_pgdata, XLOG_CONTROL_BAK_FILE); + /* + * rename (if it exist) dest control file before restoring + * if it doesn't exist, that mean, that we already restoring in a previously failed + * pgdata, where XLOG_CONTROL_BAK_FILE exist + */ + if (current.backup_mode != BACKUP_MODE_FULL && !dry_run) + { + if (!fio_access(dest_pg_control_fullpath, F_OK, FIO_LOCAL_HOST)) + { + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + + if(!fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST)) + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + fio_rename(dest_pg_control_fullpath, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } + } + /* run copy threads */ elog(INFO, "Start transferring data files"); time(&start_time); @@ -985,6 +1016,15 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); transfered_datafiles_bytes += source_pg_control_file->size; + + /* Now backup control file can be deled */ + if (current.backup_mode != BACKUP_MODE_FULL && !fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST)){ + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } } if (!catchup_isok && !dry_run) diff --git a/src/dir.c b/src/dir.c index 353ed2d43..4b1bc2816 100644 --- a/src/dir.c +++ b/src/dir.c @@ -1867,4 +1867,4 @@ set_forkname(pgFile *file) file->segno = segno; file->is_datafile = file->forkName == none; return true; -} +} \ No newline at end of file diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8246c517d..61dd2ce0e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -91,6 +91,7 @@ extern const char *PROGRAM_EMAIL; #define DATABASE_MAP "database_map" #define HEADER_MAP "page_header_map" #define HEADER_MAP_TMP "page_header_map_tmp" +#define XLOG_CONTROL_BAK_FILE XLOG_CONTROL_FILE".pbk.bak" /* default replication slot names */ #define DEFAULT_TEMP_SLOT_NAME "pg_probackup_slot"; @@ -1209,6 +1210,8 @@ extern uint32 get_xlog_seg_size(const char *pgdata_path); extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); +extern void get_control_file_or_back_file(const char *pgdata_path, fio_location location, + ControlFileData *control); extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, const char *to_fullpath, fio_location to_location, pgFile *file); diff --git a/src/restore.c b/src/restore.c index 707dca0c8..535faebfb 100644 --- a/src/restore.c +++ b/src/restore.c @@ -39,6 +39,8 @@ typedef struct int ret; } restore_files_arg; +static bool control_downloaded = false; +static ControlFileData instance_control; static void print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, @@ -501,6 +503,9 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg if (redo.checksum_version == 0) elog(ERROR, "Incremental restore in 'lsn' mode require " "data_checksums to be enabled in destination data directory"); + if (!control_downloaded) + get_control_file_or_back_file(instance_config.pgdata, FIO_DB_HOST, + &instance_control); timelines = read_timeline_history(instanceState->instance_wal_subdir_path, redo.tli, false); @@ -719,6 +724,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *pgdata_files = NULL; parray *dest_files = NULL; parray *external_dirs = NULL; + pgFile *dest_pg_control_file = NULL; + char dest_pg_control_fullpath[MAXPGPATH]; + char dest_pg_control_bak_fullpath[MAXPGPATH]; + /* arrays with meta info for multi threaded backup */ pthread_t *threads; restore_files_arg *threads_args; @@ -922,6 +931,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) redundant = true; + /* global/pg_control.pbk.bak are always keeped, because it's needed for restart failed incremental restore */ + if (file->external_dir_num == 0 && + pg_strcasecmp(file->rel_path, XLOG_CONTROL_BAK_FILE) == 0) + redundant = false; + /* do not delete the useful internal directories */ if (S_ISDIR(file->mode) && !redundant) continue; @@ -974,6 +988,42 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, dest_bytes = dest_backup->pgdata_bytes; pretty_size(dest_bytes, pretty_dest_bytes, lengthof(pretty_dest_bytes)); + /* + * [Issue #313] + * find pg_control file (in already sorted earlier dest_files, see parray_qsort(backup->files...)) + * and exclude it from list for future special processing + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(dest_files, &search_key, pgFileCompareRelPathWithExternal); + + if (control_file_elem_index < 0) + elog(ERROR, "File \"%s\" not found in backup %s", XLOG_CONTROL_FILE, base36enc(dest_backup->start_time)); + dest_pg_control_file = (pgFile *) parray_get(dest_files, control_file_elem_index); + parray_remove(dest_files, control_file_elem_index); + + join_path_components(dest_pg_control_fullpath, pgdata_path, XLOG_CONTROL_FILE); + join_path_components(dest_pg_control_bak_fullpath, pgdata_path, XLOG_CONTROL_BAK_FILE); + /* + * rename (if it exist) dest control file before restoring + * if it doesn't exist, that mean, that we already restoring in a previously failed + * pgdata, where XLOG_CONTROL_BAK_FILE exist + */ + if (params->incremental_mode != INCR_NONE) + { + if (fio_access(dest_pg_control_fullpath,F_OK,FIO_DB_HOST) == 0){ + if (fio_rename(dest_pg_control_fullpath, dest_pg_control_bak_fullpath, FIO_DB_HOST) < 0) + elog(WARNING, "Cannot rename file \"%s\" to \"%s\": %s", + dest_pg_control_fullpath, dest_pg_control_bak_fullpath, strerror(errno)); + } + } + } + elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); time(&start_time); thread_interrupted = false; @@ -1014,6 +1064,32 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, total_bytes += threads_args[i].restored_bytes; } + /* [Issue #313] copy pg_control at very end */ + if (restore_isok) + { + FILE *out = NULL; + elog(progress ? INFO : LOG, "Progress: Restore file \"%s\"", + dest_pg_control_file->rel_path); + + out = fio_fopen(dest_pg_control_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + + total_bytes += restore_non_data_file(parent_chain, + dest_backup, + dest_pg_control_file, + out, + dest_pg_control_fullpath, false); + fio_fclose(out); + /* Now backup control file can be deleted */ + if (params->incremental_mode != INCR_NONE) + { + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } + } + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -1098,6 +1174,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_free(pgdata_files); } + if(dest_pg_control_file) pgFileFree(dest_pg_control_file); + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -2230,7 +2308,10 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ elog(LOG, "Trying to read pg_control file in destination directory"); - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); + get_control_file_or_back_file(pgdata, FIO_DB_HOST, &instance_control); + control_downloaded = true; + + system_id_pgdata = instance_control.system_identifier; if (system_id_pgdata == instance_config.system_identifier) system_id_match = true; diff --git a/src/util.c b/src/util.c index 1407f03cc..3c0a33453 100644 --- a/src/util.c +++ b/src/util.c @@ -190,6 +190,26 @@ get_current_timeline_from_control(const char *pgdata_path, fio_location location return ControlFile.checkPointCopy.ThisTimeLineID; } +void +get_control_file_or_back_file(const char *pgdata_path, fio_location location, ControlFileData *control) +{ + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, true, location); + + if (!buffer || size == 0){ + /* Error read XLOG_CONTROL_FILE or file is truncated, trying read backup */ + buffer = slurpFile(pgdata_path, XLOG_CONTROL_BAK_FILE, &size, true, location); + if (!buffer) + elog(ERROR, "Could not read %s and %s files\n", XLOG_CONTROL_FILE, XLOG_CONTROL_BAK_FILE); /* Maybe it should be PANIC? */ + } + digestControlFile(control, buffer, size); + pg_free(buffer); +} + + /* * Get last check point record ptr from pg_tonrol. */ diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6b665097c..da8ece15e 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1783,7 +1783,7 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): 'ptrack_control', 'ptrack_init', 'pg_control', 'probackup_recovery.conf', 'recovery.signal', 'standby.signal', 'ptrack.map', 'ptrack.map.mmap', - 'ptrack.map.tmp' + 'ptrack.map.tmp', 'recovery.done','backup_label.old' ] if exclude_dirs: diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index f17ee95d1..eea0e313b 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -9,8 +9,9 @@ import hashlib import shutil import json -from testgres import QueryException - +from testgres import QueryException, StartNodeException +import stat +from stat import S_ISDIR class IncrRestoreTest(ProbackupTest, unittest.TestCase): @@ -2426,3 +2427,86 @@ def test_incremental_pg_filenode_map(self): 'select 1') # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn + + # @unittest.skip("skip") + def test_incr_restore_issue_313(self): + """ + Check that failed incremental restore can be restarted + """ + self._check_gdb_flag_or_skip_test + node = self.make_simple_node('node', + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale = 50) + + full_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + last_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=full_backup_id) + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', last_backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and \ + not stat.S_ISDIR(int(filelist[file]['mode'])) and \ + not filelist[file]['size'] == '0' and \ + file != 'database_map': + count += 1 + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 1) + gdb.quit() + + bak_file = os.path.join(node.data_dir, 'global', 'pg_control.pbk.bak') + self.assertTrue( + os.path.exists(bak_file), + "pg_control bak File should not exist: {0}".format(bak_file)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + if self.pg_config_version >= 120000: + self.assertIn( + "PANIC: could not read file \"global/pg_control\"", + f.read()) + else: + self.assertIn( + "PANIC: could not read from control file", + f.read()) + self.restore_node(backup_dir, 'node', node, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + node.slow_start() + self.compare_pgdata(pgdata, self.pgdata_content(node.data_dir)) diff --git a/tests/restore_test.py b/tests/restore_test.py index df836aada..b6664252e 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -3,11 +3,11 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess import sys -from time import sleep from datetime import datetime, timedelta, timezone import hashlib import shutil import json +import stat from shutil import copyfile from testgres import QueryException, StartNodeException from stat import S_ISDIR @@ -3709,66 +3709,6 @@ def test_concurrent_restore(self): self.compare_pgdata(pgdata1, pgdata2) self.compare_pgdata(pgdata2, pgdata3) - # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 - @unittest.skip("skip") - def test_restore_issue_313(self): - """ - Check that partially restored PostgreSQL instance cannot be started - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - node.cleanup() - - count = 0 - filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) - for file in filelist: - # count only nondata files - if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: - count += 1 - - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) - node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - - gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) - gdb.verbose = False - gdb.set_breakpoint('restore_non_data_file') - gdb.run_until_break() - gdb.continue_execution_until_break(count - 2) - gdb.quit() - - # emulate the user or HA taking care of PG configuration - for fname in os.listdir(node_restored.data_dir): - if fname.endswith('.conf'): - os.rename( - os.path.join(node_restored.data_dir, fname), - os.path.join(node.data_dir, fname)) - - try: - node.slow_start() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because backup is not fully restored") - except StartNodeException as e: - self.assertIn( - 'Cannot start node', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - # @unittest.skip("skip") def test_restore_with_waldir(self): """recovery using tablespace-mapping option and page backup""" @@ -3833,8 +3773,6 @@ def test_restore_to_latest_timeline(self): self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - - node.pgbench_init(scale=2) before1 = node.table_checksum("pgbench_branches") @@ -3850,8 +3788,6 @@ def test_restore_to_latest_timeline(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - - node.slow_start() pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -3925,3 +3861,72 @@ def test_restore_to_latest_timeline(self): # check for the current updates after = node.table_checksum("pgbench_branches") self.assertEqual(before1, after) + + def test_restore_issue_313(self): + """ + Check that partially restored PostgreSQL instance cannot be started + """ + self._check_gdb_flag_or_skip_test + node = self.make_simple_node('node', + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + node.cleanup() + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and \ + not stat.S_ISDIR(int(filelist[file]['mode'])) and \ + not filelist[file]['size'] == '0' and \ + file != 'database_map': + count += 1 + + node_restored = self.make_simple_node('node_restored') + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 1) + gdb.quit() + + # emulate the user or HA taking care of PG configuration + for fname in os.listdir(node_restored.data_dir): + if fname.endswith('.conf'): + os.rename( + os.path.join(node_restored.data_dir, fname), + os.path.join(node.data_dir, fname)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + if self.pg_config_version >= 120000: + self.assertIn( + "PANIC: could not read file \"global/pg_control\"", + f.read()) + else: + self.assertIn( + "PANIC: could not read from control file", + f.read()) From 343ed029a4277e0217da692dffe1a6748c80eaaa Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Fri, 23 Dec 2022 09:20:42 +0300 Subject: [PATCH 2075/2107] test_recovery_target_lsn_backup_victim - looks like it should pass and not fail --- tests/false_positive_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/false_positive_test.py b/tests/false_positive_test.py index fbb785c60..ea82cb18f 100644 --- a/tests/false_positive_test.py +++ b/tests/false_positive_test.py @@ -203,13 +203,16 @@ def test_recovery_target_time_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_lsn_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 + + @y.sokolov: looks like this test should pass. + So I commented 'expectedFailure' """ backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( From 46c8a3351dafaa48b6ed05efffd26b641d0ac48b Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 6 Dec 2023 11:59:39 +0300 Subject: [PATCH 2076/2107] Rewrite How-To into a Quick Start --- doc/pgprobackup.xml | 623 +++++++++++++++++++++++--------------------- 1 file changed, 332 insertions(+), 291 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1f764a432..466f474f5 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -171,6 +171,9 @@ doc/src/sgml/pgprobackup.sgml Overview + + Quick Start + Installation @@ -453,7 +456,267 @@ doc/src/sgml/pgprobackup.sgml - + + Quick Start + + To quickly get started with pg_probackup, complete the steps below. This will set up FULL and DELTA backups in the remote mode and demonstrate some + basic pg_probackup operations. In the following, these terms are used: + + + + + backup — PostgreSQL + role used to connect to the PostgreSQL + cluster. + + + + + backupdb — database used used to connect to the + PostgreSQL cluster. + + + + + backup_host — host with the backup catalog. + + + + + backup — user on + backup_host running all pg_probackup + operations. + + + + + /mnt/backups — directory on + backup_host where the backup catalog is stored. + + + + + postgres_host — host with the + PostgreSQL cluster. + + + + + postgres — user on + postgres_host under which + PostgreSQL cluster processes are running. + + + + + /var/lib/postgresql/16/main — + PostgreSQL data directory on + postgres_host. + + + + + Steps to perform: + + + Install <application>pg_probackup</application> on both <literal>backup_host</literal> and <literal>postgres_host</literal>. + + + <link linkend="pbk-setup-ssh">Set up an SSH connection</link> from <literal>backup_host</literal> to <literal>postgres_host</literal>. + + + <link linkend="pbk-configuring-the-database-cluster">Configure</link> your database cluster for <link linkend="pbk-setting-up-stream-backups">STREAM backups</link>. + + + Initialize the backup catalog: + +backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups +INFO: Backup catalog '/mnt/backups' successfully initialized + + + + Add a backup instance called <literal>mydb</literal> to the backup catalog: + +backup_user@backup_host:~$ pg_probackup-16 add-instance \ + -B /mnt/backups \ + -D /var/lib/postgresql/16/main \ + --instance=mydb \ + --remote-host=postgres_host \ + --remote-user=postgres +INFO: Instance 'mydb' successfully initialized + + + + Make a FULL backup: + +backup_user@backup_host:~$ pg_probackup-16 backup \ + -B /mnt/backups \ + -b FULL \ + --instance=mydb \ + --stream \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb +INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBFN, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBFN/database/pg_wal/000000010000000000000002 to be streamed +INFO: PGDATA size: 29MB +INFO: Current Start LSN: 0/2000060, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 1s +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/2003CB0 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 0 +INFO: Validating backup S6OBFN +INFO: Backup S6OBFN data files are valid +INFO: Backup S6OBFN resident size: 45MB +INFO: Backup S6OBFN completed + + + + List the backups of the instance: + +backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb +================================================================================================================================ + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================ + mydb 16 S6OBFN 2024-01-03 06:59:49+00 FULL STREAM 1/0 10s 29MB 16MB 1.00 0/2000060 0/2003CB0 OK + + + + Make an incremental backup in the DELTA mode: + +backup_user@backup_host:~$ pg_probackup-16 backup \ + -B /mnt/backups \ + -b delta \ + --instance=mydb \ + --stream \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb +INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBLG, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Parent backup: S6OBFN +INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBLG/database/pg_wal/000000010000000000000004 to be streamed +INFO: PGDATA size: 29MB +INFO: Current Start LSN: 0/4000028, TLI: 1 +INFO: Parent Start LSN: 0/2000060, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 1s +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/4000168 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 0 +INFO: Validating backup S6OBLG +INFO: Backup S6OBLG data files are valid +INFO: Backup S6OBLG resident size: 32MB +INFO: Backup S6OBLG completed + + + + Add or modify some parameters in the <application>pg_probackup</application> + configuration file, so that you do not have to specify them each time on the command line: + +backup_user@backup_host:~$ pg_probackup-16 set-config \ + -B /mnt/backups \ + --instance=mydb \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb + + + + Check the configuration of the instance: + +backup_user@backup_host:~$ pg_probackup-16 show-config -B /mnt/backups --instance=mydb +# Backup instance information +pgdata = /var/lib/postgresql/16/main +system-identifier = 7319761899046784808 +xlog-seg-size = 16777216 +# Connection parameters +pgdatabase = backupdb +pghost = postgres_host +pguser = backup +# Replica parameters +replica-timeout = 5min +# Archive parameters +archive-timeout = 5min +# Logging parameters +log-level-console = INFO +log-level-file = OFF +log-format-console = PLAIN +log-format-file = PLAIN +log-filename = pg_probackup.log +log-rotation-size = 0TB +log-rotation-age = 0d +# Retention parameters +retention-redundancy = 0 +retention-window = 0 +wal-depth = 0 +# Compression parameters +compress-algorithm = none +compress-level = 1 +# Remote access parameters +remote-proto = ssh +remote-host = postgres_host +remote-user = postgres + + + Note that the parameters not modified via set-config retain their default values. + + + + Make another incremental backup in the DELTA mode, omitting + the parameters stored in the configuration file earlier: + +backup_user@backup_host:~$ pg_probackup-16 backup -B /mnt/backups --instance=mydb -b delta --stream +INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBQO, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Parent backup: S6OBLG +INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBQO/database/pg_wal/000000010000000000000006 to be streamed +INFO: PGDATA size: 29MB +INFO: Current Start LSN: 0/6000028, TLI: 1 +INFO: Parent Start LSN: 0/4000028, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 1s +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/6000168 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 0 +INFO: Validating backup S6OBQO +INFO: Backup S6OBQO data files are valid +INFO: Backup S6OBQO resident size: 32MB +INFO: Backup S6OBQO completed + + + + List the backups of the instance again: + +backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb +================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================== + mydb 16 S6OBQO 2024-01-03 07:06:26+00 DELTA STREAM 1/1 6s 111kB 32MB 1.00 0/6000028 0/6000168 OK + mydb 16 S6OBLG 2024-01-03 07:03:18+00 DELTA STREAM 1/1 10s 127kB 32MB 1.00 0/4000028 0/4000168 OK + mydb 16 S6OBFN 2024-01-03 06:59:49+00 FULL STREAM 1/0 10s 29MB 16MB 1.00 0/2000060 0/2003CB0 OK + + + + + Installation @@ -744,7 +1007,7 @@ pg_probackup init -B backup_dir pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] - where: + Where: @@ -812,11 +1075,19 @@ pg_probackup add-instance -B backup_dir -D backup role is used as an example. + + For security reasons, it is recommended to run the configuration SQL queries below + in a separate database. + + +postgres=# CREATE DATABASE backupdb; +postgres=# \c backupdb + To perform a , the following permissions for role backup are required only in the database used for - connection to the PostgreSQL server: + connection to the PostgreSQL server. For PostgreSQL versions 11 — 14: @@ -908,7 +1179,18 @@ COMMIT; - Grant the REPLICATION privilege to the backup role: + If the backup role does not exist, create it with + the REPLICATION privilege when + Configuring the + Database Cluster: + + +CREATE ROLE backup WITH LOGIN REPLICATION; + + + + + If the backup role already exists, grant it with the REPLICATION privilege: ALTER ROLE backup WITH REPLICATION; @@ -1203,7 +1485,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; If you are going to use pg_probackup in remote mode via SSH, complete the following steps: - + Install pg_probackup on both systems: @@ -1213,7 +1495,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - For communication between the hosts set up the passwordless + For communication between the hosts set up a passwordless SSH connection between backup user on backup_host and postgres user on @@ -1222,54 +1504,64 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; [backup@backup_host] ssh-copy-id postgres@db_host + + Where: + + + + + backup_host is the system with + backup catalog. + + + + + db_host is the system with PostgreSQL + cluster. + + + + + backup is the OS user on + backup_host used to run pg_probackup. + + + + + postgres is the user on + postgres_host under which + PostgreSQL cluster processes are running. + For PostgreSQL 11 or higher a + more secure approach can be used thanks to + allow-group-access feature. + + + If you are going to rely on continuous - WAL archiving, set up passwordless SSH - connection between postgres user on - db_host and backup + WAL archiving, set up a passwordless SSH + connection between the postgres user on + db_host and the backup user on backup_host: [postgres@db_host] ssh-copy-id backup@backup_host - - - where: - - - - - backup_host is the system with - backup catalog. - - - - - db_host is the system with PostgreSQL - cluster. - - - - - backup is the OS user on - backup_host used to run pg_probackup. - - - postgres is the OS user on - db_host used to start the PostgreSQL - cluster. For PostgreSQL 11 or higher a - more secure approach can be used thanks to - allow-group-access - feature. + Make sure pg_probackup on postgres_host + can be located when a connection via SSH is made. For example, for Bash, you can + modify PATH in ~/.bashrc of the backup user. + Alternatively, for pg_probackup commands, specify the path to the directory + containing the pg_probackup binary on postgres_host via + the --remote-path option. - + pg_probackup in the remote mode via SSH works as follows: @@ -1763,7 +2055,7 @@ pg_probackup validate -B backup_dir --instance backup_dir --instance instance_name -i backup_id - where: + Where: @@ -5935,257 +6227,6 @@ pg_probackup catchup -b catchup_mode - - How-To - - All examples below assume the remote mode of operations via - SSH. If you are planning to run backup and - restore operation locally, skip the - Setup passwordless SSH connection step - and omit all options. - - - Examples are based on Ubuntu 18.04, - PostgreSQL 11, and pg_probackup - 2.2.0. - - - - - backup — PostgreSQL - role used for connection to PostgreSQL - cluster. - - - - - backupdb — database used for connection - to PostgreSQL cluster. - - - - - backup_host — host with backup catalog. - - - - - backupman — user on - backup_host running all pg_probackup - operations. - - - - - /mnt/backups — directory on - backup_host where backup catalog is stored. - - - - - postgres_host — host with PostgreSQL - cluster. - - - - - postgres — user on - postgres_host that has started the PostgreSQL cluster. - - - - - /var/lib/postgresql/11/main — PostgreSQL - data directory on postgres_host. - - - - - Minimal Setup - - This scenario illustrates setting up standalone FULL and DELTA backups. - - - - Set up passwordless SSH connection from - <literal>backup_host</literal> to - <literal>postgres_host</literal>: - -[backupman@backup_host] ssh-copy-id postgres@postgres_host - - - - Configure your <productname>PostgreSQL</productname> cluster. - - For security purposes, it is recommended to use a separate - database for backup operations. - - -postgres=# -CREATE DATABASE backupdb; - - - Connect to the backupdb database, create the - probackup role, and grant the following - permissions to this role: - - -backupdb=# -BEGIN; -CREATE ROLE backup WITH LOGIN REPLICATION; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; - - - - Initialize the backup catalog: - -[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups -INFO: Backup catalog '/mnt/backups' successfully inited - - - - Add instance <literal>pg-11</literal> to the backup catalog: - -[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main -INFO: Instance 'node' successfully inited - - - - Take a FULL backup: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YK2 -INFO: Backup PZ7YK2 data files are valid -INFO: Backup PZ7YK2 resident size: 196MB -INFO: Backup PZ7YK2 completed - - - - Let's take a look at the backup catalog: - -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance pg-11 - -BACKUP INSTANCE 'pg-11' -================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - - - - Take an incremental backup in the DELTA mode: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b delta --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YK2 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YMP -INFO: Backup PZ7YMP data files are valid -INFO: Backup PZ7YMP resident size: 32MB -INFO: Backup PZ7YMP completed - - - - Let's add some parameters to <application>pg_probackup</application> - configuration file, so that you can omit them from the command line: - -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance pg-11 --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb - - - - Take another incremental backup in the DELTA mode, omitting - some of the previous parameters: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance pg-11 -b delta --stream -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YMP -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YR5 -INFO: Backup PZ7YR5 data files are valid -INFO: Backup PZ7YR5 resident size: 32MB -INFO: Backup PZ7YR5 completed - - - - Let's take a look at the instance configuration: - -[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance pg-11 - -# Backup instance information -pgdata = /var/lib/postgresql/11/main -system-identifier = 6746586934060931492 -xlog-seg-size = 16777216 -# Connection parameters -pgdatabase = backupdb -pghost = postgres_host -pguser = backup -# Archive parameters -archive-timeout = 5min -# Logging parameters -log-level-console = INFO -log-level-file = OFF -log-format-console = PLAIN -log-format-file = PLAIN -log-filename = pg_probackup.log -log-rotation-size = 0 -log-rotation-age = 0 -# Retention parameters -retention-redundancy = 0 -retention-window = 0 -wal-depth = 0 -# Compression parameters -compress-algorithm = none -compress-level = 1 -# Remote access parameters -remote-proto = ssh -remote-host = postgres_host - - - Note that we are getting the default values for other options - that were not overwritten by the set-config command. - - - - Let's take a look at the backup catalog: - -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance pg-11 - -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - - - - - - Versioning From 36b9761fa1d92c277c5a6630c43defb3e6a4af58 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 8 Dec 2023 11:44:18 +0700 Subject: [PATCH 2077/2107] PBCKP-817 Update documentation examples to 2.7.0 --- doc/pgprobackup.xml | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 466f474f5..fb2a8f599 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -483,7 +483,7 @@ doc/src/sgml/pgprobackup.sgml - backup — user on + backup_user — user on backup_host running all pg_probackup operations. @@ -1472,12 +1472,12 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Configuring the Remote Mode - pg_probackup supports the remote mode that allows to perform - backup, restore and WAL archiving operations remotely. In this - mode, the backup catalog is stored on a local system, while - PostgreSQL instance to backup and/or to restore is located on a - remote system. Currently the only supported remote protocol is - SSH. + pg_probackup supports the remote mode that + allows to perform backup, restore and WAL archiving operations remotely. + In this mode, the backup catalog is stored on a local system, while + PostgreSQL instance to backup and/or to restore + is located on a remote system. Currently the only supported remote + protocol is SSH. Set up SSH @@ -1490,19 +1490,19 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Install pg_probackup on both systems: backup_host and - db_host. + postgres_host. For communication between the hosts set up a passwordless - SSH connection between backup user on - backup_host and + SSH connection between the backup_user user on + backup_host and the postgres user on - db_host: + postgres_host: -[backup@backup_host] ssh-copy-id postgres@db_host +[backup_user@backup_host] ssh-copy-id postgres@postgres_host Where: @@ -1516,13 +1516,13 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - db_host is the system with PostgreSQL + postgres_host is the system with the PostgreSQL cluster. - backup is the OS user on + backup_user is the OS user on backup_host used to run pg_probackup. @@ -1544,18 +1544,19 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; continuous WAL archiving, set up a passwordless SSH connection between the postgres user on - db_host and the backup + postgres_host and the backup user on backup_host: -[postgres@db_host] ssh-copy-id backup@backup_host +[postgres@postgres_host] ssh-copy-id backup_user@backup_host Make sure pg_probackup on postgres_host can be located when a connection via SSH is made. For example, for Bash, you can - modify PATH in ~/.bashrc of the backup user. + modify PATH in ~/.bashrc of the postgres user + (above the line in bashrc which exits the script for non-interactive shells). Alternatively, for pg_probackup commands, specify the path to the directory containing the pg_probackup binary on postgres_host via the --remote-path option. @@ -1611,10 +1612,10 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; The main process is usually started on backup_host and connects to - db_host, but in case of + postgres_host, but in case of archive-push and archive-get commands the main process - is started on db_host and connects to + is started on postgres_host and connects to backup_host. @@ -1635,7 +1636,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Compression is always done on - db_host, while decompression is always done on + postgres_host, while decompression is always done on backup_host. From 9587b75b1fbde7231388af2ff0e7c6d86722e55b Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Wed, 13 Dec 2023 06:33:15 +0300 Subject: [PATCH 2078/2107] A few documentation edits --- doc/pgprobackup.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index fb2a8f599..547b76b0a 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2434,7 +2434,8 @@ pg_probackup restore -B backup_dir --instance In addition to SSH connection, pg_probackup uses a regular connection to the database to manage the remote operation. - See for details of how to set up + See the section Configuring + the Database Cluster for details of how to set up a database connection. @@ -3941,7 +3942,7 @@ pg_probackup delete -B backup_dir --instance To prepare for cloning/synchronizing a PostgreSQL instance, - set up the source instance server as follows: + set up the source server as follows: @@ -3964,7 +3965,7 @@ pg_probackup delete -B backup_dir --instance Before cloning/synchronizing a PostgreSQL instance, ensure that the source - instance server is running and accepting connections. To clone/sync a PostgreSQL instance, + server is running and accepting connections. To clone/sync a PostgreSQL instance, on the server with the destination instance, you can run the command as follows: @@ -4007,7 +4008,7 @@ pg_probackup catchup -b catchup_mode --source-pgdata= By specifying the option, you can set STREAM WAL delivery mode of copying, which will include all the necessary WAL files by streaming them from - the instance server via replication protocol. + the server via replication protocol. You can use connection_options to specify @@ -4998,7 +4999,7 @@ pg_probackup catchup -b catchup_mode Copies the instance in STREAM WAL delivery mode, including all the necessary WAL files by streaming them from - the instance server via replication protocol. + the server via replication protocol. From c9439b65e26b3bd8210e2903c23f23976ec4e284 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Mon, 18 Dec 2023 15:37:22 +0300 Subject: [PATCH 2079/2107] [DOC] Fix syntax errorn in doc step elements --- doc/pgprobackup.xml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 547b76b0a..7e0787c8e 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -519,23 +519,23 @@ doc/src/sgml/pgprobackup.sgml Steps to perform: - Install <application>pg_probackup</application> on both <literal>backup_host</literal> and <literal>postgres_host</literal>. + Install pg_probackup on both backup_host and postgres_host. - <link linkend="pbk-setup-ssh">Set up an SSH connection</link> from <literal>backup_host</literal> to <literal>postgres_host</literal>. + Set up an SSH connection from backup_host to postgres_host. - <link linkend="pbk-configuring-the-database-cluster">Configure</link> your database cluster for <link linkend="pbk-setting-up-stream-backups">STREAM backups</link>. + Configure your database cluster for STREAM backups. - Initialize the backup catalog: + Initialize the backup catalog: backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized - Add a backup instance called <literal>mydb</literal> to the backup catalog: + Add a backup instance called mydb to the backup catalog: backup_user@backup_host:~$ pg_probackup-16 add-instance \ -B /mnt/backups \ @@ -547,7 +547,7 @@ INFO: Instance 'mydb' successfully initialized - Make a FULL backup: + Make a FULL backup: backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ @@ -579,7 +579,7 @@ INFO: Backup S6OBFN completed - List the backups of the instance: + List the backups of the instance: backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb ================================================================================================================================ @@ -589,7 +589,7 @@ backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb - Make an incremental backup in the DELTA mode: + Make an incremental backup in the DELTA mode: backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ @@ -623,8 +623,8 @@ INFO: Backup S6OBLG completed - Add or modify some parameters in the <application>pg_probackup</application> - configuration file, so that you do not have to specify them each time on the command line: + Add or modify some parameters in the pg_probackup + configuration file, so that you do not have to specify them each time on the command line: backup_user@backup_host:~$ pg_probackup-16 set-config \ -B /mnt/backups \ @@ -636,7 +636,7 @@ backup_user@backup_host:~$ pg_probackup-16 set-config \ - Check the configuration of the instance: + Check the configuration of the instance: backup_user@backup_host:~$ pg_probackup-16 show-config -B /mnt/backups --instance=mydb # Backup instance information @@ -676,8 +676,8 @@ remote-user = postgres - Make another incremental backup in the DELTA mode, omitting - the parameters stored in the configuration file earlier: + Make another incremental backup in the DELTA mode, omitting + the parameters stored in the configuration file earlier: backup_user@backup_host:~$ pg_probackup-16 backup -B /mnt/backups --instance=mydb -b delta --stream INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBQO, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 @@ -703,7 +703,7 @@ INFO: Backup S6OBQO completed - List the backups of the instance again: + List the backups of the instance again: backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb ================================================================================================================================== From f8c46efb5a700167bcb31cfcb3b41cfe50687ce6 Mon Sep 17 00:00:00 2001 From: Elena Indrupskaya Date: Tue, 26 Dec 2023 14:52:48 +0300 Subject: [PATCH 2080/2107] [PBCKP-865] Fix minor but grammar --- doc/pgprobackup.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7e0787c8e..14f14040e 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -472,7 +472,7 @@ doc/src/sgml/pgprobackup.sgml - backupdb — database used used to connect to the + backupdb — database used to connect to the PostgreSQL cluster. @@ -1473,7 +1473,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Configuring the Remote Mode pg_probackup supports the remote mode that - allows to perform backup, restore and WAL archiving operations remotely. + allows you to perform backup, restore and WAL archiving operations remotely. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to backup and/or to restore is located on a remote system. Currently the only supported remote @@ -1556,7 +1556,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Make sure pg_probackup on postgres_host can be located when a connection via SSH is made. For example, for Bash, you can modify PATH in ~/.bashrc of the postgres user - (above the line in bashrc which exits the script for non-interactive shells). + (above the line in bashrc that exits the script for non-interactive shells). Alternatively, for pg_probackup commands, specify the path to the directory containing the pg_probackup binary on postgres_host via the --remote-path option. @@ -2188,7 +2188,7 @@ pg_probackup restore -B backup_dir --instance LSN — read the pg_control in the - data directory to obtain redo LSN and redo TLI, which allows + data directory to obtain redo LSN and redo TLI, which allows you to determine a point in history(shiftpoint), where data directory state shifted from target backup chain history. If shiftpoint is not within reach of backup chain history, then restore is aborted. @@ -2417,7 +2417,7 @@ pg_probackup restore -B backup_dir --instance Using <application>pg_probackup</application> in the Remote Mode - pg_probackup supports the remote mode that allows to perform + pg_probackup supports the remote mode that allows you to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed @@ -3782,7 +3782,7 @@ pg_probackup merge -B backup_dir --instance pg_probackup in the remote mode. From b4035fd23d0fde7b517d2738c2506eb842f245d1 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 27 Dec 2023 05:50:42 +0300 Subject: [PATCH 2081/2107] PBCKP-817 Add an example of the restore command --- doc/pgprobackup.xml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 14f14040e..7d5ab5ccf 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -565,7 +565,7 @@ INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBFN/database/pg_wal/0000 INFO: PGDATA size: 29MB INFO: Current Start LSN: 0/2000060, TLI: 1 INFO: Start transferring data files -INFO: Data files are transferred, time elapsed: 1s +INFO: Data files are transferred, time elapsed: 0 INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed INFO: stop_lsn: 0/2003CB0 @@ -689,7 +689,7 @@ INFO: PGDATA size: 29MB INFO: Current Start LSN: 0/6000028, TLI: 1 INFO: Parent Start LSN: 0/4000028, TLI: 1 INFO: Start transferring data files -INFO: Data files are transferred, time elapsed: 1s +INFO: Data files are transferred, time elapsed: 0 INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed INFO: stop_lsn: 0/6000168 @@ -714,6 +714,27 @@ backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb mydb 16 S6OBFN 2024-01-03 06:59:49+00 FULL STREAM 1/0 10s 29MB 16MB 1.00 0/2000060 0/2003CB0 OK + + Restore the data from the latest available backup to an arbitrary location: + +backup_user@backup_host:~$ pg_probackup-16 restore -B /mnt/backups -D /var/lib/postgresql/16/staging --instance=mydb +INFO: Validating parents for backup S6OBQO +INFO: Validating backup S6OBFN +INFO: Backup S6OBFN data files are valid +INFO: Validating backup S6OBLG +INFO: Backup S6OBLG data files are valid +INFO: Validating backup S6OBQO +INFO: Backup S6OBQO data files are valid +INFO: Backup S6OBQO WAL segments are valid +INFO: Backup S6OBQO is valid. +INFO: Restoring the database from backup at 2024-01-03 07:06:24+00 +INFO: Start restoring backup files. PGDATA size: 61MB +INFO: Backup files are restored. Transfered bytes: 61MB, time elapsed: 1s +INFO: Restore incremental ratio (less is better): 100% (61MB/61MB) +INFO: Syncing restored files to disk +INFO: Restored backup files are synced, time elapsed: 0 +INFO: Restore of backup S6OBQO completed. + From 9a91ea78dd7a5694bc2688616a3028df259209cc Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 3 Jan 2024 17:12:40 +0700 Subject: [PATCH 2082/2107] Add a link to the installation section in the Quick start --- doc/pgprobackup.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 7d5ab5ccf..26dfde6e1 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -519,7 +519,7 @@ doc/src/sgml/pgprobackup.sgml Steps to perform: - Install pg_probackup on both backup_host and postgres_host. + Install pg_probackup on both backup_host and postgres_host. Set up an SSH connection from backup_host to postgres_host. @@ -734,6 +734,7 @@ INFO: Restore incremental ratio (less is better): 100% (61MB/61MB) INFO: Syncing restored files to disk INFO: Restored backup files are synced, time elapsed: 0 INFO: Restore of backup S6OBQO completed. + From 58752c5a755e29ed690ce6dea9e96eb7419d6fe4 Mon Sep 17 00:00:00 2001 From: oleg gurev Date: Fri, 26 Jan 2024 11:02:37 +0300 Subject: [PATCH 2083/2107] [PBCKP-198] Added tablespaces into show command output - Solve Issue #431 - Output all tablespaces in backup in JSON output - Output all tablespaces in backup in PLAIN output --- src/show.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++ tests/show_test.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/src/show.c b/src/show.c index 86a122698..810262df6 100644 --- a/src/show.c +++ b/src/show.c @@ -67,6 +67,7 @@ static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *timelines_list, bool show_name); static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, parray *tli_list); +static bool backup_has_tablespace_map(pgBackup *backup); static PQExpBufferData show_buf; static bool first_instance = true; @@ -479,6 +480,32 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) appendPQExpBuffer(buf, "%u", backup->content_crc); } + /* print tablespaces list */ + if (backup_has_tablespace_map(backup)) + { + parray *links = parray_new(); + + json_add_key(buf, "tablespace_map", json_level); + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + read_tablespace_map(links, backup->root_dir); + parray_qsort(links, pgFileCompareLinked); + + for (size_t i = 0; i < parray_num(links); i++){ + pgFile *link = (pgFile *) parray_get(links, i); + if (i) + appendPQExpBufferChar(buf, ','); + json_add(buf, JT_BEGIN_OBJECT, &json_level); + json_add_value(buf, "oid", link->name, json_level, true); + json_add_value(buf, "path", link->linked, json_level, true); + json_add(buf, JT_END_OBJECT, &json_level); + } + /* End of tablespaces */ + json_add(buf, JT_END_ARRAY, &json_level); + parray_walk(links, pgFileFree); + parray_free(links); + } + json_add(buf, JT_END_OBJECT, &json_level); } @@ -521,7 +548,27 @@ show_backup(InstanceState *instanceState, time_t requested_backup_id) } if (show_format == SHOW_PLAIN) + { pgBackupWriteControl(stdout, backup, false); + + /* print tablespaces list */ + if (backup_has_tablespace_map(backup)) + { + parray *links = parray_new(); + + fio_fprintf(stdout, "\ntablespace_map = '"); + + read_tablespace_map(links, backup->root_dir); + parray_qsort(links, pgFileCompareLinked); + + for (size_t i = 0; i < parray_num(links); i++){ + pgFile *link = (pgFile *) parray_get(links, i); + fio_fprintf(stdout, "%s %s%s", link->name, link->linked, (i < parray_num(links) - 1) ? "; " : "'\n"); + } + parray_walk(links, pgFileFree); + parray_free(links); + } + } else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -1174,3 +1221,10 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, first_instance = false; } + +static bool backup_has_tablespace_map(pgBackup *backup) +{ + char map_path[MAXPGPATH]; + join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); + return fileExists(map_path, FIO_BACKUP_HOST); +} diff --git a/tests/show_test.py b/tests/show_test.py index c4b96499d..27b6fab96 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -507,3 +507,39 @@ def test_color_with_no_terminal(self): '[0m', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_tablespace_print_issue_431(self): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + full_backup_id = self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir,'node', as_text=True)) + # Check that tablespace info exists. JSON + self.assertIn("tablespace_map", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn("oid", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn("path", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn(tblspc_path, self.show_pb(backup_dir, 'node', as_text=True)) + # Check that tablespace info exists. PLAIN + self.assertIn("tablespace_map", self.show_pb(backup_dir, 'node', backup_id=full_backup_id, as_text=True, as_json=False)) + self.assertIn(tblspc_path, self.show_pb(backup_dir, 'node', backup_id=full_backup_id, as_text=True, as_json=False)) + # Check that tablespace info NOT exists if backup id not provided. PLAIN + self.assertNotIn("tablespace_map", self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) From 09236c6583a89bce6231abdd38d1b4e3b5062d0b Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Tue, 19 Dec 2023 14:56:09 +0300 Subject: [PATCH 2084/2107] [PBCKP-874] Addeded "Logging options" section to help.c for - add-instance - archive-push - archive-get - catchup commands --- src/help.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/help.c b/src/help.c index 954ba6416..b3ba02160 100644 --- a/src/help.c +++ b/src/help.c @@ -983,6 +983,30 @@ help_add_instance(void) printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } static void @@ -1030,6 +1054,30 @@ help_archive_push(void) printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); @@ -1065,6 +1113,30 @@ help_archive_get(void) printf(_(" --prefetch-dir=path location of the store area for prefetched WAL files\n")); printf(_(" --no-validate-wal skip validation of prefetched WAL file before using it\n")); + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); @@ -1131,6 +1203,30 @@ help_catchup(void) printf(_(" -w, --no-password never prompt for password\n")); printf(_(" -W, --password force password prompt\n\n")); + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); From 43318d696be4dcec37eeb433ea4086ec2127a24b Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Fri, 12 Jan 2024 17:37:51 +0300 Subject: [PATCH 2085/2107] [PBCKP-874] Addeded default unit output for config-show - config-show command now has --default-units parameter - If it is provided, all time and memory - configuration options will be printed in their default units - test added, test_help_1 corrected --- src/configure.c | 7 ++++-- src/help.c | 4 ++- src/pg_probackup.c | 5 +++- src/pg_probackup.h | 2 +- src/utils/configuration.c | 41 +++++++++++++++++++++---------- src/utils/configuration.h | 1 + tests/expected/option_help.out | 1 + tests/expected/option_help_ru.out | 1 + tests/option_test.py | 19 ++++++++++++++ 9 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/configure.c b/src/configure.c index f7befb0c5..4f6774d55 100644 --- a/src/configure.c +++ b/src/configure.c @@ -269,7 +269,7 @@ static const char *current_group = NULL; * Show configure options including default values. */ void -do_show_config(void) +do_show_config(bool show_default_units) { int i; @@ -277,10 +277,13 @@ do_show_config(void) for (i = 0; instance_options[i].type; i++) { + if (show_default_units && strchr("bBiIuU", instance_options[i].type) && instance_options[i].get_value == *option_get_value) + instance_options[i].flags |= GET_VAL_IN_DEFAULT_UNITS; /* Set flag */ if (show_format == SHOW_PLAIN) show_configure_plain(&instance_options[i]); else show_configure_json(&instance_options[i]); + instance_options[i].flags &= ~(GET_VAL_IN_DEFAULT_UNITS); /* Reset flag. It was resetted in option_get_value(). Probably this reset isn't needed */ } show_configure_end(); @@ -801,6 +804,6 @@ show_configure_json(ConfigOption *opt) return; json_add_value(&show_buf, opt->lname, value, json_level, - true); + !(opt->flags & GET_VAL_IN_DEFAULT_UNITS)); pfree(value); } diff --git a/src/help.c b/src/help.c index b3ba02160..0ccae938f 100644 --- a/src/help.c +++ b/src/help.c @@ -121,6 +121,7 @@ help_pg_probackup(void) printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); + printf(_(" [--default-units]\n")); printf(_(" [--help]\n")); printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); @@ -953,7 +954,8 @@ help_show_config(void) printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" --format=format show format=PLAIN|JSON\n\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --default-units show memory and time values in default units\n\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 6653898e4..9b896d8bc 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -164,6 +164,7 @@ bool no_validate_wal = false; /* show options */ ShowFormat show_format = SHOW_PLAIN; bool show_archive = false; +static bool show_default_units = false; /* set-backup options */ int64 ttl = -1; @@ -275,6 +276,8 @@ static ConfigOption cmd_options[] = /* show options */ { 'f', 165, "format", opt_show_format, SOURCE_CMD_STRICT }, { 'b', 166, "archive", &show_archive, SOURCE_CMD_STRICT }, + /* show-config options */ + { 'b', 167, "default-units", &show_default_units,SOURCE_CMD_STRICT }, /* set-backup options */ { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, @@ -1049,7 +1052,7 @@ main(int argc, char *argv[]) do_merge(instanceState, current.backup_id, no_validate, no_sync); break; case SHOW_CONFIG_CMD: - do_show_config(); + do_show_config(show_default_units); break; case SET_CONFIG_CMD: do_set_config(instanceState, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 61dd2ce0e..bfb551ace 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -939,7 +939,7 @@ extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instanc char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ -extern void do_show_config(void); +extern void do_show_config(bool show_default_units); extern void do_set_config(InstanceState *instanceState, bool missing_ok); extern void init_config(InstanceConfig *config, const char *instance_name); extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 921555350..6b2382996 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -678,6 +678,8 @@ config_set_opt(ConfigOption options[], void *var, OptionSource source) /* * Return value of the function in the string representation. Result is * allocated string. + * We can set GET_VAL_IN_DEFAULT_UNITS flag in opt->flags + * before call option_get_value() to get option value in default units */ char * option_get_value(ConfigOption *opt) @@ -692,20 +694,33 @@ option_get_value(ConfigOption *opt) */ if (opt->flags & OPTION_UNIT) { - if (opt->type == 'i') - convert_from_base_unit(*((int32 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'I') - convert_from_base_unit(*((int64 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'u') - convert_from_base_unit_u(*((uint32 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); - else if (opt->type == 'U') - convert_from_base_unit_u(*((uint64 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); + if (opt->flags & GET_VAL_IN_DEFAULT_UNITS){ + if (opt->type == 'i') + value = *((int32 *) opt->var); + else if (opt->type == 'I') + value = *((int64 *) opt->var); + else if (opt->type == 'u') + value_u = *((uint32 *) opt->var); + else if (opt->type == 'U') + value_u = *((uint64 *) opt->var); + unit = ""; + } + else + { + if (opt->type == 'i') + convert_from_base_unit(*((int32 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'I') + convert_from_base_unit(*((int64 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'u') + convert_from_base_unit_u(*((uint32 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + else if (opt->type == 'U') + convert_from_base_unit_u(*((uint64 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + } } - /* Get string representation itself */ switch (opt->type) { diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 2c6ea3eec..f3bda65de 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -100,6 +100,7 @@ struct ConfigOption #define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) +#define GET_VAL_IN_DEFAULT_UNITS 0x80000000 /* bitflag to get memory and time values in default units*/ extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 49f79607f..618a0d156 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -39,6 +39,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup show-config -B backup-path --instance=instance_name [--format=format] + [--default-units] [--help] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 976932b9d..005c74ebb 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -39,6 +39,7 @@ pg_probackup - утилита для управления резервным к pg_probackup show-config -B backup-path --instance=instance_name [--format=format] + [--default-units] [--help] pg_probackup backup -B backup-path -b backup-mode --instance=instance_name diff --git a/tests/option_test.py b/tests/option_test.py index 66cc13746..9a829aaef 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -230,6 +230,25 @@ def test_help_6(self): self.skipTest( 'You need configure PostgreSQL with --enabled-nls option for this test') + # @unittest.skip("skip") + def test_options_default_units(self): + """check --default-units option""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # check that --default-units option works correctly + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node"]) + self.assertIn(container=output, member="archive-timeout = 5min") + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--default-units"]) + self.assertIn(container=output, member="archive-timeout = 300") + self.assertNotIn(container=output, member="archive-timeout = 300s") + # check that we have now quotes ("") in json output + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--default-units", "--format=json"]) + self.assertIn(container=output, member='"archive-timeout": 300,') + self.assertIn(container=output, member='"retention-redundancy": 0,') + self.assertNotIn(container=output, member='"archive-timeout": "300",') def check_locale(locale_name): ret=True From 031d3ebef77f293d4ac491d485e08d22e9fc7e63 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 2 Feb 2024 12:07:24 +0700 Subject: [PATCH 2086/2107] Put listitem contents in --- doc/pgprobackup.xml | 108 +++++++++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 26dfde6e1..74389f9e0 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -748,7 +748,9 @@ INFO: Restore of backup S6OBQO completed. - Add the pg_probackup repository GPG key + + Add the pg_probackup repository GPG key + sudo apt install gpg wget wget -qO - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG-PROBACKUP | \ @@ -756,7 +758,9 @@ sudo tee /etc/apt/trusted.gpg.d/pg_probackup.asc - Setup the binary package repository + + Setup the binary package repository + . /etc/os-release echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ @@ -764,24 +768,32 @@ sudo tee /etc/apt/sources.list.d/pg_probackup.list - Optionally setup the source package repository for rebuilding the binaries + + Optionally setup the source package repository for rebuilding the binaries + echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ sudo tee -a /etc/apt/sources.list.d/pg_probackup.list - List the available pg_probackup packages + + List the available pg_probackup packages + - Using apt: + + Using apt: + sudo apt update apt search pg_probackup - Using apt-get: + + Using apt-get: + sudo apt-get update apt-cache search pg_probackup @@ -790,19 +802,25 @@ apt-cache search pg_probackup - Install or upgrade a pg_probackup version of your choice + + Install or upgrade a pg_probackup version of your choice + sudo apt install pg-probackup-15 - Optionally install the debug package + + Optionally install the debug package + sudo apt install pg-probackup-15-dbg - Optionally install the source package (provided you have set up the source package repository as described above) + + Optionally install the source package (provided you have set up the source package repository as described above) + sudo apt install dpkg-dev sudo apt source pg-probackup-15 @@ -817,41 +835,55 @@ sudo apt source pg-probackup-15 - Install the pg_probackup repository + + Install the pg_probackup repository + dnf install https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm - List the available pg_probackup packages + + List the available pg_probackup packages + dnf search pg_probackup - Install or upgrade a pg_probackup version of your choice + + Install or upgrade a pg_probackup version of your choice + dnf install pg_probackup-15 - Optionally install the debug package + + Optionally install the debug package + dnf install pg_probackup-15-debuginfo - Optionally install the source package for rebuilding the binaries + + Optionally install the source package for rebuilding the binaries + - Using dnf: + + Using dnf: + dnf install 'dnf-command(download)' dnf download --source pg_probackup-15 - Using yum: + + Using yum: + yumdownloader --source pg_probackup-15 @@ -864,10 +896,14 @@ yumdownloader --source pg_probackup-15 Installation on ALT Linux - Setup the repository + + Setup the repository + - On ALT Linux 10: + + On ALT Linux 10: + . /etc/os-release echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p$VERSION_ID x86_64 vanilla" | \ @@ -875,7 +911,9 @@ sudo tee /etc/apt/sources.list.d/pg_probackup.list - On ALT Linux 8 and 9: + + On ALT Linux 8 and 9: + . /etc/os-release echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-$VERSION_ID x86_64 vanilla" | \ @@ -885,20 +923,26 @@ sudo tee /etc/apt/sources.list.d/pg_probackup.list - List the available pg_probackup packages + + List the available pg_probackup packages + sudo apt-get update apt-cache search pg_probackup - Install or upgrade a pg_probackup version of your choice + + Install or upgrade a pg_probackup version of your choice + sudo apt-get install pg_probackup-15 - Optionally install the debug package + + Optionally install the debug package + sudo apt-get install pg_probackup-15-debuginfo @@ -909,7 +953,9 @@ sudo apt-get install pg_probackup-15-debuginfo Installation on SUSE Linux - Add the pg_probackup repository GPG key + + Add the pg_probackup repository GPG key + zypper in -y gpg wget wget -O GPG-KEY-PG_PROBACKUP https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP @@ -917,25 +963,33 @@ rpm --import GPG-KEY-PG_PROBACKUP - Setup the repository + + Setup the repository + zypper in https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm - List the available pg_probackup packages + + List the available pg_probackup packages + zypper se pg_probackup - Install or upgrade a pg_probackup version of your choice + + Install or upgrade a pg_probackup version of your choice + zypper in pg_probackup-15 - Optionally install the source package for rebuilding the binaries + + Optionally install the source package for rebuilding the binaries + zypper si pg_probackup-15 From 287e7fc89f7d84fc9c3ea07f89798a64224c6b85 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Fri, 9 Feb 2024 15:50:58 +0300 Subject: [PATCH 2087/2107] Add_requirements.txt testgres can be installed with pip install -r pg_probackup/tests/requirements.txt --- tests/requirements.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/requirements.txt diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..62efb0e68 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,13 @@ +# Testgres can be installed in the following ways: +# 1. From a pip package (recommended) +# testgres==1.8.5 +# 2. From a specific Git branch, tag or commit +# git+https://github.com/postgrespro/testgres.git@ +# 3. From a local directory +# /path/to/local/directory/testgres +git+https://github.com/postgrespro/testgres.git@master#egg=testgres-pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 +allure-pytest +deprecation +pexpect +pytest +pytest-xdist From 17037baea211bf2148a09e1c3b9284847647d33b Mon Sep 17 00:00:00 2001 From: oleg gurev Date: Wed, 14 Feb 2024 10:51:47 +0300 Subject: [PATCH 2088/2107] [PBCKP-913] Fix WAL switching with huge XLogRecord - Backport of PBCKP-859 bugfix - increase current segment number when reader has already read it before - avoid error if reader has to switch WAL again - add python test for PAGE backup with huge XLog record --- src/parsexlog.c | 8 ++++++++ tests/page_test.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/parsexlog.c b/src/parsexlog.c index 7c4b5b349..7df169fbf 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -1588,9 +1588,14 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) reader_data = (XLogReaderData *) xlogreader->private_data; reader_data->need_switch = false; +start: /* Critical section */ pthread_lock(&wal_segment_mutex); Assert(segno_next); + + if (reader_data->xlogsegno > segno_next) + segno_next = reader_data->xlogsegno; + reader_data->xlogsegno = segno_next; segnum_read++; segno_next++; @@ -1604,6 +1609,7 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) GetXLogRecPtr(reader_data->xlogsegno, 0, wal_seg_size, arg->startpoint); /* We need to close previously opened file if it wasn't closed earlier */ CleanupXLogPageRead(xlogreader); + xlogreader->currRecPtr = InvalidXLogRecPtr; /* Skip over the page header and contrecord if any */ found = XLogFindNextRecord(xlogreader, arg->startpoint); @@ -1613,6 +1619,8 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) */ if (XLogRecPtrIsInvalid(found)) { + if (reader_data->need_switch) + goto start; /* * Check if we need to stop reading. We stop if other thread found a * target segment. diff --git a/tests/page_test.py b/tests/page_test.py index 99f3ce992..a66d6d413 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -1415,3 +1415,50 @@ def test_page_pg_resetxlog(self): # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) + + def test_page_huge_xlog_record(self): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_locks_per_transaction': '1000', + 'work_mem': '100MB', + 'temp_buffers': '100MB', + 'wal_buffers': '128MB', + 'wal_level' : 'logical', + }) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Do full backup + self.backup_node(backup_dir, 'node', node, backup_type='full') + show_backup = self.show_pb(backup_dir,'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Originally client had the problem at the transaction that (supposedly) + # deletes a lot of temporary tables (probably it was client disconnect). + # It generated ~40MB COMMIT WAL record. + # + # `pg_logical_emit_message` is much simpler and faster way to generate + # such huge record. + node.safe_psql( + "postgres", + "select pg_logical_emit_message(False, 'z', repeat('o', 60*1000*1000))") + + # Do page backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + show_backup = self.show_pb(backup_dir,'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") From 90a4a4f4b32128ee728c530b1fabba608b3d51eb Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 14 Feb 2024 12:57:52 +0700 Subject: [PATCH 2089/2107] Replace BACKUP_PATH in the source files --- doc/pgprobackup.xml | 10 +++++----- po/ru.po | 2 +- src/archive.c | 4 ++-- src/catalog.c | 2 +- src/help.c | 22 +++++++++++----------- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 8 ++++---- src/pg_probackup_state.h | 6 +++--- tests/option_test.py | 2 +- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 74389f9e0..49e74e626 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1128,7 +1128,7 @@ pg_probackup add-instance -B backup_dir -D backup_dir directory and at least read-only access to data_dir directory. If you specify the path to the backup catalog in the - BACKUP_PATH environment variable, you can + BACKUP_DIR environment variable, you can omit the corresponding option when running pg_probackup commands. @@ -5212,14 +5212,14 @@ pg_probackup catchup -b catchup_mode -BACKUP_PATH +BACKUP_DIR Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify - it once in the BACKUP_PATH environment variable. In this case, + it once in the BACKUP_DIR environment variable. In this case, you do not need to use this option each time on the command line. @@ -5679,7 +5679,7 @@ pg_probackup catchup -b catchup_mode lazily, when the first log message is written. - Default: $BACKUP_PATH/log/ + Default: $BACKUP_DIR/log/ @@ -5762,7 +5762,7 @@ pg_probackup catchup -b catchup_mode reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in - $BACKUP_PATH/log/log_rotation. The zero value disables + $BACKUP_DIR/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). diff --git a/po/ru.po b/po/ru.po index 1263675c2..30f50f797 100644 --- a/po/ru.po +++ b/po/ru.po @@ -811,7 +811,7 @@ msgstr "" #: src/help.c:360 src/help.c:521 src/help.c:588 src/help.c:635 src/help.c:715 #: src/help.c:761 src/help.c:833 #, c-format -msgid " directory for file logging (default: BACKUP_PATH/log)\n" +msgid " directory for file logging (default: BACKUP_DIR/log)\n" msgstr "" #: src/help.c:361 src/help.c:522 src/help.c:589 src/help.c:636 src/help.c:716 diff --git a/src/archive.c b/src/archive.c index 7d753c8b3..e97a1ade8 100644 --- a/src/archive.c +++ b/src/archive.c @@ -113,7 +113,7 @@ static parray *setup_push_filelist(const char *archive_status_dir, * set archive_command to * 'pg_probackup archive-push -B /home/anastasia/backup --wal-file-name %f', * to move backups into arclog_path. - * Where archlog_path is $BACKUP_PATH/wal/instance_name + * Where archlog_path is $BACKUP_DIR/wal/instance_name */ void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, @@ -1126,7 +1126,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha join_path_components(absolute_wal_file_path, current_dir, wal_file_path); /* full filepath to WAL file in archive directory. - * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ + * $BACKUP_DIR/wal/instance_name/000000010000000000000001 */ join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name); INSTR_TIME_SET_CURRENT(start_time); diff --git a/src/catalog.c b/src/catalog.c index b29090789..4da406af3 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1437,7 +1437,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, } /* - * Create backup directory in $BACKUP_PATH + * Create backup directory in $BACKUP_DIR * (with proposed backup->backup_id) * and initialize this directory. * If creation of directory fails, then diff --git a/src/help.c b/src/help.c index 0ccae938f..46acab886 100644 --- a/src/help.c +++ b/src/help.c @@ -372,7 +372,7 @@ help_backup(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -548,7 +548,7 @@ help_restore(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -621,7 +621,7 @@ help_validate(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -674,7 +674,7 @@ help_checkdb(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -760,7 +760,7 @@ help_delete(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -814,7 +814,7 @@ help_merge(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -890,7 +890,7 @@ help_set_config(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1002,7 +1002,7 @@ help_add_instance(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1072,7 +1072,7 @@ help_archive_push(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1131,7 +1131,7 @@ help_archive_get(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1221,7 +1221,7 @@ help_catchup(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 9b896d8bc..09817fdde 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -468,10 +468,10 @@ main(int argc, char *argv[]) if (backup_path == NULL) { /* - * If command line argument is not set, try to read BACKUP_PATH + * If command line argument is not set, try to read BACKUP_DIR * from environment variable */ - backup_path = getenv("BACKUP_PATH"); + backup_path = getenv("BACKUP_DIR"); } if (backup_path != NULL) @@ -498,7 +498,7 @@ main(int argc, char *argv[]) backup_subcmd != CATCHUP_CMD) elog(ERROR, "No backup catalog path specified.\n" - "Please specify it either using environment variable BACKUP_PATH or\n" + "Please specify it either using environment variable BACKUP_DIR or\n" "command line option --backup-path (-B)"); /* ===== catalogState (END) ======*/ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index bfb551ace..7b884c90b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -837,13 +837,13 @@ typedef struct InstanceState CatalogState *catalog_state; char instance_name[MAXPGPATH]; //previously global var instance_name - /* $BACKUP_PATH/backups/instance_name */ + /* $BACKUP_DIR/backups/instance_name */ char instance_backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ + /* $BACKUP_DIR/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ char instance_config_path[MAXPGPATH]; - - /* $BACKUP_PATH/backups/instance_name */ + + /* $BACKUP_DIR/backups/instance_name */ char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path /* TODO: Make it more specific */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 56d852537..1d1ff88d0 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -13,11 +13,11 @@ typedef struct CatalogState { - /* $BACKUP_PATH */ + /* $BACKUP_DIR */ char catalog_path[MAXPGPATH]; //previously global var backup_path - /* $BACKUP_PATH/backups */ + /* $BACKUP_DIR/backups */ char backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/wal */ + /* $BACKUP_DIR/wal */ char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path } CatalogState; diff --git a/tests/option_test.py b/tests/option_test.py index 9a829aaef..636c74327 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -34,7 +34,7 @@ def test_without_backup_path_3(self): except ProbackupException as e: self.assertIn( 'ERROR: No backup catalog path specified.\n' + \ - 'Please specify it either using environment variable BACKUP_PATH or\n' + \ + 'Please specify it either using environment variable BACKUP_DIR or\n' + \ 'command line option --backup-path (-B)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From ab05badc9d91b5aeec64c32d3d524747fbcd0a5b Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 13 Feb 2024 19:14:47 +0700 Subject: [PATCH 2090/2107] Change backup-path to backup-dir --- src/help.c | 124 +++++++++++++++--------------- tests/expected/option_help.out | 36 ++++----- tests/expected/option_help_ru.out | 36 ++++----- 3 files changed, 98 insertions(+), 98 deletions(-) diff --git a/src/help.c b/src/help.c index 46acab886..7eced19bc 100644 --- a/src/help.c +++ b/src/help.c @@ -87,9 +87,9 @@ help_pg_probackup(void) printf(_("\n %s version\n"), PROGRAM_NAME); - printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + printf(_("\n %s init -B backup-dir\n"), PROGRAM_NAME); - printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s set-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); @@ -114,17 +114,17 @@ help_pg_probackup(void) printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); - printf(_("\n %s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s set-backup -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--ttl=interval] [--expire-time=timestamp]\n")); printf(_(" [--note=text]\n")); printf(_(" [--help]\n")); - printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s show-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); printf(_(" [--default-units]\n")); printf(_(" [--help]\n")); - printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s backup -B backup-dir -b backup-mode --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); @@ -157,7 +157,7 @@ help_pg_probackup(void) printf(_(" [--help]\n")); - printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s restore -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -184,7 +184,7 @@ help_pg_probackup(void) printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); - printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n %s validate -B backup-dir [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -193,18 +193,18 @@ help_pg_probackup(void) printf(_(" [--skip-block-validation]\n")); printf(_(" [--help]\n")); - printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n %s checkdb [-B backup-dir] [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); printf(_(" [--heapallindexed] [--checkunique]\n")); printf(_(" [--help]\n")); - printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance-name [-i backup-id]]\n")); printf(_(" [--format=format] [--archive]\n")); printf(_(" [--no-color] [--help]\n")); - printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s delete -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); @@ -214,24 +214,24 @@ help_pg_probackup(void) printf(_(" [--dry-run] [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); - printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s merge -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--progress] [-j num-threads]\n")); printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); - printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n %s add-instance -B backup-dir -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); - printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n %s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [--help]\n")); - printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s archive-push -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); @@ -245,7 +245,7 @@ help_pg_probackup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); - printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s archive-get -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); @@ -295,14 +295,14 @@ help_internal(void) static void help_init(void) { - printf(_("\n%s init -B backup-path\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n\n")); + printf(_("\n%s init -B backup-dir\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n\n")); } static void help_backup(void) { - printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s backup -B backup-dir -b backup-mode --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); @@ -333,9 +333,9 @@ help_backup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); printf(_(" --stream stream the transaction log and include it in the backup\n")); @@ -442,7 +442,7 @@ help_backup(void) static void help_restore(void) { - printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s restore -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--progress] [--force] [--no-sync]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); @@ -469,8 +469,8 @@ help_restore(void) printf(_(" [--archive-host=hostname] [--archive-port=port]\n")); printf(_(" [--archive-user=username]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -i, --backup-id=backup-id backup to restore\n")); @@ -577,7 +577,7 @@ help_restore(void) static void help_validate(void) { - printf(_("\n%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s validate -B backup-dir [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -585,8 +585,8 @@ help_validate(void) printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to validate\n")); printf(_(" --progress show progress\n")); @@ -634,13 +634,13 @@ help_validate(void) static void help_checkdb(void) { - printf(_("\n%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s checkdb [-B backup-dir] [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); printf(_(" [--heapallindexed] [--checkunique]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --progress show progress\n")); @@ -695,12 +695,12 @@ help_checkdb(void) static void help_show(void) { - printf(_("\n%s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_("\n%s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance-name [-i backup-id]]\n")); printf(_(" [--format=format] [--archive]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name show info about specific instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); printf(_(" --archive show WAL archive information\n")); printf(_(" --format=format show format=PLAIN|JSON\n")); @@ -710,7 +710,7 @@ help_show(void) static void help_delete(void) { - printf(_("\n%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s delete -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n")); printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); @@ -718,8 +718,8 @@ help_delete(void) printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [--no-validate] [--no-sync]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to delete\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); @@ -773,7 +773,7 @@ help_delete(void) static void help_merge(void) { - printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s merge -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); @@ -786,8 +786,8 @@ help_merge(void) printf(_(" [--log-rotation-size=log-rotation-size]\n")); printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to merge\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); @@ -827,7 +827,7 @@ help_merge(void) static void help_set_backup(void) { - printf(_("\n%s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s set-backup -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id\n")); printf(_(" [--ttl=interval] [--expire-time=time] [--note=text]\n\n")); @@ -843,7 +843,7 @@ help_set_backup(void) static void help_set_config(void) { - printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s set-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path]\n")); printf(_(" [-E external-directories-paths]\n")); printf(_(" [--restore-command=cmdline]\n")); @@ -866,8 +866,8 @@ help_set_config(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); @@ -949,11 +949,11 @@ help_set_config(void) static void help_show_config(void) { - printf(_("\n%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s show-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" --format=format show format=PLAIN|JSON\n")); printf(_(" --default-units show memory and time values in default units\n\n")); } @@ -961,16 +961,16 @@ help_show_config(void) static void help_add_instance(void) { - printf(_("\n%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n%s add-instance -B backup-dir -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [-E external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" --instance=instance_name name of the new instance\n")); + printf(_(" --instance=instance-name name of the new instance\n")); printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); @@ -1014,16 +1014,16 @@ help_add_instance(void) static void help_del_instance(void) { - printf(_("\n%s del-instance -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s del-instance -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n\n")); } static void help_archive_push(void) { - printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s archive-push -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); @@ -1036,8 +1036,8 @@ help_archive_push(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the file to copy into WAL archive\n")); printf(_(" --wal-file-path=wal-file-path\n")); @@ -1095,7 +1095,7 @@ help_archive_push(void) static void help_archive_get(void) { - printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s archive-get -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); @@ -1104,8 +1104,8 @@ help_archive_get(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n")); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" relative destination path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 618a0d156..985ba7fec 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -5,9 +5,9 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup version - pg_probackup init -B backup-path + pg_probackup init -B backup-dir - pg_probackup set-config -B backup-path --instance=instance_name + pg_probackup set-config -B backup-dir --instance=instance-name [-D pgdata-path] [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] @@ -32,17 +32,17 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--archive-port=port] [--archive-user=username] [--help] - pg_probackup set-backup -B backup-path --instance=instance_name + pg_probackup set-backup -B backup-dir --instance=instance-name -i backup-id [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup show-config -B backup-path --instance=instance_name + pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] [--default-units] [--help] - pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name [-D pgdata-path] [-C] [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] @@ -74,7 +74,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup restore -B backup-path --instance=instance_name + pg_probackup restore -B backup-dir --instance=instance-name [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -101,7 +101,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--archive-port=port] [--archive-user=username] [--help] - pg_probackup validate -B backup-path [--instance=instance_name] + pg_probackup validate -B backup-dir [--instance=instance-name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -110,18 +110,18 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--skip-block-validation] [--help] - pg_probackup checkdb [-B backup-path] [--instance=instance_name] + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] [--heapallindexed] [--checkunique] [--help] - pg_probackup show -B backup-path - [--instance=instance_name [-i backup-id]] + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] [--format=format] [--archive] [--no-color] [--help] - pg_probackup delete -B backup-path --instance=instance_name + pg_probackup delete -B backup-dir --instance=instance-name [-j num-threads] [--progress] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -131,24 +131,24 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--dry-run] [--no-validate] [--no-sync] [--help] - pg_probackup merge -B backup-path --instance=instance_name + pg_probackup merge -B backup-dir --instance=instance-name -i backup-id [--progress] [-j num-threads] [--no-validate] [--no-sync] [--help] - pg_probackup add-instance -B backup-path -D pgdata-path - --instance=instance_name + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--help] - pg_probackup del-instance -B backup-path - --instance=instance_name + pg_probackup del-instance -B backup-dir + --instance=instance-name [--help] - pg_probackup archive-push -B backup-path --instance=instance_name + pg_probackup archive-push -B backup-dir --instance=instance-name --wal-file-name=wal-file-name [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] @@ -162,7 +162,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--help] - pg_probackup archive-get -B backup-path --instance=instance_name + pg_probackup archive-get -B backup-dir --instance=instance-name --wal-file-path=wal-file-path --wal-file-name=wal-file-name [-j num-threads] [--batch-size=batch_size] diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 005c74ebb..2fe516bdc 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -5,9 +5,9 @@ pg_probackup - утилита для управления резервным к pg_probackup version - pg_probackup init -B backup-path + pg_probackup init -B backup-dir - pg_probackup set-config -B backup-path --instance=instance_name + pg_probackup set-config -B backup-dir --instance=instance-name [-D pgdata-path] [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] @@ -32,17 +32,17 @@ pg_probackup - утилита для управления резервным к [--archive-port=port] [--archive-user=username] [--help] - pg_probackup set-backup -B backup-path --instance=instance_name + pg_probackup set-backup -B backup-dir --instance=instance-name -i backup-id [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup show-config -B backup-path --instance=instance_name + pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] [--default-units] [--help] - pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name [-D pgdata-path] [-C] [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] @@ -74,7 +74,7 @@ pg_probackup - утилита для управления резервным к [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup restore -B backup-path --instance=instance_name + pg_probackup restore -B backup-dir --instance=instance-name [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -101,7 +101,7 @@ pg_probackup - утилита для управления резервным к [--archive-port=port] [--archive-user=username] [--help] - pg_probackup validate -B backup-path [--instance=instance_name] + pg_probackup validate -B backup-dir [--instance=instance-name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -110,18 +110,18 @@ pg_probackup - утилита для управления резервным к [--skip-block-validation] [--help] - pg_probackup checkdb [-B backup-path] [--instance=instance_name] + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] [--heapallindexed] [--checkunique] [--help] - pg_probackup show -B backup-path - [--instance=instance_name [-i backup-id]] + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] [--format=format] [--archive] [--no-color] [--help] - pg_probackup delete -B backup-path --instance=instance_name + pg_probackup delete -B backup-dir --instance=instance-name [-j num-threads] [--progress] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -131,24 +131,24 @@ pg_probackup - утилита для управления резервным к [--dry-run] [--no-validate] [--no-sync] [--help] - pg_probackup merge -B backup-path --instance=instance_name + pg_probackup merge -B backup-dir --instance=instance-name -i backup-id [--progress] [-j num-threads] [--no-validate] [--no-sync] [--help] - pg_probackup add-instance -B backup-path -D pgdata-path - --instance=instance_name + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--help] - pg_probackup del-instance -B backup-path - --instance=instance_name + pg_probackup del-instance -B backup-dir + --instance=instance-name [--help] - pg_probackup archive-push -B backup-path --instance=instance_name + pg_probackup archive-push -B backup-dir --instance=instance-name --wal-file-name=wal-file-name [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] @@ -162,7 +162,7 @@ pg_probackup - утилита для управления резервным к [--ssh-options] [--help] - pg_probackup archive-get -B backup-path --instance=instance_name + pg_probackup archive-get -B backup-dir --instance=instance-name --wal-file-path=wal-file-path --wal-file-name=wal-file-name [-j num-threads] [--batch-size=batch_size] From dffc2b2fcd795cc3e8f4a0d99509ef38971ffd2f Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 4 Mar 2024 13:26:40 +0100 Subject: [PATCH 2091/2107] PBCKP-805 add unlock mutex in data.c --- src/data.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/data.c b/src/data.c index a287218ea..1a9616bae 100644 --- a/src/data.c +++ b/src/data.c @@ -2490,7 +2490,10 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->rel_path, file->hdr_off, z_len, file->hdr_crc); if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) + { + pthread_mutex_unlock(&(hdr_map->mutex)); elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); + } file->hdr_size = z_len; /* save the length of compressed headers */ hdr_map->offset += z_len; /* update current offset in map */ From 2fd0dda488520ca9507a9d60a4131a88993388cd Mon Sep 17 00:00:00 2001 From: Viktoria Shepard Date: Thu, 7 Mar 2024 23:28:44 +0300 Subject: [PATCH 2092/2107] Print remote host --- src/restore.c | 11 +++++++---- tests/requirements.txt | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/restore.c b/src/restore.c index 535faebfb..44e06f2f6 100644 --- a/src/restore.c +++ b/src/restore.c @@ -131,6 +131,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg bool cleanup_pgdata = false; bool backup_has_tblspc = true; /* backup contain tablespace */ XLogRecPtr shift_lsn = InvalidXLogRecPtr; + char timestamp[100]; if (instanceState == NULL) elog(ERROR, "Required parameter not specified: --instance"); @@ -687,6 +688,12 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg backup_id_of(dest_backup), dest_backup->server_version); + time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); + if (instance_config.remote.host) + elog(INFO, "Restoring the database from the backup starting at %s on %s", timestamp, instance_config.remote.host); + else + elog(INFO, "Restoring the database from the backup starting at %s", timestamp); + restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); @@ -720,7 +727,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, bool backup_has_tblspc) { int i; - char timestamp[100]; parray *pgdata_files = NULL; parray *dest_files = NULL; parray *external_dirs = NULL; @@ -743,9 +749,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time_t start_time, end_time; /* Preparations for actual restoring */ - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); - elog(INFO, "Restoring the database from backup at %s", timestamp); - dest_files = get_backup_filelist(dest_backup, true); /* Lock backup chain and make sanity checks */ diff --git a/tests/requirements.txt b/tests/requirements.txt index 62efb0e68..e2ac18bea 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,9 +5,9 @@ # git+https://github.com/postgrespro/testgres.git@ # 3. From a local directory # /path/to/local/directory/testgres -git+https://github.com/postgrespro/testgres.git@master#egg=testgres-pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 +git+https://github.com/postgrespro/testgres.git@archive-command-exec#egg=testgres-pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 allure-pytest deprecation pexpect -pytest +pytest==7.4.3 pytest-xdist From 3b741ee4f5f68390a89cc4e10968604cc38cbcef Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Mon, 20 Nov 2023 12:00:33 +0300 Subject: [PATCH 2093/2107] [PBCKP-804] Test_AssertionError_Python3.11 (no attribute 'errors') --- tests/helpers/ptrack_helpers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index da8ece15e..27d982856 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -423,8 +423,12 @@ def is_test_result_ok(test_case): result = test_case.defaultTestResult() # These two methods have no side effects test_case._feedErrorsToResult(result, test_case._outcome.errors) else: - # Python 3.11+ + # Python 3.11+ and pytest 5.3.5+ result = test_case._outcome.result + if not hasattr(result, 'errors'): + result.errors = [] + if not hasattr(result, 'failures'): + result.failures = [] else: # Python 2.7, 3.0-3.3 result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) @@ -2252,4 +2256,4 @@ def __init__(self, is_datafile: bool): self.is_datafile = is_datafile class ContentDir(object): - __slots__ = ('mode') \ No newline at end of file + __slots__ = ('mode') From e39a31e369ab5d6a1b6fe99e6235fa6bd647abe4 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Tue, 27 Feb 2024 21:24:13 +0300 Subject: [PATCH 2094/2107] PBCKP-964 fix bug with incremental restore of relation with more than one segment (always wrong checksum on non-zero segment) + add tests for it --- tests/incr_restore_test.py | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index eea0e313b..6a2164098 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -2510,3 +2510,123 @@ def test_incr_restore_issue_313(self): backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) node.slow_start() self.compare_pgdata(pgdata, self.pgdata_content(node.data_dir)) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_checksum(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=checksum", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_lsn(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) From 63c281066d8926aa9096ecc92520b8516a806524 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 14 Mar 2024 17:53:31 +0300 Subject: [PATCH 2095/2107] fix restoring highly compressed WAL gzip surprisingly emits a lot of zeroes when compression ratio is high. It triggered branch where FIO_PAGE_ZERO is emitted in agent but not handled in fio_sned_file_gz properly. Fix it. --- src/utils/file.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/utils/file.c b/src/utils/file.c index d39d3e320..fa08939f5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2537,11 +2537,22 @@ fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) exit_code = hdr.arg; goto cleanup; } - else if (hdr.cop == FIO_PAGE) + else if (hdr.cop == FIO_PAGE || hdr.cop == FIO_PAGE_ZERO) { int rc; - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + unsigned size; + if (hdr.cop == FIO_PAGE) + { + Assert(hdr.size <= CHUNK_SIZE); + size = hdr.size; + IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + } + else + { + Assert(hdr.arg <= CHUNK_SIZE); + size = hdr.arg; + memset(in_buf, 0, hdr.arg); + } /* We have received a chunk of compressed data, lets decompress it */ if (strm == NULL) @@ -2552,7 +2563,7 @@ fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) /* The fields next_in, avail_in initialized before init */ strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; + strm->avail_in = size; rc = inflateInit2(strm, 15 + 16); @@ -2569,7 +2580,7 @@ fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) else { strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; + strm->avail_in = size; } strm->next_out = (Bytef *)out_buf; /* output buffer */ From e9b8fcb1b25dd3a639e718444ce57770582bc259 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 23 May 2023 08:55:59 +0700 Subject: [PATCH 2096/2107] Remove the version test --- tests/expected/option_version.out | 1 - tests/option_test.py | 9 --------- 2 files changed, 10 deletions(-) delete mode 100644 tests/expected/option_version.out diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out deleted file mode 100644 index 0d50cb268..000000000 --- a/tests/expected/option_version.out +++ /dev/null @@ -1 +0,0 @@ -pg_probackup 2.5.12 diff --git a/tests/option_test.py b/tests/option_test.py index 636c74327..b74b9714f 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -15,15 +15,6 @@ def test_help_1(self): help_out.read().decode("utf-8") ) - # @unittest.skip("skip") - def test_version_2(self): - """help options""" - with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: - self.assertIn( - version_out.read().decode("utf-8").strip(), - self.run_pb(["--version"]) - ) - # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" From 91cddad61d623b869157abf816c0264dabb6b3df Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 5 Apr 2024 12:44:29 +0700 Subject: [PATCH 2097/2107] Up version --- src/pg_probackup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7b884c90b..186666a9b 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -356,7 +356,7 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.13" +#define PROGRAM_VERSION "2.5.14" /* update when remote agent API or behaviour changes */ #define AGENT_PROTOCOL_VERSION 20509 From da79091ace70cc0cad08002ff0276162bfbbb66c Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 12 Apr 2024 07:23:45 +0200 Subject: [PATCH 2098/2107] Fix test_unfinished_merge for REL_2_5 --- tests/merge_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/merge_test.py b/tests/merge_test.py index a9bc6fe68..1d40af7f7 100644 --- a/tests/merge_test.py +++ b/tests/merge_test.py @@ -2768,7 +2768,9 @@ def test_unfinished_merge(self): print(self.show_pb(backup_dir, node_name, as_json=False, as_text=True)) - for expected, real in zip(states, self.show_pb(backup_dir, node_name), strict=True): + backup_infos = self.show_pb(backup_dir, node_name) + self.assertEqual(len(backup_infos), len(states)) + for expected, real in zip(states, backup_infos): self.assertEqual(expected, real['status']) with self.assertRaisesRegex(ProbackupException, From 79d2b9d21cf6db670ec128994107f739dfdb9fd3 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Thu, 18 Apr 2024 12:54:28 +0300 Subject: [PATCH 2099/2107] Name the backup by its ID --- src/restore.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/restore.c b/src/restore.c index 44e06f2f6..f9310dcee 100644 --- a/src/restore.c +++ b/src/restore.c @@ -131,7 +131,6 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg bool cleanup_pgdata = false; bool backup_has_tblspc = true; /* backup contain tablespace */ XLogRecPtr shift_lsn = InvalidXLogRecPtr; - char timestamp[100]; if (instanceState == NULL) elog(ERROR, "Required parameter not specified: --instance"); @@ -688,11 +687,10 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg backup_id_of(dest_backup), dest_backup->server_version); - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); if (instance_config.remote.host) - elog(INFO, "Restoring the database from the backup starting at %s on %s", timestamp, instance_config.remote.host); + elog(INFO, "Restoring the database from backup %s on %s", backup_id_of(dest_backup), instance_config.remote.host); else - elog(INFO, "Restoring the database from the backup starting at %s", timestamp); + elog(INFO, "Restoring the database from backup %s", backup_id_of(dest_backup)); restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); From d29b005d23f04c5549b4f4bfd2ba60a6a53e6e32 Mon Sep 17 00:00:00 2001 From: oleg gurev Date: Mon, 6 May 2024 04:39:06 +0300 Subject: [PATCH 2100/2107] Rename default-units to no-scale-units --- doc/pgprobackup.xml | 13 +++++++++++++ src/configure.c | 10 +++++----- src/help.c | 4 ++-- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 2 +- src/utils/configuration.c | 4 ++-- src/utils/configuration.h | 2 +- tests/expected/option_help.out | 2 +- tests/expected/option_help_ru.out | 2 +- tests/option_test.py | 10 +++++----- 10 files changed, 34 insertions(+), 21 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 49e74e626..a73041f31 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -4267,6 +4267,7 @@ pg_probackup set-backup -B backup_dir --instance show-config pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] +[--no-scale-units] [logging_options] Displays the contents of the pg_probackup.conf configuration @@ -4277,6 +4278,18 @@ pg_probackup show-config -B backup_dir --instance JSON format. By default, configuration settings are shown as plain text. + + You can also specify the + option to display time and memory configuration settings in their base (unscaled) units. + Otherwise, the values are scaled to larger units for optimal display. + For example, if archive-timeout is 300, then + 5min is displayed, but if archive-timeout + is 301, then 301s is displayed. + Also, if the option is specified, configuration + settings are displayed without units and for the JSON format, + numeric and boolean values are not enclosed in quotes. This facilitates parsing + the output. + To edit pg_probackup.conf, use the command. diff --git a/src/configure.c b/src/configure.c index 4f6774d55..964548343 100644 --- a/src/configure.c +++ b/src/configure.c @@ -269,7 +269,7 @@ static const char *current_group = NULL; * Show configure options including default values. */ void -do_show_config(bool show_default_units) +do_show_config(bool show_base_units) { int i; @@ -277,13 +277,13 @@ do_show_config(bool show_default_units) for (i = 0; instance_options[i].type; i++) { - if (show_default_units && strchr("bBiIuU", instance_options[i].type) && instance_options[i].get_value == *option_get_value) - instance_options[i].flags |= GET_VAL_IN_DEFAULT_UNITS; /* Set flag */ + if (show_base_units && strchr("bBiIuU", instance_options[i].type) && instance_options[i].get_value == *option_get_value) + instance_options[i].flags |= GET_VAL_IN_BASE_UNITS; /* Set flag */ if (show_format == SHOW_PLAIN) show_configure_plain(&instance_options[i]); else show_configure_json(&instance_options[i]); - instance_options[i].flags &= ~(GET_VAL_IN_DEFAULT_UNITS); /* Reset flag. It was resetted in option_get_value(). Probably this reset isn't needed */ + instance_options[i].flags &= ~(GET_VAL_IN_BASE_UNITS); /* Reset flag. It was resetted in option_get_value(). Probably this reset isn't needed */ } show_configure_end(); @@ -804,6 +804,6 @@ show_configure_json(ConfigOption *opt) return; json_add_value(&show_buf, opt->lname, value, json_level, - !(opt->flags & GET_VAL_IN_DEFAULT_UNITS)); + !(opt->flags & GET_VAL_IN_BASE_UNITS)); pfree(value); } diff --git a/src/help.c b/src/help.c index 7eced19bc..48cc1f524 100644 --- a/src/help.c +++ b/src/help.c @@ -121,7 +121,7 @@ help_pg_probackup(void) printf(_("\n %s show-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); - printf(_(" [--default-units]\n")); + printf(_(" [--no-scale-units]\n")); printf(_(" [--help]\n")); printf(_("\n %s backup -B backup-dir -b backup-mode --instance=instance-name\n"), PROGRAM_NAME); @@ -955,7 +955,7 @@ help_show_config(void) printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); printf(_(" --instance=instance-name name of the instance\n")); printf(_(" --format=format show format=PLAIN|JSON\n")); - printf(_(" --default-units show memory and time values in default units\n\n")); + printf(_(" --no-scale-units show memory and time values in default units\n\n")); } static void diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 09817fdde..e50b05995 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -164,7 +164,7 @@ bool no_validate_wal = false; /* show options */ ShowFormat show_format = SHOW_PLAIN; bool show_archive = false; -static bool show_default_units = false; +static bool show_base_units = false; /* set-backup options */ int64 ttl = -1; @@ -277,7 +277,7 @@ static ConfigOption cmd_options[] = { 'f', 165, "format", opt_show_format, SOURCE_CMD_STRICT }, { 'b', 166, "archive", &show_archive, SOURCE_CMD_STRICT }, /* show-config options */ - { 'b', 167, "default-units", &show_default_units,SOURCE_CMD_STRICT }, + { 'b', 167, "no-scale-units", &show_base_units,SOURCE_CMD_STRICT }, /* set-backup options */ { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, @@ -1052,7 +1052,7 @@ main(int argc, char *argv[]) do_merge(instanceState, current.backup_id, no_validate, no_sync); break; case SHOW_CONFIG_CMD: - do_show_config(show_default_units); + do_show_config(show_base_units); break; case SET_CONFIG_CMD: do_set_config(instanceState, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 186666a9b..1f4780f58 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -939,7 +939,7 @@ extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instanc char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ -extern void do_show_config(bool show_default_units); +extern void do_show_config(bool show_base_units); extern void do_set_config(InstanceState *instanceState, bool missing_ok); extern void init_config(InstanceConfig *config, const char *instance_name); extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 6b2382996..f049aa1be 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -678,7 +678,7 @@ config_set_opt(ConfigOption options[], void *var, OptionSource source) /* * Return value of the function in the string representation. Result is * allocated string. - * We can set GET_VAL_IN_DEFAULT_UNITS flag in opt->flags + * We can set GET_VAL_IN_BASE_UNITS flag in opt->flags * before call option_get_value() to get option value in default units */ char * @@ -694,7 +694,7 @@ option_get_value(ConfigOption *opt) */ if (opt->flags & OPTION_UNIT) { - if (opt->flags & GET_VAL_IN_DEFAULT_UNITS){ + if (opt->flags & GET_VAL_IN_BASE_UNITS){ if (opt->type == 'i') value = *((int32 *) opt->var); else if (opt->type == 'I') diff --git a/src/utils/configuration.h b/src/utils/configuration.h index f3bda65de..59da29bd5 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -100,7 +100,7 @@ struct ConfigOption #define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) -#define GET_VAL_IN_DEFAULT_UNITS 0x80000000 /* bitflag to get memory and time values in default units*/ +#define GET_VAL_IN_BASE_UNITS 0x80000000 /* bitflag to get memory and time values in default units*/ extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 985ba7fec..f0c77ae16 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -39,7 +39,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] - [--default-units] + [--no-scale-units] [--help] pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 2fe516bdc..bd6d76970 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -39,7 +39,7 @@ pg_probackup - утилита для управления резервным к pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] - [--default-units] + [--no-scale-units] [--help] pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name diff --git a/tests/option_test.py b/tests/option_test.py index b74b9714f..e97da1ef7 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -222,21 +222,21 @@ def test_help_6(self): 'You need configure PostgreSQL with --enabled-nls option for this test') # @unittest.skip("skip") - def test_options_default_units(self): - """check --default-units option""" + def test_options_no_scale_units(self): + """check --no-scale-units option""" backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) - # check that --default-units option works correctly + # check that --no-scale-units option works correctly output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node"]) self.assertIn(container=output, member="archive-timeout = 5min") - output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--default-units"]) + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--no-scale-units"]) self.assertIn(container=output, member="archive-timeout = 300") self.assertNotIn(container=output, member="archive-timeout = 300s") # check that we have now quotes ("") in json output - output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--default-units", "--format=json"]) + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--no-scale-units", "--format=json"]) self.assertIn(container=output, member='"archive-timeout": 300,') self.assertIn(container=output, member='"retention-redundancy": 0,') self.assertNotIn(container=output, member='"archive-timeout": "300",') From 2b74971f710a799e49b10ab87e0901a6ec561d90 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Sat, 27 Apr 2024 15:54:32 +0300 Subject: [PATCH 2101/2107] [PBCKP-818] Add id's for programlisting section - to use with automatet listings update --- doc/pgprobackup.xml | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index a73041f31..ac9892c65 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -529,14 +529,14 @@ doc/src/sgml/pgprobackup.sgml Initialize the backup catalog: - + backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized Add a backup instance called mydb to the backup catalog: - + backup_user@backup_host:~$ pg_probackup-16 add-instance \ -B /mnt/backups \ -D /var/lib/postgresql/16/main \ @@ -548,7 +548,7 @@ INFO: Instance 'mydb' successfully initialized Make a FULL backup: - + backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b FULL \ @@ -580,7 +580,7 @@ INFO: Backup S6OBFN completed List the backups of the instance: - + backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb ================================================================================================================================ Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -590,7 +590,7 @@ backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb Make an incremental backup in the DELTA mode: - + backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b delta \ @@ -625,7 +625,7 @@ INFO: Backup S6OBLG completed Add or modify some parameters in the pg_probackup configuration file, so that you do not have to specify them each time on the command line: - + backup_user@backup_host:~$ pg_probackup-16 set-config \ -B /mnt/backups \ --instance=mydb \ @@ -637,7 +637,7 @@ backup_user@backup_host:~$ pg_probackup-16 set-config \ Check the configuration of the instance: - + backup_user@backup_host:~$ pg_probackup-16 show-config -B /mnt/backups --instance=mydb # Backup instance information pgdata = /var/lib/postgresql/16/main @@ -678,7 +678,7 @@ remote-user = postgres Make another incremental backup in the DELTA mode, omitting the parameters stored in the configuration file earlier: - + backup_user@backup_host:~$ pg_probackup-16 backup -B /mnt/backups --instance=mydb -b delta --stream INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBQO, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 INFO: Database backup start @@ -704,7 +704,7 @@ INFO: Backup S6OBQO completed List the backups of the instance again: - + backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb ================================================================================================================================== Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -716,7 +716,7 @@ backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb Restore the data from the latest available backup to an arbitrary location: - + backup_user@backup_host:~$ pg_probackup-16 restore -B /mnt/backups -D /var/lib/postgresql/16/staging --instance=mydb INFO: Validating parents for backup S6OBQO INFO: Validating backup S6OBFN @@ -2110,7 +2110,7 @@ pg_probackup validate -B backup_dir --instance PT8XFX backup ID up to the specified timestamp, run this command: - + pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time="2017-05-18 14:18:11+03" @@ -2305,7 +2305,7 @@ pg_probackup restore -B backup_dir --instance - + ============================================================================================================================================= Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ============================================================================================================================================= @@ -2788,7 +2788,7 @@ pg_probackup show -B backup_dir pg_probackup displays the list of all the available backups. For example: - + BACKUP INSTANCE 'node' ====================================================================================================================================== Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -2952,7 +2952,7 @@ pg_probackup show -B backup_dir --instance The sample output is as follows: - + #Configuration backup-mode = FULL stream = false @@ -3096,7 +3096,7 @@ pg_probackup show -B backup_dir --instance The sample output is as follows: - + [ { "instance": "node", @@ -3145,7 +3145,7 @@ pg_probackup show -B backup_dir [--instance pg_probackup displays the list of all the available WAL files grouped by timelines. For example: - + ARCHIVE INSTANCE 'node' =================================================================================================================================== TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status @@ -3243,7 +3243,7 @@ pg_probackup show -B backup_dir [--instance The sample output is as follows: - + [ { "instance": "replica", @@ -3599,7 +3599,7 @@ pg_probackup delete -B backup_dir --instance 7, and you have the following backups available on April 10, 2019: - + BACKUP INSTANCE 'node' =================================================================================================================================== Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -3613,17 +3613,17 @@ BACKUP INSTANCE 'node' node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK - Even though P7XDHB and P7XDHU backups are outside the + Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the - succeeding incremental backups P7XDJA and P7XDQV that are + succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the command with the - flag, only the P7XDFT full + flag, only the P7XDFT full backup will be removed. - With the option, the P7XDJA - backup is merged with the underlying P7XDHU and P7XDHB backups + With the option, the P7XDJA + backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: @@ -3631,7 +3631,7 @@ BACKUP INSTANCE 'node' pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired pg_probackup show -B backup_dir - + BACKUP INSTANCE 'node' ================================================================================================================================== Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -3688,7 +3688,7 @@ pg_probackup show -B backup_dir --instance If the backup is pinned, it has the expire-time attribute that displays its expiration time: - + ... recovery-time = '2017-05-16 12:57:31' expire-time = '2020-01-01 00:00:00+03' @@ -3766,7 +3766,7 @@ pg_probackup set-backup -B backup_dir --instance pg_probackup show -B backup_dir --instance node - + BACKUP INSTANCE 'node' ==================================================================================================================================== Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status @@ -3786,7 +3786,7 @@ BACKUP INSTANCE 'node' pg_probackup show -B backup_dir --instance node --archive - + ARCHIVE INSTANCE 'node' =============================================================================================================================== TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status @@ -3800,7 +3800,7 @@ ARCHIVE INSTANCE 'node' pg_probackup delete -B backup_dir --instance node --delete-wal - + ARCHIVE INSTANCE 'node' =============================================================================================================================== TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status @@ -3815,7 +3815,7 @@ ARCHIVE INSTANCE 'node' pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 - + ARCHIVE INSTANCE 'node' ================================================================================================================================ TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status @@ -3829,7 +3829,7 @@ ARCHIVE INSTANCE 'node' pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal - + ARCHIVE INSTANCE 'node' =============================================================================================================================== TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status From 26f1e9c052d50b8ada799dc5dd02355893d54056 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Thu, 2 May 2024 11:02:27 +0300 Subject: [PATCH 2102/2107] [PBCKP-818] Doc. Replace --instance to --instance= in the xml - Replace --instance node to --instance=node in the xml --- doc/pgprobackup.xml | 146 ++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index ac9892c65..272a0d043 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1080,7 +1080,7 @@ pg_probackup init -B backup_dir To add a new backup instance, run the following command: -pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] +pg_probackup add-instance -B backup_dir -D data_dir --instance=instance_name [remote_options] Where: @@ -1355,7 +1355,7 @@ ALTER ROLE backup WITH REPLICATION; parameter, as follows: -archive_command = '"install_dir/pg_probackup" archive-push -B "backup_dir" --instance instance_name --wal-file-name=%f [remote_options]' +archive_command = '"install_dir/pg_probackup" archive-push -B "backup_dir" --instance=instance_name --wal-file-name=%f [remote_options]' @@ -1804,7 +1804,7 @@ CREATE EXTENSION ptrack; To create a backup, run the following command: -pg_probackup backup -B backup_dir --instance instance_name -b backup_mode +pg_probackup backup -B backup_dir --instance=instance_name -b backup_mode Where backup_mode can take one of the @@ -1830,7 +1830,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL +pg_probackup backup -B backup_dir --instance=instance_name -b FULL ARCHIVE backups rely on @@ -1860,7 +1860,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --stream --temp-slot The optional flag ensures that @@ -1953,7 +1953,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 Similarly, to include C:\dir1 and @@ -1961,7 +1961,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 pg_probackup recursively copies the contents @@ -1989,7 +1989,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] [connection_options] +pg_probackup checkdb [-B backup_dir [--instance=instance_name]] [-D data_dir] [connection_options] @@ -2087,7 +2087,7 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ this command: -pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 +pg_probackup validate -B backup_dir --instance=instance_name --recovery-target-xid=4242 If validation completes successfully, pg_probackup displays the @@ -2111,7 +2111,7 @@ pg_probackup validate -B backup_dir --instance -pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time="2017-05-18 14:18:11+03" +pg_probackup validate -B backup_dir --instance=instance_name -i PT8XFX --recovery-target-time="2017-05-18 14:18:11+03" If you specify the backup_id of an incremental backup, @@ -2129,7 +2129,7 @@ pg_probackup validate -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name -i backup_id +pg_probackup restore -B backup_dir --instance=instance_name -i backup_id Where: @@ -2213,7 +2213,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir +pg_probackup restore -B backup_dir --instance=instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir @@ -2245,7 +2245,7 @@ pg_probackup restore -B backup_dir --instance command with the following options: -pg_probackup restore -B backup_dir --instance instance_name -D data_dir -I incremental_mode +pg_probackup restore -B backup_dir --instance=instance_name -D data_dir -I incremental_mode Where incremental_mode can take one of the @@ -2314,7 +2314,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name +pg_probackup restore -B backup_dir --instance=instance_name --db-include=database_name The option can be specified @@ -2360,14 +2360,14 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 +pg_probackup restore -B backup_dir --instance=instance_name --db-include=db1 --db-include=db2 To exclude one or more databases from restore, use the option: -pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name +pg_probackup restore -B backup_dir --instance=instance_name --db-exclude=database_name The option can be specified @@ -2376,7 +2376,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --db-exclude=db1 --db-exclude=db2 +pg_probackup restore -B backup_dir --instance=instance_name --db-exclude=db1 --db-exclude=db2 Partial restore relies on lax behavior of PostgreSQL recovery @@ -2438,7 +2438,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time="2017-05-18 14:18:11+03" +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-time="2017-05-18 14:18:11+03" @@ -2447,7 +2447,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-xid option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-xid=687 @@ -2456,7 +2456,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-lsn option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-lsn=16/B374D848 @@ -2465,7 +2465,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-name option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name="before_app_upgrade" +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-name="before_app_upgrade" @@ -2475,7 +2475,7 @@ pg_probackup restore -B backup_dir --instance latest value: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target="latest" +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target="latest" @@ -2485,7 +2485,7 @@ pg_probackup restore -B backup_dir --instance immediate value: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target='immediate' @@ -2569,7 +2569,7 @@ pg_probackup restore -B backup_dir --instance 2302, run: -pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 To restore the latest available backup on a remote system with host address @@ -2577,7 +2577,7 @@ pg_probackup backup -B backup_dir --instance 2302, run: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 Restoring an ARCHIVE backup or performing PITR in the remote mode @@ -2604,20 +2604,20 @@ pg_probackup restore -B backup_dir --instance 2303, run: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup Provided arguments will be used to construct the restore_command: -restore_command = '"install_dir/pg_probackup" archive-get -B "backup_dir" --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +restore_command = '"install_dir/pg_probackup" archive-get -B "backup_dir" --instance=instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' Alternatively, you can use the option to provide the entire restore_command: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='"install_dir/pg_probackup" archive-get -B "backup_dir" --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='"install_dir/pg_probackup" archive-get -B "backup_dir" --instance=instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' @@ -2646,7 +2646,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL -j 4 @@ -2707,14 +2707,14 @@ pg_probackup backup -B backup_dir --instance set-config command: -pg_probackup set-config -B backup_dir --instance instance_name +pg_probackup set-config -B backup_dir --instance=instance_name [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] To view the current settings, run the following command: -pg_probackup show-config -B backup_dir --instance instance_name +pg_probackup show-config -B backup_dir --instance=instance_name You can override the settings defined in pg_probackup.conf when @@ -2947,7 +2947,7 @@ BACKUP INSTANCE 'node' show command with the backup ID: -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id The sample output is as follows: @@ -3091,7 +3091,7 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod in the JSON format: -pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id +pg_probackup show -B backup_dir --instance=instance_name --format=json -i backup_id The sample output is as follows: @@ -3139,7 +3139,7 @@ pg_probackup show -B backup_dir --instance -pg_probackup show -B backup_dir [--instance instance_name] --archive +pg_probackup show -B backup_dir [--instance=instance_name] --archive pg_probackup displays the list of all the available WAL files @@ -3238,7 +3238,7 @@ ARCHIVE INSTANCE 'node' format, run the command: -pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json +pg_probackup show -B backup_dir [--instance=instance_name] --archive --format=json The sample output is as follows: @@ -3540,7 +3540,7 @@ pg_probackup show -B backup_dir [--instance -pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 +pg_probackup set-config -B backup_dir --instance=instance_name --retention-redundancy=2 --retention-window=7 @@ -3558,7 +3558,7 @@ pg_probackup set-config -B backup_dir --instance --delete-expired flag: -pg_probackup delete -B backup_dir --instance instance_name --delete-expired +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired If you would like to also remove the WAL files that are no @@ -3566,7 +3566,7 @@ pg_probackup delete -B backup_dir --instance --delete-wal flag: -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --delete-wal @@ -3577,7 +3577,7 @@ pg_probackup delete -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --retention-window=7 --retention-redundancy=2 Since incremental backups require that their parent full @@ -3628,7 +3628,7 @@ BACKUP INSTANCE 'node' expired backups anymore: -pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired +pg_probackup delete -B backup_dir --instance=node --delete-expired --merge-expired pg_probackup show -B backup_dir @@ -3654,7 +3654,7 @@ BACKUP INSTANCE 'node' for arbitrary time. For example: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --ttl=30d This command sets the expiration time of the @@ -3666,7 +3666,7 @@ pg_probackup set-backup -B backup_dir --instance --expire-time option. For example: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time="2020-01-01 00:00:00+03" +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --expire-time="2020-01-01 00:00:00+03" Alternatively, you can use the and @@ -3675,14 +3675,14 @@ pg_probackup set-backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d -pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time="2020-01-01 00:00:00+03" +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --ttl=30d +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --expire-time="2020-01-01 00:00:00+03" To check if the backup is pinned, run the command: -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id @@ -3700,7 +3700,7 @@ data-bytes = 22288792 You can unpin the backup by setting the option to zero: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --ttl=0 @@ -3764,7 +3764,7 @@ pg_probackup set-backup -B backup_dir --instance : -pg_probackup show -B backup_dir --instance node +pg_probackup show -B backup_dir --instance=node BACKUP INSTANCE 'node' @@ -3784,7 +3784,7 @@ BACKUP INSTANCE 'node' flag: -pg_probackup show -B backup_dir --instance node --archive +pg_probackup show -B backup_dir --instance=node --archive ARCHIVE INSTANCE 'node' @@ -3798,7 +3798,7 @@ ARCHIVE INSTANCE 'node' achieve much, only one segment is removed: -pg_probackup delete -B backup_dir --instance node --delete-wal +pg_probackup delete -B backup_dir --instance=node --delete-wal ARCHIVE INSTANCE 'node' @@ -3813,7 +3813,7 @@ ARCHIVE INSTANCE 'node' option to 1: -pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 +pg_probackup delete -B backup_dir --instance=node --delete-wal --wal-depth=1 ARCHIVE INSTANCE 'node' @@ -3827,7 +3827,7 @@ ARCHIVE INSTANCE 'node' option with the command: -pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal +pg_probackup backup -B backup_dir --instance=node -b DELTA --wal-depth=1 --delete-wal ARCHIVE INSTANCE 'node' @@ -3848,7 +3848,7 @@ ARCHIVE INSTANCE 'node' recent incremental backup you would like to merge: -pg_probackup merge -B backup_dir --instance instance_name -i backup_id +pg_probackup merge -B backup_dir --instance=instance_name -i backup_id This command merges backups that belong to a common incremental backup @@ -3870,7 +3870,7 @@ pg_probackup merge -B backup_dir --instance -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id If the merge is still in progress, the backup status is @@ -3888,7 +3888,7 @@ pg_probackup show -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name -i backup_id +pg_probackup delete -B backup_dir --instance=instance_name -i backup_id This command will delete the backup with the specified @@ -3904,7 +3904,7 @@ pg_probackup delete -B backup_dir --instance --delete-wal flag: -pg_probackup delete -B backup_dir --instance instance_name --delete-wal +pg_probackup delete -B backup_dir --instance=instance_name --delete-wal To delete backups that are expired according to the current @@ -3912,7 +3912,7 @@ pg_probackup delete -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name --delete-expired +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired Expired backups cannot be removed while at least one @@ -3923,7 +3923,7 @@ pg_probackup delete -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --merge-expired In this case, pg_probackup searches for the oldest incremental @@ -3943,7 +3943,7 @@ pg_probackup delete -B backup_dir --instance --status: -pg_probackup delete -B backup_dir --instance instance_name --status=ERROR +pg_probackup delete -B backup_dir --instance=instance_name --status=ERROR @@ -4181,7 +4181,7 @@ pg_probackup init -B backup_dir [--help] add-instance -pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] +pg_probackup add-instance -B backup_dir -D data_dir --instance=instance_name [--help] Initializes a new backup instance inside the backup catalog @@ -4199,7 +4199,7 @@ pg_probackup add-instance -B backup_dir -D del-instance -pg_probackup del-instance -B backup_dir --instance instance_name [--help] +pg_probackup del-instance -B backup_dir --instance=instance_name [--help] Deletes all backups and WAL files associated with the @@ -4209,7 +4209,7 @@ pg_probackup del-instance -B backup_dir --instance set-config -pg_probackup set-config -B backup_dir --instance instance_name +pg_probackup set-config -B backup_dir --instance=instance_name [--help] [--pgdata=pgdata-path] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] @@ -4235,7 +4235,7 @@ pg_probackup set-config -B backup_dir --instance set-backup -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id {--ttl=ttl | --expire-time=time} [--note=backup_note] [--help] @@ -4266,7 +4266,7 @@ pg_probackup set-backup -B backup_dir --instance show-config -pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] +pg_probackup show-config -B backup_dir --instance=instance_name [--format=plain|json] [--no-scale-units] [logging_options] @@ -4299,7 +4299,7 @@ pg_probackup show-config -B backup_dir --instance show pg_probackup show -B backup_dir -[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] [--no-color] +[--help] [--instance=instance_name [-i backup_id | --archive]] [--format=plain|json] [--no-color] Shows the contents of the backup catalog. If @@ -4328,7 +4328,7 @@ pg_probackup show -B backup_dir backup -pg_probackup backup -B backup_dir -b backup_mode --instance instance_name +pg_probackup backup -B backup_dir -b backup_mode --instance=instance_name [--help] [-j num_threads] [--progress] [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] [--no-validate] [--skip-block-validation] @@ -4511,7 +4511,7 @@ pg_probackup backup -B backup_dir -b bac restore -pg_probackup restore -B backup_dir --instance instance_name +pg_probackup restore -B backup_dir --instance=instance_name [--help] [-D data_dir] [-i backup_id] [-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] @@ -4722,7 +4722,7 @@ pg_probackup restore -B backup_dir --instance checkdb pg_probackup checkdb -[-B backup_dir] [--instance instance_name] [-D data_dir] +[-B backup_dir] [--instance=instance_name] [-D data_dir] [--help] [-j num_threads] [--progress] [--amcheck [--skip-block-validation] [--checkunique] [--heapallindexed]] [connection_options] [logging_options] @@ -4812,7 +4812,7 @@ pg_probackup checkdb validate pg_probackup validate -B backup_dir -[--help] [--instance instance_name] [-i backup_id] +[--help] [--instance=instance_name] [-i backup_id] [-j num_threads] [--progress] [--skip-block-validation] [recovery_target_options] [logging_options] @@ -4840,7 +4840,7 @@ pg_probackup validate -B backup_dir merge -pg_probackup merge -B backup_dir --instance instance_name -i backup_id +pg_probackup merge -B backup_dir --instance=instance_name -i backup_id [--help] [-j num_threads] [--progress] [--no-validate] [--no-sync] [logging_options] @@ -4884,7 +4884,7 @@ pg_probackup merge -B backup_dir --instance delete -pg_probackup delete -B backup_dir --instance instance_name +pg_probackup delete -B backup_dir --instance=instance_name [--help] [-j num_threads] [--progress] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status=backup_status} @@ -4931,7 +4931,7 @@ pg_probackup delete -B backup_dir --instance archive-push -pg_probackup archive-push -B backup_dir --instance instance_name +pg_probackup archive-push -B backup_dir --instance=instance_name --wal-file-name=wal_file_name [--wal-file-path=wal_file_path] [--help] [--no-sync] [--compress] [--no-ready-rename] [--overwrite] [-j num_threads] [--batch-size=batch_size] @@ -4997,7 +4997,7 @@ pg_probackup archive-push -B backup_dir --instance archive-get -pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name +pg_probackup archive-get -B backup_dir --instance=instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [-j num_threads] [--batch-size=batch_size] [--prefetch-dir=prefetch_dir_path] [--no-validate-wal] [--help] [remote_options] [logging_options] From 69379ab7a66c15720f066cd770bbc0ced80c05b1 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Thu, 2 May 2024 11:38:22 +0300 Subject: [PATCH 2103/2107] [PBCKP-818] Documentation update script work result. --- doc/pgprobackup.xml | 866 ++++++++++++++++++++++---------------------- 1 file changed, 424 insertions(+), 442 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 272a0d043..f5b2a93eb 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2,7 +2,6 @@ doc/src/sgml/pgprobackup.sgml &project; documentation --> - pg_probackup @@ -530,105 +529,111 @@ doc/src/sgml/pgprobackup.sgml Initialize the backup catalog: -backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups +backup_user@backup_host:~$ pg_probackup init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized Add a backup instance called mydb to the backup catalog: -backup_user@backup_host:~$ pg_probackup-16 add-instance \ +backup_user@backup_host:~$ pg_probackup add-instance \ -B /mnt/backups \ - -D /var/lib/postgresql/16/main \ - --instance=mydb \ + -D /var/lib/pgpro/std-16/data \ + --instance=node \ --remote-host=postgres_host \ --remote-user=postgres -INFO: Instance 'mydb' successfully initialized +INFO: Instance 'node' successfully initialized Make a FULL backup: -backup_user@backup_host:~$ pg_probackup-16 backup \ +backup_user@backup_host:~$ pg_probackup backup \ -B /mnt/backups \ -b FULL \ - --instance=mydb \ + --instance=node \ --stream \ + --compress-algorithm=zlib \ --remote-host=postgres_host \ --remote-user=postgres \ -U backup \ -d backupdb -INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBFN, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN1Q, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() -INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBFN/database/pg_wal/000000010000000000000002 to be streamed -INFO: PGDATA size: 29MB -INFO: Current Start LSN: 0/2000060, TLI: 1 +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN1Q/database/pg_wal/000000010000000000000008 to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/8000028, TLI: 1 INFO: Start transferring data files -INFO: Data files are transferred, time elapsed: 0 +INFO: Data files are transferred, time elapsed: 1s INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed -INFO: stop_lsn: 0/2003CB0 +INFO: stop_lsn: 0/800BBD0 INFO: Getting the Recovery Time from WAL INFO: Syncing backup files to disk -INFO: Backup files are synced, time elapsed: 0 -INFO: Validating backup S6OBFN -INFO: Backup S6OBFN data files are valid -INFO: Backup S6OBFN resident size: 45MB -INFO: Backup S6OBFN completed +INFO: Backup files are synced, time elapsed: 1s +INFO: Validating backup SCUN1Q +INFO: Backup SCUN1Q data files are valid +INFO: Backup SCUN1Q resident size: 56MB +INFO: Backup SCUN1Q completed List the backups of the instance: -backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb +backup_user@backup_host:~$ pg_probackup show \ + -B /mnt/backups \ + --instance=node ================================================================================================================================ Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ================================================================================================================================ - mydb 16 S6OBFN 2024-01-03 06:59:49+00 FULL STREAM 1/0 10s 29MB 16MB 1.00 0/2000060 0/2003CB0 OK + node 16 SCUN1Q 2024-05-02 11:17:53+03 FULL STREAM 1/0 12s 40MB 16MB 2.42 0/8000028 0/800BBD0 OK Make an incremental backup in the DELTA mode: -backup_user@backup_host:~$ pg_probackup-16 backup \ - -B /mnt/backups \ - -b delta \ - --instance=mydb \ - --stream \ - --remote-host=postgres_host \ - --remote-user=postgres \ - -U backup \ - -d backupdb -INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBLG, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +backup_user@backup_host:~$ pg_probackup backup \ + -B /mnt/backups \ + -b DELTA \ + --instance=node \ + --stream \ + --compress-algorithm=zlib \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb +INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN22, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() -INFO: Parent backup: S6OBFN -INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBLG/database/pg_wal/000000010000000000000004 to be streamed -INFO: PGDATA size: 29MB -INFO: Current Start LSN: 0/4000028, TLI: 1 -INFO: Parent Start LSN: 0/2000060, TLI: 1 +INFO: Parent backup: SCUN1Q +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN22/database/pg_wal/000000010000000000000009 to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/9000028, TLI: 1 +INFO: Parent Start LSN: 0/8000028, TLI: 1 INFO: Start transferring data files INFO: Data files are transferred, time elapsed: 1s INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed -INFO: stop_lsn: 0/4000168 +INFO: stop_lsn: 0/9000168 INFO: Getting the Recovery Time from WAL INFO: Syncing backup files to disk -INFO: Backup files are synced, time elapsed: 0 -INFO: Validating backup S6OBLG -INFO: Backup S6OBLG data files are valid -INFO: Backup S6OBLG resident size: 32MB -INFO: Backup S6OBLG completed +INFO: Backup files are synced, time elapsed: 1s +INFO: Validating backup SCUN22 +INFO: Backup SCUN22 data files are valid +INFO: Backup SCUN22 resident size: 34MB +INFO: Backup SCUN22 completed Add or modify some parameters in the pg_probackup configuration file, so that you do not have to specify them each time on the command line: -backup_user@backup_host:~$ pg_probackup-16 set-config \ +backup_user@backup_host:~$ pg_probackup set-config \ -B /mnt/backups \ - --instance=mydb \ + --instance=node \ --remote-host=postgres_host \ --remote-user=postgres \ -U backup \ @@ -638,10 +643,12 @@ backup_user@backup_host:~$ pg_probackup-16 set-config \ Check the configuration of the instance: -backup_user@backup_host:~$ pg_probackup-16 show-config -B /mnt/backups --instance=mydb +backup_user@backup_host:~$ pg_probackup show-config \ + -B /mnt/backups \ + --instance=node # Backup instance information -pgdata = /var/lib/postgresql/16/main -system-identifier = 7319761899046784808 +pgdata = /var/lib/pgpro/std-16/data +system-identifier = 7364313570668255886 xlog-seg-size = 16777216 # Connection parameters pgdatabase = backupdb @@ -679,61 +686,72 @@ remote-user = postgres Make another incremental backup in the DELTA mode, omitting the parameters stored in the configuration file earlier: -backup_user@backup_host:~$ pg_probackup-16 backup -B /mnt/backups --instance=mydb -b delta --stream -INFO: Backup start, pg_probackup version: 2.5.13, instance: mydb, backup ID: S6OBQO, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 +backup_user@backup_host:~$ pg_probackup backup \ + -B /mnt/backups \ + -b DELTA \ + --instance=node \ + --stream \ + --compress-algorithm=zlib +INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN2C, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() -INFO: Parent backup: S6OBLG -INFO: Wait for WAL segment /mnt/backups/backups/mydb/S6OBQO/database/pg_wal/000000010000000000000006 to be streamed -INFO: PGDATA size: 29MB -INFO: Current Start LSN: 0/6000028, TLI: 1 -INFO: Parent Start LSN: 0/4000028, TLI: 1 +INFO: Parent backup: SCUN22 +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN2C/database/pg_wal/00000001000000000000000B to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/B000028, TLI: 1 +INFO: Parent Start LSN: 0/9000028, TLI: 1 INFO: Start transferring data files INFO: Data files are transferred, time elapsed: 0 INFO: wait for pg_stop_backup() INFO: pg_stop backup() successfully executed -INFO: stop_lsn: 0/6000168 +INFO: stop_lsn: 0/B000168 INFO: Getting the Recovery Time from WAL INFO: Syncing backup files to disk INFO: Backup files are synced, time elapsed: 0 -INFO: Validating backup S6OBQO -INFO: Backup S6OBQO data files are valid -INFO: Backup S6OBQO resident size: 32MB -INFO: Backup S6OBQO completed +INFO: Validating backup SCUN2C +INFO: Backup SCUN2C data files are valid +INFO: Backup SCUN2C resident size: 17MB +INFO: Backup SCUN2C completed List the backups of the instance again: -backup_user@backup_host:~$ pg_probackup-16 show -B /mnt/backups --instance=mydb -================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - mydb 16 S6OBQO 2024-01-03 07:06:26+00 DELTA STREAM 1/1 6s 111kB 32MB 1.00 0/6000028 0/6000168 OK - mydb 16 S6OBLG 2024-01-03 07:03:18+00 DELTA STREAM 1/1 10s 127kB 32MB 1.00 0/4000028 0/4000168 OK - mydb 16 S6OBFN 2024-01-03 06:59:49+00 FULL STREAM 1/0 10s 29MB 16MB 1.00 0/2000060 0/2003CB0 OK +backup_user@backup_host:~$ pg_probackup show \ + -B /mnt/backups \ + --instance=node +=================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +=================================================================================================================================== + node 16 SCUN2C 2024-05-02 11:18:13+03 DELTA STREAM 1/1 10s 1139kB 16MB 1.00 0/B000028 0/B000168 OK + node 16 SCUN22 2024-05-02 11:18:04+03 DELTA STREAM 1/1 10s 2357kB 32MB 1.02 0/9000028 0/9000168 OK + node 16 SCUN1Q 2024-05-02 11:17:53+03 FULL STREAM 1/0 12s 40MB 16MB 2.42 0/8000028 0/800BBD0 OK Restore the data from the latest available backup to an arbitrary location: -backup_user@backup_host:~$ pg_probackup-16 restore -B /mnt/backups -D /var/lib/postgresql/16/staging --instance=mydb -INFO: Validating parents for backup S6OBQO -INFO: Validating backup S6OBFN -INFO: Backup S6OBFN data files are valid -INFO: Validating backup S6OBLG -INFO: Backup S6OBLG data files are valid -INFO: Validating backup S6OBQO -INFO: Backup S6OBQO data files are valid -INFO: Backup S6OBQO WAL segments are valid -INFO: Backup S6OBQO is valid. -INFO: Restoring the database from backup at 2024-01-03 07:06:24+00 -INFO: Start restoring backup files. PGDATA size: 61MB -INFO: Backup files are restored. Transfered bytes: 61MB, time elapsed: 1s -INFO: Restore incremental ratio (less is better): 100% (61MB/61MB) +backup_user@backup_host:~$ pg_probackup restore \ + -B /mnt/backups \ + -D /var/lib/pgpro/std-16/staging-data \ + --instance=node +INFO: Validating parents for backup SCUN2C +INFO: Validating backup SCUN1Q +INFO: Backup SCUN1Q data files are valid +INFO: Validating backup SCUN22 +INFO: Backup SCUN22 data files are valid +INFO: Validating backup SCUN2C +INFO: Backup SCUN2C data files are valid +INFO: Backup SCUN2C WAL segments are valid +INFO: Backup SCUN2C is valid. +INFO: Restoring the database from backup SCUN2C on localhost +INFO: Start restoring backup files. PGDATA size: 112MB +INFO: Backup files are restored. Transfered bytes: 112MB, time elapsed: 0 +INFO: Restore incremental ratio (less is better): 100% (112MB/112MB) INFO: Syncing restored files to disk -INFO: Restored backup files are synced, time elapsed: 0 -INFO: Restore of backup S6OBQO completed. +INFO: Restored backup files are synced, time elapsed: 2s +INFO: Restore of backup SCUN2C completed. @@ -806,7 +824,7 @@ apt-cache search pg_probackup Install or upgrade a pg_probackup version of your choice -sudo apt install pg-probackup-15 +sudo apt install pg-probackup-16 @@ -814,7 +832,7 @@ sudo apt install pg-probackup-15 Optionally install the debug package -sudo apt install pg-probackup-15-dbg +sudo apt install pg-probackup-16-dbg @@ -823,7 +841,7 @@ sudo apt install pg-probackup-15-dbg sudo apt install dpkg-dev -sudo apt source pg-probackup-15 +sudo apt source pg-probackup-16 @@ -855,7 +873,7 @@ dnf search pg_probackup Install or upgrade a pg_probackup version of your choice -dnf install pg_probackup-15 +dnf install pg_probackup-16 @@ -863,7 +881,7 @@ dnf install pg_probackup-15 Optionally install the debug package -dnf install pg_probackup-15-debuginfo +dnf install pg_probackup-16-debuginfo @@ -877,7 +895,7 @@ dnf install pg_probackup-15-debuginfo dnf install 'dnf-command(download)' -dnf download --source pg_probackup-15 +dnf download --source pg_probackup-16 @@ -885,7 +903,7 @@ dnf download --source pg_probackup-15 Using yum: -yumdownloader --source pg_probackup-15 +yumdownloader --source pg_probackup-16 @@ -936,7 +954,7 @@ apt-cache search pg_probackup Install or upgrade a pg_probackup version of your choice -sudo apt-get install pg_probackup-15 +sudo apt-get install pg_probackup-16 @@ -944,7 +962,7 @@ sudo apt-get install pg_probackup-15 Optionally install the debug package -sudo apt-get install pg_probackup-15-debuginfo +sudo apt-get install pg_probackup-16-debuginfo @@ -983,7 +1001,7 @@ zypper se pg_probackup Install or upgrade a pg_probackup version of your choice -zypper in pg_probackup-15 +zypper in pg_probackup-16 @@ -991,7 +1009,7 @@ zypper in pg_probackup-15 Optionally install the source package for rebuilding the binaries -zypper si pg_probackup-15 +zypper si pg_probackup-16 @@ -2107,11 +2125,11 @@ pg_probackup validate -B backup_dir --instance= For example, to check that you can restore the database cluster - from a backup copy with the PT8XFX backup ID up to the + from a backup copy with the SCUN2C backup ID up to the specified timestamp, run this command: -pg_probackup validate -B backup_dir --instance=instance_name -i PT8XFX --recovery-target-time="2017-05-18 14:18:11+03" +pg_probackup validate -B backup_dir --instance=instance_name -i SCUN2C --recovery-target-time="2024-05-03 11:18:13+03" If you specify the backup_id of an incremental backup, @@ -2306,28 +2324,35 @@ pg_probackup restore -B backup_dir --instance= -============================================================================================================================================= - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -============================================================================================================================================= - node 12 QBRNBP 2020-06-11 17:40:58+03 DELTA ARCHIVE 16/15 40s 194MB 16MB 8.26 15/2C000028 15/2D000128 OK - node 12 QBRIDX 2020-06-11 15:51:42+03 PAGE ARCHIVE 15/15 11s 18MB 16MB 5.10 14/DC000028 14/DD0000B8 OK - node 12 QBRIAJ 2020-06-11 15:51:08+03 PAGE ARCHIVE 15/15 20s 141MB 96MB 6.22 14/D4BABFE0 14/DA9871D0 OK - node 12 QBRHT8 2020-06-11 15:45:56+03 FULL ARCHIVE 15/0 2m:11s 1371MB 416MB 10.93 14/9D000028 14/B782E9A0 OK - -pg_probackup restore -B /backup --instance=node -R -I lsn -INFO: Running incremental restore into nonempty directory: "/var/lib/pgsql/12/data" -INFO: Destination directory redo point 15/2E000028 on tli 16 is within reach of backup QBRIDX with Stop LSN 14/DD0000B8 on tli 15 -INFO: shift LSN: 14/DD0000B8 -INFO: Restoring the database from backup at 2020-06-11 17:40:58+03 -INFO: Extracting the content of destination directory for incremental restore -INFO: Destination directory content extracted, time elapsed: 1s -INFO: Removing redundant files in destination directory -INFO: Redundant files are removed, time elapsed: 1s -INFO: Start restoring backup files. PGDATA size: 15GB -INFO: Backup files are restored. Transfered bytes: 1693MB, time elapsed: 43s -INFO: Restore incremental ratio (less is better): 11% (1693MB/15GB) -INFO: Restore of backup QBRNBP completed. - +====================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 16 SCUN3Y 2024-05-02 11:19:16+03 DELTA STREAM 1/1 7s 92MB 208MB 2.27 0/3C0043A8 0/46159C70 OK + node 16 SCUN3M 2024-05-02 11:19:01+03 PTRACK STREAM 1/1 10s 30MB 16MB 2.23 0/32000028 0/32005ED0 OK + node 16 SCUN39 2024-05-02 11:18:50+03 PAGE STREAM 1/1 12s 46MB 32MB 1.44 0/2A000028 0/2B0000B8 OK + node 16 SCUN2V 2024-05-02 11:18:38+03 FULL STREAM 1/0 11s 154MB 16MB 2.32 0/23000028 0/23000168 OK + +backup_user@backup_host:~$ pg_probackup restore -B /mnt/backups --instance=node -R -I lsn +INFO: Destination directory and tablespace directories are empty, disable incremental restore +INFO: Validating parents for backup SCUN3Y +INFO: Validating backup SCUN2V +INFO: Backup SCUN2V data files are valid +INFO: Validating backup SCUN39 +INFO: Backup SCUN39 data files are valid +INFO: Validating backup SCUN3M +INFO: Backup SCUN3M data files are valid +INFO: Validating backup SCUN3Y +INFO: Backup SCUN3Y data files are valid +INFO: Backup SCUN3Y WAL segments are valid +INFO: Backup SCUN3Y is valid. +INFO: Restoring the database from backup SCUN3Y +INFO: Start restoring backup files. PGDATA size: 759MB +INFO: Backup files are restored. Transfered bytes: 759MB, time elapsed: 3s +INFO: Restore incremental ratio (less is better): 100% (759MB/759MB) +INFO: Syncing restored files to disk +INFO: Restored backup files are synced, time elapsed: 1s +INFO: Restore of backup SCUN3Y completed. + Incremental restore is possible only for backups with @@ -2438,7 +2463,7 @@ pg_probackup restore -B backup_dir --instance= -pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-time="2017-05-18 14:18:11+03" +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-time="2024-05-03 11:18:13+03" @@ -2791,13 +2816,13 @@ pg_probackup show -B backup_dir BACKUP INSTANCE 'node' ====================================================================================================================================== - Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ====================================================================================================================================== - node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK + node 16 SCUN4E 2024-05-02 11:19:37+03 FULL ARCHIVE 1/0 13s 239MB 16MB 2.31 0/4C000028 0/4D0000B8 OK + node 16 SCUN3Y 2024-05-02 11:19:16+03 DELTA STREAM 1/1 7s 92MB 208MB 2.27 0/3C0043A8 0/46159C70 OK + node 16 SCUN3M 2024-05-02 11:19:01+03 PTRACK STREAM 1/1 10s 30MB 16MB 2.23 0/32000028 0/32005ED0 OK + node 16 SCUN39 2024-05-02 11:18:50+03 PAGE STREAM 1/1 12s 46MB 32MB 1.44 0/2A000028 0/2B0000B8 OK + node 16 SCUN2V 2024-05-02 11:18:38+03 FULL STREAM 1/0 11s 154MB 16MB 2.32 0/23000028 0/23000168 OK For each backup, the following information is provided: @@ -2962,27 +2987,26 @@ from-replica = false #Compatibility block-size = 8192 -wal-block-size = 8192 +xlog-block-size = 8192 checksum-version = 1 -program-version = 2.1.3 -server-version = 10 +program-version = 2.5.14 +server-version = 16 #Result backup info timelineid = 1 -start-lsn = 0/04000028 -stop-lsn = 0/040000f8 -start-time = '2017-05-16 12:57:29' -end-time = '2017-05-16 12:57:31' -recovery-xid = 597 -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-05-16 12:57:31' -data-bytes = 22288792 +start-lsn = 0/4C000028 +stop-lsn = 0/4D0000B8 +start-time = '2024-05-02 11:19:26+03' +end-time = '2024-05-02 11:19:39+03' +recovery-xid = 743 +recovery-time = '2024-05-02 11:19:37+03' +data-bytes = 250827955 wal-bytes = 16777216 -uncompressed-bytes = 39961833 -pgdata-bytes = 39859393 +uncompressed-bytes = 578216425 +pgdata-bytes = 578216107 status = OK -parent-backup-id = 'PT8XFX' -primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' +primary_conninfo = 'user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable' +content-crc = 802820606 Detailed output has additional attributes: @@ -3098,37 +3122,39 @@ pg_probackup show -B backup_dir --instance= [ - { - "instance": "node", - "backups": [ - { - "id": "PT91HZ", - "parent-backup-id": "PT8XFX", - "backup-mode": "DELTA", - "wal": "ARCHIVE", - "compress-alg": "zlib", - "compress-level": 1, - "from-replica": false, - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.3", - "server-version": "10", - "current-tli": 16, - "parent-tli": 2, - "start-lsn": "0/8000028", - "stop-lsn": "0/8000160", - "start-time": "2019-06-17 18:25:11+03", - "end-time": "2019-06-17 18:25:16+03", - "recovery-xid": 0, - "recovery-time": "2019-06-17 18:25:15+03", - "data-bytes": 106733, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } + { + "instance": "node", + "backups": [ + { + "id": "SCUN4E", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/4C000028", + "stop-lsn": "0/4D0000B8", + "start-time": "2024-05-02 11:19:26+03", + "end-time": "2024-05-02 11:19:39+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:37+03", + "data-bytes": 250827955, + "wal-bytes": 16777216, + "uncompressed-bytes": 578216425, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 802820606 + } + ] + } ] @@ -3146,15 +3172,12 @@ pg_probackup show -B backup_dir [--instance= + ARCHIVE INSTANCE 'node' -=================================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=================================================================================================================================== - 5 1 0/B000000 00000005000000000000000B 00000005000000000000000C 2 685kB 48.00 0 OK - 4 3 0/18000000 000000040000000000000018 00000004000000000000001A 3 648kB 77.00 0 OK - 3 2 0/15000000 000000030000000000000015 000000030000000000000017 3 648kB 77.00 0 OK - 2 1 0/B000108 00000002000000000000000B 000000020000000000000015 5 892kB 94.00 1 DEGRADED - 1 0 0/0 000000010000000000000001 00000001000000000000000A 10 8774kB 19.00 1 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000019 00000001000000000000004D 53 848MB 1.00 5 OK For each timeline, the following information is provided: @@ -3245,212 +3268,169 @@ pg_probackup show -B backup_dir [--instance= [ - { - "instance": "replica", - "timelines": [ - { - "tli": 5, - "parent-tli": 1, - "switchpoint": "0/B000000", - "min-segno": "00000005000000000000000B", - "max-segno": "00000005000000000000000C", - "n-segments": 2, - "size": 685320, - "zratio": 48.00, - "closest-backup-id": "PXS92O", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 4, - "parent-tli": 3, - "switchpoint": "0/18000000", - "min-segno": "000000040000000000000018", - "max-segno": "00000004000000000000001A", - "n-segments": 3, - "size": 648625, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 3, - "parent-tli": 2, - "switchpoint": "0/15000000", - "min-segno": "000000030000000000000015", - "max-segno": "000000030000000000000017", - "n-segments": 3, - "size": 648911, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 2, - "parent-tli": 1, - "switchpoint": "0/B000108", - "min-segno": "00000002000000000000000B", - "max-segno": "000000020000000000000015", - "n-segments": 5, - "size": 892173, - "zratio": 94.00, - "closest-backup-id": "PXS92O", - "status": "DEGRADED", - "lost-segments": [ - { - "begin-segno": "00000002000000000000000D", - "end-segno": "00000002000000000000000E" - }, - { - "begin-segno": "000000020000000000000010", - "end-segno": "000000020000000000000012" - } - ], - "backups": [ - { - "id": "PXS9CE", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 2, - "parent-tli": 0, - "start-lsn": "0/C000028", - "stop-lsn": "0/C000160", - "start-time": "2019-09-13 21:43:26+03", - "end-time": "2019-09-13 21:43:30+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:43:29+03", - "data-bytes": 104674852, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - }, - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "000000010000000000000001", - "max-segno": "00000001000000000000000A", - "n-segments": 10, - "size": 8774805, - "zratio": 19.00, - "closest-backup-id": "", - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92O", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "true", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/4000028", - "stop-lsn": "0/6000028", - "start-time": "2019-09-13 21:37:36+03", - "end-time": "2019-09-13 21:38:45+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 25987319, - "wal-bytes": 50331648, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - }, - { - "instance": "master", - "timelines": [ - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "000000010000000000000001", - "max-segno": "00000001000000000000000B", - "n-segments": 11, - "size": 8860892, - "zratio": 20.00, - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92H", - "parent-backup-id": "PXS92C", - "backup-mode": "PAGE", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 1, - "start-lsn": "0/4000028", - "stop-lsn": "0/50000B8", - "start-time": "2019-09-13 21:37:29+03", - "end-time": "2019-09-13 21:37:31+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 1328461, - "wal-bytes": 33554432, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - }, - { - "id": "PXS92C", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/2000028", - "stop-lsn": "0/2000160", - "start-time": "2019-09-13 21:37:24+03", - "end-time": "2019-09-13 21:37:29+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:28+03", - "data-bytes": 24871902, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - } + { + "instance": "node", + "timelines": [ + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "000000010000000000000019", + "max-segno": "00000001000000000000004D", + "n-segments": 53, + "size": 889192448, + "zratio": 1.00, + "closest-backup-id": "", + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "SCUN4E", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/4C000028", + "stop-lsn": "0/4D0000B8", + "start-time": "2024-05-02 11:19:26+03", + "end-time": "2024-05-02 11:19:39+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:37+03", + "data-bytes": 250827955, + "wal-bytes": 16777216, + "uncompressed-bytes": 578216425, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 802820606 + }, + { + "id": "SCUN3Y", + "parent-backup-id": "SCUN3M", + "backup-mode": "DELTA", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/3C0043A8", + "stop-lsn": "0/46159C70", + "start-time": "2024-05-02 11:19:10+03", + "end-time": "2024-05-02 11:19:17+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:16+03", + "data-bytes": 96029293, + "wal-bytes": 218103808, + "uncompressed-bytes": 217639806, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3074300814 + }, + { + "id": "SCUN3M", + "parent-backup-id": "SCUN39", + "backup-mode": "PTRACK", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/32000028", + "stop-lsn": "0/32005ED0", + "start-time": "2024-05-02 11:18:58+03", + "end-time": "2024-05-02 11:19:08+03", + "recovery-xid": 742, + "recovery-time": "2024-05-02 11:19:01+03", + "data-bytes": 31205704, + "wal-bytes": 16777216, + "uncompressed-bytes": 69585790, + "pgdata-bytes": 509927595, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3446949708 + }, + { + "id": "SCUN39", + "parent-backup-id": "SCUN2V", + "backup-mode": "PAGE", + "wal": "STREAM", + "compress-alg": "pglz", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/2A000028", + "stop-lsn": "0/2B0000B8", + "start-time": "2024-05-02 11:18:45+03", + "end-time": "2024-05-02 11:18:57+03", + "recovery-xid": 741, + "recovery-time": "2024-05-02 11:18:50+03", + "data-bytes": 48381612, + "wal-bytes": 33554432, + "uncompressed-bytes": 69569406, + "pgdata-bytes": 441639083, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3492989773 + }, + { + "id": "SCUN2V", + "backup-mode": "FULL", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.14", + "server-version": "16", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/23000028", + "stop-lsn": "0/23000168", + "start-time": "2024-05-02 11:18:31+03", + "end-time": "2024-05-02 11:18:42+03", + "recovery-xid": 740, + "recovery-time": "2024-05-02 11:18:38+03", + "data-bytes": 161084290, + "wal-bytes": 16777216, + "uncompressed-bytes": 373359081, + "pgdata-bytes": 373358763, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 1621343133 + } + ] + } + ] + } ] @@ -3597,33 +3577,33 @@ pg_probackup delete -B backup_dir --instance=backup_dir directory, with the option set to 7, and you have the following backups - available on April 10, 2019: + available on May 02, 2024: BACKUP INSTANCE 'node' -=================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -=================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK - -------------------------------------------------------retention window-------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK - - - Even though P7XDHB and P7XDHU backups are outside the +===================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +===================================================================================================================================== + node 16 SCUN6L 2024-05-02 11:20:48+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/46000028 0/470000B8 OK + node 16 SCQXUI 2024-04-30 11:20:45+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/44000028 0/450000F0 OK + node 16 SCFTUG 2024-04-24 11:20:43+03 DELTA ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/42000028 0/430000B8 OK +----------------------------------------------------------retention window----------------------------------------------------------- + node 16 SCDZ6D 2024-04-23 11:20:40+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/40000028 0/410000B8 OK + node 16 SCC4HX 2024-04-22 11:20:24+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3E000028 0/3F0000F0 OK + node 16 SC8F5G 2024-04-20 11:20:07+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3C0000D8 0/3D00BB58 OK + + + Even though SCC4HX and SCDZ6D backups are outside the retention window, they cannot be removed as it invalidates the - succeeding incremental backups P7XDJA and P7XDQV that are + succeeding incremental backups SCFTUG and SCQXUI that are still required, so, if you run the command with the - flag, only the P7XDFT full + flag, only the SC8F5G full backup will be removed. - With the option, the P7XDJA - backup is merged with the underlying P7XDHU and P7XDHB backups + With the option, the SCC4HX + backup is merged with the underlying SCC4HX and SCC4HX backups and becomes a full one, so there is no need to keep these expired backups anymore: @@ -3633,12 +3613,14 @@ pg_probackup show -B backup_dir BACKUP INSTANCE 'node' -================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK +===================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +===================================================================================================================================== + node 16 SCUN6L 2024-05-02 11:20:48+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/46000028 0/470000B8 OK + node 16 SCQXUI 2024-04-30 11:20:45+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/44000028 0/450000F0 OK + node 16 SCFTUG 2024-04-24 11:20:43+03 DELTA ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/42000028 0/430000B8 OK + node 16 SCDZ6D 2024-04-23 11:20:40+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/40000028 0/410000B8 OK + node 16 SCC4HX 2024-04-22 11:20:24+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3E000028 0/3F0000F0 OK The Time field for the merged backup displays the time @@ -3666,7 +3648,7 @@ pg_probackup set-backup -B backup_dir --instance=--expire-time option. For example: -pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --expire-time="2020-01-01 00:00:00+03" +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --expire-time="2027-05-02 11:21:00+00" Alternatively, you can use the and @@ -3676,7 +3658,7 @@ pg_probackup set-backup -B backup_dir --instance= pg_probackup backup -B backup_dir --instance=instance_name -b FULL --ttl=30d -pg_probackup backup -B backup_dir --instance=instance_name -b FULL --expire-time="2020-01-01 00:00:00+03" +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --expire-time="2027-05-02 11:21:00+00" To check if the backup is pinned, @@ -3690,8 +3672,8 @@ pg_probackup show -B backup_dir --instance= ... -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-01-01 00:00:00+03' +recovery-time = '2024-05-02 11:21:00+00' +expire-time = '2027-05-02 11:21:00+00' data-bytes = 22288792 ... @@ -3767,16 +3749,15 @@ pg_probackup set-backup -B backup_dir --instance=backup_dir --instance=node -BACKUP INSTANCE 'node' -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK - node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK +====================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 16 SCUN92 2024-05-02 11:22:16+03 DELTA STREAM 1/1 9s 1162kB 32MB 1.08 0/7C000028 0/7C000168 OK + node 16 SCUN8N 2024-05-02 11:22:09+03 FULL STREAM 1/0 12s 296MB 16MB 2.30 0/7A000028 0/7A009A08 OK + node 16 SCUN8I 2024-05-02 11:21:55+03 DELTA STREAM 1/1 5s 1148kB 32MB 1.01 0/78000028 0/78000168 OK + node 16 SCUN86 2024-05-02 11:21:47+03 DELTA STREAM 1/1 11s 120MB 16MB 2.27 0/76000028 0/760001A0 OK + node 16 SCUN7I 2024-05-02 11:21:29+03 FULL STREAM 1/0 22s 296MB 288MB 2.30 0/63012FE8 0/74E7ADA0 OK + node 16 SCUN71 2024-05-02 11:21:12+03 FULL STREAM 1/0 13s 296MB 272MB 2.30 0/49000028 0/573683B8 OK You can check the state of the WAL archive by running the @@ -3787,11 +3768,12 @@ BACKUP INSTANCE 'node' pg_probackup show -B backup_dir --instance=node --archive + ARCHIVE INSTANCE 'node' -=============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================================== - 1 0 0/0 000000010000000000000001 000000010000000000000047 71 36MB 31.00 6 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000048 00000001000000000000007C 53 848MB 1.00 6 OK WAL purge without cannot @@ -3801,11 +3783,12 @@ ARCHIVE INSTANCE 'node' pg_probackup delete -B backup_dir --instance=node --delete-wal + ARCHIVE INSTANCE 'node' -=============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================================== - 1 0 0/0 000000010000000000000002 000000010000000000000047 70 34MB 32.00 6 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000049 00000001000000000000007C 52 832MB 1.00 6 OK If you would like, for example, to keep only those WAL @@ -3816,11 +3799,12 @@ ARCHIVE INSTANCE 'node' pg_probackup delete -B backup_dir --instance=node --delete-wal --wal-depth=1 + ARCHIVE INSTANCE 'node' -================================================================================================================================ - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -================================================================================================================================ - 1 0 0/0 000000010000000000000046 000000010000000000000047 2 143kB 228.00 6 OK +=============================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================================== + 1 0 0/0 00000001000000000000007C 00000001000000000000007C 1 16MB 1.00 6 OK Alternatively, you can use the @@ -3830,11 +3814,12 @@ ARCHIVE INSTANCE 'node' pg_probackup backup -B backup_dir --instance=node -b DELTA --wal-depth=1 --delete-wal + ARCHIVE INSTANCE 'node' =============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status =============================================================================================================================== - 1 0 0/0 000000010000000000000048 000000010000000000000049 1 72kB 228.00 7 OK + 1 0 0/0 00000001000000000000007E 00000001000000000000007E 1 16MB 1.00 7 OK @@ -3997,10 +3982,7 @@ pg_probackup delete -B backup_dir --instance= DDL commands - CREATE TABLESPACE/DROP TABLESPACE + CREATE TABLESPACE/DROP TABLESPACE cannot be run simultaneously with catchup. @@ -5375,7 +5357,7 @@ pg_probackup catchup -b catchup_mode If the time zone offset is not specified, the local time zone is used. - Example: --recovery-target-time="2020-01-01 00:00:00+03" + Example: --recovery-target-time="2027-05-02 11:21:00+00" @@ -5565,7 +5547,7 @@ pg_probackup catchup -b catchup_mode If the time zone offset is not specified, the local time zone is used. - Example: --expire-time="2020-01-01 00:00:00+03" + Example: --expire-time="2027-05-02 11:21:00+00" From 28bc0f6358b0af4958788fe1f36f6af6bac6f6b1 Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Fri, 3 May 2024 10:49:50 +0300 Subject: [PATCH 2104/2107] [PBCKP-818] Refinery of documentation manually --- doc/pgprobackup.xml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index f5b2a93eb..95dd1a1d8 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2327,10 +2327,10 @@ pg_probackup restore -B backup_dir --instance=backup_dir --instance= - With the option, the SCC4HX - backup is merged with the underlying SCC4HX and SCC4HX backups + With the option, the SCFTUG + backup is merged with the underlying SCDZ6D and SCC4HX backups and becomes a full one, so there is no need to keep these expired backups anymore: @@ -3618,9 +3618,7 @@ BACKUP INSTANCE 'node' ===================================================================================================================================== node 16 SCUN6L 2024-05-02 11:20:48+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/46000028 0/470000B8 OK node 16 SCQXUI 2024-04-30 11:20:45+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/44000028 0/450000F0 OK - node 16 SCFTUG 2024-04-24 11:20:43+03 DELTA ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/42000028 0/430000B8 OK - node 16 SCDZ6D 2024-04-23 11:20:40+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/40000028 0/410000B8 OK - node 16 SCC4HX 2024-04-22 11:20:24+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3E000028 0/3F0000F0 OK + node 16 SCFTUG 2024-04-24 11:20:43+03 FULL ARCHIVE 1/1 5s 296MB 16MB 1.00 0/42000028 0/430000B8 OK The Time field for the merged backup displays the time From f361dda7f3df31fe65f3fb5477ee92d04404abd2 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 16 May 2024 14:08:04 +0300 Subject: [PATCH 2105/2107] Revert "Replace BACKUP_PATH in the source files" This reverts commit 90a4a4f4b32128ee728c530b1fabba608b3d51eb. --- doc/pgprobackup.xml | 10 +++++----- po/ru.po | 2 +- src/archive.c | 4 ++-- src/catalog.c | 2 +- src/help.c | 22 +++++++++++----------- src/pg_probackup.c | 6 +++--- src/pg_probackup.h | 8 ++++---- src/pg_probackup_state.h | 6 +++--- tests/option_test.py | 2 +- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 95dd1a1d8..f8f269b7c 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -1146,7 +1146,7 @@ pg_probackup add-instance -B backup_dir -D backup_dir directory and at least read-only access to data_dir directory. If you specify the path to the backup catalog in the - BACKUP_DIR environment variable, you can + BACKUP_PATH environment variable, you can omit the corresponding option when running pg_probackup commands. @@ -5205,14 +5205,14 @@ pg_probackup catchup -b catchup_mode -BACKUP_DIR +BACKUP_PATH Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify - it once in the BACKUP_DIR environment variable. In this case, + it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. @@ -5672,7 +5672,7 @@ pg_probackup catchup -b catchup_mode lazily, when the first log message is written. - Default: $BACKUP_DIR/log/ + Default: $BACKUP_PATH/log/ @@ -5755,7 +5755,7 @@ pg_probackup catchup -b catchup_mode reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in - $BACKUP_DIR/log/log_rotation. The zero value disables + $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). diff --git a/po/ru.po b/po/ru.po index 30f50f797..1263675c2 100644 --- a/po/ru.po +++ b/po/ru.po @@ -811,7 +811,7 @@ msgstr "" #: src/help.c:360 src/help.c:521 src/help.c:588 src/help.c:635 src/help.c:715 #: src/help.c:761 src/help.c:833 #, c-format -msgid " directory for file logging (default: BACKUP_DIR/log)\n" +msgid " directory for file logging (default: BACKUP_PATH/log)\n" msgstr "" #: src/help.c:361 src/help.c:522 src/help.c:589 src/help.c:636 src/help.c:716 diff --git a/src/archive.c b/src/archive.c index e97a1ade8..7d753c8b3 100644 --- a/src/archive.c +++ b/src/archive.c @@ -113,7 +113,7 @@ static parray *setup_push_filelist(const char *archive_status_dir, * set archive_command to * 'pg_probackup archive-push -B /home/anastasia/backup --wal-file-name %f', * to move backups into arclog_path. - * Where archlog_path is $BACKUP_DIR/wal/instance_name + * Where archlog_path is $BACKUP_PATH/wal/instance_name */ void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, @@ -1126,7 +1126,7 @@ do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const cha join_path_components(absolute_wal_file_path, current_dir, wal_file_path); /* full filepath to WAL file in archive directory. - * $BACKUP_DIR/wal/instance_name/000000010000000000000001 */ + * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name); INSTR_TIME_SET_CURRENT(start_time); diff --git a/src/catalog.c b/src/catalog.c index 4da406af3..b29090789 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1437,7 +1437,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, } /* - * Create backup directory in $BACKUP_DIR + * Create backup directory in $BACKUP_PATH * (with proposed backup->backup_id) * and initialize this directory. * If creation of directory fails, then diff --git a/src/help.c b/src/help.c index 48cc1f524..e18706a13 100644 --- a/src/help.c +++ b/src/help.c @@ -372,7 +372,7 @@ help_backup(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -548,7 +548,7 @@ help_restore(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -621,7 +621,7 @@ help_validate(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -674,7 +674,7 @@ help_checkdb(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -760,7 +760,7 @@ help_delete(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -814,7 +814,7 @@ help_merge(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -890,7 +890,7 @@ help_set_config(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1002,7 +1002,7 @@ help_add_instance(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1072,7 +1072,7 @@ help_archive_push(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1131,7 +1131,7 @@ help_archive_get(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); @@ -1221,7 +1221,7 @@ help_catchup(void) printf(_(" --error-log-filename=error-log-filename\n")); printf(_(" filename for error logging (default: none)\n")); printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_DIR/log)\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); printf(_(" --log-rotation-size=log-rotation-size\n")); printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); diff --git a/src/pg_probackup.c b/src/pg_probackup.c index e50b05995..fa67ddff5 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -468,10 +468,10 @@ main(int argc, char *argv[]) if (backup_path == NULL) { /* - * If command line argument is not set, try to read BACKUP_DIR + * If command line argument is not set, try to read BACKUP_PATH * from environment variable */ - backup_path = getenv("BACKUP_DIR"); + backup_path = getenv("BACKUP_PATH"); } if (backup_path != NULL) @@ -498,7 +498,7 @@ main(int argc, char *argv[]) backup_subcmd != CATCHUP_CMD) elog(ERROR, "No backup catalog path specified.\n" - "Please specify it either using environment variable BACKUP_DIR or\n" + "Please specify it either using environment variable BACKUP_PATH or\n" "command line option --backup-path (-B)"); /* ===== catalogState (END) ======*/ diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 1f4780f58..668f183a7 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -837,13 +837,13 @@ typedef struct InstanceState CatalogState *catalog_state; char instance_name[MAXPGPATH]; //previously global var instance_name - /* $BACKUP_DIR/backups/instance_name */ + /* $BACKUP_PATH/backups/instance_name */ char instance_backup_subdir_path[MAXPGPATH]; - /* $BACKUP_DIR/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ + /* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ char instance_config_path[MAXPGPATH]; - - /* $BACKUP_DIR/backups/instance_name */ + + /* $BACKUP_PATH/backups/instance_name */ char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path /* TODO: Make it more specific */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h index 1d1ff88d0..56d852537 100644 --- a/src/pg_probackup_state.h +++ b/src/pg_probackup_state.h @@ -13,11 +13,11 @@ typedef struct CatalogState { - /* $BACKUP_DIR */ + /* $BACKUP_PATH */ char catalog_path[MAXPGPATH]; //previously global var backup_path - /* $BACKUP_DIR/backups */ + /* $BACKUP_PATH/backups */ char backup_subdir_path[MAXPGPATH]; - /* $BACKUP_DIR/wal */ + /* $BACKUP_PATH/wal */ char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path } CatalogState; diff --git a/tests/option_test.py b/tests/option_test.py index e97da1ef7..d1e8cb3a6 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -25,7 +25,7 @@ def test_without_backup_path_3(self): except ProbackupException as e: self.assertIn( 'ERROR: No backup catalog path specified.\n' + \ - 'Please specify it either using environment variable BACKUP_DIR or\n' + \ + 'Please specify it either using environment variable BACKUP_PATH or\n' + \ 'command line option --backup-path (-B)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) From 721d5d231118587ed4bd30725b86c8de8366dea7 Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Thu, 16 May 2024 14:37:37 +0300 Subject: [PATCH 2106/2107] Up version to 2.5.15 --- doc/pgprobackup.xml | 20 ++++++++++---------- src/pg_probackup.h | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index f8f269b7c..1491059c5 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -558,7 +558,7 @@ backup_user@backup_host:~$ pg_probackup backup \ --remote-user=postgres \ -U backup \ -d backupdb -INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN1Q, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN1Q, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() @@ -604,7 +604,7 @@ backup_user@backup_host:~$ pg_probackup backup \ --remote-user=postgres \ -U backup \ -d backupdb -INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN22, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN22, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() @@ -692,7 +692,7 @@ backup_user@backup_host:~$ pg_probackup backup \ --instance=node \ --stream \ --compress-algorithm=zlib -INFO: Backup start, pg_probackup version: 2.5.14, instance: node, backup ID: SCUN2C, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN2C, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected INFO: Database backup start INFO: wait for pg_backup_start() @@ -2989,7 +2989,7 @@ from-replica = false block-size = 8192 xlog-block-size = 8192 checksum-version = 1 -program-version = 2.5.14 +program-version = 2.5.15 server-version = 16 #Result backup info @@ -3135,7 +3135,7 @@ pg_probackup show -B backup_dir --instance=backup_dir [--instance=backup_dir [--instance=backup_dir [--instance=backup_dir [--instance=backup_dir [--instance= Date: Thu, 16 May 2024 16:24:43 +0300 Subject: [PATCH 2107/2107] [nojira] Change binary name to pg_probackup-16 --- doc/pgprobackup.xml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 1491059c5..10e766239 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -529,14 +529,14 @@ doc/src/sgml/pgprobackup.sgml Initialize the backup catalog: -backup_user@backup_host:~$ pg_probackup init -B /mnt/backups +backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups INFO: Backup catalog '/mnt/backups' successfully initialized Add a backup instance called mydb to the backup catalog: -backup_user@backup_host:~$ pg_probackup add-instance \ +backup_user@backup_host:~$ pg_probackup-16 add-instance \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/data \ --instance=node \ @@ -548,7 +548,7 @@ INFO: Instance 'node' successfully initialized Make a FULL backup: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b FULL \ --instance=node \ @@ -582,7 +582,7 @@ INFO: Backup SCUN1Q completed List the backups of the instance: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node ================================================================================================================================ @@ -594,7 +594,7 @@ backup_user@backup_host:~$ pg_probackup show \ Make an incremental backup in the DELTA mode: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -631,7 +631,7 @@ INFO: Backup SCUN22 completed Add or modify some parameters in the pg_probackup configuration file, so that you do not have to specify them each time on the command line: -backup_user@backup_host:~$ pg_probackup set-config \ +backup_user@backup_host:~$ pg_probackup-16 set-config \ -B /mnt/backups \ --instance=node \ --remote-host=postgres_host \ @@ -643,7 +643,7 @@ backup_user@backup_host:~$ pg_probackup set-config \ Check the configuration of the instance: -backup_user@backup_host:~$ pg_probackup show-config \ +backup_user@backup_host:~$ pg_probackup-16 show-config \ -B /mnt/backups \ --instance=node # Backup instance information @@ -686,7 +686,7 @@ remote-user = postgres Make another incremental backup in the DELTA mode, omitting the parameters stored in the configuration file earlier: -backup_user@backup_host:~$ pg_probackup backup \ +backup_user@backup_host:~$ pg_probackup-16 backup \ -B /mnt/backups \ -b DELTA \ --instance=node \ @@ -718,7 +718,7 @@ INFO: Backup SCUN2C completed List the backups of the instance again: -backup_user@backup_host:~$ pg_probackup show \ +backup_user@backup_host:~$ pg_probackup-16 show \ -B /mnt/backups \ --instance=node =================================================================================================================================== @@ -732,7 +732,7 @@ backup_user@backup_host:~$ pg_probackup show \ Restore the data from the latest available backup to an arbitrary location: -backup_user@backup_host:~$ pg_probackup restore \ +backup_user@backup_host:~$ pg_probackup-16 restore \ -B /mnt/backups \ -D /var/lib/pgpro/std-16/staging-data \ --instance=node @@ -2332,7 +2332,7 @@ pg_probackup restore -B backup_dir --instance=